Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

Inject zinject(8) a percentage amount of dev errs 

In the original form of device error injection, it was an all or nothing
situation.  To help simulate intermittent error conditions, you can now
specify a real number percentage value. This is also very useful for our
ZFS fault diagnosis testing and for injecting intermittent errors during
load testing.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Don Brady <don.brady@intel.com>
Closes #6227
This commit is contained in:
Don Brady 2017-06-16 18:21:11 -06:00 committed by Brian Behlendorf
parent 05a5357a6c
commit 0241e491a0
4 changed files with 76 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
cmd/zinject/zinject.c
View File

@ -21,6 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2015 by Delphix. All rights reserved.
* Copyright (c) 2017, Intel Corporation.
*/
/*
@ -124,7 +125,7 @@
* cache.
*
* The '-f' flag controls the frequency of errors injected, expressed as a
* integer percentage between 1 and 100. The default is 100.
* real number percentage between 0.0001 and 100. The default is 100.
*
* The this form is responsible for actually injecting the handler into the
* framework. It takes the arguments described above, translates them to the
@ -230,11 +231,13 @@ usage(void)
"\t\tspa_vdev_exit() will trigger a panic.\n"
"\n"
"\tzinject -d device [-e errno] [-L <nvlist|uber|pad1|pad2>] [-F]\n"
"\t [-T <read|write|free|claim|all> pool\n"
"\t [-T <read|write|free|claim|all>] [-f frequency] pool\n"
"\t\tInject a fault into a particular device or the device's\n"
"\t\tlabel. Label injection can either be 'nvlist', 'uber',\n "
"\t\t'pad1', or 'pad2'.\n"
"\t\t'errno' can be 'nxio' (the default), 'io', or 'dtl'.\n"
"\t\t'frequency' is a value between 0.0001 and 100.0 that limits\n"
"\t\tdevice error injection to a percentage of the IOs.\n"
"\n"
"\tzinject -d device -A <degrade|fault> -D <delay secs> pool\n"
"\t\tPerform a specific action on a particular device.\n"
@ -305,7 +308,7 @@ usage(void)
"\t\t-u\tUnload the associated pool. Can be specified with only\n"
"\t\t\ta pool object.\n"
"\t\t-f\tOnly inject errors a fraction of the time. Expressed as\n"
"\t\t\ta percentage between 1 and 100.\n"
"\t\t\ta percentage between 0.0001 and 100.\n"
"\n"
"\t-t data\t\tInject an error into the plain file contents of a\n"
"\t\t\tfile. The object must be specified as a complete path\n"
@ -645,6 +648,27 @@ parse_delay(char *str, uint64_t *delay, uint64_t *nlanes)
return (0);
}
static int
parse_frequency(const char *str, uint32_t *percent)
{
double val;
char *post;
val = strtod(str, &post);
if (post == NULL || *post != '\0')
return (EINVAL);
/* valid range is [0.0001, 100.0] */
val /= 100.0f;
if (val < 0.000001f || val > 1.0f)
return (ERANGE);
/* convert to an integer for use by kernel */
*percent = ((uint32_t)(val * ZI_PERCENTAGE_MAX));
return (0);
}
int
main(int argc, char **argv)
{
@ -760,10 +784,12 @@ main(int argc, char **argv)
}
break;
case 'f':
record.zi_freq = atoi(optarg);
if (record.zi_freq < 1 || record.zi_freq > 100) {
(void) fprintf(stderr, "frequency range must "
"be in the range (0, 100]\n");
ret = parse_frequency(optarg, &record.zi_freq);
if (ret != 0) {
(void) fprintf(stderr, "%sfrequency value must "
"be in the range [0.0001, 100.0]\n",
ret == EINVAL ? "invalid value: " :
ret == ERANGE ? "out of range: " : "");
libzfs_fini(g_zfs);
return (1);
}
@ -898,7 +924,8 @@ main(int argc, char **argv)
* '-c' is invalid with any other options.
*/
if (raw != NULL || range != NULL || type != TYPE_INVAL ||
level != 0 || record.zi_cmd != ZINJECT_UNINITIALIZED) {
level != 0 || record.zi_cmd != ZINJECT_UNINITIALIZED ||
record.zi_freq > 0) {
(void) fprintf(stderr, "cancel (-c) incompatible with "
"any other options\n");
usage();
@ -972,7 +999,8 @@ main(int argc, char **argv)
} else if (raw != NULL) {
if (range != NULL || type != TYPE_INVAL || level != 0 ||
record.zi_cmd != ZINJECT_UNINITIALIZED) {
record.zi_cmd != ZINJECT_UNINITIALIZED ||
record.zi_freq > 0) {
(void) fprintf(stderr, "raw (-b) format with "
"any other options\n");
usage();
@ -1007,7 +1035,7 @@ main(int argc, char **argv)
error = EIO;
} else if (record.zi_cmd == ZINJECT_PANIC) {
if (raw != NULL || range != NULL || type != TYPE_INVAL ||
level != 0 || device != NULL) {
level != 0 || device != NULL || record.zi_freq > 0) {
(void) fprintf(stderr, "panic (-p) incompatible with "
"other options\n");
usage();

5
include/sys/zfs_ioctl.h
View File

@ -22,6 +22,7 @@
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2015 by Delphix. All rights reserved.
* Copyright 2016 RackTop Systems.
* Copyright (c) 2017, Intel Corporation.
*/
#ifndef _SYS_ZFS_IOCTL_H
@ -338,6 +339,10 @@ typedef struct zinject_record {
#define ZEVENT_SEEK_START 0
#define ZEVENT_SEEK_END UINT64_MAX
/* scaled frequency ranges */
#define ZI_PERCENTAGE_MIN 4294UL
#define ZI_PERCENTAGE_MAX UINT32_MAX
typedef enum zinject_type {
ZINJECT_UNINITIALIZED,
ZINJECT_DATA_FAULT,

6
man/man8/zinject.8
View File

@ -76,7 +76,7 @@ create 3 lanes on the device; one lane with a latency
of 10 ms and two lanes with a 25 ms latency.
.TP
.B "zinject \-d \fIvdev\fB [\-e \fIdevice_error\fB] [\-L \fIlabel_error\fB] [\-T \fIfailure\fB] [\-F] \fIpool\fB"
.B "zinject \-d \fIvdev\fB [\-e \fIdevice_error\fB] [\-L \fIlabel_error\fB] [\-T \fIfailure\fB] [\-f \fIfrequency\fB] [\-F] \fIpool\fB"
Force a vdev error.
.TP
.B "zinject \-I [\-s \fIseconds\fB | \-g \fItxgs\fB] \fIpool\fB"
@ -113,8 +113,8 @@ Specify
.BR "nxio" " for an ENXIO error where reopening the device will fail."
.TP
.BI "\-f" " frequency"
Only inject errors a fraction of the time. Expressed as an integer
percentage between 1 and 100.
Only inject errors a fraction of the time. Expressed as a real number
percentage between 0.0001 and 100.
.TP
.BI "\-F"
Fail faster. Do fewer checks.

37
module/zfs/zio_inject.c
View File

@ -21,6 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2015 by Delphix. All rights reserved.
* Copyright (c) 2017, Intel Corporation.
*/
/*
@ -98,6 +99,26 @@ static kmutex_t inject_delay_mtx;
*/
static int inject_next_id = 1;
/*
* Test if the requested frequency was triggered
*/
static boolean_t
freq_triggered(uint32_t frequency)
{
/*
* zero implies always (100%)
*/
if (frequency == 0)
return (B_TRUE);
/*
* Note: we still handle legacy (unscaled) frequecy values
*/
uint32_t maximum = (frequency <= 100) ? 100 : ZI_PERCENTAGE_MAX;
return (spa_get_random(maximum) < frequency);
}
/*
* Returns true if the given record matches the I/O in progress.
*/
@ -113,8 +134,7 @@ zio_match_handler(zbookmark_phys_t *zb, uint64_t type,
record->zi_object == DMU_META_DNODE_OBJECT) {
if (record->zi_type == DMU_OT_NONE ||
type == record->zi_type)
return (record->zi_freq == 0 ||
spa_get_random(100) < record->zi_freq);
return (freq_triggered(record->zi_freq));
else
return (B_FALSE);
}
@ -128,8 +148,7 @@ zio_match_handler(zbookmark_phys_t *zb, uint64_t type,
zb->zb_blkid >= record->zi_start &&
zb->zb_blkid <= record->zi_end &&
error == record->zi_error)
return (record->zi_freq == 0 ||
spa_get_random(100) < record->zi_freq);
return (freq_triggered(record->zi_freq));
return (B_FALSE);
}
@ -293,6 +312,12 @@ zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
continue;
if (handler->zi_record.zi_error == error) {
/*
* limit error injection if requested
*/
if (!freq_triggered(handler->zi_record.zi_freq))
continue;
/*
* For a failed open, pretend like the device
* has gone away.
@ -466,10 +491,8 @@ zio_handle_io_delay(zio_t *zio)
if (handler->zi_record.zi_cmd != ZINJECT_DELAY_IO)
continue;
if (handler->zi_record.zi_freq != 0 &&
spa_get_random(100) >= handler->zi_record.zi_freq) {
if (!freq_triggered(handler->zi_record.zi_freq))
continue;
}
if (vd->vdev_guid != handler->zi_record.zi_guid)
continue;