Configure zed's diagnosis engine with vdev properties

Introduce four new vdev properties:
    checksum_n
    checksum_t
    io_n
    io_t

These properties can be used for configuring the thresholds of zed's
diagnosis engine and are interpeted as <N> events in T <seconds>.

When this property is set to a non-default value on a top-level vdev,
those thresholds will also apply to its leaf vdevs. This behavior can be
overridden by explicitly setting the property on the leaf vdev.

Note that, these properties do not persist across vdev replacement. For
this reason, it is advisable to set the property on the top-level vdev
instead of the leaf vdev.

The default values for zed's diagnosis engine (10 events, 600 seconds)
remains unchanged.

Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Ryan Moeller <ryan@iXsystems.com>
Reviewed-by: Allan Jude <allan@klarasystems.com>
Signed-off-by: Rob Wing <rob.wing@klarasystems.com>
Sponsored-by: Seagate Technology LLC
Closes #13805

cmd/zed/agents/zfs_diagnosis.c

@@ -39,6 +39,15 @@
 #include "zfs_agents.h"
 #include "fmd_api.h"
 
+/*
+ * Default values for the serd engine when processing checksum or io errors. The
+ * semantics are N <events> in T <seconds>.
+ */
+#define	DEFAULT_CHECKSUM_N	10	/* events */
+#define	DEFAULT_CHECKSUM_T	600	/* seconds */
+#define	DEFAULT_IO_N		10	/* events */
+#define	DEFAULT_IO_T		600	/* seconds */
+
 /*
  * Our serd engines are named 'zfs_<pool_guid>_<vdev_guid>_{checksum,io}'.  This
  * #define reserves enough space for two 64-bit hex values plus the length of
@@ -448,6 +457,8 @@ zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
 	zfs_case_t *zcp, *dcp;
 	int32_t pool_state;
 	uint64_t ena, pool_guid, vdev_guid;
+	uint64_t checksum_n, checksum_t;
+	uint64_t io_n, io_t;
 	er_timeval_t pool_load;
 	er_timeval_t er_when;
 	nvlist_t *detector;
@@ -784,11 +795,21 @@ zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
 		if (fmd_nvl_class_match(hdl, nvl,
 		    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
 			if (zcp->zc_data.zc_serd_io[0] == '\0') {
+				if (nvlist_lookup_uint64(nvl,
+				    FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
+				    &io_n) != 0) {
+					io_n = DEFAULT_IO_N;
+				}
+				if (nvlist_lookup_uint64(nvl,
+				    FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
+				    &io_t) != 0) {
+					io_t = DEFAULT_IO_T;
+				}
 				zfs_serd_name(zcp->zc_data.zc_serd_io,
 				    pool_guid, vdev_guid, "io");
 				fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
-				    fmd_prop_get_int32(hdl, "io_N"),
+				    io_n,
-				    fmd_prop_get_int64(hdl, "io_T"));
+				    SEC2NSEC(io_t));
 				zfs_case_serialize(zcp);
 			}
 			if (fmd_serd_record(hdl, zcp->zc_data.zc_serd_io, ep))
@@ -813,12 +834,23 @@ zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
 			}
 
 			if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
+				if (nvlist_lookup_uint64(nvl,
+				    FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
+				    &checksum_n) != 0) {
+					checksum_n = DEFAULT_CHECKSUM_N;
+				}
+				if (nvlist_lookup_uint64(nvl,
+				    FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
+				    &checksum_t) != 0) {
+					checksum_t = DEFAULT_CHECKSUM_T;
+				}
+
 				zfs_serd_name(zcp->zc_data.zc_serd_checksum,
 				    pool_guid, vdev_guid, "checksum");
 				fmd_serd_create(hdl,
 				    zcp->zc_data.zc_serd_checksum,
-				    fmd_prop_get_int32(hdl, "checksum_N"),
+				    checksum_n,
-				    fmd_prop_get_int64(hdl, "checksum_T"));
+				    SEC2NSEC(checksum_t));
 				zfs_case_serialize(zcp);
 			}
 			if (fmd_serd_record(hdl,

include/sys/fm/fs/zfs.h

@@ -78,6 +78,10 @@ extern "C" {
 #define	FM_EREPORT_PAYLOAD_ZFS_VDEV_READ_ERRORS	"vdev_read_errors"
 #define	FM_EREPORT_PAYLOAD_ZFS_VDEV_WRITE_ERRORS "vdev_write_errors"
 #define	FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_ERRORS "vdev_cksum_errors"
+#define	FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N	"vdev_cksum_n"
+#define	FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T	"vdev_cksum_t"
+#define	FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N	"vdev_io_n"
+#define	FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T	"vdev_io_t"
 #define	FM_EREPORT_PAYLOAD_ZFS_VDEV_DELAYS	"vdev_delays"
 #define	FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID	"parent_guid"
 #define	FM_EREPORT_PAYLOAD_ZFS_PARENT_TYPE	"parent_type"

include/sys/fs/zfs.h

@@ -356,6 +356,10 @@ typedef enum {
 	VDEV_PROP_REMOVING,
 	VDEV_PROP_ALLOCATING,
 	VDEV_PROP_FAILFAST,
+	VDEV_PROP_CHECKSUM_N,
+	VDEV_PROP_CHECKSUM_T,
+	VDEV_PROP_IO_N,
+	VDEV_PROP_IO_T,
 	VDEV_NUM_PROPS
 } vdev_prop_t;
 

include/sys/vdev_impl.h

@@ -469,6 +469,14 @@ struct vdev {
 	zfs_ratelimit_t vdev_delay_rl;
 	zfs_ratelimit_t vdev_deadman_rl;
 	zfs_ratelimit_t vdev_checksum_rl;
+
+	/*
+	 * Checksum and IO thresholds for tuning ZED
+	 */
+	uint64_t	vdev_checksum_n;
+	uint64_t	vdev_checksum_t;
+	uint64_t	vdev_io_n;
+	uint64_t	vdev_io_t;
 };
 
 #define	VDEV_PAD_SIZE		(8 << 10)

lib/libzfs/libzfs.abi

@@ -3223,7 +3223,11 @@
       <enumerator name='VDEV_PROP_REMOVING' value='39'/>
       <enumerator name='VDEV_PROP_ALLOCATING' value='40'/>
       <enumerator name='VDEV_PROP_FAILFAST' value='41'/>
-      <enumerator name='VDEV_NUM_PROPS' value='42'/>
+      <enumerator name='VDEV_PROP_CHECKSUM_N' value='42'/>
+      <enumerator name='VDEV_PROP_CHECKSUM_T' value='43'/>
+      <enumerator name='VDEV_PROP_IO_N' value='44'/>
+      <enumerator name='VDEV_PROP_IO_T' value='45'/>
+      <enumerator name='VDEV_NUM_PROPS' value='46'/>
     </enum-decl>
     <typedef-decl name='vdev_prop_t' type-id='1573bec8' id='5aa5c90c'/>
     <enum-decl name='vdev_state' id='21566197'>

lib/libzfs/libzfs_pool.c

@@ -5002,6 +5002,17 @@ zpool_get_vdev_prop_value(nvlist_t *nvprop, vdev_prop_t prop, char *prop_name,
 				    (u_longlong_t)intval);
 			}
 			break;
+		case VDEV_PROP_CHECKSUM_N:
+		case VDEV_PROP_CHECKSUM_T:
+		case VDEV_PROP_IO_N:
+		case VDEV_PROP_IO_T:
+			if (intval == UINT64_MAX) {
+				(void) strlcpy(buf, "-", len);
+			} else {
+				(void) snprintf(buf, len, "%llu",
+				    (u_longlong_t)intval);
+			}
+			break;
 		case VDEV_PROP_FRAGMENTATION:
 			if (intval == UINT64_MAX) {
 				(void) strlcpy(buf, "-", len);

lib/libzfs/libzfs_util.c

@@ -1681,6 +1681,18 @@ zprop_parse_value(libzfs_handle_t *hdl, nvpair_t *elem, int prop,
 			*ivalp = UINT64_MAX;
 		}
 
+		/*
+		 * Special handling for "checksum_*=none". In this case it's not
+		 * 0 but UINT64_MAX.
+		 */
+		if ((type & ZFS_TYPE_VDEV) && isnone &&
+		    (prop == VDEV_PROP_CHECKSUM_N ||
+		    prop == VDEV_PROP_CHECKSUM_T ||
+		    prop == VDEV_PROP_IO_N ||
+		    prop == VDEV_PROP_IO_T)) {
+			*ivalp = UINT64_MAX;
+		}
+
 		/*
 		 * Special handling for setting 'refreservation' to 'auto'.  Use
 		 * UINT64_MAX to tell the caller to use zfs_fix_auto_resv().

man/man7/vdevprops.7

@@ -43,7 +43,8 @@ section, below.
 .Ss Native Properties
 Every vdev has a set of properties that export statistics about the vdev
 as well as control various behaviors.
-Properties are NOT inherited from top-level vdevs.
+Properties are not inherited from top-level vdevs, with the exception of
+checksum_n, checksum_t, io_n, and io_t.
 .Pp
 The values of numeric properties can be specified using human-readable suffixes
 .Po for example,
@@ -114,9 +115,19 @@ The cumulative size of all operations of each type performed by this vdev
 If this device is currently being removed from the pool
 .El
 .Pp
-The following native properties can be used to change the behavior of a ZFS
+The following native properties can be used to change the behavior of a vdev.
-dataset.
 .Bl -tag -width "allocating"
+.It Sy checksum_n , checksum_t , io_n , io_t
+Tune the fault management daemon by specifying checksum/io thresholds of <N>
+errors in <T> seconds, respectively.
+These properties can be set on leaf and top-level vdevs.
+When the property is set on the leaf and top-level vdev, the value of the leaf
+vdev will be used.
+If the property is only set on the top-level vdev, this value will be used.
+The value of these properties do not persist across vdev replacement.
+For this reason, it is advisable to set the property on the top-level vdev -
+not on the leaf vdev itself.
+The default values are 10 errors in 600 seconds.
 .It Sy comment
 A text comment up to 8192 characters long
 .It Sy bootsize

module/zcommon/zpool_prop.c

@@ -410,6 +410,18 @@ vdev_prop_init(void)
 	    sfeatures);
 
 	/* default numeric properties */
+	zprop_register_number(VDEV_PROP_CHECKSUM_N, "checksum_n", UINT64_MAX,
+	    PROP_DEFAULT, ZFS_TYPE_VDEV, "<events>", "CKSUM_N", B_FALSE,
+	    sfeatures);
+	zprop_register_number(VDEV_PROP_CHECKSUM_T, "checksum_t", UINT64_MAX,
+	    PROP_DEFAULT, ZFS_TYPE_VDEV, "<seconds>", "CKSUM_T", B_FALSE,
+	    sfeatures);
+	zprop_register_number(VDEV_PROP_IO_N, "io_n", UINT64_MAX,
+	    PROP_DEFAULT, ZFS_TYPE_VDEV, "<events>", "IO_N", B_FALSE,
+	    sfeatures);
+	zprop_register_number(VDEV_PROP_IO_T, "io_t", UINT64_MAX,
+	    PROP_DEFAULT, ZFS_TYPE_VDEV, "<seconds>", "IO_T", B_FALSE,
+	    sfeatures);
 
 	/* default index (boolean) properties */
 	zprop_register_index(VDEV_PROP_REMOVING, "removing", 0,

module/zfs/vdev.c

@@ -389,6 +389,31 @@ vdev_get_nparity(vdev_t *vd)
 	return (nparity);
 }
 
+static int
+vdev_prop_get_int(vdev_t *vd, vdev_prop_t prop, uint64_t *value)
+{
+	spa_t *spa = vd->vdev_spa;
+	objset_t *mos = spa->spa_meta_objset;
+	uint64_t objid;
+	int err;
+
+	if (vd->vdev_top_zap != 0) {
+		objid = vd->vdev_top_zap;
+	} else if (vd->vdev_leaf_zap != 0) {
+		objid = vd->vdev_leaf_zap;
+	} else {
+		return (EINVAL);
+	}
+
+	err = zap_lookup(mos, objid, vdev_prop_to_name(prop),
+	    sizeof (uint64_t), 1, value);
+
+	if (err == ENOENT)
+		*value = vdev_prop_default_numeric(prop);
+
+	return (err);
+}
+
 /*
  * Get the number of data disks for a top-level vdev.
  */
@@ -642,6 +667,14 @@ vdev_alloc_common(spa_t *spa, uint_t id, uint64_t guid, vdev_ops_t *ops)
 	zfs_ratelimit_init(&vd->vdev_checksum_rl,
 	    &zfs_checksum_events_per_second, 1);
 
+	/*
+	 * Default Thresholds for tuning ZED
+	 */
+	vd->vdev_checksum_n = vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_N);
+	vd->vdev_checksum_t = vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_T);
+	vd->vdev_io_n = vdev_prop_default_numeric(VDEV_PROP_IO_N);
+	vd->vdev_io_t = vdev_prop_default_numeric(VDEV_PROP_IO_T);
+
 	list_link_init(&vd->vdev_config_dirty_node);
 	list_link_init(&vd->vdev_state_dirty_node);
 	list_link_init(&vd->vdev_initialize_node);
@@ -3597,6 +3630,39 @@ vdev_load(vdev_t *vd)
 		}
 	}
 
+	if (vd->vdev_top_zap != 0 || vd->vdev_leaf_zap != 0) {
+		uint64_t zapobj;
+
+		if (vd->vdev_top_zap != 0)
+			zapobj = vd->vdev_top_zap;
+		else
+			zapobj = vd->vdev_leaf_zap;
+
+		error = vdev_prop_get_int(vd, VDEV_PROP_CHECKSUM_N,
+		    &vd->vdev_checksum_n);
+		if (error && error != ENOENT)
+			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
+			    "failed [error=%d]", (u_longlong_t)zapobj, error);
+
+		error = vdev_prop_get_int(vd, VDEV_PROP_CHECKSUM_T,
+		    &vd->vdev_checksum_t);
+		if (error && error != ENOENT)
+			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
+			    "failed [error=%d]", (u_longlong_t)zapobj, error);
+
+		error = vdev_prop_get_int(vd, VDEV_PROP_IO_N,
+		    &vd->vdev_io_n);
+		if (error && error != ENOENT)
+			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
+			    "failed [error=%d]", (u_longlong_t)zapobj, error);
+
+		error = vdev_prop_get_int(vd, VDEV_PROP_IO_T,
+		    &vd->vdev_io_t);
+		if (error && error != ENOENT)
+			vdev_dbgmsg(vd, "vdev_load: zap_lookup(zap=%llu) "
+			    "failed [error=%d]", (u_longlong_t)zapobj, error);
+	}
+
 	/*
 	 * If this is a top-level vdev, initialize its metaslabs.
 	 */
@@ -5736,6 +5802,34 @@ vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl)
 			}
 			vd->vdev_failfast = intval & 1;
 			break;
+		case VDEV_PROP_CHECKSUM_N:
+			if (nvpair_value_uint64(elem, &intval) != 0) {
+				error = EINVAL;
+				break;
+			}
+			vd->vdev_checksum_n = intval;
+			break;
+		case VDEV_PROP_CHECKSUM_T:
+			if (nvpair_value_uint64(elem, &intval) != 0) {
+				error = EINVAL;
+				break;
+			}
+			vd->vdev_checksum_t = intval;
+			break;
+		case VDEV_PROP_IO_N:
+			if (nvpair_value_uint64(elem, &intval) != 0) {
+				error = EINVAL;
+				break;
+			}
+			vd->vdev_io_n = intval;
+			break;
+		case VDEV_PROP_IO_T:
+			if (nvpair_value_uint64(elem, &intval) != 0) {
+				error = EINVAL;
+				break;
+			}
+			vd->vdev_io_t = intval;
+			break;
 		default:
 			/* Most processing is done in vdev_props_set_sync */
 			break;
@@ -6025,28 +6119,25 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl)
 				continue;
 			/* Numeric Properites */
 			case VDEV_PROP_ALLOCATING:
-				src = ZPROP_SRC_LOCAL;
-				strval = NULL;
-
-				err = zap_lookup(mos, objid, nvpair_name(elem),
-				    sizeof (uint64_t), 1, &intval);
-				if (err == ENOENT) {
-					intval =
-					    vdev_prop_default_numeric(prop);
-					err = 0;
-				} else if (err)
-					break;
-				if (intval == vdev_prop_default_numeric(prop))
-					src = ZPROP_SRC_DEFAULT;
-
 				/* Leaf vdevs cannot have this property */
 				if (vd->vdev_mg == NULL &&
 				    vd->vdev_top != NULL) {
 					src = ZPROP_SRC_NONE;
 					intval = ZPROP_BOOLEAN_NA;
+				} else {
+					err = vdev_prop_get_int(vd, prop,
+					    &intval);
+					if (err && err != ENOENT)
+						break;
+
+					if (intval ==
+					    vdev_prop_default_numeric(prop))
+						src = ZPROP_SRC_DEFAULT;
+					else
+						src = ZPROP_SRC_LOCAL;
 				}
 
-				vdev_prop_add_list(outnvl, propname, strval,
+				vdev_prop_add_list(outnvl, propname, NULL,
 				    intval, src);
 				break;
 			case VDEV_PROP_FAILFAST:
@@ -6068,6 +6159,22 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl)
 				vdev_prop_add_list(outnvl, propname, strval,
 				    intval, src);
 				break;
+			case VDEV_PROP_CHECKSUM_N:
+			case VDEV_PROP_CHECKSUM_T:
+			case VDEV_PROP_IO_N:
+			case VDEV_PROP_IO_T:
+				err = vdev_prop_get_int(vd, prop, &intval);
+				if (err && err != ENOENT)
+					break;
+
+				if (intval == vdev_prop_default_numeric(prop))
+					src = ZPROP_SRC_DEFAULT;
+				else
+					src = ZPROP_SRC_LOCAL;
+
+				vdev_prop_add_list(outnvl, propname, NULL,
+				    intval, src);
+				break;
 			/* Text Properties */
 			case VDEV_PROP_COMMENT:
 				/* Exists in the ZAP below */

module/zfs/zfs_fm.c

@@ -200,6 +200,42 @@ recent_events_compare(const void *a, const void *b)
 	return (0);
 }
 
+/*
+ * workaround: vdev properties don't have inheritance
+ */
+static uint64_t
+vdev_prop_get_inherited(vdev_t *vd, vdev_prop_t prop)
+{
+	uint64_t propdef, propval;
+
+	propdef = vdev_prop_default_numeric(prop);
+	switch (prop) {
+		case VDEV_PROP_CHECKSUM_N:
+			propval = vd->vdev_checksum_n;
+			break;
+		case VDEV_PROP_CHECKSUM_T:
+			propval = vd->vdev_checksum_t;
+			break;
+		case VDEV_PROP_IO_N:
+			propval = vd->vdev_io_n;
+			break;
+		case VDEV_PROP_IO_T:
+			propval = vd->vdev_io_t;
+			break;
+		default:
+			propval = propdef;
+			break;
+	}
+
+	if (propval != propdef)
+		return (propval);
+
+	if (vd->vdev_parent == NULL)
+		return (propdef);
+
+	return (vdev_prop_get_inherited(vd->vdev_parent, prop));
+}
+
 static void zfs_ereport_schedule_cleaner(void);
 
 /*
@@ -662,6 +698,49 @@ zfs_ereport_start(nvlist_t **ereport_out, nvlist_t **detector_out,
 		    DATA_TYPE_UINT64, zb->zb_blkid, NULL);
 	}
 
+	/*
+	 * Payload for tuning the zed
+	 */
+	if (vd != NULL && strcmp(subclass, FM_EREPORT_ZFS_CHECKSUM) == 0) {
+		uint64_t cksum_n, cksum_t;
+
+		cksum_n = vdev_prop_get_inherited(vd, VDEV_PROP_CHECKSUM_N);
+		if (cksum_n != vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_N))
+			fm_payload_set(ereport,
+			    FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_N,
+			    DATA_TYPE_UINT64,
+			    cksum_n,
+			    NULL);
+
+		cksum_t = vdev_prop_get_inherited(vd, VDEV_PROP_CHECKSUM_T);
+		if (cksum_t != vdev_prop_default_numeric(VDEV_PROP_CHECKSUM_T))
+			fm_payload_set(ereport,
+			    FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_T,
+			    DATA_TYPE_UINT64,
+			    cksum_t,
+			    NULL);
+	}
+
+	if (vd != NULL && strcmp(subclass, FM_EREPORT_ZFS_IO) == 0) {
+		uint64_t io_n, io_t;
+
+		io_n = vdev_prop_get_inherited(vd, VDEV_PROP_IO_N);
+		if (io_n != vdev_prop_default_numeric(VDEV_PROP_IO_N))
+			fm_payload_set(ereport,
+			    FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_N,
+			    DATA_TYPE_UINT64,
+			    io_n,
+			    NULL);
+
+		io_t = vdev_prop_get_inherited(vd, VDEV_PROP_IO_T);
+		if (io_t != vdev_prop_default_numeric(VDEV_PROP_IO_T))
+			fm_payload_set(ereport,
+			    FM_EREPORT_PAYLOAD_ZFS_VDEV_IO_T,
+			    DATA_TYPE_UINT64,
+			    io_t,
+			    NULL);
+	}
+
 	mutex_exit(&spa->spa_errlist_lock);
 
 	*ereport_out = ereport;

tests/runfiles/linux.run

@@ -87,7 +87,7 @@ tags = ['functional', 'devices']
 
 [tests/functional/events:Linux]
 tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter', 'zed_fd_spill',
-    'zed_cksum_reported']
+    'zed_cksum_reported', 'zed_cksum_config', 'zed_io_config']
 tags = ['functional', 'events']
 
 [tests/functional/fadvise:Linux]

tests/zfs-tests/tests/Makefile.am

@@ -1367,8 +1367,10 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/events/events_001_pos.ksh \
 	functional/events/events_002_pos.ksh \
 	functional/events/setup.ksh \
+	functional/events/zed_cksum_config.ksh \
 	functional/events/zed_cksum_reported.ksh \
 	functional/events/zed_fd_spill.ksh \
+	functional/events/zed_io_config.ksh \
 	functional/events/zed_rc_filter.ksh \
 	functional/exec/cleanup.ksh \
 	functional/exec/exec_001_pos.ksh \

tests/zfs-tests/tests/functional/events/zed_cksum_config.ksh

@@ -0,0 +1,158 @@
+#!/bin/ksh -p
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2022, Klara Inc.
+#
+
+# DESCRIPTION:
+#	Verify that vdev properties, checksum_n and checksum_t, work with ZED.
+#
+# STRATEGY:
+#	1. Create a pool with single vdev
+#	2. Set checksum_n/checksum_t to non-default values
+#	3. Inject checksum errors
+#	4. Verify that ZED degrades vdev
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+MOUNTDIR="$TEST_BASE_DIR/checksum_mount"
+FILEPATH="$MOUNTDIR/checksum_file"
+VDEV="$TEST_BASE_DIR/vdevfile.$$"
+POOL="checksum_pool"
+FILESIZE="10M"
+
+function cleanup
+{
+	log_must zed_stop
+
+	log_must zinject -c all
+	if poolexists $POOL ; then
+		destroy_pool $POOL
+	fi
+	log_must rm -fd $VDEV $MOUNTDIR
+}
+
+log_onexit cleanup
+
+log_assert "Test ZED checksum_N and checksum_T configurability"
+
+function do_setup
+{
+	log_must zpool create -f -m $MOUNTDIR $POOL $VDEV
+	log_must zpool events -c
+	log_must truncate -s 0 $ZED_DEBUG_LOG
+	log_must zfs set compression=off $POOL
+	log_must zfs set primarycache=none $POOL
+	log_must zfs set recordsize=512 $POOL
+}
+
+function do_clean
+{
+	log_must zinject -c all
+	log_must zpool destroy $POOL
+}
+
+function must_degrade
+{
+	log_must wait_vdev_state $POOL $VDEV "DEGRADED" 60
+}
+
+function mustnot_degrade
+{
+	log_must file_wait $ZED_DEBUG_LOG 5
+	log_must wait_vdev_state $POOL $VDEV "ONLINE" 60
+}
+
+# Test default settings of ZED:
+#   checksum_n=10
+#   checksum_t=600
+# fire 10 events, should degrade.
+function default_degrade
+{
+	do_setup
+
+	log_must mkfile $FILESIZE $FILEPATH
+	log_must zinject -a -t data -e checksum -T read -f 100 $FILEPATH
+
+	blk=0
+	for _ in {1..10}; do
+		dd if=$FILEPATH of=/dev/null bs=1 count=1 skip=$blk 2>/dev/null
+		blk=$((blk+512))
+	done
+
+	must_degrade
+
+	do_clean
+}
+
+# Set checksum_t=1
+# fire 10 events over 2.5 seconds, should not degrade.
+function checksum_t_no_degrade
+{
+	do_setup
+
+	log_must zpool set checksum_t=1 $POOL $VDEV
+	log_must mkfile $FILESIZE $FILEPATH
+	log_must zinject -a -t data -e checksum -T read -f 100 $FILEPATH
+
+	blk=0
+	for _ in {1..10}; do
+		dd if=$FILEPATH of=/dev/null bs=1 count=1 skip=$blk 2>/dev/null
+		blk=$((blk+512))
+		sleep 0.25
+	done
+
+	mustnot_degrade
+
+	do_clean
+}
+
+# Set checksum_n=1
+# fire 1 event, should degrade.
+function checksum_n_degrade
+{
+	do_setup
+
+	log_must zpool set checksum_n=1 $POOL $VDEV
+	log_must mkfile $FILESIZE $FILEPATH
+	log_must zinject -a -t data -e checksum -T read -f 100 $FILEPATH
+
+	dd if=$FILEPATH of=/dev/null bs=1 count=1 2>/dev/null
+
+	must_degrade
+
+	do_clean
+}
+
+log_must truncate -s $MINVDEVSIZE $VDEV
+log_must mkdir -p $MOUNTDIR
+
+log_must zed_start
+default_degrade
+checksum_n_degrade
+checksum_t_no_degrade
+
+log_pass "Test ZED checksum_N and checksum_T configurability"

tests/zfs-tests/tests/functional/events/zed_io_config.ksh

@@ -0,0 +1,150 @@
+#!/bin/ksh -p
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2022, Klara Inc.
+#
+
+# DESCRIPTION:
+#	Verify that vdev properties, io_n and io_t, work with ZED.
+#
+# STRATEGY:
+#	1. Create a mirrored pool.
+#	3. Set io_n/io_t to non-default values
+#	3. Inject io errors
+#	4. Verify that ZED degrades vdev
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+MOUNTDIR="$TEST_BASE_DIR/io_mount"
+FILEPATH="$MOUNTDIR/io_file"
+VDEV="$TEST_BASE_DIR/vdevfile.$$"
+VDEV1="$TEST_BASE_DIR/vdevfile1.$$"
+POOL="io_pool"
+
+function cleanup
+{
+	log_must zed_stop
+
+	log_must zinject -c all
+	if poolexists $POOL ; then
+		destroy_pool $POOL
+	fi
+	log_must rm -fd $VDEV $VDEV1 $MOUNTDIR
+	log_must set_tunable32 PREFETCH_DISABLE $zfsprefetch
+}
+log_onexit cleanup
+
+log_assert "Test ZED io_n and io_t configurability"
+
+zfsprefetch=$(get_tunable PREFETCH_DISABLE)
+log_must set_tunable32 PREFETCH_DISABLE 1
+
+function setup_pool
+{
+	log_must zpool create -f -m $MOUNTDIR $POOL mirror $VDEV $VDEV1
+	log_must zpool events -c
+	log_must truncate -s 0 $ZED_DEBUG_LOG
+	log_must zfs set compression=off $POOL
+	log_must zfs set primarycache=none $POOL
+	log_must zfs set recordsize=512 $POOL
+}
+
+function do_clean
+{
+	log_must zinject -c all
+	log_must zpool destroy $POOL
+}
+
+# Test default ZED settings:
+#   io_n=10	(events)
+#   io_t=600	(seconds)
+# fire 10 events over 2.5 seconds, should degrade.
+function default_degrade
+{
+	setup_pool
+
+	log_must dd if=/dev/urandom of=$FILEPATH bs=1M count=64
+	log_must zinject -a -d $VDEV -e io -T read -f 100 $POOL
+
+	blk=0
+	for _ in {1..10}; do
+		dd if=$FILEPATH of=/dev/null bs=1 count=1 skip=$blk 2>/dev/null
+		blk=$((blk+512))
+		sleep 0.25
+	done
+
+	log_must wait_vdev_state $POOL $VDEV "FAULTED" 60
+	do_clean
+}
+
+# set io_n=1
+# fire 1 event, should degrade
+function io_n_degrade
+{
+	setup_pool
+
+	log_must zpool set io_n=1 $POOL $VDEV
+	log_must dd if=/dev/urandom of=$FILEPATH bs=1M count=64
+	log_must zinject -a -d $VDEV -e io -T read -f 100 $POOL
+
+	dd if=$FILEPATH of=/dev/null bs=1 count=1 2>/dev/null
+
+	log_must wait_vdev_state $POOL $VDEV "FAULTED" 60
+	do_clean
+}
+
+# set io_t=1
+# fire 10 events over 2.5 seconds, should not degrade
+function io_t_nodegrade
+{
+	setup_pool
+
+	log_must zpool set io_t=1 $POOL $VDEV
+	log_must dd if=/dev/urandom of=$FILEPATH bs=1M count=64
+	log_must zinject -a -d $VDEV -e io -T read -f 100 $POOL
+
+	blk=0
+	for _ in {1..10}; do
+		dd if=$FILEPATH of=/dev/null bs=1 count=1 skip=$blk 2>/dev/null
+		blk=$((blk+512))
+		sleep 0.25
+	done
+
+	log_must file_wait $ZED_DEBUG_LOG 30
+	log_must wait_vdev_state $POOL $VDEV "ONLINE" 1
+
+	do_clean
+}
+
+log_must truncate -s $MINVDEVSIZE $VDEV
+log_must truncate -s $MINVDEVSIZE $VDEV1
+log_must mkdir -p $MOUNTDIR
+
+log_must zed_start
+default_degrade
+io_n_degrade
+io_t_nodegrade
+
+log_pass "Test ZED io_n and io_t configurability"