zed: misc vdev_enc_sysfs_path fixes

There have been rare cases where the VDEV_ENC_SYSFS_PATH value that zed
gets passed is stale.  To mitigate this, dynamically check the sysfs
path at the time of zed event processing, and use the dynamic value if
possible.  Note that there will be other times when we can not
dynamically detect the sysfs path (like if a disk disappears) and have
to rely on the old value for things like turning on the fault LED.  That
is to say, we can't just blindly use the dynamic path in every case.

Also:
	- Add enclosure sysfs entry when running 'zpool add'
	- Fix 'slot' and 'enc' zpool.d scripts for nvme

Reviewed-by: Don Brady <dev.fs.zfs@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Closes #15462

cmd/zed/agents/zfs_mod.c

@@ -233,8 +233,12 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	}
 
 	(void) nvlist_lookup_string(vdev, ZPOOL_CONFIG_PHYS_PATH, &physpath);
+
+	update_vdev_config_dev_sysfs_path(vdev, path,
+	    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
 	(void) nvlist_lookup_string(vdev, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
 	    &enc_sysfs_path);
+
 	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_WHOLE_DISK, &wholedisk);
 	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_OFFLINE, &offline);
 	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_FAULTED, &faulted);

cmd/zed/zed_event.c

@@ -35,6 +35,7 @@
 #include "zed_strings.h"
 
 #include "agents/zfs_agents.h"
+#include <libzutil.h>
 
 #define	MAXBUF	4096
 
@@ -922,6 +923,25 @@ _zed_event_add_time_strings(uint64_t eid, zed_strings_t *zsp, int64_t etime[])
 	}
 }
 
+
+static void
+_zed_event_update_enc_sysfs_path(nvlist_t *nvl)
+{
+	const char *vdev_path;
+
+	if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_PATH,
+	    &vdev_path) != 0) {
+		return; /* some other kind of event, ignore it */
+	}
+
+	if (vdev_path == NULL) {
+		return;
+	}
+
+	update_vdev_config_dev_sysfs_path(nvl, vdev_path,
+	    FM_EREPORT_PAYLOAD_ZFS_VDEV_ENC_SYSFS_PATH);
+}
+
 /*
  * Service the next zevent, blocking until one is available.
  */
@@ -969,6 +989,17 @@ zed_event_service(struct zed_conf *zcp)
 		zed_log_msg(LOG_WARNING,
 		    "Failed to lookup zevent class (eid=%llu)", eid);
 	} else {
+		/*
+		 * Special case: If we can dynamically detect an enclosure sysfs
+		 * path, then use that value rather than the one stored in the
+		 * vd->vdev_enc_sysfs_path.  There have been rare cases where
+		 * vd->vdev_enc_sysfs_path becomes outdated.  However, there
+		 * will be other times when we can not dynamically detect the
+		 * sysfs path (like if a disk disappears) and have to rely on
+		 * the old value for things like turning on the fault LED.
+		 */
+		_zed_event_update_enc_sysfs_path(nvl);
+
 		/* let internal modules see this event first */
 		zfs_agent_post_event(class, NULL, nvl);
 

cmd/zpool/zpool.d/ses

@@ -33,10 +33,18 @@ for i in $scripts ; do
 	val=""
 	case $i in
 	enc)
-		val=$(ls "$VDEV_ENC_SYSFS_PATH/../../" 2>/dev/null)
+		if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then
+			val="$VDEV_ENC_SYSFS_PATH"
+		else
+			val="$(ls """$VDEV_ENC_SYSFS_PATH/../../""" 2>/dev/null)"
+		fi
 		;;
 	slot)
-		val=$(cat "$VDEV_ENC_SYSFS_PATH/slot" 2>/dev/null)
+		if echo "$VDEV_ENC_SYSFS_PATH" | grep -q '/sys/bus/pci/slots' ; then
+			val="$(basename """$VDEV_ENC_SYSFS_PATH""")"
+		else
+			val="$(cat """$VDEV_ENC_SYSFS_PATH/slot""" 2>/dev/null)"
+		fi
 		;;
 	encdev)
 		val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null)

cmd/zpool/zpool_vdev.c

@@ -372,6 +372,10 @@ make_leaf_vdev(nvlist_t *props, const char *arg, boolean_t is_primary)
 	verify(nvlist_add_string(vdev, ZPOOL_CONFIG_PATH, path) == 0);
 	verify(nvlist_add_string(vdev, ZPOOL_CONFIG_TYPE, type) == 0);
 
+	/* Lookup and add the enclosure sysfs path (if exists) */
+	update_vdev_config_dev_sysfs_path(vdev, path,
+	    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
+
 	if (strcmp(type, VDEV_TYPE_DISK) == 0)
 		verify(nvlist_add_uint64(vdev, ZPOOL_CONFIG_WHOLE_DISK,
 		    (uint64_t)wholedisk) == 0);

include/libzutil.h

@@ -208,6 +208,8 @@ int for_each_vdev_cb(void *zhp, nvlist_t *nv, pool_vdev_iter_f func,
 int for_each_vdev_in_nvlist(nvlist_t *nvroot, pool_vdev_iter_f func,
     void *data);
 void update_vdevs_config_dev_sysfs_path(nvlist_t *config);
+_LIBZUTIL_H void update_vdev_config_dev_sysfs_path(nvlist_t *nv,
+    const char *path, const char *key);
 #ifdef	__cplusplus
 }
 #endif

lib/libzfs/libzfs.abi

@@ -260,6 +260,7 @@
     <elf-symbol name='tpool_suspended' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='tpool_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='update_vdev_config_dev_strs' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='update_vdev_config_dev_sysfs_path' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='use_color' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='vdev_expand_proplist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='vdev_name_to_prop' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -8329,6 +8330,12 @@
       <parameter type-id='b59d7dce' name='buflen'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='update_vdev_config_dev_sysfs_path' mangled-name='update_vdev_config_dev_sysfs_path' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='update_vdev_config_dev_sysfs_path'>
+      <parameter type-id='5ce45b60' name='nv'/>
+      <parameter type-id='80f4b756' name='path'/>
+      <parameter type-id='80f4b756' name='key'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-type size-in-bits='64' id='2ec2411e'>
       <parameter type-id='eaa32e2f'/>
       <parameter type-id='5ce45b60'/>

lib/libzutil/os/freebsd/zutil_import_os.c

@@ -249,6 +249,15 @@ zfs_dev_flush(int fd)
 	return (0);
 }
 
+void
+update_vdev_config_dev_sysfs_path(nvlist_t *nv, const char *path,
+    const char *key)
+{
+	(void) nv;
+	(void) path;
+	(void) key;
+}
+
 void
 update_vdevs_config_dev_sysfs_path(nvlist_t *config)
 {

lib/libzutil/os/linux/zutil_import_os.c

@@ -766,9 +766,12 @@ no_dev:
  * Rescan the enclosure sysfs path for turning on enclosure LEDs and store it
  * in the nvlist * (if applicable).  Like:
  *    vdev_enc_sysfs_path: '/sys/class/enclosure/11:0:1:0/SLOT 4'
+ *
+ * key: The nvlist_t name (like ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH)
  */
-static void
-update_vdev_config_dev_sysfs_path(nvlist_t *nv, const char *path)
+void
+update_vdev_config_dev_sysfs_path(nvlist_t *nv, const char *path,
+    const char *key)
 {
 	char *upath, *spath;
 
@@ -777,9 +780,9 @@ update_vdev_config_dev_sysfs_path(nvlist_t *nv, const char *path)
 	spath = zfs_get_enclosure_sysfs_path(upath);
 
 	if (spath) {
-		nvlist_add_string(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH, spath);
+		(void) nvlist_add_string(nv, key, spath);
 	} else {
-		nvlist_remove_all(nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
+		(void) nvlist_remove_all(nv, key);
 	}
 
 	free(upath);
@@ -799,7 +802,8 @@ sysfs_path_pool_vdev_iter_f(void *hdl_data, nvlist_t *nv, void *data)
 		return (1);
 
 	/* Rescan our enclosure sysfs path for this vdev */
-	update_vdev_config_dev_sysfs_path(nv, path);
+	update_vdev_config_dev_sysfs_path(nv, path,
+	    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
 	return (0);
 }
 
@@ -888,7 +892,8 @@ update_vdev_config_dev_strs(nvlist_t *nv)
 			(void) nvlist_add_string(nv, ZPOOL_CONFIG_PHYS_PATH,
 			    vds.vds_devphys);
 		}
-		update_vdev_config_dev_sysfs_path(nv, path);
+		update_vdev_config_dev_sysfs_path(nv, path,
+		    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
 	} else {
 		/* Clear out any stale entries. */
 		(void) nvlist_remove_all(nv, ZPOOL_CONFIG_DEVID);