Prebaked scripts for zpool status/iostat -c

This patch updates the "zpool status/iostat -c" commands to only run
"pre-baked" scripts from the /etc/zfs/zpool.d directory (or wherever
you install to).  The scripts can only be run from -c as an unprivileged
user (unless the ZPOOL_SCRIPTS_AS_ROOT environment var is
set by root).  This was done to encourage scripts to be written is such
a way that normal users can use them, and to be cautious.  If your
script needs to run a privileged command, consider adding the
appropriate line in /etc/sudoers.  See zpool(8) for an example of how
to do this.

The patch also allows the scripts to output custom column names.  If
the script outputs a line like:

name=value

then "name" is used for the column name, and "value" is its value.
Multiple columns can be specified by outputting multiple lines.  Column
names and values can have spaces.  If the value is empty, a dash (-) is
printed instead.

After all the "name=value" lines are read (if any), zpool will take the
next the next line of output (if any) and print it without a column
header.  After that, no more lines will be processed. This can be
useful for printing errors.

Lastly, this patch also disables the -c option with the latency and
request size histograms, since it produced awkward output and made the
code harder to maintain.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Giuseppe Di Natale <dinatale2@llnl.gov>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Closes #5852

Makefile.am

@@ -57,7 +57,8 @@ shellcheck:
 			scripts/zfs-tests.sh \
 			scripts/zfs.sh \
 			scripts/commitcheck.sh \
-			$$(find cmd/zed/zed.d/*.sh -type f); \
+			$$(find cmd/zed/zed.d/*.sh -type f) \
+			$$(find cmd/zpool/zpool.d/* -executable); \
 	fi
 
 lint: cppcheck paxcheck

cmd/zpool/Makefile.am

@@ -20,3 +20,54 @@ zpool_LDADD = \
 	$(top_builddir)/lib/libzfs/libzfs.la \
 	$(top_builddir)/lib/libzfs_core/libzfs_core.la \
 	-lm $(LIBBLKID)
+
+zpoolconfdir = $(sysconfdir)/zfs/zpool.d
+zpoolexecdir = $(libexecdir)/zfs/zpool.d
+
+EXTRA_DIST = zpool.d/README
+
+dist_zpoolexec_SCRIPTS = \
+	zpool.d/enc \
+	zpool.d/encdev \
+	zpool.d/fault_led \
+	zpool.d/iostat \
+	zpool.d/iostat-1s \
+	zpool.d/iostat-10s \
+	zpool.d/label \
+	zpool.d/locate_led \
+	zpool.d/lsblk \
+	zpool.d/model \
+	zpool.d/serial \
+	zpool.d/ses \
+	zpool.d/size \
+	zpool.d/slaves \
+	zpool.d/slot \
+	zpool.d/upath \
+	zpool.d/vendor
+
+zpoolconfdefaults = \
+	enc \
+	encdev \
+	fault_led \
+	iostat \
+	iostat-1s \
+	iostat-10s \
+	label \
+	locate_led \
+	lsblk \
+	model \
+	serial \
+	ses \
+	size \
+	slaves \
+	slot \
+	upath \
+	vendor
+
+install-data-hook:
+	$(MKDIR_P) "$(DESTDIR)$(zpoolconfdir)"
+	for f in $(zpoolconfdefaults); do \
+	  test -f "$(DESTDIR)$(zpoolconfdir)/$${f}" -o \
+	       -L "$(DESTDIR)$(zpoolconfdir)/$${f}" || \
+	    ln -s "$(zpoolexecdir)/$${f}" "$(DESTDIR)$(zpoolconfdir)"; \
+	done

cmd/zpool/zpool.d/README

@@ -0,0 +1,9 @@
+This directory contains scripts that can be run the zpool status/iostat
+-c option:
+
+	zpool status -c script1,script2, ...
+
+	zpool iostat -vc script1,script2, ...
+
+Some scripts output different values depending on the symlink name that is
+used to run them.  See the zpool(8) man page for more details.

cmd/zpool/zpool.d/enc

@@ -0,0 +1 @@
+ses

cmd/zpool/zpool.d/encdev

@@ -0,0 +1 @@
+ses

cmd/zpool/zpool.d/fault_led

@@ -0,0 +1 @@
+ses

cmd/zpool/zpool.d/iostat

@@ -0,0 +1,65 @@
+#!/bin/sh
+#
+# Display most relevant iostat bandwidth/latency numbers.  The output is
+# dependent on the name of the script/symlink used to call it.
+#
+
+helpstr="
+iostat:		Show iostat values since boot (summary page).
+iostat-1s:	Do a single 1-second iostat sample and show values.
+iostat-10s:	Do a single 10-second iostat sample and show values."
+
+script=$(basename "$0")
+if [ "$1" = "-h" ] ; then
+	echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
+	exit
+fi
+
+if [ "$script" = "iostat-1s" ] ; then
+	# Do a single one-second sample
+	extra="1 1"
+	# Don't show summary stats
+	y="-y"
+elif [ "$script" = "iostat-10s" ] ; then
+	# Do a single ten-second sample
+	extra="10 1"
+	# Don't show summary stats
+	y="-y"
+fi
+
+if [ -f "$VDEV_UPATH" ] ; then
+	# We're a file-based vdev, iostat doesn't work on us.  Do nothing.
+	exit
+fi
+
+out=$(eval "iostat $y -k -x $VDEV_UPATH $extra")
+
+# Sample output (we want the last two lines):
+#
+# Linux 2.6.32-642.13.1.el6.x86_64 (centos68) 	03/09/2017 	_x86_64_	(6 CPU)
+#
+# avg-cpu:  %user   %nice %system %iowait  %steal   %idle
+#           0.00    0.00    0.00    0.00    0.00  100.00
+#
+# Device:         rrqm/s   wrqm/s     r/s     w/s    rkB/s    wkB/s avgrq-sz avgqu-sz   await r_await w_await  svctm  %util
+# sdb               0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00    0.00    0.00   0.00   0.00
+#
+
+# Get the column names
+cols=$(echo "$out" | grep Device)
+
+# Get the values and tab separate them to make them cut-able.
+vals="$(echo "$out" | grep -A1 Device | tail -n 1 | sed -r 's/[[:blank:]]+/\t/g')"
+
+i=0
+for col in $cols ; do
+	i=$((i+1))
+	# Skip the first column since it's just the device name
+	if [ "$col" = "Device:" ] ; then
+		continue
+	fi
+
+	# Get i'th value
+	val=$(echo "$vals" | cut -f "$i")
+	echo "$col=$val"
+done

cmd/zpool/zpool.d/iostat-10s

@@ -0,0 +1 @@
+iostat

cmd/zpool/zpool.d/iostat-1s

@@ -0,0 +1 @@
+iostat

cmd/zpool/zpool.d/label

@@ -0,0 +1 @@
+lsblk

cmd/zpool/zpool.d/locate_led

@@ -0,0 +1 @@
+ses

cmd/zpool/zpool.d/lsblk

@@ -0,0 +1,84 @@
+#!/bin/sh
+#
+# Print some common lsblk values
+#
+# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk
+# as one of its --output names.  Here's a partial list of --output names
+# from the lsblk binary:
+#
+# Available columns (for --output):
+#        NAME  device name
+#       KNAME  internal kernel device name
+#     MAJ:MIN  major:minor device number
+#      FSTYPE  filesystem type
+#  MOUNTPOINT  where the device is mounted
+#       LABEL  filesystem LABEL
+#        UUID  filesystem UUID
+#          RA  read-ahead of the device
+#          RO  read-only device
+#          RM  removable device
+#       MODEL  device identifier
+#        SIZE  size of the device
+#       STATE  state of the device
+#       OWNER  user name
+#       GROUP  group name
+#        MODE  device node permissions
+#   ALIGNMENT  alignment offset
+#      MIN-IO  minimum I/O size
+#      OPT-IO  optimal I/O size
+#     PHY-SEC  physical sector size
+#     LOG-SEC  logical sector size
+#        ROTA  rotational device
+#       SCHED  I/O scheduler name
+#     RQ-SIZE  request queue size
+#        TYPE  device type
+#    DISC-ALN  discard alignment offset
+#   DISC-GRAN  discard granularity
+#    DISC-MAX  discard max bytes
+#   DISC-ZERO  discard zeroes data
+#
+# If the script is run as just 'lsblk' then print out disk size, vendor,
+# model number and serial number.
+
+
+helpstr="
+label:	Show filesystem label.
+model:	Show disk model number.
+serial:	Show disk serial number.
+size:	Show the disk capacity.
+vendor:	Show the disk vendor.
+lsblk:	Show the disk size, vendor, model number, and serial number."
+
+script=$(basename "$0")
+
+if [ "$1" = "-h" ] ; then
+        echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
+        exit
+fi
+
+if [ "$script" = "lsblk" ] ; then
+	list="size vendor model serial"
+else
+	list=$(echo "$script" | tr '[:upper:]' '[:lower:]')
+fi
+
+# Older versions of lsblk don't support all these values (like SERIAL).
+for i in $list ; do
+
+	# Special case: Looking up the size of a file-based vdev can't
+	# be done with lsblk.
+	if [ "$i" = "size" ] && [ -f "$VDEV_UPATH" ] ; then
+		size="$(du -h --apparent-size $VDEV_UPATH | cut -f 1)"
+		echo "size=$size"
+		continue
+	fi
+
+
+	val=""
+	if val=$(eval "lsblk -dl -n -o $i $VDEV_UPATH 2>/dev/null") ; then
+		# Remove leading/trailing whitespace from value
+		val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \
+		     -e 's/[[:space:]]*$//')
+	fi
+	echo "$i=$val"
+done

cmd/zpool/zpool.d/model

@@ -0,0 +1 @@
+lsblk

cmd/zpool/zpool.d/serial

@@ -0,0 +1 @@
+lsblk

cmd/zpool/zpool.d/ses

@@ -0,0 +1,52 @@
+#!/bin/sh
+#
+# Print SCSI Enclosure Services (SES) info. The output is dependent on the name
+# of the script/symlink used to call it.
+#
+helpstr="
+enc:		Show disk enclosure w:x:y:z value.
+slot:		Show disk slot number as reported by the enclosure.
+encdev:		Show the /dev/sg* device for the enclosure associated with the disk slot.
+fault_led:	Show the value of the disk enclosure slot fault LED.
+locate_led:	Show the value of the disk enclosure slot locate LED.
+ses:		Show disk's enclosure, enclosure dev, slot number, and fault/locate LED values."
+
+script=$(basename "$0")
+if [ "$1" = "-h" ] ; then
+	echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
+	exit
+fi
+
+if [ "$script" = "ses" ] ; then
+	scripts='enc encdev slot fault_led locate_led'
+else
+	scripts="$script"
+fi
+
+for i in $scripts ; do
+	if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then
+		echo "$i="
+		continue
+	fi
+
+	val=""
+	case $i in
+	enc)
+		val=$(ls "$VDEV_ENC_SYSFS_PATH/../../" 2>/dev/null)
+		;;
+	slot)
+		val=$(cat "$VDEV_ENC_SYSFS_PATH/slot" 2>/dev/null)
+		;;
+	encdev)
+		val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null)
+		;;
+	fault_led)
+		val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null)
+		;;
+	locate_led)
+		val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null)
+		;;
+	esac
+	echo "$i=$val"
+done
+

cmd/zpool/zpool.d/size

@@ -0,0 +1 @@
+lsblk

cmd/zpool/zpool.d/slaves

@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Show device mapper slave devices.  This is useful for looking up the
+# /dev/sd* devices associated with a dm or multipath device.  For example:
+#
+# $ ls /sys/block/dm-113/slaves/
+# sddt  sdjw
+#
+
+if [ "$1" = "-h" ] ; then
+	echo "Show device mapper slave devices."
+	exit
+fi
+
+dev="$VDEV_PATH"
+
+# If the VDEV path is a symlink, resolve it to a real device
+if [ -L "$dev" ] ; then
+	dev=$(readlink "$dev")
+fi
+
+dev=$(basename "$dev")
+val=""
+if [ -d "/sys/class/block/$dev/slaves" ] ; then
+	# ls -C: output in columns, no newlines
+	val=$(ls -C "/sys/class/block/$dev/slaves")
+
+	# ls -C will print two spaces between files; change to one space.
+	val=$(echo "$val" | sed -r 's/[[:blank:]]+/ /g')
+fi
+
+echo "slaves=$val"

cmd/zpool/zpool.d/slot

@@ -0,0 +1 @@
+ses

cmd/zpool/zpool.d/upath

@@ -0,0 +1,7 @@
+#!/bin/sh
+if [ "$1" = "-h" ] ; then
+	echo "Show the underlying path for a device."
+	exit
+fi
+
+echo upath="$VDEV_UPATH"

cmd/zpool/zpool.d/vendor

@@ -0,0 +1 @@
+lsblk

cmd/zpool/zpool_iter.c

@@ -36,6 +36,7 @@
 
 #include <libzfs.h>
 #include <sys/zfs_context.h>
+#include <sys/wait.h>
 
 #include "zpool_util.h"
 
@@ -321,41 +322,229 @@ for_each_vdev(zpool_handle_t *zhp, pool_vdev_iter_f func, void *data)
 	return (for_each_vdev_cb(zhp, nvroot, func, data));
 }
 
+/*
+ * Process the vcdl->vdev_cmd_data[] array to figure out all the unique column
+ * names and their widths.  When this function is done, vcdl->uniq_cols,
+ * vcdl->uniq_cols_cnt, and vcdl->uniq_cols_width will be filled in.
+ */
+static void
+process_unique_cmd_columns(vdev_cmd_data_list_t *vcdl)
+{
+	char **uniq_cols = NULL, **tmp = NULL;
+	int *uniq_cols_width;
+	vdev_cmd_data_t *data;
+	int cnt = 0;
+	int k;
+
+	/* For each vdev */
+	for (int i = 0; i < vcdl->count; i++) {
+		data = &vcdl->data[i];
+		/* For each column the vdev reported */
+		for (int j = 0; j < data->cols_cnt; j++) {
+			/* Is this column in our list of unique column names? */
+			for (k = 0; k < cnt; k++) {
+				if (strcmp(data->cols[j], uniq_cols[k]) == 0)
+					break; /* yes it is */
+			}
+			if (k == cnt) {
+				/* No entry for column, add to list */
+				tmp = realloc(uniq_cols, sizeof (*uniq_cols) *
+				    (cnt + 1));
+				if (tmp == NULL)
+					break; /* Nothing we can do... */
+				uniq_cols = tmp;
+				uniq_cols[cnt] = data->cols[j];
+				cnt++;
+			}
+		}
+	}
+
+	/*
+	 * We now have a list of all the unique column names.  Figure out the
+	 * max width of each column by looking at the column name and all its
+	 * values.
+	 */
+	uniq_cols_width = safe_malloc(sizeof (*uniq_cols_width) * cnt);
+	for (int i = 0; i < cnt; i++) {
+		/* Start off with the column title's width */
+		uniq_cols_width[i] = strlen(uniq_cols[i]);
+		/* For each vdev */
+		for (int j = 0; j < vcdl->count; j++) {
+			/* For each of the vdev's values in a column */
+			data = &vcdl->data[j];
+			for (k = 0; k < data->cols_cnt; k++) {
+				/* Does this vdev have a value for this col? */
+				if (strcmp(data->cols[k], uniq_cols[i]) == 0) {
+					/* Is the value width larger? */
+					uniq_cols_width[i] =
+					    MAX(uniq_cols_width[i],
+					    strlen(data->lines[k]));
+				}
+			}
+		}
+	}
+
+	vcdl->uniq_cols = uniq_cols;
+	vcdl->uniq_cols_cnt = cnt;
+	vcdl->uniq_cols_width = uniq_cols_width;
+}
+
+
+/*
+ * Process a line of command output
+ *
+ * When running 'zpool iostat|status -c' the lines of output can either be
+ * in the form of:
+ *
+ *	column_name=value
+ *
+ * Or just:
+ *
+ *	value
+ *
+ * Process the column_name (if any) and value.
+ *
+ * Returns 0 if line was processed, and there are more lines can still be
+ * processed.
+ *
+ * Returns 1 if this was the last line to process, or error.
+ */
+static int
+vdev_process_cmd_output(vdev_cmd_data_t *data, char *line)
+{
+	char *col = NULL;
+	char *val = line;
+	char *equals;
+	char **tmp;
+
+	if (line == NULL)
+		return (1);
+
+	equals = strchr(line, '=');
+	if (equals != NULL) {
+		/*
+		 * We have a 'column=value' type line.  Split it into the
+		 * column and value strings by turning the '=' into a '\0'.
+		 */
+		*equals = '\0';
+		col = line;
+		val = equals + 1;
+	} else {
+		val = line;
+	}
+
+	/* Do we already have a column by this name?  If so, skip it. */
+	if (col != NULL) {
+		for (int i = 0; i < data->cols_cnt; i++) {
+			if (strcmp(col, data->cols[i]) == 0)
+				return (0); /* Duplicate, skip */
+		}
+	}
+
+	if (val != NULL) {
+		tmp = realloc(data->lines,
+		    (data->lines_cnt + 1) * sizeof (*data->lines));
+		if (tmp == NULL)
+			return (1);
+
+		data->lines = tmp;
+		data->lines[data->lines_cnt] = strdup(val);
+		data->lines_cnt++;
+	}
+
+	if (col != NULL) {
+		tmp = realloc(data->cols,
+		    (data->cols_cnt + 1) * sizeof (*data->cols));
+		if (tmp == NULL)
+			return (1);
+
+		data->cols = tmp;
+		data->cols[data->cols_cnt] = strdup(col);
+		data->cols_cnt++;
+	}
+
+	if (val != NULL && col == NULL)
+		return (1);
+
+	return (0);
+}
+
+/*
+ * Run the cmd and store results in *data.
+ */
+static void
+vdev_run_cmd(vdev_cmd_data_t *data, char *cmd)
+{
+	int rc;
+	char *argv[2] = {cmd, 0};
+	char *env[5] = {"PATH=/bin:/sbin:/usr/bin:/usr/sbin", NULL, NULL, NULL,
+	    NULL};
+	char **lines = NULL;
+	int lines_cnt = 0;
+	int i;
+
+	/* Setup our custom environment variables */
+	rc = asprintf(&env[1], "VDEV_PATH=%s",
+	    data->path ? data->path : "");
+	if (rc == -1)
+		goto out;
+
+	rc = asprintf(&env[2], "VDEV_UPATH=%s",
+	    data->upath ? data->upath : "");
+	if (rc == -1)
+		goto out;
+
+	rc = asprintf(&env[3], "VDEV_ENC_SYSFS_PATH=%s",
+	    data->vdev_enc_sysfs_path ?
+	    data->vdev_enc_sysfs_path : "");
+	if (rc == -1)
+		goto out;
+
+	/* Run the command */
+	rc = libzfs_run_process_get_stdout_nopath(cmd, argv, env, &lines,
+	    &lines_cnt);
+	if (rc != 0)
+		goto out;
+
+	/* Process the output we got */
+	for (i = 0; i < lines_cnt; i++)
+		if (vdev_process_cmd_output(data, lines[i]) != 0)
+			break;
+
+out:
+	if (lines != NULL)
+		libzfs_free_str_array(lines, lines_cnt);
+
+	/* Start with i = 1 since env[0] was statically allocated */
+	for (i = 1; i < ARRAY_SIZE(env); i++)
+		if (env[i] != NULL)
+			free(env[i]);
+}
+
 /* Thread function run for each vdev */
 static void
 vdev_run_cmd_thread(void *cb_cmd_data)
 {
 	vdev_cmd_data_t *data = cb_cmd_data;
-	char *pos = NULL;
-	FILE *fp;
-	size_t len = 0;
-	char cmd[_POSIX_ARG_MAX];
+	const char *sep = ",";
+	char *cmd = NULL, *cmddup, *rest;
+	char fullpath[MAXPATHLEN];
 
-	/* Set our VDEV_PATH and VDEV_UPATH env vars and run command */
-	if (snprintf(cmd, sizeof (cmd), "VDEV_PATH=%s && VDEV_UPATH=\"%s\" && "
-	    "VDEV_ENC_SYSFS_PATH=\"%s\" && %s", data->path ? data->path : "",
-	    data->upath ? data->upath : "",
-	    data->vdev_enc_sysfs_path ? data->vdev_enc_sysfs_path : "",
-	    data->cmd) >= sizeof (cmd)) {
-		/* Our string was truncated */
-		return;
-	}
-
-	fp = popen(cmd, "r");
-	if (fp == NULL)
+	cmddup = strdup(data->cmd);
+	if (cmddup == NULL)
 		return;
 
-	data->line = NULL;
+	rest = cmddup;
+	while ((cmd = strtok_r(rest, sep, &rest))) {
+		if (snprintf(fullpath, sizeof (fullpath), "%s/%s",
+		    ZPOOL_SCRIPTS_DIR, cmd) == -1)
+			continue;
 
-	/* Save the first line of output from the command */
-	if (getline(&data->line, &len, fp) != -1) {
-		/* Success.  Remove newline from the end, if necessary. */
-		if ((pos = strchr(data->line, '\n')) != NULL)
-			*pos = '\0';
-	} else {
-		data->line = NULL;
+		/* Does the script exist in our zpool scripts dir? */
+		if (access(fullpath, X_OK) == 0)
+			vdev_run_cmd(data, fullpath);
 	}
-	pclose(fp);
+	free(cmddup);
 }
 
 /* For each vdev in the pool run a command */
@@ -412,6 +601,8 @@ for_each_vdev_run_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_vcdl)
 	data->path = strdup(path);
 	data->upath = zfs_get_underlying_path(path);
 	data->cmd = vcdl->cmd;
+	data->lines = data->cols = NULL;
+	data->lines_cnt = data->cols_cnt = 0;
 	if (vdev_enc_sysfs_path)
 		data->vdev_enc_sysfs_path = strdup(vdev_enc_sysfs_path);
 	else
@@ -463,6 +654,7 @@ all_pools_for_each_vdev_run_vcdl(vdev_cmd_data_list_t *vcdl)
 	taskq_wait(t);
 	taskq_destroy(t);
 	thread_fini();
+
 }
 
 /*
@@ -495,6 +687,13 @@ all_pools_for_each_vdev_run(int argc, char **argv, char *cmd,
 	/* Run command on all vdevs in all pools */
 	all_pools_for_each_vdev_run_vcdl(vcdl);
 
+	/*
+	 * vcdl->data[] now contains all the column names and values for each
+	 * vdev.  We need to process that into a master list of unique column
+	 * names, and figure out the width of each column.
+	 */
+	process_unique_cmd_columns(vcdl);
+
 	return (vcdl);
 }
 
@@ -504,12 +703,23 @@ all_pools_for_each_vdev_run(int argc, char **argv, char *cmd,
 void
 free_vdev_cmd_data_list(vdev_cmd_data_list_t *vcdl)
 {
-	int i;
-	for (i = 0; i < vcdl->count; i++) {
+	free(vcdl->uniq_cols);
+	free(vcdl->uniq_cols_width);
+
+	for (int i = 0; i < vcdl->count; i++) {
 		free(vcdl->data[i].path);
 		free(vcdl->data[i].pool);
 		free(vcdl->data[i].upath);
-		free(vcdl->data[i].line);
+
+		for (int j = 0; j < vcdl->data[i].lines_cnt; j++)
+			free(vcdl->data[i].lines[j]);
+
+		free(vcdl->data[i].lines);
+
+		for (int j = 0; j < vcdl->data[i].cols_cnt; j++)
+			free(vcdl->data[i].cols[j]);
+
+		free(vcdl->data[i].cols);
 		free(vcdl->data[i].vdev_enc_sysfs_path);
 	}
 	free(vcdl->data);

cmd/zpool/zpool_main.c

@@ -45,6 +45,7 @@
 #include <unistd.h>
 #include <pwd.h>
 #include <zone.h>
+#include <sys/wait.h>
 #include <zfs_prop.h>
 #include <sys/fs/zfs.h>
 #include <sys/stat.h>
@@ -52,6 +53,7 @@
 #include <sys/fm/util.h>
 #include <sys/fm/protocol.h>
 #include <sys/zfs_ioctl.h>
+
 #include <math.h>
 
 #include <libzfs.h>
@@ -314,10 +316,10 @@ get_usage(zpool_help_t idx)
 		    "[-R root] [-F [-n]]\n"
 		    "\t    <pool | id> [newpool]\n"));
 	case HELP_IOSTAT:
-		return (gettext("\tiostat [-c CMD] [-T d | u] [-ghHLpPvy] "
-		    "[[-lq]|[-r|-w]]\n"
-		    "\t    [[pool ...]|[pool vdev ...]|[vdev ...]] "
-		    "[interval [count]]\n"));
+		return (gettext("\tiostat [[[-c [script1,script2,...]"
+		    "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
+		    "\t    [[pool ...]|[pool vdev ...]|[vdev ...]]"
+		    " [interval [count]]\n"));
 	case HELP_LABELCLEAR:
 		return (gettext("\tlabelclear [-f] <vdev>\n"));
 	case HELP_LIST:
@@ -337,8 +339,8 @@ get_usage(zpool_help_t idx)
 	case HELP_SCRUB:
 		return (gettext("\tscrub [-s] <pool> ...\n"));
 	case HELP_STATUS:
-		return (gettext("\tstatus [-c CMD] [-gLPvxD] [-T d|u] [pool]"
-		    " ... [interval [count]]\n"));
+		return (gettext("\tstatus [-c [script1,script2,...]] [-gLPvxD]"
+		    "[-T d|u] [pool] ... [interval [count]]\n"));
 	case HELP_UPGRADE:
 		return (gettext("\tupgrade\n"
 		    "\tupgrade -v\n"
@@ -1542,17 +1544,68 @@ typedef struct status_cbdata {
 	vdev_cmd_data_list_t	*vcdl;
 } status_cbdata_t;
 
-/* Print output line for specific vdev in a specific pool */
+/* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
+static int
+is_blank_str(char *str)
+{
+	while (str != NULL && *str != '\0') {
+		if (!isblank(*str))
+			return (0);
+		str++;
+	}
+	return (1);
+}
+
+/* Print command output lines for specific vdev in a specific pool */
 static void
 zpool_print_cmd(vdev_cmd_data_list_t *vcdl, const char *pool, char *path)
 {
-	int i;
+	vdev_cmd_data_t *data;
+	int i, j;
+	char *val;
+
 	for (i = 0; i < vcdl->count; i++) {
-		if ((strcmp(vcdl->data[i].path, path) == 0) &&
-		    (strcmp(vcdl->data[i].pool, pool) == 0)) {
-			printf("%s", vcdl->data[i].line);
-			break;
+		if ((strcmp(vcdl->data[i].path, path) != 0) ||
+		    (strcmp(vcdl->data[i].pool, pool) != 0)) {
+			/* Not the vdev we're looking for */
+			continue;
 		}
+
+		data = &vcdl->data[i];
+		/* Print out all the output values for this vdev */
+		for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
+			val = NULL;
+			/* Does this vdev have values for this column? */
+			for (int k = 0; k < data->cols_cnt; k++) {
+				if (strcmp(data->cols[k],
+				    vcdl->uniq_cols[j]) == 0) {
+					/* yes it does, record the value */
+					val = data->lines[k];
+					break;
+				}
+			}
+			/*
+			 * Mark empty values with dashes to make output
+			 * awk-able.
+			 */
+			if (is_blank_str(val))
+				val = "-";
+
+			printf("%*s", vcdl->uniq_cols_width[j], val);
+			if (j < vcdl->uniq_cols_cnt - 1)
+				printf("  ");
+		}
+
+		/* Print out any values that aren't in a column at the end */
+		for (j = data->cols_cnt; j < data->lines_cnt; j++) {
+			/* Did we have any columns?  If so print a spacer. */
+			if (vcdl->uniq_cols_cnt > 0)
+				printf("  ");
+
+			val = data->lines[j];
+			printf("%s", val ? val : "");
+		}
+		break;
 	}
 }
 
@@ -2799,9 +2852,54 @@ print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
 
 		}
 	}
-	printf("\n");
 }
 
+
+/*
+ * print_cmd_columns - Print custom column titles from -c
+ *
+ * If the user specified the "zpool status|iostat -c" then print their custom
+ * column titles in the header.  For example, print_cmd_columns() would print
+ * the "  col1  col2" part of this:
+ *
+ * $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
+ * ...
+ *	      capacity     operations     bandwidth
+ * pool        alloc   free   read  write   read  write  col1  col2
+ * ----------  -----  -----  -----  -----  -----  -----  ----  ----
+ * mypool       269K  1008M      0      0    107    946
+ *   mirror     269K  1008M      0      0    107    946
+ *     sdb         -      -      0      0    102    473  val1  val2
+ *     sdc         -      -      0      0      5    473  val1  val2
+ * ----------  -----  -----  -----  -----  -----  -----  ----  ----
+ */
+void
+print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
+{
+	int i, j;
+	vdev_cmd_data_t *data = &vcdl->data[0];
+
+	if (vcdl->count == 0 || data == NULL)
+		return;
+
+	/*
+	 * Each vdev cmd should have the same column names unless the user did
+	 * something weird with their cmd.  Just take the column names from the
+	 * first vdev and assume it works for all of them.
+	 */
+	for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
+		printf("  ");
+		if (use_dashes) {
+			for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
+				printf("-");
+		} else {
+			printf("%*s", vcdl->uniq_cols_width[i],
+			    vcdl->uniq_cols[i]);
+		}
+	}
+}
+
+
 /*
  * Utility function to print out a line of dashes like:
  *
@@ -2870,7 +2968,6 @@ print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
 				    "--------------------");
 		}
 	}
-	printf("\n");
 }
 
 
@@ -2912,12 +3009,22 @@ print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
 
 
 	print_iostat_labels(cb, force_column_width, iostat_top_labels);
+	printf("\n");
 
 	printf("%-*s", namewidth, title);
 
 	print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
+	if (cb->vcdl != NULL)
+		print_cmd_columns(cb->vcdl, 0);
+
+	printf("\n");
 
 	print_iostat_separator_impl(cb, force_column_width);
+
+	if (cb->vcdl != NULL)
+		print_cmd_columns(cb->vcdl, 1);
+
+	printf("\n");
 }
 
 static void
@@ -3451,11 +3558,8 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
 		char *path;
 		if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
 		    &path) == 0) {
-			if (!(cb->cb_flags & IOS_ANYHISTO_M))
-				printf("  ");
+			printf("  ");
 			zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
-			if (cb->cb_flags & IOS_ANYHISTO_M)
-				printf("\n");
 		}
 	}
 
@@ -3602,6 +3706,10 @@ print_iostat(zpool_handle_t *zhp, void *data)
 	if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
 	    !cb->cb_scripted && cb->cb_verbose && !cb->cb_vdev_names_count) {
 		print_iostat_separator(cb);
+		if (cb->vcdl != NULL) {
+			print_cmd_columns(cb->vcdl, 1);
+		}
+		printf("\n");
 	}
 
 	return (ret);
@@ -3990,11 +4098,72 @@ fsleep(float sec)
 	nanosleep(&req, NULL);
 }
 
+/*
+ * Run one of the zpool status/iostat -c scripts with the help (-h) option and
+ * print the result.
+ *
+ * name:	Short name of the script ('iostat').
+ * path:	Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
+ */
+static void
+print_zpool_script_help(char *name, char *path)
+{
+	char *argv[] = {path, "-h", NULL};
+	char **lines = NULL;
+	int lines_cnt = 0;
+	int rc;
+
+	rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
+	    &lines_cnt);
+	if (rc != 0 || lines == NULL || lines_cnt <= 0)
+		return;
+
+	for (int i = 0; i < lines_cnt; i++)
+		if (!is_blank_str(lines[i]))
+			printf("  %-14s  %s\n", name, lines[i]);
+
+	libzfs_free_str_array(lines, lines_cnt);
+}
+
 
 /*
- * zpool iostat [-c CMD] [-ghHLpPvy] [[-lq]|[-r|-w]] [-n name] [-T d|u]
- *		[[ pool ...]|[pool vdev ...]|[vdev ...]]
- *		[interval [count]]
+ * Go though the list of all the zpool status/iostat -c scripts, run their
+ * help option (-h), and print out the results.
+ */
+static void
+print_zpool_script_list(void)
+{
+	DIR *dir;
+	struct dirent *ent;
+	char fullpath[MAXPATHLEN];
+	struct stat dir_stat;
+
+	if ((dir = opendir(ZPOOL_SCRIPTS_DIR)) != NULL) {
+		printf("\n");
+		/* print all the files and directories within directory */
+		while ((ent = readdir(dir)) != NULL) {
+			sprintf(fullpath, "%s/%s", ZPOOL_SCRIPTS_DIR,
+			    ent->d_name);
+
+			/* Print the scripts */
+			if (stat(fullpath, &dir_stat) == 0)
+				if (dir_stat.st_mode & S_IXUSR &&
+				    S_ISREG(dir_stat.st_mode))
+					print_zpool_script_help(ent->d_name,
+					    fullpath);
+		}
+		printf("\n");
+		closedir(dir);
+	} else {
+		fprintf(stderr, gettext("Can't open %s scripts dir\n"),
+		    ZPOOL_SCRIPTS_DIR);
+	}
+}
+
+/*
+ * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
+ *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
+ *              [interval [count]]
  *
  *	-c CMD  For each vdev, run command CMD
  *	-g	Display guid for individual vdev name.
@@ -4046,7 +4215,20 @@ zpool_do_iostat(int argc, char **argv)
 	while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwH")) != -1) {
 		switch (c) {
 		case 'c':
+			if (cmd != NULL) {
+				fprintf(stderr,
+				    gettext("Can't set -c flag twice\n"));
+				exit(1);
+			}
+			if ((getuid() <= 0 || geteuid() <= 0) &&
+			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
+				fprintf(stderr, gettext(
+				    "Can't run -c with root privileges "
+				    "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
+				exit(1);
+			}
 			cmd = optarg;
+			verbose = B_TRUE;
 			break;
 		case 'g':
 			guid = B_TRUE;
@@ -4089,8 +4271,11 @@ zpool_do_iostat(int argc, char **argv)
 			break;
 		case '?':
 			if (optopt == 'c') {
-				fprintf(stderr,
-				    gettext("Missing CMD for -c\n"));
+				fprintf(stderr, gettext(
+				    "Current scripts in %s:\n"),
+				    ZPOOL_SCRIPTS_DIR);
+				print_zpool_script_list();
+				exit(0);
 			} else {
 				fprintf(stderr,
 				    gettext("invalid option '%c'\n"), optopt);
@@ -4185,10 +4370,10 @@ zpool_do_iostat(int argc, char **argv)
 		return (1);
 	}
 
-	if ((l_histo || rq_histo) && (queues || latency)) {
+	if ((l_histo || rq_histo) && (cmd != NULL || latency || queues)) {
 		pool_list_free(list);
 		(void) fprintf(stderr,
-		    gettext("[-r|-w] isn't allowed with [-q|-l]\n"));
+		    gettext("[-r|-w] isn't allowed with [-c|-l|-q]\n"));
 		usage(B_FALSE);
 		return (1);
 	}
@@ -4276,6 +4461,15 @@ zpool_do_iostat(int argc, char **argv)
 			if (timestamp_fmt != NODATE)
 				print_timestamp(timestamp_fmt);
 
+			if (cmd != NULL && cb.cb_verbose &&
+			    !(cb.cb_flags & IOS_ANYHISTO_M)) {
+				cb.vcdl = all_pools_for_each_vdev_run(argc,
+				    argv, cmd, g_zfs, cb.cb_vdev_names,
+				    cb.cb_vdev_names_count, cb.cb_name_flags);
+			} else {
+				cb.vcdl = NULL;
+			}
+
 			/*
 			 * If it's the first time and we're not skipping it,
 			 * or either skip or verbose mode, print the header.
@@ -4294,16 +4488,9 @@ zpool_do_iostat(int argc, char **argv)
 				continue;
 			}
 
-			if (cmd != NULL && cb.cb_verbose)
-				cb.vcdl = all_pools_for_each_vdev_run(argc,
-				    argv, cmd, g_zfs, cb.cb_vdev_names,
-				    cb.cb_vdev_names_count, cb.cb_name_flags);
 
 			pool_list_iter(list, B_FALSE, print_iostat, &cb);
 
-			if (cb.vcdl != NULL)
-				free_vdev_cmd_data_list(cb.vcdl);
-
 			/*
 			 * If there's more than one pool, and we're not in
 			 * verbose mode (which prints a separator for us),
@@ -4318,7 +4505,14 @@ zpool_do_iostat(int argc, char **argv)
 			    cb.cb_vdev_names_count)) &&
 			    !cb.cb_scripted) {
 				print_iostat_separator(&cb);
+				if (cb.vcdl != NULL)
+					print_cmd_columns(cb.vcdl, 1);
+				printf("\n");
 			}
+
+			if (cb.vcdl != NULL)
+				free_vdev_cmd_data_list(cb.vcdl);
+
 		}
 
 		/*
@@ -6036,9 +6230,14 @@ status_callback(zpool_handle_t *zhp, void *data)
 			cbp->cb_namewidth = 10;
 
 		(void) printf(gettext("config:\n\n"));
-		(void) printf(gettext("\t%-*s  %-8s %5s %5s %5s\n"),
+		(void) printf(gettext("\t%-*s  %-8s %5s %5s %5s"),
 		    cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
 		    "CKSUM");
+
+		if (cbp->vcdl != NULL)
+			print_cmd_columns(cbp->vcdl, 0);
+
+		printf("\n");
 		print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
 		    B_FALSE);
 
@@ -6098,7 +6297,8 @@ status_callback(zpool_handle_t *zhp, void *data)
 }
 
 /*
- * zpool status [-c CMD] [-gLPvx] [-T d|u] [pool] ... [interval [count]]
+ * zpool status [-c [script1,script2,...]] [-gLPvx] [-T d|u] [pool] ...
+ *              [interval [count]]
  *
  *	-c CMD	For each vdev, run command CMD
  *	-g	Display guid for individual vdev name.
@@ -6125,6 +6325,18 @@ zpool_do_status(int argc, char **argv)
 	while ((c = getopt(argc, argv, "c:gLPvxDT:")) != -1) {
 		switch (c) {
 		case 'c':
+			if (cmd != NULL) {
+				fprintf(stderr,
+				    gettext("Can't set -c flag twice\n"));
+				exit(1);
+			}
+			if ((getuid() <= 0 || geteuid() <= 0) &&
+			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
+				fprintf(stderr, gettext(
+				    "Can't run -c with root privileges "
+				    "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
+				exit(1);
+			}
 			cmd = optarg;
 			break;
 		case 'g':
@@ -6150,8 +6362,11 @@ zpool_do_status(int argc, char **argv)
 			break;
 		case '?':
 			if (optopt == 'c') {
-				fprintf(stderr,
-				    gettext("Missing CMD for -c\n"));
+				fprintf(stderr, gettext(
+				    "Current scripts in %s:\n"),
+				    ZPOOL_SCRIPTS_DIR);
+				print_zpool_script_list();
+				exit(0);
 			} else {
 				fprintf(stderr,
 				    gettext("invalid option '%c'\n"), optopt);

cmd/zpool/zpool_util.h

@@ -32,6 +32,9 @@
 extern "C" {
 #endif
 
+/* Path to scripts you can run with "zpool status/iostat -c" */
+#define	ZPOOL_SCRIPTS_DIR SYSCONFDIR"/zfs/zpool.d"
+
 /*
  * Basic utility functions
  */
@@ -75,7 +78,13 @@ libzfs_handle_t *g_zfs;
 
 typedef	struct vdev_cmd_data
 {
-	char *line;	/* cmd output */
+	char **lines;	/* Array of lines of output, minus the column name */
+	int lines_cnt;	/* Number of lines in the array */
+
+	char **cols;	/* Array of column names */
+	int cols_cnt;	/* Number of column names */
+
+
 	char *path;	/* vdev path */
 	char *upath;	/* vdev underlying path */
 	char *pool;	/* Pool name */
@@ -96,6 +105,11 @@ typedef struct vdev_cmd_data_list
 
 	vdev_cmd_data_t *data;	/* Array of vdevs */
 
+	/* List of unique column names and widths */
+	char **uniq_cols;
+	int uniq_cols_cnt;
+	int *uniq_cols_width;
+
 } vdev_cmd_data_list_t;
 
 vdev_cmd_data_list_t *all_pools_for_each_vdev_run(int argc, char **argv,

include/libzfs.h

@@ -793,8 +793,17 @@ extern int zfs_nicestrtonum(libzfs_handle_t *, const char *, uint64_t *);
  */
 #define	STDOUT_VERBOSE	0x01
 #define	STDERR_VERBOSE	0x02
+#define	NO_DEFAULT_PATH	0x04 /* Don't use $PATH to lookup the command */
 
 int libzfs_run_process(const char *, char **, int flags);
+int libzfs_run_process_get_stdout(const char *path, char *argv[], char *env[],
+    char **lines[], int *lines_cnt);
+int libzfs_run_process_get_stdout_nopath(const char *path, char *argv[],
+    char *env[], char **lines[], int *lines_cnt);
+
+void libzfs_free_str_array(char **strs, int count);
+
+int libzfs_envvar_is_set(char *envvar);
 
 /*
  * Given a device or file, determine if it is part of a pool.

lib/libzfs/libzfs_util.c

@@ -726,30 +726,106 @@ libzfs_module_loaded(const char *module)
 	return (access(path, F_OK) == 0);
 }
 
-int
-libzfs_run_process(const char *path, char *argv[], int flags)
+
+/*
+ * Read lines from an open file descriptor and store them in an array of
+ * strings until EOF.  lines[] will be allocated and populated with all the
+ * lines read.  All newlines are replaced with NULL terminators for
+ * convenience.  lines[] must be freed after use with libzfs_free_str_array().
+ *
+ * Returns the number of lines read.
+ */
+static int
+libzfs_read_stdout_from_fd(int fd, char **lines[])
+{
+
+	FILE *fp;
+	int lines_cnt = 0;
+	size_t len = 0;
+	char *line = NULL;
+	char **tmp_lines = NULL, **tmp;
+	char *nl = NULL;
+	int rc;
+
+	fp = fdopen(fd, "r");
+	if (fp == NULL)
+		return (0);
+	while (1) {
+		rc = getline(&line, &len, fp);
+		if (rc == -1)
+			break;
+
+		tmp = realloc(tmp_lines, sizeof (*tmp_lines) * (lines_cnt + 1));
+		if (tmp == NULL) {
+			/* Return the lines we were able to process */
+			break;
+		}
+		tmp_lines = tmp;
+
+		/* Terminate newlines */
+		if ((nl = strchr(line, '\n')) != NULL)
+			*nl = '\0';
+		tmp_lines[lines_cnt] = line;
+		lines_cnt++;
+		line = NULL;
+	}
+	fclose(fp);
+	*lines = tmp_lines;
+	return (lines_cnt);
+}
+
+static int
+libzfs_run_process_impl(const char *path, char *argv[], char *env[], int flags,
+    char **lines[], int *lines_cnt)
 {
 	pid_t pid;
 	int error, devnull_fd;
+	int link[2];
+
+	/*
+	 * Setup a pipe between our child and parent process if we're
+	 * reading stdout.
+	 */
+	if ((lines != NULL) && pipe(link) == -1)
+		return (-ESTRPIPE);
 
 	pid = vfork();
 	if (pid == 0) {
+		/* Child process */
 		devnull_fd = open("/dev/null", O_WRONLY);
 
 		if (devnull_fd < 0)
 			_exit(-1);
 
-		if (!(flags & STDOUT_VERBOSE))
+		if (!(flags & STDOUT_VERBOSE) && (lines == NULL))
 			(void) dup2(devnull_fd, STDOUT_FILENO);
+		else if (lines != NULL) {
+			/* Save the output to lines[] */
+			dup2(link[1], STDOUT_FILENO);
+			close(link[0]);
+			close(link[1]);
+		}
 
 		if (!(flags & STDERR_VERBOSE))
 			(void) dup2(devnull_fd, STDERR_FILENO);
 
 		close(devnull_fd);
 
-		(void) execvp(path, argv);
+		if (flags & NO_DEFAULT_PATH) {
+			if (env == NULL)
+				execv(path, argv);
+			else
+				execve(path, argv, env);
+		} else {
+			if (env == NULL)
+				execvp(path, argv);
+			else
+				execvpe(path, argv, env);
+		}
+
 		_exit(-1);
 	} else if (pid > 0) {
+		/* Parent process */
 		int status;
 
 		while ((error = waitpid(pid, &status, 0)) == -1 &&
@@ -757,12 +833,78 @@ libzfs_run_process(const char *path, char *argv[], int flags)
 		if (error < 0 || !WIFEXITED(status))
 			return (-1);
 
+		if (lines != NULL) {
+			close(link[1]);
+			*lines_cnt = libzfs_read_stdout_from_fd(link[0], lines);
+		}
 		return (WEXITSTATUS(status));
 	}
 
 	return (-1);
 }
 
+int
+libzfs_run_process(const char *path, char *argv[], int flags)
+{
+	return (libzfs_run_process_impl(path, argv, NULL, flags, NULL, NULL));
+}
+
+/*
+ * Run a command and store its stdout lines in an array of strings (lines[]).
+ * lines[] is allocated and populated for you, and the number of lines is set in
+ * lines_cnt.  lines[] must be freed after use with libzfs_free_str_array().
+ * All newlines (\n) in lines[] are terminated for convenience.
+ */
+int
+libzfs_run_process_get_stdout(const char *path, char *argv[], char *env[],
+    char **lines[], int *lines_cnt)
+{
+	return (libzfs_run_process_impl(path, argv, env, 0, lines, lines_cnt));
+}
+
+/*
+ * Same as libzfs_run_process_get_stdout(), but run without $PATH set.  This
+ * means that *path needs to be the full path to the executable.
+ */
+int
+libzfs_run_process_get_stdout_nopath(const char *path, char *argv[],
+    char *env[], char **lines[], int *lines_cnt)
+{
+	return (libzfs_run_process_impl(path, argv, env, NO_DEFAULT_PATH,
+	    lines, lines_cnt));
+}
+
+/*
+ * Free an array of strings.  Free both the strings contained in the array and
+ * the array itself.
+ */
+void
+libzfs_free_str_array(char **strs, int count)
+{
+	while (--count >= 0)
+		free(strs[count]);
+
+	free(strs);
+}
+
+/*
+ * Returns 1 if environment variable is set to "YES", "yes", "ON", "on", or
+ * a non-zero number.
+ *
+ * Returns 0 otherwise.
+ */
+int
+libzfs_envvar_is_set(char *envvar)
+{
+	char *env = getenv(envvar);
+	if (env && (strtoul(env, NULL, 0) > 0 ||
+	    (!strncasecmp(env, "YES", 3) && strnlen(env, 4) == 3) ||
+	    (!strncasecmp(env, "ON", 2) && strnlen(env, 3) == 2)))
+		return (1);
+
+	return (0);
+}
+
 /*
  * Verify the required ZFS_DEV device is available and optionally attempt
  * to load the ZFS modules.  Under normal circumstances the modules

man/man8/zpool.8

@@ -96,7 +96,7 @@ zpool \- configures ZFS storage pools
 
 .LP
 .nf
-\fB\fBzpool iostat\fR [\fB-c\fR \fBCMD\fR] [\fB-T\fR \fBd\fR | \fBu\fR] [\fB-ghHLpPvy\fR] [\fB-lq\fR]|[\fB-r\fR|-\fBw\fR]]
+\fB\fBzpool iostat\fR [[[\fB-c\fR \fBSCRIPT\fR] [\fB-lq\fR]] | \fB-rw\fR] [\fB-T\fR \fBd\fR | \fBu\fR] [\fB-ghHLpPvy\fR]
      [[\fIpool\fR ...]|[\fIpool vdev\fR ...]|[\fIvdev\fR ...]] [\fIinterval\fR[\fIcount\fR]]\fR
 
 .fi
@@ -159,7 +159,7 @@ zpool \- configures ZFS storage pools
 
 .LP
 .nf
-\fBzpool status\fR [\fB-c\fR \fBCMD\fR] [\fB-gLPvxD\fR] [\fB-T\fR d | u] [\fIpool\fR] ... [\fIinterval\fR [\fIcount\fR]]
+\fBzpool status\fR [\fB-c\fR \fBSCRIPT\fR] [\fB-gLPvxD\fR] [\fB-T\fR d | u] [\fIpool\fR] ... [\fIinterval\fR [\fIcount\fR]]
 .fi
 
 .LP
@@ -1523,7 +1523,7 @@ Scan using the default search path, the libblkid cache will not be consulted.  A
 .sp
 .ne 2
 .na
-\fB\fBzpool iostat\fR [\fB-c\fR \fBCMD\fR] [\fB-T\fR \fBd\fR | \fBu\fR] [\fB-ghHLpPvy\fR] [[\fB-lq\fR]|[\fB-r\fR|\fB-w\fR]] [[\fIpool\fR ...]|[\fIpool vdev\fR ...]|[\fIvdev\fR ...]] [\fIinterval\fR[\fIcount\fR]]\fR
+\fB\fBzpool iostat\fR [[[\fB-c\fR \fBSCRIPT\fR] [\fB-lq\fR]] | \fB-rw\fR] [\fB-T\fR \fBd\fR | \fBu\fR] [\fB-ghHLpPvy\fR] [[\fIpool\fR ...]|[\fIpool vdev\fR ...]|[\fIvdev\fR ...]] [\fIinterval\fR[\fIcount\fR]]\fR
 
 .ad
 .sp .6
@@ -1542,14 +1542,33 @@ base 1024.  To get the raw values, use the \fB-p\fR flag.
 .sp
 .ne 2
 .na
-\fB\fB-c\fR \fBCMD\fR
+\fB\fB-c\fR \fB[SCRIPT1,SCRIPT2,...]\fR
 .ad
 .RS 12n
-Run a command on each vdev and include first line of output
+Run a script (or scripts) on each vdev and include the output in zpool iostat
 .sp
-The \fB-c\fR option allows you to run an arbitrary command on each vdev and
-display the first line of output in zpool iostat.  The following environment
-vars are set before running each command:
+The \fB-c\fR option allows you to run script(s) for each vdev and display the
+output in zpool iostat.  For security reasons, a user can only execute scripts
+found in the /<etc>/zfs/zpool.d directory as an unprivileged user.  However, a
+privileged user can run \fB-c\fR if they have the ZPOOL_SCRIPTS_AS_ROOT
+environment variable set.  If a script requires the use of a privileged
+command (like smartctl) then it's recommended you allow the user access to it in
+/etc/sudoers.  For example, to allow user "zfsuser" access to "smartctl -a", add
+the following to /etc/sudoers:
+
+zfsuser ALL=NOPASSWD: /usr/sbin/smartctl -a /dev/sd[a-z]*, NOEXEC: /usr/sbin/smartctl -a /dev/sd[a-z]*`
+
+If \fB-c\fR is passed without a script name, it prints a list of all scripts.
+\fB-c\fR also sets verbose mode (\fB-v\fR).
+
+Script output should be in the form of "name=value".  The column name is
+set to "name" and the value is set to "value".  Multiple lines can be used to
+output multiple columns.  The first line of output not in the "name=value"
+format is displayed without a column title, and no more output after that is
+displayed.  This can be useful for printing error messages.  Blank or NULL
+values are printed as a '-' to make output awk-able.
+
+The following environment variables are set before running each script:
 .sp
 \fB$VDEV_PATH\fR: Full path to the vdev.
 .LP
@@ -2103,7 +2122,7 @@ Sets the specified property for \fInewpool\fR. See the “Properties” section
 .sp
 .ne 2
 .na
-\fBzpool status\fR [\fB-c\fR \fBCMD\fR] [\fB-gLPvxD\fR] [\fB-T\fR d | u] [\fIpool\fR] ... [\fIinterval\fR [\fIcount\fR]]
+\fBzpool status\fR [\fB-c\fR \fB[SCRIPT1,SCRIPT2,...] \fR] [\fB-gLPvxD\fR] [\fB-T\fR d | u] [\fIpool\fR] ... [\fIinterval\fR [\fIcount\fR]]
 .ad
 .sp .6
 .RS 4n
@@ -2114,14 +2133,32 @@ If a scrub or resilver is in progress, this command reports the percentage done
 .sp
 .ne 2
 .na
-\fB\fB-c\fR \fBCMD\fR
+\fB\fB-c\fR \fB[SCRIPT1,SCRIPT2,...]\fR
 .ad
 .RS 12n
-Run a command on each vdev and include first line of output
+Run a script (or scripts) on each vdev and include the output in zpool status
 .sp
-The \fB-c\fR option allows you to run an arbitrary command on each vdev and
-display the first line of output in zpool iostat.  The following environment
-vars are set before running each command:
+The \fB-c\fR option allows you to run script(s) for each vdev and display the
+output in zpool iostat.  For security reasons, a user can only execute scripts
+found in the /<etc>/zfs/zpool.d directory as an unprivileged user.  However, a
+privileged user can run \fB-c\fR if they have the ZPOOL_SCRIPTS_AS_ROOT
+environment variable set.  If a script requires the use of a privileged
+command (like smartctl) then it's recommended you allow the user access to it in
+/etc/sudoers.  For example, to allow user "zfsuser" access to "smartctl -a", add
+the following to /etc/sudoers:
+
+zfsuser ALL=NOPASSWD: /usr/sbin/smartctl -a /dev/sd[a-z]*, NOEXEC: /usr/sbin/smartctl -a /dev/sd[a-z]*`
+
+If \fB-c\fR is passed without a script name, it prints a list of all scripts.
+
+Script output should be in the form of "name=value".  The column name is
+set to "name" and the value is set to "value".  Multiple lines can be used to
+output multiple columns.  The first line of output not in the "name=value"
+format is displayed without a column title, and no more output after that is
+displayed.  This can be useful for printing error messages.  Blank or NULL
+values are printed as a '-' to make output awk-able.
+
+The following environment variables are set before running each command:
 .sp
 \fB$VDEV_PATH\fR: Full path to the vdev.
 .LP
@@ -2567,32 +2604,38 @@ data        23.9G  14.6G  9.30G    48%         -    61%  1.00x  ONLINE  -
 \fBExample 16 \fRRunning commands in zpool status and zpool iostat with -c
 .sp
 .LP
-Some examples of using the command (-c) option with zpool status and zpool
-iostat:
 .sp
 .in +2
 .nf
-# \fBzpool status -c  \[aq]echo I am $VDEV_PATH, $VDEV_UPATH\[aq]\fR
-NAME        STATE     READ WRITE CKSUM
-mypool      ONLINE       0     0     0
+# zpool status -c vendor,model,size,enc
+...
+NAME        STATE     READ WRITE CKSUM   vendor         model  size      enc
+tank        ONLINE       0     0     0
   mirror-0  ONLINE       0     0     0
-    mpatha  ONLINE       0     0     0  I am /dev/mapper/mpatha, /dev/sdc
-    sdb     ONLINE       0     0     0  I am /dev/sdb1, /dev/sdb
+    U1      ONLINE       0     0     0  SEAGATE  ST8000NM0075  7.3T  0:0:0:0
+    U10     ONLINE       0     0     0  SEAGATE  ST8000NM0075  7.3T  0:0:0:0
+    U11     ONLINE       0     0     0  SEAGATE  ST8000NM0075  7.3T  0:0:0:0
+    U12     ONLINE       0     0     0  SEAGATE  ST8000NM0075  7.3T  0:0:0:0
+    U13     ONLINE       0     0     0  SEAGATE  ST8000NM0075  7.3T  0:0:0:0
+    U14     ONLINE       0     0     0  SEAGATE  ST8000NM0075  7.3T  0:0:0:0
 .fi
 .in -2
 
 .sp
 .in +2
 .nf
-# \fBzpool iostat -v -c \[aq]smartctl -a $VDEV_UPATH | grep "Current Drive Temperature"\[aq]\fR
-mypool       997M  7.25T      0      0   105K   106K
-  mirror     997M  7.25T      0      0   105K   106K
-    B0          -      -      0      0  17.4K  15.2K  Current Drive Temperature:     25 C
-    B1          -      -      0      0  17.4K  15.2K  Current Drive Temperature:     24 C
-    B2          -      -      0      0  17.5K  15.2K  Current Drive Temperature:     24 C
-    B3          -      -      0      0      0  15.1K  Current Drive Temperature:     24 C
-logs            -      -      -      -      -      -
-  B8            0  7.25T      0      0  1.14K  20.2K  Current Drive Temperature:     23 C
+# zpool iostat -vc slaves,locate_led
+              capacity     operations     bandwidth 
+pool        alloc   free   read  write   read  write     slaves  locate_led
+----------  -----  -----  -----  -----  -----  -----  ---------  ----------
+tank        20.4G  7.23T     26    152  20.7M  21.6M
+  mirror    20.4G  7.23T     26    152  20.7M  21.6M
+    U1          -      -      0     31  1.46K  20.6M   sdb sdff           0
+    U10         -      -      0      1  3.77K  13.3K  sdas sdgw           0
+    U11         -      -      0      1   288K  13.3K  sdat sdgx           1
+    U12         -      -      0      1  78.4K  13.3K  sdau sdgy           0
+    U13         -      -      0      1   128K  13.3K  sdav sdgz           0
+    U14         -      -      0      1  63.2K  13.3K   sdfk sdg           0
 .fi
 .in -2
 
@@ -2657,6 +2700,10 @@ This would also be true for future Linux based pools.
 
 A pool can be stripped of any "devid" values on import or prevented from adding
 them on \fBzpool create\fR or \fBzpool add\fR by setting ZFS_VDEV_DEVID_OPT_OUT.
+.TP
+.B "ZPOOL_SCRIPTS_AS_ROOT"
+Allow a privilaged user to run the \fBzpool status/iostat\fR with the \fB-c\fR
+option.  Normally, only unprivilaged users are allowed to run \fB-c\fR.
 
 .SH SEE ALSO
 .sp

rpm/generic/zfs.spec.in

@@ -95,6 +95,10 @@ Requires(postun): systemd
 BuildRequires: systemd
 %endif
 
+# The zpool iostat/status -c scripts call some utilities like lsblk and iostat
+Requires:  util-linux
+Requires:  sysstat
+
 %description
 This package contains the ZFS command line utilities.
 

tests/runfiles/linux.run

@@ -335,7 +335,8 @@ pre =
 post =
 
 [tests/functional/cli_root/zpool_status]
-tests = ['zpool_status_001_pos', 'zpool_status_002_pos']
+tests = ['zpool_status_001_pos', 'zpool_status_002_pos','zpool_status_003_pos']
+user =
 
 # DISABLED:
 # zpool_upgrade_002_pos - https://github.com/zfsonlinux/zfs/issues/4034

tests/zfs-tests/include/Makefile.am

@@ -4,4 +4,5 @@ dist_pkgdata_SCRIPTS = \
 	default.cfg \
 	libtest.shlib \
 	math.shlib \
-	properties.shlib
+	properties.shlib \
+	zpool_script.shlib

tests/zfs-tests/include/default.cfg

@@ -34,6 +34,7 @@
 
 # ZFS Directories
 export ZEDLETDIR=${ZEDLETDIR:-/etc/zfs/zed.d}
+export ZPOOLSCRIPTDIR=${ZPOOLSCRIPTDIR:-/etc/zfs/zpool.d}
 
 # Define run length constants
 export RT_LONG="3"

tests/zfs-tests/include/zpool_script.shlib

@@ -0,0 +1,50 @@
+#!/bin/ksh -p
+#
+# Common functions used by the zpool_status and zpool_iostat tests for running
+# scripts with the -c option.
+#
+# Copyright (c) 2017 Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+function test_zpool_script {
+	script="$1"
+	testpool="$2"
+	cmd="$3"
+	wholecmd="$cmd $script $testpool"
+	out="$($wholecmd)"
+
+	# Default number of columns that get printed without -c
+	if echo "$cmd" | grep -q iostat ; then
+		# iostat
+		dcols=7
+	else
+		
+		# status
+		dcols=5
+	fi
+		
+	# Get the new column name that the script created
+	col="$(echo "$out" | \
+	    awk '/^pool +alloc +free +read +write +/ {print $8} \
+	    /NAME +STATE +READ +WRITE +CKSUM/ {print $6}')"
+
+	if [ -z "$col" ] ; then
+		log_fail "'$wholecmd' created no new columns"
+	fi
+
+	# Count the number of columns for each vdev.  Each script should produce
+	# at least one new column value.  Even if scripts return blank, zpool
+	# will convert the blank to a '-' to make things awk-able.  Normal
+	# zpool iostat -v output is 7 columns, so if the script ran correctly
+	# we should see more than that.
+	if ! newcols=$(echo "$out" | \
+	    awk '/\/dev/{print NF-'$dcols'; if (NF <= '$dcols') {exit 1}}' | \
+	    head -n 1) ; \
+	    then
+		log_fail "'$wholecmd' didn't create a new column value"
+	else
+		log_note "'$wholecmd' passed ($newcols new columns)"
+	fi
+}

tests/zfs-tests/tests/functional/cli_root/zpool_status/Makefile.am

@@ -3,4 +3,5 @@ dist_pkgdata_SCRIPTS = \
 	setup.ksh \
 	cleanup.ksh \
 	zpool_status_001_pos.ksh \
-	zpool_status_002_pos.ksh
+	zpool_status_002_pos.ksh \
+	zpool_status_003_pos.ksh

tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh

@@ -0,0 +1,76 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013 by Delphix. All rights reserved.
+#
+
+#
+# Copyright (c) 2016-2017 by Lawrence Livermore National Security, LLC.
+#
+
+# DESCRIPTION:
+# Verify zpool status command mode (-c) works for all pre-baked scripts.
+#
+# STRATEGY:
+# 1. Make sure each script creates at least one new column.
+# 2. Make sure the new column values exist.
+# 3. Make sure we can run multiple scripts in one -c line
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/include/zpool_script.shlib
+
+verify_runnable "both"
+
+typeset testpool
+if is_global_zone ; then
+	testpool="$TESTPOOL"
+else
+	testpool="${TESTPOOL%%/*}"
+fi
+
+files="$(ls $ZPOOLSCRIPTDIR)"
+scripts=""
+for i in $files ; do
+	if [ ! -x "$ZPOOLSCRIPTDIR/$i" ] ; then
+		# Skip non-executables
+		continue
+	fi
+
+	# Collect executable script names
+	scripts="$scripts $i"
+
+	# Run each one with -c
+	test_zpool_script "$i" "$testpool" "zpool status -P -c"
+done
+
+# Test that we can run multiple scripts separated with a commma by running
+# all the scripts in a single -c line.
+allscripts="$(echo $scripts | sed -r 's/[[:blank:]]+/,/g')"
+test_zpool_script "$allscripts" "$testpool" "zpool status -P -c"
+
+exit 0

tests/zfs-tests/tests/functional/cli_user/misc/zpool_status_001_neg.ksh

@@ -26,7 +26,7 @@
 #
 
 #
-# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2013 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -69,23 +69,4 @@ check_pool_status
 log_must eval "zpool status -v $TESTPOOL > /tmp/pool-status.$$"
 check_pool_status
 
-# Make sure -c option works, and that VDEV_PATH and VDEV_UPATH get set.
-#
-# grep for '^\s+/' to just get the vdevs (not pools).  All vdevs will start with
-# a '/' when we specify the path (-P) flag. We check for "{}" to see if one
-# of the VDEV variables isn't set.
-C1=$(zpool status -P | grep -E '^\s+/' | wc -l)
-C2=$(zpool status -P -c 'echo vdev_test{$VDEV_PATH}{$VDEV_UPATH}' | \
-    grep -E '^\s+/' | grep -v '{}' | wc -l)
-
-if [ "$C1" != "$C2" ] ; then
-	log_fail "zpool status -c option failed.  Expected $C1 vdevs, got $C2"
-else
-	log_pass "zpool status -c option passed.  Expected $C1 vdevs, got $C2"
-fi
-
-# $TESTPOOL.virt has an offline device, so -x will show it
-log_must eval "zpool status -x $TESTPOOL.virt > /tmp/pool-status.$$"
-check_pool_status
-
 log_pass "zpool status works when run as a user"

tests/zfs-tests/tests/functional/cli_user/zpool_iostat/zpool_iostat_005_pos.ksh

@@ -30,11 +30,19 @@
 #
 
 #
-# Copyright (c) 2016 by Lawrence Livermore National Security, LLC.
+# Copyright (c) 2016-2017 by Lawrence Livermore National Security, LLC.
 #
 
+# DESCRIPTION:
+# Verify zpool iostat command mode (-c) works for all pre-baked scripts.
+#
+# STRATEGY:
+# 1. Make sure each script creates at least one new column.
+# 2. Make sure the new column values exist.
+# 3. Make sure we can run multiple scripts in one -c line
 
 . $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/include/zpool_script.shlib
 
 verify_runnable "both"
 
@@ -45,36 +53,24 @@ else
 	testpool=${TESTPOOL%%/*}
 fi
 
-#
-# DESCRIPTION:
-# Verify 'zpool iostat -c CMD' works, and that VDEV_PATH and VDEV_UPATH get set.
-#
-# STRATEGY:
-# grep for '^\s+/' to just get the vdevs (not pools).  All vdevs will start with
-# a '/' when we specify the path (-P) flag. We check for "{}" to see if one
-# of the VDEV variables isn't set.
-#
-C1=$(zpool iostat -Pv $testpool | grep -E '^\s+/' | wc -l)
-C2=$(zpool iostat -Pv -c 'echo vdev_test{$VDEV_PATH}{$VDEV_UPATH}' $testpool \
-    | grep -E '^\s+/' | grep -v '{}' | wc -l)
-if [ "$C1" != "$C2" ] ; then
-	log_fail "zpool iostat -c failed, expected $C1 vdevs, got $C2"
-else
-	log_note "zpool iostat -c passed, expected $C1 vdevs, got $C2"
-fi
+files="$(ls $ZPOOLSCRIPTDIR)"
+scripts=""
+for i in $files ; do
+	if [ ! -x "$ZPOOLSCRIPTDIR/$i" ] ; then
+		# Skip non-executables
+		continue
+	fi
 
-# Call iostat on only a specific vdev, and verify that the command only gets
-# run on the vdev.  We write the command results to a temp file to verify that
-# the command actually gets run, rather than just verifying that the results
-# are *displayed* for the specific vdev.
-TMP=$(mktemp)
-FIRST_VDEV=$(zpool iostat -Pv $testpool | grep -Eo '^\s+/[^ ]+' | head -n 1)
-log_must zpool iostat -Pv -c "echo \$VDEV_PATH >> $TMP" $testpool \
-    $FIRST_VDEV > /dev/null
-C2=$(wc -w < $TMP)
-rm $TMP
-if [ "$C2" != "1" ] ; then
-	log_fail "zpool iostat -c <VDEV> failed, expected 1 vdev, got $C2"
-else
-	log_note "zpool iostat -c <VDEV> passed, expected 1 vdev, got $C2"
-fi
+	# Collect executable script names
+	scripts="$scripts $i"
+
+	# Run each one with -c
+	test_zpool_script "$i" "$testpool" "zpool iostat -Pv -c"
+done
+
+# Test that we can run multiple scripts separated with a commma by running
+# all the scripts in a single -c line.
+allscripts="$(echo $scripts | sed -r 's/[[:blank:]]+/,/g')"
+test_zpool_script "$allscripts" "$testpool" "zpool iostat -Pv -c"
+
+exit 0

zfs-script-config.sh.in

@@ -21,6 +21,7 @@ export TESTSDIR=${SRCDIR}/tests
 export RUNFILEDIR=${TESTSDIR}/runfiles
 export UDEVRULEDIR=${BUILDDIR}/udev/rules.d
 export ZEDLETDIR=${SRCDIR}/cmd/zed/zed.d
+export ZPOOLSCRIPTDIR=${SRCDIR}/cmd/zpool/zpool.d
 
 export ZDB=${CMDDIR}/zdb/zdb
 export ZFS=${CMDDIR}/zfs/zfs