Add a diagnostic kstat for obtaining pool status

This kstat output does not require taking the spa_namespace
lock, as in the case for 'zpool status'. It can be used for
investigations when pools are in a hung state while holding
global locks required for a traditional 'zpool status' to
proceed.

This kstat is not safe to use in conditions where pools are
in the process of configuration changes (i.e., adding/removing
devices).  Therefore, this kstat is not intended to be a general
replacement or alternative to using 'zpool status'.

Sponsored-by: Wasabi Technology, Inc.
Sponsored-By: Klara Inc.

Co-authored-by: Don Brady <don.brady@klarasystems.com>
Signed-off-by: Don Brady <don.brady@klarasystems.com>

include/Makefile.am

@@ -175,6 +175,8 @@ COMMON_H = \
 
 
 KERNEL_H = \
+	sys/jprint.h \
+	sys/spa_json_stats.h \
 	sys/zfs_ioctl.h \
 	sys/zfs_ioctl_impl.h \
 	sys/zfs_onexit.h \

include/os/freebsd/spl/sys/kstat.h

@@ -67,8 +67,8 @@ struct list_head {};
 #define	KSTAT_FLAG_VIRTUAL	0x01
 #define	KSTAT_FLAG_VAR_SIZE	0x02
 #define	KSTAT_FLAG_WRITABLE	0x04
-#define	KSTAT_FLAG_PERSISTENT	0x08
+#define	KSTAT_FLAG_RESTRICTED	0x08
-#define	KSTAT_FLAG_DORMANT	0x10
+#define	KSTAT_FLAG_UNUSED	0x10
 #define	KSTAT_FLAG_INVALID	0x20
 #define	KSTAT_FLAG_LONGSTRINGS	0x40
 #define	KSTAT_FLAG_NO_HEADERS	0x80

include/os/linux/spl/sys/kstat.h

@@ -66,8 +66,8 @@
 #define	KSTAT_FLAG_VIRTUAL	0x01
 #define	KSTAT_FLAG_VAR_SIZE	0x02
 #define	KSTAT_FLAG_WRITABLE	0x04
-#define	KSTAT_FLAG_PERSISTENT	0x08
+#define	KSTAT_FLAG_RESTRICTED	0x08
-#define	KSTAT_FLAG_DORMANT	0x10
+#define	KSTAT_FLAG_UNUSED	0x10
 #define	KSTAT_FLAG_INVALID	0x20
 #define	KSTAT_FLAG_LONGSTRINGS	0x40
 #define	KSTAT_FLAG_NO_HEADERS	0x80

include/sys/jprint.h

@@ -0,0 +1,70 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2024, Klara Inc.
+ */
+
+#ifndef _SYS_JPRINT_H
+#define	_SYS_JPRINT_H
+
+/* maximum stack nesting */
+#define	JP_MAX_STACK	32
+
+enum jp_type {
+	JP_OBJECT = 1,
+	JP_ARRAY
+};
+
+struct jp_stack {
+	enum jp_type type;
+	int nelem;
+};
+
+typedef struct jprint {
+	char *buffer;		/* pointer to application's buffer */
+	size_t buflen;		/* length of buffer */
+	char *bufp;		/* current write position in buffer */
+	char tmpbuf[32];	/* local buffer for conversions */
+	int error;		/* error code */
+	int ncall;		/* API call number on which error occurred */
+	struct jp_stack		/* stack of array/object nodes */
+	    stack[JP_MAX_STACK];
+	int stackp;
+} jprint_t;
+
+/* error return codes */
+#define	JPRINT_OK		0	/* no error */
+#define	JPRINT_BUF_FULL		1	/* output buffer full */
+#define	JPRINT_NEST_ERROR	2	/* nesting error */
+#define	JPRINT_STACK_FULL	3	/* array/object nesting  */
+#define	JPRINT_STACK_EMPTY	4	/* stack underflow error */
+#define	JPRINT_OPEN		5	/* not all objects closed */
+#define	JPRINT_FMT		6	/* format error */
+
+const char *jp_errorstring(int err);
+int jp_error(jprint_t *jp);
+void jp_open(jprint_t *jp, char *buffer, size_t buflen);
+int jp_close(jprint_t *jp);
+int jp_errorpos(jprint_t *jp);
+int jp_printf(jprint_t *jp, const char *fmt, ...);
+
+#endif	/* _SYS_JPRINT_H */

include/sys/spa.h

@@ -771,6 +771,9 @@ extern void spa_scan_stat_init(spa_t *spa);
 extern int spa_scan_get_stats(spa_t *spa, pool_scan_stat_t *ps);
 extern int bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx);
 extern int bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx);
+extern void spa_add_spares(spa_t *spa, nvlist_t *config);
+extern void spa_add_l2cache(spa_t *spa, nvlist_t *config);
+extern void spa_add_feature_stats(spa_t *spa, nvlist_t *config);
 
 #define	SPA_ASYNC_CONFIG_UPDATE			0x01
 #define	SPA_ASYNC_REMOVE			0x02

include/sys/spa_impl.h

@@ -35,6 +35,7 @@
 #include <sys/spa.h>
 #include <sys/spa_checkpoint.h>
 #include <sys/spa_log_spacemap.h>
+#include <sys/spa_json_stats.h>
 #include <sys/vdev.h>
 #include <sys/vdev_rebuild.h>
 #include <sys/vdev_removal.h>
@@ -441,6 +442,7 @@ struct spa {
 	uint64_t	spa_autotrim;		/* automatic background trim? */
 	uint64_t	spa_errata;		/* errata issues detected */
 	spa_stats_t	spa_stats;		/* assorted spa statistics */
+	spa_json_stats_t spa_json_stats;	/* diagnostic status in JSON */
 	spa_keystore_t	spa_keystore;		/* loaded crypto keys */
 
 	/* arc_memory_throttle() parameters during low memory condition */

include/sys/spa_json_stats.h

@@ -0,0 +1,40 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2024, Klara Inc.
+ */
+
+#ifndef _SYS_SPA_JSON_STATS_H
+#define	_SYS_SPA_JSON_STATS_H
+
+#include <sys/zfs_context.h>
+#include <sys/spa_impl.h>
+#include <sys/kstat.h>
+
+typedef struct spa_json_stats {
+	kmutex_t	lock;
+	kstat_t		*kstat;
+} spa_json_stats_t;
+
+extern int spa_generate_json_stats(spa_t *spa, char *buf, size_t size);
+
+#endif	/* _SYS_SPA_JSON_STATS_H */

lib/libspl/include/sys/kstat.h

@@ -239,24 +239,13 @@ typedef struct kstat {
  *		The ks_snapshot routine (see below) does not need to check for
  *		this; permission checking is handled in the kstat driver.
  *
- *	KSTAT_FLAG_PERSISTENT:
+ *	KSTAT_FLAG_RESTRICTED:
  *
- *		Indicates that this kstat is to be persistent over time.
+ *		Indicates that this kstat has restricted access and is
- *		For persistent kstats, kstat_delete() simply marks the
+ *		not world readable.
- *		kstat as dormant; a subsequent kstat_create() reactivates
- *		the kstat.  This feature is provided so that statistics
- *		are not lost across driver close/open (e.g., raw disk I/O
- *		on a disk with no mounted partitions.)
- *		NOTE: Persistent kstats cannot be virtual, since ks_data
- *		points to garbage as soon as the driver goes away.
  *
  * The following flags are maintained by the kstat framework:
  *
- *	KSTAT_FLAG_DORMANT:
- *
- *		For persistent kstats, indicates that the kstat is in the
- *		dormant state (e.g., the corresponding device is closed).
- *
  *	KSTAT_FLAG_INVALID:
  *
  *		This flag is set when a kstat is in a transitional state,
@@ -268,8 +257,8 @@ typedef struct kstat {
 #define	KSTAT_FLAG_VIRTUAL		0x01
 #define	KSTAT_FLAG_VAR_SIZE		0x02
 #define	KSTAT_FLAG_WRITABLE		0x04
-#define	KSTAT_FLAG_PERSISTENT		0x08
+#define	KSTAT_FLAG_RESTRICTED		0x08
-#define	KSTAT_FLAG_DORMANT		0x10
+#define	KSTAT_FLAG_UNUSED		0x10
 #define	KSTAT_FLAG_INVALID		0x20
 #define	KSTAT_FLAG_LONGSTRINGS		0x40
 #define	KSTAT_FLAG_NO_HEADERS		0x80
@@ -722,33 +711,8 @@ extern void	kstat_init(void);	/* initialize kstat framework */
  * you must NOT be holding that kstat's ks_lock.  Otherwise, you may
  * deadlock with a kstat reader.
  *
- * Persistent kstats
+ * Persistent kstats are not implemented since there is no persistent
- *
+ * namespace for them to reside.
- * From the provider's point of view, persistence is transparent.  The only
- * difference between ephemeral (normal) kstats and persistent kstats
- * is that you pass KSTAT_FLAG_PERSISTENT to kstat_create().  Magically,
- * this has the effect of making your data visible even when you're
- * not home.  Persistence is important to tools like iostat, which want
- * to get a meaningful picture of disk activity.  Without persistence,
- * raw disk i/o statistics could never accumulate: they would come and
- * go with each open/close of the raw device.
- *
- * The magic of persistence works by slightly altering the behavior of
- * kstat_create() and kstat_delete().  The first call to kstat_create()
- * creates a new kstat, as usual.  However, kstat_delete() does not
- * actually delete the kstat: it performs one final update of the data
- * (i.e., calls the ks_update routine), marks the kstat as dormant, and
- * sets the ks_lock, ks_update, ks_private, and ks_snapshot fields back
- * to their default values (since they might otherwise point to garbage,
- * e.g. if the provider is going away).  kstat clients can still access
- * the dormant kstat just like a live kstat; they just continue to see
- * the final data values as long as the kstat remains dormant.
- * All subsequent kstat_create() calls simply find the already-existing,
- * dormant kstat and return a pointer to it, without altering any fields.
- * The provider then performs its usual initialization sequence, and
- * calls kstat_install().  kstat_install() uses the old data values to
- * initialize the native data (i.e., ks_update is called with KSTAT_WRITE),
- * thus making it seem like you were never gone.
  */
 
 extern kstat_t *kstat_create(const char *, int, const char *, const char *,

lib/libzpool/Makefile.am

@@ -112,6 +112,7 @@ nodist_libzpool_la_SOURCES = \
 	module/zfs/fm.c \
 	module/zfs/gzip.c \
 	module/zfs/hkdf.c \
+	module/zfs/jprint.c \
 	module/zfs/lz4.c \
 	module/zfs/lz4_zfs.c \
 	module/zfs/lzjb.c \
@@ -131,6 +132,7 @@ nodist_libzpool_la_SOURCES = \
 	module/zfs/spa_config.c \
 	module/zfs/spa_errlog.c \
 	module/zfs/spa_history.c \
+	module/zfs/spa_json_stats.c \
 	module/zfs/spa_log_spacemap.c \
 	module/zfs/spa_misc.c \
 	module/zfs/spa_stats.c \

module/Kbuild.in

@@ -353,6 +353,7 @@ ZFS_OBJS := \
 	fm.o \
 	gzip.o \
 	hkdf.o \
+	jprint.o \
 	lz4.o \
 	lz4_zfs.o \
 	lzjb.o \
@@ -372,6 +373,7 @@ ZFS_OBJS := \
 	spa_config.o \
 	spa_errlog.o \
 	spa_history.o \
+	spa_json_stats.o \
 	spa_log_spacemap.o \
 	spa_misc.o \
 	spa_stats.o \

module/Makefile.bsd

@@ -282,6 +282,7 @@ SRCS+=	abd.c \
 	edonr_zfs.c \
 	fm.c \
 	gzip.c \
+	jprint.c \
 	lz4.c \
 	lz4_zfs.c \
 	lzjb.c \
@@ -303,6 +304,7 @@ SRCS+=	abd.c \
 	spa_config.c \
 	spa_errlog.c \
 	spa_history.c \
+	spa_json_stats.c \
 	spa_log_spacemap.c \
 	spa_misc.c \
 	spa_stats.c \

module/os/linux/spl/spl-kstat.c

@@ -654,7 +654,7 @@ __kstat_install(kstat_t *ksp)
 	ASSERT(ksp);
 	mode_t mode;
 	/* Specify permission modes for different kstats */
-	if (strncmp(ksp->ks_proc.kpe_name, "dbufs", KSTAT_STRLEN) == 0) {
+	if (ksp->ks_flags & KSTAT_FLAG_RESTRICTED) {
 		mode = 0600;
 	} else {
 		mode = 0644;

module/zfs/dbuf_stats.c

@@ -189,7 +189,7 @@ dbuf_stats_hash_table_init(dbuf_hash_table_t *hash)
 	dsh->hash = hash;
 
 	ksp = kstat_create("zfs", 0, "dbufs", "misc",
-	    KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
+	    KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_RESTRICTED);
 	dsh->kstat = ksp;
 
 	if (ksp) {

module/zfs/jprint.c

@@ -0,0 +1,423 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2024, Klara Inc.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/jprint.h>
+
+/* Formats for int64_t and uint64_t */
+#ifndef PRId64
+#define	PRId64		"lld"	/* %D, int64_t */
+#define	PRIu64		"llu"	/* %U, uint64_t */
+#endif
+
+/* literal key length maximum */
+#define	KEYLEN		255
+
+/* return error position (call number of jp_printf) */
+int
+jp_errorpos(jprint_t *jp)
+{
+	return (jp->ncall);
+}
+
+/* return string for error code */
+const char *
+jp_errorstring(int err)
+{
+	switch (err) {
+	case JPRINT_OK:			return ("jprint ok");
+	case JPRINT_BUF_FULL:		return ("jprint buffer full");
+	case JPRINT_NEST_ERROR:		return ("jprint nest error");
+	case JPRINT_STACK_FULL:		return ("jprint stack full");
+	case JPRINT_STACK_EMPTY:	return ("jprint stack empty");
+	case JPRINT_OPEN:		return ("jprint open");
+	case JPRINT_FMT:		return ("jprint format");
+	default:			return ("jprint unknown error");
+	}
+	return ("jprint unknown error");
+}
+
+/* return error from jprint_t */
+int
+jp_error(jprint_t *jp)
+{
+	return (jp->error);
+}
+
+/* open json using buffer of length buflen */
+void
+jp_open(jprint_t *jp, char *buffer, size_t buflen)
+{
+	jp->buffer = jp->bufp = buffer;
+	jp->buflen = buflen;
+	jp->error = JPRINT_OK;
+	jp->ncall = 0;
+	jp->stackp = -1;
+	*buffer = '\0';
+}
+
+
+/* close json (return out of memory error) */
+int
+jp_close(jprint_t *jp)
+{
+	if (jp->error != JPRINT_OK)
+		return (jp->error);
+	if (jp->stackp != -1)
+		jp->error = JPRINT_OPEN;
+	return (jp->error);
+}
+
+
+/* put character to json */
+static void
+jp_putc(jprint_t *jp, char c)
+{
+	if (jp->error == JPRINT_OK) {
+		if ((jp->bufp - jp->buffer + 1) >= jp->buflen)
+			jp->error = JPRINT_BUF_FULL;
+		else {
+			*jp->bufp++ = c;
+			*jp->bufp = '\0';
+		}
+	}
+}
+
+
+/* put string to json */
+static void
+jp_puts(jprint_t *jp, char *s)
+{
+	while (*s && (jp->error == JPRINT_OK))
+		jp_putc(jp, *s++);
+}
+
+
+/* put quoted string to json */
+static void
+jp_putsq(jprint_t *jp, char *s)
+{
+	static const char *hex = "0123456789ABCDEF";
+	int c;
+
+	if (s == NULL) {
+		jp_puts(jp, (char *)"null");
+		return;
+	}
+	jp_putc(jp, '\"');
+	while (*s && (jp->error == JPRINT_OK)) {
+		c = (int)*s++;
+		/* formfeed, newline, return, tab, backspace */
+		if (c == 12)
+			jp_puts(jp, (char *)"\\f");
+		else if (c == 10)
+			jp_puts(jp, (char *)"\\n");
+		else if (c == 13)
+			jp_puts(jp, (char *)"\\r");
+		else if (c == 9)
+			jp_puts(jp, (char *)"\\t");
+		else if (c == 8)
+			jp_puts(jp, (char *)"\\b");
+		/*
+		 * all characters from 0x00 to 0x1f, and 0x7f are
+		 * escaped as: \u00xx
+		 */
+		else if (((0 <= c) && (c <= 0x1f)) || (c == 0x7f)) {
+			jp_puts(jp, (char *)"\\u00");
+			jp_putc(jp, hex[(c >> 4) & 0x0f]);
+			jp_putc(jp, hex[c & 0x0f]);
+		/* * " \ / */
+		} else if (c == '"')
+			jp_puts(jp, (char *)"\\\"");
+		else if (c == '\\')
+			jp_puts(jp, (char *)"\\\\");
+		else if (c == '/')
+			jp_puts(jp, (char *)"\\/");
+		/*
+		 * all other printable characters ' ' to '~', and
+		 * any utf-8 sequences (high bit set):
+		 * 1xxxxxxx 10xxxxxx ...
+		 * is a utf-8 sequence (10xxxxxx may occur 1 to 3 times).
+		 * Note that this is simply distinguished here as high
+		 * bit set.
+		 */
+		else
+			jp_putc(jp, (char)c);
+	}
+	jp_putc(jp, '\"');
+}
+
+
+/* put out key if object open. error if nothing open */
+static int
+jp_key(jprint_t *jp, char *key)
+{
+	if (jp->error != JPRINT_OK)
+		goto err;
+	/* at top level, no frame exists yet, no error */
+	if (jp->stackp == -1)
+		goto err;
+	/* stackp has been "popped" too many times */
+	if (jp->stackp < -1) {
+		jp->error = JPRINT_STACK_EMPTY;
+		goto err;
+	}
+	/* put comma separator in (both object and array) */
+	if (++jp->stack[jp->stackp].nelem > 1)
+		jp_putc(jp, ',');
+	/* if its in an object, put out the key and separator */
+	if (jp->stack[jp->stackp].type == JP_OBJECT) {
+		jp_putsq(jp, key);
+		jp_putc(jp, ':');
+	}
+err:
+	return (jp->error);
+}
+
+
+/* printf to json */
+int
+jp_printf(jprint_t *jp, const char *fmt, ...)
+{
+	char key[KEYLEN + 1];
+	int k, i;
+	va_list ap;
+	int n;
+	unsigned int u;
+	int64_t n64;
+	uint64_t u64;
+	boolean_t b;
+	char *s;
+	char *start = jp->bufp;
+
+	if (jp->error != JPRINT_OK)
+		return (-1);
+	++jp->ncall;
+	va_start(ap, fmt);
+	key[k = 0] = '\0';
+	while (*fmt && (jp->error == JPRINT_OK)) {
+		/*
+		 * If we need to debug jprint format,
+		 * zfs_dbgmsg("====> jprint char = %c\n", *fmt);
+		 */
+		switch (*fmt) {
+		case '%':
+			++fmt;
+			switch (*fmt) {
+			case 'k': /* next parameter is key */
+				if (jp->stackp < 0) {
+					jp->error = JPRINT_STACK_EMPTY;
+					break;
+				}
+				s = va_arg(ap, char *);
+				if (strlen(s) <= KEYLEN)
+					strcpy(key, s);
+				else
+					jp->error = JPRINT_FMT;
+				break;
+			case 'd': /* next parameter is int */
+				if (jp->stackp < 0) {
+					jp->error = JPRINT_STACK_EMPTY;
+					break;
+				}
+				n = va_arg(ap, int);
+				i = snprintf(
+				    jp->tmpbuf, sizeof (jp->tmpbuf),
+				    "%d", n);
+				if (jp_key(jp, key) == JPRINT_OK) {
+					if ((i >= sizeof (jp->tmpbuf)) ||
+					    (i < 0))
+						jp_puts(jp, (char *)"####");
+					else
+						jp_puts(jp, jp->tmpbuf);
+				}
+				key[k = 0] = '\0';
+				break;
+			case 'u': /* next parameter is unsigned int */
+				if (jp->stackp < 0) {
+					jp->error = JPRINT_STACK_EMPTY;
+					break;
+				}
+				u = va_arg(ap, unsigned int);
+				i = snprintf(
+				    jp->tmpbuf, sizeof (jp->tmpbuf),
+				    "%u", u);
+				if (jp_key(jp, key) == JPRINT_OK) {
+					if ((i >= sizeof (jp->tmpbuf)) ||
+					    (i < 0))
+						jp_puts(jp, (char *)"####");
+					else
+						jp_puts(jp, jp->tmpbuf);
+				}
+				key[k = 0] = '\0';
+				break;
+			case 'U': /* next parameter is uint64_t */
+				if (jp->stackp < 0) {
+					jp->error = JPRINT_STACK_EMPTY;
+					break;
+				}
+				u64 = va_arg(ap, uint64_t);
+				i = snprintf(
+				    jp->tmpbuf, sizeof (jp->tmpbuf),
+				    "%" PRIu64, u64);
+				if (jp_key(jp, key) == JPRINT_OK) {
+					if ((i >= sizeof (jp->tmpbuf)) ||
+					    (i < 0))
+						jp_puts(jp, (char *)"####");
+					else
+						jp_puts(jp, jp->tmpbuf);
+				}
+				key[k = 0] = '\0';
+				break;
+			case 'D': /* next parameter is int64_t */
+				if (jp->stackp < 0) {
+					jp->error = JPRINT_STACK_EMPTY;
+					break;
+				}
+				n64 = va_arg(ap, int64_t);
+				i = snprintf(
+				    jp->tmpbuf, sizeof (jp->tmpbuf),
+				    "%" PRId64, n64);
+				if (jp_key(jp, key) == JPRINT_OK) {
+					if ((i >= sizeof (jp->tmpbuf)) ||
+					    (i < 0))
+						jp_puts(jp, (char *)"####");
+					else
+						jp_puts(jp, jp->tmpbuf);
+				}
+				key[k = 0] = '\0';
+				break;
+			case 's': /* next parameter is string */
+				if (jp->stackp < 0) {
+					jp->error = JPRINT_STACK_EMPTY;
+					break;
+				}
+				s = va_arg(ap, char *);
+				if (jp_key(jp, key) == JPRINT_OK)
+					jp_putsq(jp, s);
+				key[k = 0] = '\0';
+				break;
+			case 'b': /* next parameter is boolean */
+				if (jp->stackp < 0) {
+					jp->error = JPRINT_STACK_EMPTY;
+					break;
+				}
+				if (jp_key(jp, key) == JPRINT_OK) {
+					b = (boolean_t)va_arg(ap, int);
+					s = b ?
+					    (char *)"true" :
+					    (char *)"false";
+					jp_puts(jp, s);
+				}
+				key[k = 0] = '\0';
+				break;
+			case '%': /* literal % */
+				if (k < KEYLEN) {
+					key[k++] = '%';
+					key[k] = '\0';
+				} else
+					jp->error = JPRINT_FMT;
+				break;
+			default:
+				jp->error = JPRINT_FMT;
+			}
+			break;
+		case '{': /* open object */
+			if (jp->stackp >= (JP_MAX_STACK - 1))
+				jp->error = JPRINT_STACK_FULL;
+			else {
+				(void) jp_key(jp, key);
+				++jp->stackp;
+				jp->stack[jp->stackp].type = JP_OBJECT;
+				jp->stack[jp->stackp].nelem = 0;
+				jp_putc(jp, '{');
+			}
+			break;
+		case '}': /* close object */
+			if (jp->stackp < 0)
+				jp->error = JPRINT_STACK_EMPTY;
+			else if (jp->stack[jp->stackp].type != JP_OBJECT)
+				jp->error = JPRINT_NEST_ERROR;
+			else {
+				--jp->stackp;
+				jp_putc(jp, '}');
+			}
+			break;
+		case '[': /* open array */
+			if (jp->stackp >= (JP_MAX_STACK - 1))
+				jp->error = JPRINT_STACK_FULL;
+			else {
+				(void) jp_key(jp, key);
+				++jp->stackp;
+				jp->stack[jp->stackp].type = JP_ARRAY;
+				jp->stack[jp->stackp].nelem = 0;
+				jp_putc(jp, '[');
+			}
+			break;
+		case ']': /* close array */
+			if (jp->stackp < 0)
+				jp->error = JPRINT_STACK_EMPTY;
+			else if (jp->stack[jp->stackp].type != JP_ARRAY)
+				jp->error = JPRINT_NEST_ERROR;
+			else {
+				--jp->stackp;
+				jp_putc(jp, ']');
+			}
+			break;
+
+		case ',': /* ,: space tab are ignored */
+		case ':':
+		case ' ':
+		case '\t':
+			break;
+		case '\\':
+			/* allow inclusion of ,: space tab to key */
+			if (fmt[1] == '\0')
+				jp->error = JPRINT_FMT;
+			else {
+				++fmt;
+				if (k < KEYLEN) {
+					key[k++] = *fmt;
+					key[k] = '\0';
+				} else
+					jp->error = JPRINT_FMT;
+			}
+			break;
+		default:
+			if (k < KEYLEN) {
+				key[k++] = *fmt;
+				key[k] = '\0';
+			} else
+				jp->error = JPRINT_FMT;
+			break;
+		}
+		++fmt;
+	}
+	va_end(ap);
+	if (jp->error != JPRINT_OK)
+		return (-1);
+
+	return ((int)(jp->bufp - start));
+}

module/zfs/spa.c

@@ -5956,7 +5956,7 @@ spa_inject_delref(spa_t *spa)
 /*
  * Add spares device information to the nvlist.
  */
-static void
+void
 spa_add_spares(spa_t *spa, nvlist_t *config)
 {
 	nvlist_t **spares;
@@ -6006,7 +6006,7 @@ spa_add_spares(spa_t *spa, nvlist_t *config)
 /*
  * Add l2cache device information to the nvlist, including vdev stats.
  */
-static void
+void
 spa_add_l2cache(spa_t *spa, nvlist_t *config)
 {
 	nvlist_t **l2cache;
@@ -6118,7 +6118,7 @@ spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features)
  * ensures we don't block here on I/O on a suspended pool so 'zpool
  * clear' can resume the pool.
  */
-static void
+void
 spa_add_feature_stats(spa_t *spa, nvlist_t *config)
 {
 	nvlist_t *features;

module/zfs/spa_json_stats.c

@@ -0,0 +1,649 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2024, Klara Inc.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/spa_impl.h>
+#include <sys/vdev_impl.h>
+#include <sys/spa.h>
+#include <zfs_comutil.h>
+#include <sys/jprint.h>
+#include <sys/spa_json_stats.h>
+#include <sys/nvpair_impl.h>
+
+#define	JSON_STATUS_VERSION	4
+
+/*
+ * Return string for datatype -- this guides us in implementing
+ * json translation.
+ */
+static const char *
+datatype_string(data_type_t t)
+{
+	switch (t) {
+	case DATA_TYPE_UNKNOWN:		return "DATA_TYPE_UNKNOWN";
+	case DATA_TYPE_BOOLEAN:		return "DATA_TYPE_BOOLEAN";
+	case DATA_TYPE_BYTE:		return "DATA_TYPE_BYTE";
+	case DATA_TYPE_INT16:		return "DATA_TYPE_INT16";
+	case DATA_TYPE_UINT16:		return "DATA_TYPE_UINT16";
+	case DATA_TYPE_INT32:		return "DATA_TYPE_INT32";
+	case DATA_TYPE_UINT32:		return "DATA_TYPE_UINT32";
+	case DATA_TYPE_INT64:		return "DATA_TYPE_INT64";
+	case DATA_TYPE_UINT64:		return "DATA_TYPE_UINT64";
+	case DATA_TYPE_STRING:		return "DATA_TYPE_STRING";
+	case DATA_TYPE_BYTE_ARRAY:	return "DATA_TYPE_BYTE_ARRAY";
+	case DATA_TYPE_INT16_ARRAY:	return "DATA_TYPE_INT16_ARRAY";
+	case DATA_TYPE_UINT16_ARRAY:	return "DATA_TYPE_UINT16_ARRAY";
+	case DATA_TYPE_INT32_ARRAY:	return "DATA_TYPE_INT32_ARRAY";
+	case DATA_TYPE_UINT32_ARRAY:	return "DATA_TYPE_UINT32_ARRAY";
+	case DATA_TYPE_INT64_ARRAY:	return "DATA_TYPE_INT64_ARRAY";
+	case DATA_TYPE_UINT64_ARRAY:	return "DATA_TYPE_UINT64_ARRAY";
+	case DATA_TYPE_STRING_ARRAY:	return "DATA_TYPE_STRING_ARRAY";
+	case DATA_TYPE_HRTIME:		return "DATA_TYPE_HRTIME";
+	case DATA_TYPE_NVLIST:		return "DATA_TYPE_NVLIST";
+	case DATA_TYPE_NVLIST_ARRAY:	return "DATA_TYPE_NVLIST_ARRAY";
+	case DATA_TYPE_BOOLEAN_VALUE:	return "DATA_TYPE_BOOLEAN_VALUE";
+	case DATA_TYPE_INT8:		return "DATA_TYPE_INT8";
+	case DATA_TYPE_UINT8:		return "DATA_TYPE_UINT8";
+	case DATA_TYPE_BOOLEAN_ARRAY:	return "DATA_TYPE_BOOLEAN_ARRAY";
+	case DATA_TYPE_INT8_ARRAY:	return "DATA_TYPE_INT8_ARRAY";
+	case DATA_TYPE_UINT8_ARRAY:	return "DATA_TYPE_UINT8_ARRAY";
+	default:			return "UNKNOWN";
+	}
+}
+
+/*
+ * nvlist_to_json takes a filter function. If the functions returns
+ * B_TRUE, the case has been handled. If it returns B_FALSE, the
+ * case has not been handled, and will be handled. Invoking nvlist_to_json
+ * with a NULL filter chooses the default filter, which does nothing.
+ *
+ * The filtering is passed the jprint_t in case the nesting level is
+ * important, name, data type and value.
+ */
+typedef boolean_t nvj_filter_t(jprint_t *, const char *, data_type_t, void *);
+
+static boolean_t
+null_filter(jprint_t *jp, const char *name, data_type_t type, void *value)
+{
+	jp = jp; name = name; type = type; value = value;
+	return (B_FALSE);
+}
+
+static void nvlist_to_json(nvlist_t *nvl, jprint_t *jp, nvj_filter_t f);
+
+/*
+ * Convert source (src) to string -- up to 105 characters, so pass in 256
+ * byte buffer (for future)
+ */
+static void
+source_to_string(uint64_t src, char *buf)
+{
+	buf[0] = '\0';
+	if (src & ZPROP_SRC_NONE) {
+		if (buf[0] != '\0')
+			strcat(buf, "|");
+		strcat(buf, "ZPROP_SRC_NONE");
+	}
+	if (src & ZPROP_SRC_DEFAULT) {
+		if (buf[0] != '\0')
+			strcat(buf, "|");
+		strcat(buf, "ZPROP_SRC_DEFAULT");
+	}
+	if (src & ZPROP_SRC_TEMPORARY) {
+		if (buf[0] != '\0')
+			strcat(buf, "|");
+		strcat(buf, "ZPROP_SRC_TEMPORARY");
+	}
+	if (src & ZPROP_SRC_INHERITED) {
+		if (buf[0] != '\0')
+			strcat(buf, "|");
+		strcat(buf, "ZPROP_SRC_INHERITED");
+	}
+	if (src & ZPROP_SRC_RECEIVED) {
+		if (buf[0] != '\0')
+			strcat(buf, "|");
+		strcat(buf, "ZPROP_SRC_RECEIVED");
+	}
+}
+
+/*
+ * spa_props_filter replace source: with string. The way source is
+ * defined it could be bitmap -- so generate the | sequence as
+ * needed.
+ */
+static boolean_t
+spa_props_filter(jprint_t *jp, const char *name, data_type_t type, void *value)
+{
+	if ((strcmp(name, "source") == 0) &&
+	    (type == DATA_TYPE_UINT64)) {
+		uint64_t src = *(uint64_t *)value;
+		char buf[256];
+		source_to_string(src, buf);
+		jp_printf(jp, "source: %s", buf);
+		return (B_TRUE);
+	}
+	return (B_FALSE);
+}
+
+/*
+ * stats_filter removes parts of the nvlist we don't want to visit.
+ */
+static boolean_t
+stats_filter(jprint_t *jp, const char *name, data_type_t type, void *value)
+{
+	/*
+	 * Suppress root object state:
+	 */
+	if ((jp->stackp == 0) &&
+	    (type == DATA_TYPE_UINT64) &&
+	    (strcmp(name, "state") == 0))
+		return (B_TRUE);
+
+	/*
+	 * Suppress root object vdev_children: -- we will
+	 * output at one level down.
+	 */
+	if ((jp->stackp == 0) &&
+	    (type == DATA_TYPE_UINT64) &&
+	    (strcmp(name, "vdev_children") == 0))
+		return (B_TRUE);
+
+	/*
+	 * We are going to suppress vdev_tree:, and generate the
+	 * data ourselves.
+	 * It does seem like a bit of a waste going through this
+	 * twice... but for now, this seems prudent.
+	 */
+	if ((jp->stackp == 0) &&
+	    (type == DATA_TYPE_NVLIST) &&
+	    (strcmp(name, "vdev_tree") == 0))
+		return (B_TRUE);
+
+	/*
+	 * Process spa_props:. Here we recurse, but with a filter that
+	 * modifies source.
+	 */
+	if ((jp->stackp == 0) &&
+	    (type == DATA_TYPE_NVLIST) &&
+	    (strcmp(name, "spa_props") == 0)) {
+		jp_printf(jp, "spa_props: {");
+		nvlist_to_json((nvlist_t *)value, jp, spa_props_filter);
+		jp_printf(jp, "}");
+		return (B_TRUE);
+	}
+
+	return (B_FALSE);
+}
+
+/*
+ * This code is NOT hightly abstracted -- just some duplication, until I
+ * actually understand what is needed.
+ *
+ * In avoiding early abstraction, we find the need for a "filter". Which
+ * is now implemented. This does appear in the spirit of other ZFS
+ * coding.
+ */
+static void
+nvlist_to_json(nvlist_t *nvl, jprint_t *jp, nvj_filter_t f)
+{
+	const nvpriv_t *priv;
+	const i_nvp_t *curr;
+	uint64_t *u = NULL;
+	nvlist_t **a = NULL;
+
+	if (f == NULL)
+		f = null_filter;
+
+	if ((priv = (const nvpriv_t *)(uintptr_t)nvl->nvl_priv) == NULL)
+		return;
+
+	for (curr = priv->nvp_list; curr != NULL; curr = curr->nvi_next) {
+		const nvpair_t *nvp = &curr->nvi_nvp;
+		const char *name = (const char *)NVP_NAME(nvp);
+		data_type_t type = NVP_TYPE(nvp);
+		void *p = NVP_VALUE(nvp);
+
+		if (jp_error(jp) != JPRINT_OK)
+			return;
+
+		if (f(jp, name, type, p))
+			continue;
+		switch (type) {
+
+		/*
+		 * Array types
+		 */
+		case DATA_TYPE_UINT64_ARRAY:
+			u = (uint64_t *)p;
+			jp_printf(jp, "%k: [", name);
+			for (int i = 0; i < NVP_NELEM(nvp); ++i) {
+				if (jp_error(jp) != JPRINT_OK)
+					break;
+				jp_printf(jp, "%U", u[i]);
+			}
+			jp_printf(jp, "]");
+			break;
+		case DATA_TYPE_NVLIST_ARRAY:
+			a = (nvlist_t **)p;
+			jp_printf(jp, "%k: [", name);
+			for (int i = 0; i < NVP_NELEM(nvp); ++i)  {
+				if (jp_error(jp) != JPRINT_OK)
+					break;
+				jp_printf(jp, "{");
+				nvlist_to_json(a[i], jp, f);
+				jp_printf(jp, "}");
+			}
+			jp_printf(jp, "]");
+			break;
+
+		/*
+		 * Primitive types
+		 */
+		case DATA_TYPE_UINT64:
+			jp_printf(jp, "%k: %U", name, *(uint64_t *)p);
+			break;
+		case DATA_TYPE_INT64:
+			jp_printf(jp, "%k: %D", name, *(int64_t *)p);
+			break;
+		case DATA_TYPE_UINT32:
+			jp_printf(jp, "%k: %u", name, *(uint32_t *)p);
+			break;
+		case DATA_TYPE_INT32:
+			jp_printf(jp, "%k: %d", name, *(int32_t *)p);
+			break;
+		case DATA_TYPE_STRING:
+			jp_printf(jp, "%k: %s", name, (char *)p);
+			break;
+		case DATA_TYPE_BOOLEAN:
+			jp_printf(jp, "%k: %b", name, B_TRUE);
+			break;
+		case DATA_TYPE_BOOLEAN_VALUE:
+			jp_printf(jp, "%k: %b", name, *(boolean_t *)p);
+			break;
+
+		/*
+		 * Object types
+		 */
+		case DATA_TYPE_NVLIST:
+			jp_printf(jp, "%k: {", name);
+			nvlist_to_json((nvlist_t *)p, jp, f);
+			jp_printf(jp, "}");
+			break;
+
+		/*
+		 * Default -- tell us what we are missing. This is done to
+		 * simply avoid writing ALL the cases, YAGNI (yah ain't
+		 * gonna need it).
+		 */
+		default:
+			jp_printf(jp, "%k: %s", name, datatype_string(type));
+			zfs_dbgmsg("name = %s type = %d %s", name,
+			    (int)type, datatype_string(type));
+		break;
+		}
+	}
+}
+
+static const char *
+vdev_state_string(vdev_state_t state, vdev_aux_t aux)
+{
+	const char *s;
+	switch (state) {
+	case VDEV_STATE_UNKNOWN:	s = "HEALTHY";    break;
+	case VDEV_STATE_CLOSED:		s = "CLOSED";	  break;
+	case VDEV_STATE_OFFLINE:	s = "OFFLINE";    break;
+	case VDEV_STATE_REMOVED:	s = "REMOVED";    break;
+	case VDEV_STATE_CANT_OPEN:
+		if (aux == VDEV_AUX_CORRUPT_DATA || aux == VDEV_AUX_BAD_LOG)
+			s = "FAULTED";
+		else if (aux == VDEV_AUX_SPLIT_POOL)
+			s = "SPLIT";
+		else
+			s = "UNAVAIL";
+		break;
+	case VDEV_STATE_FAULTED:	s = "FAULTED";    break;
+	case VDEV_STATE_DEGRADED:	s = "DEGRADED";   break;
+	case VDEV_STATE_HEALTHY:	s = "HEALTHY";    break;
+	default:			s = "?";
+	}
+	return (s);
+}
+
+static void
+vdev_to_json(vdev_t *vd, pool_scan_stat_t *ps, jprint_t *jp)
+{
+	uint64_t i, n;
+	int nparity = vdev_get_nparity(vd);
+	vdev_t **a;
+	const char *s;
+
+	jp_printf(jp, "type: %s", vd->vdev_ops->vdev_op_type);
+
+	/* dRAID vdevs have additional config keys */
+	if (vd->vdev_ops == &vdev_draid_ops &&
+	    vd->vdev_ops->vdev_op_config_generate != NULL) {
+		nvlist_t *nvl = fnvlist_alloc();
+		vd->vdev_ops->vdev_op_config_generate(vd, nvl);
+		nvlist_to_json(nvl, jp, null_filter);
+		nvlist_free(nvl);
+	} else if (nparity > 0) {
+		/* RAIDZ parity */
+		jp_printf(jp, "nparity: %U", nparity);
+	}
+	jp_printf(jp, "id: %U", vd->vdev_id);
+	jp_printf(jp, "guid: %U", vd->vdev_guid);
+	if (strcmp(vd->vdev_ops->vdev_op_type, "root") != 0) {
+		jp_printf(jp, "asize: %U", vd->vdev_asize);
+		jp_printf(jp, "ashift: %U", vd->vdev_ashift);
+		if (vd->vdev_ops->vdev_op_leaf) {
+			jp_printf(jp, "whole_disk: %b",
+			    (vd->vdev_wholedisk == 0) ? B_FALSE : B_TRUE);
+		}
+		jp_printf(jp, "offline: %b",
+		    (vd->vdev_offline == 0) ? B_FALSE : B_TRUE);
+		jp_printf(jp, "faulted: %b",
+		    (vd->vdev_faulted == 0) ? B_FALSE : B_TRUE);
+		jp_printf(jp, "degraded: %b",
+		    (vd->vdev_degraded == 0) ? B_FALSE : B_TRUE);
+		jp_printf(jp, "removed: %b",
+		    (vd->vdev_removed == 0) ? B_FALSE : B_TRUE);
+		jp_printf(jp, "not_present: %b",
+		    (vd->vdev_not_present == 0) ? B_FALSE : B_TRUE);
+		jp_printf(jp, "is_log: %b",
+		    (vd->vdev_islog == 0) ? B_FALSE : B_TRUE);
+
+		if (vd->vdev_path != NULL)
+			jp_printf(jp, "path: %s", vd->vdev_path);
+		if (vd->vdev_devid != NULL)
+			jp_printf(jp, "devid: %s",  vd->vdev_devid);
+		if (vd->vdev_physpath != NULL)
+			jp_printf(jp, "physpath: %s", vd->vdev_physpath);
+		if (vd->vdev_enc_sysfs_path != NULL)
+			jp_printf(jp, "enc_sysfs_path: %s",
+			    vd->vdev_enc_sysfs_path);
+		jp_printf(jp, "state: %s", vdev_state_string(vd->vdev_state,
+		    vd->vdev_stat.vs_aux));
+		/*
+		 * Try for some of the extended status annotations that
+		 * zpool status provides.
+		 */
+		jp_printf(jp, "vs_scan_removing: %b",
+		    vd->vdev_stat.vs_scan_removing != 0);
+		jp_printf(jp, "vs_noalloc: %b", vd->vdev_stat.vs_noalloc != 0);
+		jp_printf(jp, "vs_resilver_deferred: %b",
+		    vd->vdev_stat.vs_resilver_deferred);
+		s = "none";
+		if ((vd->vdev_state == VDEV_STATE_UNKNOWN) ||
+		    (vd->vdev_state == VDEV_STATE_HEALTHY)) {
+			if (vd->vdev_stat.vs_scan_processed != 0) {
+				if (ps &&
+				    (ps->pss_state == DSS_SCANNING)) {
+					s = (ps->pss_func ==
+					    POOL_SCAN_RESILVER) ?
+					    "resilvering" : "repairing";
+				} else if (ps &&
+				    vd->vdev_stat.vs_resilver_deferred) {
+					s = "awaiting resilver";
+				}
+			}
+		}
+		jp_printf(jp, "resilver_repair: %s", s);
+		jp_printf(jp, "initialize_state: {");
+		s = "VDEV_INITIALIZE_NONE";
+		if (vd->vdev_stat.vs_initialize_state == VDEV_INITIALIZE_ACTIVE)
+			s = "VDEV_INITIALIZE_ACTIVE";
+		if (vd->vdev_stat.vs_initialize_state ==
+		    VDEV_INITIALIZE_SUSPENDED)
+			s = "VDEV_INITIALIZE_SUSPENDED";
+		if (vd->vdev_stat.vs_initialize_state ==
+		    VDEV_INITIALIZE_COMPLETE)
+			s = "VDEV_INITIALIZE_COMPLETE";
+		jp_printf(jp, "vs_initialize_state: %s", s);
+		jp_printf(jp, "vs_initialize_bytes_done: %U",
+		    vd->vdev_stat.vs_initialize_bytes_done);
+		jp_printf(jp, "vs_initialize_bytes_est: %U",
+		    vd->vdev_stat.vs_initialize_bytes_est);
+		jp_printf(jp, "vs_initialize_action_time: %U",
+		    vd->vdev_stat.vs_initialize_action_time);
+		jp_printf(jp, "}");
+
+		jp_printf(jp, "trim_state: {");
+		s = "VDEV_UNTRIMMED";
+		if (vd->vdev_stat.vs_trim_state == VDEV_TRIM_ACTIVE)
+			s = "VDEV_TRIM_ACTIVE";
+		if (vd->vdev_stat.vs_trim_state == VDEV_TRIM_SUSPENDED)
+			s = "VDEV_TRIM_SUSPENDED";
+		if (vd->vdev_stat.vs_trim_state == VDEV_TRIM_COMPLETE)
+			s = "VDEV_TRIM_COMPLETE";
+		if (vd->vdev_stat.vs_trim_notsup)
+			s = "VDEV_TRIM_UNSUPPORTED";
+		jp_printf(jp, "vs_trim_state: %s", s);
+		if (!vd->vdev_stat.vs_trim_notsup) {
+			jp_printf(jp, "vs_trim_action_time: %U",
+			    vd->vdev_stat.vs_trim_action_time);
+			jp_printf(jp, "vs_trim_bytes_done: %U",
+			    vd->vdev_stat.vs_trim_bytes_done);
+			jp_printf(jp, "vs_trim_bytes_est: %U",
+			    vd->vdev_stat.vs_trim_bytes_est);
+		}
+		jp_printf(jp, "}");
+
+		jp_printf(jp, "read_errors: %U",
+		    vd->vdev_stat.vs_read_errors);
+		jp_printf(jp, "write_errors: %U",
+		    vd->vdev_stat.vs_write_errors);
+		jp_printf(jp, "checksum_errors: %U",
+		    vd->vdev_stat.vs_checksum_errors);
+		jp_printf(jp, "slow_ios: %U",
+		    vd->vdev_stat.vs_slow_ios);
+		jp_printf(jp, "trim_errors: %U",
+		    vd->vdev_stat.vs_trim_errors);
+	}
+	n = vd->vdev_children;
+	a = vd->vdev_child;
+	if (n != 0) {
+		jp_printf(jp, "vdev_children: %U", n);
+		jp_printf(jp, "children: [");
+		for (i = 0; i < n; ++i) {
+			jp_printf(jp, "{");
+			vdev_to_json(a[i], ps, jp);
+			jp_printf(jp, "}");
+		}
+		jp_printf(jp, "]");
+	}
+}
+
+static void
+iterate_vdevs(spa_t *spa, pool_scan_stat_t *ps, jprint_t *jp)
+{
+	vdev_t *v = spa->spa_root_vdev;
+	if (v == NULL) {
+		jp_printf(jp, "error: %s", "NO ROOT VDEV");
+		return;
+	}
+	jp_printf(jp, "vdev_tree: {");
+	vdev_to_json(v, ps, jp);
+	jp_printf(jp, "}");
+}
+
+static const char *
+pss_func_to_string(uint64_t n)
+{
+	const char *s = "?";
+	switch (n) {
+		case POOL_SCAN_NONE:		s = "NONE";	break;
+		case POOL_SCAN_SCRUB:		s = "SCRUB";	break;
+		case POOL_SCAN_RESILVER:	s = "RESILVER";	break;
+		case POOL_SCAN_FUNCS:		s = "?";
+	}
+	return (s);
+}
+
+static const char *pss_state_to_string(uint64_t n)
+{
+	const char *s = "?";
+	switch (n) {
+		case DSS_NONE:		s = "NONE";	break;
+		case DSS_SCANNING:	s = "SCANNING";	break;
+		case DSS_FINISHED:	s = "FINISHED";	break;
+		case DSS_CANCELED:	s = "CANCELED";	break;
+		case DSS_NUM_STATES:	s = "?";
+	}
+	return (s);
+}
+
+/*
+ * Collect the spa status without any locking and return as a JSON string.
+ *
+ * Currently used by the 'zfs/<pool>/stats.json' kstat.
+ */
+int
+spa_generate_json_stats(spa_t *spa, char *buf, size_t size)
+{
+	int error = 0;
+	int ps_error = 0;
+	jprint_t jp;
+	nvlist_t *nvl, *pnvl;
+	uint64_t loadtimes[2];
+	pool_scan_stat_t ps;
+	int scl_config_lock;
+
+	if (nvlist_dup(spa->spa_config, &nvl, 0) != 0) {
+		zfs_dbgmsg("json_data: nvlist_dup failed");
+		return (0);
+	}
+	fnvlist_add_nvlist(nvl, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info);
+
+	scl_config_lock =
+	    spa_config_tryenter(spa, SCL_CONFIG, FTAG, RW_READER);
+
+	ps_error = spa_scan_get_stats(spa, &ps);
+
+	error = spa_prop_get(spa, &pnvl);
+	fnvlist_add_nvlist(nvl, "spa_props", pnvl);
+
+	loadtimes[0] = spa->spa_loaded_ts.tv_sec;
+	loadtimes[1] = spa->spa_loaded_ts.tv_nsec;
+	fnvlist_add_uint64_array(nvl, ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2);
+	fnvlist_add_uint64(nvl, ZPOOL_CONFIG_ERRCOUNT,
+	    spa_approx_errlog_size(spa));
+	fnvlist_add_boolean_value(nvl, ZPOOL_CONFIG_SUSPENDED,
+	    spa_suspended(spa));
+	if (spa_suspended(spa)) {
+		const char *failmode;
+		switch (spa->spa_failmode) {
+		case ZIO_FAILURE_MODE_WAIT:
+			failmode = "wait";
+			break;
+		case ZIO_FAILURE_MODE_CONTINUE:
+			failmode = "continue";
+			break;
+		case ZIO_FAILURE_MODE_PANIC:
+			failmode = "panic";
+			break;
+		default:
+			failmode = "???";
+		}
+		fnvlist_add_string(nvl, "failmode", failmode);
+		if (spa->spa_suspended != ZIO_SUSPEND_NONE) {
+			fnvlist_add_string(nvl, ZPOOL_CONFIG_SUSPENDED_REASON,
+			    (spa->spa_suspended == ZIO_SUSPEND_MMP) ?
+			    "MMP" : "IO");
+		}
+	}
+
+	jp_open(&jp, buf, size);
+	jp_printf(&jp, "{");
+
+	jp_printf(&jp, "status_json_version: %d", JSON_STATUS_VERSION);
+	jp_printf(&jp, "scl_config_lock: %b", scl_config_lock != 0);
+	jp_printf(&jp, "scan_error: %d", ps_error);
+	jp_printf(&jp, "scan_stats: {");
+	if (ps_error == 0) {
+		jp_printf(&jp, "func: %s", pss_func_to_string(ps.pss_func));
+		jp_printf(&jp, "state: %s", pss_state_to_string(ps.pss_state));
+		jp_printf(&jp, "start_time: %U", ps.pss_start_time);
+		jp_printf(&jp, "end_time: %U", ps.pss_end_time);
+		jp_printf(&jp, "to_examine: %U", ps.pss_to_examine);
+		jp_printf(&jp, "examined: %U", ps.pss_examined);
+		jp_printf(&jp, "processed: %U", ps.pss_processed);
+		jp_printf(&jp, "errors: %U", ps.pss_errors);
+
+		jp_printf(&jp, "pass_exam: %U", ps.pss_pass_exam);
+		jp_printf(&jp, "pass_start: %U", ps.pss_pass_start);
+		jp_printf(&jp, "pass_scrub_pause: %U", ps.pss_pass_scrub_pause);
+		jp_printf(&jp, "pass_scrub_spent_paused: %U",
+		    ps.pss_pass_scrub_spent_paused);
+		jp_printf(&jp, "pass_issued: %U", ps.pss_pass_issued);
+		jp_printf(&jp, "issued: %U", ps.pss_issued);
+	} else if (ps_error == ENOENT) {
+		jp_printf(&jp, "func: %s", "NONE");
+		jp_printf(&jp, "state: %s", "NONE");
+	} else {
+		jp_printf(&jp, "func: %s", "?");
+		jp_printf(&jp, "state: %s", "?");
+	}
+	jp_printf(&jp, "}");
+
+	jp_printf(&jp, "state: %s", spa_state_to_name(spa));
+
+	spa_add_spares(spa, nvl);
+	spa_add_l2cache(spa, nvl);
+	spa_add_feature_stats(spa, nvl);
+
+	/* iterate and transfer nvl to json */
+	nvlist_to_json(nvl, &jp, stats_filter);
+
+	iterate_vdevs(spa, &ps, &jp);
+
+	/*
+	 * close the root object
+	 */
+	jp_printf(&jp, "}");
+
+	if (scl_config_lock)
+		spa_config_exit(spa, SCL_CONFIG, FTAG);
+	nvlist_free(nvl);
+
+	error = jp_close(&jp);
+	if (error == JPRINT_BUF_FULL) {
+		error = SET_ERROR(ENOMEM);
+	} else if (error != 0) {
+		/*
+		 * Another error from jprint, format an error message
+		 * but this is not ever to happen (this would be a
+		 * defect elsewhere).
+		 *
+		 * If this does happen, we simply put the string where
+		 * the json should go... this is expected to trigger
+		 * a json decode error, and report "upstream"
+		 */
+		snprintf(buf, size,
+		    "jprint error %s (%d) callno %d, size %ld\n",
+		    jp_errorstring(error), error, jp_errorpos(&jp), size);
+		error = 0;
+	}
+	return (error);
+}

module/zfs/spa_stats.c

@@ -24,6 +24,7 @@
 #include <sys/vdev_impl.h>
 #include <sys/spa.h>
 #include <zfs_comutil.h>
+#include <sys/spa_json_stats.h>
 
 /*
  * Keeps stats on last N reads per spa_t, disabled by default.
@@ -988,6 +989,52 @@ spa_iostats_destroy(spa_t *spa)
 	mutex_destroy(&shk->lock);
 }
 
+static void *
+spa_json_addr(kstat_t *ksp, loff_t n)
+{
+	if (n == 0)
+		return (ksp->ks_private);
+	return (NULL);
+}
+
+static int
+spa_json_data(char *buf, size_t size, void *data)
+{
+	spa_t *spa = (spa_t *)data;
+
+	return (spa_generate_json_stats(spa, buf, size));
+}
+
+static void
+spa_json_stats_init(spa_t *spa)
+{
+	char *name;
+	kstat_t *ksp;
+
+	mutex_init(&spa->spa_json_stats.lock, NULL, MUTEX_DEFAULT, NULL);
+	name = kmem_asprintf("zfs/%s", spa_name(spa));
+	ksp = kstat_create(name, 0, "status.json", "misc", KSTAT_TYPE_RAW, 0,
+	    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_RESTRICTED | KSTAT_FLAG_NO_HEADERS);
+	spa->spa_json_stats.kstat = ksp;
+	if (ksp) {
+		ksp->ks_lock = &spa->spa_json_stats.lock;
+		ksp->ks_data = NULL;
+		ksp->ks_private = spa;
+		kstat_set_raw_ops(ksp, NULL, spa_json_data, spa_json_addr);
+		kstat_install(ksp);
+	}
+
+	kmem_strfree(name);
+}
+
+static void
+spa_json_stats_destroy(spa_t *spa)
+{
+	if (spa->spa_json_stats.kstat)
+		kstat_delete(spa->spa_json_stats.kstat);
+	mutex_destroy(&spa->spa_json_stats.lock);
+}
+
 void
 spa_stats_init(spa_t *spa)
 {
@@ -998,11 +1045,13 @@ spa_stats_init(spa_t *spa)
 	spa_state_init(spa);
 	spa_guid_init(spa);
 	spa_iostats_init(spa);
+	spa_json_stats_init(spa);
 }
 
 void
 spa_stats_destroy(spa_t *spa)
 {
+	spa_json_stats_destroy(spa);
 	spa_iostats_destroy(spa);
 	spa_health_destroy(spa);
 	spa_tx_assign_destroy(spa);

tests/runfiles/common.run

@@ -559,7 +559,8 @@ tests = ['zpool_status_001_pos', 'zpool_status_002_pos',
     'zpool_status_003_pos', 'zpool_status_004_pos',
     'zpool_status_005_pos', 'zpool_status_006_pos',
     'zpool_status_007_pos', 'zpool_status_008_pos',
-    'zpool_status_features_001_pos']
+    'zpool_status_features_001_pos',
+    'zpool_status_kstat_pos']
 tags = ['functional', 'cli_root', 'zpool_status']
 
 [tests/functional/cli_root/zpool_sync]

tests/zfs-tests/tests/Makefile.am

@@ -1256,6 +1256,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zpool_status/zpool_status_007_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_008_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_features_001_pos.ksh \
+	functional/cli_root/zpool_status/zpool_status_kstat_pos.ksh \
 	functional/cli_root/zpool_sync/cleanup.ksh \
 	functional/cli_root/zpool_sync/setup.ksh \
 	functional/cli_root/zpool_sync/zpool_sync_001_pos.ksh \

tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_kstat_pos.ksh

@@ -0,0 +1,66 @@
+#!/bin/ksh -p
+
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2024 Klara
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# Verify the status.json kstat
+#
+# STRATEGY:
+# 1. Create zpool
+# 2. Confirm the output of the kstat is valid json
+# 3. Confirm that some expected keys are present
+#
+
+function cleanup
+{
+	poolexists $TESTPOOL2 && destroy_pool $TESTPOOL2
+	log_must rm -f $all_vdevs
+	[[ -f $tmpfile ]] && rm -f $tmpfile
+}
+
+log_assert "Verify status.json kstat"
+
+log_onexit cleanup
+
+all_vdevs=$(echo $TESTDIR/vdev{1..6})
+log_must mkdir -p $TESTDIR
+log_must truncate -s $MINVDEVSIZE $all_vdevs
+tmpfile=$TEST_BASE_DIR/tmpfile.$$
+
+for raid_type in "draid2:3d:6c:1s" "raidz2"; do
+
+	log_must zpool create -f $TESTPOOL2 $raid_type $all_vdevs
+
+	# Verify that the JSON output is valid
+	log_must eval "kstat ${TESTPOOL2}/status.json | python3 -m json.tool > $tmpfile"
+
+	# Verify that some of the expected keys are present
+	log_must eval "grep '\"vdev_children\": 6' $tmpfile"
+	log_must eval "grep '\"nparity\": 2' $tmpfile"
+	log_must eval "grep '\"state\": \"ONLINE\"' $tmpfile"
+	log_must eval "grep '\"name\": \"$TESTPOOL2\"' $tmpfile"
+
+	zpool destroy $TESTPOOL2
+done
+
+log_pass "Verify status.json kstat"