zinject: inject device errors into ioctls

Adds 'ioctl' as a valid IO type for device error injection, so we can
simulate a flush error (which OpenZFS currently ignores, but that's by
the by).

To support this, adding ZIO_STAGE_VDEV_IO_DONE to ZIO_IOCTL_PIPELINE,
since that's where device error injection happens. This needs a small
exclusion to avoid the vdev_queue, since flushes are not queued, and I'm
assuming that the various failure responses are still reasonable for
flush failures (probes, media change, etc). This seems reasonable to me,
as a flush failure is not unlike a write failure in this regard, however
this may be too aggressive or subtle to assume in just this change.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>
Closes #16061

cmd/zinject/zinject.c

@@ -265,7 +265,7 @@ usage(void)
 	    "\t\tspa_vdev_exit() will trigger a panic.\n"
 	    "\n"
 	    "\tzinject -d device [-e errno] [-L <nvlist|uber|pad1|pad2>] [-F]\n"
-	    "\t\t[-T <read|write|free|claim|all>] [-f frequency] pool\n\n"
+	    "\t\t[-T <read|write|free|claim|ioctl|all>] [-f frequency] pool\n\n"
 	    "\t\tInject a fault into a particular device or the device's\n"
 	    "\t\tlabel.  Label injection can either be 'nvlist', 'uber',\n "
 	    "\t\t'pad1', or 'pad2'.\n"
@@ -978,12 +978,14 @@ main(int argc, char **argv)
 				io_type = ZIO_TYPE_FREE;
 			} else if (strcasecmp(optarg, "claim") == 0) {
 				io_type = ZIO_TYPE_CLAIM;
+			} else if (strcasecmp(optarg, "ioctl") == 0) {
+				io_type = ZIO_TYPE_IOCTL;
 			} else if (strcasecmp(optarg, "all") == 0) {
 				io_type = ZIO_TYPES;
 			} else {
 				(void) fprintf(stderr, "invalid I/O type "
 				    "'%s': must be 'read', 'write', 'free', "
-				    "'claim' or 'all'\n", optarg);
+				    "'claim', 'ioctl' or 'all'\n", optarg);
 				usage();
 				libzfs_fini(g_zfs);
 				return (1);

include/sys/zio_impl.h

@@ -153,7 +153,7 @@ enum zio_stage {
 	ZIO_STAGE_READY			= 1 << 20,	/* RWFCIT */
 
 	ZIO_STAGE_VDEV_IO_START		= 1 << 21,	/* RW--IT */
-	ZIO_STAGE_VDEV_IO_DONE		= 1 << 22,	/* RW---T */
+	ZIO_STAGE_VDEV_IO_DONE		= 1 << 22,	/* RW--IT */
 	ZIO_STAGE_VDEV_IO_ASSESS	= 1 << 23,	/* RW--IT */
 
 	ZIO_STAGE_CHECKSUM_VERIFY	= 1 << 24,	/* R----- */
@@ -261,8 +261,7 @@ enum zio_stage {
 
 #define	ZIO_IOCTL_PIPELINE			\
 	(ZIO_INTERLOCK_STAGES |			\
-	ZIO_STAGE_VDEV_IO_START |		\
+	ZIO_VDEV_IO_STAGES)
-	ZIO_STAGE_VDEV_IO_ASSESS)
 
 #define	ZIO_TRIM_PIPELINE			\
 	(ZIO_INTERLOCK_STAGES |			\

man/man8/zinject.8

@@ -19,10 +19,11 @@
 .\" CDDL HEADER END
 .\"
 .\" Copyright 2013 Darik Horn <dajhorn@vanadac.com>. All rights reserved.
+.\" Copyright (c) 2024, Klara Inc.
 .\"
 .\" lint-ok: WARNING: sections out of conventional order: Sh SYNOPSIS
 .\"
-.Dd May 26, 2021
+.Dd April 4, 2024
 .Dt ZINJECT 8
 .Os
 .
@@ -257,6 +258,7 @@ Run for this many seconds before reporting failure.
 .It Fl T Ar failure
 Set the failure type to one of
 .Sy all ,
+.Sy ioctl ,
 .Sy claim ,
 .Sy free ,
 .Sy read ,

man/man8/zpool-events.8

@@ -404,7 +404,7 @@ ZIO_STAGE_DVA_CLAIM:0x00080000:---C--
 ZIO_STAGE_READY:0x00100000:RWFCIT
 
 ZIO_STAGE_VDEV_IO_START:0x00200000:RW--IT
-ZIO_STAGE_VDEV_IO_DONE:0x00400000:RW---T
+ZIO_STAGE_VDEV_IO_DONE:0x00400000:RW--IT
 ZIO_STAGE_VDEV_IO_ASSESS:0x00800000:RW--IT
 
 ZIO_STAGE_CHECKSUM_VERIFY:0x01000000:R-----

module/zfs/zio.c

@@ -4086,13 +4086,16 @@ zio_vdev_io_done(zio_t *zio)
 	}
 
 	ASSERT(zio->io_type == ZIO_TYPE_READ ||
-	    zio->io_type == ZIO_TYPE_WRITE || zio->io_type == ZIO_TYPE_TRIM);
+	    zio->io_type == ZIO_TYPE_WRITE ||
+	    zio->io_type == ZIO_TYPE_IOCTL ||
+	    zio->io_type == ZIO_TYPE_TRIM);
 
 	if (zio->io_delay)
 		zio->io_delay = gethrtime() - zio->io_delay;
 
 	if (vd != NULL && vd->vdev_ops->vdev_op_leaf &&
 	    vd->vdev_ops != &vdev_draid_spare_ops) {
+		if (zio->io_type != ZIO_TYPE_IOCTL)
 			vdev_queue_io_done(zio);
 
 		if (zio_injection_enabled && zio->io_error == 0)

module/zfs/zio_inject.c

@@ -364,10 +364,10 @@ zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2)
 	int ret = 0;
 
 	/*
-	 * We skip over faults in the labels unless it's during
+	 * We skip over faults in the labels unless it's during device open
-	 * device open (i.e. zio == NULL).
+	 * (i.e. zio == NULL) or a device flush (offset is meaningless)
 	 */
-	if (zio != NULL) {
+	if (zio != NULL && zio->io_type != ZIO_TYPE_IOCTL) {
 		uint64_t offset = zio->io_offset;
 
 		if (offset < VDEV_LABEL_START_SIZE ||