Introduce two types of root ZIOs

The zio_root() interface recently removed the READY stage in its
pipeline. However, there might be workloads that rely on zio_root()
calling the READY pipeline phase even though a callback is not invoked.

This change re-introduces the original behavior of zio_root() so that
any features that may have relied on the old behavior are not impacted.
Additionally, it adds a new interface, zio_root_done(), to provide
the performance benefits that consumers like the ZIL might benefit from.

Signed-off-by: George Wilson <gwilson@delphix.com>

include/sys/zio.h

@@ -547,6 +547,8 @@ extern zio_t *zio_null(zio_t *pio, spa_t *spa, vdev_t *vd,
 
 extern zio_t *zio_root(spa_t *spa,
     zio_done_func_t *done, void *priv, zio_flag_t flags);
+extern zio_t *zio_root_done(spa_t *spa,
+    zio_done_func_t *done, void *priv, zio_flag_t flags);
 
 extern void zio_destroy(zio_t *zio);
 

include/sys/zio_impl.h

@@ -159,7 +159,7 @@ enum zio_stage {
 	ZIO_STAGE_DONE			= 1 << 25	/* RWFCI */
 };
 
-#define	ZIO_ROOT_PIPELINE			\
+#define	ZIO_DONE_PIPELINE			\
 	ZIO_STAGE_DONE
 
 #define	ZIO_INTERLOCK_STAGES			\

module/zfs/spa.c

@@ -3548,7 +3548,7 @@ spa_ld_parse_config(spa_t *spa, spa_import_type_t type)
 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
 	    KM_SLEEP);
 	for (int i = 0; i < max_ncpus; i++) {
-		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
+		spa->spa_async_zio_root[i] = zio_root_done(spa, NULL, NULL,
 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
 		    ZIO_FLAG_GODFATHER);
 	}
@@ -6016,7 +6016,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
 	spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
 	    KM_SLEEP);
 	for (int i = 0; i < max_ncpus; i++) {
-		spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
+		spa->spa_async_zio_root[i] = zio_root_done(spa, NULL, NULL,
 		    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
 		    ZIO_FLAG_GODFATHER);
 	}

module/zfs/zil.c

@@ -1787,7 +1787,7 @@ zil_lwb_write_issue(zilog_t *zilog, lwb_t *lwb)
 		zil_lwb_commit(zilog, lwb, itx);
 	lwb->lwb_nused = lwb->lwb_nfilled;
 
-	lwb->lwb_root_zio = zio_root(spa, zil_lwb_flush_vdevs_done, lwb,
+	lwb->lwb_root_zio = zio_root_done(spa, zil_lwb_flush_vdevs_done, lwb,
 	    ZIO_FLAG_CANFAIL);
 
 	/*

module/zfs/zio.c

@@ -950,19 +950,33 @@ zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done,
 }
 
 /*
- * ZIO intended to be a root of a tree.  Unlike null ZIO does not have a
+ * Root ZIO that runs through all the interlock stages. Although it does
- * READY pipeline stage (is ready on creation), so it should not be used
+ * not provide a callback, it can be used to drive functionality and
- * as child of any ZIO that may need waiting for grandchildren READY stage
+ * synchronization in the READY and DONE stages of the pipeline.
- * (any other ZIO type).
  */
 zio_t *
 zio_root(spa_t *spa, zio_done_func_t *done, void *private, zio_flag_t flags)
+{
+	return (zio_null(NULL, spa, NULL, done, private, flags));
+}
+
+/*
+ * Optimized Root ZIO that can be used when READY stage interlocking
+ * is not necessary or required. Unlike a standard root ZIO, this ZIO
+ * is READY on creation and immediates moves to the DONE stage when
+ * the pipeline is invoked. Unlike a null ZIO or a root ZIO, this ZIO
+ * should not be used as child of any ZIO that may need waiting for
+ * grandchildren READY stage (any other ZIO type).
+ */
+zio_t *
+zio_root_done(spa_t *spa, zio_done_func_t *done, void *private,
+    zio_flag_t flags)
 {
 	zio_t *zio;
 
 	zio = zio_create(NULL, spa, 0, NULL, NULL, 0, 0, done, private,
 	    ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, NULL, 0, NULL,
-	    ZIO_STAGE_OPEN, ZIO_ROOT_PIPELINE);
+	    ZIO_STAGE_OPEN, ZIO_DONE_PIPELINE);
 
 	return (zio);
 }