From 416ea41982fa8ec73faf2ba9d3756fca469136d6 Mon Sep 17 00:00:00 2001 From: George Wilson Date: Mon, 6 Nov 2023 18:26:55 -0700 Subject: [PATCH] Introduce two types of root ZIOs The zio_root() interface recently removed the READY stage in its pipeline. However, there might be workloads that rely on zio_root() calling the READY pipeline phase even though a callback is not invoked. This change re-introduces the original behavior of zio_root() so that any features that may have relied on the old behavior are not impacted. Additionally, it adds a new interface, zio_root_done(), to provide the performance benefits that consumers like the ZIL might benefit from. Signed-off-by: George Wilson --- include/sys/zio.h | 2 ++ include/sys/zio_impl.h | 2 +- module/zfs/spa.c | 4 ++-- module/zfs/zil.c | 2 +- module/zfs/zio.c | 24 +++++++++++++++++++----- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/include/sys/zio.h b/include/sys/zio.h index e1f4d5c044..f8b6a4b250 100644 --- a/include/sys/zio.h +++ b/include/sys/zio.h @@ -547,6 +547,8 @@ extern zio_t *zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, extern zio_t *zio_root(spa_t *spa, zio_done_func_t *done, void *priv, zio_flag_t flags); +extern zio_t *zio_root_done(spa_t *spa, + zio_done_func_t *done, void *priv, zio_flag_t flags); extern void zio_destroy(zio_t *zio); diff --git a/include/sys/zio_impl.h b/include/sys/zio_impl.h index febe0a87b4..a449dac074 100644 --- a/include/sys/zio_impl.h +++ b/include/sys/zio_impl.h @@ -159,7 +159,7 @@ enum zio_stage { ZIO_STAGE_DONE = 1 << 25 /* RWFCI */ }; -#define ZIO_ROOT_PIPELINE \ +#define ZIO_DONE_PIPELINE \ ZIO_STAGE_DONE #define ZIO_INTERLOCK_STAGES \ diff --git a/module/zfs/spa.c b/module/zfs/spa.c index aa97144f16..1df669992f 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -3548,7 +3548,7 @@ spa_ld_parse_config(spa_t *spa, spa_import_type_t type) spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), KM_SLEEP); for (int i = 0; i < max_ncpus; i++) { - spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, + spa->spa_async_zio_root[i] = zio_root_done(spa, NULL, NULL, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_GODFATHER); } @@ -6016,7 +6016,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), KM_SLEEP); for (int i = 0; i < max_ncpus; i++) { - spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, + spa->spa_async_zio_root[i] = zio_root_done(spa, NULL, NULL, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_GODFATHER); } diff --git a/module/zfs/zil.c b/module/zfs/zil.c index ce2cb8b144..d82014c438 100644 --- a/module/zfs/zil.c +++ b/module/zfs/zil.c @@ -1787,7 +1787,7 @@ zil_lwb_write_issue(zilog_t *zilog, lwb_t *lwb) zil_lwb_commit(zilog, lwb, itx); lwb->lwb_nused = lwb->lwb_nfilled; - lwb->lwb_root_zio = zio_root(spa, zil_lwb_flush_vdevs_done, lwb, + lwb->lwb_root_zio = zio_root_done(spa, zil_lwb_flush_vdevs_done, lwb, ZIO_FLAG_CANFAIL); /* diff --git a/module/zfs/zio.c b/module/zfs/zio.c index 4eb276352a..c852a84260 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -950,19 +950,33 @@ zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, } /* - * ZIO intended to be a root of a tree. Unlike null ZIO does not have a - * READY pipeline stage (is ready on creation), so it should not be used - * as child of any ZIO that may need waiting for grandchildren READY stage - * (any other ZIO type). + * Root ZIO that runs through all the interlock stages. Although it does + * not provide a callback, it can be used to drive functionality and + * synchronization in the READY and DONE stages of the pipeline. */ zio_t * zio_root(spa_t *spa, zio_done_func_t *done, void *private, zio_flag_t flags) +{ + return (zio_null(NULL, spa, NULL, done, private, flags)); +} + +/* + * Optimized Root ZIO that can be used when READY stage interlocking + * is not necessary or required. Unlike a standard root ZIO, this ZIO + * is READY on creation and immediates moves to the DONE stage when + * the pipeline is invoked. Unlike a null ZIO or a root ZIO, this ZIO + * should not be used as child of any ZIO that may need waiting for + * grandchildren READY stage (any other ZIO type). + */ +zio_t * +zio_root_done(spa_t *spa, zio_done_func_t *done, void *private, + zio_flag_t flags) { zio_t *zio; zio = zio_create(NULL, spa, 0, NULL, NULL, 0, 0, done, private, ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, NULL, 0, NULL, - ZIO_STAGE_OPEN, ZIO_ROOT_PIPELINE); + ZIO_STAGE_OPEN, ZIO_DONE_PIPELINE); return (zio); }