From 1f147a2f54e9271767ea8fc3ed0391a677adda07 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Thu, 12 Aug 2010 15:16:55 -0700 Subject: [PATCH 1/3] Add fix-stack-zio_execute topic branch --- .topdeps | 1 + .topmsg | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 .topdeps create mode 100644 .topmsg diff --git a/.topdeps b/.topdeps new file mode 100644 index 0000000000..1f7391f92b --- /dev/null +++ b/.topdeps @@ -0,0 +1 @@ +master diff --git a/.topmsg b/.topmsg new file mode 100644 index 0000000000..492fda1eda --- /dev/null +++ b/.topmsg @@ -0,0 +1,10 @@ +From: Ned Bass +Subject: [PATCH] fix stack zio_execute + +Implement zio_execute() as a wrapper around the static function +__zio_execute() so that we can force __zio_execute() to be inlined. +This reduces stack overhead which is important because __zio_execute() +is called recursively in several zio code paths. zio_execute() itself +cannot be inlined because it is externally visible. + +Signed-off-by: Brian Behlendorf From 2e3c7a06c2c3c7066619eb15a95d1149621c3f48 Mon Sep 17 00:00:00 2001 From: Ned Bass Date: Thu, 12 Aug 2010 14:39:57 -0700 Subject: [PATCH 2/3] Wrap zio_execute() to enable inlining Implement zio_execute() as a wrapper around the static function __zio_execute() so that we can force __zio_execute() to be inlined. This reduces stack overhead which is important because __zio_execute() is called recursively in several zio code paths. zio_execute() itself cannot be inlined because it is externally visible. Signed-off-by: Brian Behlendorf --- module/zfs/zio.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/module/zfs/zio.c b/module/zfs/zio.c index 88d80af4e9..b03f1d073a 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -93,6 +93,8 @@ int zio_buf_debug_limit = 16384; int zio_buf_debug_limit = 0; #endif +static inline void __zio_execute(zio_t *zio); + void zio_init(void) { @@ -449,7 +451,7 @@ zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) if (--*countp == 0 && pio->io_stall == countp) { pio->io_stall = NULL; mutex_exit(&pio->io_lock); - zio_execute(pio); + __zio_execute(pio); } else { mutex_exit(&pio->io_lock); } @@ -1118,8 +1120,23 @@ zio_interrupt(zio_t *zio) */ static zio_pipe_stage_t *zio_pipeline[]; +/* + * zio_execute() is a wrapper around the static function + * __zio_execute() so that we can force __zio_execute() to be + * inlined. This reduces stack overhead which is important + * because __zio_execute() is called recursively in several zio + * code paths. zio_execute() itself cannot be inlined because + * it is externally visible. + */ void zio_execute(zio_t *zio) +{ + __zio_execute(zio); +} + +__attribute__((always_inline)) +static inline void +__zio_execute(zio_t *zio) { zio->io_executor = curthread; @@ -1165,6 +1182,7 @@ zio_execute(zio_t *zio) } } + /* * ========================================================================== * Initiate I/O, either sync or async @@ -1180,7 +1198,7 @@ zio_wait(zio_t *zio) zio->io_waiter = curthread; - zio_execute(zio); + __zio_execute(zio); mutex_enter(&zio->io_lock); while (zio->io_executor != NULL) @@ -1210,7 +1228,7 @@ zio_nowait(zio_t *zio) zio_add_child(spa->spa_async_zio_root, zio); } - zio_execute(zio); + __zio_execute(zio); } /* @@ -1264,7 +1282,7 @@ zio_reexecute(zio_t *pio) * responsibility of the caller to wait on him. */ if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) - zio_execute(pio); + __zio_execute(pio); } void From e47d393490ddf8f4e8cf126460a381157c9554aa Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 13 Aug 2010 10:56:48 -0700 Subject: [PATCH 3/3] New TopGit dependency: fix-stack-zio_execute --- .topdeps | 1 + 1 file changed, 1 insertion(+) diff --git a/.topdeps b/.topdeps index dfcc9e73d2..a41cef572a 100644 --- a/.topdeps +++ b/.topdeps @@ -31,3 +31,4 @@ fix-stack-dbuf_hold_impl fix-bpobj_close fix-zfs_ioc_objset_stats fix-commit-callback +fix-stack-zio_execute