From a76f3d0437e5e974f0f748f8735af3539443b388 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 16 Mar 2018 16:46:06 -0700 Subject: [PATCH] Fix deadlock in IO pipeline In vdev_queue_aggregate() the zio_execute() bypass should not be called under the vdev queue lock. This can result in a deadlock as shown in the stack traces below. Drop the vdev queue lock then walk the parents of the aggregate IO to determine the list of component IOs to be bypassed. This can be done safely without holding the io_lock since the new aggregate IO has not yet been returned and its parents cannot change. --- THREAD 1 --- arc_read() zio_nowait() zio_vdev_io_start() vdev_queue_io() <--- mutex_enter(vq->vq_lock) vdev_queue_io_to_issue() vdev_queue_aggregate() zio_execute() zio_vdev_io_assess() zio_wait_for_children() <- mutex_enter(zio->io_lock) --- THREAD 2 --- (inverse order) arc_read() zio_change_priority() <- mutex_enter(zio->zio_lock) vdev_queue_change_io_priority() <- mutex_enter(vq->vq_lock) Reviewed-by: Tom Caputi Reviewed-by: Don Brady Signed-off-by: Brian Behlendorf Closes #7307 --- module/zfs/vdev_queue.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/module/zfs/vdev_queue.c b/module/zfs/vdev_queue.c index f121eddc7a..5d2c980133 100644 --- a/module/zfs/vdev_queue.c +++ b/module/zfs/vdev_queue.c @@ -517,6 +517,7 @@ static zio_t * vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio) { zio_t *first, *last, *aio, *dio, *mandatory, *nio; + zio_link_t *zl = NULL; uint64_t maxgap = 0; uint64_t size; uint64_t limit; @@ -665,9 +666,18 @@ vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio) zio_add_child(dio, aio); vdev_queue_io_remove(vq, dio); + } while (dio != last); + + /* + * We need to drop the vdev queue's lock to avoid a deadlock that we + * could encounter since this I/O will complete immediately. + */ + mutex_exit(&vq->vq_lock); + while ((dio = zio_walk_parents(aio, &zl)) != NULL) { zio_vdev_io_bypass(dio); zio_execute(dio); - } while (dio != last); + } + mutex_enter(&vq->vq_lock); return (aio); }