From 1c44a5c97fabc669885df84c3e9b6b5e16f0cd35 Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Thu, 18 Jul 2019 12:55:29 -0700 Subject: [PATCH] hdr_recl calls zthr_wakeup() on destroyed zthr There exists a race condition were hdr_recl() calls zthr_wakeup() on a destroyed zthr. The timeline is the following: [1] hdr_recl() runs first and goes intro zthr_wakeup() because arc_initialized is set. [2] arc_fini() is called by another thread, zeroes that flag, destroying the zthr, and goes into buf_init(). [3] hdr_recl() tries to enter the destroyed mutex and we blow up. This patch ensures that the ARC's zthrs are not offloaded any new work once arc_initialized is set and then destroys them after all of the ARC state has been deleted. Reviewed by: Matt Ahrens Reviewed by: Brian Behlendorf Signed-off-by: Serapheim Dimitropoulos Closes #9047 --- module/zfs/arc.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 98bafeee28..90a731bffa 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2018, Joyent, Inc. - * Copyright (c) 2011, 2018 by Delphix. All rights reserved. + * Copyright (c) 2011, 2019 by Delphix. All rights reserved. * Copyright (c) 2014 by Saso Kiselkov. All rights reserved. * Copyright 2017 Nexenta Systems, Inc. All rights reserved. */ @@ -5086,6 +5086,9 @@ arc_kmem_reap_soon(void) static boolean_t arc_adjust_cb_check(void *arg, zthr_t *zthr) { + if (!arc_initialized) + return (B_FALSE); + /* * This is necessary so that any changes which may have been made to * many of the zfs_arc_* module parameters will be propagated to @@ -5173,6 +5176,9 @@ arc_adjust_cb(void *arg, zthr_t *zthr) static boolean_t arc_reap_cb_check(void *arg, zthr_t *zthr) { + if (!arc_initialized) + return (B_FALSE); + int64_t free_memory = arc_available_memory(); /* @@ -7933,11 +7939,9 @@ arc_fini(void) list_destroy(&arc_prune_list); mutex_destroy(&arc_prune_mtx); - (void) zthr_cancel(arc_adjust_zthr); - zthr_destroy(arc_adjust_zthr); + (void) zthr_cancel(arc_adjust_zthr); (void) zthr_cancel(arc_reap_zthr); - zthr_destroy(arc_reap_zthr); mutex_destroy(&arc_adjust_lock); cv_destroy(&arc_adjust_waiters_cv); @@ -7950,6 +7954,14 @@ arc_fini(void) buf_fini(); arc_state_fini(); + /* + * We destroy the zthrs after all the ARC state has been + * torn down to avoid the case of them receiving any + * wakeup() signals after they are destroyed. + */ + zthr_destroy(arc_adjust_zthr); + zthr_destroy(arc_reap_zthr); + ASSERT0(arc_loaned_bytes); }