hdr_recl calls zthr_wakeup() on destroyed zthr

There exists a race condition were hdr_recl() calls
zthr_wakeup() on a destroyed zthr. The timeline is the
following:

[1] hdr_recl() runs first and goes intro zthr_wakeup()
    because arc_initialized is set.
[2] arc_fini() is called by another thread, zeroes
    that flag, destroying the zthr, and goes into
    buf_init().
[3] hdr_recl() tries to enter the destroyed mutex
    and we blow up.

This patch ensures that the ARC's zthrs are not offloaded
any new work once arc_initialized is set and then destroys
them after all of the ARC state has been deleted.

Reviewed by: Matt Ahrens <matt@delphix.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Serapheim Dimitropoulos <serapheim@delphix.com>
Closes #9047
This commit is contained in:
Serapheim Dimitropoulos 2019-07-18 12:55:29 -07:00 committed by Tony Hutter
parent 3c144b9267
commit bbbe4b0a98
1 changed files with 16 additions and 4 deletions

View File

@ -21,7 +21,7 @@
/* /*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, Joyent, Inc. * Copyright (c) 2018, Joyent, Inc.
* Copyright (c) 2011, 2018 by Delphix. All rights reserved. * Copyright (c) 2011, 2019 by Delphix. All rights reserved.
* Copyright (c) 2014 by Saso Kiselkov. All rights reserved. * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
* Copyright 2017 Nexenta Systems, Inc. All rights reserved. * Copyright 2017 Nexenta Systems, Inc. All rights reserved.
*/ */
@ -5079,6 +5079,9 @@ arc_kmem_reap_soon(void)
static boolean_t static boolean_t
arc_adjust_cb_check(void *arg, zthr_t *zthr) arc_adjust_cb_check(void *arg, zthr_t *zthr)
{ {
if (!arc_initialized)
return (B_FALSE);
/* /*
* This is necessary so that any changes which may have been made to * This is necessary so that any changes which may have been made to
* many of the zfs_arc_* module parameters will be propagated to * many of the zfs_arc_* module parameters will be propagated to
@ -5166,6 +5169,9 @@ arc_adjust_cb(void *arg, zthr_t *zthr)
static boolean_t static boolean_t
arc_reap_cb_check(void *arg, zthr_t *zthr) arc_reap_cb_check(void *arg, zthr_t *zthr)
{ {
if (!arc_initialized)
return (B_FALSE);
int64_t free_memory = arc_available_memory(); int64_t free_memory = arc_available_memory();
/* /*
@ -7924,11 +7930,9 @@ arc_fini(void)
list_destroy(&arc_prune_list); list_destroy(&arc_prune_list);
mutex_destroy(&arc_prune_mtx); mutex_destroy(&arc_prune_mtx);
(void) zthr_cancel(arc_adjust_zthr);
zthr_destroy(arc_adjust_zthr);
(void) zthr_cancel(arc_adjust_zthr);
(void) zthr_cancel(arc_reap_zthr); (void) zthr_cancel(arc_reap_zthr);
zthr_destroy(arc_reap_zthr);
mutex_destroy(&arc_adjust_lock); mutex_destroy(&arc_adjust_lock);
cv_destroy(&arc_adjust_waiters_cv); cv_destroy(&arc_adjust_waiters_cv);
@ -7941,6 +7945,14 @@ arc_fini(void)
buf_fini(); buf_fini();
arc_state_fini(); arc_state_fini();
/*
* We destroy the zthrs after all the ARC state has been
* torn down to avoid the case of them receiving any
* wakeup() signals after they are destroyed.
*/
zthr_destroy(arc_adjust_zthr);
zthr_destroy(arc_reap_zthr);
ASSERT0(arc_loaned_bytes); ASSERT0(arc_loaned_bytes);
} }