From c6d1112bf4125e5a22eb47ceb7b8cee01f0df9a1 Mon Sep 17 00:00:00 2001 From: Jorgen Lundman Date: Fri, 2 Jul 2021 00:28:15 +0900 Subject: [PATCH] Fix abd leak, kmem_free correct size of abd_t Fix a leak of abd_t that manifested mostly when using raidzN with at least as many columns as N (e.g. a four-disk raidz2 but not a three-disk raidz2). Sufficiently heavy raidz use would eventually run a system out of memory. Additionally: * Switch abd_cache arena to FIRSTFIT, which empirically improves perofrmance. * Make abd_chunk_cache more performant and debuggable. * Allocate the abd_zero_buf from abd_chunk_cache rather than the heap. * Don't try to reap non-existent qcaches in abd_cache arena. * KM_PUSHPAGE->KM_SLEEP when allocating chunks from their own arena Reviewed-by: Matthew Ahrens Reviewed-by: Alexander Motin Signed-off-by: Jorgen Lundman Co-authored-by: Sean Doran Closes #12295 --- include/sys/abd_impl.h | 2 +- module/os/freebsd/zfs/abd_os.c | 9 ++++++--- module/os/linux/zfs/abd_os.c | 3 ++- module/zfs/abd.c | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/include/sys/abd_impl.h b/include/sys/abd_impl.h index 6bce08cfa3..113700cd72 100644 --- a/include/sys/abd_impl.h +++ b/include/sys/abd_impl.h @@ -64,7 +64,7 @@ void abd_free_struct(abd_t *); */ abd_t *abd_alloc_struct_impl(size_t); -abd_t *abd_get_offset_scatter(abd_t *, abd_t *, size_t); +abd_t *abd_get_offset_scatter(abd_t *, abd_t *, size_t, size_t); void abd_free_struct_impl(abd_t *); void abd_alloc_chunks(abd_t *, size_t); void abd_free_chunks(abd_t *); diff --git a/module/os/freebsd/zfs/abd_os.c b/module/os/freebsd/zfs/abd_os.c index 47adc2278d..4f5b33d945 100644 --- a/module/os/freebsd/zfs/abd_os.c +++ b/module/os/freebsd/zfs/abd_os.c @@ -374,14 +374,17 @@ abd_alloc_for_io(size_t size, boolean_t is_metadata) } abd_t * -abd_get_offset_scatter(abd_t *abd, abd_t *sabd, size_t off) +abd_get_offset_scatter(abd_t *abd, abd_t *sabd, size_t off, + size_t size) { abd_verify(sabd); ASSERT3U(off, <=, sabd->abd_size); size_t new_offset = ABD_SCATTER(sabd).abd_offset + off; - uint_t chunkcnt = abd_scatter_chunkcnt(sabd) - - (new_offset / zfs_abd_chunk_size); + size_t chunkcnt = abd_chunkcnt_for_bytes( + (new_offset % zfs_abd_chunk_size) + size); + + ASSERT3U(chunkcnt, <=, abd_scatter_chunkcnt(sabd)); /* * If an abd struct is provided, it is only the minimum size. If we diff --git a/module/os/linux/zfs/abd_os.c b/module/os/linux/zfs/abd_os.c index af543d6e3f..d1d238a4e3 100644 --- a/module/os/linux/zfs/abd_os.c +++ b/module/os/linux/zfs/abd_os.c @@ -835,7 +835,8 @@ abd_alloc_for_io(size_t size, boolean_t is_metadata) } abd_t * -abd_get_offset_scatter(abd_t *abd, abd_t *sabd, size_t off) +abd_get_offset_scatter(abd_t *abd, abd_t *sabd, size_t off, + size_t size) { int i = 0; struct scatterlist *sg = NULL; diff --git a/module/zfs/abd.c b/module/zfs/abd.c index d5fafccd08..cc2d3575db 100644 --- a/module/zfs/abd.c +++ b/module/zfs/abd.c @@ -531,7 +531,7 @@ abd_get_offset_impl(abd_t *abd, abd_t *sabd, size_t off, size_t size) } ASSERT3U(left, ==, 0); } else { - abd = abd_get_offset_scatter(abd, sabd, off); + abd = abd_get_offset_scatter(abd, sabd, off, size); } ASSERT3P(abd, !=, NULL);