Archive-Team
/
zfs
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 103 Wiki Activity

ARC: Remove b_bufcnt/b_ebufcnt from ARC headers 

In most cases we do not care about exact number of buffers linked
to the header, we just need to know if it is zero, non-zero or one.
That can easily be checked just looking on b_buf pointer or in some
cases derefencing it.

b_ebufcnt is read only once, and in that case we already traverse
the list as part of arc_buf_remove(), so second traverse should not
be expensive.

This reduces L1 ARC header size by 8 bytes and full crypto header by
16 bytes, down to 176 and 232 bytes on FreeBSD respectively.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by:	Alexander Motin <mav@FreeBSD.org>
Sponsored by:	iXsystems, Inc.
Closes #15350
This commit is contained in:
Alexander Motin 2023-10-06 11:56:17 -04:00 committed by GitHub
parent 82d2291398
commit 96b9cf42e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 42 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/os/linux/zfs/sys/trace_arc.h
View File

@ -51,7 +51,6 @@ DECLARE_EVENT_CLASS(zfs_arc_buf_hdr_class,
__array(uint64_t, hdr_dva_word, 2) __array(uint64_t, hdr_dva_word, 2)
__field(uint64_t, hdr_birth) __field(uint64_t, hdr_birth)
__field(uint32_t, hdr_flags) __field(uint32_t, hdr_flags)
__field(uint32_t, hdr_bufcnt)
__field(arc_buf_contents_t, hdr_type) __field(arc_buf_contents_t, hdr_type)
__field(uint16_t, hdr_psize) __field(uint16_t, hdr_psize)
__field(uint16_t, hdr_lsize) __field(uint16_t, hdr_lsize)
@ -70,7 +69,6 @@ DECLARE_EVENT_CLASS(zfs_arc_buf_hdr_class,
__entry->hdr_dva_word[1] = ab->b_dva.dva_word[1]; __entry->hdr_dva_word[1] = ab->b_dva.dva_word[1];
__entry->hdr_birth = ab->b_birth; __entry->hdr_birth = ab->b_birth;
__entry->hdr_flags = ab->b_flags; __entry->hdr_flags = ab->b_flags;
__entry->hdr_bufcnt = ab->b_l1hdr.b_bufcnt;
__entry->hdr_psize = ab->b_psize; __entry->hdr_psize = ab->b_psize;
__entry->hdr_lsize = ab->b_lsize; __entry->hdr_lsize = ab->b_lsize;
__entry->hdr_spa = ab->b_spa; __entry->hdr_spa = ab->b_spa;
@ -84,12 +82,12 @@ DECLARE_EVENT_CLASS(zfs_arc_buf_hdr_class,
__entry->hdr_refcount = ab->b_l1hdr.b_refcnt.rc_count; __entry->hdr_refcount = ab->b_l1hdr.b_refcnt.rc_count;
), ),
TP_printk("hdr { dva 0x%llx:0x%llx birth %llu " TP_printk("hdr { dva 0x%llx:0x%llx birth %llu "
"flags 0x%x bufcnt %u type %u psize %u lsize %u spa %llu " "flags 0x%x type %u psize %u lsize %u spa %llu "
"state_type %u access %lu mru_hits %u mru_ghost_hits %u " "state_type %u access %lu mru_hits %u mru_ghost_hits %u "
"mfu_hits %u mfu_ghost_hits %u l2_hits %u refcount %lli }", "mfu_hits %u mfu_ghost_hits %u l2_hits %u refcount %lli }",
__entry->hdr_dva_word[0], __entry->hdr_dva_word[1], __entry->hdr_dva_word[0], __entry->hdr_dva_word[1],
__entry->hdr_birth, __entry->hdr_flags, __entry->hdr_birth, __entry->hdr_flags,
__entry->hdr_bufcnt, __entry->hdr_type, __entry->hdr_psize, __entry->hdr_type, __entry->hdr_psize,
__entry->hdr_lsize, __entry->hdr_spa, __entry->hdr_state_type, __entry->hdr_lsize, __entry->hdr_spa, __entry->hdr_state_type,
__entry->hdr_access, __entry->hdr_mru_hits, __entry->hdr_access, __entry->hdr_mru_hits,
__entry->hdr_mru_ghost_hits, __entry->hdr_mfu_hits, __entry->hdr_mru_ghost_hits, __entry->hdr_mfu_hits,
@ -192,7 +190,6 @@ DECLARE_EVENT_CLASS(zfs_arc_miss_class,
__array(uint64_t, hdr_dva_word, 2) __array(uint64_t, hdr_dva_word, 2)
__field(uint64_t, hdr_birth) __field(uint64_t, hdr_birth)
__field(uint32_t, hdr_flags) __field(uint32_t, hdr_flags)
__field(uint32_t, hdr_bufcnt)
__field(arc_buf_contents_t, hdr_type) __field(arc_buf_contents_t, hdr_type)
__field(uint16_t, hdr_psize) __field(uint16_t, hdr_psize)
__field(uint16_t, hdr_lsize) __field(uint16_t, hdr_lsize)
@ -223,7 +220,6 @@ DECLARE_EVENT_CLASS(zfs_arc_miss_class,
__entry->hdr_dva_word[1] = hdr->b_dva.dva_word[1]; __entry->hdr_dva_word[1] = hdr->b_dva.dva_word[1];
__entry->hdr_birth = hdr->b_birth; __entry->hdr_birth = hdr->b_birth;
__entry->hdr_flags = hdr->b_flags; __entry->hdr_flags = hdr->b_flags;
__entry->hdr_bufcnt = hdr->b_l1hdr.b_bufcnt;
__entry->hdr_psize = hdr->b_psize; __entry->hdr_psize = hdr->b_psize;
__entry->hdr_lsize = hdr->b_lsize; __entry->hdr_lsize = hdr->b_lsize;
__entry->hdr_spa = hdr->b_spa; __entry->hdr_spa = hdr->b_spa;
@ -255,7 +251,7 @@ DECLARE_EVENT_CLASS(zfs_arc_miss_class,
__entry->zb_blkid = zb->zb_blkid; __entry->zb_blkid = zb->zb_blkid;
), ),
TP_printk("hdr { dva 0x%llx:0x%llx birth %llu " TP_printk("hdr { dva 0x%llx:0x%llx birth %llu "
"flags 0x%x bufcnt %u psize %u lsize %u spa %llu state_type %u " "flags 0x%x psize %u lsize %u spa %llu state_type %u "
"access %lu mru_hits %u mru_ghost_hits %u mfu_hits %u " "access %lu mru_hits %u mru_ghost_hits %u mfu_hits %u "
"mfu_ghost_hits %u l2_hits %u refcount %lli } " "mfu_ghost_hits %u l2_hits %u refcount %lli } "
"bp { dva0 0x%llx:0x%llx dva1 0x%llx:0x%llx dva2 " "bp { dva0 0x%llx:0x%llx dva1 0x%llx:0x%llx dva2 "
@ -264,7 +260,7 @@ DECLARE_EVENT_CLASS(zfs_arc_miss_class,
"blkid %llu }", "blkid %llu }",
__entry->hdr_dva_word[0], __entry->hdr_dva_word[1], __entry->hdr_dva_word[0], __entry->hdr_dva_word[1],
__entry->hdr_birth, __entry->hdr_flags, __entry->hdr_birth, __entry->hdr_flags,
__entry->hdr_bufcnt, __entry->hdr_psize, __entry->hdr_lsize, __entry->hdr_psize, __entry->hdr_lsize,
__entry->hdr_spa, __entry->hdr_state_type, __entry->hdr_access, __entry->hdr_spa, __entry->hdr_state_type, __entry->hdr_access,
__entry->hdr_mru_hits, __entry->hdr_mru_ghost_hits, __entry->hdr_mru_hits, __entry->hdr_mru_ghost_hits,
__entry->hdr_mfu_hits, __entry->hdr_mfu_ghost_hits, __entry->hdr_mfu_hits, __entry->hdr_mfu_ghost_hits,

8
include/sys/arc_impl.h
View File

@ -159,8 +159,6 @@ struct arc_write_callback {
* these two allocation states. * these two allocation states.
*/ */
typedef struct l1arc_buf_hdr { typedef struct l1arc_buf_hdr {
uint8_t b_byteswap;
/* protected by arc state mutex */ /* protected by arc state mutex */
arc_state_t *b_state; arc_state_t *b_state;
multilist_node_t b_arc_node; multilist_node_t b_arc_node;
@ -171,7 +169,7 @@ typedef struct l1arc_buf_hdr {
uint32_t b_mru_ghost_hits; uint32_t b_mru_ghost_hits;
uint32_t b_mfu_hits; uint32_t b_mfu_hits;
uint32_t b_mfu_ghost_hits; uint32_t b_mfu_ghost_hits;
uint32_t b_bufcnt; uint8_t b_byteswap;
arc_buf_t *b_buf; arc_buf_t *b_buf;
/* self protecting */ /* self protecting */
@ -434,12 +432,12 @@ typedef struct l2arc_dev {
*/ */
typedef struct arc_buf_hdr_crypt { typedef struct arc_buf_hdr_crypt {
abd_t *b_rabd; /* raw encrypted data */ abd_t *b_rabd; /* raw encrypted data */
dmu_object_type_t b_ot; /* object type */
uint32_t b_ebufcnt; /* count of encrypted buffers */
/* dsobj for looking up encryption key for l2arc encryption */ /* dsobj for looking up encryption key for l2arc encryption */
uint64_t b_dsobj; uint64_t b_dsobj;
dmu_object_type_t b_ot; /* object type */
/* encryption parameters */ /* encryption parameters */
uint8_t b_salt[ZIO_DATA_SALT_LEN]; uint8_t b_salt[ZIO_DATA_SALT_LEN];
uint8_t b_iv[ZIO_DATA_IV_LEN]; uint8_t b_iv[ZIO_DATA_IV_LEN];

101
module/zfs/arc.c
View File

@ -1993,7 +1993,6 @@ arc_buf_untransform_in_place(arc_buf_t *buf)
arc_buf_size(buf)); arc_buf_size(buf));
buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED; buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED; buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
hdr->b_crypt_hdr.b_ebufcnt -= 1;
} }
/* /*
@ -2228,7 +2227,6 @@ arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
ASSERT(HDR_HAS_L1HDR(hdr)); ASSERT(HDR_HAS_L1HDR(hdr));
if (GHOST_STATE(state)) { if (GHOST_STATE(state)) {
ASSERT0(hdr->b_l1hdr.b_bufcnt);
ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
ASSERT(!HDR_HAS_RABD(hdr)); ASSERT(!HDR_HAS_RABD(hdr));
@ -2268,7 +2266,6 @@ arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
ASSERT(HDR_HAS_L1HDR(hdr)); ASSERT(HDR_HAS_L1HDR(hdr));
if (GHOST_STATE(state)) { if (GHOST_STATE(state)) {
ASSERT0(hdr->b_l1hdr.b_bufcnt);
ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
ASSERT(!HDR_HAS_RABD(hdr)); ASSERT(!HDR_HAS_RABD(hdr));
@ -2384,7 +2381,9 @@ arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index)
l2hdr = &hdr->b_l2hdr; l2hdr = &hdr->b_l2hdr;
if (l1hdr) { if (l1hdr) {
abi->abi_bufcnt = l1hdr->b_bufcnt; abi->abi_bufcnt = 0;
for (arc_buf_t *buf = l1hdr->b_buf; buf; buf = buf->b_next)
abi->abi_bufcnt++;
abi->abi_access = l1hdr->b_arc_access; abi->abi_access = l1hdr->b_arc_access;
abi->abi_mru_hits = l1hdr->b_mru_hits; abi->abi_mru_hits = l1hdr->b_mru_hits;
abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits; abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits;
@ -2412,7 +2411,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
{ {
arc_state_t *old_state; arc_state_t *old_state;
int64_t refcnt; int64_t refcnt;
uint32_t bufcnt;
boolean_t update_old, update_new; boolean_t update_old, update_new;
arc_buf_contents_t type = arc_buf_type(hdr); arc_buf_contents_t type = arc_buf_type(hdr);
@ -2426,19 +2424,16 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
if (HDR_HAS_L1HDR(hdr)) { if (HDR_HAS_L1HDR(hdr)) {
old_state = hdr->b_l1hdr.b_state; old_state = hdr->b_l1hdr.b_state;
refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt); refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);
bufcnt = hdr->b_l1hdr.b_bufcnt; update_old = (hdr->b_l1hdr.b_buf != NULL ||
update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL || hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
HDR_HAS_RABD(hdr));
IMPLY(GHOST_STATE(old_state), bufcnt == 0);
IMPLY(GHOST_STATE(new_state), bufcnt == 0);
IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL); IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL);
IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL); IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL);
IMPLY(old_state == arc_anon, bufcnt <= 1); IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL ||
ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
} else { } else {
old_state = arc_l2c_only; old_state = arc_l2c_only;
refcnt = 0; refcnt = 0;
bufcnt = 0;
update_old = B_FALSE; update_old = B_FALSE;
} }
update_new = update_old; update_new = update_old;
@ -2486,14 +2481,12 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
if (update_new && new_state != arc_l2c_only) { if (update_new && new_state != arc_l2c_only) {
ASSERT(HDR_HAS_L1HDR(hdr)); ASSERT(HDR_HAS_L1HDR(hdr));
if (GHOST_STATE(new_state)) { if (GHOST_STATE(new_state)) {
ASSERT0(bufcnt);
/* /*
* When moving a header to a ghost state, we first * When moving a header to a ghost state, we first
* remove all arc buffers. Thus, we'll have a * remove all arc buffers. Thus, we'll have no arc
* bufcnt of zero, and no arc buffer to use for * buffer to use for the reference. As a result, we
* the reference. As a result, we use the arc * use the arc header pointer for the reference.
* header pointer for the reference.
*/ */
(void) zfs_refcount_add_many( (void) zfs_refcount_add_many(
&new_state->arcs_size[type], &new_state->arcs_size[type],
@ -2501,7 +2494,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
ASSERT(!HDR_HAS_RABD(hdr)); ASSERT(!HDR_HAS_RABD(hdr));
} else { } else {
uint32_t buffers = 0;
/* /*
* Each individual buffer holds a unique reference, * Each individual buffer holds a unique reference,
@ -2510,8 +2502,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
*/ */
for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
buf = buf->b_next) { buf = buf->b_next) {
ASSERT3U(bufcnt, !=, 0);
buffers++;
/* /*
* When the arc_buf_t is sharing the data * When the arc_buf_t is sharing the data
@ -2527,7 +2517,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
&new_state->arcs_size[type], &new_state->arcs_size[type],
arc_buf_size(buf), buf); arc_buf_size(buf), buf);
} }
ASSERT3U(bufcnt, ==, buffers);
if (hdr->b_l1hdr.b_pabd != NULL) { if (hdr->b_l1hdr.b_pabd != NULL) {
(void) zfs_refcount_add_many( (void) zfs_refcount_add_many(
@ -2546,7 +2535,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
if (update_old && old_state != arc_l2c_only) { if (update_old && old_state != arc_l2c_only) {
ASSERT(HDR_HAS_L1HDR(hdr)); ASSERT(HDR_HAS_L1HDR(hdr));
if (GHOST_STATE(old_state)) { if (GHOST_STATE(old_state)) {
ASSERT0(bufcnt);
ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
ASSERT(!HDR_HAS_RABD(hdr)); ASSERT(!HDR_HAS_RABD(hdr));
@ -2562,7 +2550,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
&old_state->arcs_size[type], &old_state->arcs_size[type],
HDR_GET_LSIZE(hdr), hdr); HDR_GET_LSIZE(hdr), hdr);
} else { } else {
uint32_t buffers = 0;
/* /*
* Each individual buffer holds a unique reference, * Each individual buffer holds a unique reference,
@ -2571,8 +2558,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
*/ */
for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
buf = buf->b_next) { buf = buf->b_next) {
ASSERT3U(bufcnt, !=, 0);
buffers++;
/* /*
* When the arc_buf_t is sharing the data * When the arc_buf_t is sharing the data
@ -2588,7 +2573,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
&old_state->arcs_size[type], &old_state->arcs_size[type],
arc_buf_size(buf), buf); arc_buf_size(buf), buf);
} }
ASSERT3U(bufcnt, ==, buffers);
ASSERT(hdr->b_l1hdr.b_pabd != NULL || ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
HDR_HAS_RABD(hdr)); HDR_HAS_RABD(hdr));
@ -2836,9 +2820,6 @@ arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb,
VERIFY3P(buf->b_data, !=, NULL); VERIFY3P(buf->b_data, !=, NULL);
hdr->b_l1hdr.b_buf = buf; hdr->b_l1hdr.b_buf = buf;
hdr->b_l1hdr.b_bufcnt += 1;
if (encrypted)
hdr->b_crypt_hdr.b_ebufcnt += 1;
/* /*
* If the user wants the data from the hdr, we need to either copy or * If the user wants the data from the hdr, we need to either copy or
@ -3080,8 +3061,6 @@ arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
} }
buf->b_next = NULL; buf->b_next = NULL;
ASSERT3P(lastbuf, !=, buf); ASSERT3P(lastbuf, !=, buf);
IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL);
IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL);
IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf)); IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));
return (lastbuf); return (lastbuf);
@ -3120,22 +3099,20 @@ arc_buf_destroy_impl(arc_buf_t *buf)
} }
buf->b_data = NULL; buf->b_data = NULL;
ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
hdr->b_l1hdr.b_bufcnt -= 1;
if (ARC_BUF_ENCRYPTED(buf)) {
hdr->b_crypt_hdr.b_ebufcnt -= 1;
/* /*
* If we have no more encrypted buffers and we've * If we have no more encrypted buffers and we've already
* already gotten a copy of the decrypted data we can * gotten a copy of the decrypted data we can free b_rabd
* free b_rabd to save some space. * to save some space.
*/ */
if (hdr->b_crypt_hdr.b_ebufcnt == 0 && if (ARC_BUF_ENCRYPTED(buf) && HDR_HAS_RABD(hdr) &&
HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd != NULL && hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
!HDR_IO_IN_PROGRESS(hdr)) { arc_buf_t *b;
arc_hdr_free_abd(hdr, B_TRUE); for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) {
if (b != buf && ARC_BUF_ENCRYPTED(b))
break;
} }
if (b == NULL)
arc_hdr_free_abd(hdr, B_TRUE);
} }
} }
@ -3323,7 +3300,6 @@ arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
hdr->b_l1hdr.b_mru_ghost_hits = 0; hdr->b_l1hdr.b_mru_ghost_hits = 0;
hdr->b_l1hdr.b_mfu_hits = 0; hdr->b_l1hdr.b_mfu_hits = 0;
hdr->b_l1hdr.b_mfu_ghost_hits = 0; hdr->b_l1hdr.b_mfu_ghost_hits = 0;
hdr->b_l1hdr.b_bufcnt = 0;
hdr->b_l1hdr.b_buf = NULL; hdr->b_l1hdr.b_buf = NULL;
ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
@ -3380,7 +3356,6 @@ arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
ASSERT(!HDR_HAS_RABD(hdr)); ASSERT(!HDR_HAS_RABD(hdr));
} else { } else {
ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
ASSERT0(hdr->b_l1hdr.b_bufcnt);
#ifdef ZFS_DEBUG #ifdef ZFS_DEBUG
ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
#endif #endif
@ -3496,7 +3471,6 @@ arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt)
#ifdef ZFS_DEBUG #ifdef ZFS_DEBUG
nhdr->b_l1hdr.b_freeze_cksum = hdr->b_l1hdr.b_freeze_cksum; nhdr->b_l1hdr.b_freeze_cksum = hdr->b_l1hdr.b_freeze_cksum;
#endif #endif
nhdr->b_l1hdr.b_bufcnt = hdr->b_l1hdr.b_bufcnt;
nhdr->b_l1hdr.b_byteswap = hdr->b_l1hdr.b_byteswap; nhdr->b_l1hdr.b_byteswap = hdr->b_l1hdr.b_byteswap;
nhdr->b_l1hdr.b_state = hdr->b_l1hdr.b_state; nhdr->b_l1hdr.b_state = hdr->b_l1hdr.b_state;
nhdr->b_l1hdr.b_arc_access = hdr->b_l1hdr.b_arc_access; nhdr->b_l1hdr.b_arc_access = hdr->b_l1hdr.b_arc_access;
@ -3539,7 +3513,6 @@ arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt)
hdr->b_l1hdr.b_freeze_cksum = NULL; hdr->b_l1hdr.b_freeze_cksum = NULL;
#endif #endif
hdr->b_l1hdr.b_buf = NULL; hdr->b_l1hdr.b_buf = NULL;
hdr->b_l1hdr.b_bufcnt = 0;
hdr->b_l1hdr.b_byteswap = 0; hdr->b_l1hdr.b_byteswap = 0;
hdr->b_l1hdr.b_state = NULL; hdr->b_l1hdr.b_state = NULL;
hdr->b_l1hdr.b_arc_access = 0; hdr->b_l1hdr.b_arc_access = 0;
@ -3553,7 +3526,6 @@ arc_hdr_realloc_crypt(arc_buf_hdr_t *hdr, boolean_t need_crypt)
if (ocache == hdr_full_crypt_cache) { if (ocache == hdr_full_crypt_cache) {
ASSERT(!HDR_HAS_RABD(hdr)); ASSERT(!HDR_HAS_RABD(hdr));
hdr->b_crypt_hdr.b_ot = DMU_OT_NONE; hdr->b_crypt_hdr.b_ot = DMU_OT_NONE;
hdr->b_crypt_hdr.b_ebufcnt = 0;
hdr->b_crypt_hdr.b_dsobj = 0; hdr->b_crypt_hdr.b_dsobj = 0;
memset(hdr->b_crypt_hdr.b_salt, 0, ZIO_DATA_SALT_LEN); memset(hdr->b_crypt_hdr.b_salt, 0, ZIO_DATA_SALT_LEN);
memset(hdr->b_crypt_hdr.b_iv, 0, ZIO_DATA_IV_LEN); memset(hdr->b_crypt_hdr.b_iv, 0, ZIO_DATA_IV_LEN);
@ -3787,8 +3759,6 @@ static void
arc_hdr_destroy(arc_buf_hdr_t *hdr) arc_hdr_destroy(arc_buf_hdr_t *hdr)
{ {
if (HDR_HAS_L1HDR(hdr)) { if (HDR_HAS_L1HDR(hdr)) {
ASSERT(hdr->b_l1hdr.b_buf == NULL ||
hdr->b_l1hdr.b_bufcnt > 0);
ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
} }
@ -3869,7 +3839,8 @@ arc_buf_destroy(arc_buf_t *buf, const void *tag)
arc_buf_hdr_t *hdr = buf->b_hdr; arc_buf_hdr_t *hdr = buf->b_hdr;
if (hdr->b_l1hdr.b_state == arc_anon) { if (hdr->b_l1hdr.b_state == arc_anon) {
ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
ASSERT(ARC_BUF_LAST(buf));
ASSERT(!HDR_IO_IN_PROGRESS(hdr)); ASSERT(!HDR_IO_IN_PROGRESS(hdr));
VERIFY0(remove_reference(hdr, tag)); VERIFY0(remove_reference(hdr, tag));
return; return;
@ -3879,7 +3850,7 @@ arc_buf_destroy(arc_buf_t *buf, const void *tag)
mutex_enter(hash_lock); mutex_enter(hash_lock);
ASSERT3P(hdr, ==, buf->b_hdr); ASSERT3P(hdr, ==, buf->b_hdr);
ASSERT(hdr->b_l1hdr.b_bufcnt > 0); ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon); ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
ASSERT3P(buf->b_data, !=, NULL); ASSERT3P(buf->b_data, !=, NULL);
@ -3922,7 +3893,6 @@ arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted)
ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
ASSERT(HDR_HAS_L1HDR(hdr)); ASSERT(HDR_HAS_L1HDR(hdr));
ASSERT(!HDR_IO_IN_PROGRESS(hdr)); ASSERT(!HDR_IO_IN_PROGRESS(hdr));
ASSERT0(hdr->b_l1hdr.b_bufcnt);
ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt)); ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));
@ -6320,7 +6290,8 @@ arc_release(arc_buf_t *buf, const void *tag)
ASSERT(!HDR_IN_HASH_TABLE(hdr)); ASSERT(!HDR_IN_HASH_TABLE(hdr));
ASSERT(!HDR_HAS_L2HDR(hdr)); ASSERT(!HDR_HAS_L2HDR(hdr));
ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
ASSERT(ARC_BUF_LAST(buf));
ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1); ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
@ -6371,7 +6342,7 @@ arc_release(arc_buf_t *buf, const void *tag)
/* /*
* Do we have more than one buf? * Do we have more than one buf?
*/ */
if (hdr->b_l1hdr.b_bufcnt > 1) { if (hdr->b_l1hdr.b_buf != buf || !ARC_BUF_LAST(buf)) {
arc_buf_hdr_t *nhdr; arc_buf_hdr_t *nhdr;
uint64_t spa = hdr->b_spa; uint64_t spa = hdr->b_spa;
uint64_t psize = HDR_GET_PSIZE(hdr); uint64_t psize = HDR_GET_PSIZE(hdr);
@ -6452,10 +6423,6 @@ arc_release(arc_buf_t *buf, const void *tag)
arc_buf_size(buf), buf); arc_buf_size(buf), buf);
} }
hdr->b_l1hdr.b_bufcnt -= 1;
if (ARC_BUF_ENCRYPTED(buf))
hdr->b_crypt_hdr.b_ebufcnt -= 1;
arc_cksum_verify(buf); arc_cksum_verify(buf);
arc_buf_unwatch(buf); arc_buf_unwatch(buf);
@ -6468,15 +6435,11 @@ arc_release(arc_buf_t *buf, const void *tag)
nhdr = arc_hdr_alloc(spa, psize, lsize, protected, nhdr = arc_hdr_alloc(spa, psize, lsize, protected,
compress, hdr->b_complevel, type); compress, hdr->b_complevel, type);
ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL); ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
ASSERT0(nhdr->b_l1hdr.b_bufcnt);
ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt)); ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
VERIFY3U(nhdr->b_type, ==, type); VERIFY3U(nhdr->b_type, ==, type);
ASSERT(!HDR_SHARED_DATA(nhdr)); ASSERT(!HDR_SHARED_DATA(nhdr));
nhdr->b_l1hdr.b_buf = buf; nhdr->b_l1hdr.b_buf = buf;
nhdr->b_l1hdr.b_bufcnt = 1;
if (ARC_BUF_ENCRYPTED(buf))
nhdr->b_crypt_hdr.b_ebufcnt = 1;
(void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag); (void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
buf->b_hdr = nhdr; buf->b_hdr = nhdr;
@ -6527,7 +6490,7 @@ arc_write_ready(zio_t *zio)
ASSERT(HDR_HAS_L1HDR(hdr)); ASSERT(HDR_HAS_L1HDR(hdr));
ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt)); ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
ASSERT(hdr->b_l1hdr.b_bufcnt > 0); ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
/* /*
* If we're reexecuting this zio because the pool suspended, then * If we're reexecuting this zio because the pool suspended, then
@ -6666,7 +6629,8 @@ arc_write_ready(zio_t *zio)
} else { } else {
ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd)); ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf)); ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
ASSERT(ARC_BUF_LAST(buf));
arc_share_buf(hdr, buf); arc_share_buf(hdr, buf);
} }
@ -6747,7 +6711,8 @@ arc_write_done(zio_t *zio)
(void *)hdr, (void *)exists); (void *)hdr, (void *)exists);
} else { } else {
/* Dedup */ /* Dedup */
ASSERT(hdr->b_l1hdr.b_bufcnt == 1); ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
ASSERT(hdr->b_l1hdr.b_state == arc_anon); ASSERT(hdr->b_l1hdr.b_state == arc_anon);
ASSERT(BP_GET_DEDUP(zio->io_bp)); ASSERT(BP_GET_DEDUP(zio->io_bp));
ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
@ -6788,7 +6753,7 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
ASSERT(!HDR_IO_ERROR(hdr)); ASSERT(!HDR_IO_ERROR(hdr));
ASSERT(!HDR_IO_IN_PROGRESS(hdr)); ASSERT(!HDR_IO_IN_PROGRESS(hdr));
ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
if (uncached) if (uncached)
arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED); arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
else if (l2arc) else if (l2arc)