diff --git a/include/sys/abd.h b/include/sys/abd.h index 19fe96292d..7b7d84b528 100644 --- a/include/sys/abd.h +++ b/include/sys/abd.h @@ -137,6 +137,7 @@ void abd_copy_from_buf_off(abd_t *, const void *, size_t, size_t); void abd_copy_to_buf_off(void *, abd_t *, size_t, size_t); int abd_cmp(abd_t *, abd_t *); int abd_cmp_buf_off(abd_t *, const void *, size_t, size_t); +int abd_cmp_zero_off(abd_t *, size_t, size_t); void abd_zero_off(abd_t *, size_t, size_t); void abd_verify(abd_t *); @@ -183,6 +184,12 @@ abd_zero(abd_t *abd, size_t size) abd_zero_off(abd, 0, size); } +static inline int +abd_cmp_zero(abd_t *abd, size_t size) +{ + return (abd_cmp_zero_off(abd, 0, size)); +} + /* * ABD type check functions */ diff --git a/module/zfs/abd.c b/module/zfs/abd.c index 2c0cda25db..94f492522f 100644 --- a/module/zfs/abd.c +++ b/module/zfs/abd.c @@ -1050,6 +1050,31 @@ abd_cmp(abd_t *dabd, abd_t *sabd) abd_cmp_cb, NULL)); } +/* + * Check if ABD content is all-zeroes. + */ +static int +abd_cmp_zero_off_cb(void *data, size_t len, void *private) +{ + (void) private; + + /* This function can only check whole uint64s. Enforce that. */ + ASSERT0(P2PHASE(len, 8)); + + uint64_t *end = (uint64_t *)((char *)data + len); + for (uint64_t *word = (uint64_t *)data; word < end; word++) + if (*word != 0) + return (1); + + return (0); +} + +int +abd_cmp_zero_off(abd_t *abd, size_t off, size_t size) +{ + return (abd_iterate_func(abd, off, size, abd_cmp_zero_off_cb, NULL)); +} + /* * Iterate over code ABDs and a data ABD and call @func_raidz_gen. * diff --git a/module/zfs/zio.c b/module/zfs/zio.c index fd69136f70..6d08d4bd16 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -1859,8 +1859,13 @@ zio_write_compress(zio_t *zio) if (compress != ZIO_COMPRESS_OFF && !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) { void *cbuf = NULL; - psize = zio_compress_data(compress, zio->io_abd, &cbuf, lsize, - zp->zp_complevel); + if (abd_cmp_zero(zio->io_abd, lsize) == 0) + psize = 0; + else if (compress == ZIO_COMPRESS_EMPTY) + psize = lsize; + else + psize = zio_compress_data(compress, zio->io_abd, &cbuf, + lsize, zp->zp_complevel); if (psize == 0) { compress = ZIO_COMPRESS_OFF; } else if (psize >= lsize) { @@ -1924,10 +1929,12 @@ zio_write_compress(zio_t *zio) * receive, we must check whether the block can be compressed * to a hole. */ - psize = zio_compress_data(ZIO_COMPRESS_EMPTY, - zio->io_abd, NULL, lsize, zp->zp_complevel); - if (psize == 0 || psize >= lsize) + if (abd_cmp_zero(zio->io_abd, lsize) == 0) { + psize = 0; compress = ZIO_COMPRESS_OFF; + } else { + psize = lsize; + } } else if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS && !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) { /* diff --git a/module/zfs/zio_compress.c b/module/zfs/zio_compress.c index c8a10db748..e12d5498cc 100644 --- a/module/zfs/zio_compress.c +++ b/module/zfs/zio_compress.c @@ -111,19 +111,6 @@ zio_compress_select(spa_t *spa, enum zio_compress child, return (result); } -static int -zio_compress_zeroed_cb(void *data, size_t len, void *private) -{ - (void) private; - - uint64_t *end = (uint64_t *)((char *)data + len); - for (uint64_t *word = (uint64_t *)data; word < end; word++) - if (*word != 0) - return (1); - - return (0); -} - size_t zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len, uint8_t level) @@ -132,18 +119,9 @@ zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len, uint8_t complevel; zio_compress_info_t *ci = &zio_compress_table[c]; - ASSERT((uint_t)c < ZIO_COMPRESS_FUNCTIONS); - ASSERT((uint_t)c == ZIO_COMPRESS_EMPTY || ci->ci_compress != NULL); - - /* - * If the data is all zeroes, we don't even need to allocate - * a block for it. We indicate this by returning zero size. - */ - if (abd_iterate_func(src, 0, s_len, zio_compress_zeroed_cb, NULL) == 0) - return (0); - - if (c == ZIO_COMPRESS_EMPTY) - return (s_len); + ASSERT3U(c, <, ZIO_COMPRESS_FUNCTIONS); + ASSERT3U(ci->ci_compress, !=, NULL); + ASSERT3U(s_len, >, 0); /* Compress at least 12.5% */ d_len = s_len - (s_len >> 3);