abd: lift ABD zero scan from zio_compress_data() to abd_cmp_zero()

It's now the caller's responsibility do special handling for holes if
that's something it wants.

This also makes zio_compress_data() and zio_decompress_data() properly
the inverse of each other.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Tino Reichardt <milky-zfs@mcmilk.de>
Reviewed-by: Jason Lee <jasonlee@lanl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>
Closes #16326

include/sys/abd.h

@@ -137,6 +137,7 @@ void abd_copy_from_buf_off(abd_t *, const void *, size_t, size_t);
 void abd_copy_to_buf_off(void *, abd_t *, size_t, size_t);
 int abd_cmp(abd_t *, abd_t *);
 int abd_cmp_buf_off(abd_t *, const void *, size_t, size_t);
+int abd_cmp_zero_off(abd_t *, size_t, size_t);
 void abd_zero_off(abd_t *, size_t, size_t);
 void abd_verify(abd_t *);
 
@@ -183,6 +184,12 @@ abd_zero(abd_t *abd, size_t size)
 	abd_zero_off(abd, 0, size);
 }
 
+static inline int
+abd_cmp_zero(abd_t *abd, size_t size)
+{
+	return (abd_cmp_zero_off(abd, 0, size));
+}
+
 /*
  * ABD type check functions
  */

module/zfs/abd.c

@@ -1050,6 +1050,31 @@ abd_cmp(abd_t *dabd, abd_t *sabd)
 	    abd_cmp_cb, NULL));
 }
 
+/*
+ * Check if ABD content is all-zeroes.
+ */
+static int
+abd_cmp_zero_off_cb(void *data, size_t len, void *private)
+{
+	(void) private;
+
+	/* This function can only check whole uint64s. Enforce that. */
+	ASSERT0(P2PHASE(len, 8));
+
+	uint64_t *end = (uint64_t *)((char *)data + len);
+	for (uint64_t *word = (uint64_t *)data; word < end; word++)
+		if (*word != 0)
+			return (1);
+
+	return (0);
+}
+
+int
+abd_cmp_zero_off(abd_t *abd, size_t off, size_t size)
+{
+	return (abd_iterate_func(abd, off, size, abd_cmp_zero_off_cb, NULL));
+}
+
 /*
  * Iterate over code ABDs and a data ABD and call @func_raidz_gen.
  *

module/zfs/zio.c

@@ -1859,8 +1859,13 @@ zio_write_compress(zio_t *zio)
 	if (compress != ZIO_COMPRESS_OFF &&
 	    !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {
 		void *cbuf = NULL;
-		psize = zio_compress_data(compress, zio->io_abd, &cbuf, lsize,
+		if (abd_cmp_zero(zio->io_abd, lsize) == 0)
-		    zp->zp_complevel);
+			psize = 0;
+		else if (compress == ZIO_COMPRESS_EMPTY)
+			psize = lsize;
+		else
+			psize = zio_compress_data(compress, zio->io_abd, &cbuf,
+			    lsize, zp->zp_complevel);
 		if (psize == 0) {
 			compress = ZIO_COMPRESS_OFF;
 		} else if (psize >= lsize) {
@@ -1924,10 +1929,12 @@ zio_write_compress(zio_t *zio)
 		 * receive, we must check whether the block can be compressed
 		 * to a hole.
 		 */
-		psize = zio_compress_data(ZIO_COMPRESS_EMPTY,
+		if (abd_cmp_zero(zio->io_abd, lsize) == 0) {
-		    zio->io_abd, NULL, lsize, zp->zp_complevel);
+			psize = 0;
-		if (psize == 0 || psize >= lsize)
 			compress = ZIO_COMPRESS_OFF;
+		} else {
+			psize = lsize;
+		}
 	} else if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS &&
 	    !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) {
 		/*

module/zfs/zio_compress.c

@@ -111,19 +111,6 @@ zio_compress_select(spa_t *spa, enum zio_compress child,
 	return (result);
 }
 
-static int
-zio_compress_zeroed_cb(void *data, size_t len, void *private)
-{
-	(void) private;
-
-	uint64_t *end = (uint64_t *)((char *)data + len);
-	for (uint64_t *word = (uint64_t *)data; word < end; word++)
-		if (*word != 0)
-			return (1);
-
-	return (0);
-}
-
 size_t
 zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len,
     uint8_t level)
@@ -132,18 +119,9 @@ zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len,
 	uint8_t complevel;
 	zio_compress_info_t *ci = &zio_compress_table[c];
 
-	ASSERT((uint_t)c < ZIO_COMPRESS_FUNCTIONS);
+	ASSERT3U(c, <, ZIO_COMPRESS_FUNCTIONS);
-	ASSERT((uint_t)c == ZIO_COMPRESS_EMPTY || ci->ci_compress != NULL);
+	ASSERT3U(ci->ci_compress, !=, NULL);
-
+	ASSERT3U(s_len, >, 0);
-	/*
-	 * If the data is all zeroes, we don't even need to allocate
-	 * a block for it.  We indicate this by returning zero size.
-	 */
-	if (abd_iterate_func(src, 0, s_len, zio_compress_zeroed_cb, NULL) == 0)
-		return (0);
-
-	if (c == ZIO_COMPRESS_EMPTY)
-		return (s_len);
 
 	/* Compress at least 12.5% */
 	d_len = s_len - (s_len >> 3);