diff --git a/cmd/zstream/zstream_decompress.c b/cmd/zstream/zstream_decompress.c index f5f66080d0..f8f439d462 100644 --- a/cmd/zstream/zstream_decompress.c +++ b/cmd/zstream/zstream_decompress.c @@ -288,10 +288,12 @@ zstream_do_decompress(int argc, char *argv[]) char *lzbuf = safe_calloc(payload_size); (void) sfread(lzbuf, payload_size, stdin); - abd_t sabd; + abd_t sabd, dabd; abd_get_from_buf_struct(&sabd, lzbuf, payload_size); - int err = zio_decompress_data(c, &sabd, buf, + abd_get_from_buf_struct(&dabd, buf, payload_size); + int err = zio_decompress_data(c, &sabd, &dabd, payload_size, payload_size, NULL); + abd_free(&dabd); abd_free(&sabd); if (err != 0) { diff --git a/include/sys/zio_compress.h b/include/sys/zio_compress.h index 56376fdd10..d41b5dfd44 100644 --- a/include/sys/zio_compress.h +++ b/include/sys/zio_compress.h @@ -22,7 +22,7 @@ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2019, Allan Jude - * Copyright (c) 2019, Klara Inc. + * Copyright (c) 2019, 2024, Klara, Inc. * Use is subject to license terms. * Copyright (c) 2015, 2016 by Delphix. All rights reserved. */ @@ -122,13 +122,13 @@ enum zio_zstd_levels { struct zio_prop; /* Common signature for all zio compress functions. */ -typedef size_t zio_compress_func_t(void *src, void *dst, +typedef size_t zio_compress_func_t(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int); /* Common signature for all zio decompress functions. */ -typedef int zio_decompress_func_t(void *src, void *dst, +typedef int zio_decompress_func_t(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int); /* Common signature for all zio decompress and get level functions. */ -typedef int zio_decompresslevel_func_t(void *src, void *dst, +typedef int zio_decompresslevel_func_t(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, uint8_t *level); /* @@ -153,21 +153,21 @@ extern void lz4_fini(void); /* * Compression routines. */ -extern size_t zfs_lzjb_compress(void *src, void *dst, size_t s_len, +extern size_t zfs_lzjb_compress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); -extern int zfs_lzjb_decompress(void *src, void *dst, size_t s_len, +extern int zfs_lzjb_decompress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); -extern size_t zfs_gzip_compress(void *src, void *dst, size_t s_len, +extern size_t zfs_gzip_compress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); -extern int zfs_gzip_decompress(void *src, void *dst, size_t s_len, +extern int zfs_gzip_decompress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); -extern size_t zfs_zle_compress(void *src, void *dst, size_t s_len, +extern size_t zfs_zle_compress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); -extern int zfs_zle_decompress(void *src, void *dst, size_t s_len, +extern int zfs_zle_decompress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); -extern size_t zfs_lz4_compress(void *src, void *dst, size_t s_len, +extern size_t zfs_lz4_compress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); -extern int zfs_lz4_decompress(void *src, void *dst, size_t s_len, +extern int zfs_lz4_decompress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); /* @@ -179,6 +179,40 @@ extern int zio_decompress_data(enum zio_compress c, abd_t *src, void *dst, size_t s_len, size_t d_len, uint8_t *level); extern int zio_compress_to_feature(enum zio_compress comp); +#define ZFS_COMPRESS_WRAP_DECL(name) \ +size_t \ +name(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int n) \ +{ \ + void *s_buf = abd_borrow_buf_copy(src, s_len); \ + void *d_buf = abd_borrow_buf(dst, d_len); \ + size_t c_len = name##_buf(s_buf, d_buf, s_len, d_len, n); \ + abd_return_buf(src, s_buf, s_len); \ + abd_return_buf_copy(dst, d_buf, d_len); \ + return (c_len); \ +} +#define ZFS_DECOMPRESS_WRAP_DECL(name) \ +int \ +name(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int n) \ +{ \ + void *s_buf = abd_borrow_buf_copy(src, s_len); \ + void *d_buf = abd_borrow_buf(dst, d_len); \ + int err = name##_buf(s_buf, d_buf, s_len, d_len, n); \ + abd_return_buf(src, s_buf, s_len); \ + abd_return_buf_copy(dst, d_buf, d_len); \ + return (err); \ +} +#define ZFS_DECOMPRESS_LEVEL_WRAP_DECL(name) \ +int \ +name(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, uint8_t *n) \ +{ \ + void *s_buf = abd_borrow_buf_copy(src, s_len); \ + void *d_buf = abd_borrow_buf(dst, d_len); \ + int err = name##_buf(s_buf, d_buf, s_len, d_len, n); \ + abd_return_buf(src, s_buf, s_len); \ + abd_return_buf_copy(dst, d_buf, d_len); \ + return (err); \ +} + #ifdef __cplusplus } #endif diff --git a/include/sys/zstd/zstd.h b/include/sys/zstd/zstd.h index f9e7ac0b32..6d212b082f 100644 --- a/include/sys/zstd/zstd.h +++ b/include/sys/zstd/zstd.h @@ -90,12 +90,12 @@ typedef struct zfs_zstd_meta { int zstd_init(void); void zstd_fini(void); -size_t zfs_zstd_compress(void *s_start, void *d_start, size_t s_len, +size_t zfs_zstd_compress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); int zfs_zstd_get_level(void *s_start, size_t s_len, uint8_t *level); -int zfs_zstd_decompress_level(void *s_start, void *d_start, size_t s_len, +int zfs_zstd_decompress_level(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, uint8_t *level); -int zfs_zstd_decompress(void *s_start, void *d_start, size_t s_len, +int zfs_zstd_decompress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int n); void zfs_zstd_cache_reap_now(void); diff --git a/module/zfs/ddt_zap.c b/module/zfs/ddt_zap.c index 8e78ec3277..e96984b86f 100644 --- a/module/zfs/ddt_zap.c +++ b/module/zfs/ddt_zap.c @@ -53,8 +53,12 @@ ddt_zap_compress(const void *src, uchar_t *dst, size_t s_len, size_t d_len) ASSERT3U(d_len, >=, s_len + 1); /* no compression plus version byte */ /* Call compress function directly to avoid hole detection. */ - c_len = ci->ci_compress((void *)src, dst, s_len, d_len - 1, - ci->ci_level); + abd_t sabd, dabd; + abd_get_from_buf_struct(&sabd, (void *)src, s_len); + abd_get_from_buf_struct(&dabd, dst, d_len); + c_len = ci->ci_compress(&sabd, &dabd, s_len, d_len - 1, ci->ci_level); + abd_free(&dabd); + abd_free(&sabd); if (c_len == s_len) { cpfunc = ZIO_COMPRESS_OFF; diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c index e62ecdb259..042725b235 100644 --- a/module/zfs/dsl_dataset.c +++ b/module/zfs/dsl_dataset.c @@ -2426,8 +2426,13 @@ get_receive_resume_token_impl(dsl_dataset_t *ds) compressed = kmem_alloc(packed_size, KM_SLEEP); /* Call compress function directly to avoid hole detection. */ - compressed_size = zfs_gzip_compress(packed, compressed, + abd_t pabd, cabd; + abd_get_from_buf_struct(&pabd, packed, packed_size); + abd_get_from_buf_struct(&cabd, compressed, packed_size); + compressed_size = zfs_gzip_compress(&pabd, &cabd, packed_size, packed_size, 6); + abd_free(&cabd); + abd_free(&pabd); zio_cksum_t cksum; fletcher_4_native_varsize(compressed, compressed_size, &cksum); diff --git a/module/zfs/gzip.c b/module/zfs/gzip.c index 0ca66c2bd6..e7fd6f63c4 100644 --- a/module/zfs/gzip.c +++ b/module/zfs/gzip.c @@ -47,8 +47,8 @@ typedef uLongf zlen_t; #endif -size_t -zfs_gzip_compress(void *s_start, void *d_start, size_t s_len, +static size_t +zfs_gzip_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { int ret; @@ -83,8 +83,8 @@ zfs_gzip_compress(void *s_start, void *d_start, size_t s_len, return ((size_t)dstlen); } -int -zfs_gzip_decompress(void *s_start, void *d_start, size_t s_len, +static int +zfs_gzip_decompress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { (void) n; @@ -105,3 +105,6 @@ zfs_gzip_decompress(void *s_start, void *d_start, size_t s_len, return (0); } + +ZFS_COMPRESS_WRAP_DECL(zfs_gzip_compress) +ZFS_DECOMPRESS_WRAP_DECL(zfs_gzip_decompress) diff --git a/module/zfs/lz4_zfs.c b/module/zfs/lz4_zfs.c index 698ed69956..0033b5e50d 100644 --- a/module/zfs/lz4_zfs.c +++ b/module/zfs/lz4_zfs.c @@ -52,8 +52,8 @@ int LZ4_uncompress_unknownOutputSize(const char *source, char *dest, static kmem_cache_t *lz4_cache; -size_t -zfs_lz4_compress(void *s_start, void *d_start, size_t s_len, +static size_t +zfs_lz4_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { (void) n; @@ -80,8 +80,8 @@ zfs_lz4_compress(void *s_start, void *d_start, size_t s_len, return (bufsiz + sizeof (bufsiz)); } -int -zfs_lz4_decompress(void *s_start, void *d_start, size_t s_len, +static int +zfs_lz4_decompress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { (void) n; @@ -100,6 +100,9 @@ zfs_lz4_decompress(void *s_start, void *d_start, size_t s_len, d_start, bufsiz, d_len) < 0); } +ZFS_COMPRESS_WRAP_DECL(zfs_lz4_compress) +ZFS_DECOMPRESS_WRAP_DECL(zfs_lz4_decompress) + /* * LZ4 API Description: * diff --git a/module/zfs/lzjb.c b/module/zfs/lzjb.c index b246693120..2db549b162 100644 --- a/module/zfs/lzjb.c +++ b/module/zfs/lzjb.c @@ -45,8 +45,8 @@ #define OFFSET_MASK ((1 << (16 - MATCH_BITS)) - 1) #define LEMPEL_SIZE 1024 -size_t -zfs_lzjb_compress(void *s_start, void *d_start, size_t s_len, +static size_t +zfs_lzjb_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { (void) n; @@ -101,8 +101,8 @@ zfs_lzjb_compress(void *s_start, void *d_start, size_t s_len, return (dst - (uchar_t *)d_start); } -int -zfs_lzjb_decompress(void *s_start, void *d_start, +static int +zfs_lzjb_decompress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { (void) s_len, (void) n; @@ -132,3 +132,6 @@ zfs_lzjb_decompress(void *s_start, void *d_start, } return (0); } + +ZFS_COMPRESS_WRAP_DECL(zfs_lzjb_compress) +ZFS_DECOMPRESS_WRAP_DECL(zfs_lzjb_decompress) diff --git a/module/zfs/zio_compress.c b/module/zfs/zio_compress.c index 9553a9377c..118003bd29 100644 --- a/module/zfs/zio_compress.c +++ b/module/zfs/zio_compress.c @@ -29,7 +29,7 @@ /* * Copyright (c) 2013, 2018 by Delphix. All rights reserved. - * Copyright (c) 2019, Klara Inc. + * Copyright (c) 2019, 2024, Klara, Inc. * Copyright (c) 2019, Allan Jude */ @@ -160,10 +160,10 @@ zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len, if (*dst == NULL) *dst = zio_buf_alloc(s_len); - /* No compression algorithms can read from ABDs directly */ - void *tmp = abd_borrow_buf_copy(src, s_len); - c_len = ci->ci_compress(tmp, *dst, s_len, d_len, complevel); - abd_return_buf(src, tmp, s_len); + abd_t dabd; + abd_get_from_buf_struct(&dabd, dst, d_len); + c_len = ci->ci_compress(src, &dabd, s_len, d_len, complevel); + abd_free(&dabd); if (c_len > d_len) return (s_len); @@ -180,15 +180,16 @@ zio_decompress_data(enum zio_compress c, abd_t *src, void *dst, if ((uint_t)c >= ZIO_COMPRESS_FUNCTIONS || ci->ci_decompress == NULL) return (SET_ERROR(EINVAL)); - void *sbuf = abd_borrow_buf_copy(src, s_len); + abd_t dabd; + abd_get_from_buf_struct(&dabd, dst, d_len); int err; if (ci->ci_decompress_level != NULL && level != NULL) - err = ci->ci_decompress_level(sbuf, dst, s_len, d_len, level); + err = ci->ci_decompress_level(src, &dabd, s_len, d_len, level); else - err = ci->ci_decompress(sbuf, dst, s_len, d_len, ci->ci_level); + err = ci->ci_decompress(src, &dabd, s_len, d_len, ci->ci_level); - abd_return_buf(src, sbuf, s_len); + abd_free(&dabd); /* * Decompression shouldn't fail, because we've already verified diff --git a/module/zfs/zle.c b/module/zfs/zle.c index 32b5fe18ce..7810161966 100644 --- a/module/zfs/zle.c +++ b/module/zfs/zle.c @@ -34,8 +34,8 @@ #include #include -size_t -zfs_zle_compress(void *s_start, void *d_start, size_t s_len, +static size_t +zfs_zle_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { uchar_t *src = s_start; @@ -65,8 +65,8 @@ zfs_zle_compress(void *s_start, void *d_start, size_t s_len, return (src == s_end ? dst - (uchar_t *)d_start : s_len); } -int -zfs_zle_decompress(void *s_start, void *d_start, size_t s_len, +static int +zfs_zle_decompress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) { uchar_t *src = s_start; @@ -91,3 +91,6 @@ zfs_zle_decompress(void *s_start, void *d_start, size_t s_len, } return (dst == d_end ? 0 : -1); } + +ZFS_COMPRESS_WRAP_DECL(zfs_zle_compress) +ZFS_DECOMPRESS_WRAP_DECL(zfs_zle_decompress) diff --git a/module/zstd/zfs_zstd.c b/module/zstd/zfs_zstd.c index 34ab8fd8a4..8d1d53d234 100644 --- a/module/zstd/zfs_zstd.c +++ b/module/zstd/zfs_zstd.c @@ -536,8 +536,8 @@ zfs_zstd_compress_impl(void *s_start, void *d_start, size_t s_len, size_t d_len, } -size_t -zfs_zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, +static size_t +zfs_zstd_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int level) { int16_t zstd_level; @@ -569,7 +569,10 @@ zfs_zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, if (zstd_earlyabort_pass > 0 && zstd_level >= zstd_cutoff_level && s_len >= actual_abort_size) { int pass_len = 1; - pass_len = zfs_lz4_compress(s_start, d_start, s_len, d_len, 0); + abd_t sabd; + abd_get_from_buf_struct(&sabd, s_start, s_len); + pass_len = zfs_lz4_compress(&sabd, d_start, s_len, d_len, 0); + abd_free(&sabd); if (pass_len < d_len) { ZSTDSTAT_BUMP(zstd_stat_lz4pass_allowed); goto keep_trying; @@ -595,8 +598,8 @@ keep_trying: } /* Decompress block using zstd and return its stored level */ -int -zfs_zstd_decompress_level(void *s_start, void *d_start, size_t s_len, +static int +zfs_zstd_decompress_level_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, uint8_t *level) { ZSTD_DCtx *dctx; @@ -671,15 +674,20 @@ zfs_zstd_decompress_level(void *s_start, void *d_start, size_t s_len, } /* Decompress datablock using zstd */ -int -zfs_zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, - int level __maybe_unused) +static int +zfs_zstd_decompress_buf(void *s_start, void *d_start, size_t s_len, + size_t d_len, int level __maybe_unused) { - return (zfs_zstd_decompress_level(s_start, d_start, s_len, d_len, + return (zfs_zstd_decompress_level_buf(s_start, d_start, s_len, d_len, NULL)); } +ZFS_COMPRESS_WRAP_DECL(zfs_zstd_compress) +ZFS_DECOMPRESS_WRAP_DECL(zfs_zstd_decompress) +ZFS_DECOMPRESS_LEVEL_WRAP_DECL(zfs_zstd_decompress_level) + + /* Allocator for zstd compression context using mempool_allocator */ static void * zstd_alloc(void *opaque __maybe_unused, size_t size)