compress: change compression providers API to use ABDs

This commit changes the provider compress and decompress API to take ABD
pointers instead of buffer pointers for both data source and
destination. It then updates all providers to match.

This doesn't actually change the providers to do chunked compression,
just changes the API to allow such an update in the future. Helper
macros are added to easily adapt the ABD functions to their buffer-based
implementations.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>

module/zfs/ddt_zap.c

@@ -53,8 +53,12 @@ ddt_zap_compress(const void *src, uchar_t *dst, size_t s_len, size_t d_len)
 	ASSERT3U(d_len, >=, s_len + 1);	/* no compression plus version byte */
 
 	/* Call compress function directly to avoid hole detection. */
-	c_len = ci->ci_compress((void *)src, dst, s_len, d_len - 1,
-	    ci->ci_level);
+	abd_t sabd, dabd;
+	abd_get_from_buf_struct(&sabd, (void *)src, s_len);
+	abd_get_from_buf_struct(&dabd, dst, d_len);
+	c_len = ci->ci_compress(&sabd, &dabd, s_len, d_len - 1, ci->ci_level);
+	abd_free(&dabd);
+	abd_free(&sabd);
 
 	if (c_len == s_len) {
 		cpfunc = ZIO_COMPRESS_OFF;

module/zfs/dsl_dataset.c

@@ -2426,8 +2426,13 @@ get_receive_resume_token_impl(dsl_dataset_t *ds)
 	compressed = kmem_alloc(packed_size, KM_SLEEP);
 
 	/* Call compress function directly to avoid hole detection. */
-	compressed_size = zfs_gzip_compress(packed, compressed,
+	abd_t pabd, cabd;
+	abd_get_from_buf_struct(&pabd, packed, packed_size);
+	abd_get_from_buf_struct(&cabd, compressed, packed_size);
+	compressed_size = zfs_gzip_compress(&pabd, &cabd,
 	    packed_size, packed_size, 6);
+	abd_free(&cabd);
+	abd_free(&pabd);
 
 	zio_cksum_t cksum;
 	fletcher_4_native_varsize(compressed, compressed_size, &cksum);

module/zfs/gzip.c

@@ -47,8 +47,8 @@ typedef uLongf zlen_t;
 
 #endif
 
-size_t
-zfs_gzip_compress(void *s_start, void *d_start, size_t s_len,
+static size_t
+zfs_gzip_compress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	int ret;
@@ -83,8 +83,8 @@ zfs_gzip_compress(void *s_start, void *d_start, size_t s_len,
 	return ((size_t)dstlen);
 }
 
-int
-zfs_gzip_decompress(void *s_start, void *d_start, size_t s_len,
+static int
+zfs_gzip_decompress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	(void) n;
@@ -105,3 +105,6 @@ zfs_gzip_decompress(void *s_start, void *d_start, size_t s_len,
 
 	return (0);
 }
+
+ZFS_COMPRESS_WRAP_DECL(zfs_gzip_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_gzip_decompress)

module/zfs/lz4_zfs.c

@@ -52,8 +52,8 @@ int LZ4_uncompress_unknownOutputSize(const char *source, char *dest,
 
 static kmem_cache_t *lz4_cache;
 
-size_t
-zfs_lz4_compress(void *s_start, void *d_start, size_t s_len,
+static size_t
+zfs_lz4_compress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	(void) n;
@@ -80,8 +80,8 @@ zfs_lz4_compress(void *s_start, void *d_start, size_t s_len,
 	return (bufsiz + sizeof (bufsiz));
 }
 
-int
-zfs_lz4_decompress(void *s_start, void *d_start, size_t s_len,
+static int
+zfs_lz4_decompress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	(void) n;
@@ -100,6 +100,9 @@ zfs_lz4_decompress(void *s_start, void *d_start, size_t s_len,
 	    d_start, bufsiz, d_len) < 0);
 }
 
+ZFS_COMPRESS_WRAP_DECL(zfs_lz4_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_lz4_decompress)
+
 /*
  * LZ4 API Description:
  *

module/zfs/lzjb.c

@@ -45,8 +45,8 @@
 #define	OFFSET_MASK	((1 << (16 - MATCH_BITS)) - 1)
 #define	LEMPEL_SIZE	1024
 
-size_t
-zfs_lzjb_compress(void *s_start, void *d_start, size_t s_len,
+static size_t
+zfs_lzjb_compress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	(void) n;
@@ -101,8 +101,8 @@ zfs_lzjb_compress(void *s_start, void *d_start, size_t s_len,
 	return (dst - (uchar_t *)d_start);
 }
 
-int
-zfs_lzjb_decompress(void *s_start, void *d_start,
+static int
+zfs_lzjb_decompress_buf(void *s_start, void *d_start,
     size_t s_len, size_t d_len, int n)
 {
 	(void) s_len, (void) n;
@@ -132,3 +132,6 @@ zfs_lzjb_decompress(void *s_start, void *d_start,
 	}
 	return (0);
 }
+
+ZFS_COMPRESS_WRAP_DECL(zfs_lzjb_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_lzjb_decompress)

module/zfs/zio_compress.c

@@ -29,7 +29,7 @@
 
 /*
  * Copyright (c) 2013, 2018 by Delphix. All rights reserved.
- * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, 2024, Klara, Inc.
  * Copyright (c) 2019, Allan Jude
  */
 
@@ -160,10 +160,10 @@ zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len,
 	if (*dst == NULL)
 		*dst = zio_buf_alloc(s_len);
 
-	/* No compression algorithms can read from ABDs directly */
-	void *tmp = abd_borrow_buf_copy(src, s_len);
-	c_len = ci->ci_compress(tmp, *dst, s_len, d_len, complevel);
-	abd_return_buf(src, tmp, s_len);
+	abd_t dabd;
+	abd_get_from_buf_struct(&dabd, dst, d_len);
+	c_len = ci->ci_compress(src, &dabd, s_len, d_len, complevel);
+	abd_free(&dabd);
 
 	if (c_len > d_len)
 		return (s_len);
@@ -180,15 +180,16 @@ zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
 	if ((uint_t)c >= ZIO_COMPRESS_FUNCTIONS || ci->ci_decompress == NULL)
 		return (SET_ERROR(EINVAL));
 
-	void *sbuf = abd_borrow_buf_copy(src, s_len);
+	abd_t dabd;
+	abd_get_from_buf_struct(&dabd, dst, d_len);
 
 	int err;
 	if (ci->ci_decompress_level != NULL && level != NULL)
-		err = ci->ci_decompress_level(sbuf, dst, s_len, d_len, level);
+		err = ci->ci_decompress_level(src, &dabd, s_len, d_len, level);
 	else
-		err = ci->ci_decompress(sbuf, dst, s_len, d_len, ci->ci_level);
+		err = ci->ci_decompress(src, &dabd, s_len, d_len, ci->ci_level);
 
-	abd_return_buf(src, sbuf, s_len);
+	abd_free(&dabd);
 
 	/*
 	 * Decompression shouldn't fail, because we've already verified

module/zfs/zle.c

@@ -34,8 +34,8 @@
 #include <sys/sysmacros.h>
 #include <sys/zio_compress.h>
 
-size_t
-zfs_zle_compress(void *s_start, void *d_start, size_t s_len,
+static size_t
+zfs_zle_compress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	uchar_t *src = s_start;
@@ -65,8 +65,8 @@ zfs_zle_compress(void *s_start, void *d_start, size_t s_len,
 	return (src == s_end ? dst - (uchar_t *)d_start : s_len);
 }
 
-int
-zfs_zle_decompress(void *s_start, void *d_start, size_t s_len,
+static int
+zfs_zle_decompress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	uchar_t *src = s_start;
@@ -91,3 +91,6 @@ zfs_zle_decompress(void *s_start, void *d_start, size_t s_len,
 	}
 	return (dst == d_end ? 0 : -1);
 }
+
+ZFS_COMPRESS_WRAP_DECL(zfs_zle_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_zle_decompress)

module/zstd/zfs_zstd.c

@@ -536,8 +536,8 @@ zfs_zstd_compress_impl(void *s_start, void *d_start, size_t s_len, size_t d_len,
 }
 
 
-size_t
-zfs_zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len,
+static size_t
+zfs_zstd_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len,
     int level)
 {
 	int16_t zstd_level;
@@ -569,7 +569,10 @@ zfs_zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len,
 	if (zstd_earlyabort_pass > 0 && zstd_level >= zstd_cutoff_level &&
 	    s_len >= actual_abort_size) {
 		int pass_len = 1;
-		pass_len = zfs_lz4_compress(s_start, d_start, s_len, d_len, 0);
+		abd_t sabd;
+		abd_get_from_buf_struct(&sabd, s_start, s_len);
+		pass_len = zfs_lz4_compress(&sabd, d_start, s_len, d_len, 0);
+		abd_free(&sabd);
 		if (pass_len < d_len) {
 			ZSTDSTAT_BUMP(zstd_stat_lz4pass_allowed);
 			goto keep_trying;
@@ -595,8 +598,8 @@ keep_trying:
 }
 
 /* Decompress block using zstd and return its stored level */
-int
-zfs_zstd_decompress_level(void *s_start, void *d_start, size_t s_len,
+static int
+zfs_zstd_decompress_level_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, uint8_t *level)
 {
 	ZSTD_DCtx *dctx;
@@ -671,15 +674,20 @@ zfs_zstd_decompress_level(void *s_start, void *d_start, size_t s_len,
 }
 
 /* Decompress datablock using zstd */
-int
-zfs_zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len,
-    int level __maybe_unused)
+static int
+zfs_zstd_decompress_buf(void *s_start, void *d_start, size_t s_len,
+    size_t d_len, int level __maybe_unused)
 {
 
-	return (zfs_zstd_decompress_level(s_start, d_start, s_len, d_len,
+	return (zfs_zstd_decompress_level_buf(s_start, d_start, s_len, d_len,
 	    NULL));
 }
 
+ZFS_COMPRESS_WRAP_DECL(zfs_zstd_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_zstd_decompress)
+ZFS_DECOMPRESS_LEVEL_WRAP_DECL(zfs_zstd_decompress_level)
+
+
 /* Allocator for zstd compression context using mempool_allocator */
 static void *
 zstd_alloc(void *opaque __maybe_unused, size_t size)