From f62e6e1f985b5cc197940dcd2dc839aab0708ca2 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Thu, 4 Jul 2024 14:48:38 +1000 Subject: [PATCH] compress: change zio_compress API to use ABDs This commit changes the frontend zio_compress_data and zio_decompress_data APIs to take ABD points instead of buffer pointers. All callers are updated to match. Any that already have an appropriate ABD nearby now use it directly, while at the rest we create an one. Internally, the ABDs are passed through to the provider directly. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Signed-off-by: Rob Norris --- cmd/zdb/zdb.c | 37 +++++++++----- cmd/zstream/zstream_recompress.c | 17 ++++--- include/sys/zio_compress.h | 4 +- module/zfs/arc.c | 84 ++++++++++++++------------------ module/zfs/blkptr.c | 12 +++-- module/zfs/ddt_zap.c | 6 ++- module/zfs/dmu_recv.c | 7 ++- module/zfs/zio.c | 35 +++++++------ module/zfs/zio_compress.c | 20 +++----- module/zstd/zfs_zstd.c | 6 ++- 10 files changed, 116 insertions(+), 112 deletions(-) diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index c72df3909..41c2b6765 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -4657,7 +4657,6 @@ dump_l2arc_log_blocks(int fd, const l2arc_dev_hdr_phys_t *l2dhdr, l2arc_log_blk_phys_t this_lb; uint64_t asize; l2arc_log_blkptr_t lbps[2]; - abd_t *abd; zio_cksum_t cksum; int failed = 0; l2arc_dev_t dev; @@ -4711,20 +4710,25 @@ dump_l2arc_log_blocks(int fd, const l2arc_dev_hdr_phys_t *l2dhdr, switch (L2BLK_GET_COMPRESS((&lbps[0])->lbp_prop)) { case ZIO_COMPRESS_OFF: break; - default: - abd = abd_alloc_for_io(asize, B_TRUE); + default: { + abd_t *abd = abd_alloc_linear(asize, B_TRUE); abd_copy_from_buf_off(abd, &this_lb, 0, asize); - if (zio_decompress_data(L2BLK_GET_COMPRESS( - (&lbps[0])->lbp_prop), abd, &this_lb, - asize, sizeof (this_lb), NULL) != 0) { + abd_t dabd; + abd_get_from_buf_struct(&dabd, &this_lb, + sizeof (this_lb)); + int err = zio_decompress_data(L2BLK_GET_COMPRESS( + (&lbps[0])->lbp_prop), abd, &dabd, + asize, sizeof (this_lb), NULL); + abd_free(&dabd); + abd_free(abd); + if (err != 0) { (void) printf("L2ARC block decompression " "failed\n"); - abd_free(abd); goto out; } - abd_free(abd); break; } + } if (this_lb.lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC)) byteswap_uint64_array(&this_lb, sizeof (this_lb)); @@ -8618,13 +8622,22 @@ try_decompress_block(abd_t *pabd, uint64_t lsize, uint64_t psize, memset(lbuf, 0x00, lsize); memset(lbuf2, 0xff, lsize); + abd_t labd, labd2; + abd_get_from_buf_struct(&labd, lbuf, lsize); + abd_get_from_buf_struct(&labd2, lbuf2, lsize); + + boolean_t ret = B_FALSE; if (zio_decompress_data(cfunc, pabd, - lbuf, psize, lsize, NULL) == 0 && + &labd, psize, lsize, NULL) == 0 && zio_decompress_data(cfunc, pabd, - lbuf2, psize, lsize, NULL) == 0 && + &labd2, psize, lsize, NULL) == 0 && memcmp(lbuf, lbuf2, lsize) == 0) - return (B_TRUE); - return (B_FALSE); + ret = B_TRUE; + + abd_free(&labd2); + abd_free(&labd); + + return (ret); } static uint64_t diff --git a/cmd/zstream/zstream_recompress.c b/cmd/zstream/zstream_recompress.c index 0e5cc9cd8..32ef6fa54 100644 --- a/cmd/zstream/zstream_recompress.c +++ b/cmd/zstream/zstream_recompress.c @@ -259,12 +259,13 @@ zstream_do_recompress(int argc, char *argv[]) /* Read and decompress the payload */ (void) sfread(cbuf, payload_size, stdin); if (dtype != ZIO_COMPRESS_OFF) { - abd_t cabd; + abd_t cabd, dabd; abd_get_from_buf_struct(&cabd, cbuf, payload_size); - if (zio_decompress_data(dtype, &cabd, dbuf, - payload_size, - MIN(bufsz, drrw->drr_logical_size), + abd_get_from_buf_struct(&dabd, dbuf, + MIN(bufsz, drrw->drr_logical_size)); + if (zio_decompress_data(dtype, &cabd, &dabd, + payload_size, abd_get_size(&dabd), NULL) != 0) { warnx("decompression type %d failed " "for ino %llu offset %llu", @@ -274,17 +275,20 @@ zstream_do_recompress(int argc, char *argv[]) exit(4); } payload_size = drrw->drr_logical_size; + abd_free(&dabd); abd_free(&cabd); free(cbuf); } /* Recompress the payload */ if (ctype != ZIO_COMPRESS_OFF) { - abd_t dabd; + abd_t dabd, abd; abd_get_from_buf_struct(&dabd, dbuf, drrw->drr_logical_size); + abd_t *pabd = + abd_get_from_buf_struct(&abd, buf, bufsz); payload_size = P2ROUNDUP(zio_compress_data( - ctype, &dabd, (void **)&buf, + ctype, &dabd, &pabd, drrw->drr_logical_size, level), SPA_MINBLOCKSIZE); if (payload_size != drrw->drr_logical_size) { @@ -296,6 +300,7 @@ zstream_do_recompress(int argc, char *argv[]) drrw->drr_compressiontype = 0; drrw->drr_compressed_size = 0; } + abd_free(&abd); abd_free(&dabd); free(dbuf); } else { diff --git a/include/sys/zio_compress.h b/include/sys/zio_compress.h index d41b5dfd4..31602039a 100644 --- a/include/sys/zio_compress.h +++ b/include/sys/zio_compress.h @@ -173,9 +173,9 @@ extern int zfs_lz4_decompress(abd_t *src, abd_t *dst, size_t s_len, /* * Compress and decompress data if necessary. */ -extern size_t zio_compress_data(enum zio_compress c, abd_t *src, void **dst, +extern size_t zio_compress_data(enum zio_compress c, abd_t *src, abd_t **dst, size_t s_len, uint8_t level); -extern int zio_decompress_data(enum zio_compress c, abd_t *src, void *dst, +extern int zio_decompress_data(enum zio_compress c, abd_t *src, abd_t *abd, size_t s_len, size_t d_len, uint8_t *level); extern int zio_compress_to_feature(enum zio_compress comp); diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 3c657c979..714a30e86 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -1767,12 +1767,12 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj) uint64_t csize; uint64_t lsize = HDR_GET_LSIZE(hdr); uint64_t psize = HDR_GET_PSIZE(hdr); - void *tmpbuf = NULL; abd_t *abd = hdr->b_l1hdr.b_pabd; + boolean_t free_abd = B_FALSE; ASSERT(HDR_EMPTY_OR_LOCKED(hdr)); ASSERT(HDR_AUTHENTICATED(hdr)); - ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); + ASSERT3P(abd, !=, NULL); /* * The MAC is calculated on the compressed data that is stored on disk. @@ -1784,14 +1784,13 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj) */ if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) { - + abd = NULL; csize = zio_compress_data(HDR_GET_COMPRESS(hdr), - hdr->b_l1hdr.b_pabd, &tmpbuf, lsize, hdr->b_complevel); - ASSERT3P(tmpbuf, !=, NULL); + hdr->b_l1hdr.b_pabd, &abd, lsize, hdr->b_complevel); + ASSERT3P(abd, !=, NULL); ASSERT3U(csize, <=, psize); - abd = abd_get_from_buf(tmpbuf, lsize); - abd_take_ownership_of_buf(abd, B_TRUE); abd_zero_off(abd, csize, psize - csize); + free_abd = B_TRUE; } /* @@ -1810,16 +1809,10 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj) if (ret == 0) arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH); - else if (ret != ENOENT) - goto error; + else if (ret == ENOENT) + ret = 0; - if (tmpbuf != NULL) - abd_free(abd); - - return (0); - -error: - if (tmpbuf != NULL) + if (free_abd) abd_free(abd); return (ret); @@ -1836,7 +1829,6 @@ arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb) { int ret; abd_t *cabd = NULL; - void *tmp = NULL; boolean_t no_crypt = B_FALSE; boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS); @@ -1871,17 +1863,14 @@ arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb) * linear buffer and wrapping it in an abd later. */ cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0); - tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr)); ret = zio_decompress_data(HDR_GET_COMPRESS(hdr), - hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr), + hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr), &hdr->b_complevel); if (ret != 0) { - abd_return_buf(cabd, tmp, arc_hdr_size(hdr)); goto error; } - abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr)); arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, arc_hdr_size(hdr), hdr); hdr->b_l1hdr.b_pabd = cabd; @@ -2123,10 +2112,14 @@ arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb, /* Skip byteswapping and checksumming (already done) */ return (0); } else { + abd_t dabd; + abd_get_from_buf_struct(&dabd, buf->b_data, + HDR_GET_LSIZE(hdr)); error = zio_decompress_data(HDR_GET_COMPRESS(hdr), - hdr->b_l1hdr.b_pabd, buf->b_data, + hdr->b_l1hdr.b_pabd, &dabd, HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr), &hdr->b_complevel); + abd_free(&dabd); /* * Absent hardware errors or software bugs, this should @@ -8531,18 +8524,15 @@ l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb) !HDR_COMPRESSION_ENABLED(hdr)) { abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, ARC_HDR_USE_RESERVE); - void *tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr)); ret = zio_decompress_data(HDR_GET_COMPRESS(hdr), - hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr), + hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr), &hdr->b_complevel); if (ret != 0) { - abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr)); arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr); goto error; } - abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr)); arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, arc_hdr_size(hdr), hdr); hdr->b_l1hdr.b_pabd = cabd; @@ -9037,9 +9027,8 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize, } if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) { - size_t bufsize = MAX(size, asize); - void *buf = zio_buf_alloc(bufsize); - uint64_t csize = zio_compress_data(compress, to_write, &buf, + cabd = abd_alloc_for_io(MAX(size, asize), ismd); + uint64_t csize = zio_compress_data(compress, to_write, &cabd, size, hdr->b_complevel); if (csize > psize) { /* @@ -9047,13 +9036,12 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize, * psize. Even if it fits into asize, it does not * matter, since checksum will never match on read. */ - zio_buf_free(buf, bufsize); + abd_free(cabd); return (SET_ERROR(EIO)); } if (asize > csize) - memset((char *)buf + csize, 0, asize - csize); - to_write = cabd = abd_get_from_buf(buf, bufsize); - abd_take_ownership_of_buf(cabd, B_TRUE); + abd_zero_off(cabd, csize, asize - csize); + to_write = cabd; } if (HDR_ENCRYPTED(hdr)) { @@ -10184,7 +10172,6 @@ l2arc_log_blk_read(l2arc_dev_t *dev, { int err = 0; zio_cksum_t cksum; - abd_t *abd = NULL; uint64_t asize; ASSERT(this_lbp != NULL && next_lbp != NULL); @@ -10246,16 +10233,22 @@ l2arc_log_blk_read(l2arc_dev_t *dev, switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) { case ZIO_COMPRESS_OFF: break; - case ZIO_COMPRESS_LZ4: - abd = abd_alloc_for_io(asize, B_TRUE); + case ZIO_COMPRESS_LZ4: { + abd_t *abd = abd_alloc_linear(asize, B_TRUE); abd_copy_from_buf_off(abd, this_lb, 0, asize); - if ((err = zio_decompress_data( + abd_t dabd; + abd_get_from_buf_struct(&dabd, this_lb, sizeof (*this_lb)); + err = zio_decompress_data( L2BLK_GET_COMPRESS((this_lbp)->lbp_prop), - abd, this_lb, asize, sizeof (*this_lb), NULL)) != 0) { + abd, &dabd, asize, sizeof (*this_lb), NULL); + abd_free(&dabd); + abd_free(abd); + if (err != 0) { err = SET_ERROR(EINVAL); goto cleanup; } break; + } default: err = SET_ERROR(EINVAL); goto cleanup; @@ -10272,8 +10265,6 @@ cleanup: l2arc_log_blk_fetch_abort(*next_io); *next_io = NULL; } - if (abd != NULL) - abd_free(abd); return (err); } @@ -10509,7 +10500,7 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb) uint64_t psize, asize; zio_t *wzio; l2arc_lb_abd_buf_t *abd_buf; - uint8_t *tmpbuf = NULL; + abd_t *abd = NULL; l2arc_lb_ptr_buf_t *lb_ptr_buf; VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries); @@ -10532,7 +10523,7 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb) /* try to compress the buffer */ psize = zio_compress_data(ZIO_COMPRESS_LZ4, - abd_buf->abd, (void **) &tmpbuf, sizeof (*lb), 0); + abd_buf->abd, &abd, sizeof (*lb), 0); /* a log block is never entirely zero */ ASSERT(psize != 0); @@ -10558,27 +10549,26 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb) ZIO_CHECKSUM_FLETCHER_4); if (asize < sizeof (*lb)) { /* compression succeeded */ - memset(tmpbuf + psize, 0, asize - psize); + abd_zero_off(abd, psize, asize - psize); L2BLK_SET_COMPRESS( (&l2dhdr->dh_start_lbps[0])->lbp_prop, ZIO_COMPRESS_LZ4); } else { /* compression failed */ - memcpy(tmpbuf, lb, sizeof (*lb)); + abd_copy_from_buf_off(abd, lb, 0, sizeof (*lb)); L2BLK_SET_COMPRESS( (&l2dhdr->dh_start_lbps[0])->lbp_prop, ZIO_COMPRESS_OFF); } /* checksum what we're about to write */ - fletcher_4_native(tmpbuf, asize, NULL, + abd_fletcher_4_native(abd, asize, NULL, &l2dhdr->dh_start_lbps[0].lbp_cksum); abd_free(abd_buf->abd); /* perform the write itself */ - abd_buf->abd = abd_get_from_buf(tmpbuf, sizeof (*lb)); - abd_take_ownership_of_buf(abd_buf->abd, B_TRUE); + abd_buf->abd = abd; wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand, asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE); diff --git a/module/zfs/blkptr.c b/module/zfs/blkptr.c index 6a6f06c73..ac801c2bc 100644 --- a/module/zfs/blkptr.c +++ b/module/zfs/blkptr.c @@ -142,11 +142,13 @@ decode_embedded_bp(const blkptr_t *bp, void *buf, int buflen) if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { uint8_t dstbuf[BPE_PAYLOAD_SIZE]; decode_embedded_bp_compressed(bp, dstbuf); - abd_t dstabd; - abd_get_from_buf_struct(&dstabd, dstbuf, psize); - VERIFY0(zio_decompress_data(BP_GET_COMPRESS(bp), &dstabd, - buf, psize, buflen, NULL)); - abd_free(&dstabd); + abd_t cabd, dabd; + abd_get_from_buf_struct(&cabd, dstbuf, psize); + abd_get_from_buf_struct(&dabd, buf, buflen); + VERIFY0(zio_decompress_data(BP_GET_COMPRESS(bp), &cabd, + &dabd, psize, buflen, NULL)); + abd_free(&dabd); + abd_free(&cabd); } else { ASSERT3U(lsize, ==, psize); decode_embedded_bp_compressed(bp, buf); diff --git a/module/zfs/ddt_zap.c b/module/zfs/ddt_zap.c index e96984b86..d96dc505c 100644 --- a/module/zfs/ddt_zap.c +++ b/module/zfs/ddt_zap.c @@ -83,9 +83,11 @@ ddt_zap_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len) return; } - abd_t sabd; + abd_t sabd, dabd; abd_get_from_buf_struct(&sabd, src, s_len); - VERIFY0(zio_decompress_data(cpfunc, &sabd, dst, s_len, d_len, NULL)); + abd_get_from_buf_struct(&dabd, dst, d_len); + VERIFY0(zio_decompress_data(cpfunc, &sabd, &dabd, s_len, d_len, NULL)); + abd_free(&dabd); abd_free(&sabd); if (((version & DDT_ZAP_COMPRESS_BYTEORDER_MASK) != 0) != diff --git a/module/zfs/dmu_recv.c b/module/zfs/dmu_recv.c index 0119191d7..a1752650f 100644 --- a/module/zfs/dmu_recv.c +++ b/module/zfs/dmu_recv.c @@ -1391,7 +1391,7 @@ do_corrective_recv(struct receive_writer_arg *rwa, struct drr_write *drrw, abd_t *dabd = abd_alloc_linear( drrw->drr_logical_size, B_FALSE); err = zio_decompress_data(drrw->drr_compressiontype, - abd, abd_to_buf(dabd), abd_get_size(abd), + abd, dabd, abd_get_size(abd), abd_get_size(dabd), NULL); if (err != 0) { @@ -1407,9 +1407,8 @@ do_corrective_recv(struct receive_writer_arg *rwa, struct drr_write *drrw, /* Recompress the data */ abd_t *cabd = abd_alloc_linear(BP_GET_PSIZE(bp), B_FALSE); - void *buf = abd_to_buf(cabd); uint64_t csize = zio_compress_data(BP_GET_COMPRESS(bp), - abd, &buf, abd_get_size(abd), + abd, &cabd, abd_get_size(abd), rwa->os->os_complevel); abd_zero_off(cabd, csize, BP_GET_PSIZE(bp) - csize); /* Swap in newly compressed data into the abd */ @@ -2221,7 +2220,7 @@ flush_write_batch_impl(struct receive_writer_arg *rwa) err = zio_decompress_data( drrw->drr_compressiontype, - abd, abd_to_buf(decomp_abd), + abd, decomp_abd, abd_get_size(abd), abd_get_size(decomp_abd), NULL); diff --git a/module/zfs/zio.c b/module/zfs/zio.c index 73252c2da..a841e0a79 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -487,11 +487,9 @@ static void zio_decompress(zio_t *zio, abd_t *data, uint64_t size) { if (zio->io_error == 0) { - void *tmp = abd_borrow_buf(data, size); int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), - zio->io_abd, tmp, zio->io_size, size, + zio->io_abd, data, zio->io_size, size, &zio->io_prop.zp_complevel); - abd_return_buf_copy(data, tmp, size); if (zio_injection_enabled && ret == 0) ret = zio_handle_fault_injection(zio, EINVAL); @@ -538,17 +536,18 @@ zio_decrypt(zio_t *zio, abd_t *data, uint64_t size) * from the indirect block. We decompress it now and * throw away the result after we are finished. */ - tmp = zio_buf_alloc(lsize); + abd_t *abd = abd_alloc_linear(lsize, B_TRUE); ret = zio_decompress_data(BP_GET_COMPRESS(bp), - zio->io_abd, tmp, zio->io_size, lsize, + zio->io_abd, abd, zio->io_size, lsize, &zio->io_prop.zp_complevel); if (ret != 0) { + abd_free(abd); ret = SET_ERROR(EIO); goto error; } - ret = zio_crypt_do_indirect_mac_checksum(B_FALSE, - tmp, lsize, BP_SHOULD_BYTESWAP(bp), mac); - zio_buf_free(tmp, lsize); + ret = zio_crypt_do_indirect_mac_checksum_abd(B_FALSE, + abd, lsize, BP_SHOULD_BYTESWAP(bp), mac); + abd_free(abd); } else { ret = zio_crypt_do_indirect_mac_checksum_abd(B_FALSE, zio->io_abd, size, BP_SHOULD_BYTESWAP(bp), mac); @@ -1866,30 +1865,32 @@ zio_write_compress(zio_t *zio) /* If it's a compressed write that is not raw, compress the buffer. */ if (compress != ZIO_COMPRESS_OFF && !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) { - void *cbuf = NULL; + abd_t *cabd = NULL; if (abd_cmp_zero(zio->io_abd, lsize) == 0) psize = 0; else if (compress == ZIO_COMPRESS_EMPTY) psize = lsize; else - psize = zio_compress_data(compress, zio->io_abd, &cbuf, + psize = zio_compress_data(compress, zio->io_abd, &cabd, lsize, zp->zp_complevel); if (psize == 0) { compress = ZIO_COMPRESS_OFF; } else if (psize >= lsize) { compress = ZIO_COMPRESS_OFF; - if (cbuf != NULL) - zio_buf_free(cbuf, lsize); + if (cabd != NULL) + abd_free(cabd); } else if (!zp->zp_dedup && !zp->zp_encrypt && psize <= BPE_PAYLOAD_SIZE && zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) && spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) { + void *cbuf = abd_borrow_buf_copy(cabd, lsize); encode_embedded_bp_compressed(bp, cbuf, compress, lsize, psize); BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA); BP_SET_TYPE(bp, zio->io_prop.zp_type); BP_SET_LEVEL(bp, zio->io_prop.zp_level); - zio_buf_free(cbuf, lsize); + abd_return_buf(cabd, cbuf, lsize); + abd_free(cabd); BP_SET_LOGICAL_BIRTH(bp, zio->io_txg); zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; ASSERT(spa_feature_is_active(spa, @@ -1908,14 +1909,12 @@ zio_write_compress(zio_t *zio) psize); if (rounded >= lsize) { compress = ZIO_COMPRESS_OFF; - zio_buf_free(cbuf, lsize); + abd_free(cabd); psize = lsize; } else { - abd_t *cdata = abd_get_from_buf(cbuf, lsize); - abd_take_ownership_of_buf(cdata, B_TRUE); - abd_zero_off(cdata, psize, rounded - psize); + abd_zero_off(cabd, psize, rounded - psize); psize = rounded; - zio_push_transform(zio, cdata, + zio_push_transform(zio, cabd, psize, lsize, NULL); } } diff --git a/module/zfs/zio_compress.c b/module/zfs/zio_compress.c index 118003bd2..faf430972 100644 --- a/module/zfs/zio_compress.c +++ b/module/zfs/zio_compress.c @@ -128,7 +128,7 @@ zio_compress_select(spa_t *spa, enum zio_compress child, } size_t -zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len, +zio_compress_data(enum zio_compress c, abd_t *src, abd_t **dst, size_t s_len, uint8_t level) { size_t c_len, d_len; @@ -158,12 +158,9 @@ zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len, } if (*dst == NULL) - *dst = zio_buf_alloc(s_len); + *dst = abd_alloc_sametype(src, s_len); - abd_t dabd; - abd_get_from_buf_struct(&dabd, dst, d_len); - c_len = ci->ci_compress(src, &dabd, s_len, d_len, complevel); - abd_free(&dabd); + c_len = ci->ci_compress(src, *dst, s_len, d_len, complevel); if (c_len > d_len) return (s_len); @@ -173,23 +170,18 @@ zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len, } int -zio_decompress_data(enum zio_compress c, abd_t *src, void *dst, +zio_decompress_data(enum zio_compress c, abd_t *src, abd_t *dst, size_t s_len, size_t d_len, uint8_t *level) { zio_compress_info_t *ci = &zio_compress_table[c]; if ((uint_t)c >= ZIO_COMPRESS_FUNCTIONS || ci->ci_decompress == NULL) return (SET_ERROR(EINVAL)); - abd_t dabd; - abd_get_from_buf_struct(&dabd, dst, d_len); - int err; if (ci->ci_decompress_level != NULL && level != NULL) - err = ci->ci_decompress_level(src, &dabd, s_len, d_len, level); + err = ci->ci_decompress_level(src, dst, s_len, d_len, level); else - err = ci->ci_decompress(src, &dabd, s_len, d_len, ci->ci_level); - - abd_free(&dabd); + err = ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level); /* * Decompression shouldn't fail, because we've already verified diff --git a/module/zstd/zfs_zstd.c b/module/zstd/zfs_zstd.c index 8d1d53d23..e113962f6 100644 --- a/module/zstd/zfs_zstd.c +++ b/module/zstd/zfs_zstd.c @@ -569,9 +569,11 @@ zfs_zstd_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, if (zstd_earlyabort_pass > 0 && zstd_level >= zstd_cutoff_level && s_len >= actual_abort_size) { int pass_len = 1; - abd_t sabd; + abd_t sabd, dabd; abd_get_from_buf_struct(&sabd, s_start, s_len); - pass_len = zfs_lz4_compress(&sabd, d_start, s_len, d_len, 0); + abd_get_from_buf_struct(&dabd, d_start, d_len); + pass_len = zfs_lz4_compress(&sabd, &dabd, s_len, d_len, 0); + abd_free(&dabd); abd_free(&sabd); if (pass_len < d_len) { ZSTDSTAT_BUMP(zstd_stat_lz4pass_allowed);