From bff26b022066c92dfdb405c3c67d841269b5a2a3 Mon Sep 17 00:00:00 2001 From: Richard Yao Date: Mon, 27 Feb 2023 17:41:02 -0500 Subject: [PATCH] Skip memory allocation when compressing holes Hole detection in the zio compression code allows us to opportunistically skip compression on holes. We can go a step further by not doing memory allocations on holes either. Reviewed-by: Brian Atkinson Reviewed-by: Alexander Motin Signed-off-by: Richard Yao Sponsored-by: Wasabi Technology, Inc. Closes #14500 --- include/sys/zio_compress.h | 2 +- module/zfs/arc.c | 16 ++++++++-------- module/zfs/dmu_recv.c | 3 ++- module/zfs/zio.c | 11 +++++++---- module/zfs/zio_compress.c | 7 +++++-- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/include/sys/zio_compress.h b/include/sys/zio_compress.h index 19fc44952..691d7b624 100644 --- a/include/sys/zio_compress.h +++ b/include/sys/zio_compress.h @@ -183,7 +183,7 @@ extern int lz4_decompress_zfs(void *src, void *dst, size_t s_len, size_t d_len, /* * Compress and decompress data if necessary. */ -extern size_t zio_compress_data(enum zio_compress c, abd_t *src, void *dst, +extern size_t zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len, uint8_t level); extern int zio_decompress_data(enum zio_compress c, abd_t *src, void *dst, size_t s_len, size_t d_len, uint8_t *level); diff --git a/module/zfs/arc.c b/module/zfs/arc.c index aa806706d..d851e919e 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -1816,12 +1816,13 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj) */ if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) { - tmpbuf = zio_buf_alloc(lsize); + + csize = zio_compress_data(HDR_GET_COMPRESS(hdr), + hdr->b_l1hdr.b_pabd, &tmpbuf, lsize, hdr->b_complevel); + ASSERT3P(tmpbuf, !=, NULL); + ASSERT3U(csize, <=, psize); abd = abd_get_from_buf(tmpbuf, lsize); abd_take_ownership_of_buf(abd, B_TRUE); - csize = zio_compress_data(HDR_GET_COMPRESS(hdr), - hdr->b_l1hdr.b_pabd, tmpbuf, lsize, hdr->b_complevel); - ASSERT3U(csize, <=, psize); abd_zero_off(abd, csize, psize - csize); } @@ -9402,7 +9403,7 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize, cabd = abd_alloc_for_io(size, ismd); tmp = abd_borrow_buf(cabd, size); - psize = zio_compress_data(compress, to_write, tmp, size, + psize = zio_compress_data(compress, to_write, &tmp, size, hdr->b_complevel); if (psize >= asize) { @@ -10867,12 +10868,11 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb) uint64_t psize, asize; zio_t *wzio; l2arc_lb_abd_buf_t *abd_buf; - uint8_t *tmpbuf; + uint8_t *tmpbuf = NULL; l2arc_lb_ptr_buf_t *lb_ptr_buf; VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries); - tmpbuf = zio_buf_alloc(sizeof (*lb)); abd_buf = zio_buf_alloc(sizeof (*abd_buf)); abd_buf->abd = abd_get_from_buf(lb, sizeof (*lb)); lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP); @@ -10891,7 +10891,7 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb) /* try to compress the buffer */ psize = zio_compress_data(ZIO_COMPRESS_LZ4, - abd_buf->abd, tmpbuf, sizeof (*lb), 0); + abd_buf->abd, (void **) &tmpbuf, sizeof (*lb), 0); /* a log block is never entirely zero */ ASSERT(psize != 0); diff --git a/module/zfs/dmu_recv.c b/module/zfs/dmu_recv.c index ddaa4a5c7..1684d9d92 100644 --- a/module/zfs/dmu_recv.c +++ b/module/zfs/dmu_recv.c @@ -1398,8 +1398,9 @@ do_corrective_recv(struct receive_writer_arg *rwa, struct drr_write *drrw, /* Recompress the data */ abd_t *cabd = abd_alloc_linear(BP_GET_PSIZE(bp), B_FALSE); + void *buf = abd_to_buf(cabd); uint64_t csize = zio_compress_data(BP_GET_COMPRESS(bp), - abd, abd_to_buf(cabd), abd_get_size(abd), + abd, &buf, abd_get_size(abd), rwa->os->os_complevel); abd_zero_off(cabd, csize, BP_GET_PSIZE(bp) - csize); /* Swap in newly compressed data into the abd */ diff --git a/module/zfs/zio.c b/module/zfs/zio.c index 19b222dfe..d17ee60dc 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -1704,12 +1704,15 @@ zio_write_compress(zio_t *zio) /* If it's a compressed write that is not raw, compress the buffer. */ if (compress != ZIO_COMPRESS_OFF && !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) { - void *cbuf = zio_buf_alloc(lsize); - psize = zio_compress_data(compress, zio->io_abd, cbuf, lsize, + void *cbuf = NULL; + psize = zio_compress_data(compress, zio->io_abd, &cbuf, lsize, zp->zp_complevel); - if (psize == 0 || psize >= lsize) { + if (psize == 0) { compress = ZIO_COMPRESS_OFF; - zio_buf_free(cbuf, lsize); + } else if (psize >= lsize) { + compress = ZIO_COMPRESS_OFF; + if (cbuf != NULL) + zio_buf_free(cbuf, lsize); } else if (!zp->zp_dedup && !zp->zp_encrypt && psize <= BPE_PAYLOAD_SIZE && zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) && diff --git a/module/zfs/zio_compress.c b/module/zfs/zio_compress.c index 0fb91ac81..c8a10db74 100644 --- a/module/zfs/zio_compress.c +++ b/module/zfs/zio_compress.c @@ -125,7 +125,7 @@ zio_compress_zeroed_cb(void *data, size_t len, void *private) } size_t -zio_compress_data(enum zio_compress c, abd_t *src, void *dst, size_t s_len, +zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len, uint8_t level) { size_t c_len, d_len; @@ -163,9 +163,12 @@ zio_compress_data(enum zio_compress c, abd_t *src, void *dst, size_t s_len, ASSERT3U(complevel, !=, ZIO_COMPLEVEL_INHERIT); } + if (*dst == NULL) + *dst = zio_buf_alloc(s_len); + /* No compression algorithms can read from ABDs directly */ void *tmp = abd_borrow_buf_copy(src, s_len); - c_len = ci->ci_compress(tmp, dst, s_len, d_len, complevel); + c_len = ci->ci_compress(tmp, *dst, s_len, d_len, complevel); abd_return_buf(src, tmp, s_len); if (c_len > d_len)