c/mirror_zfs
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_zfs.git synced 2026-05-23 19:04:45 +03:00
Code Issues Packages Projects Releases Wiki Activity

Implementation of block cloning for ZFS

Browse Source 
Block Cloning allows to manually clone a file (or a subset of its
blocks) into another (or the same) file by just creating additional
references to the data blocks without copying the data itself.
Those references are kept in the Block Reference Tables (BRTs).

The whole design of block cloning is documented in module/zfs/brt.c.

Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Christian Schwarz <christian.schwarz@nutanix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Rich Ercolani <rincebrain@gmail.com>
Signed-off-by: Pawel Jakub Dawidek <pawel@dawidek.net>
Closes #13392
This commit is contained in:
Pawel Jakub Dawidek
2023-03-10 20:59:53 +01:00
committed by GitHub
parent da19d919a8
commit 67a1b03791
51 changed files with 3480 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
module/zfs/brt.c
+1884
View File
File diff suppressed because it is too large Load Diff
module/zfs/dbuf.c
+87 -39
View File
@@ -26,6 +26,7 @@
* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
* Copyright (c) 2019, Klara Inc.
* Copyright (c) 2019, Allan Jude
* Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
*/
#include <sys/zfs_context.h>
@@ -49,6 +50,7 @@
#include <sys/trace_zfs.h>
#include <sys/callb.h>
#include <sys/abd.h>
#include <sys/brt.h>
#include <sys/vdev.h>
#include <cityhash.h>
#include <sys/spa_impl.h>
@@ -1427,7 +1429,7 @@ dbuf_read_bonus(dmu_buf_impl_t *db, dnode_t *dn, uint32_t flags)
}
static void
dbuf_handle_indirect_hole(dmu_buf_impl_t *db, dnode_t *dn)
dbuf_handle_indirect_hole(dmu_buf_impl_t *db, dnode_t *dn, blkptr_t *dbbp)
{
blkptr_t *bps = db->db.db_data;
uint32_t indbs = 1ULL << dn->dn_indblkshift;
@@ -1436,12 +1438,12 @@ dbuf_handle_indirect_hole(dmu_buf_impl_t *db, dnode_t *dn)
for (int i = 0; i < n_bps; i++) {
blkptr_t *bp = &bps[i];
ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, indbs);
BP_SET_LSIZE(bp, BP_GET_LEVEL(db->db_blkptr) == 1 ?
dn->dn_datablksz : BP_GET_LSIZE(db->db_blkptr));
BP_SET_TYPE(bp, BP_GET_TYPE(db->db_blkptr));
BP_SET_LEVEL(bp, BP_GET_LEVEL(db->db_blkptr) - 1);
BP_SET_BIRTH(bp, db->db_blkptr->blk_birth, 0);
ASSERT3U(BP_GET_LSIZE(dbbp), ==, indbs);
BP_SET_LSIZE(bp, BP_GET_LEVEL(dbbp) == 1 ?
dn->dn_datablksz : BP_GET_LSIZE(dbbp));
BP_SET_TYPE(bp, BP_GET_TYPE(dbbp));
BP_SET_LEVEL(bp, BP_GET_LEVEL(dbbp) - 1);
BP_SET_BIRTH(bp, dbbp->blk_birth, 0);
}
}
@@ -1451,30 +1453,27 @@ dbuf_handle_indirect_hole(dmu_buf_impl_t *db, dnode_t *dn)
* was taken, ENOENT if no action was taken.
*/
static int
dbuf_read_hole(dmu_buf_impl_t *db, dnode_t *dn)
dbuf_read_hole(dmu_buf_impl_t *db, dnode_t *dn, blkptr_t *bp)
{
ASSERT(MUTEX_HELD(&db->db_mtx));
int is_hole = db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr);
int is_hole = bp == NULL || BP_IS_HOLE(bp);
/*
* For level 0 blocks only, if the above check fails:
* Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync()
* processes the delete record and clears the bp while we are waiting
* for the dn_mtx (resulting in a "no" from block_freed).
*/
if (!is_hole && db->db_level == 0) {
is_hole = dnode_block_freed(dn, db->db_blkid) ||
BP_IS_HOLE(db->db_blkptr);
}
if (!is_hole && db->db_level == 0)
is_hole = dnode_block_freed(dn, db->db_blkid) || BP_IS_HOLE(bp);
if (is_hole) {
dbuf_set_data(db, dbuf_alloc_arcbuf(db));
memset(db->db.db_data, 0, db->db.db_size);
if (db->db_blkptr != NULL && db->db_level > 0 &&
BP_IS_HOLE(db->db_blkptr) &&
db->db_blkptr->blk_birth != 0) {
dbuf_handle_indirect_hole(db, dn);
if (bp != NULL && db->db_level > 0 && BP_IS_HOLE(bp) &&
bp->blk_birth != 0) {
dbuf_handle_indirect_hole(db, dn, bp);
}
db->db_state = DB_CACHED;
DTRACE_SET_STATE(db, "hole read satisfied");
@@ -1551,12 +1550,13 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags,
zbookmark_phys_t zb;
uint32_t aflags = ARC_FLAG_NOWAIT;
int err, zio_flags;
blkptr_t bp, *bpp;
DB_DNODE_ENTER(db);
dn = DB_DNODE(db);
ASSERT(!zfs_refcount_is_zero(&db->db_holds));
ASSERT(MUTEX_HELD(&db->db_mtx));
ASSERT(db->db_state == DB_UNCACHED);
ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
ASSERT(db->db_buf == NULL);
ASSERT(db->db_parent == NULL ||
RW_LOCK_HELD(&db->db_parent->db_rwlock));
@@ -1566,16 +1566,46 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags,
goto early_unlock;
}
err = dbuf_read_hole(db, dn);
if (db->db_state == DB_UNCACHED) {
if (db->db_blkptr == NULL) {
bpp = NULL;
} else {
bp = *db->db_blkptr;
bpp = &bp;
}
} else {
struct dirty_leaf *dl;
dbuf_dirty_record_t *dr;
ASSERT3S(db->db_state, ==, DB_NOFILL);
dr = list_head(&db->db_dirty_records);
if (dr == NULL) {
err = EIO;
goto early_unlock;
} else {
dl = &dr->dt.dl;
if (!dl->dr_brtwrite) {
err = EIO;
goto early_unlock;
}
bp = dl->dr_overridden_by;
bpp = &bp;
}
}
err = dbuf_read_hole(db, dn, bpp);
if (err == 0)
goto early_unlock;
ASSERT(bpp != NULL);
/*
* Any attempt to read a redacted block should result in an error. This
* will never happen under normal conditions, but can be useful for
* debugging purposes.
*/
if (BP_IS_REDACTED(db->db_blkptr)) {
if (BP_IS_REDACTED(bpp)) {
ASSERT(dsl_dataset_feature_is_active(
db->db_objset->os_dsl_dataset,
SPA_FEATURE_REDACTED_DATASETS));
@@ -1590,7 +1620,7 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags,
* All bps of an encrypted os should have the encryption bit set.
* If this is not true it indicates tampering and we report an error.
*/
if (db->db_objset->os_encrypted && !BP_USES_CRYPT(db->db_blkptr)) {
if (db->db_objset->os_encrypted && !BP_USES_CRYPT(bpp)) {
spa_log_error(db->db_objset->os_spa, &zb);
zfs_panic_recover("unencrypted block in encrypted "
"object set %llu", dmu_objset_id(db->db_objset));
@@ -1621,15 +1651,14 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags,
if ((flags & DB_RF_NO_DECRYPT) && BP_IS_PROTECTED(db->db_blkptr))
zio_flags |= ZIO_FLAG_RAW;
/*
* The zio layer will copy the provided blkptr later, but we need to
* do this now so that we can release the parent's rwlock. We have to
* do that now so that if dbuf_read_done is called synchronously (on
* The zio layer will copy the provided blkptr later, but we have our
* own copy so that we can release the parent's rwlock. We have to
* do that so that if dbuf_read_done is called synchronously (on
* an l1 cache hit) we don't acquire the db_mtx while holding the
* parent's rwlock, which would be a lock ordering violation.
*/
blkptr_t bp = *db->db_blkptr;
dmu_buf_unlock_parent(db, dblt, tag);
(void) arc_read(zio, db->db_objset->os_spa, &bp,
(void) arc_read(zio, db->db_objset->os_spa, bpp,
dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, zio_flags,
&aflags, &zb);
return (err);
@@ -1731,9 +1760,6 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
*/
ASSERT(!zfs_refcount_is_zero(&db->db_holds));
if (db->db_state == DB_NOFILL)
return (SET_ERROR(EIO));
DB_DNODE_ENTER(db);
dn = DB_DNODE(db);
@@ -1780,13 +1806,13 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
}
DB_DNODE_EXIT(db);
DBUF_STAT_BUMP(hash_hits);
} else if (db->db_state == DB_UNCACHED) {
} else if (db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL) {
boolean_t need_wait = B_FALSE;
db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);
if (zio == NULL &&
db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) {
if (zio == NULL && (db->db_state == DB_NOFILL ||
(db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)))) {
spa_t *spa = dn->dn_objset->os_spa;
zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
need_wait = B_TRUE;
@@ -1913,7 +1939,8 @@ dbuf_unoverride(dbuf_dirty_record_t *dr)
* the buf thawed to save the effort of freezing &
* immediately re-thawing it.
*/
arc_release(dr->dt.dl.dr_data, db);
if (!dr->dt.dl.dr_brtwrite)
arc_release(dr->dt.dl.dr_data, db);
}
/*
@@ -1996,6 +2023,11 @@ dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
db->db_blkid > dn->dn_maxblkid)
dn->dn_maxblkid = db->db_blkid;
dbuf_unoverride(dr);
if (dr->dt.dl.dr_brtwrite) {
ASSERT(db->db.db_data == NULL);
mutex_exit(&db->db_mtx);
continue;
}
} else {
/*
* This dbuf is not dirty in the open context.
@@ -2285,7 +2317,7 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
if (db->db_blkid != DMU_BONUS_BLKID) {
if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) {
dmu_objset_willuse_space(os, db->db.db_size, tx);
}
@@ -2328,8 +2360,9 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
sizeof (dbuf_dirty_record_t),
offsetof(dbuf_dirty_record_t, dr_dirty_node));
}
if (db->db_blkid != DMU_BONUS_BLKID)
if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) {
dr->dr_accounted = db->db.db_size;
}
dr->dr_dbuf = db;
dr->dr_txg = tx->tx_txg;
list_insert_before(&db->db_dirty_records, dr_next, dr);
@@ -2489,6 +2522,7 @@ static boolean_t
dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
{
uint64_t txg = tx->tx_txg;
boolean_t brtwrite;
ASSERT(txg != 0);
@@ -2513,6 +2547,16 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
return (B_FALSE);
ASSERT(dr->dr_dbuf == db);
brtwrite = dr->dt.dl.dr_brtwrite;
if (brtwrite) {
/*
* We are freeing a block that we cloned in the same
* transaction group.
*/
brt_pending_remove(dmu_objset_spa(db->db_objset),
&dr->dt.dl.dr_overridden_by, tx);
}
dnode_t *dn = dr->dr_dnode;
dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
@@ -2542,7 +2586,7 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
mutex_exit(&dn->dn_mtx);
}
if (db->db_state != DB_NOFILL) {
if (db->db_state != DB_NOFILL && !brtwrite) {
dbuf_unoverride(dr);
ASSERT(db->db_buf != NULL);
@@ -2557,7 +2601,8 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
db->db_dirtycnt -= 1;
if (zfs_refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
ASSERT(db->db_state == DB_NOFILL || arc_released(db->db_buf));
ASSERT(db->db_state == DB_NOFILL || brtwrite ||
arc_released(db->db_buf));
dbuf_destroy(db);
return (B_TRUE);
}
@@ -4748,8 +4793,10 @@ dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
ASSERT(db->db_blkid != DMU_BONUS_BLKID);
ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
if (db->db_state != DB_NOFILL) {
if (dr->dt.dl.dr_data != db->db_buf)
if (dr->dt.dl.dr_data != NULL &&
dr->dt.dl.dr_data != db->db_buf) {
arc_buf_destroy(dr->dt.dl.dr_data, db);
}
}
} else {
ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
@@ -5046,7 +5093,8 @@ dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx)
mutex_enter(&db->db_mtx);
dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by,
dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite);
dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite,
dr->dt.dl.dr_brtwrite);
mutex_exit(&db->db_mtx);
} else if (db->db_state == DB_NOFILL) {
ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF ||
module/zfs/ddt.c
+55
View File
@@ -22,6 +22,7 @@
/*
* Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2016 by Delphix. All rights reserved.
* Copyright (c) 2022 by Pawel Jakub Dawidek
*/
#include <sys/zfs_context.h>
@@ -1180,5 +1181,59 @@ ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde)
return (SET_ERROR(ENOENT));
}
/*
* This function is used by Block Cloning (brt.c) to increase reference
* counter for the DDT entry if the block is already in DDT.
*
* Return false if the block, despite having the D bit set, is not present
* in the DDT. Currently this is not possible but might be in the future.
* See the comment below.
*/
boolean_t
ddt_addref(spa_t *spa, const blkptr_t *bp)
{
ddt_t *ddt;
ddt_entry_t *dde;
boolean_t result;
spa_config_enter(spa, SCL_ZIO, FTAG, RW_READER);
ddt = ddt_select(spa, bp);
ddt_enter(ddt);
dde = ddt_lookup(ddt, bp, B_TRUE);
ASSERT(dde != NULL);
if (dde->dde_type < DDT_TYPES) {
ddt_phys_t *ddp;
ASSERT3S(dde->dde_class, <, DDT_CLASSES);
ddp = &dde->dde_phys[BP_GET_NDVAS(bp)];
if (ddp->ddp_refcnt == 0) {
/* This should never happen? */
ddt_phys_fill(ddp, bp);
}
ddt_phys_addref(ddp);
result = B_TRUE;
} else {
/*
* At the time of implementating this if the block has the
* DEDUP flag set it must exist in the DEDUP table, but
* there are many advocates that want ability to remove
* entries from DDT with refcnt=1. If this will happen,
* we may have a block with the DEDUP set, but which doesn't
* have a corresponding entry in the DDT. Be ready.
*/
ASSERT3S(dde->dde_class, ==, DDT_CLASSES);
ddt_remove(ddt, dde);
result = B_FALSE;
}
ddt_exit(ddt);
spa_config_exit(spa, SCL_ZIO, FTAG);
return (result);
}
ZFS_MODULE_PARAM(zfs_dedup, zfs_dedup_, prefetch, INT, ZMOD_RW,
"Enable prefetching dedup-ed blks");
module/zfs/dmu.c
+152 -1
View File
@@ -29,6 +29,7 @@
* Copyright (c) 2019, Klara Inc.
* Copyright (c) 2019, Allan Jude
* Copyright (c) 2022 Hewlett Packard Enterprise Development LP.
* Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
*/
#include <sys/dmu.h>
@@ -52,6 +53,7 @@
#include <sys/sa.h>
#include <sys/zfeature.h>
#include <sys/abd.h>
#include <sys/brt.h>
#include <sys/trace_zfs.h>
#include <sys/zfs_racct.h>
#include <sys/zfs_rlock.h>
@@ -513,7 +515,7 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
zio_t *zio = NULL;
boolean_t missed = B_FALSE;
ASSERT(length <= DMU_MAX_ACCESS);
ASSERT(!read || length <= DMU_MAX_ACCESS);
/*
* Note: We directly notify the prefetch code of this read, so that
@@ -2165,6 +2167,155 @@ restart:
return (err);
}
int
dmu_read_l0_bps(objset_t *os, uint64_t object, uint64_t offset, uint64_t length,
dmu_tx_t *tx, blkptr_t *bps, size_t *nbpsp)
{
dmu_buf_t **dbp, *dbuf;
dmu_buf_impl_t *db;
blkptr_t *bp;
int error, numbufs;
error = dmu_buf_hold_array(os, object, offset, length, FALSE, FTAG,
&numbufs, &dbp);
if (error != 0) {
if (error == ESRCH) {
error = SET_ERROR(ENXIO);
}
return (error);
}
ASSERT3U(numbufs, <=, *nbpsp);
for (int i = 0; i < numbufs; i++) {
dbuf = dbp[i];
db = (dmu_buf_impl_t *)dbuf;
bp = db->db_blkptr;
/*
* If the block is not on the disk yet, it has no BP assigned.
* There is not much we can do...
*/
if (!list_is_empty(&db->db_dirty_records)) {
dbuf_dirty_record_t *dr;
dr = list_head(&db->db_dirty_records);
if (dr->dt.dl.dr_brtwrite) {
/*
* This is very special case where we clone a
* block and in the same transaction group we
* read its BP (most likely to clone the clone).
*/
bp = &dr->dt.dl.dr_overridden_by;
} else {
/*
* The block was modified in the same
* transaction group.
*/
error = SET_ERROR(EAGAIN);
goto out;
}
}
if (bp == NULL) {
/*
* The block was created in this transaction group,
* so it has no BP yet.
*/
error = SET_ERROR(EAGAIN);
goto out;
}
if (dmu_buf_is_dirty(dbuf, tx)) {
error = SET_ERROR(EAGAIN);
goto out;
}
/*
* Make sure we clone only data blocks.
*/
if (BP_IS_METADATA(bp) && !BP_IS_HOLE(bp)) {
error = SET_ERROR(EINVAL);
goto out;
}
bps[i] = *bp;
}
*nbpsp = numbufs;
out:
dmu_buf_rele_array(dbp, numbufs, FTAG);
return (error);
}
void
dmu_brt_clone(objset_t *os, uint64_t object, uint64_t offset, uint64_t length,
dmu_tx_t *tx, const blkptr_t *bps, size_t nbps, boolean_t replay)
{
spa_t *spa;
dmu_buf_t **dbp, *dbuf;
dmu_buf_impl_t *db;
struct dirty_leaf *dl;
dbuf_dirty_record_t *dr;
const blkptr_t *bp;
int numbufs;
spa = os->os_spa;
VERIFY0(dmu_buf_hold_array(os, object, offset, length, FALSE, FTAG,
&numbufs, &dbp));
ASSERT3U(nbps, ==, numbufs);
for (int i = 0; i < numbufs; i++) {
dbuf = dbp[i];
db = (dmu_buf_impl_t *)dbuf;
bp = &bps[i];
ASSERT0(db->db_level);
ASSERT(db->db_blkid != DMU_BONUS_BLKID);
ASSERT(BP_IS_HOLE(bp) || dbuf->db_size == BP_GET_LSIZE(bp));
if (db->db_state == DB_UNCACHED) {
/*
* XXX-PJD: If the dbuf is already cached, calling
* dmu_buf_will_not_fill() will panic on assertion
* (db->db_buf == NULL) in dbuf_clear_data(),
* which is called from dbuf_noread() in DB_NOFILL
* case. I'm not 100% sure this is the right thing
* to do, but it seems to work.
*/
dmu_buf_will_not_fill(dbuf, tx);
}
dr = list_head(&db->db_dirty_records);
ASSERT3U(dr->dr_txg, ==, tx->tx_txg);
dl = &dr->dt.dl;
dl->dr_overridden_by = *bp;
dl->dr_brtwrite = B_TRUE;
dl->dr_override_state = DR_OVERRIDDEN;
if (BP_IS_HOLE(bp)) {
dl->dr_overridden_by.blk_birth = 0;
dl->dr_overridden_by.blk_phys_birth = 0;
} else {
dl->dr_overridden_by.blk_birth = dr->dr_txg;
dl->dr_overridden_by.blk_phys_birth =
BP_PHYSICAL_BIRTH(bp);
}
/*
* When data in embedded into BP there is no need to create
* BRT entry as there is no data block. Just copy the BP as
* it contains the data.
* Also, when replaying ZIL we don't want to bump references
* in the BRT as it was already done during ZIL claim.
*/
if (!replay && !BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) {
brt_pending_add(spa, bp, tx);
}
}
dmu_buf_rele_array(dbp, numbufs, FTAG);
}
void
__dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
{
module/zfs/dmu_tx.c
+38 -9
View File
@@ -349,7 +349,7 @@ dmu_tx_mark_netfree(dmu_tx_t *tx)
}
static void
dmu_tx_hold_free_impl(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
dmu_tx_count_free(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
{
dmu_tx_t *tx = txh->txh_tx;
dnode_t *dn = txh->txh_dnode;
@@ -357,15 +357,11 @@ dmu_tx_hold_free_impl(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
ASSERT(tx->tx_txg == 0);
dmu_tx_count_dnode(txh);
if (off >= (dn->dn_maxblkid + 1) * dn->dn_datablksz)
return;
if (len == DMU_OBJECT_END)
len = (dn->dn_maxblkid + 1) * dn->dn_datablksz - off;
dmu_tx_count_dnode(txh);
/*
* For i/o error checking, we read the first and last level-0
* blocks if they are not aligned, and all the level-1 blocks.
@@ -445,8 +441,10 @@ dmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off, uint64_t len)
txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
object, THT_FREE, off, len);
if (txh != NULL)
(void) dmu_tx_hold_free_impl(txh, off, len);
if (txh != NULL) {
dmu_tx_count_dnode(txh);
dmu_tx_count_free(txh, off, len);
}
}
void
@@ -455,8 +453,35 @@ dmu_tx_hold_free_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, uint64_t len)
dmu_tx_hold_t *txh;
txh = dmu_tx_hold_dnode_impl(tx, dn, THT_FREE, off, len);
if (txh != NULL)
(void) dmu_tx_hold_free_impl(txh, off, len);
if (txh != NULL) {
dmu_tx_count_dnode(txh);
dmu_tx_count_free(txh, off, len);
}
}
static void
dmu_tx_count_clone(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
{
/*
* Reuse dmu_tx_count_free(), it does exactly what we need for clone.
*/
dmu_tx_count_free(txh, off, len);
}
void
dmu_tx_hold_clone_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, int len)
{
dmu_tx_hold_t *txh;
ASSERT0(tx->tx_txg);
ASSERT(len == 0 || UINT64_MAX - off >= len - 1);
txh = dmu_tx_hold_dnode_impl(tx, dn, THT_CLONE, off, len);
if (txh != NULL) {
dmu_tx_count_dnode(txh);
dmu_tx_count_clone(txh, off, len);
}
}
static void
@@ -667,6 +692,10 @@ dmu_tx_dirty_buf(dmu_tx_t *tx, dmu_buf_impl_t *db)
case THT_NEWOBJECT:
match_object = TRUE;
break;
case THT_CLONE:
if (blkid >= beginblk && blkid <= endblk)
match_offset = TRUE;
break;
default:
cmn_err(CE_PANIC, "bad txh_type %d",
txh->txh_type);
module/zfs/dsl_scan.c
+5 -3
View File
@@ -47,6 +47,7 @@
#include <sys/vdev_impl.h>
#include <sys/zil_impl.h>
#include <sys/zio_checksum.h>
#include <sys/brt.h>
#include <sys/ddt.h>
#include <sys/sa.h>
#include <sys/sa_impl.h>
@@ -3499,11 +3500,12 @@ dsl_process_async_destroys(dsl_pool_t *dp, dmu_tx_t *tx)
scn->scn_dedup_frees_this_txg = 0;
/*
* Write out changes to the DDT that may be required as a
* result of the blocks freed. This ensures that the DDT
* is clean when a scrub/resilver runs.
* Write out changes to the DDT and the BRT that may be required
* as a result of the blocks freed. This ensures that the DDT
* and the BRT are clean when a scrub/resilver runs.
*/
ddt_sync(spa, tx->tx_txg);
brt_sync(spa, tx->tx_txg);
}
if (err != 0)
return (err);
module/zfs/spa.c
+39
View File
@@ -52,6 +52,7 @@
#include <sys/dmu_tx.h>
#include <sys/zap.h>
#include <sys/zil.h>
#include <sys/brt.h>
#include <sys/ddt.h>
#include <sys/vdev_impl.h>
#include <sys/vdev_removal.h>
@@ -341,6 +342,12 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp)
spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL,
ddt_get_pool_dedup_ratio(spa), src);
spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONEUSED, NULL,
brt_get_used(spa), src);
spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONESAVED, NULL,
brt_get_saved(spa), src);
spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONERATIO, NULL,
brt_get_ratio(spa), src);
spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL,
rvd->vdev_state, src);
@@ -1707,6 +1714,7 @@ spa_unload(spa_t *spa)
}
ddt_unload(spa);
brt_unload(spa);
spa_unload_log_sm_metadata(spa);
/*
@@ -4414,6 +4422,21 @@ spa_ld_load_dedup_tables(spa_t *spa)
return (0);
}
static int
spa_ld_load_brt(spa_t *spa)
{
int error = 0;
vdev_t *rvd = spa->spa_root_vdev;
error = brt_load(spa);
if (error != 0) {
spa_load_failed(spa, "brt_load failed [error=%d]", error);
return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
}
return (0);
}
static int
spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, const char **ereport)
{
@@ -4895,6 +4918,10 @@ spa_load_impl(spa_t *spa, spa_import_type_t type, const char **ereport)
if (error != 0)
return (error);
error = spa_ld_load_brt(spa);
if (error != 0)
return (error);
/*
* Verify the logs now to make sure we don't have any unexpected errors
* when we claim log blocks later.
@@ -5963,6 +5990,10 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
* Create DDTs (dedup tables).
*/
ddt_create(spa);
/*
* Create BRT table and BRT table object.
*/
brt_create(spa);
spa_update_dspace(spa);
@@ -9138,6 +9169,7 @@ spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx)
&spa->spa_deferred_bpobj, tx);
}
brt_sync(spa, txg);
ddt_sync(spa, txg);
dsl_scan_sync(dp, tx);
svr_sync(spa, tx);
@@ -9262,6 +9294,13 @@ spa_sync(spa_t *spa, uint64_t txg)
spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL,
ZIO_FLAG_CANFAIL);
/*
* Now that there can be no more cloning in this transaction group,
* but we are still before issuing frees, we can process pending BRT
* updates.
*/
brt_pending_apply(spa, txg);
/*
* Lock out configuration changes.
*/
module/zfs/spa_misc.c
+4 -1
View File
@@ -57,6 +57,7 @@
#include <sys/fs/zfs.h>
#include <sys/metaslab_impl.h>
#include <sys/arc.h>
#include <sys/brt.h>
#include <sys/ddt.h>
#include <sys/kstat.h>
#include "zfs_prop.h"
@@ -1834,7 +1835,7 @@ void
spa_update_dspace(spa_t *spa)
{
spa->spa_dspace = metaslab_class_get_dspace(spa_normal_class(spa)) +
ddt_get_dedup_dspace(spa);
ddt_get_dedup_dspace(spa) + brt_get_dspace(spa);
if (spa->spa_nonallocating_dspace > 0) {
/*
* Subtract the space provided by all non-allocating vdevs that
@@ -2410,6 +2411,7 @@ spa_init(spa_mode_t mode)
unique_init();
zfs_btree_init();
metaslab_stat_init();
brt_init();
ddt_init();
zio_init();
dmu_init();
@@ -2446,6 +2448,7 @@ spa_fini(void)
dmu_fini();
zio_fini();
ddt_fini();
brt_fini();
metaslab_stat_fini();
zfs_btree_fini();
unique_fini();
module/zfs/zfs_ioctl.c
+1 -1
View File
@@ -23,7 +23,7 @@
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Portions Copyright 2011 Martin Matuska
* Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved.
* Portions Copyright 2012 Pawel Jakub Dawidek <pawel@dawidek.net>
* Copyright (c) 2012 Pawel Jakub Dawidek
* Copyright (c) 2014, 2016 Joyent, Inc. All rights reserved.
* Copyright 2016 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2014, Joyent, Inc. All rights reserved.
module/zfs/zfs_log.c
+52
View File
@@ -21,6 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2018 by Delphix. All rights reserved.
* Copyright (c) 2022 by Pawel Jakub Dawidek
*/
@@ -891,5 +892,56 @@ zfs_log_acl(zilog_t *zilog, dmu_tx_t *tx, znode_t *zp,
zil_itx_assign(zilog, itx, tx);
}
/*
* Handles TX_CLONE_RANGE transactions.
*/
void
zfs_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype, znode_t *zp,
uint64_t off, uint64_t len, uint64_t blksz, const blkptr_t *bps,
size_t nbps)
{
itx_t *itx;
lr_clone_range_t *lr;
uint64_t partlen, max_log_data;
size_t i, partnbps;
VERIFY(!zil_replaying(zilog, tx));
if (zp->z_unlinked)
return;
max_log_data = zil_max_log_data(zilog, sizeof (lr_clone_range_t));
while (nbps > 0) {
partnbps = MIN(nbps, max_log_data / sizeof (bps[0]));
partlen = 0;
for (i = 0; i < partnbps; i++) {
partlen += BP_GET_LSIZE(&bps[i]);
}
partlen = MIN(partlen, len);
itx = zil_itx_create(txtype,
sizeof (*lr) + sizeof (bps[0]) * partnbps);
lr = (lr_clone_range_t *)&itx->itx_lr;
lr->lr_foid = zp->z_id;
lr->lr_offset = off;
lr->lr_length = partlen;
lr->lr_blksz = blksz;
lr->lr_nbps = partnbps;
memcpy(lr->lr_bps, bps, sizeof (bps[0]) * partnbps);
itx->itx_sync = (zp->z_sync_cnt != 0);
zil_itx_assign(zilog, itx, tx);
bps += partnbps;
ASSERT3U(nbps, >=, partnbps);
nbps -= partnbps;
off += partlen;
ASSERT3U(len, >=, partlen);
len -= partlen;
}
}
ZFS_MODULE_PARAM(zfs, zfs_, immediate_write_sz, S64, ZMOD_RW,
"Largest data block to write to zil");
module/zfs/zfs_quota.c
+1 -2
View File
@@ -20,8 +20,7 @@
*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>.
* All rights reserved.
* Copyright (c) 2011 Pawel Jakub Dawidek
* Copyright (c) 2012, 2015, 2018 by Delphix. All rights reserved.
* Copyright (c) 2014 Integros [integros.com]
* Copyright 2016 Nexenta Systems, Inc. All rights reserved.
module/zfs/zfs_replay.c
+30
View File
@@ -22,6 +22,7 @@
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012 Cyril Plisko. All rights reserved.
* Copyright (c) 2013, 2017 by Delphix. All rights reserved.
* Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
*/
#include <sys/types.h>
@@ -1162,6 +1163,34 @@ zfs_replay_acl(void *arg1, void *arg2, boolean_t byteswap)
return (error);
}
static int
zfs_replay_clone_range(void *arg1, void *arg2, boolean_t byteswap)
{
zfsvfs_t *zfsvfs = arg1;
lr_clone_range_t *lr = arg2;
znode_t *zp;
int error;
if (byteswap)
byteswap_uint64_array(lr, sizeof (*lr));
if ((error = zfs_zget(zfsvfs, lr->lr_foid, &zp)) != 0) {
/*
* Clones can be logged out of order, so don't be surprised if
* the file is gone - just return success.
*/
if (error == ENOENT)
error = 0;
return (error);
}
error = zfs_clone_range_replay(zp, lr->lr_offset, lr->lr_length,
lr->lr_blksz, lr->lr_bps, lr->lr_nbps);
zrele(zp);
return (error);
}
/*
* Callback vectors for replaying records
*/
@@ -1190,4 +1219,5 @@ zil_replay_func_t *const zfs_replay_vector[TX_MAX_TYPE] = {
zfs_replay_setsaxattr, /* TX_SETSAXATTR */
zfs_replay_rename_exchange, /* TX_RENAME_EXCHANGE */
zfs_replay_rename_whiteout, /* TX_RENAME_WHITEOUT */
zfs_replay_clone_range, /* TX_CLONE_RANGE */
};
module/zfs/zfs_vnops.c
+466 -1
View File
@@ -24,6 +24,7 @@
* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
* Copyright (c) 2015 by Chunwei Chen. All rights reserved.
* Copyright 2017 Nexenta Systems, Inc.
* Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
*/
/* Portions Copyright 2007 Jeremy Teo */
@@ -50,6 +51,7 @@
#include <sys/txg.h>
#include <sys/dbuf.h>
#include <sys/policy.h>
#include <sys/zfeature.h>
#include <sys/zfs_vnops.h>
#include <sys/zfs_quota.h>
#include <sys/zfs_vfsops.h>
@@ -501,7 +503,7 @@ zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER);
}
if (zn_rlimit_fsize(zp, uio)) {
if (zn_rlimit_fsize_uio(zp, uio)) {
zfs_rangelock_exit(lr);
zfs_exit(zfsvfs, FTAG);
return (SET_ERROR(EFBIG));
@@ -995,6 +997,467 @@ zfs_get_done(zgd_t *zgd, int error)
kmem_free(zgd, sizeof (zgd_t));
}
static int
zfs_enter_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
{
int error;
/* Swap. Not sure if the order of zfs_enter()s is important. */
if (zfsvfs1 > zfsvfs2) {
zfsvfs_t *tmpzfsvfs;
tmpzfsvfs = zfsvfs2;
zfsvfs2 = zfsvfs1;
zfsvfs1 = tmpzfsvfs;
}
error = zfs_enter(zfsvfs1, tag);
if (error != 0)
return (error);
if (zfsvfs1 != zfsvfs2) {
error = zfs_enter(zfsvfs2, tag);
if (error != 0) {
zfs_exit(zfsvfs1, tag);
return (error);
}
}
return (0);
}
static void
zfs_exit_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
{
zfs_exit(zfsvfs1, tag);
if (zfsvfs1 != zfsvfs2)
zfs_exit(zfsvfs2, tag);
}
/*
* We split each clone request in chunks that can fit into a single ZIL
* log entry. Each ZIL log entry can fit 130816 bytes for a block cloning
* operation (see zil_max_log_data() and zfs_log_clone_range()). This gives
* us room for storing 1022 block pointers.
*
* On success, the function return the number of bytes copied in *lenp.
* Note, it doesn't return how much bytes are left to be copied.
*/
int
zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp,
uint64_t *outoffp, uint64_t *lenp, cred_t *cr)
{
zfsvfs_t *inzfsvfs, *outzfsvfs;
objset_t *inos, *outos;
zfs_locked_range_t *inlr, *outlr;
dmu_buf_impl_t *db;
dmu_tx_t *tx;
zilog_t *zilog;
uint64_t inoff, outoff, len, done;
uint64_t outsize, size;
int error;
int count = 0;
sa_bulk_attr_t bulk[3];
uint64_t mtime[2], ctime[2];
uint64_t uid, gid, projid;
blkptr_t *bps;
size_t maxblocks, nbps;
uint_t inblksz;
uint64_t clear_setid_bits_txg = 0;
inoff = *inoffp;
outoff = *outoffp;
len = *lenp;
done = 0;
inzfsvfs = ZTOZSB(inzp);
outzfsvfs = ZTOZSB(outzp);
inos = inzfsvfs->z_os;
outos = outzfsvfs->z_os;
/*
* Both source and destination have to belong to the same storage pool.
*/
if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) {
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (SET_ERROR(EXDEV));
}
/*
* We need to call zfs_enter() potentially on two different datasets,
* so we need a dedicated function for that.
*/
error = zfs_enter_two(inzfsvfs, outzfsvfs, FTAG);
if (error != 0)
return (error);
ASSERT(!outzfsvfs->z_replay);
error = zfs_verify_zp(inzp);
if (error == 0)
error = zfs_verify_zp(outzp);
if (error != 0) {
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (error);
}
if (!spa_feature_is_enabled(dmu_objset_spa(outos),
SPA_FEATURE_BLOCK_CLONING)) {
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (SET_ERROR(EXDEV));
}
/*
* We don't copy source file's flags that's why we don't allow to clone
* files that are in quarantine.
*/
if (inzp->z_pflags & ZFS_AV_QUARANTINED) {
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (SET_ERROR(EACCES));
}
if (inoff >= inzp->z_size) {
*lenp = 0;
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (0);
}
if (len > inzp->z_size - inoff) {
len = inzp->z_size - inoff;
}
if (len == 0) {
*lenp = 0;
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (0);
}
/*
* Callers might not be able to detect properly that we are read-only,
* so check it explicitly here.
*/
if (zfs_is_readonly(outzfsvfs)) {
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (SET_ERROR(EROFS));
}
/*
* If immutable or not appending then return EPERM.
* Intentionally allow ZFS_READONLY through here.
* See zfs_zaccess_common()
*/
if ((outzp->z_pflags & ZFS_IMMUTABLE) != 0) {
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (SET_ERROR(EPERM));
}
/*
* No overlapping if we are cloning within the same file.
*/
if (inzp == outzp) {
if (inoff < outoff + len && outoff < inoff + len) {
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (SET_ERROR(EINVAL));
}
}
/*
* Maintain predictable lock order.
*/
if (inzp < outzp || (inzp == outzp && inoff < outoff)) {
inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
RL_READER);
outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
RL_WRITER);
} else {
outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
RL_WRITER);
inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
RL_READER);
}
inblksz = inzp->z_blksz;
/*
* We cannot clone into files with different block size.
*/
if (inblksz != outzp->z_blksz && outzp->z_size > inblksz) {
error = SET_ERROR(EXDEV);
goto unlock;
}
/*
* Offsets and len must be at block boundries.
*/
if ((inoff % inblksz) != 0 || (outoff % inblksz) != 0) {
error = SET_ERROR(EXDEV);
goto unlock;
}
/*
* Length must be multipe of blksz, except for the end of the file.
*/
if ((len % inblksz) != 0 &&
(len < inzp->z_size - inoff || len < outzp->z_size - outoff)) {
error = SET_ERROR(EXDEV);
goto unlock;
}
error = zn_rlimit_fsize(outoff + len);
if (error != 0) {
goto unlock;
}
if (inoff >= MAXOFFSET_T || outoff >= MAXOFFSET_T) {
error = SET_ERROR(EFBIG);
goto unlock;
}
SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(outzfsvfs), NULL,
&mtime, 16);
SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(outzfsvfs), NULL,
&ctime, 16);
SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(outzfsvfs), NULL,
&outzp->z_size, 8);
zilog = outzfsvfs->z_log;
maxblocks = zil_max_log_data(zilog, sizeof (lr_clone_range_t)) /
sizeof (bps[0]);
uid = KUID_TO_SUID(ZTOUID(outzp));
gid = KGID_TO_SGID(ZTOGID(outzp));
projid = outzp->z_projid;
bps = kmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP);
/*
* Clone the file in reasonable size chunks. Each chunk is cloned
* in a separate transaction; this keeps the intent log records small
* and allows us to do more fine-grained space accounting.
*/
while (len > 0) {
size = MIN(inblksz * maxblocks, len);
if (zfs_id_overblockquota(outzfsvfs, DMU_USERUSED_OBJECT,
uid) ||
zfs_id_overblockquota(outzfsvfs, DMU_GROUPUSED_OBJECT,
gid) ||
(projid != ZFS_DEFAULT_PROJID &&
zfs_id_overblockquota(outzfsvfs, DMU_PROJECTUSED_OBJECT,
projid))) {
error = SET_ERROR(EDQUOT);
break;
}
/*
* Start a transaction.
*/
tx = dmu_tx_create(outos);
nbps = maxblocks;
error = dmu_read_l0_bps(inos, inzp->z_id, inoff, size, tx, bps,
&nbps);
if (error != 0) {
dmu_tx_abort(tx);
/*
* If we are tyring to clone a block that was created
* in the current transaction group. Return an error,
* so the caller can fallback to just copying the data.
*/
if (error == EAGAIN) {
error = SET_ERROR(EXDEV);
}
break;
}
/*
* Encrypted data is fine as long as it comes from the same
* dataset.
* TODO: We want to extend it in the future to allow cloning to
* datasets with the same keys, like clones or to be able to
* clone a file from a snapshot of an encrypted dataset into the
* dataset itself.
*/
if (BP_IS_PROTECTED(&bps[0])) {
if (inzfsvfs != outzfsvfs) {
dmu_tx_abort(tx);
error = SET_ERROR(EXDEV);
break;
}
}
dmu_tx_hold_sa(tx, outzp->z_sa_hdl, B_FALSE);
db = (dmu_buf_impl_t *)sa_get_db(outzp->z_sa_hdl);
DB_DNODE_ENTER(db);
dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), outoff, size);
DB_DNODE_EXIT(db);
zfs_sa_upgrade_txholds(tx, outzp);
error = dmu_tx_assign(tx, TXG_WAIT);
if (error != 0) {
dmu_tx_abort(tx);
break;
}
/*
* Copy source znode's block size. This only happens on the
* first iteration since zfs_rangelock_reduce() will shrink down
* lr_len to the appropriate size.
*/
if (outlr->lr_length == UINT64_MAX) {
zfs_grow_blocksize(outzp, inblksz, tx);
/*
* Round range lock up to the block boundary, so we
* prevent appends until we are done.
*/
zfs_rangelock_reduce(outlr, outoff,
((len - 1) / inblksz + 1) * inblksz);
}
dmu_brt_clone(outos, outzp->z_id, outoff, size, tx, bps, nbps,
B_FALSE);
zfs_clear_setid_bits_if_necessary(outzfsvfs, outzp, cr,
&clear_setid_bits_txg, tx);
zfs_tstamp_update_setup(outzp, CONTENT_MODIFIED, mtime, ctime);
/*
* Update the file size (zp_size) if it has changed;
* account for possible concurrent updates.
*/
while ((outsize = outzp->z_size) < outoff + size) {
(void) atomic_cas_64(&outzp->z_size, outsize,
outoff + size);
}
error = sa_bulk_update(outzp->z_sa_hdl, bulk, count, tx);
zfs_log_clone_range(zilog, tx, TX_CLONE_RANGE, outzp, outoff,
size, inblksz, bps, nbps);
dmu_tx_commit(tx);
if (error != 0)
break;
inoff += size;
outoff += size;
len -= size;
done += size;
}
kmem_free(bps, sizeof (bps[0]) * maxblocks);
zfs_znode_update_vfs(outzp);
unlock:
zfs_rangelock_exit(outlr);
zfs_rangelock_exit(inlr);
if (done > 0) {
/*
* If we have made at least partial progress, reset the error.
*/
error = 0;
ZFS_ACCESSTIME_STAMP(inzfsvfs, inzp);
if (outos->os_sync == ZFS_SYNC_ALWAYS) {
zil_commit(zilog, outzp->z_id);
}
*inoffp += done;
*outoffp += done;
*lenp = done;
}
zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
return (error);
}
/*
* Usual pattern would be to call zfs_clone_range() from zfs_replay_clone(),
* but we cannot do that, because when replaying we don't have source znode
* available. This is why we need a dedicated replay function.
*/
int
zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz,
const blkptr_t *bps, size_t nbps)
{
zfsvfs_t *zfsvfs;
dmu_buf_impl_t *db;
dmu_tx_t *tx;
int error;
int count = 0;
sa_bulk_attr_t bulk[3];
uint64_t mtime[2], ctime[2];
ASSERT3U(off, <, MAXOFFSET_T);
ASSERT3U(len, >, 0);
ASSERT3U(nbps, >, 0);
zfsvfs = ZTOZSB(zp);
ASSERT(spa_feature_is_enabled(dmu_objset_spa(zfsvfs->z_os),
SPA_FEATURE_BLOCK_CLONING));
if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
return (error);
ASSERT(zfsvfs->z_replay);
ASSERT(!zfs_is_readonly(zfsvfs));
if ((off % blksz) != 0) {
zfs_exit(zfsvfs, FTAG);
return (SET_ERROR(EINVAL));
}
SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
&zp->z_size, 8);
/*
* Start a transaction.
*/
tx = dmu_tx_create(zfsvfs->z_os);
dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
DB_DNODE_ENTER(db);
dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), off, len);
DB_DNODE_EXIT(db);
zfs_sa_upgrade_txholds(tx, zp);
error = dmu_tx_assign(tx, TXG_WAIT);
if (error != 0) {
dmu_tx_abort(tx);
zfs_exit(zfsvfs, FTAG);
return (error);
}
if (zp->z_blksz < blksz)
zfs_grow_blocksize(zp, blksz, tx);
dmu_brt_clone(zfsvfs->z_os, zp->z_id, off, len, tx, bps, nbps, B_TRUE);
zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
if (zp->z_size < off + len)
zp->z_size = off + len;
error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
/*
* zil_replaying() not only check if we are replaying ZIL, but also
* updates the ZIL header to record replay progress.
*/
VERIFY(zil_replaying(zfsvfs->z_log, tx));
dmu_tx_commit(tx);
zfs_znode_update_vfs(zp);
zfs_exit(zfsvfs, FTAG);
return (error);
}
EXPORT_SYMBOL(zfs_access);
EXPORT_SYMBOL(zfs_fsync);
EXPORT_SYMBOL(zfs_holey);
@@ -1002,6 +1465,8 @@ EXPORT_SYMBOL(zfs_read);
EXPORT_SYMBOL(zfs_write);
EXPORT_SYMBOL(zfs_getsecattr);
EXPORT_SYMBOL(zfs_setsecattr);
EXPORT_SYMBOL(zfs_clone_range);
EXPORT_SYMBOL(zfs_clone_range_replay);
ZFS_MODULE_PARAM(zfs_vnops, zfs_vnops_, read_chunk_size, U64, ZMOD_RW,
"Bytes to read per chunk");
module/zfs/zil.c
+111 -15
View File
@@ -43,6 +43,7 @@
#include <sys/metaslab.h>
#include <sys/trace_zfs.h>
#include <sys/abd.h>
#include <sys/brt.h>
#include <sys/wmsum.h>
/*
@@ -578,14 +579,12 @@ zil_claim_log_block(zilog_t *zilog, const blkptr_t *bp, void *tx,
}
static int
zil_claim_log_record(zilog_t *zilog, const lr_t *lrc, void *tx,
uint64_t first_txg)
zil_claim_write(zilog_t *zilog, const lr_t *lrc, void *tx, uint64_t first_txg)
{
lr_write_t *lr = (lr_write_t *)lrc;
int error;
if (lrc->lrc_txtype != TX_WRITE)
return (0);
ASSERT(lrc->lrc_txtype == TX_WRITE);
/*
* If the block is not readable, don't claim it. This can happen
@@ -604,6 +603,57 @@ zil_claim_log_record(zilog_t *zilog, const lr_t *lrc, void *tx,
return (zil_claim_log_block(zilog, &lr->lr_blkptr, tx, first_txg));
}
static int
zil_claim_clone_range(zilog_t *zilog, const lr_t *lrc, void *tx)
{
const lr_clone_range_t *lr = (const lr_clone_range_t *)lrc;
const blkptr_t *bp;
spa_t *spa;
uint_t ii;
ASSERT(lrc->lrc_txtype == TX_CLONE_RANGE);
if (tx == NULL) {
return (0);
}
/*
* XXX: Do we need to byteswap lr?
*/
spa = zilog->zl_spa;
for (ii = 0; ii < lr->lr_nbps; ii++) {
bp = &lr->lr_bps[ii];
/*
* When data in embedded into BP there is no need to create
* BRT entry as there is no data block. Just copy the BP as
* it contains the data.
*/
if (!BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) {
brt_pending_add(spa, bp, tx);
}
}
return (0);
}
static int
zil_claim_log_record(zilog_t *zilog, const lr_t *lrc, void *tx,
uint64_t first_txg)
{
switch (lrc->lrc_txtype) {
case TX_WRITE:
return (zil_claim_write(zilog, lrc, tx, first_txg));
case TX_CLONE_RANGE:
return (zil_claim_clone_range(zilog, lrc, tx));
default:
return (0);
}
}
static int
zil_free_log_block(zilog_t *zilog, const blkptr_t *bp, void *tx,
uint64_t claim_txg)
@@ -616,23 +666,70 @@ zil_free_log_block(zilog_t *zilog, const blkptr_t *bp, void *tx,
}
static int
zil_free_log_record(zilog_t *zilog, const lr_t *lrc, void *tx,
uint64_t claim_txg)
zil_free_write(zilog_t *zilog, const lr_t *lrc, void *tx, uint64_t claim_txg)
{
lr_write_t *lr = (lr_write_t *)lrc;
blkptr_t *bp = &lr->lr_blkptr;
ASSERT(lrc->lrc_txtype == TX_WRITE);
/*
* If we previously claimed it, we need to free it.
*/
if (claim_txg != 0 && lrc->lrc_txtype == TX_WRITE &&
bp->blk_birth >= claim_txg && zil_bp_tree_add(zilog, bp) == 0 &&
!BP_IS_HOLE(bp))
if (bp->blk_birth >= claim_txg && zil_bp_tree_add(zilog, bp) == 0 &&
!BP_IS_HOLE(bp)) {
zio_free(zilog->zl_spa, dmu_tx_get_txg(tx), bp);
}
return (0);
}
static int
zil_free_clone_range(zilog_t *zilog, const lr_t *lrc, void *tx)
{
const lr_clone_range_t *lr = (const lr_clone_range_t *)lrc;
const blkptr_t *bp;
spa_t *spa;
uint_t ii;
ASSERT(lrc->lrc_txtype == TX_CLONE_RANGE);
if (tx == NULL) {
return (0);
}
spa = zilog->zl_spa;
for (ii = 0; ii < lr->lr_nbps; ii++) {
bp = &lr->lr_bps[ii];
if (!BP_IS_HOLE(bp)) {
zio_free(spa, dmu_tx_get_txg(tx), bp);
}
}
return (0);
}
static int
zil_free_log_record(zilog_t *zilog, const lr_t *lrc, void *tx,
uint64_t claim_txg)
{
if (claim_txg == 0) {
return (0);
}
switch (lrc->lrc_txtype) {
case TX_WRITE:
return (zil_free_write(zilog, lrc, tx, claim_txg));
case TX_CLONE_RANGE:
return (zil_free_clone_range(zilog, lrc, tx));
default:
return (0);
}
}
static int
zil_lwb_vdev_compare(const void *x1, const void *x2)
{
@@ -1798,13 +1895,12 @@ zil_lwb_write_issue(zilog_t *zilog, lwb_t *lwb)
}
/*
* Maximum amount of write data that can be put into single log block.
* Maximum amount of data that can be put into single log block.
*/
uint64_t
zil_max_log_data(zilog_t *zilog)
zil_max_log_data(zilog_t *zilog, size_t hdrsize)
{
return (zilog->zl_max_block_size -
sizeof (zil_chain_t) - sizeof (lr_write_t));
return (zilog->zl_max_block_size - sizeof (zil_chain_t) - hdrsize);
}
/*
@@ -1814,7 +1910,7 @@ zil_max_log_data(zilog_t *zilog)
static inline uint64_t
zil_max_waste_space(zilog_t *zilog)
{
return (zil_max_log_data(zilog) / 8);
return (zil_max_log_data(zilog, sizeof (lr_write_t)) / 8);
}
/*
@@ -1887,7 +1983,7 @@ cont:
* For WR_NEED_COPY optimize layout for minimal number of chunks.
*/
lwb_sp = lwb->lwb_sz - lwb->lwb_nused;
max_log_data = zil_max_log_data(zilog);
max_log_data = zil_max_log_data(zilog, sizeof (lr_write_t));
if (reclen > lwb_sp || (reclen + dlen > lwb_sp &&
lwb_sp < zil_max_waste_space(zilog) &&
(dlen % max_log_data == 0 ||
module/zfs/zio.c
+48 -7
View File
@@ -41,6 +41,7 @@
#include <sys/zio_checksum.h>
#include <sys/dmu_objset.h>
#include <sys/arc.h>
#include <sys/brt.h>
#include <sys/ddt.h>
#include <sys/blkptr.h>
#include <sys/zfeature.h>
@@ -1176,12 +1177,14 @@ zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, abd_t *data,
}
void
zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite)
zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite,
boolean_t brtwrite)
{
ASSERT(zio->io_type == ZIO_TYPE_WRITE);
ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
ASSERT(zio->io_stage == ZIO_STAGE_OPEN);
ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa));
ASSERT(!brtwrite || !nopwrite);
/*
* We must reset the io_prop to match the values that existed
@@ -1190,6 +1193,7 @@ zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite)
*/
zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup;
zio->io_prop.zp_nopwrite = nopwrite;
zio->io_prop.zp_brtwrite = brtwrite;
zio->io_prop.zp_copies = copies;
zio->io_bp_override = bp;
}
@@ -1222,7 +1226,8 @@ zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp)
BP_GET_DEDUP(bp) ||
txg != spa->spa_syncing_txg ||
(spa_sync_pass(spa) >= zfs_sync_pass_deferred_free &&
!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP))) {
!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) ||
brt_maybe_exists(spa, bp)) {
metaslab_check_free(spa, bp);
bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);
} else {
@@ -1249,11 +1254,13 @@ zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
arc_freed(spa, bp);
dsl_scan_freed(spa, bp);
if (BP_IS_GANG(bp) || BP_GET_DEDUP(bp)) {
if (BP_IS_GANG(bp) ||
BP_GET_DEDUP(bp) ||
brt_maybe_exists(spa, bp)) {
/*
* GANG and DEDUP blocks can induce a read (for the gang block
* header, or the DDT), so issue them asynchronously so that
* this thread is not tied up.
* GANG, DEDUP and BRT blocks can induce a read (for the gang
* block header, the DDT or the BRT), so issue them
* asynchronously so that this thread is not tied up.
*/
enum zio_stage stage =
ZIO_FREE_PIPELINE | ZIO_STAGE_ISSUE_ASYNC;
@@ -1594,11 +1601,15 @@ zio_write_bp_init(zio_t *zio)
zio_prop_t *zp = &zio->io_prop;
ASSERT(bp->blk_birth != zio->io_txg);
ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0);
*bp = *zio->io_bp_override;
zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
if (zp->zp_brtwrite)
return (zio);
ASSERT(!BP_GET_DEDUP(zio->io_bp_override));
if (BP_IS_EMBEDDED(bp))
return (zio);
@@ -3042,6 +3053,35 @@ zio_nop_write(zio_t *zio)
return (zio);
}
/*
* ==========================================================================
* Block Reference Table
* ==========================================================================
*/
static zio_t *
zio_brt_free(zio_t *zio)
{
blkptr_t *bp;
bp = zio->io_bp;
if (BP_GET_LEVEL(bp) > 0 ||
BP_IS_METADATA(bp) ||
!brt_maybe_exists(zio->io_spa, bp)) {
return (zio);
}
if (!brt_entry_decref(zio->io_spa, bp)) {
/*
* This isn't the last reference, so we cannot free
* the data yet.
*/
zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
}
return (zio);
}
/*
* ==========================================================================
* Dedup
@@ -4894,6 +4934,7 @@ static zio_pipe_stage_t *zio_pipeline[] = {
zio_encrypt,
zio_checksum_generate,
zio_nop_write,
zio_brt_free,
zio_ddt_read_start,
zio_ddt_read_done,
zio_ddt_write,
module/zfs/zvol.c
+55
View File
@@ -482,6 +482,60 @@ zvol_replay_write(void *arg1, void *arg2, boolean_t byteswap)
return (error);
}
/*
* Replay a TX_CLONE_RANGE ZIL transaction that didn't get committed
* after a system failure.
*
* TODO: For now we drop block cloning transations for ZVOLs as they are
* unsupported, but we still need to inform BRT about that as we
* claimed them during pool import.
* This situation can occur when we try to import a pool from a ZFS
* version supporting block cloning for ZVOLs into a system that
* has this ZFS version, that doesn't support block cloning for ZVOLs.
*/
static int
zvol_replay_clone_range(void *arg1, void *arg2, boolean_t byteswap)
{
char name[ZFS_MAX_DATASET_NAME_LEN];
zvol_state_t *zv = arg1;
objset_t *os = zv->zv_objset;
lr_clone_range_t *lr = arg2;
blkptr_t *bp;
dmu_tx_t *tx;
spa_t *spa;
uint_t ii;
int error;
dmu_objset_name(os, name);
cmn_err(CE_WARN, "ZFS dropping block cloning transaction for %s.",
name);
if (byteswap)
byteswap_uint64_array(lr, sizeof (*lr));
tx = dmu_tx_create(os);
error = dmu_tx_assign(tx, TXG_WAIT);
if (error) {
dmu_tx_abort(tx);
return (error);
}
spa = os->os_spa;
for (ii = 0; ii < lr->lr_nbps; ii++) {
bp = &lr->lr_bps[ii];
if (!BP_IS_HOLE(bp)) {
zio_free(spa, dmu_tx_get_txg(tx), bp);
}
}
(void) zil_replaying(zv->zv_zilog, tx);
dmu_tx_commit(tx);
return (0);
}
static int
zvol_replay_err(void *arg1, void *arg2, boolean_t byteswap)
{
@@ -516,6 +570,7 @@ zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE] = {
zvol_replay_err, /* TX_SETSAXATTR */
zvol_replay_err, /* TX_RENAME_EXCHANGE */
zvol_replay_err, /* TX_RENAME_WHITEOUT */
zvol_replay_clone_range /* TX_CLONE_RANGE */
};
/*