2015-07-02 01:23:09 +03:00
|
|
|
#
|
|
|
|
# CDDL HEADER START
|
|
|
|
#
|
|
|
|
# The contents of this file are subject to the terms of the
|
|
|
|
# Common Development and Distribution License (the "License").
|
|
|
|
# You may not use this file except in compliance with the License.
|
|
|
|
#
|
|
|
|
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
2022-07-12 00:16:13 +03:00
|
|
|
# or https://opensource.org/licenses/CDDL-1.0.
|
2015-07-02 01:23:09 +03:00
|
|
|
# See the License for the specific language governing permissions
|
|
|
|
# and limitations under the License.
|
|
|
|
#
|
|
|
|
# When distributing Covered Code, include this CDDL HEADER in each
|
|
|
|
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
|
|
|
# If applicable, add the following below this CDDL HEADER, with the
|
|
|
|
# fields enclosed by brackets "[]" replaced with your own identifying
|
|
|
|
# information: Portions Copyright [yyyy] [name of copyright owner]
|
|
|
|
#
|
|
|
|
# CDDL HEADER END
|
|
|
|
#
|
|
|
|
|
|
|
|
#
|
|
|
|
# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
|
|
|
|
# Use is subject to license terms.
|
|
|
|
#
|
|
|
|
|
|
|
|
#
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 19:48:13 +03:00
|
|
|
# Copyright (c) 2013, 2018 by Delphix. All rights reserved.
|
2020-07-24 03:44:03 +03:00
|
|
|
# Copyright (c) 2020 by Datto Inc. All rights reserved.
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
|
|
|
|
|
|
|
. $STF_SUITE/include/libtest.shlib
|
2017-04-12 00:56:54 +03:00
|
|
|
. $STF_SUITE/include/math.shlib
|
2019-05-08 01:18:44 +03:00
|
|
|
. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
|
2015-07-02 01:23:09 +03:00
|
|
|
. $STF_SUITE/tests/functional/rsend/rsend.cfg
|
|
|
|
|
|
|
|
#
|
|
|
|
# Set up test model which includes various datasets
|
|
|
|
#
|
|
|
|
# @final
|
|
|
|
# @snapB
|
|
|
|
# @init
|
|
|
|
# |
|
|
|
|
# ______ pclone
|
|
|
|
# | /
|
|
|
|
# |@psnap
|
|
|
|
# || @final
|
|
|
|
# ||@final @final @snapC
|
|
|
|
# ||@snapC @snapC @snapB
|
|
|
|
# ||@snapA @snapB @snapA
|
|
|
|
# ||@init @init @init
|
|
|
|
# ||| | |
|
|
|
|
# $pool -------- $FS ------- fs1 ------- fs2
|
|
|
|
# \ \\_____ \ |
|
|
|
|
# vol vol \____ \ @fsnap
|
|
|
|
# | | \ \ \
|
|
|
|
# @init @vsnap | ------------ fclone
|
|
|
|
# @snapA @init \ | |
|
|
|
|
# @final @snapB \ | @init
|
|
|
|
# @snapC vclone @snapA
|
|
|
|
# @final | @final
|
|
|
|
# @init
|
|
|
|
# @snapC
|
|
|
|
# @final
|
|
|
|
#
|
|
|
|
# $1 pool name
|
|
|
|
#
|
|
|
|
function setup_test_model
|
|
|
|
{
|
|
|
|
typeset pool=$1
|
|
|
|
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs create -p $pool/$FS/fs1/fs2
|
2015-07-02 01:23:09 +03:00
|
|
|
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs snapshot $pool@psnap
|
|
|
|
log_must zfs clone $pool@psnap $pool/pclone
|
2015-07-02 01:23:09 +03:00
|
|
|
|
|
|
|
if is_global_zone ; then
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs create -V 16M $pool/vol
|
|
|
|
log_must zfs create -V 16M $pool/$FS/vol
|
2016-01-07 00:22:48 +03:00
|
|
|
block_device_wait
|
2015-07-02 01:23:09 +03:00
|
|
|
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs snapshot $pool/$FS/vol@vsnap
|
|
|
|
log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
|
2016-01-07 00:22:48 +03:00
|
|
|
block_device_wait
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
|
|
|
|
log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
|
|
|
|
log_must zfs snapshot -r $pool@init
|
2015-07-02 01:23:09 +03:00
|
|
|
|
|
|
|
log_must snapshot_tree $pool@snapA
|
|
|
|
log_must snapshot_tree $pool@snapC
|
|
|
|
log_must snapshot_tree $pool/pclone@snapB
|
|
|
|
log_must snapshot_tree $pool/$FS@snapB
|
|
|
|
log_must snapshot_tree $pool/$FS@snapC
|
|
|
|
log_must snapshot_tree $pool/$FS/fs1@snapA
|
|
|
|
log_must snapshot_tree $pool/$FS/fs1@snapB
|
|
|
|
log_must snapshot_tree $pool/$FS/fs1@snapC
|
|
|
|
log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
|
|
|
|
|
|
|
|
if is_global_zone ; then
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs snapshot $pool/vol@snapA
|
|
|
|
log_must zfs snapshot $pool/$FS/vol@snapB
|
|
|
|
log_must zfs snapshot $pool/$FS/vol@snapC
|
|
|
|
log_must zfs snapshot $pool/$FS/vclone@snapC
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs snapshot -r $pool@final
|
2015-07-02 01:23:09 +03:00
|
|
|
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Cleanup the BACKDIR and given pool content and all the sub datasets
|
|
|
|
#
|
|
|
|
# $1 pool name
|
|
|
|
#
|
|
|
|
function cleanup_pool
|
|
|
|
{
|
|
|
|
typeset pool=$1
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must rm -rf $BACKDIR/*
|
2015-07-02 01:23:09 +03:00
|
|
|
|
|
|
|
if is_global_zone ; then
|
2021-12-22 22:05:07 +03:00
|
|
|
#
|
|
|
|
# Linux: Issuing a `df` seems to properly force any negative
|
|
|
|
# dcache entries to be invalidated preventing failures when
|
|
|
|
# accessing the mount point. Additional investigation required.
|
|
|
|
#
|
|
|
|
# https://github.com/openzfs/zfs/issues/6143
|
|
|
|
#
|
2022-03-09 15:39:34 +03:00
|
|
|
log_must eval "df >/dev/null"
|
2017-08-03 18:57:43 +03:00
|
|
|
log_must_busy zfs destroy -Rf $pool
|
2015-07-02 01:23:09 +03:00
|
|
|
else
|
2017-04-12 18:47:42 +03:00
|
|
|
typeset list=$(zfs list -H -r -t all -o name $pool)
|
2015-07-02 01:23:09 +03:00
|
|
|
for ds in $list ; do
|
|
|
|
if [[ $ds != $pool ]] ; then
|
|
|
|
if datasetexists $ds ; then
|
2017-08-03 18:57:43 +03:00
|
|
|
log_must_busy zfs destroy -Rf $ds
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
fi
|
|
|
|
|
|
|
|
typeset mntpnt=$(get_prop mountpoint $pool)
|
|
|
|
if ! ismounted $pool ; then
|
|
|
|
# Make sure mountpoint directory is empty
|
|
|
|
if [[ -d $mntpnt ]]; then
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must rm -rf $mntpnt/*
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs mount $pool
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
if [[ -d $mntpnt ]]; then
|
2017-04-12 18:47:42 +03:00
|
|
|
rm -rf $mntpnt/*
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
OpenZFS 9438 - Holes can lose birth time info if a block has a mix of birth times
As reported by https://github.com/zfsonlinux/zfs/issues/4996, there is
yet another hole birth issue. In this one, if a block is entirely holes,
but the birth times are not all the same, we lose that information by
creating one hole with the current txg as its birth time.
The ZoL PR's fix approach is incorrect. Ultimately, the problem here is
that when you truncate and write a file in the same transaction group,
the dbuf for the indirect block will be zeroed out to deal with the
truncation, and then written for the write. During this process, we will
lose hole birth time information for any holes in the range. In the case
where a dnode is being freed, we need to determine whether the block
should be converted to a higher-level hole in the zio pipeline, and if
so do it when the dnode is being synced out.
Porting Notes:
* The DMU_OBJECT_END change in zfs_znode.c was already applied.
* Added test cases from #5675 provided by @rincebrain for hole_birth
issues. These test cases should be pushed upstream to OpenZFS.
* Updated mk_files which is used by several rsend tests so the
files created are a little more interesting and may contain holes.
Authored by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Matt Ahrens <matt@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Ported-by: Brian Behlendorf <behlendorf1@llnl.gov>
OpenZFS-issue: https://www.illumos.org/issues/9438
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/738e2a3c
External-issue: DLPX-46861
Closes #7746
2016-09-20 20:02:29 +03:00
|
|
|
function cmp_md5s {
|
|
|
|
typeset file1=$1
|
|
|
|
typeset file2=$2
|
|
|
|
|
2022-03-23 03:52:39 +03:00
|
|
|
[ "$(md5digest $file1)" = "$(md5digest $file2)" ]
|
OpenZFS 9438 - Holes can lose birth time info if a block has a mix of birth times
As reported by https://github.com/zfsonlinux/zfs/issues/4996, there is
yet another hole birth issue. In this one, if a block is entirely holes,
but the birth times are not all the same, we lose that information by
creating one hole with the current txg as its birth time.
The ZoL PR's fix approach is incorrect. Ultimately, the problem here is
that when you truncate and write a file in the same transaction group,
the dbuf for the indirect block will be zeroed out to deal with the
truncation, and then written for the write. During this process, we will
lose hole birth time information for any holes in the range. In the case
where a dnode is being freed, we need to determine whether the block
should be converted to a higher-level hole in the zio pipeline, and if
so do it when the dnode is being synced out.
Porting Notes:
* The DMU_OBJECT_END change in zfs_znode.c was already applied.
* Added test cases from #5675 provided by @rincebrain for hole_birth
issues. These test cases should be pushed upstream to OpenZFS.
* Updated mk_files which is used by several rsend tests so the
files created are a little more interesting and may contain holes.
Authored by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Matt Ahrens <matt@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Ported-by: Brian Behlendorf <behlendorf1@llnl.gov>
OpenZFS-issue: https://www.illumos.org/issues/9438
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/738e2a3c
External-issue: DLPX-46861
Closes #7746
2016-09-20 20:02:29 +03:00
|
|
|
}
|
|
|
|
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
|
|
|
# Detect if the given two filesystems have same sub-datasets
|
|
|
|
#
|
|
|
|
# $1 source filesystem
|
|
|
|
# $2 destination filesystem
|
|
|
|
#
|
|
|
|
function cmp_ds_subs
|
|
|
|
{
|
|
|
|
typeset src_fs=$1
|
|
|
|
typeset dst_fs=$2
|
|
|
|
|
2022-03-23 03:52:39 +03:00
|
|
|
diff \
|
|
|
|
<(zfs list -rHt all -o name $src_fs | sed "s:^$src_fs:PREFIX:g") \
|
|
|
|
<(zfs list -rHt all -o name $dst_fs | sed "s:^$dst_fs:PREFIX:g")
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#
|
2017-01-03 20:31:18 +03:00
|
|
|
# Compare all the directories and files in two filesystems
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
|
|
|
# $1 source filesystem
|
|
|
|
# $2 destination filesystem
|
|
|
|
#
|
|
|
|
function cmp_ds_cont
|
|
|
|
{
|
|
|
|
typeset src_fs=$1
|
|
|
|
typeset dst_fs=$2
|
|
|
|
|
|
|
|
typeset srcdir dstdir
|
|
|
|
srcdir=$(get_prop mountpoint $src_fs)
|
|
|
|
dstdir=$(get_prop mountpoint $dst_fs)
|
|
|
|
|
2022-03-01 21:05:32 +03:00
|
|
|
replay_directory_diff $srcdir $dstdir
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Compare the given two dataset properties
|
|
|
|
#
|
|
|
|
# $1 dataset 1
|
|
|
|
# $2 dataset 2
|
2022-03-23 23:07:33 +03:00
|
|
|
# $3 -n == don't track property source
|
|
|
|
# $4 -n == don't track the origin property
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
|
|
|
function cmp_ds_prop
|
|
|
|
{
|
|
|
|
typeset dtst1=$1
|
|
|
|
typeset dtst2=$2
|
2022-03-23 23:07:33 +03:00
|
|
|
typeset nosource=$3
|
|
|
|
typeset noorigin=$4
|
|
|
|
typeset source=",source"; [ -n "$nosource" ] && source=
|
|
|
|
typeset origin=",origin"; [ -n "$noorigin" ] && origin=
|
|
|
|
typeset props="type$origin,volblocksize,acltype,dnodesize"
|
2022-03-23 03:52:39 +03:00
|
|
|
props+=",atime,canmount,checksum,compression,copies,devices"
|
|
|
|
props+=",exec,quota,readonly,recordsize,reservation,setuid"
|
|
|
|
props+=",snapdir,version,volsize,xattr,mountpoint"
|
2020-01-31 19:52:29 +03:00
|
|
|
if is_freebsd; then
|
2022-03-23 03:52:39 +03:00
|
|
|
props+=",jailed"
|
2020-01-31 19:52:29 +03:00
|
|
|
else
|
2022-03-23 03:52:39 +03:00
|
|
|
props+=",zoned"
|
2020-01-31 19:52:29 +03:00
|
|
|
fi
|
|
|
|
|
2022-03-23 03:52:39 +03:00
|
|
|
diff \
|
2022-03-15 03:00:03 +03:00
|
|
|
<(zfs get -Ho property,value$source $props $dtst1 | sed -e "s:$dtst1:PREFIX:g" -e 's/^origin [^@]*/origin POOL/' -e 's/ inherited from [^/]*/ inherited from POOL/') \
|
|
|
|
<(zfs get -Ho property,value$source $props $dtst2 | sed -e "s:$dtst2:PREFIX:g" -e 's/^origin [^@]*/origin POOL/' -e 's/ inherited from [^/]*/ inherited from POOL/')
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Random create directories and files
|
|
|
|
#
|
|
|
|
# $1 directory
|
|
|
|
#
|
|
|
|
function random_tree
|
|
|
|
{
|
|
|
|
typeset dir=$1
|
|
|
|
|
|
|
|
if [[ -d $dir ]]; then
|
2017-04-06 03:18:22 +03:00
|
|
|
rm -rf $dir
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
2017-04-06 03:18:22 +03:00
|
|
|
mkdir -p $dir
|
2015-07-02 01:23:09 +03:00
|
|
|
typeset -i ret=$?
|
|
|
|
|
|
|
|
typeset -i nl nd nf
|
|
|
|
((nl = RANDOM % 6 + 1))
|
|
|
|
((nd = RANDOM % 3 ))
|
|
|
|
((nf = RANDOM % 5 ))
|
2017-04-06 03:18:22 +03:00
|
|
|
mktree -b $dir -l $nl -d $nd -f $nf
|
2015-07-02 01:23:09 +03:00
|
|
|
((ret |= $?))
|
|
|
|
|
|
|
|
return $ret
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Put data in filesystem and take snapshot
|
|
|
|
#
|
|
|
|
# $1 snapshot name
|
|
|
|
#
|
|
|
|
function snapshot_tree
|
|
|
|
{
|
|
|
|
typeset snap=$1
|
|
|
|
typeset ds=${snap%%@*}
|
|
|
|
typeset type=$(get_prop "type" $ds)
|
|
|
|
|
|
|
|
typeset -i ret=0
|
|
|
|
if [[ $type == "filesystem" ]]; then
|
|
|
|
typeset mntpnt=$(get_prop mountpoint $ds)
|
|
|
|
|
|
|
|
if ((ret == 0)) ; then
|
|
|
|
eval random_tree $mntpnt/${snap##$ds}
|
|
|
|
((ret |= $?))
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ((ret == 0)) ; then
|
2017-04-06 03:18:22 +03:00
|
|
|
zfs snapshot $snap
|
2015-07-02 01:23:09 +03:00
|
|
|
((ret |= $?))
|
|
|
|
fi
|
|
|
|
|
|
|
|
return $ret
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Destroy the given snapshot and stuff
|
|
|
|
#
|
|
|
|
# $1 snapshot
|
|
|
|
#
|
|
|
|
function destroy_tree
|
|
|
|
{
|
|
|
|
typeset -i ret=0
|
|
|
|
typeset snap
|
|
|
|
for snap in "$@" ; do
|
2017-08-03 18:57:43 +03:00
|
|
|
log_must_busy zfs destroy $snap
|
2015-07-02 01:23:09 +03:00
|
|
|
|
|
|
|
typeset ds=${snap%%@*}
|
|
|
|
typeset type=$(get_prop "type" $ds)
|
|
|
|
if [[ $type == "filesystem" ]]; then
|
|
|
|
typeset mntpnt=$(get_prop mountpoint $ds)
|
2017-08-03 18:57:43 +03:00
|
|
|
if [[ -n $mntpnt ]]; then
|
|
|
|
rm -rf $mntpnt/$snap
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Get all the sub-datasets of give dataset with specific suffix
|
|
|
|
#
|
|
|
|
# $1 Given dataset
|
|
|
|
# $2 Suffix
|
|
|
|
#
|
|
|
|
function getds_with_suffix
|
|
|
|
{
|
|
|
|
typeset ds=$1
|
|
|
|
typeset suffix=$2
|
|
|
|
|
2022-03-23 03:52:39 +03:00
|
|
|
zfs list -rHt all -o name $ds | grep "$suffix$"
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#
|
2019-08-31 02:52:00 +03:00
|
|
|
# Output inherited properties which is edited for file system
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
|
|
|
function fs_inherit_prop
|
|
|
|
{
|
|
|
|
typeset fs_prop
|
|
|
|
if is_global_zone ; then
|
2017-04-06 03:18:22 +03:00
|
|
|
fs_prop=$(zfs inherit 2>&1 | \
|
|
|
|
awk '$2=="YES" && $3=="YES" {print $1}')
|
2015-07-02 01:23:09 +03:00
|
|
|
if ! is_te_enabled ; then
|
2017-04-06 03:18:22 +03:00
|
|
|
fs_prop=$(echo $fs_prop | grep -v "mlslabel")
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
else
|
2017-04-06 03:18:22 +03:00
|
|
|
fs_prop=$(zfs inherit 2>&1 | \
|
2022-03-12 01:54:08 +03:00
|
|
|
awk '$2=="YES" && $3=="YES" && !/devices|mlslabel|sharenfs|sharesmb|zoned/ {print $1}')
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
|
2017-04-06 03:18:22 +03:00
|
|
|
echo $fs_prop
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Output inherited properties for volume
|
|
|
|
#
|
|
|
|
function vol_inherit_prop
|
|
|
|
{
|
2022-03-23 03:52:39 +03:00
|
|
|
echo checksum readonly
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Get the destination dataset to compare
|
|
|
|
#
|
|
|
|
function get_dst_ds
|
|
|
|
{
|
|
|
|
typeset srcfs=$1
|
|
|
|
typeset dstfs=$2
|
|
|
|
|
|
|
|
#
|
|
|
|
# If the srcfs is not pool
|
|
|
|
#
|
2017-04-06 03:18:22 +03:00
|
|
|
if ! zpool list $srcfs > /dev/null 2>&1 ; then
|
2015-07-02 01:23:09 +03:00
|
|
|
eval dstfs="$dstfs/${srcfs#*/}"
|
|
|
|
fi
|
|
|
|
|
2017-04-06 03:18:22 +03:00
|
|
|
echo $dstfs
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Make test files
|
|
|
|
#
|
|
|
|
# $1 Number of files to create
|
|
|
|
# $2 Maximum file size
|
|
|
|
# $3 File ID offset
|
|
|
|
# $4 File system to create the files on
|
|
|
|
#
|
|
|
|
function mk_files
|
|
|
|
{
|
|
|
|
nfiles=$1
|
|
|
|
maxsize=$2
|
|
|
|
file_id_offset=$3
|
|
|
|
fs=$4
|
OpenZFS 9438 - Holes can lose birth time info if a block has a mix of birth times
As reported by https://github.com/zfsonlinux/zfs/issues/4996, there is
yet another hole birth issue. In this one, if a block is entirely holes,
but the birth times are not all the same, we lose that information by
creating one hole with the current txg as its birth time.
The ZoL PR's fix approach is incorrect. Ultimately, the problem here is
that when you truncate and write a file in the same transaction group,
the dbuf for the indirect block will be zeroed out to deal with the
truncation, and then written for the write. During this process, we will
lose hole birth time information for any holes in the range. In the case
where a dnode is being freed, we need to determine whether the block
should be converted to a higher-level hole in the zio pipeline, and if
so do it when the dnode is being synced out.
Porting Notes:
* The DMU_OBJECT_END change in zfs_znode.c was already applied.
* Added test cases from #5675 provided by @rincebrain for hole_birth
issues. These test cases should be pushed upstream to OpenZFS.
* Updated mk_files which is used by several rsend tests so the
files created are a little more interesting and may contain holes.
Authored by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Matt Ahrens <matt@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Ported-by: Brian Behlendorf <behlendorf1@llnl.gov>
OpenZFS-issue: https://www.illumos.org/issues/9438
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/738e2a3c
External-issue: DLPX-46861
Closes #7746
2016-09-20 20:02:29 +03:00
|
|
|
bs=512
|
2015-07-02 01:23:09 +03:00
|
|
|
|
|
|
|
for ((i=0; i<$nfiles; i=i+1)); do
|
OpenZFS 9438 - Holes can lose birth time info if a block has a mix of birth times
As reported by https://github.com/zfsonlinux/zfs/issues/4996, there is
yet another hole birth issue. In this one, if a block is entirely holes,
but the birth times are not all the same, we lose that information by
creating one hole with the current txg as its birth time.
The ZoL PR's fix approach is incorrect. Ultimately, the problem here is
that when you truncate and write a file in the same transaction group,
the dbuf for the indirect block will be zeroed out to deal with the
truncation, and then written for the write. During this process, we will
lose hole birth time information for any holes in the range. In the case
where a dnode is being freed, we need to determine whether the block
should be converted to a higher-level hole in the zio pipeline, and if
so do it when the dnode is being synced out.
Porting Notes:
* The DMU_OBJECT_END change in zfs_znode.c was already applied.
* Added test cases from #5675 provided by @rincebrain for hole_birth
issues. These test cases should be pushed upstream to OpenZFS.
* Updated mk_files which is used by several rsend tests so the
files created are a little more interesting and may contain holes.
Authored by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Matt Ahrens <matt@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Robert Mustacchi <rm@joyent.com>
Ported-by: Brian Behlendorf <behlendorf1@llnl.gov>
OpenZFS-issue: https://www.illumos.org/issues/9438
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/738e2a3c
External-issue: DLPX-46861
Closes #7746
2016-09-20 20:02:29 +03:00
|
|
|
file_name="/$fs/file-$maxsize-$((i+$file_id_offset))"
|
|
|
|
file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
|
|
|
|
|
|
|
|
#
|
|
|
|
# Create an interesting mix of files which contain both
|
|
|
|
# data blocks and holes for more realistic test coverage.
|
|
|
|
# Half the files are created as sparse then partially filled,
|
|
|
|
# the other half is dense then a hole is punched in the file.
|
|
|
|
#
|
|
|
|
if [ $((RANDOM % 2)) -eq 0 ]; then
|
|
|
|
truncate -s $file_size $file_name || \
|
|
|
|
log_fail "Failed to create $file_name"
|
|
|
|
dd if=/dev/urandom of=$file_name \
|
|
|
|
bs=$bs count=$(($file_size / 2 / $bs)) \
|
|
|
|
seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
|
|
|
|
conv=notrunc >/dev/null 2>&1 || \
|
|
|
|
log_fail "Failed to create $file_name"
|
|
|
|
else
|
|
|
|
dd if=/dev/urandom of=$file_name \
|
|
|
|
bs=$file_size count=1 >/dev/null 2>&1 || \
|
|
|
|
log_fail "Failed to create $file_name"
|
|
|
|
dd if=/dev/zero of=$file_name \
|
|
|
|
bs=$bs count=$(($file_size / 2 / $bs)) \
|
|
|
|
seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
|
|
|
|
conv=notrunc >/dev/null 2>&1 || \
|
|
|
|
log_fail "Failed to create $file_name"
|
|
|
|
fi
|
2015-07-02 01:23:09 +03:00
|
|
|
done
|
2017-04-06 03:18:22 +03:00
|
|
|
echo Created $nfiles files of random sizes up to $maxsize bytes
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Remove test files
|
|
|
|
#
|
|
|
|
# $1 Number of files to remove
|
|
|
|
# $2 Maximum file size
|
|
|
|
# $3 File ID offset
|
|
|
|
# $4 File system to remove the files from
|
|
|
|
#
|
|
|
|
function rm_files
|
|
|
|
{
|
|
|
|
nfiles=$1
|
|
|
|
maxsize=$2
|
|
|
|
file_id_offset=$3
|
|
|
|
fs=$4
|
|
|
|
|
|
|
|
for ((i=0; i<$nfiles; i=i+1)); do
|
2017-04-06 03:18:22 +03:00
|
|
|
rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
|
2015-07-02 01:23:09 +03:00
|
|
|
done
|
2017-04-06 03:18:22 +03:00
|
|
|
echo Removed $nfiles files of random sizes up to $maxsize bytes
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
2019-04-13 00:28:04 +03:00
|
|
|
#
|
|
|
|
# Simulate a random set of operations which could be reasonably expected
|
|
|
|
# to occur on an average filesystem.
|
|
|
|
#
|
|
|
|
# $1 Number of files to modify
|
|
|
|
# $2 Maximum file size
|
|
|
|
# $3 File system to modify the file on
|
2019-04-10 20:17:36 +03:00
|
|
|
# $4 Enabled xattrs (optional)
|
2019-04-13 00:28:04 +03:00
|
|
|
#
|
|
|
|
function churn_files
|
|
|
|
{
|
|
|
|
nfiles=$1
|
|
|
|
maxsize=$2
|
|
|
|
fs=$3
|
2019-04-10 20:17:36 +03:00
|
|
|
xattrs=${4:-1}
|
2019-04-13 00:28:04 +03:00
|
|
|
|
|
|
|
#
|
|
|
|
# Remove roughly half of the files in order to make it more
|
|
|
|
# likely that a dnode will be reallocated.
|
|
|
|
#
|
|
|
|
for ((i=0; i<$nfiles; i=i+1)); do
|
|
|
|
file_name="/$fs/file-$i"
|
|
|
|
|
|
|
|
if [[ -e $file_name ]]; then
|
|
|
|
if [ $((RANDOM % 2)) -eq 0 ]; then
|
|
|
|
rm $file_name || \
|
|
|
|
log_fail "Failed to remove $file_name"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
|
|
|
#
|
|
|
|
# Remount the filesystem to simulate normal usage. This resets
|
|
|
|
# the last allocated object id allowing for new objects to be
|
|
|
|
# reallocated in the locations of previously freed objects.
|
|
|
|
#
|
|
|
|
log_must zfs unmount $fs
|
|
|
|
log_must zfs mount $fs
|
|
|
|
|
|
|
|
for i in {0..$nfiles}; do
|
|
|
|
file_name="/$fs/file-$i"
|
|
|
|
file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
|
|
|
|
|
|
|
|
#
|
|
|
|
# When the file exists modify it in one of five ways to
|
|
|
|
# simulate normal usage:
|
|
|
|
# - (20%) Remove and set and extended attribute on the file
|
|
|
|
# - (20%) Overwrite the existing file
|
|
|
|
# - (20%) Truncate the existing file to a random length
|
|
|
|
# - (20%) Truncate the existing file to zero length
|
|
|
|
# - (20%) Remove the file
|
|
|
|
#
|
|
|
|
# Otherwise create the missing file. 20% of the created
|
|
|
|
# files will be small and use embedded block pointers, the
|
|
|
|
# remainder with have random sizes up to the maximum size.
|
|
|
|
# Three extended attributes are attached to all of the files.
|
|
|
|
#
|
|
|
|
if [[ -e $file_name ]]; then
|
|
|
|
value=$((RANDOM % 5))
|
2019-04-10 20:17:36 +03:00
|
|
|
if [ $value -eq 0 -a $xattrs -ne 0 ]; then
|
2019-04-13 00:28:04 +03:00
|
|
|
attrname="testattr$((RANDOM % 3))"
|
2019-05-08 01:18:44 +03:00
|
|
|
attrlen="$(((RANDOM % 1000) + 1))"
|
|
|
|
attrvalue="$(random_string VALID_NAME_CHAR \
|
|
|
|
$attrlen)"
|
2020-01-11 00:24:59 +03:00
|
|
|
rm_xattr $attrname $file_name || \
|
2019-04-13 00:28:04 +03:00
|
|
|
log_fail "Failed to remove $attrname"
|
2020-01-11 00:24:59 +03:00
|
|
|
set_xattr $attrname "$attrvalue" $file_name || \
|
2019-04-13 00:28:04 +03:00
|
|
|
log_fail "Failed to set $attrname"
|
|
|
|
elif [ $value -eq 1 ]; then
|
|
|
|
dd if=/dev/urandom of=$file_name \
|
|
|
|
bs=$file_size count=1 >/dev/null 2>&1 || \
|
|
|
|
log_fail "Failed to overwrite $file_name"
|
|
|
|
elif [ $value -eq 2 ]; then
|
|
|
|
truncate -s $file_size $file_name || \
|
|
|
|
log_fail "Failed to truncate $file_name"
|
|
|
|
elif [ $value -eq 3 ]; then
|
|
|
|
truncate -s 0 $file_name || \
|
|
|
|
log_fail "Failed to truncate $file_name"
|
|
|
|
else
|
|
|
|
rm $file_name || \
|
|
|
|
log_fail "Failed to remove $file_name"
|
|
|
|
fi
|
|
|
|
else
|
|
|
|
if [ $((RANDOM % 5)) -eq 0 ]; then
|
|
|
|
file_size=$((($RANDOM % 64) + 1))
|
|
|
|
fi
|
|
|
|
|
|
|
|
dd if=/dev/urandom of=$file_name \
|
|
|
|
bs=$file_size count=1 >/dev/null 2>&1 || \
|
|
|
|
log_fail "Failed to create $file_name"
|
|
|
|
|
2019-04-10 20:17:36 +03:00
|
|
|
if [ $xattrs -ne 0 ]; then
|
|
|
|
for j in {0..2}; do
|
|
|
|
attrname="testattr$j"
|
2019-05-08 01:18:44 +03:00
|
|
|
attrlen="$(((RANDOM % 1000) + 1))"
|
|
|
|
attrvalue="$(random_string \
|
|
|
|
VALID_NAME_CHAR $attrlen)"
|
2020-01-11 00:24:59 +03:00
|
|
|
set_xattr $attrname \
|
|
|
|
"$attrvalue" $file_name || \
|
2019-05-08 01:18:44 +03:00
|
|
|
log_fail "Failed to set $attrname"
|
2019-04-10 20:17:36 +03:00
|
|
|
done
|
|
|
|
fi
|
2019-04-13 00:28:04 +03:00
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
2020-01-10 21:16:58 +03:00
|
|
|
# Mess up a send file's contents
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
2020-01-10 21:16:58 +03:00
|
|
|
# $1 The send file path
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
2020-01-10 21:16:58 +03:00
|
|
|
function mess_send_file
|
2015-07-02 01:23:09 +03:00
|
|
|
{
|
2022-03-13 23:16:49 +03:00
|
|
|
typeset -i minsize=2072
|
2015-07-02 01:23:09 +03:00
|
|
|
file=$1
|
|
|
|
|
2019-12-18 23:29:43 +03:00
|
|
|
filesize=$(stat_size $file)
|
2022-03-13 23:16:49 +03:00
|
|
|
if [ $filesize -lt $minsize ]; then
|
|
|
|
log_fail "Send file too small: $filesize < $minsize"
|
|
|
|
fi
|
2019-12-18 23:29:43 +03:00
|
|
|
|
2022-03-13 23:16:49 +03:00
|
|
|
# Truncate the send stream at a random offset after the DRR_BEGIN
|
|
|
|
# record (beyond 2072 bytes), any smaller than this and the receiving
|
|
|
|
# system won't have enough info to create the partial dataset at all.
|
|
|
|
# We use zstream dump to verify there is an intact DRR_BEGIN record.
|
|
|
|
offset=$(((($RANDOM * $RANDOM) % ($filesize - $minsize)) + $minsize))
|
2021-06-24 19:38:33 +03:00
|
|
|
nr_begins=$(head -c $offset $file | zstream dump | \
|
2022-03-12 01:54:08 +03:00
|
|
|
awk '/DRR_BEGIN/ { print $5 }')
|
|
|
|
log_must [ "$nr_begins" -eq 1 ]
|
2020-01-10 21:16:58 +03:00
|
|
|
|
2015-07-02 01:23:09 +03:00
|
|
|
if (($RANDOM % 7 <= 1)); then
|
|
|
|
#
|
|
|
|
# We corrupt 2 bytes to minimize the chance that we
|
|
|
|
# write the same value that's already there.
|
|
|
|
#
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must eval "dd if=/dev/urandom of=$file conv=notrunc " \
|
2016-01-07 00:22:48 +03:00
|
|
|
"bs=1 count=2 seek=$offset >/dev/null 2>&1"
|
2015-07-02 01:23:09 +03:00
|
|
|
else
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must truncate -s $offset $file
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Diff the send/receive filesystems
|
|
|
|
#
|
|
|
|
# $1 The sent filesystem
|
|
|
|
# $2 The received filesystem
|
|
|
|
#
|
|
|
|
function file_check
|
|
|
|
{
|
|
|
|
sendfs=$1
|
|
|
|
recvfs=$2
|
|
|
|
|
|
|
|
if [[ -d /$recvfs/.zfs/snapshot/a && -d \
|
|
|
|
/$sendfs/.zfs/snapshot/a ]]; then
|
2022-03-23 03:52:39 +03:00
|
|
|
log_must directory_diff /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
if [[ -d /$recvfs/.zfs/snapshot/b && -d \
|
|
|
|
/$sendfs/.zfs/snapshot/b ]]; then
|
2022-03-23 03:52:39 +03:00
|
|
|
log_must directory_diff /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
|
2015-07-02 01:23:09 +03:00
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Resume test helper
|
|
|
|
#
|
|
|
|
# $1 The ZFS send command
|
|
|
|
# $2 The filesystem where the streams are sent
|
|
|
|
# $3 The receive filesystem
|
2020-10-03 03:40:46 +03:00
|
|
|
# $4 Test dry-run (optional)
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
|
|
|
function resume_test
|
|
|
|
{
|
2020-10-03 03:40:46 +03:00
|
|
|
typeset sendcmd=$1
|
|
|
|
typeset streamfs=$2
|
|
|
|
typeset recvfs=$3
|
|
|
|
typeset dryrun=${4:-1}
|
2015-07-02 01:23:09 +03:00
|
|
|
|
|
|
|
stream_num=1
|
|
|
|
log_must eval "$sendcmd >/$streamfs/$stream_num"
|
|
|
|
|
|
|
|
for ((i=0; i<2; i=i+1)); do
|
2020-01-10 21:16:58 +03:00
|
|
|
mess_send_file /$streamfs/$stream_num
|
2022-03-09 15:39:34 +03:00
|
|
|
log_mustnot eval "zfs recv -suv $recvfs </$streamfs/$stream_num"
|
2015-07-02 01:23:09 +03:00
|
|
|
stream_num=$((stream_num+1))
|
|
|
|
|
2022-03-15 01:47:38 +03:00
|
|
|
token=$(get_prop receive_resume_token $recvfs)
|
2020-10-03 03:40:46 +03:00
|
|
|
|
|
|
|
# Do a dry-run
|
|
|
|
[ $dryrun -ne 0 ] && \
|
|
|
|
log_must eval "zfs send -nvt $token > /dev/null"
|
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 19:48:13 +03:00
|
|
|
log_must eval "zfs send -t $token >/$streamfs/$stream_num"
|
2015-07-02 01:23:09 +03:00
|
|
|
done
|
2022-03-09 15:39:34 +03:00
|
|
|
log_must eval "zfs recv -suv $recvfs </$streamfs/$stream_num"
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
|
|
|
|
2020-07-24 03:44:03 +03:00
|
|
|
function get_resume_token
|
|
|
|
{
|
|
|
|
sendcmd=$1
|
|
|
|
streamfs=$2
|
|
|
|
recvfs=$3
|
|
|
|
|
|
|
|
log_must eval "$sendcmd > /$streamfs/1"
|
|
|
|
mess_send_file /$streamfs/1
|
2022-03-09 15:39:34 +03:00
|
|
|
log_mustnot eval "zfs recv -suv $recvfs < /$streamfs/1 2>&1"
|
|
|
|
get_prop receive_resume_token $recvfs > /$streamfs/resume_token
|
2020-07-24 03:44:03 +03:00
|
|
|
}
|
|
|
|
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
|
|
|
# Setup filesystems for the resumable send/receive tests
|
|
|
|
#
|
2017-04-12 18:47:42 +03:00
|
|
|
# $1 The "send" filesystem
|
|
|
|
# $2 The "recv" filesystem
|
2015-07-02 01:23:09 +03:00
|
|
|
#
|
|
|
|
function test_fs_setup
|
|
|
|
{
|
2017-04-12 18:47:42 +03:00
|
|
|
typeset sendfs=$1
|
|
|
|
typeset recvfs=$2
|
2017-10-11 01:22:05 +03:00
|
|
|
typeset streamfs=$3
|
2017-04-12 18:47:42 +03:00
|
|
|
typeset sendpool=${sendfs%%/*}
|
|
|
|
typeset recvpool=${recvfs%%/*}
|
2015-07-02 01:23:09 +03:00
|
|
|
|
2017-08-03 18:57:43 +03:00
|
|
|
datasetexists $sendfs && log_must_busy zfs destroy -r $sendpool
|
|
|
|
datasetexists $recvfs && log_must_busy zfs destroy -r $recvpool
|
2017-10-11 01:22:05 +03:00
|
|
|
datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
|
2015-07-02 01:23:09 +03:00
|
|
|
|
2020-05-21 04:02:41 +03:00
|
|
|
if datasetexists $sendfs || zfs create -o compress=lz4 $sendfs; then
|
2015-07-02 01:23:09 +03:00
|
|
|
mk_files 1000 256 0 $sendfs &
|
|
|
|
mk_files 1000 131072 0 $sendfs &
|
|
|
|
mk_files 100 1048576 0 $sendfs &
|
|
|
|
mk_files 10 10485760 0 $sendfs &
|
|
|
|
mk_files 1 104857600 0 $sendfs &
|
2017-04-12 00:56:54 +03:00
|
|
|
log_must wait
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs snapshot $sendfs@a
|
2015-07-02 01:23:09 +03:00
|
|
|
|
|
|
|
rm_files 200 256 0 $sendfs &
|
|
|
|
rm_files 200 131072 0 $sendfs &
|
|
|
|
rm_files 20 1048576 0 $sendfs &
|
|
|
|
rm_files 2 10485760 0 $sendfs &
|
2017-04-12 00:56:54 +03:00
|
|
|
log_must wait
|
2015-07-02 01:23:09 +03:00
|
|
|
|
|
|
|
mk_files 400 256 0 $sendfs &
|
|
|
|
mk_files 400 131072 0 $sendfs &
|
|
|
|
mk_files 40 1048576 0 $sendfs &
|
|
|
|
mk_files 4 10485760 0 $sendfs &
|
2017-04-12 00:56:54 +03:00
|
|
|
log_must wait
|
2015-07-02 01:23:09 +03:00
|
|
|
|
2017-04-06 03:18:22 +03:00
|
|
|
log_must zfs snapshot $sendfs@b
|
|
|
|
log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
|
|
|
|
log_must eval "zfs send -v -i @a $sendfs@b " \
|
2015-07-02 01:23:09 +03:00
|
|
|
">/$sendpool/incremental.zsend"
|
|
|
|
fi
|
|
|
|
|
2017-10-11 01:22:05 +03:00
|
|
|
log_must zfs create -o compress=lz4 $streamfs
|
2015-07-02 01:23:09 +03:00
|
|
|
}
|
2017-04-12 00:56:54 +03:00
|
|
|
|
|
|
|
#
|
|
|
|
# Check to see if the specified features are set in a send stream.
|
|
|
|
# The values for these features are found in include/sys/zfs_ioctl.h
|
|
|
|
#
|
|
|
|
# $1 The stream file
|
|
|
|
# $2-$n The flags expected in the stream
|
|
|
|
#
|
|
|
|
function stream_has_features
|
|
|
|
{
|
|
|
|
typeset file=$1
|
|
|
|
shift
|
|
|
|
|
|
|
|
[[ -f $file ]] || log_fail "Couldn't find file: $file"
|
2022-03-15 01:47:38 +03:00
|
|
|
typeset flags=$(zstream dump $file | \
|
2017-04-12 00:56:54 +03:00
|
|
|
awk '/features =/ {features = $3} END {print features}')
|
|
|
|
typeset -A feature
|
|
|
|
feature[dedup]="1"
|
|
|
|
feature[dedupprops]="2"
|
|
|
|
feature[sa_spill]="4"
|
|
|
|
feature[embed_data]="10000"
|
|
|
|
feature[lz4]="20000"
|
|
|
|
feature[mooch_byteswap]="40000"
|
|
|
|
feature[large_blocks]="80000"
|
|
|
|
feature[resuming]="100000"
|
|
|
|
feature[redacted]="200000"
|
|
|
|
feature[compressed]="400000"
|
|
|
|
|
|
|
|
typeset flag known derived=0
|
|
|
|
for flag in "$@"; do
|
|
|
|
known=${feature[$flag]}
|
|
|
|
[[ -z $known ]] && log_fail "Unknown feature: $flag"
|
|
|
|
|
|
|
|
derived=$(printf "%x" $((0x${flags} & 0x${feature[$flag]})))
|
|
|
|
[[ $derived = $known ]] || return 1
|
|
|
|
done
|
|
|
|
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Given a send stream, verify that the size of the stream matches what's
|
|
|
|
# expected based on the source or target dataset. If the stream is an
|
|
|
|
# incremental stream, subtract the size of the source snapshot before
|
|
|
|
# comparing. This function does not currently handle incremental streams
|
|
|
|
# that remove data.
|
|
|
|
#
|
2021-06-24 19:38:33 +03:00
|
|
|
# $1 The zstream dump output file
|
2017-04-12 00:56:54 +03:00
|
|
|
# $2 The dataset to compare against
|
|
|
|
# This can be a source of a send or recv target (fs, not snapshot)
|
|
|
|
# $3 The percentage below which verification is deemed a failure
|
|
|
|
# $4 The source snapshot of an incremental send
|
|
|
|
#
|
|
|
|
|
|
|
|
function verify_stream_size
|
|
|
|
{
|
|
|
|
typeset stream=$1
|
|
|
|
typeset ds=$2
|
|
|
|
typeset percent=${3:-90}
|
|
|
|
typeset inc_src=$4
|
|
|
|
|
|
|
|
[[ -f $stream ]] || log_fail "No such file: $stream"
|
|
|
|
datasetexists $ds || log_fail "No such dataset: $ds"
|
|
|
|
|
2022-03-15 01:47:38 +03:00
|
|
|
typeset stream_size=$(zstream dump $stream | sed -n \
|
2019-06-23 02:33:44 +03:00
|
|
|
's/ Total payload size = \(.*\) (0x.*)/\1/p')
|
2017-04-12 00:56:54 +03:00
|
|
|
|
|
|
|
typeset inc_size=0
|
|
|
|
if [[ -n $inc_src ]]; then
|
|
|
|
inc_size=$(get_prop lrefer $inc_src)
|
|
|
|
if stream_has_features $stream compressed; then
|
|
|
|
inc_size=$(get_prop refer $inc_src)
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if stream_has_features $stream compressed; then
|
|
|
|
ds_size=$(get_prop refer $ds)
|
|
|
|
else
|
|
|
|
ds_size=$(get_prop lrefer $ds)
|
|
|
|
fi
|
|
|
|
ds_size=$((ds_size - inc_size))
|
|
|
|
|
2022-03-15 01:47:38 +03:00
|
|
|
log_must within_percent $stream_size $ds_size $percent
|
2017-04-12 00:56:54 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
# Cleanup function for tests involving resumable send
|
|
|
|
function resume_cleanup
|
|
|
|
{
|
|
|
|
typeset sendfs=$1
|
|
|
|
typeset streamfs=$2
|
2017-10-11 01:22:05 +03:00
|
|
|
typeset sendpool=${sendfs%%/*}
|
2017-04-12 00:56:54 +03:00
|
|
|
|
2017-08-03 18:57:43 +03:00
|
|
|
datasetexists $sendfs && log_must_busy zfs destroy -r $sendfs
|
|
|
|
datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
|
2017-04-12 00:56:54 +03:00
|
|
|
cleanup_pool $POOL2
|
2017-10-11 01:22:05 +03:00
|
|
|
rm -f /$sendpool/initial.zsend /$sendpool/incremental.zsend
|
2017-04-12 00:56:54 +03:00
|
|
|
}
|
2019-04-13 00:28:04 +03:00
|
|
|
|
|
|
|
# Randomly set the property to one of the enumerated values.
|
|
|
|
function rand_set_prop
|
|
|
|
{
|
|
|
|
typeset dtst=$1
|
|
|
|
typeset prop=$2
|
|
|
|
shift 2
|
|
|
|
typeset value=$(random_get $@)
|
|
|
|
|
|
|
|
log_must eval "zfs set $prop='$value' $dtst"
|
|
|
|
}
|
|
|
|
|
2019-05-08 01:18:44 +03:00
|
|
|
# Generate a recursive checksum of a filesystem which includes the file
|
2019-12-18 23:29:43 +03:00
|
|
|
# contents and any associated extended attributes.
|
2019-04-13 00:28:04 +03:00
|
|
|
function recursive_cksum
|
|
|
|
{
|
2019-12-18 23:29:43 +03:00
|
|
|
case "$(uname)" in
|
|
|
|
FreeBSD)
|
|
|
|
find $1 -type f -exec sh -c 'sha256 -q {}; lsextattr -q \
|
|
|
|
system {} | sha256 -q; lsextattr -q user {} | sha256 -q' \
|
|
|
|
\; | sort | sha256 -q
|
|
|
|
;;
|
|
|
|
*)
|
|
|
|
find $1 -type f -exec sh -c 'sha256sum {}; getfattr \
|
|
|
|
--absolute-names --only-values -d {} | sha256sum' \; | \
|
|
|
|
sort -k 2 | awk '{ print $1 }' | sha256sum | \
|
|
|
|
awk '{ print $1 }'
|
|
|
|
;;
|
|
|
|
esac
|
2019-04-13 00:28:04 +03:00
|
|
|
}
|