mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2026-05-22 02:27:36 +03:00
Remove deduplicated send/receive code
Deduplicated send streams (i.e. `zfs send -D` and `zfs receive` of such streams) are deprecated. Deduplicated send streams can be received by first converting them to non-deduplicated with the `zstream redup` command. This commit removes the code for sending and receiving deduplicated send streams. `zfs send -D` will now print a warning, ignore the `-D` flag, and generate a regular (non-deduplicated) send stream. `zfs receive` of a deduplicated send stream will print an error message and fail. The resulting code simplification (especially in the kernel's support for receiving dedup streams) should help enable future performance enhancements. Several new tests are added which leverage `zstream redup`. Reviewed-by: Paul Dagnelie <pcd@delphix.com> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: Matthew Ahrens <mahrens@delphix.com> Issue #7887 Issue #10117 Issue #10156 Closes #10212
This commit is contained in:
Matthew Ahrens
@@ -766,22 +766,22 @@ tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
|
||||
tags = ['functional', 'rootpool']
|
||||
|
||||
[tests/functional/rsend]
|
||||
tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
|
||||
'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos',
|
||||
'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos',
|
||||
'rsend_013_pos', 'rsend_014_pos', 'rsend_016_neg',
|
||||
'rsend_019_pos', 'rsend_020_pos',
|
||||
tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
|
||||
'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos',
|
||||
'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', 'rsend_009_pos',
|
||||
'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', 'rsend_013_pos',
|
||||
'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos',
|
||||
'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos',
|
||||
'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props',
|
||||
'send-c_incremental', 'send-c_volume', 'send-c_zstreamdump',
|
||||
'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
|
||||
'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
|
||||
'send-c_mixed_compression', 'send-c_stream_size_estimate',
|
||||
'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
|
||||
'send-c_recv_dedup', 'send_encrypted_hierarchy',
|
||||
'send_encrypted_props', 'send_encrypted_truncated_files',
|
||||
'send_freeobjects', 'send_realloc_files',
|
||||
'send_realloc_encrypted_files', 'send_spill_block', 'send_holds',
|
||||
'send_hole_birth', 'send_mixed_raw', 'send-wDR_encrypted_zvol',
|
||||
'send_hole_birth', 'send_mixed_raw', 'send-wR_encrypted_zvol',
|
||||
'send_partial_dataset']
|
||||
tags = ['functional', 'rsend']
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/rsend
|
||||
dist_pkgdata_SCRIPTS = \
|
||||
setup.ksh \
|
||||
cleanup.ksh \
|
||||
recv_dedup.ksh \
|
||||
recv_dedup_encrypted_zvol.ksh \
|
||||
rsend_001_pos.ksh \
|
||||
rsend_002_pos.ksh \
|
||||
rsend_003_pos.ksh \
|
||||
@@ -25,7 +27,6 @@ dist_pkgdata_SCRIPTS = \
|
||||
send_encrypted_hierarchy.ksh \
|
||||
send_encrypted_props.ksh \
|
||||
send_encrypted_truncated_files.ksh \
|
||||
send-cD.ksh \
|
||||
send-c_embedded_blocks.ksh \
|
||||
send-c_incremental.ksh \
|
||||
send-c_lz4_disabled.ksh \
|
||||
@@ -49,8 +50,12 @@ dist_pkgdata_SCRIPTS = \
|
||||
send_holds.ksh \
|
||||
send_hole_birth.ksh \
|
||||
send_mixed_raw.ksh \
|
||||
send-wDR_encrypted_zvol.ksh
|
||||
send-wR_encrypted_zvol.ksh
|
||||
|
||||
dist_pkgdata_DATA = \
|
||||
dedup.zsend.bz2 \
|
||||
dedup_encrypted_zvol.bz2 \
|
||||
dedup_encrypted_zvol.zsend.bz2 \
|
||||
fs.tar.gz \
|
||||
rsend.cfg \
|
||||
rsend.kshlib
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+53
@@ -0,0 +1,53 @@
|
||||
#!/bin/ksh -p
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# This file and its contents are supplied under the terms of the
|
||||
# Common Development and Distribution License ("CDDL"), version 1.0.
|
||||
# You may only use this file in accordance with the terms of version
|
||||
# 1.0 of the CDDL.
|
||||
#
|
||||
# A full copy of the text of the CDDL should have accompanied this
|
||||
# source. A copy of the CDDL is also available via the Internet at
|
||||
# http://www.illumos.org/license/CDDL.
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2020 by Delphix. All rights reserved.
|
||||
#
|
||||
|
||||
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Verifies that we can receive a dedup send stream by processing it with
|
||||
# "zstream redup".
|
||||
#
|
||||
|
||||
verify_runnable "both"
|
||||
|
||||
function cleanup
|
||||
{
|
||||
destroy_dataset $TESTPOOL/recv "-r"
|
||||
rm -r /$TESTPOOL/tar
|
||||
rm $sendfile
|
||||
}
|
||||
log_onexit cleanup
|
||||
|
||||
log_assert "Verify zfs can receive dedup send streams with 'zstream redup'"
|
||||
|
||||
typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup.zsend.bz2
|
||||
typeset sendfile=/$TESTPOOL/dedup.zsend
|
||||
typeset tarfile=$STF_SUITE/tests/functional/rsend/fs.tar.gz
|
||||
|
||||
log_must eval "bzcat <$sendfile_compressed >$sendfile"
|
||||
log_must zfs create $TESTPOOL/recv
|
||||
log_must eval "zstream redup $sendfile | zfs recv -d $TESTPOOL/recv"
|
||||
|
||||
log_must mkdir /$TESTPOOL/tar
|
||||
log_must tar --directory /$TESTPOOL/tar -xzf $tarfile
|
||||
log_must diff -r /$TESTPOOL/tar /$TESTPOOL/recv
|
||||
|
||||
log_pass "zfs can receive dedup send streams with 'zstream redup'"
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/bin/ksh -p
|
||||
#
|
||||
# CDDL HEADER START
|
||||
#
|
||||
# This file and its contents are supplied under the terms of the
|
||||
# Common Development and Distribution License ("CDDL"), version 1.0.
|
||||
# You may only use this file in accordance with the terms of version
|
||||
# 1.0 of the CDDL.
|
||||
#
|
||||
# A full copy of the text of the CDDL should have accompanied this
|
||||
# source. A copy of the CDDL is also available via the Internet at
|
||||
# http://www.illumos.org/license/CDDL.
|
||||
#
|
||||
# CDDL HEADER END
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2020 by Delphix. All rights reserved.
|
||||
#
|
||||
|
||||
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Verifies that we can receive a dedup send stream of a zvol by processing it
|
||||
# with "zstream redup".
|
||||
#
|
||||
|
||||
verify_runnable "both"
|
||||
|
||||
function cleanup
|
||||
{
|
||||
destroy_dataset $TESTPOOL/recv "-r"
|
||||
rm $sendfile
|
||||
rm $volfile
|
||||
rm $keyfile
|
||||
}
|
||||
log_onexit cleanup
|
||||
|
||||
log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams"
|
||||
|
||||
typeset keyfile=/$TESTPOOL/pkey
|
||||
typeset recvdev=$ZVOL_DEVDIR/$TESTPOOL/recv
|
||||
typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2
|
||||
typeset sendfile=/$TESTPOOL/dedup_encrypted_zvol.zsend
|
||||
typeset volfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.bz2
|
||||
typeset volfile=/$TESTPOOL/dedup_encrypted_zvol
|
||||
|
||||
log_must eval "echo 'password' > $keyfile"
|
||||
|
||||
log_must eval "bzcat <$sendfile_compressed >$sendfile"
|
||||
log_must eval "zstream redup $sendfile | zfs recv $TESTPOOL/recv"
|
||||
|
||||
log_must zfs load-key $TESTPOOL/recv
|
||||
block_device_wait
|
||||
|
||||
log_must eval "bzcat <$volfile_compressed >$volfile"
|
||||
log_must diff $volfile $recvdev
|
||||
|
||||
log_pass "zfs can receive raw, recursive, and deduplicated send streams"
|
||||
@@ -1,89 +0,0 @@
|
||||
#!/bin/ksh -p
|
||||
|
||||
#
|
||||
# This file and its contents are supplied under the terms of the
|
||||
# Common Development and Distribution License ("CDDL"), version 1.0.
|
||||
# You may only use this file in accordance with the terms of version
|
||||
# 1.0 of the CDDL.
|
||||
#
|
||||
# A full copy of the text of the CDDL should have accompanied this
|
||||
# source. A copy of the CDDL is also available via the Internet at
|
||||
# http://www.illumos.org/license/CDDL.
|
||||
#
|
||||
|
||||
#
|
||||
# Copyright (c) 2015, 2018 by Delphix. All rights reserved.
|
||||
#
|
||||
|
||||
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
|
||||
|
||||
#
|
||||
# Description:
|
||||
# Verify that the -c and -D flags do not interfere with each other.
|
||||
#
|
||||
# Strategy:
|
||||
# 1. Write unique data to a filesystem and create a compressed, deduplicated
|
||||
# full stream.
|
||||
# 2. Verify that the stream and send dataset show the same size
|
||||
# 3. Make several copies of the original data, and create both full and
|
||||
# incremental compressed, deduplicated send streams
|
||||
# 4. Verify the full stream is no bigger than the stream from step 1
|
||||
# 5. Verify the streams can be received correctly.
|
||||
#
|
||||
|
||||
verify_runnable "both"
|
||||
|
||||
log_assert "Verify that the -c and -D flags do not interfere with each other"
|
||||
log_onexit cleanup_pool $POOL2
|
||||
|
||||
typeset sendfs=$POOL2/sendfs
|
||||
typeset recvfs=$POOL2/recvfs
|
||||
typeset stream0=$BACKDIR/stream.0
|
||||
typeset stream1=$BACKDIR/stream.1
|
||||
typeset inc=$BACKDIR/stream.inc
|
||||
|
||||
log_must zfs create -o compress=lz4 $sendfs
|
||||
log_must zfs create -o compress=lz4 $recvfs
|
||||
typeset dir=$(get_prop mountpoint $sendfs)
|
||||
# Don't use write_compressible: we want compressible but undeduplicable data.
|
||||
log_must eval "dd if=/dev/urandom bs=1024k count=4 | base64 >$dir/file"
|
||||
log_must zfs snapshot $sendfs@snap0
|
||||
log_must eval "zfs send -D -c $sendfs@snap0 >$stream0"
|
||||
|
||||
# The stream size should match at this point because the data is all unique
|
||||
verify_stream_size $stream0 $sendfs
|
||||
|
||||
for i in {0..3}; do
|
||||
log_must cp $dir/file $dir/file.$i
|
||||
done
|
||||
log_must zfs snapshot $sendfs@snap1
|
||||
|
||||
# The stream sizes should match, since the second stream contains no new blocks
|
||||
log_must eval "zfs send -D -c $sendfs@snap1 >$stream1"
|
||||
typeset size0=$(stat_size $stream0)
|
||||
typeset size1=$(stat_size $stream1)
|
||||
within_percent $size0 $size1 90 || log_fail "$size0 and $size1"
|
||||
|
||||
# make sure the receive works correctly.
|
||||
log_must eval "zfs send -D -c -i snap0 $sendfs@snap1 >$inc"
|
||||
log_must eval "zfs recv -d $recvfs <$stream0"
|
||||
log_must eval "zfs recv -d $recvfs <$inc"
|
||||
cmp_ds_cont $sendfs $recvfs
|
||||
|
||||
# check receive with redup.
|
||||
log_must zfs destroy -r $recvfs
|
||||
log_must zfs create -o compress=lz4 $recvfs
|
||||
log_must eval "zstream redup $stream0 | zfs recv -d $recvfs"
|
||||
log_must eval "zstream redup $inc | zfs recv -d $recvfs"
|
||||
cmp_ds_cont $sendfs $recvfs
|
||||
|
||||
# The size of the incremental should be the same as the initial send.
|
||||
typeset size2=$(stat_size $inc)
|
||||
within_percent $size0 $size2 90 || log_fail "$size0 and $size1"
|
||||
|
||||
# The redup'ed size should be 4x
|
||||
typeset size3=$(zstream redup $inc | wc -c)
|
||||
let size4=size0*4
|
||||
within_percent $size4 $size3 90 || log_fail "$size4 and $size3"
|
||||
|
||||
log_pass "The -c and -D flags do not interfere with each other"
|
||||
+8
-7
@@ -16,20 +16,21 @@
|
||||
|
||||
#
|
||||
# Copyright (c) 2018 by Datto Inc. All rights reserved.
|
||||
# Copyright (c) 2020 by Delphix. All rights reserved.
|
||||
#
|
||||
|
||||
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
|
||||
|
||||
#
|
||||
# DESCRIPTION:
|
||||
# Verify that zvols with dedup=on and encryption=on can be sent and received
|
||||
# with a deduplicated raw send stream.
|
||||
# Verify that zvols with encryption=on can be sent and received with a raw
|
||||
# send stream.
|
||||
#
|
||||
# STRATEGY:
|
||||
# 1. Create a zvol with dedup and encryption on and put a filesystem on it
|
||||
# 1. Create a zvol with encryption on and put a filesystem on it
|
||||
# 2. Copy a file into the zvol a few times and take a snapshot
|
||||
# 3. Repeat step 2 a few times to create more snapshots
|
||||
# 4. Send all snapshots in a recursive, raw, deduplicated send stream
|
||||
# 4. Send all snapshots in a recursive, raw send stream
|
||||
# 5. Mount the received zvol and verify that all of the data there is correct
|
||||
#
|
||||
|
||||
@@ -48,7 +49,7 @@ function cleanup
|
||||
}
|
||||
log_onexit cleanup
|
||||
|
||||
log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams"
|
||||
log_assert "Verify zfs can receive raw, recursive send streams"
|
||||
|
||||
typeset keyfile=/$TESTPOOL/pkey
|
||||
typeset snap_count=5
|
||||
@@ -93,7 +94,7 @@ for ((i = 1; i <= $snap_count; i++)); do
|
||||
log_must mount $remount_rw $zdev $mntpnt
|
||||
done
|
||||
|
||||
log_must eval "zfs send -wDR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile"
|
||||
log_must eval "zfs send -wR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile"
|
||||
log_must eval "zfs recv $TESTPOOL/recv < $sendfile"
|
||||
log_must zfs load-key $TESTPOOL/recv
|
||||
block_device_wait
|
||||
@@ -104,4 +105,4 @@ md5_1=$(cat $mntpnt/* | md5digest)
|
||||
md5_2=$(cat $recvmnt/* | md5digest)
|
||||
[[ "$md5_1" == "$md5_2" ]] || log_fail "md5 mismatch: $md5_1 != $md5_2"
|
||||
|
||||
log_pass "zfs can receive raw, recursive, and deduplicated send streams"
|
||||
log_pass "zfs can receive raw, recursive send streams"
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
#
|
||||
# Copyright (c) 2019 Datto Inc.
|
||||
# Copyright (c) 2020 by Delphix. All rights reserved.
|
||||
#
|
||||
|
||||
. $STF_SUITE/include/libtest.shlib
|
||||
@@ -98,7 +99,6 @@ set -A badargs \
|
||||
"-R $POOL/recvfs" \
|
||||
"-p $POOL/recvfs" \
|
||||
"-I $POOL/recvfs" \
|
||||
"-D $POOL/recvfs" \
|
||||
"-h $POOL/recvfs"
|
||||
|
||||
while (( i < ${#badargs[*]} ))
|
||||
|
||||
Reference in New Issue
Block a user