mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2025-10-26 18:05:04 +03:00
c618f87cd2
Deduplicated send and receive is deprecated. To ease migration to the new dedup-send-less world, the commit adds a `zstream redup` utility to convert deduplicated send streams to normal streams, so that they can continue to be received indefinitely. The new `zstream` command also replaces the functionality of `zstreamdump`, by way of the `zstream dump` subcommand. The `zstreamdump` command is replaced by a shell script which invokes `zstream dump`. The way that `zstream redup` works under the hood is that as we read the send stream, we build up a hash table which maps from `<GUID, object, offset> -> <file_offset>`. Whenever we see a WRITE record, we add a new entry to the hash table, which indicates where in the stream file to find the WRITE record for this block. (The key is `drr_toguid, drr_object, drr_offset`.) For entries other than WRITE_BYREF, we pass them through unchanged (except for the running checksum, which is recalculated). For WRITE_BYREF records, we change them to WRITE records. We find the referenced WRITE record by looking in the hash table (for the record with key `drr_refguid, drr_refobject, drr_refoffset`), and then reading the record header and payload from the specified offset in the stream file. This is why the stream can not be a pipe. The found WRITE record replaces the WRITE_BYREF record, with its `drr_toguid`, `drr_object`, and `drr_offset` fields changed to be the same as the WRITE_BYREF's (i.e. we are writing the same logical block, but with the data supplied by the previous WRITE record). This algorithm requires memory proportional to the number of WRITE records (same as `zfs send -D`), but the size per WRITE record is relatively low (40 bytes, vs. 72 for `zfs send -D`). A 1TB send stream with 8KB blocks (`recordsize=8k`) would use around 5GB of RAM to "redup". Reviewed-by: Jorgen Lundman <lundman@lundman.net> Reviewed-by: Paul Dagnelie <pcd@delphix.com> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: Matthew Ahrens <mahrens@delphix.com> Closes #10124 Closes #10156
215 lines
2.7 KiB
INI
215 lines
2.7 KiB
INI
#
|
|
# Copyright (c) 2016, 2019 by Delphix. All rights reserved.
|
|
# These variables are used by zfs-tests.sh to constrain which utilities
|
|
# may be used by the suite. The suite will create a directory which is
|
|
# the only element of $PATH and create symlinks from that dir to the
|
|
# binaries listed below.
|
|
#
|
|
# Please keep the contents of each variable sorted for ease of reading
|
|
# and maintenance.
|
|
#
|
|
export SYSTEM_FILES_COMMON='arp
|
|
awk
|
|
base64
|
|
basename
|
|
bc
|
|
bunzip2
|
|
bzcat
|
|
cat
|
|
chgrp
|
|
chmod
|
|
chown
|
|
cksum
|
|
cmp
|
|
cp
|
|
cpio
|
|
cut
|
|
date
|
|
dd
|
|
df
|
|
diff
|
|
dirname
|
|
dmesg
|
|
du
|
|
echo
|
|
egrep
|
|
expr
|
|
false
|
|
file
|
|
find
|
|
fio
|
|
getconf
|
|
getent
|
|
getfacl
|
|
grep
|
|
gunzip
|
|
gzip
|
|
head
|
|
hostname
|
|
id
|
|
iostat
|
|
kill
|
|
ksh
|
|
ln
|
|
logname
|
|
ls
|
|
mkdir
|
|
mknod
|
|
mktemp
|
|
mount
|
|
mv
|
|
net
|
|
od
|
|
openssl
|
|
pax
|
|
pgrep
|
|
ping
|
|
pkill
|
|
printenv
|
|
printf
|
|
ps
|
|
pwd
|
|
python
|
|
python2
|
|
python3
|
|
quotaon
|
|
readlink
|
|
rm
|
|
rmdir
|
|
scp
|
|
script
|
|
sed
|
|
seq
|
|
setfacl
|
|
sh
|
|
sleep
|
|
sort
|
|
ssh
|
|
stat
|
|
strings
|
|
su
|
|
sudo
|
|
sum
|
|
swapoff
|
|
swapon
|
|
sync
|
|
tail
|
|
tar
|
|
tee
|
|
timeout
|
|
touch
|
|
tr
|
|
true
|
|
truncate
|
|
umask
|
|
umount
|
|
uname
|
|
uuidgen
|
|
vmstat
|
|
wait
|
|
wc
|
|
which
|
|
xargs'
|
|
|
|
export SYSTEM_FILES_FREEBSD='chflags
|
|
compress
|
|
diskinfo
|
|
dumpon
|
|
env
|
|
fsck
|
|
getextattr
|
|
gpart
|
|
lsextattr
|
|
md5
|
|
mdconfig
|
|
mkfifo
|
|
newfs
|
|
pw
|
|
rmextattr
|
|
setextattr
|
|
sha256
|
|
showmount
|
|
swapctl
|
|
sysctl
|
|
uncompress'
|
|
|
|
export SYSTEM_FILES_LINUX='attr
|
|
bash
|
|
blkid
|
|
blockdev
|
|
chattr
|
|
dmidecode
|
|
exportfs
|
|
fallocate
|
|
fdisk
|
|
free
|
|
getfattr
|
|
groupadd
|
|
groupdel
|
|
groupmod
|
|
hostid
|
|
losetup
|
|
lsattr
|
|
lsblk
|
|
lscpu
|
|
lsmod
|
|
lsscsi
|
|
md5sum
|
|
mkswap
|
|
modprobe
|
|
mpstat
|
|
nproc
|
|
parted
|
|
perf
|
|
setenforce
|
|
setfattr
|
|
sha256sum
|
|
udevadm
|
|
useradd
|
|
userdel
|
|
usermod'
|
|
|
|
export ZFS_FILES='zdb
|
|
zfs
|
|
zhack
|
|
zinject
|
|
zpool
|
|
ztest
|
|
raidz_test
|
|
arc_summary
|
|
arc_summary3
|
|
arcstat
|
|
dbufstat
|
|
zed
|
|
zgenhostid
|
|
zstream
|
|
zstreamdump'
|
|
|
|
export ZFSTEST_FILES='btree_test
|
|
chg_usr_exec
|
|
devname2devid
|
|
dir_rd_update
|
|
file_check
|
|
file_trunc
|
|
file_write
|
|
get_diff
|
|
largest_file
|
|
libzfs_input_check
|
|
mkbusy
|
|
mkfile
|
|
mkfiles
|
|
mktree
|
|
mmap_exec
|
|
mmap_libaio
|
|
mmapwrite
|
|
nvlist_to_lua
|
|
randfree_file
|
|
randwritecomp
|
|
readmmap
|
|
rename_dir
|
|
rm_lnkcnt_zero_file
|
|
threadsappend
|
|
user_ns_exec
|
|
xattrtest
|
|
stride_dd'
|