ZTS: Replace MD5 and SHA256 wit XXH128

For data integrity checks as done in ZTS, the verification for
unintended data corruption with xxhash128 should be a lot faster
and perfectly usable.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>
Closes #16577
This commit is contained in:
Tino Reichardt 2024-09-28 20:24:05 +04:00 committed by GitHub
parent 834b90fb81
commit b1958b531b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
35 changed files with 117 additions and 138 deletions

View File

@ -16,7 +16,7 @@ function archlinux() {
sudo pacman -Sy --noconfirm base-devel bc cpio dhclient dkms fakeroot \
fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils parted \
pax perf python-packaging python-setuptools qemu-guest-agent ksh samba \
sysstat rng-tools rsync wget
sysstat rng-tools rsync wget xxhash
echo "##[endgroup]"
}
@ -38,7 +38,7 @@ function debian() {
lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \
python3-cffi python3-dev python3-distlib python3-packaging \
python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \
rsync samba sysstat uuid-dev watchdog wget xfslibs-dev zlib1g-dev
rsync samba sysstat uuid-dev watchdog wget xfslibs-dev xxhash zlib1g-dev
echo "##[endgroup]"
}
@ -48,8 +48,7 @@ function freebsd() {
echo "##[group]Install Development Tools"
sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \
gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \
pkgconf python python3 pamtester pamtester qemu-guest-agent rsync \
sysutils/coreutils
pkgconf python python3 pamtester pamtester qemu-guest-agent rsync xxhash
sudo pkg install -xy \
'^samba4[[:digit:]]+$' \
'^py3[[:digit:]]+-cffi$' \
@ -76,7 +75,7 @@ function rhel() {
lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester parted perf \
python3 python3-cffi python3-devel python3-packaging kernel-devel \
python3-setuptools qemu-guest-agent rng-tools rpcgen rpm-build rsync \
samba sysstat systemd watchdog wget xfsprogs-devel zlib-devel
samba sysstat systemd watchdog wget xfsprogs-devel xxhash zlib-devel
echo "##[endgroup]"
}

View File

@ -100,7 +100,8 @@ export SYSTEM_FILES_COMMON='awk
uniq
vmstat
wc
xargs'
xargs
xxh128sum'
export SYSTEM_FILES_FREEBSD='chflags
compress
@ -112,13 +113,11 @@ export SYSTEM_FILES_FREEBSD='chflags
jexec
jls
lsextattr
md5
mdconfig
newfs
pw
rmextattr
setextattr
sha256
showmount
swapctl
sysctl
@ -146,7 +145,6 @@ export SYSTEM_FILES_LINUX='attr
lscpu
lsmod
lsscsi
md5sum
mkswap
modprobe
mountpoint
@ -156,7 +154,6 @@ export SYSTEM_FILES_LINUX='attr
perf
setfattr
setpriv
sha256sum
udevadm
unshare
useradd

View File

@ -3455,35 +3455,26 @@ function tunable_exists
}
#
# Compute MD5 digest for given file or stdin if no file given.
# Compute xxh128sum for given file or stdin if no file given.
# Note: file path must not contain spaces
#
function md5digest
function xxh128digest
{
openssl md5 -r $1 | awk '{print $1}'
xxh128sum $1 | awk '{print $1}'
}
#
# Compare the MD5 digest of two files.
# Compare the xxhash128 digest of two files.
#
function cmp_md5s {
function cmp_xxh128 {
typeset file1=$1
typeset file2=$2
typeset sum1=$(md5digest $file1)
typeset sum2=$(md5digest $file2)
typeset sum1=$(xxh128digest $file1)
typeset sum2=$(xxh128digest $file2)
test "$sum1" = "$sum2"
}
#
# Compute SHA256 digest for given file or stdin if no file given.
# Note: file path must not contain spaces
#
function sha256digest
{
openssl sha256 -r $1 | awk '{print $1}'
}
function new_fs #<args>
{
case "$UNAME" in

View File

@ -77,7 +77,7 @@ function test_file_integrity
typeset -r clone=$2
typeset -r filesize=$3
typeset -r clone_checksum=$(sha256digest $clone)
typeset -r clone_checksum=$(xxh128digest $clone)
if [[ $original_checksum != $clone_checksum ]]; then
log_fail "Clone $clone is corrupted with file size $filesize"
@ -171,7 +171,7 @@ function bclone_test
dsize=0
fi
typeset -r original_checksum=$(sha256digest $original)
typeset -r original_checksum=$(xxh128digest $original)
sync_pool $TESTPOOL

View File

@ -32,14 +32,14 @@ function first_half_checksum
{
typeset -r file=$1
dd if=$file bs=$HALFRECORDSIZE count=1 2>/dev/null | sha256digest
dd if=$file bs=$HALFRECORDSIZE count=1 2>/dev/null | xxh128digest
}
function second_half_checksum
{
typeset -r file=$1
dd if=$file bs=$HALFRECORDSIZE count=1 skip=1 2>/dev/null | sha256digest
dd if=$file bs=$HALFRECORDSIZE count=1 skip=1 2>/dev/null | xxh128digest
}
function bclone_corner_cases_init
@ -66,7 +66,7 @@ function bclone_corner_cases_init
export SECOND_HALF_ORIG0_CHECKSUM=$(second_half_checksum $ORIG0)
export SECOND_HALF_ORIG1_CHECKSUM=$(second_half_checksum $ORIG1)
export SECOND_HALF_ORIG2_CHECKSUM=$(second_half_checksum $ORIG2)
export ZEROS_CHECKSUM=$(dd if=/dev/zero bs=$HALFRECORDSIZE count=1 2>/dev/null | sha256digest)
export ZEROS_CHECKSUM=$(dd if=/dev/zero bs=$HALFRECORDSIZE count=1 2>/dev/null | xxh128digest)
export FIRST_HALF_CHECKSUM=""
export SECOND_HALF_CHECKSUM=""
}

View File

@ -28,8 +28,8 @@
function have_same_content
{
typeset hash1=$(md5digest $1)
typeset hash2=$(md5digest $2)
typeset hash1=$(xxh128digest $1)
typeset hash2=$(xxh128digest $2)
log_must [ "$hash1" = "$hash2" ]
}

View File

@ -110,9 +110,9 @@ log_note "Cloning entire file with copy_file_range across different enc" \
clone_and_check "file" "clone" $DS1 $DS2 "" true
log_note "check if the file is still readable and the same after" \
"unmount and key unload, shouldn't fail"
typeset hash1=$(md5digest "/$DS1/file")
typeset hash1=$(xxh128digest "/$DS1/file")
log_must zfs umount $DS1 && zfs unload-key $DS1
typeset hash2=$(md5digest "/$DS2/clone")
typeset hash2=$(xxh128digest "/$DS2/clone")
log_must [ "$hash1" = "$hash2" ]
cleanup_enc
@ -144,12 +144,12 @@ log_must sync_pool $TESTPOOL
log_must rm -f "/$DS1/file" "/$DS2/file"
log_must sync_pool $TESTPOOL
clone_and_check "file" "clone" "$DS2" "$DS1" "" true "s1"
typeset hash1=$(md5digest "/$DS1/.zfs/snapshot/s1/file")
typeset hash1=$(xxh128digest "/$DS1/.zfs/snapshot/s1/file")
log_note "destroy the snapshot and check if the file is still readable and" \
"has the same content"
log_must zfs destroy -r $DS2@s1
log_must sync_pool $TESTPOOL
typeset hash2=$(md5digest "/$DS1/file")
typeset hash2=$(xxh128digest "/$DS1/file")
log_must [ "$hash1" = "$hash2" ]
cleanup_enc

View File

@ -47,8 +47,8 @@ sync_pool $TESTPOOL
log_must eval "zfs send -ecL $snap > $tmpfile.1"
log_must eval "zdb -B $TESTPOOL/$objsetid ecL > $tmpfile.2"
typeset sum1=$(md5digest $tmpfile.1)
typeset sum2=$(md5digest $tmpfile.2)
typeset sum1=$(xxh128digest $tmpfile.1)
typeset sum2=$(xxh128digest $tmpfile.2)
log_must test "$sum1" = "$sum2"

View File

@ -73,7 +73,7 @@ function test_corrective_recv
log_must zpool status -v $TESTPOOL
log_mustnot eval "zpool status -v $TESTPOOL | \
grep \"Permanent errors have been detected\""
typeset cksum=$(md5digest $file)
typeset cksum=$(xxh128digest $file)
[[ "$cksum" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum != $checksum)"
}
@ -95,7 +95,7 @@ log_must zfs create -o primarycache=none \
log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync
log_must eval "echo 'aaaaaaaa' >> "$file
typeset checksum=$(md5digest $file)
typeset checksum=$(xxh128digest $file)
log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1

View File

@ -72,7 +72,7 @@ function test_corrective_recv
log_must zpool status -v $TESTPOOL
log_mustnot eval "zpool status -v $TESTPOOL | \
grep \"Permanent errors have been detected\""
typeset cksum=$(md5digest $file)
typeset cksum=$(xxh128digest $file)
[[ "$cksum" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum != $checksum)"
}
@ -94,7 +94,7 @@ log_must zfs create -o primarycache=none \
log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync
log_must eval "echo 'aaaaaaaa' >> "$file
typeset checksum=$(md5digest $file)
typeset checksum=$(xxh128digest $file)
log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1
@ -177,7 +177,7 @@ log_must zpool scrub -w $TESTPOOL
log_must zpool status -v $TESTPOOL
log_mustnot eval "zpool status -v $TESTPOOL | \
grep \"Permanent errors have been detected\""
typeset cksum=$(md5digest $file)
typeset cksum=$(xxh128digest $file)
[[ "$cksum" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum != $checksum)"

View File

@ -59,7 +59,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
"-o keyformat=passphrase $TESTPOOL/$TESTFS2"
log_must mkfile 1M /$TESTPOOL/$TESTFS2/$TESTFILE0
typeset checksum=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
typeset checksum=$(xxh128digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
log_must zfs snapshot $snap
@ -69,14 +69,14 @@ log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c1"
crypt=$(get_prop encryption $TESTPOOL/$TESTFS1/c1)
[[ "$crypt" == "off" ]] || log_fail "Received unencrypted stream as encrypted"
typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
typeset cksum1=$(xxh128digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
[[ "$cksum1" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum1 != $checksum)"
log_note "Verify ZFS can receive into an encrypted child"
log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS2/c1"
typeset cksum2=$(md5digest /$TESTPOOL/$TESTFS2/c1/$TESTFILE0)
typeset cksum2=$(xxh128digest /$TESTPOOL/$TESTFS2/c1/$TESTFILE0)
[[ "$cksum2" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum2 != $checksum)"

View File

@ -60,7 +60,7 @@ log_must zfs create -o compress=zstd-$random_level $TESTPOOL/$TESTFS1
# Make a 5kb compressible file
log_must eval cat $src_data $src_data $src_data $src_data $src_data \
"> /$TESTPOOL/$TESTFS1/$TESTFILE0"
typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
typeset checksum=$(xxh128digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
log_must zfs snapshot $snap
@ -79,7 +79,7 @@ log_note "ZSTD src: size=$zstd_size1 version=$zstd_version1 level=$zstd_level1"
log_note "Verify ZFS can receive the ZSTD compressed stream"
log_must eval "zfs send -ec $snap | zfs receive $TESTPOOL/$TESTFS2"
typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
typeset cksum1=$(xxh128digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
[[ "$cksum1" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum1 != $checksum)"

View File

@ -73,7 +73,7 @@ function test_corrective_recv
log_must zpool status -v $TESTPOOL
log_mustnot eval "zpool status -v $TESTPOOL | \
grep \"Permanent errors have been detected\""
typeset cksum=$(md5digest $file)
typeset cksum=$(xxh128digest $file)
[[ "$cksum" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum != $checksum)"
}
@ -96,7 +96,7 @@ log_must zfs create -o recordsize=1m -o primarycache=none \
log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync
log_must eval "echo 'aaaaaaaa' >> "$file
typeset checksum=$(md5digest $file)
typeset checksum=$(xxh128digest $file)
log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1

View File

@ -61,7 +61,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
"-o keyformat=passphrase $TESTPOOL/$TESTFS1"
log_must mkfile 1M /$TESTPOOL/$TESTFS1/$TESTFILE0
typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
typeset checksum=$(xxh128digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
log_must zfs snapshot $snap
@ -74,7 +74,7 @@ keystatus=$(get_prop keystatus $TESTPOOL/$TESTFS2)
log_must eval "echo $passphrase | zfs mount -l $TESTPOOL/$TESTFS2"
typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
typeset cksum1=$(xxh128digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
[[ "$cksum1" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum1 != $checksum)"
@ -85,7 +85,7 @@ keystatus=$(get_prop keystatus $TESTPOOL/$TESTFS1/c1)
log_fail "Expected keystatus unavailable, got $keystatus"
log_must eval "echo $passphrase | zfs mount -l $TESTPOOL/$TESTFS1/c1"
typeset cksum2=$(md5digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
typeset cksum2=$(xxh128digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
[[ "$cksum2" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum2 != $checksum)"

View File

@ -69,7 +69,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
log_must zfs snapshot $snap1
log_must mkfile 1M /$TESTPOOL/$TESTFS1/$TESTFILE0
typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
typeset checksum=$(xxh128digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
log_must zfs snapshot $snap2
@ -89,7 +89,7 @@ log_must zfs unload-key $TESTPOOL/$TESTFS2
log_must eval "zfs receive $TESTPOOL/$TESTFS2 < $ibackup"
log_must eval "echo $passphrase2 | zfs mount -l $TESTPOOL/$TESTFS2"
typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
typeset cksum1=$(xxh128digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
[[ "$cksum1" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum1 != $checksum)"

View File

@ -50,7 +50,7 @@ function dev_checksum
log_note "Compute checksum of '$dev'"
md5digest $dev ||
xxh128digest $dev ||
log_fail "Failed to compute checksum of '$dev'"
}

View File

@ -79,11 +79,11 @@ function test_devices_missing
log_must set_spa_load_verify_data 0
log_must zpool import -o readonly=on -d $DEVICE_DIR $TESTPOOL1
log_must verify_data_md5sums $MD5FILE
log_must verify_data_hashsums $MD5FILE
log_note "Try reading second batch of data, make sure pool doesn't" \
"get suspended."
verify_data_md5sums $MD5FILE >/dev/null 2>&1
verify_data_hashsums $MD5FILE >/dev/null 2>&1
log_must_busy zpool export $TESTPOOL1
@ -95,8 +95,8 @@ function test_devices_missing
log_must set_zfs_max_missing_tvds 0
log_must zpool import -d $DEVICE_DIR $TESTPOOL1
log_must verify_data_md5sums $MD5FILE
log_must verify_data_md5sums $MD5FILE2
log_must verify_data_hashsums $MD5FILE
log_must verify_data_hashsums $MD5FILE2
# Cleanup
log_must zpool destroy $TESTPOOL1

View File

@ -23,7 +23,7 @@
#
# STRATEGY:
# 1. Create a pool.
# 2. Generate files and remember their md5sum.
# 2. Generate files and remember their hashsum.
# 3. Note last synced txg.
# 4. Take a snapshot to make sure old blocks are not overwritten.
# 5. Perform zpool add/attach/detach/remove operation.
@ -134,7 +134,7 @@ function test_common
log_must zpool export $TESTPOOL1
if zpool import -d $DEVICE_DIR -T $txg $TESTPOOL1; then
verify_data_md5sums $MD5FILE && retval=0
verify_data_hashsums $MD5FILE && retval=0
log_must check_pool_config $TESTPOOL1 "$poolcheck"
log_must zpool destroy $TESTPOOL1

View File

@ -23,7 +23,7 @@
#
# STRATEGY:
# 1. Create a pool.
# 2. Generate files and remember their md5sum.
# 2. Generate files and remember their hashsum.
# 3. Sync a few times and note last synced txg.
# 4. Take a snapshot to make sure old blocks are not overwritten.
# 5. Initiate device replacement and export the pool. Special care must
@ -117,7 +117,7 @@ function test_replace_vdev
log_must zpool import -d $DEVICE_DIR -o readonly=on -T $txg $TESTPOOL1
log_must check_pool_config $TESTPOOL1 "$poolcreate"
log_must verify_data_md5sums $MD5FILE
log_must verify_data_hashsums $MD5FILE
log_must zpool export $TESTPOOL1
@ -137,7 +137,7 @@ function test_replace_vdev
log_must zpool import -d $DEVICE_DIR -T $txg $TESTPOOL1
log_must check_pool_config $TESTPOOL1 "$poolcreate"
log_must verify_data_md5sums $MD5FILE
log_must verify_data_hashsums $MD5FILE
# Cleanup
log_must zpool destroy $TESTPOOL1

View File

@ -45,8 +45,8 @@ export MYTESTFILE=$STF_SUITE/include/libtest.shlib
export CPATH=$TEST_BASE_DIR/cachefile.$$
export CPATHBKP=$TEST_BASE_DIR/cachefile.$$.bkp
export CPATHBKP2=$TEST_BASE_DIR/cachefile.$$.bkp2
export MD5FILE=$TEST_BASE_DIR/md5sums.$$
export MD5FILE2=$TEST_BASE_DIR/md5sums.$$.2
export MD5FILE=$TEST_BASE_DIR/hashsums.$$
export MD5FILE2=$TEST_BASE_DIR/hashsums.$$.2
export GROUP_NUM=3
typeset -i num=0

View File

@ -79,29 +79,29 @@ function write_some_data
# Checksum all the files and store digests in a file.
#
# newdata: overwrite existing files if false.
# md5file: file where to store md5 digests
# hashfile: file where to store xxh128 digests
# datasetname: base name for datasets
#
function _generate_data_common
{
typeset pool=$1
typeset newdata=$2
typeset md5file=$3
typeset hashfile=$3
typeset datasetname=$4
typeset -i datasets=3
typeset -i files=5
typeset -i blocks=10
[[ -n $md5file ]] && rm -f $md5file
[[ -n $hashfile ]] && rm -f $hashfile
for i in {1..$datasets}; do
( $newdata ) && log_must zfs create "$pool/$datasetname$i"
for j in {1..$files}; do
typeset file="/$pool/$datasetname$i/file$j"
dd if=/dev/urandom of=$file bs=128k count=$blocks > /dev/null
if [[ -n $md5file ]]; then
typeset cksum=$(md5digest $file)
echo $cksum $file >> $md5file
if [[ -n $hashfile ]]; then
typeset cksum=$(xxh128digest $file)
echo $cksum $file >> $hashfile
fi
done
( $newdata ) && sync_pool "$pool"
@ -113,39 +113,39 @@ function _generate_data_common
function generate_data
{
typeset pool=$1
typeset md5file="$2"
typeset hashfile="$2"
typeset datasetname=${3:-ds}
_generate_data_common $pool true "$md5file" $datasetname
_generate_data_common $pool true "$hashfile" $datasetname
}
function overwrite_data
{
typeset pool=$1
typeset md5file="$2"
typeset hashfile="$2"
typeset datasetname=${3:-ds}
_generate_data_common $1 false "$md5file" $datasetname
_generate_data_common $1 false "$hashfile" $datasetname
}
#
# Verify md5sums of every file in md5sum file $1.
# Verify hashsums of every file in hashsum file $1.
#
function verify_data_md5sums
function verify_data_hashsums
{
typeset md5file=$1
typeset hashfile=$1
if [[ ! -f $md5file ]]; then
log_note "md5 sums file '$md5file' doesn't exist"
if [[ ! -f $hashfile ]]; then
log_note "md5 sums file '$hashfile' doesn't exist"
return 1
fi
while read -r digest file; do
typeset digest1=$(md5digest $file)
typeset digest1=$(xxh128digest $file)
if [[ "$digest1" != "$digest" ]]; then
return 1
fi
done < $md5file
done < $hashfile
return 0
}

View File

@ -65,7 +65,7 @@ log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "unavail"
TESTFILE=/$TESTPOOL/data
log_must generate_random_file /$TESTPOOL/data $LARGE_FILE_SIZE
sync_pool $TESTPOOL
TESTFILE_MD5=$(md5digest $TESTFILE)
TESTFILE_MD5=$(xxh128digest $TESTFILE)
# 4. Execute scrub.
# add delay to I/O requests for remaining disk in pool
@ -89,7 +89,7 @@ log_must is_scan_restarted $TESTPOOL
# 8. Put another device offline and check if the test file checksum is correct.
log_must zpool offline $TESTPOOL $DISK2
CHECK_MD5=$(md5digest $TESTFILE)
CHECK_MD5=$(xxh128digest $TESTFILE)
[[ $CHECK_MD5 == $TESTFILE_MD5 ]] || \
log_fail "Checksums differ ($CHECK_MD5 != $TESTFILE_MD5)"
log_must zpool online $TESTPOOL $DISK2

View File

@ -61,8 +61,8 @@ function cleanup
function verify_copy
{
src_cksum=$(sha256digest $1)
dst_cksum=$(sha256digest $2)
src_cksum=$(xxh128digest $1)
dst_cksum=$(xxh128digest $2)
if [[ "$src_cksum" != "$dst_cksum" ]]; then
log_must ls -l $CP_TESTDIR

View File

@ -83,7 +83,7 @@ for ibs in "512" "$page_size" "131072"; do
-c $oblocks
log_must stride_dd -i $new_file -o $tmp_file -b $ibs \
-c $iblocks $iflags
log_must cmp_md5s $new_file $tmp_file
log_must cmp_xxh128 $new_file $tmp_file
log_must rm -f $new_file $tmp_file
# Verify direct write followed by a buffered read.
@ -91,7 +91,7 @@ for ibs in "512" "$page_size" "131072"; do
-c $oblocks $oflags
log_must stride_dd -i $new_file -o $tmp_file -b $ibs \
-c $iblocks
log_must cmp_md5s $new_file $tmp_file
log_must cmp_xxh128 $new_file $tmp_file
log_must rm -f $new_file $tmp_file
# Verify direct write followed by a direct read.
@ -99,7 +99,7 @@ for ibs in "512" "$page_size" "131072"; do
-c $oblocks $oflags
log_must stride_dd -i $new_file -o $tmp_file -b $ibs \
-c $iblocks $iflags
log_must cmp_md5s $new_file $tmp_file
log_must cmp_xxh128 $new_file $tmp_file
log_must rm -f $new_file $tmp_file
done
done

View File

@ -43,7 +43,7 @@ log_assert "ensure single-disk pool resumes properly after suspend and clear"
# create a file, and take a checksum, so we can compare later
log_must dd if=/dev/urandom of=$DATAFILE bs=128K count=1
typeset sum1=$(md5digest $DATAFILE)
typeset sum1=$(xxh128digest $DATAFILE)
# make a debug device that we can "unplug"
load_scsi_debug 100 1 1 1 '512b'
@ -94,7 +94,7 @@ log_must zpool export $TESTPOOL
log_must zpool import $TESTPOOL
# sum the file we wrote earlier
typeset sum2=$(md5digest /$TESTPOOL/file)
typeset sum2=$(xxh128digest /$TESTPOOL/file)
# make sure the checksums match
log_must test "$sum1" = "$sum2"

View File

@ -64,7 +64,7 @@ log_must zpool create $spool $VDEV0
log_must zfs create $spool/$sfs
typeset -i orig_count=$(zpool history $spool | wc -l)
typeset orig_md5=$(zpool history $spool | head -2 | md5digest)
typeset orig_hash=$(zpool history $spool | head -2 | xxh128digest)
typeset -i i=0
while ((i < 300)); do
zfs set compression=off $spool/$sfs
@ -79,7 +79,7 @@ done
TMPFILE=$TEST_BASE_DIR/spool.$$
zpool history $spool >$TMPFILE
typeset -i entry_count=$(wc -l < $TMPFILE)
typeset final_md5=$(head -2 $TMPFILE | md5digest)
typeset final_hash=$(head -2 $TMPFILE | xxh128digest)
grep -q 'zpool create' $TMPFILE ||
log_fail "'zpool create' was not found in pool history"
@ -91,7 +91,7 @@ grep -q 'zfs set compress' $TMPFILE ||
log_fail "'zfs set compress' was found in pool history"
# Verify that the creation of the pool was preserved in the history.
if [[ $orig_md5 != $final_md5 ]]; then
if [[ $orig_hash != $final_hash ]]; then
log_fail "zpool creation history was not preserved."
fi

View File

@ -41,7 +41,7 @@ bs=1024k
count=512
log_must dd if=/dev/urandom of=$TESTDIR/data bs=$bs count=$count
data_checksum=$(sha256digest $TESTDIR/data)
data_checksum=$(xxh128digest $TESTDIR/data)
# Test common large block configuration.
log_must zfs create -o recordsize=1m -o primarycache=metadata $TESTPOOL/gang
@ -50,7 +50,7 @@ mntpnt=$(get_prop mountpoint $TESTPOOL/gang)
log_must dd if=$TESTDIR/data of=$mntpnt/file bs=$bs count=$count
sync_pool $TESTPOOL
log_must dd if=$mntpnt/file of=$TESTDIR/out bs=$bs count=$count
out_checksum=$(sha256digest $TESTDIR/out)
out_checksum=$(xxh128digest $TESTDIR/out)
if [[ "$data_checksum" != "$out_checksum" ]]; then
log_fail "checksum mismatch ($data_checksum != $out_checksum)"
@ -74,7 +74,7 @@ mntpnt=$(get_prop mountpoint $TESTPOOL/gang)
log_must dd if=$TESTDIR/data of=$mntpnt/file bs=$bs count=$count
sync_pool $TESTPOOL
log_must dd if=$mntpnt/file of=$TESTDIR/out bs=$bs count=$count
out_checksum=$(sha256digest $TESTDIR/out)
out_checksum=$(xxh128digest $TESTDIR/out)
if [[ "$data_checksum" != "$out_checksum" ]]; then
log_fail "checksum mismatch ($data_checksum != $out_checksum)"

View File

@ -801,15 +801,15 @@ function recursive_cksum
{
case "$(uname)" in
FreeBSD)
find $1 -type f -exec sh -c 'sha256 -q {}; lsextattr -q \
system {} | sha256 -q; lsextattr -q user {} | sha256 -q' \
\; | sort | sha256 -q
find $1 -type f -exec sh -c 'xxh128sum {}; \
lsextattr -q system {} | xxh128sum; \
lsextattr -q user {} | xxh128sum' \; \
| sort -k 2 | awk '{ print $1 }' | xxh128digest
;;
*)
find $1 -type f -exec sh -c 'sha256sum {}; getfattr \
--absolute-names --only-values -d {} | sha256sum' \; | \
sort -k 2 | awk '{ print $1 }' | sha256sum | \
awk '{ print $1 }'
find $1 -type f -exec sh -c 'xxh128sum {}; getfattr \
--absolute-names --only-values -d {} | xxh128sum' \
\; | sort -k 2 | awk '{ print $1 }' | xxh128digest
;;
esac
}

View File

@ -50,8 +50,8 @@ typeset megs=8
log_must zfs create -V 256m -o compress=lz4 $vol
write_compressible $BACKDIR ${megs}m 2
md5_1=$(md5digest $data1)
md5_2=$(md5digest $data2)
hash1=$(xxh128digest $data1)
hash2=$(xxh128digest $data2)
log_must dd if=$data1 of=$voldev bs=1024k
log_must zfs snapshot $vol@snap
@ -63,8 +63,8 @@ verify_stream_size $BACKDIR/full $vol
verify_stream_size $BACKDIR/full $vol2
block_device_wait $voldev2
log_must dd if=$voldev2 of=$BACKDIR/copy bs=1024k count=$megs
md5=$(md5digest $BACKDIR/copy)
[[ $md5 = $md5_1 ]] || log_fail "md5 mismatch: $md5 != $md5_1"
hash=$(xxh128digest $BACKDIR/copy)
[[ $hash = $hash1 ]] || log_fail "hash mismatch: $hash != $hash1"
# Repeat, for an incremental send
log_must dd seek=$megs if=$data2 of=$voldev bs=1024k
@ -77,7 +77,7 @@ verify_stream_size $BACKDIR/inc $vol 90 $vol@snap
verify_stream_size $BACKDIR/inc $vol2 90 $vol2@snap
block_device_wait $voldev2
log_must dd skip=$megs if=$voldev2 of=$BACKDIR/copy bs=1024k count=$megs
md5=$(md5digest $BACKDIR/copy)
[[ $md5 = $md5_2 ]] || log_fail "md5 mismatch: $md5 != $md5_2"
hash=$(xxh128digest $BACKDIR/copy)
[[ $hash = $hash2 ]] || log_fail "hash mismatch: $hash != $hash2"
log_pass "Verify compressed send works with volumes"

View File

@ -101,8 +101,8 @@ block_device_wait
log_must mount $recvdev $recvmnt
md5_1=$(cat $mntpnt/* | md5digest)
md5_2=$(cat $recvmnt/* | md5digest)
[[ "$md5_1" == "$md5_2" ]] || log_fail "md5 mismatch: $md5_1 != $md5_2"
hash1=$(cat $mntpnt/* | xxh128digest)
hash2=$(cat $recvmnt/* | xxh128digest)
[[ "$hash1" == "$hash2" ]] || log_fail "hash mismatch: $hash1 != $hash2"
log_pass "zfs can receive raw, recursive send streams"

View File

@ -76,7 +76,7 @@ log_must zfs create -o keyformat=passphrase -o keylocation=file://$keyfile \
log_must mkfile 1M /$TESTPOOL/ds/$TESTFILE0
log_must cp /$TESTPOOL/ds/$TESTFILE0 /$TESTPOOL/crypt/$TESTFILE0
typeset cksum=$(md5digest /$TESTPOOL/ds/$TESTFILE0)
typeset cksum=$(xxh128digest /$TESTPOOL/ds/$TESTFILE0)
log_must zfs snap -r $snap
log_must zfs snap -r $snap2
@ -129,7 +129,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5digest /$ds/$TESTFILE0)
recv_cksum=$(xxh128digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds
@ -153,7 +153,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5digest /$ds/$TESTFILE0)
recv_cksum=$(xxh128digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds
@ -171,7 +171,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5digest /$ds/$TESTFILE0)
recv_cksum=$(xxh128digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds
@ -185,7 +185,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt"
log_must test "$(get_prop 'encryption' $ds)" == "aes-256-gcm"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5digest /$ds/$TESTFILE0)
recv_cksum=$(xxh128digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds
@ -199,7 +199,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt"
log_must test "$(get_prop 'encryption' $ds)" == "aes-256-gcm"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5digest /$ds/$TESTFILE0)
recv_cksum=$(xxh128digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds
@ -213,7 +213,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt"
log_must test "$(get_prop 'encryption' $ds)" == "aes-256-gcm"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5digest /$ds/$TESTFILE0)
recv_cksum=$(xxh128digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds

View File

@ -52,16 +52,8 @@ log_onexit cleanup
function recursive_cksum
{
case "$(uname)" in
FreeBSD)
find $1 -type f -exec sha256 -q {} + | \
sort | sha256digest
;;
*)
find $1 -type f -exec sha256sum {} + | \
sort -k 2 | awk '{ print $1 }' | sha256digest
;;
esac
find $1 -type f -exec xxh128sum {} + | \
sort -k 2 | awk '{ print $1 }' | xxh128digest
}
log_assert "Verify 'zfs send -w' works with many different file layouts"

View File

@ -65,7 +65,7 @@ function send_and_verify
">$BACKDIR/pool-snap1-snap2"
log_must eval "zfs receive $recvfs < $BACKDIR/pool-snap1-snap2"
log_must cmp_md5s /$sendfs/file1 /$recvfs/file1
log_must cmp_xxh128 /$sendfs/file1 /$recvfs/file1
}
# By default sending hole_birth times is disabled. This functionality needs

View File

@ -118,7 +118,7 @@ log_must rmdir /$TESTPOOL/$TESTFS/dir_to_delete
log_must mkdir -p $TESTDIR
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS/payload \
oflag=sync bs=1k count=8
typeset checksum=$(sha256digest /$TESTPOOL/$TESTFS/payload)
typeset checksum=$(xxh128digest /$TESTPOOL/$TESTFS/payload)
# TX_WRITE (small file with ordering)
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS/small_file \
@ -251,7 +251,7 @@ log_note "Verify working set diff:"
log_must replay_directory_diff $TESTDIR/copy /$TESTPOOL/$TESTFS
log_note "Verify file checksum:"
typeset checksum1=$(sha256digest /$TESTPOOL/$TESTFS/payload)
typeset checksum1=$(xxh128digest /$TESTPOOL/$TESTFS/payload)
[[ "$checksum1" == "$checksum" ]] || \
log_fail "checksum mismatch ($checksum1 != $checksum)"

View File

@ -140,7 +140,7 @@ fi
#
# 4. Generate checksums for all ext4 files.
#
typeset checksum=$(cat $MNTPNT/* | sha256digest)
typeset checksum=$(cat $MNTPNT/* | xxh128digest)
#
# 5. Unmount filesystem and export the pool
@ -172,7 +172,7 @@ log_note "Verify current block usage:"
log_must zdb -bcv $TESTPOOL
log_note "Verify checksums"
typeset checksum1=$(cat $MNTPNT/* | sha256digest)
typeset checksum1=$(cat $MNTPNT/* | xxh128digest)
[[ "$checksum1" == "$checksum" ]] || \
log_fail "checksum mismatch ($checksum1 != $checksum)"