c/mirror_zfs
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_zfs.git synced 2024-12-26 19:19:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

Refactor checksum operations in tests

Browse Source 
md5sum in particular but also sha256sum to a lesser extent is used
in several areas of the test suite for computing checksums. The vast
majority of invocations are followed by `| awk '{ print $1 }'`.

Introduce functions to wrap up `md5sum $file | awk '{ print $1 }'` and
likewise for sha256sum. These also serve as a convenient interface for
alternative implementations on other platforms.

Reviewed-by: Igor Kozhukhov <igor@dilos.org>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Ryan Moeller <ryan@ixsystems.com>
Closes #9280
This commit is contained in:
Ryan Moeller 2019-09-05 12:51:59 -04:00 committed by Tony Hutter
parent 8c9c049502
commit 27dda98b88
14 changed files with 76 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tests/zfs-tests/include/libtest.shlib
View File

@ -3562,3 +3562,25 @@ function mdb_ctf_set_int
return 0
}
#
# Compute MD5 digest for given file or stdin if no file given.
# Note: file path must not contain spaces
#
function md5digest
{
typeset file=$1
md5sum -b $file | awk '{ print $1 }'
}
#
# Compute SHA256 digest for given file or stdin if no file given.
# Note: file path must not contain spaces
#
function sha256digest
{
typeset file=$1
sha256sum -b $file | awk '{ print $1 }'
}

6
tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_from_encrypted.ksh
View File

@ -59,7 +59,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
"-o keyformat=passphrase $TESTPOOL/$TESTFS2"
log_must mkfile 1M /$TESTPOOL/$TESTFS2/$TESTFILE0
typeset checksum=$(md5sum /$TESTPOOL/$TESTFS2/$TESTFILE0 | awk '{ print $1 }')
typeset checksum=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
log_must zfs snapshot $snap
@ -69,14 +69,14 @@ log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c1"
crypt=$(get_prop encryption $TESTPOOL/$TESTFS1/c1)
[[ "$crypt" == "off" ]] || log_fail "Received unencrypted stream as encrypted"
typeset cksum1=$(md5sum /$TESTPOOL/$TESTFS1/c1/$TESTFILE0 | awk '{ print $1 }')
typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
[[ "$cksum1" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum1 != $checksum)"
log_note "Verify ZFS can receive into an encrypted child"
log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS2/c1"
typeset cksum2=$(md5sum /$TESTPOOL/$TESTFS2/c1/$TESTFILE0 | awk '{ print $1 }')
typeset cksum2=$(md5digest /$TESTPOOL/$TESTFS2/c1/$TESTFILE0)
[[ "$cksum2" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum2 != $checksum)"

8
tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_raw.ksh
View File

@ -60,8 +60,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
"-o keyformat=passphrase $TESTPOOL/$TESTFS1"
log_must mkfile 1M /$TESTPOOL/$TESTFS1/$TESTFILE0
typeset checksum=$(md5sum /$TESTPOOL/$TESTFS1/$TESTFILE0 | \
awk '{ print $1 }')
typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
log_must zfs snapshot $snap
@ -74,7 +73,7 @@ keystatus=$(get_prop keystatus $TESTPOOL/$TESTFS2)
log_must eval "echo $passphrase | zfs mount -l $TESTPOOL/$TESTFS2"
typeset cksum1=$(md5sum /$TESTPOOL/$TESTFS2/$TESTFILE0 | awk '{ print $1 }')
typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
[[ "$cksum1" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum1 != $checksum)"
@ -85,8 +84,7 @@ keystatus=$(get_prop keystatus $TESTPOOL/$TESTFS1/c1)
log_fail "Expected keystatus unavailable, got $keystatus"
log_must eval "echo $passphrase | zfs mount -l $TESTPOOL/$TESTFS1/c1"
typeset cksum2=$(md5sum /$TESTPOOL/$TESTFS1/c1/$TESTFILE0 | \
awk '{ print $1 }')
typeset cksum2=$(md5digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
[[ "$cksum2" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum2 != $checksum)"

4
tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_raw_incremental.ksh
View File

@ -69,7 +69,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
log_must zfs snapshot $snap1
log_must mkfile 1M /$TESTPOOL/$TESTFS1/$TESTFILE0
typeset checksum=$(md5sum /$TESTPOOL/$TESTFS1/$TESTFILE0 | awk '{ print $1 }')
typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
log_must zfs snapshot $snap2
@ -89,7 +89,7 @@ log_must zfs unload-key $TESTPOOL/$TESTFS2
log_must eval "zfs receive $TESTPOOL/$TESTFS2 < $ibackup"
log_must eval "echo $passphrase2 | zfs mount -l $TESTPOOL/$TESTFS2"
typeset cksum1=$(md5sum /$TESTPOOL/$TESTFS2/$TESTFILE0 | awk '{ print $1 }')
typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
[[ "$cksum1" == "$checksum" ]] || \
log_fail "Checksums differ ($cksum1 != $checksum)"

2
tests/zfs-tests/tests/functional/cli_root/zpool_import/import_cachefile_shared_device.ksh
View File

@ -50,7 +50,7 @@ function dev_checksum
log_note "Compute checksum of '$dev'"
checksum=$(md5sum $dev)
checksum=$(md5digest $dev)
if [[ $? -ne 0 ]]; then
log_fail "Failed to compute checksum of '$dev'"
return 1

20
tests/zfs-tests/tests/functional/cli_root/zpool_import/zpool_import.kshlib
View File

@ -79,10 +79,10 @@ function write_some_data
#
# Create/overwrite a few datasets with files.
# Apply md5sum on all the files and store checksums in a file.
# Checksum all the files and store digests in a file.
#
# newdata: overwrite existing files if false.
# md5file: file where to store md5sums
# md5file: file where to store md5 digests
# datasetname: base name for datasets
#
function _generate_data_common
@ -102,7 +102,10 @@ function _generate_data_common
for j in {1..$files}; do
typeset file="/$pool/$datasetname$i/file$j"
dd if=/dev/urandom of=$file bs=128k count=$blocks > /dev/null
[[ -n $md5file ]] && md5sum $file >> $md5file
if [[ -n $md5file ]]; then
typeset cksum=$(md5digest $file)
echo $cksum $file >> $md5file
fi
done
( $newdata ) && sync_pool "$pool"
done
@ -140,8 +143,15 @@ function verify_data_md5sums
return 1
fi
md5sum -c --quiet $md5file
return $?
cat $md5file | \
while read digest file; do
typeset digest1=$(md5digest $file)
if [[ "$digest1" != "$digest" ]]; then
return 1
fi
done
return 0
}
#

9
tests/zfs-tests/tests/functional/cli_root/zpool_reopen/zpool_reopen_003_pos.ksh
View File

@ -42,7 +42,6 @@ verify_runnable "global"
function cleanup
{
log_must zinject -c all
rm -f $TESTFILE_MD5 2>/dev/null
# bring back removed disk online for further tests
insert_disk $REMOVED_DISK $scsi_host
poolexists $TESTPOOL && destroy_pool $TESTPOOL
@ -64,9 +63,8 @@ log_must check_state $TESTPOOL "$REMOVED_DISK_ID" "unavail"
# 3. Write a test file to the pool and calculate its checksum.
TESTFILE=/$TESTPOOL/data
TESTFILE_MD5=$(mktemp --tmpdir=/var/tmp)
log_must generate_random_file /$TESTPOOL/data $LARGE_FILE_SIZE
log_must md5sum $TESTFILE > $TESTFILE_MD5
TESTFILE_MD5=$(md5digest $TESTFILE)
# 4. Execute scrub.
# add delay to I/O requests for remaining disk in pool
@ -90,12 +88,13 @@ log_must is_scan_restarted $TESTPOOL
# 8. Put another device offline and check if the test file checksum is correct.
log_must zpool offline $TESTPOOL $DISK2
log_must md5sum -c $TESTFILE_MD5
CHECK_MD5=$(md5digest $TESTFILE)
[[ $CHECK_MD5 == $TESTFILE_MD5 ]] || \
log_fail "Checksums differ ($CHECK_MD5 != $TESTFILE_MD5)"
log_must zpool online $TESTPOOL $DISK2
sleep 1
# clean up
rm -f $TESTFILE_MD5 2>/dev/null
log_must zpool destroy $TESTPOOL
log_pass "Zpool reopen test successful"

6
tests/zfs-tests/tests/functional/history/history_003_pos.ksh
View File

@ -65,9 +65,7 @@ log_must zpool create $spool $VDEV0
log_must zfs create $spool/$sfs
typeset -i orig_count=$(zpool history $spool | wc -l)
typeset orig_md5=$(zpool history $spool | head -2 | md5sum | \
awk '{print $1}')
typeset orig_md5=$(zpool history $spool | head -2 | md5digest)
typeset -i i=0
while ((i < 300)); do
zfs set compression=off $spool/$sfs
@ -82,7 +80,7 @@ done
TMPFILE=$TEST_BASE_DIR/spool.$$
zpool history $spool >$TMPFILE
typeset -i entry_count=$(wc -l $TMPFILE | awk '{print $1}')
typeset final_md5=$(head -2 $TMPFILE | md5sum | awk '{print $1}')
typeset final_md5=$(head -2 $TMPFILE | md5digest)
grep 'zpool create' $TMPFILE >/dev/null 2>&1 ||
log_fail "'zpool create' was not found in pool history"

11
tests/zfs-tests/tests/functional/rsend/rsend.kshlib
View File

@ -158,14 +158,9 @@ function cmp_md5s {
typeset file1=$1
typeset file2=$2
eval md5sum $file1 | awk '{ print $1 }' > $BACKDIR/md5_file1
eval md5sum $file2 | awk '{ print $1 }' > $BACKDIR/md5_file2
diff $BACKDIR/md5_file1 $BACKDIR/md5_file2
typeset -i ret=$?
rm -f $BACKDIR/md5_file1 $BACKDIR/md5_file2
return $ret
typeset sum1=$(md5digest $file1)
typeset sum2=$(md5digest $file2)
test "$sum1" = "$sum2"
}
#

10
tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh
View File

@ -49,8 +49,8 @@ typeset megs=8
log_must zfs create -V 256m -o compress=lz4 $vol
write_compressible $BACKDIR ${megs}m 2
md5_1=$(md5sum $data1 | awk '{print $1}')
md5_2=$(md5sum $data2 | awk '{print $1}')
md5_1=$(md5digest $data1)
md5_2=$(md5digest $data2)
log_must dd if=$data1 of=$voldev bs=1024k
log_must zfs snapshot $vol@snap
@ -60,8 +60,7 @@ log_must eval "zfs recv -d $POOL2 <$BACKDIR/full"
verify_stream_size $BACKDIR/full $vol
verify_stream_size $BACKDIR/full $vol2
md5=$(dd if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5sum | \
awk '{print $1}')
md5=$(dd if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5digest)
[[ $md5 = $md5_1 ]] || log_fail "md5 mismatch: $md5 != $md5_1"
# Repeat, for an incremental send
@ -73,8 +72,7 @@ log_must eval "zfs recv -d $POOL2 <$BACKDIR/inc"
verify_stream_size $BACKDIR/inc $vol 90 $vol@snap
verify_stream_size $BACKDIR/inc $vol2 90 $vol2@snap
md5=$(dd skip=$megs if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5sum | \
awk '{print $1}')
md5=$(dd skip=$megs if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5digest)
[[ $md5 = $md5_2 ]] || log_fail "md5 mismatch: $md5 != $md5_2"
log_pass "Verify compressed send works with volumes"

4
tests/zfs-tests/tests/functional/rsend/send-wDR_encrypted_zvol.ksh
View File

@ -86,8 +86,8 @@ block_device_wait
log_must mount $recvdev $recvmnt
md5_1=$(cat $mntpnt/* | md5sum | awk '{print $1}')
md5_2=$(cat $recvmnt/* | md5sum | awk '{print $1}')
md5_1=$(cat $mntpnt/* | md5digest)
md5_2=$(cat $recvmnt/* | md5digest)
[[ "$md5_1" == "$md5_2" ]] || log_fail "md5 mismatch: $md5_1 != $md5_2"
log_pass "zfs can receive raw, recursive, and deduplicated send streams"

12
tests/zfs-tests/tests/functional/rsend/send_encrypted_props.ksh
View File

@ -75,7 +75,7 @@ log_must zfs create -o keyformat=passphrase -o keylocation=file://$keyfile \
log_must mkfile 1M /$TESTPOOL/ds/$TESTFILE0
log_must cp /$TESTPOOL/ds/$TESTFILE0 /$TESTPOOL/crypt/$TESTFILE0
typeset cksum=$(md5sum /$TESTPOOL/ds/$TESTFILE0 | awk '{ print $1 }')
typeset cksum=$(md5digest /$TESTPOOL/ds/$TESTFILE0)
log_must zfs snap -r $snap
log_must zfs snap -r $esnap
@ -127,7 +127,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
recv_cksum=$(md5digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds
@ -143,7 +143,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
recv_cksum=$(md5digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds
@ -161,7 +161,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$ds"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'keylocation' $ds)" == "file://$keyfile"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
recv_cksum=$(md5digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds
@ -175,7 +175,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt"
log_must test "$(get_prop 'encryption' $ds)" == "aes-256-ccm"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
recv_cksum=$(md5digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds
@ -189,7 +189,7 @@ log_must test "$(get_prop 'encryptionroot' $ds)" == "$TESTPOOL/crypt"
log_must test "$(get_prop 'encryption' $ds)" == "aes-256-ccm"
log_must test "$(get_prop 'keyformat' $ds)" == "passphrase"
log_must test "$(get_prop 'mounted' $ds)" == "yes"
recv_cksum=$(md5sum /$ds/$TESTFILE0 | awk '{ print $1 }')
recv_cksum=$(md5digest /$ds/$TESTFILE0)
log_must test "$recv_cksum" == "$cksum"
log_must zfs destroy -r $ds

14
tests/zfs-tests/tests/functional/slog/slog_replay_fs_001.ksh
View File

@ -58,14 +58,8 @@
verify_runnable "global"
function cleanup_fs
{
rm -f $TESTDIR/checksum
cleanup
}
log_assert "Replay of intent log succeeds."
log_onexit cleanup_fs
log_onexit cleanup
log_must setup
#
@ -115,7 +109,7 @@ log_must rmdir /$TESTPOOL/$TESTFS/dir_to_delete
# Create a simple validation payload
log_must mkdir -p $TESTDIR
log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS/payload bs=1k count=8
log_must eval "sha256sum -b /$TESTPOOL/$TESTFS/payload >$TESTDIR/checksum"
typeset checksum=$(sha256digest /$TESTPOOL/$TESTFS/payload)
# TX_WRITE (small file with ordering)
log_must mkfile 1k /$TESTPOOL/$TESTFS/small_file
@ -210,6 +204,8 @@ log_note "Verify working set diff:"
log_must diff -r /$TESTPOOL/$TESTFS $TESTDIR/copy
log_note "Verify file checksum:"
log_must sha256sum -c $TESTDIR/checksum
typeset checksum1=$(sha256digest /$TESTPOOL/$TESTFS/payload)
[[ "$checksum1" == "$checksum" ]] || \
log_fail "checksum mismatch ($checksum1 != $checksum)"
log_pass "Replay of intent log succeeds."

6
tests/zfs-tests/tests/functional/slog/slog_replay_volume.ksh
View File

@ -128,7 +128,7 @@ fi
#
# 4. Generate checksums for all ext4 files.
#
log_must sha256sum -b $MNTPNT/* >$TESTDIR/checksum
typeset checksum=$(cat $MNTPNT/* | sha256digest)
#
# 5. Unmount filesystem and export the pool
@ -160,6 +160,8 @@ log_note "Verify current block usage:"
log_must zdb -bcv $TESTPOOL
log_note "Verify checksums"
log_must sha256sum -c $TESTDIR/checksum
typeset checksum1=$(cat $MNTPNT/* | sha256digest)
[[ "$checksum1" == "$checksum" ]] || \
log_fail "checksum mismatch ($checksum1 != $checksum)"
log_pass "Replay of intent log succeeds."