mirror of
https://git.proxmox.com/git/mirror_zfs.git
synced 2026-05-23 02:44:41 +03:00
OpenZFS 7252 - compressed zfs send / receive
OpenZFS 7252 - compressed zfs send / receive OpenZFS 7628 - create long versions of ZFS send / receive options Authored by: Dan Kimmel <dan.kimmel@delphix.com> Reviewed by: George Wilson <george.wilson@delphix.com> Reviewed by: John Kennedy <john.kennedy@delphix.com> Reviewed by: Matthew Ahrens <mahrens@delphix.com> Reviewed by: Paul Dagnelie <pcd@delphix.com> Reviewed by: Pavel Zakharov <pavel.zakharov@delphix.com> Reviewed by: Sebastien Roy <sebastien.roy@delphix.com> Reviewed by: David Quigley <dpquigl@davequigley.com> Reviewed by: Thomas Caputi <tcaputi@datto.com> Approved by: Dan McDonald <danmcd@omniti.com> Reviewed by: David Quigley <dpquigl@davequigley.com> Reviewed-by: loli10K <ezomori.nozomu@gmail.com> Ported-by: bunder2015 <omfgbunder@gmail.com> Ported-by: Don Brady <don.brady@intel.com> Ported-by: Brian Behlendorf <behlendorf1@llnl.gov> Porting Notes: - Most of 7252 was already picked up during ABD work. This commit represents the gap from the final commit to openzfs. - Fixed split_large_blocks check in do_dump() - An alternate version of the write_compressible() function was implemented for Linux which does not depend on fio. The behavior of fio differs significantly based on the exact version. - mkholes was replaced with truncate for Linux. OpenZFS-issue: https://www.illumos.org/issues/7252 OpenZFS-commit: https://github.com/openzfs/openzfs/commit/5602294 Closes #6067
This commit is contained in:
Dan Kimmel
committed by
Brian Behlendorf
Brian Behlendorf
parent
7a25f0891e
commit
a7004725d0
@@ -134,7 +134,8 @@ export ZFS_FILES='zdb
|
||||
arc_summary.py
|
||||
arcstat.py
|
||||
dbufstat.py
|
||||
zed'
|
||||
zed
|
||||
zstreamdump'
|
||||
|
||||
export ZFSTEST_FILES='chg_usr_exec
|
||||
devname2devid
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
#
|
||||
|
||||
. ${STF_TOOLS}/include/logapi.shlib
|
||||
. ${STF_SUITE}/include/math.shlib
|
||||
|
||||
#
|
||||
# Apply constrained path when available. This is required since the
|
||||
@@ -3111,6 +3112,73 @@ function get_min
|
||||
echo $min
|
||||
}
|
||||
|
||||
#
|
||||
# Generate a random number between 1 and the argument.
|
||||
#
|
||||
function random
|
||||
{
|
||||
typeset max=$1
|
||||
echo $(( ($RANDOM % $max) + 1 ))
|
||||
}
|
||||
|
||||
# Write data that can be compressed into a directory
|
||||
function write_compressible
|
||||
{
|
||||
typeset dir=$1
|
||||
typeset megs=$2
|
||||
typeset nfiles=${3:-1}
|
||||
typeset bs=${4:-1024k}
|
||||
typeset fname=${5:-file}
|
||||
|
||||
[[ -d $dir ]] || log_fail "No directory: $dir"
|
||||
|
||||
# Under Linux fio is not currently used since its behavior can
|
||||
# differ significantly across versions. This includes missing
|
||||
# command line options and cases where the --buffer_compress_*
|
||||
# options fail to behave as expected.
|
||||
if is_linux; then
|
||||
typeset file_bytes=$(to_bytes $megs)
|
||||
typeset bs_bytes=4096
|
||||
typeset blocks=$(($file_bytes / $bs_bytes))
|
||||
|
||||
for (( i = 0; i < $nfiles; i++ )); do
|
||||
truncate -s $file_bytes $dir/$fname.$i
|
||||
|
||||
# Write every third block to get 66% compression.
|
||||
for (( j = 0; j < $blocks; j += 3 )); do
|
||||
dd if=/dev/urandom of=$dir/$fname.$i \
|
||||
seek=$j bs=$bs_bytes count=1 \
|
||||
conv=notrunc >/dev/null 2>&1
|
||||
done
|
||||
done
|
||||
else
|
||||
log_must eval "fio \
|
||||
--name=job \
|
||||
--fallocate=0 \
|
||||
--minimal \
|
||||
--randrepeat=0 \
|
||||
--buffer_compress_percentage=66 \
|
||||
--buffer_compress_chunk=4096 \
|
||||
--directory=$dir \
|
||||
--numjobs=$nfiles \
|
||||
--nrfiles=$nfiles \
|
||||
--rw=write \
|
||||
--bs=$bs \
|
||||
--filesize=$megs \
|
||||
--filename_format='$fname.\$jobnum' >/dev/null"
|
||||
fi
|
||||
}
|
||||
|
||||
function get_objnum
|
||||
{
|
||||
typeset pathname=$1
|
||||
typeset objnum
|
||||
|
||||
[[ -e $pathname ]] || log_fail "No such file or directory: $pathname"
|
||||
objnum=$(stat -c %i $pathname)
|
||||
echo $objnum
|
||||
}
|
||||
|
||||
#
|
||||
# Synchronize all the data in pool
|
||||
#
|
||||
|
||||
@@ -41,3 +41,28 @@ function within_percent
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
#
|
||||
# Return 0 if the human readable string of the form <value>[suffix] can
|
||||
# be converted to bytes. Allow suffixes are shown in the table below.
|
||||
#
|
||||
function to_bytes
|
||||
{
|
||||
typeset size=$1
|
||||
typeset value=$(echo "$size" | grep -o '[0-9]\+')
|
||||
|
||||
case $size in
|
||||
*PB|*pb|*P|*p) factor='1024^5' ;;
|
||||
*TB|*tb|*T|*t) factor='1024^4' ;;
|
||||
*GB|*gb|*G|*g) factor='1024^3' ;;
|
||||
*MB|*mb|*M|*m) factor='1024^2' ;;
|
||||
*KB|*kb|*K|*k) factor='1024^1' ;;
|
||||
*B|*b) factor='1024^0' ;;
|
||||
*[!0-9.]*) return 1 ;;
|
||||
*) factor='1024^0' ;;
|
||||
esac
|
||||
|
||||
echo "$value * ($factor)" | bc
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -13,11 +13,26 @@
|
||||
# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
|
||||
#
|
||||
|
||||
typeset -a compress_props=('on' 'off' 'lzjb' 'gzip' 'gzip-1' 'gzip-2' 'gzip-3'
|
||||
'gzip-4' 'gzip-5' 'gzip-6' 'gzip-7' 'gzip-8' 'gzip-9' 'zle')
|
||||
typeset -a compress_prop_vals=('on' 'off' 'lzjb' 'gzip' 'gzip-1' 'gzip-2'
|
||||
'gzip-3' 'gzip-4' 'gzip-5' 'gzip-6' 'gzip-7' 'gzip-8' 'gzip-9' 'zle' 'lz4')
|
||||
typeset -a checksum_prop_vals=('on' 'off' 'fletcher2' 'fletcher4' 'sha256'
|
||||
'noparity' 'sha512' 'skein' 'edonr')
|
||||
typeset -a recsize_prop_vals=('512' '1024' '2048' '4096' '8192' '16384'
|
||||
'32768' '65536' '131072' '262144' '524288' '1048576')
|
||||
typeset -a canmount_prop_vals=('on' 'off' 'noauto')
|
||||
typeset -a copies_prop_vals=('1' '2' '3')
|
||||
typeset -a logbias_prop_vals=('latency' 'throughput')
|
||||
typeset -a primarycache_prop_vals=('all' 'none' 'metadata')
|
||||
typeset -a redundant_metadata_prop_vals=('all' 'most')
|
||||
typeset -a secondarycache_prop_vals=('all' 'none' 'metadata')
|
||||
typeset -a snapdir_prop_vals=('hidden' 'visible')
|
||||
typeset -a sync_prop_vals=('standard' 'always' 'disabled')
|
||||
|
||||
typeset -a checksum_props=('on' 'off' 'fletcher2' 'fletcher4' 'sha256' 'sha512'
|
||||
'edonr' 'skein' 'noparity')
|
||||
typeset -a fs_props=('compress' 'checksum' 'recsize'
|
||||
'canmount' 'copies' 'logbias' 'primarycache' 'redundant_metadata'
|
||||
'secondarycache' 'snapdir' 'sync')
|
||||
typeset -a vol_props=('compress' 'checksum' 'copies' 'logbias' 'primarycache'
|
||||
'secondarycache' 'redundant_metadata' 'sync')
|
||||
|
||||
#
|
||||
# Given the property array passed in, return 'num_props' elements to the
|
||||
@@ -45,20 +60,81 @@ function get_rand_prop
|
||||
|
||||
function get_rand_compress
|
||||
{
|
||||
get_rand_prop compress_props $1 2
|
||||
get_rand_prop compress_prop_vals $1 2
|
||||
}
|
||||
|
||||
function get_rand_compress_any
|
||||
{
|
||||
get_rand_prop compress_props $1 0
|
||||
get_rand_prop compress_prop_vals $1 0
|
||||
}
|
||||
|
||||
function get_rand_checksum
|
||||
{
|
||||
get_rand_prop checksum_props $1 2
|
||||
get_rand_prop checksum_prop_vals $1 2
|
||||
}
|
||||
|
||||
function get_rand_checksum_any
|
||||
{
|
||||
get_rand_prop checksum_props $1 0
|
||||
get_rand_prop checksum_prop_vals $1 0
|
||||
}
|
||||
|
||||
function get_rand_recsize
|
||||
{
|
||||
get_rand_prop recsize_prop_vals $1 0
|
||||
}
|
||||
|
||||
function get_rand_large_recsize
|
||||
{
|
||||
get_rand_prop recsize_prop_vals $1 9
|
||||
}
|
||||
|
||||
#
|
||||
# Functions to toggle on/off properties
|
||||
#
|
||||
typeset -a binary_props=('atime' 'devices' 'exec' 'nbmand' 'readonly' 'setuid'
|
||||
'xattr' 'zoned')
|
||||
|
||||
function toggle_prop
|
||||
{
|
||||
typeset ds=$1
|
||||
typeset prop=$2
|
||||
|
||||
datasetexists $ds || log_fail "$ds does not exist"
|
||||
typeset val=$(get_prop $prop $ds)
|
||||
typeset newval='off'
|
||||
|
||||
[[ $val = $newval ]] && newval='on'
|
||||
log_must zfs set $prop=$newval $ds
|
||||
}
|
||||
|
||||
function toggle_binary_props
|
||||
{
|
||||
typeset ds=$1
|
||||
typeset prop
|
||||
|
||||
for prop in "${binary_props[@]}"; do
|
||||
toggle_prop $ds $prop
|
||||
done
|
||||
}
|
||||
|
||||
function randomize_ds_props
|
||||
{
|
||||
typeset ds=$1
|
||||
typeset prop proplist val
|
||||
|
||||
datasetexists $ds || log_fail "$ds does not exist"
|
||||
if ds_is_volume $ds; then
|
||||
toggle_prop $ds readonly
|
||||
proplist="${vol_props[@]}"
|
||||
elif ds_is_filesystem $ds; then
|
||||
toggle_binary_props $ds
|
||||
proplist="${fs_props[@]}"
|
||||
else
|
||||
log_fail "$ds is neither a volume nor a file system"
|
||||
fi
|
||||
|
||||
for prop in $proplist; do
|
||||
typeset val=$(get_rand_prop "${prop}_prop_vals" 1 0)
|
||||
log_must zfs set $prop=$val $ds
|
||||
done
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user