OpenZFS 7252 - compressed zfs send / receive

OpenZFS 7252 - compressed zfs send / receive
OpenZFS 7628 - create long versions of ZFS send / receive options

Authored by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: John Kennedy <john.kennedy@delphix.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Paul Dagnelie <pcd@delphix.com>
Reviewed by: Pavel Zakharov <pavel.zakharov@delphix.com>
Reviewed by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed by: David Quigley <dpquigl@davequigley.com>
Reviewed by: Thomas Caputi <tcaputi@datto.com>
Approved by: Dan McDonald <danmcd@omniti.com>
Reviewed by: David Quigley <dpquigl@davequigley.com>
Reviewed-by: loli10K <ezomori.nozomu@gmail.com>
Ported-by: bunder2015 <omfgbunder@gmail.com>
Ported-by: Don Brady <don.brady@intel.com>
Ported-by: Brian Behlendorf <behlendorf1@llnl.gov>

Porting Notes:
- Most of 7252 was already picked up during ABD work.  This
  commit represents the gap from the final commit to openzfs.
- Fixed split_large_blocks check in do_dump()
- An alternate version of the write_compressible() function was
  implemented for Linux which does not depend on fio.  The behavior
  of fio differs significantly based on the exact version.
- mkholes was replaced with truncate for Linux.

OpenZFS-issue: https://www.illumos.org/issues/7252
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/5602294
Closes #6067

tests/runfiles/linux.run

@@ -568,15 +568,21 @@ tests = ['reservation_001_pos', 'reservation_002_pos', 'reservation_003_pos',
 tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
 
 # DISABLED:
-# rsend_008_pos - Fails for OpenZFS on illumos
+# rsend_008_pos - https://github.com/zfsonlinux/zfs/issues/6066
 # rsend_009_pos - https://github.com/zfsonlinux/zfs/issues/5887
-# rsend_020_pos - ASSERTs in dump_record()
 [tests/functional/rsend]
 tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
     'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos',
     'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos',
-    'rsend_013_pos', 'rsend_014_pos', 'rsend_019_pos',
-    'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos']
+    'rsend_013_pos', 'rsend_014_pos',
+    'rsend_019_pos', 'rsend_020_pos',
+    'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos',
+    'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props',
+    'send-c_incremental', 'send-c_volume', 'send-c_zstreamdump',
+    'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
+    'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
+    'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
+    'send-c_recv_dedup']
 
 [tests/functional/scrub_mirror]
 tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos',

tests/zfs-tests/include/commands.cfg

@@ -134,7 +134,8 @@ export ZFS_FILES='zdb
     arc_summary.py
     arcstat.py
     dbufstat.py
-    zed'
+    zed
+    zstreamdump'
 
 export ZFSTEST_FILES='chg_usr_exec
     devname2devid

tests/zfs-tests/include/libtest.shlib

@@ -29,6 +29,7 @@
 #
 
 . ${STF_TOOLS}/include/logapi.shlib
+. ${STF_SUITE}/include/math.shlib
 
 #
 # Apply constrained path when available.  This is required since the
@@ -3111,6 +3112,73 @@ function get_min
 	echo $min
 }
 
+#
+# Generate a random number between 1 and the argument.
+#
+function random
+{
+        typeset max=$1
+        echo $(( ($RANDOM % $max) + 1 ))
+}
+
+# Write data that can be compressed into a directory
+function write_compressible
+{
+	typeset dir=$1
+	typeset megs=$2
+	typeset nfiles=${3:-1}
+	typeset bs=${4:-1024k}
+	typeset fname=${5:-file}
+
+	[[ -d $dir ]] || log_fail "No directory: $dir"
+
+	# Under Linux fio is not currently used since its behavior can
+	# differ significantly across versions.  This includes missing
+	# command line options and cases where the --buffer_compress_*
+	# options fail to behave as expected.
+	if is_linux; then
+		typeset file_bytes=$(to_bytes $megs)
+		typeset bs_bytes=4096
+		typeset blocks=$(($file_bytes / $bs_bytes))
+
+		for (( i = 0; i < $nfiles; i++ )); do
+			truncate -s $file_bytes $dir/$fname.$i
+
+			# Write every third block to get 66% compression.
+			for (( j = 0; j < $blocks; j += 3 )); do
+				dd if=/dev/urandom of=$dir/$fname.$i \
+				    seek=$j bs=$bs_bytes count=1 \
+				    conv=notrunc >/dev/null 2>&1
+			done
+		done
+	else
+		log_must eval "fio \
+		    --name=job \
+		    --fallocate=0 \
+		    --minimal \
+		    --randrepeat=0 \
+		    --buffer_compress_percentage=66 \
+		    --buffer_compress_chunk=4096 \
+		    --directory=$dir \
+		    --numjobs=$nfiles \
+		    --nrfiles=$nfiles \
+		    --rw=write \
+		    --bs=$bs \
+		    --filesize=$megs \
+		    --filename_format='$fname.\$jobnum' >/dev/null"
+	fi
+}
+
+function get_objnum
+{
+	typeset pathname=$1
+	typeset objnum
+
+	[[ -e $pathname ]] || log_fail "No such file or directory: $pathname"
+	objnum=$(stat -c %i $pathname)
+	echo $objnum
+}
+
 #
 # Synchronize all the data in pool
 #

tests/zfs-tests/include/math.shlib

@@ -41,3 +41,28 @@ function within_percent
 
 	return 1
 }
+
+#
+# Return 0 if the human readable string of the form <value>[suffix] can
+# be converted to bytes.  Allow suffixes are shown in the table below.
+#
+function to_bytes
+{
+	typeset size=$1
+	typeset value=$(echo "$size" | grep -o '[0-9]\+')
+
+	case $size in
+		*PB|*pb|*P|*p)	factor='1024^5'	;;
+		*TB|*tb|*T|*t)	factor='1024^4'	;;
+		*GB|*gb|*G|*g)	factor='1024^3'	;;
+		*MB|*mb|*M|*m)	factor='1024^2'	;;
+		*KB|*kb|*K|*k)	factor='1024^1'	;;
+		*B|*b)		factor='1024^0'	;;
+		*[!0-9.]*)	return 1 ;;
+		*)		factor='1024^0'	;;
+	esac
+
+	echo "$value * ($factor)" | bc
+
+	return 0
+}

tests/zfs-tests/include/properties.shlib

@@ -13,11 +13,26 @@
 # Copyright (c) 2012, 2016 by Delphix. All rights reserved.
 #
 
-typeset -a compress_props=('on' 'off' 'lzjb' 'gzip' 'gzip-1' 'gzip-2' 'gzip-3'
-    'gzip-4' 'gzip-5' 'gzip-6' 'gzip-7' 'gzip-8' 'gzip-9' 'zle')
+typeset -a compress_prop_vals=('on' 'off' 'lzjb' 'gzip' 'gzip-1' 'gzip-2'
+    'gzip-3' 'gzip-4' 'gzip-5' 'gzip-6' 'gzip-7' 'gzip-8' 'gzip-9' 'zle' 'lz4')
+typeset -a checksum_prop_vals=('on' 'off' 'fletcher2' 'fletcher4' 'sha256'
+    'noparity' 'sha512' 'skein' 'edonr')
+typeset -a recsize_prop_vals=('512' '1024' '2048' '4096' '8192' '16384'
+    '32768' '65536' '131072' '262144' '524288' '1048576')
+typeset -a canmount_prop_vals=('on' 'off' 'noauto')
+typeset -a copies_prop_vals=('1' '2' '3')
+typeset -a logbias_prop_vals=('latency' 'throughput')
+typeset -a primarycache_prop_vals=('all' 'none' 'metadata')
+typeset -a redundant_metadata_prop_vals=('all' 'most')
+typeset -a secondarycache_prop_vals=('all' 'none' 'metadata')
+typeset -a snapdir_prop_vals=('hidden' 'visible')
+typeset -a sync_prop_vals=('standard' 'always' 'disabled')
 
-typeset -a checksum_props=('on' 'off' 'fletcher2' 'fletcher4' 'sha256' 'sha512'
-    'edonr' 'skein' 'noparity')
+typeset -a fs_props=('compress' 'checksum' 'recsize'
+    'canmount' 'copies' 'logbias' 'primarycache' 'redundant_metadata'
+    'secondarycache' 'snapdir' 'sync')
+typeset -a vol_props=('compress' 'checksum' 'copies' 'logbias' 'primarycache'
+    'secondarycache' 'redundant_metadata' 'sync')
 
 #
 # Given the property array passed in, return 'num_props' elements to the
@@ -45,20 +60,81 @@ function get_rand_prop
 
 function get_rand_compress
 {
-	get_rand_prop compress_props $1 2
+	get_rand_prop compress_prop_vals $1 2
 }
 
 function get_rand_compress_any
 {
-	get_rand_prop compress_props $1 0
+	get_rand_prop compress_prop_vals $1 0
 }
 
 function get_rand_checksum
 {
-	get_rand_prop checksum_props $1 2
+	get_rand_prop checksum_prop_vals $1 2
 }
 
 function get_rand_checksum_any
 {
-	get_rand_prop checksum_props $1 0
+	get_rand_prop checksum_prop_vals $1 0
+}
+
+function get_rand_recsize
+{
+	get_rand_prop recsize_prop_vals $1 0
+}
+
+function get_rand_large_recsize
+{
+	get_rand_prop recsize_prop_vals $1 9
+}
+
+#
+# Functions to toggle on/off properties
+#
+typeset -a binary_props=('atime' 'devices' 'exec' 'nbmand' 'readonly' 'setuid'
+    'xattr' 'zoned')
+
+function toggle_prop
+{
+	typeset ds=$1
+	typeset prop=$2
+
+	datasetexists $ds || log_fail "$ds does not exist"
+	typeset val=$(get_prop $prop $ds)
+	typeset newval='off'
+
+	[[ $val = $newval ]] && newval='on'
+	log_must zfs set $prop=$newval $ds
+}
+
+function toggle_binary_props
+{
+	typeset ds=$1
+	typeset prop
+
+	for prop in "${binary_props[@]}"; do
+		toggle_prop $ds $prop
+	done
+}
+
+function randomize_ds_props
+{
+	typeset ds=$1
+	typeset prop proplist val
+
+	datasetexists $ds || log_fail "$ds does not exist"
+	if ds_is_volume $ds; then
+		toggle_prop $ds readonly
+		proplist="${vol_props[@]}"
+	elif ds_is_filesystem $ds; then
+		toggle_binary_props $ds
+		proplist="${fs_props[@]}"
+	else
+		log_fail "$ds is neither a volume nor a file system"
+	fi
+
+	for prop in $proplist; do
+		typeset val=$(get_rand_prop "${prop}_prop_vals" 1 0)
+		log_must zfs set $prop=$val $ds
+	done
 }

tests/zfs-tests/tests/functional/rsend/Makefile.am

@@ -22,4 +22,19 @@ dist_pkgdata_SCRIPTS = \
 	rsend_020_pos.ksh \
 	rsend_021_pos.ksh \
 	rsend_022_pos.ksh \
-	rsend_024_pos.ksh
+	rsend_024_pos.ksh \
+	send-cD.ksh \
+	send-c_embedded_blocks.ksh \
+	send-c_incremental.ksh \
+	send-c_lz4_disabled.ksh \
+	send-c_mixed_compression.ksh \
+	send-c_props.ksh \
+	send-c_recv_dedup.ksh \
+	send-c_recv_lz4_disabled.ksh \
+	send-c_resume.ksh \
+	send-c_stream_size_estimate.ksh \
+	send-c_verify_contents.ksh \
+	send-c_verify_ratio.ksh \
+	send-c_volume.ksh \
+	send-c_zstreamdump.ksh \
+	send-cpL_varied_recsize.ksh

tests/zfs-tests/tests/functional/rsend/rsend.cfg

@@ -29,11 +29,11 @@
 
 export BACKDIR=${TEST_BASE_DIR%%/}/backdir-rsend
 
-export DISK1=${DISKS%% *}
+export DISK1=$(echo $DISKS | awk '{print $1}')
 export DISK2=$(echo $DISKS | awk '{print $2}')
 export DISK3=$(echo $DISKS | awk '{print $3}')
 
 export POOL=$TESTPOOL
-export POOL2=$TESTPOOL1
-export POOL3=$TESTPOOL2
+export POOL2=$TESTPOOL2
+export POOL3=$TESTPOOL3
 export FS=$TESTFS

tests/zfs-tests/tests/functional/rsend/rsend.kshlib

@@ -29,6 +29,7 @@
 #
 
 . $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/include/math.shlib
 . $STF_SUITE/tests/functional/rsend/rsend.cfg
 
 #
@@ -207,11 +208,10 @@ function cmp_ds_prop
 	typeset dtst1=$1
 	typeset dtst2=$2
 
-	for item in "type" "origin" "volblocksize" "aclinherit" "acltype" \
+	for item in "type" "origin" "volblocksize" "acltype" "dnodesize" \
 	    "atime" "canmount" "checksum" "compression" "copies" "devices" \
-	    "dnodesize" "exec" "quota" "readonly" "recordsize" "reservation" \
-	    "setuid" "snapdir" "version" "volsize" "xattr" "zoned" \
-	    "mountpoint";
+	    "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \
+	    "snapdir" "version" "volsize" "xattr" "zoned" "mountpoint";
 	do
 		zfs get -H -o property,value,source $item $dtst1 >> \
 		    $BACKDIR/dtst1
@@ -525,20 +525,20 @@ function test_fs_setup
 		mk_files 100 1048576 0 $sendfs &
 		mk_files 10 10485760 0 $sendfs &
 		mk_files 1 104857600 0 $sendfs &
-		wait
+		log_must wait
 		log_must zfs snapshot $sendfs@a
 
 		rm_files 200 256 0 $sendfs &
 		rm_files 200 131072 0 $sendfs &
 		rm_files 20 1048576 0 $sendfs &
 		rm_files 2 10485760 0 $sendfs &
-		wait
+		log_must wait
 
 		mk_files 400 256 0 $sendfs &
 		mk_files 400 131072 0 $sendfs &
 		mk_files 40 1048576 0 $sendfs &
 		mk_files 4 10485760 0 $sendfs &
-		wait
+		log_must wait
 
 		log_must zfs snapshot $sendfs@b
 		log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
@@ -551,3 +551,121 @@ function test_fs_setup
 	fi
 	log_must zfs create -o compress=lz4 $sendpool/stream
 }
+
+#
+# Check to see if the specified features are set in a send stream.
+# The values for these features are found in include/sys/zfs_ioctl.h
+#
+# $1 The stream file
+# $2-$n The flags expected in the stream
+#
+function stream_has_features
+{
+	typeset file=$1
+	shift
+
+	[[ -f $file ]] || log_fail "Couldn't find file: $file"
+	typeset flags=$(cat $file | zstreamdump | \
+	    awk '/features =/ {features = $3} END {print features}')
+	typeset -A feature
+	feature[dedup]="1"
+	feature[dedupprops]="2"
+	feature[sa_spill]="4"
+	feature[embed_data]="10000"
+	feature[lz4]="20000"
+	feature[mooch_byteswap]="40000"
+	feature[large_blocks]="80000"
+	feature[resuming]="100000"
+	feature[redacted]="200000"
+	feature[compressed]="400000"
+
+	typeset flag known derived=0
+	for flag in "$@"; do
+		known=${feature[$flag]}
+		[[ -z $known ]] && log_fail "Unknown feature: $flag"
+
+		derived=$(printf "%x" $((0x${flags} & 0x${feature[$flag]})))
+		[[ $derived = $known ]] || return 1
+	done
+
+	return 0
+}
+
+#
+# Parse zstreamdump -v output.  The output varies for each kind of record:
+# BEGIN records are simply output as "BEGIN"
+# END records are output as "END"
+# OBJECT records become "OBJECT <object num>"
+# FREEOBJECTS records become "FREEOBJECTS <startobj> <numobjs>"
+# FREE records become "<record type> <start> <length>"
+# WRITE records become:
+# "<record type> <compression type> <start> <logical size> <compressed size>
+#  <data size>"
+#
+function parse_dump
+{
+	sed '/^WRITE/{N;s/\n/ /;}' | grep "^[A-Z]" | awk '{
+	    if ($1 == "BEGIN" || $1 == "END") print $1
+	    if ($1 == "OBJECT") print $1" "$4
+	    if ($1 == "FREEOBJECTS") print $1" "$4" "$7
+	    if ($1 == "FREE") print $1" "$7" "$10
+	    if ($1 == "WRITE") print $1" "$15" "$18" "$21" "$24" "$27}'
+}
+
+#
+# Given a send stream, verify that the size of the stream matches what's
+# expected based on the source or target dataset. If the stream is an
+# incremental stream, subtract the size of the source snapshot before
+# comparing. This function does not currently handle incremental streams
+# that remove data.
+#
+# $1 The zstreamdump output file
+# $2 The dataset to compare against
+#    This can be a source of a send or recv target (fs, not snapshot)
+# $3 The percentage below which verification is deemed a failure
+# $4 The source snapshot of an incremental send
+#
+
+function verify_stream_size
+{
+	typeset stream=$1
+	typeset ds=$2
+	typeset percent=${3:-90}
+	typeset inc_src=$4
+
+	[[ -f $stream ]] || log_fail "No such file: $stream"
+	datasetexists $ds || log_fail "No such dataset: $ds"
+
+	typeset stream_size=$(cat $stream | zstreamdump | sed -n \
+	    's/	Total write size = \(.*\) (0x.*)/\1/p')
+
+	typeset inc_size=0
+	if [[ -n $inc_src ]]; then
+		inc_size=$(get_prop lrefer $inc_src)
+		if stream_has_features $stream compressed; then
+			inc_size=$(get_prop refer $inc_src)
+		fi
+	fi
+
+	if stream_has_features $stream compressed; then
+		ds_size=$(get_prop refer $ds)
+	else
+		ds_size=$(get_prop lrefer $ds)
+	fi
+	ds_size=$((ds_size - inc_size))
+
+	within_percent $stream_size $ds_size $percent || log_fail \
+	    "$stream_size $ds_size differed by too much"
+}
+
+# Cleanup function for tests involving resumable send
+function resume_cleanup
+{
+	typeset sendfs=$1
+	typeset streamfs=$2
+
+	datasetexists $sendfs && log_must zfs destroy -r $sendfs
+	datasetexists $streamfs && log_must zfs destroy -r $streamfs
+	cleanup_pool $POOL2
+	rm -f /$POOL/initial.zsend /$POOL/incremental.zsend
+}

tests/zfs-tests/tests/functional/rsend/rsend_019_pos.ksh

@@ -37,8 +37,9 @@ verify_runnable "both"
 
 log_assert "Verify resumability of a full and incremental ZFS send/receive " \
     "in the presence of a corrupted stream"
-log_onexit cleanup_pools $POOL2 $POOL3
+log_onexit resume_cleanup $sendfs $streamfs
 
+sendfs=$POOL/sendfs
 recvfs=$POOL3/recvfs
 streamfs=$POOL2/stream
 

tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh

@@ -35,12 +35,13 @@ verify_runnable "both"
 
 log_assert "Verify resumability of full ZFS send/receive with the -D " \
     "(dedup) flag"
-log_onexit cleanup_pool $POOL2
 
 sendfs=$POOL/sendfs
 recvfs=$POOL2/recvfs
 streamfs=$POOL/stream
 
+log_onexit resume_cleanup $sendfs $streamfs
+
 test_fs_setup $sendfs $recvfs
 resume_test "zfs send -D -v $sendfs@a" $streamfs $recvfs
 file_check $sendfs $recvfs

tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh

@@ -37,12 +37,13 @@ verify_runnable "both"
 
 log_assert "Verify resumability of a full and incremental ZFS send/receive " \
     "with the -e (embedded) flag"
-log_onexit cleanup_pool $POOL2
 
 sendfs=$POOL/sendfs
 recvfs=$POOL2/recvfs
 streamfs=$POOL/stream
 
+log_onexit resume_cleanup $sendfs $streamfs
+
 test_fs_setup $sendfs $recvfs
 resume_test "zfs send -v -e $sendfs@a" $streamfs $recvfs
 resume_test "zfs send -v -e -i @a $sendfs@b" $streamfs $recvfs

tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh

@@ -45,12 +45,13 @@ fi
 
 log_assert "Verify resumability of an incremental ZFS send/receive with ZFS " \
     "bookmarks"
-log_onexit cleanup_pool $POOL2
 
 sendfs=$POOL/sendfs
 recvfs=$POOL2/recvfs
 streamfs=$POOL/stream
 
+log_onexit resume_cleanup $sendfs $streamfs
+
 test_fs_setup $sendfs $recvfs
 log_must zfs bookmark $sendfs@a $sendfs#bm_a
 log_must zfs destroy $sendfs@a

tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh

@@ -42,12 +42,13 @@ fi
 
 log_assert "Verify resumability of a full ZFS send/receive with the source " \
     "filesystem unmounted"
-log_onexit cleanup_pool $POOL2
 
 sendfs=$POOL/sendfs
 recvfs=$POOL2/recvfs
 streamfs=$POOL/stream
 
+log_onexit resume_cleanup $sendfs $streamfs
+
 test_fs_setup $sendfs $recvfs
 log_must zfs unmount $sendfs
 resume_test "zfs send $sendfs" $streamfs $recvfs

tests/zfs-tests/tests/functional/rsend/send-cD.ksh

@@ -0,0 +1,77 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify that the -c and -D flags do not interfere with each other.
+#
+# Strategy:
+# 1. Write unique data to a filesystem and create a compressed, deduplicated
+#    full stream.
+# 2. Verify that the stream and send dataset show the same size
+# 3. Make several copies of the original data, and create both full and
+#    incremental compressed, deduplicated send streams
+# 4. Verify the full stream is no bigger than the stream from step 1
+# 5. Verify the streams can be received correctly.
+#
+
+verify_runnable "both"
+
+log_assert "Verify that the -c and -D flags do not interfere with each other"
+log_onexit cleanup_pool $POOL2
+
+typeset sendfs=$POOL2/sendfs
+typeset recvfs=$POOL2/recvfs
+typeset stream0=$BACKDIR/stream.0
+typeset stream1=$BACKDIR/stream.1
+typeset inc=$BACKDIR/stream.inc
+
+log_must zfs create -o compress=lz4 $sendfs
+log_must zfs create -o compress=lz4 $recvfs
+typeset dir=$(get_prop mountpoint $sendfs)
+# Don't use write_compressible: we want compressible but undedupable data here.
+log_must file_write -o overwrite -f $dir/file -d R -b 4096 -c 1000
+log_must zfs snapshot $sendfs@snap0
+log_must eval "zfs send -D -c $sendfs@snap0 >$stream0"
+
+# The stream size should match at this point because the data is all unique
+verify_stream_size $stream0 $sendfs
+
+for i in {0..3}; do
+	log_must cp $dir/file $dir/file.$i
+done
+log_must zfs snapshot $sendfs@snap1
+
+# The stream sizes should match, since the second stream contains no new blocks
+log_must eval "zfs send -D -c $sendfs@snap1 >$stream1"
+typeset size0=$(stat -c %s $stream0)
+typeset size1=$(stat -c %s $stream1)
+within_percent $size0 $size1 90 || log_fail "$size0 and $size1"
+
+# Finally, make sure the receive works correctly.
+log_must eval "zfs send -D -c -i snap0 $sendfs@snap1 >$inc"
+log_must eval "zfs recv -d $recvfs <$stream0"
+log_must eval "zfs recv -d $recvfs <$inc"
+cmp_ds_cont $sendfs $recvfs
+
+# The size of the incremental should be the same as the initial send.
+typeset size2=$(stat -c %s $inc)
+within_percent $size0 $size2 90 || log_fail "$size0 and $size1"
+
+log_pass "The -c and -D flags do not interfere with each other"

tests/zfs-tests/tests/functional/rsend/send-c_embedded_blocks.ksh

@@ -0,0 +1,109 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+. $STF_SUITE/include/properties.shlib
+
+#
+# Description:
+# Verify that compressed streams can contain embedded blocks.
+#
+# Strategy:
+# 1. Create a filesystem with compressible data and embedded blocks.
+# 2. Verify the created streams can be received correctly.
+# 3. Verify the presence / absence of embedded blocks in the compressed stream,
+#    as well as the receiving file system.
+#
+
+verify_runnable "both"
+
+log_assert "Verify that compressed streams can contain embedded blocks."
+log_onexit cleanup_pool $POOL2
+
+typeset objs obj recsize
+typeset sendfs=$POOL2/sendfs
+typeset recvfs=$POOL2/recvfs
+typeset stream=$BACKDIR/stream
+typeset dump=$BACKDIR/dump
+typeset recvfs2=$POOL2/recvfs2
+typeset stream2=$BACKDIR/stream2
+typeset dump2=$BACKDIR/dump2
+log_must zfs create -o compress=lz4 $sendfs
+log_must zfs create -o compress=lz4 $recvfs
+log_must zfs create -o compress=lz4 $recvfs2
+typeset dir=$(get_prop mountpoint $sendfs)
+
+# Populate the send dataset with compressible data and embedded block files.
+write_compressible $dir 16m
+for recsize in "${recsize_prop_vals[@]}"; do
+	# For lz4, this method works for blocks up to 16k, but not larger
+	[[ $recsize -eq $((32 * 1024)) ]] && break
+
+	if is_linux; then
+		log_must truncate -s $recsize $dir/$recsize
+		log_must dd if=/dev/urandom of=$dir/$recsize \
+		    seek=$((recsize - 8)) bs=1 count=8 conv=notrunc
+	else
+		log_must mkholes -h 0:$((recsize - 8)) -d $((recsize - 8)):8 \
+		    $dir/$recsize
+	fi
+done
+
+# Generate the streams and zstreamdump output.
+log_must zfs snapshot $sendfs@now
+log_must eval "zfs send -c $sendfs@now >$stream"
+log_must eval "zstreamdump -v <$stream >$dump"
+log_must eval "zfs recv -d $recvfs <$stream"
+cmp_ds_cont $sendfs $recvfs
+verify_stream_size $stream $sendfs
+log_mustnot stream_has_features $stream embed_data
+
+log_must eval "zfs send -c -e $sendfs@now >$stream2"
+log_must eval "zstreamdump -v <$stream2 >$dump2"
+log_must eval "zfs recv -d $recvfs2 <$stream2"
+cmp_ds_cont $sendfs $recvfs2
+verify_stream_size $stream2 $sendfs
+log_must stream_has_features $stream2 embed_data
+
+# Verify embedded blocks are present only when expected.
+for recsize in "${recsize_prop_vals[@]}"; do
+	[[ $recsize -eq $((32 * 1024)) ]] && break
+
+	typeset send_obj=$(get_objnum $(get_prop mountpoint $sendfs)/$recsize)
+	typeset recv_obj=$(get_objnum \
+	    $(get_prop mountpoint $recvfs/sendfs)/$recsize)
+	typeset recv2_obj=$(get_objnum \
+	    $(get_prop mountpoint $recvfs2/sendfs)/$recsize)
+
+	log_must eval "zdb -ddddd $sendfs $send_obj >$BACKDIR/sendfs.zdb"
+	log_must eval "zdb -ddddd $recvfs/sendfs $recv_obj >$BACKDIR/recvfs.zdb"
+	log_must eval "zdb -ddddd $recvfs2/sendfs $recv2_obj >$BACKDIR/recvfs2.zdb"
+
+	grep -q "EMBEDDED" $BACKDIR/sendfs.zdb || \
+	    log_fail "Obj $send_obj not embedded in $sendfs"
+	grep -q "EMBEDDED" $BACKDIR/recvfs.zdb || \
+	    log_fail "Obj $recv_obj not embedded in $recvfs"
+	grep -q "EMBEDDED" $BACKDIR/recvfs2.zdb || \
+	    log_fail "Obj $recv2_obj not embedded in $recvfs2"
+
+	grep -q "WRITE_EMBEDDED object = $send_obj offset = 0" $dump && \
+	    log_fail "Obj $obj embedded in zstreamdump output"
+	grep -q "WRITE_EMBEDDED object = $send_obj offset = 0" $dump2 || \
+	    log_fail "Obj $obj not embedded in zstreamdump output"
+done
+
+log_pass "Compressed streams can contain embedded blocks."

tests/zfs-tests/tests/functional/rsend/send-c_incremental.ksh

@@ -0,0 +1,100 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify that compressed send works correctly with incremental sends.
+#
+# Strategy:
+# 1. Randomly choose either a -i or -I incremental.
+# 2. Generate compressed incremental replication streams for a pool, a
+#    descendant dataset, and a volume.
+# 3. Receive these streams verifying both the contents, and intermediate
+#    snapshots are present or absent as appropriate to the -i or -I option.
+#
+
+verify_runnable "both"
+
+log_assert "Verify compressed send works with incremental send streams."
+log_onexit cleanup_pool $POOL2
+
+typeset opt=$(random_get "-i" "-I")
+typeset final dstlist list vol
+
+log_must eval "zfs send -R $POOL@final > $BACKDIR/final"
+log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/final"
+
+function do_checks
+{
+	log_must cmp_ds_cont $POOL $POOL2
+	[[ $opt = "-I" ]] && log_must cmp_ds_subs $POOL $POOL2
+	[[ $opt = "-i" ]] && log_mustnot cmp_ds_subs $POOL $POOL2
+
+	[[ $1 != "clean" ]] && return
+
+	cleanup_pool $POOL2
+	log_must eval "zfs send -R $POOL@final > $BACKDIR/final"
+	log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/final"
+}
+
+if is_global_zone; then
+	# Send from the pool root
+	final=$(getds_with_suffix $POOL2 @final)
+	list="$final $(getds_with_suffix $POOL2 @snapA)"
+	list="$list $(getds_with_suffix $POOL2 @snapB)"
+	list="$list $(getds_with_suffix $POOL2 @snapC)"
+
+	log_must eval "zfs send -c -R $opt @init $POOL2@final >$BACKDIR/pool"
+	log_must destroy_tree $list
+	log_must eval "zfs recv -d -F $POOL2 <$BACKDIR/pool"
+
+	dstlist=$(getds_with_suffix $POOL2 @final)
+	[[ $final != $dstlist ]] && log_fail "$final != $dstlist"
+
+	do_checks clean
+
+	# Send of a volume
+	vol=$POOL2/$FS/vol
+	final=$(getds_with_suffix $vol @final)
+	log_must eval "zfs send -c -R $opt @init $vol@final >$BACKDIR/vol"
+	log_must destroy_tree $vol@snapB $vol@snapC $vol@final
+	log_must eval "zfs recv -d -F $POOL2 <$BACKDIR/vol"
+
+	dstlist=$(getds_with_suffix $POOL2/$FS/vol @final)
+	[[ $final != $dstlist ]] && log_fail "$final != $dstlist"
+
+	do_checks clean
+fi
+
+# Send of a descendant fs
+final=$(getds_with_suffix $POOL2/$FS @final)
+list="$final $(getds_with_suffix $POOL2/$FS @snapA)"
+list="$list $(getds_with_suffix $POOL2/$FS @snapB)"
+list="$list $(getds_with_suffix $POOL2/$FS @snapC)"
+
+log_must eval "zfs send -c -R $opt @init $POOL2/$FS@final >$BACKDIR/fs"
+log_must destroy_tree $list
+log_must eval "zfs recv -d -F $POOL2 <$BACKDIR/fs"
+
+dstlist=$(getds_with_suffix $POOL2/$FS @final)
+[[ $final != $dstlist ]] && log_fail "$final != $dstlist"
+
+do_checks
+
+log_pass "Compressed send works with incremental send streams."

tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh

@@ -0,0 +1,73 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify a pool without the lz4 feature enabled can create compressed send
+# streams, and that they can be received into pools with or without the
+# lz4 feature.
+#
+# Strategy:
+# 1. For each of an uncompressed, and gzip dataset created from a pool with
+#    the lz4 feature disabled, receive the stream into a pool with and without
+#    the feature enabled.
+#
+
+verify_runnable "both"
+
+log_assert "Verify compressed streams are rejected if incompatible."
+
+typeset send_ds=$POOL2/testds
+typeset recv_ds=$POOL3/testds
+
+function cleanup
+{
+	poolexists $POOL2 && destroy_pool $POOL2
+	poolexists $POOL3 && destroy_pool $POOL3
+	log_must zpool create $POOL2 $DISK2
+}
+log_onexit cleanup
+
+datasetexists $POOL2 && log_must zpool destroy $POOL2
+log_must zpool create -d $POOL2 $DISK2
+
+for compress in off gzip; do
+	for pool_opt in '' -d; do
+		poolexists $POOL3 && destroy_pool $POOL3
+		log_must zpool create $pool_opt $POOL3 $DISK3
+
+		datasetexists $send_ds && log_must zfs destroy -r $send_ds
+		datasetexists $recv_ds && log_must zfs destroy -r $recv_ds
+
+		log_must zfs create -o compress=$compress $send_ds
+		typeset dir=$(get_prop mountpoint $send_ds)
+		write_compressible $dir 16m
+		log_must zfs snapshot $send_ds@full
+
+		log_must eval "zfs send -c $send_ds@full >$BACKDIR/full-c"
+		log_must eval "zfs recv $recv_ds <$BACKDIR/full-c"
+
+		log_must zfs destroy -r $recv_ds
+
+		log_must eval "zfs send $send_ds@full >$BACKDIR/full"
+		log_must eval "zfs recv $recv_ds <$BACKDIR/full"
+	done
+done
+
+log_pass "Compressed streams are rejected if incompatible."

tests/zfs-tests/tests/functional/rsend/send-c_mixed_compression.ksh

@@ -0,0 +1,54 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+. $STF_SUITE/include/properties.shlib
+
+#
+# Description:
+# Verify datasets using mixed compression algorithms can be received.
+#
+# Strategy:
+# 1. Write data with each of the available compression algorithms
+# 2. Receive a full compressed send, and verify the data and compression ratios
+#
+
+verify_runnable "both"
+
+log_assert "Verify datasets using mixed compression algorithms can be received."
+log_onexit cleanup_pool $POOL2
+
+send_ds=$POOL2/sendfs
+recv_ds=$POOL2/recvfs
+
+log_must zfs create $send_ds
+
+for prop in "${compress_prop_vals[@]}"; do
+	log_must zfs set compress=$prop $send_ds
+	write_compressible $(get_prop mountpoint $send_ds) 16m
+done
+
+log_must zfs set compress=off $send_ds
+log_must zfs snapshot $send_ds@full
+log_must eval "zfs send -c $send_ds@full >$BACKDIR/full"
+log_must eval "zfs recv $recv_ds <$BACKDIR/full"
+
+verify_stream_size $BACKDIR/full $send_ds
+verify_stream_size $BACKDIR/full $recv_ds
+log_must cmp_ds_cont $send_ds $recv_ds
+
+log_pass "Datasets using mixed compression algorithms can be received."

tests/zfs-tests/tests/functional/rsend/send-c_props.ksh

@@ -0,0 +1,67 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+. $STF_SUITE/include/properties.shlib
+
+#
+# Description:
+# Verify compressed send streams can still preserve properties
+#
+# Strategy:
+# 1. Randomly modify the properties in the src pool
+# 2. Send a full compressed stream with -p to preserve properties
+# 3. Verify all the received properties match the source datasets
+# 4. Repeat the process with -R instead of -p
+#
+
+verify_runnable "global"
+
+function cleanup
+{
+	destroy_pool $POOL
+	destroy_pool $POOL2
+	log_must zpool create $POOL $DISK1
+	log_must zpool create $POOL2 $DISK2
+	log_must setup_test_model $POOL
+}
+
+log_assert "Compressed send doesn't interfere with preservation of properties"
+log_onexit cleanup
+
+typeset -a datasets=("" "/pclone" "/$FS" "/$FS/fs1" "/$FS/fs1/fs2"
+    "/$FS/fs1/fclone" "/vol" "/$FS/vol")
+
+typeset ds
+for opt in "-p" "-R"; do
+	for ds in ${datasets[@]}; do
+		randomize_ds_props $POOL$ds
+	done
+
+	log_must eval "zfs send -c $opt $POOL@final > $BACKDIR/pool-final$opt"
+	log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final$opt"
+
+	for ds in ${datasets[@]}; do
+		log_must cmp_ds_prop $POOL$ds $POOL2$ds
+		log_must cmp_ds_prop $POOL$ds@final $POOL2$ds@final
+	done
+
+	# Don't cleanup the second time, since we do that on exit anyway.
+	[[ $opt = "-p" ]] && cleanup
+done
+
+log_pass "Compressed send doesn't interfere with preservation of properties"

tests/zfs-tests/tests/functional/rsend/send-c_recv_dedup.ksh

@@ -0,0 +1,55 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2016 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify that we can receive a compressed stream into a deduped filesystem.
+#
+# Strategy:
+# 1. Write heavily duplicated data to a filesystem and create a compressed
+#    full stream.
+# 2. Verify that the stream can be received correctly into a dedup=verify
+#    filesystem.
+#
+
+verify_runnable "both"
+
+log_pass "Verify a compressed stream can be received into a deduped filesystem"
+log_onexit cleanup_pool $POOL2
+
+typeset sendfs=$POOL2/sendfs
+typeset recvfs=$POOL2/recvfs
+typeset stream0=$BACKDIR/stream.0
+typeset stream1=$BACKDIR/stream.1
+typeset inc=$BACKDIR/stream.inc
+
+log_must zfs create -o compress=lz4 $sendfs
+log_must zfs create -o compress=lz4 -o dedup=verify $recvfs
+typeset dir=$(get_prop mountpoint $sendfs)
+for i in {0..10}; do
+    log_must file_write -o overwrite -f $dir/file.$i -d R -b 4096 -c 1000
+done
+log_must zfs snapshot $sendfs@snap0
+log_must eval "zfs send -c $sendfs@snap0 >$stream0"
+
+# Finally, make sure the receive works correctly.
+log_must eval "zfs recv -d $recvfs <$stream0"
+cmp_ds_cont $sendfs $recvfs
+
+log_pass "The compressed stream could be received into a deduped filesystem"

tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh

@@ -0,0 +1,68 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify a pool without the lz4 feature gracefully rejects a compressed stream
+# because on any sending pool that supports it, metadata will be compressed
+# with lz4 even if user data is not compressed.
+#
+# Strategy:
+# 1. For each of an uncompressed, gzip and lz4 dataset, do the following
+#    receives into a pool without the lz4 feature:
+# 2. Attempt to receive the compressed stream (should fail)
+# 3. Attempt to receive the uncompressed stream (should succeed)
+#
+
+verify_runnable "both"
+
+log_assert "Verify compressed streams are rejected if incompatible."
+
+typeset compress_types="off gzip lz4"
+typeset send_ds=$POOL2/testds
+typeset recv_ds=$POOL3/testds
+
+function cleanup
+{
+	poolexists $POOL2 && destroy_pool $POOL2
+	poolexists $POOL3 && destroy_pool $POOL3
+	log_must zpool create $POOL2 $DISK2
+}
+log_onexit cleanup
+
+datasetexists $POOL3 && log_must zpool destroy $POOL3
+log_must zpool create -d $POOL3 $DISK3
+
+for compress in $compress_types; do
+	datasetexists $send_ds && log_must zfs destroy -r $send_ds
+	datasetexists $recv_ds && log_must zfs destroy -r $recv_ds
+
+	log_must zfs create -o compress=$compress $send_ds
+	typeset dir=$(get_prop mountpoint $send_ds)
+	write_compressible $dir 16m
+	log_must zfs snapshot $send_ds@full
+
+	log_must eval "zfs send -c $send_ds@full >$BACKDIR/full-c"
+	log_mustnot eval "zfs recv $recv_ds <$BACKDIR/full-c"
+
+	log_must eval "zfs send $send_ds@full >$BACKDIR/full"
+	log_must eval "zfs recv $recv_ds <$BACKDIR/full"
+done
+
+log_pass "Compressed streams are rejected if incompatible."

tests/zfs-tests/tests/functional/rsend/send-c_resume.ksh

@@ -0,0 +1,49 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify resumability of full and incremental ZFS send/receive with the -c
+# (compress) flag in the presence of a corrupted stream.
+#
+# Strategy:
+# 1. Start a full ZFS send with the -c flag (compress), redirect output to
+#    a file
+# 2. Mess up the contents of the stream state file on disk
+# 3. Try ZFS receive, which should fail with a checksum mismatch error
+# 4. ZFS send to the stream state file again using the receive_resume_token
+# 5. ZFS receieve and verify the receive completes successfully
+# 6. Repeat steps on an incremental ZFS send
+#
+
+verify_runnable "both"
+
+sendfs=$POOL/sendfs
+recvfs=$POOL2/recvfs
+streamfs=$POOL/stream
+
+log_assert "Verify compressed send streams can be resumed if interrupted"
+log_onexit resume_cleanup $sendfs $streamfs
+
+test_fs_setup $sendfs $recvfs
+resume_test "zfs send -c -v $sendfs@a" $streamfs $recvfs
+resume_test "zfs send -c -v -i @a $sendfs@b" $streamfs $recvfs
+file_check $sendfs $recvfs
+
+log_pass "Compressed send streams can be resumed if interrupted"

tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh

@@ -0,0 +1,91 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify the stream size estimate given by -P accounts for compressed send.
+# Verify the stream size given by -P accounts for compressed send."
+#
+# Strategy:
+# 1. For datasets of varied compression types do the following:
+# 2. Write data, verify stream size estimates with and without -c
+#
+
+verify_runnable "both"
+typeset compress_types="off gzip lz4"
+typeset send_ds="$POOL2/testfs"
+typeset send_vol="$POOL2/vol"
+typeset send_voldev="$ZVOL_DEVDIR/$POOL2/vol"
+typeset file="$BACKDIR/file.0"
+typeset megs="16"
+typeset compress
+
+function get_estimated_size
+{
+	typeset cmd=$1
+	typeset ds=${cmd##* }
+	typeset tmpfile=$(mktemp -p $BACKDIR)
+
+	eval "$cmd >$tmpfile"
+	[[ $? -eq 0 ]] || log_fail "get_estimated_size: $cmd"
+	typeset size=$(eval "awk '\$2 == \"$ds\" {print \$3}' $tmpfile")
+	rm -f $tmpfile
+
+	echo $size
+}
+
+log_assert "Verify the stream size given by -P accounts for compressed send."
+log_onexit cleanup_pool $POOL2
+
+write_compressible $BACKDIR ${megs}m
+
+for compress in $compress_types; do
+	datasetexists $send_ds && log_must zfs destroy -r $send_ds
+	datasetexists $send_vol && log_must zfs destroy -r $send_vol
+	log_must zfs create -o compress=$compress $send_ds
+	log_must zfs create -V 1g -o compress=$compress $send_vol
+
+	typeset dir=$(get_prop mountpoint $send_ds)
+	log_must cp $file $dir
+	log_must zfs snapshot $send_ds@snap
+	log_must dd if=$file of=$send_voldev
+	log_must zfs snapshot $send_vol@snap
+
+	typeset ds_size=$(get_estimated_size "zfs send -nP $send_ds@snap")
+	typeset ds_lrefer=$(get_prop lrefer $send_ds)
+	within_percent $ds_size $ds_lrefer 90 || log_fail \
+	    "$ds_size and $ds_lrefer differed by too much"
+
+	typeset vol_size=$(get_estimated_size "zfs send -nP $send_vol@snap")
+	typeset vol_lrefer=$(get_prop lrefer $send_vol)
+	within_percent $vol_size $vol_lrefer 90 || log_fail \
+	    "$vol_size and $vol_lrefer differed by too much"
+
+	typeset ds_csize=$(get_estimated_size "zfs send -nP -c $send_ds@snap")
+	typeset ds_refer=$(get_prop refer $send_ds)
+	within_percent $ds_csize $ds_refer 90 || log_fail \
+	    "$ds_csize and $ds_refer differed by too much"
+
+	typeset vol_csize=$(get_estimated_size "zfs send -nP -c $send_vol@snap")
+	typeset vol_refer=$(get_prop refer $send_vol)
+	within_percent $vol_csize $vol_refer 90 || log_fail \
+	    "$vol_csize and $vol_refer differed by too much"
+done
+
+log_pass "The the stream size given by -P accounts for compressed send."

tests/zfs-tests/tests/functional/rsend/send-c_verify_contents.ksh

@@ -0,0 +1,55 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify compressed send streams replicate data and datasets
+#
+# Strategy:
+# 1. Back up all the data from POOL/FS
+# 2. Verify all the datasets and data can be recovered in POOL2
+# 3. Back up all the data from root filesystem POOL2
+# 4. Verify all the data can be recovered, too
+#
+
+verify_runnable "both"
+
+log_assert "zfs send -c -R send replication stream up to the named snap."
+log_onexit cleanup_pool $POOL2
+
+# Verify the entire pool and descendants can be backed up and restored.
+log_must eval "zfs send -c -R $POOL@final > $BACKDIR/pool-final-R"
+log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R"
+
+dstds=$(get_dst_ds $POOL $POOL2)
+log_must cmp_ds_subs $POOL $dstds
+log_must cmp_ds_cont $POOL $dstds
+
+# Cleanup POOL2
+log_must cleanup_pool $POOL2
+
+# Verify all the filesystems and descendants can be backed up and restored.
+log_must eval "zfs send -c -R $POOL/$FS@final > $BACKDIR/fs-final-R"
+log_must eval "zfs receive -d $POOL2 < $BACKDIR/fs-final-R"
+
+dstds=$(get_dst_ds $POOL/$FS $POOL2)
+log_must cmp_ds_subs $POOL/$FS $dstds
+log_must cmp_ds_cont $POOL/$FS $dstds
+
+log_pass "zfs send -c -R send replication stream up to the named snap."

tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh

@@ -0,0 +1,66 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+. $STF_SUITE/include/properties.shlib
+
+#
+# Description:
+# Verify that the amount of data in a send -c stream matches compressratio.
+#
+# Strategy:
+# 1. For random compression types, and compressible / incompressible data:
+# 2. Create a snap with data
+# 3. Compare the size of the stream with the data on the dataset, adjusted
+#    by compressratio for normal send, and compared to used for send -c.
+#
+
+verify_runnable "both"
+
+log_assert "Verify send -c streams are compressed"
+log_onexit cleanup_pool $POOL2
+
+typeset sendfs=$POOL2/$FS
+typeset megs=128
+
+for prop in $(get_rand_compress_any 6); do
+	for compressible in 'yes' 'no'; do
+		log_must zfs create -o compress=$prop $sendfs
+
+		if [[ $compressible = 'yes' ]]; then
+			write_compressible $(get_prop mountpoint $sendfs) \
+			    ${megs}m
+		else
+			typeset file="$(get_prop mountpoint $sendfs)/ddfile"
+			log_must dd if=/dev/urandom of=$file bs=1024k count=$megs
+		fi
+
+		log_must zfs snapshot $sendfs@snap
+
+		# Calculate the sizes and verify the compression ratio.
+		log_must eval "zfs send $sendfs@snap >$BACKDIR/uncompressed"
+		verify_stream_size $BACKDIR/uncompressed $sendfs
+
+		log_must eval "zfs send -c $sendfs@snap >$BACKDIR/compressed"
+		verify_stream_size $BACKDIR/compressed $sendfs
+
+		log_must rm $BACKDIR/uncompressed $BACKDIR/compressed
+		log_must zfs destroy -r $sendfs
+	done
+done
+
+log_pass "Verify send -c streams are compressed"

tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh

@@ -0,0 +1,80 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify that compressed send correctly handles volumes
+#
+# Strategy:
+# 1. Write compressible data into a volume, take a snap
+# 2. Verify the compressed stream is the correct size, and has the correct data
+# 3. Repeat step 2 for an incremental compressed stream
+#
+
+function cleanup
+{
+	log_must zfs destroy -r $vol
+	cleanup_pool $POOL2
+}
+
+verify_runnable "both"
+
+log_assert "Verify compressed send works with volumes"
+log_onexit cleanup
+
+typeset vol="$POOL/newvol"
+typeset vol2="$POOL2/newvol"
+typeset voldev="$ZVOL_DEVDIR/$POOL/newvol"
+typeset voldev2="$ZVOL_DEVDIR/$POOL2/newvol"
+typeset data1=$BACKDIR/file.0
+typeset data2=$BACKDIR/file.1
+typeset megs=8
+
+log_must zfs create -V 256m -o compress=lz4 $vol
+
+write_compressible $BACKDIR ${megs}m 2
+md5_1=$(md5sum $data1 | awk '{print $1}')
+md5_2=$(md5sum $data2 | awk '{print $1}')
+
+log_must dd if=$data1 of=$voldev bs=1024k
+log_must zfs snapshot $vol@snap
+
+log_must eval "zfs send -c $vol@snap >$BACKDIR/full"
+log_must eval "zfs recv -d $POOL2 <$BACKDIR/full"
+
+verify_stream_size $BACKDIR/full $vol
+verify_stream_size $BACKDIR/full $vol2
+md5=$(dd if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5sum | \
+    awk '{print $1}')
+[[ $md5 = $md5_1 ]] || log_fail "md5 mismatch: $md5 != $md5_1"
+
+# Repeat, for an incremental send
+log_must dd seek=$megs if=$data2 of=$voldev bs=1024k
+log_must zfs snapshot $vol@snap2
+
+log_must eval "zfs send -c -i snap $vol@snap2 >$BACKDIR/inc"
+log_must eval "zfs recv -d $POOL2 <$BACKDIR/inc"
+
+verify_stream_size $BACKDIR/inc $vol 90 $vol@snap
+verify_stream_size $BACKDIR/inc $vol2 90 $vol2@snap
+md5=$(dd skip=$megs if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5sum | \
+    awk '{print $1}')
+[[ $md5 = $md5_2 ]] || log_fail "md5 mismatch: $md5 != $md5_2"
+
+log_pass "Verify compressed send works with volumes"

tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh

@@ -0,0 +1,59 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+. $STF_SUITE/include/math.shlib
+
+#
+# Description:
+# Verify compression features show up in zstreamdump
+#
+# Strategy:
+# 1. Create a full compressed send stream
+# 2. Verify zstreamdump shows this stream has the relevant features
+# 3. Verify zstreamdump's accounting of logical and compressed size is correct
+#
+
+verify_runnable "both"
+
+log_assert "Verify zstreamdump correctly interprets compressed send streams."
+log_onexit cleanup_pool $POOL2
+
+typeset sendfs=$POOL2/fs
+
+log_must zfs create -o compress=lz4 $sendfs
+typeset dir=$(get_prop mountpoint $sendfs)
+write_compressible $dir 16m
+log_must zfs snapshot $sendfs@full
+
+log_must eval "zfs send -c $sendfs@full >$BACKDIR/full"
+log_must stream_has_features $BACKDIR/full lz4 compressed
+cat $BACKDIR/full | zstreamdump -v | parse_dump > $BACKDIR/dump.out
+
+lsize=$(awk '/^WRITE [^0]/ {lsize += $4} END {printf("%d", lsize)}' \
+    $BACKDIR/dump.out)
+lsize_prop=$(get_prop logicalused $sendfs)
+within_percent $lsize $lsize_prop 90 || log_fail \
+    "$lsize and $lsize_prop differed by too much"
+
+csize=$(awk '/^WRITE [^0]/ {csize += $5} END {printf("%d", csize)}' \
+    $BACKDIR/dump.out)
+csize_prop=$(get_prop used $sendfs)
+within_percent $csize $csize_prop 90 || log_fail \
+    "$csize and $csize_prop differed by too much"
+
+log_pass "zstreamdump correctly interprets compressed send streams."

tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh

@@ -0,0 +1,199 @@
+#!/bin/ksh -p
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2015 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify compressed send works correctly with datasets of varying recsize.
+#
+# Strategy:
+# 1. Check the recv behavior (into pools with features enabled and disabled)
+#    of all combinations of -c -p and -L. Verify the stream is compressed,
+#    and that the recsize property and that of a received file is correct
+#    according to this matrix:
+#
+# +---------+--------+------------+------------+-----------+-----------+
+# | send    | send   | received   | received   | received  | received  |
+# | stream  | stream | file bs    | prop       | file bs   | props     |
+# | recsize | flags  | (disabled) | (disabled) | (enabled) | (enabled) |
+# +---------+--------+------------+------------+-----------+-----------+
+# |    128k |        |       128k |       128k |      128k |      128k |
+# |    128k |     -c |      Fails |      Fails |      128k |      128k |
+# |    128k |     -p |       128k |       128k |      128k |      128k |
+# |    128k |     -L |       128k |       128k |      128k |      128k |
+# |    128k |    -cp |      Fails |      Fails |      128k |      128k |
+# |    128k |    -cL |      Fails |      Fails |      128k |      128k |
+# |    128k |    -pL |       128k |       128k |      128k |      128k |
+# |    128k |   -cpL |      Fails |      Fails |      128k |      128k |
+# |      1m |        |      Fails |      Fails |      128k |      128k |
+# |      1m |     -c |      Fails |      Fails |      128k |      128k |
+# |      1m |     -p |       128k |       128k |      128k |        1m |
+# |      1m |     -L |      Fails |      Fails |        1m |      128k |
+# |      1m |    -cp |      Fails |      Fails |      128k |        1m |
+# |      1m |    -cL |      Fails |      Fails |        1m |      128k |
+# |      1m |    -pL |      Fails |      Fails |        1m |        1m |
+# |      1m |   -cpL |      Fails |      Fails |        1m |        1m |
+# +---------+--------+------------+------------+-----------+-----------+
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	datasetexists $TESTPOOL/128k && log_must zfs destroy $TESTPOOL/128k
+	datasetexists $TESTPOOL/1m && log_must zfs destroy $TESTPOOL/1m
+	cleanup_pool $POOL2
+	destroy_pool $POOL3
+}
+
+# For a received stream, verify the recsize (prop and file) match expectations.
+function check_recsize
+{
+	typeset recv_ds=$1
+	typeset expected_file_bs=$2
+	typeset expected_recsize=$3
+	typeset file="$(get_prop mountpoint $recv_ds)/testfile"
+
+	[[ -f $file ]] || log_fail "file '$file' doesn't exist"
+
+	typeset read_recsize=$(get_prop recsize $recv_ds)
+	typeset read_file_bs=$(stat $file | sed -n \
+	    's/.*IO Block: \([0-9]*\).*/\1/p')
+
+	[[ $read_recsize = $expected_recsize ]] || log_fail \
+	    "read_recsize: $read_recsize expected_recsize: $expected_recsize"
+	[[ $read_file_bs = $expected_file_bs ]] || log_fail \
+	    "read_file_bs: $read_file_bs expected_file_bs: $expected_file_bs"
+}
+
+#
+# This function does a zfs send and receive according to the parameters
+# below, and verifies the data shown in the strategy section.
+#
+# -[cpL] flags to pass through to 'zfs send'
+# -d Receive into a pool with all features disabled
+#
+# $1 The recordsize of the send dataset
+# $2 Whether or not the recv should work.
+# $3 The blocksize expected in a received file (default 128k)
+# $4 The recordsize property expected in a received dataset (default 128k)
+#
+function check
+{
+	typeset recv_pool=$POOL2
+	typeset flags='-'
+
+	while getopts "cdpL" opt; do
+		case $opt in
+		c)
+			flags+='c'
+			;;
+		d)
+			recv_pool=$POOL3
+			;;
+		p)
+			flags+='p'
+			;;
+		L)
+			flags+='L'
+			;;
+		esac
+	done
+	shift $(($OPTIND - 1))
+	[[ ${#flags} -eq 1 ]] && flags=''
+
+	typeset recsize=$1
+	typeset verify=$2
+	typeset expected_file_bs=${3-131072}
+	typeset expected_recsize=${4-131072}
+	typeset send_ds=$TESTPOOL/$recsize
+	typeset send_snap=$send_ds@snap
+	typeset recv_ds=$recv_pool/$recsize
+	typeset stream=$BACKDIR/stream.out
+
+	datasetexists $send_ds || log_fail "send ds: $send_ds doesn't exist"
+	[[ -f $stream ]] && log_must rm $stream
+	log_must eval "zfs send $flags $send_snap >$stream"
+	$verify eval "zfs recv $recv_ds <$stream"
+	typeset stream_size=$(cat $stream | zstreamdump | sed -n \
+	    's/	Total write size = \(.*\) (0x.*)/\1/p')
+
+	#
+	# Special case: For a send dataset with large blocks, don't try to
+	# verify the stream size is correct if the compress flag is present
+	# but the large blocks flag isn't. In these cases, the user data
+	# isn't compressed in the stream (though metadata is) so the
+	# verification would fail.
+	#
+	typeset do_size_test=true
+	[[ $recsize = $large && $flags =~ 'c' && ! $flags =~ 'L' ]] && \
+	    do_size_test=false
+
+	$do_size_test && verify_stream_size $stream $send_ds
+
+	if [[ $verify = "log_mustnot" ]]; then
+		datasetnonexists $recv_ds || log_fail "$recv_ds shouldn't exist"
+		return
+	fi
+
+	check_recsize $recv_ds $expected_file_bs $expected_recsize
+	$do_size_test && verify_stream_size $stream $recv_ds
+	log_must zfs destroy -r $recv_ds
+}
+
+log_assert "Verify compressed send works with datasets of varying recsize."
+log_onexit cleanup
+typeset recsize opts dir
+typeset small=$((128 * 1024))
+typeset large=$((1024 * 1024))
+
+# Create POOL3 with features disabled and datasets to create test send streams
+datasetexists $POOL3 && log_must zpool destroy $POOL3
+log_must zpool create -d $POOL3 $DISK3
+write_compressible $BACKDIR 32m
+for recsize in $small $large; do
+	log_must zfs create -o compress=gzip -o recsize=$recsize \
+	    $TESTPOOL/$recsize
+	dir=$(get_prop mountpoint $TESTPOOL/$recsize)
+	log_must cp $BACKDIR/file.0 $dir/testfile
+	log_must zfs snapshot $TESTPOOL/$recsize@snap
+done
+
+# Run tests for send streams without large blocks
+for opts in '' -d -c -p -dp -L -dL -cp -cL -pL -dpL -cpL; do
+	check $opts $small log_must
+done
+for opts in -dc -dcp -dcL -dcpL; do
+	check $opts $small log_mustnot
+done
+
+# Run tests for send streams with large blocks
+for opts in '' -d -dp -c; do
+	check $opts $large log_must
+done
+for opts in -dc -dL -dcp -dcL -dpL -dcpL; do
+	check $opts $large log_mustnot
+done
+check -p $large log_must $small $large
+check -L $large log_must $large $small
+check -cp $large log_must $small $large
+check -cL $large log_must $large $small
+check -pL $large log_must $large $large
+check -cpL $large log_must $large $large
+
+log_pass "Compressed send works with datasets of varying recsize."

tests/zfs-tests/tests/perf/fio/mkfiles.fio

@@ -10,7 +10,7 @@
 #
 
 #
-# Copyright (c) 2015 by Delphix. All rights reserved.
+# Copyright (c) 2016 by Delphix. All rights reserved.
 #
 
 [global]
@@ -24,7 +24,7 @@ thread=1
 directory=/${TESTFS}
 numjobs=${NUMJOBS}
 filesize=${FILE_SIZE}
-buffer_compress_percentage=33
+buffer_compress_percentage=66
 buffer_compress_chunk=4096
 
 [job]

tests/zfs-tests/tests/perf/fio/random_readwrite.fio

@@ -10,7 +10,7 @@
 #
 
 #
-# Copyright (c) 2015 by Delphix. All rights reserved.
+# Copyright (c) 2015, 2016 by Delphix. All rights reserved.
 #
 
 [global]
@@ -29,7 +29,7 @@ bssplit=4k/50:8k/30:128k/10:1m/10
 ioengine=psync
 sync=${SYNC_TYPE}
 numjobs=${NUMJOBS}
-buffer_compress_percentage=33
+buffer_compress_percentage=66
 buffer_compress_chunk=4096
 
 [job]

tests/zfs-tests/tests/perf/fio/random_writes.fio

@@ -10,7 +10,7 @@
 #
 
 #
-# Copyright (c) 2015 by Delphix. All rights reserved.
+# Copyright (c) 2015, 2016 by Delphix. All rights reserved.
 #
 
 [global]
@@ -27,7 +27,7 @@ ioengine=psync
 sync=${SYNC_TYPE}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
-buffer_compress_percentage=33
+buffer_compress_percentage=66
 buffer_compress_chunk=4096
 
 [job]

tests/zfs-tests/tests/perf/fio/sequential_writes.fio

@@ -10,7 +10,7 @@
 #
 
 #
-# Copyright (c) 2015 by Delphix. All rights reserved.
+# Copyright (c) 2015, 2016 by Delphix. All rights reserved.
 #
 
 [global]
@@ -27,7 +27,7 @@ ioengine=psync
 sync=${SYNC_TYPE}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
-buffer_compress_percentage=33
+buffer_compress_percentage=66
 buffer_compress_chunk=4096
 
 [job]