Add 'zfs send --saved' flag

This commit adds the --saved (-S) to the 'zfs send' command.
This flag allows a user to send a partially received dataset,
which can be useful when migrating a backup server to new
hardware. This flag is compatible with resumable receives, so
even if the saved send is interrupted, it can be resumed.
The flag does not require any user / kernel ABI changes or any
new feature flags in the send stream format.

Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: Alek Pinchuk <apinchuk@datto.com>
Reviewed-by: Paul Zuchowski <pzuchowski@datto.com>
Reviewed-by: Christian Schwarz <me@cschwarz.com>
Reviewed-by: Matt Ahrens <matt@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes #9007

tests/runfiles/common.run

@@ -786,7 +786,8 @@ tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
     'send_encrypted_props', 'send_encrypted_truncated_files',
     'send_freeobjects', 'send_realloc_files',
     'send_realloc_encrypted_files', 'send_spill_block', 'send_holds',
-    'send_hole_birth', 'send_mixed_raw', 'send-wDR_encrypted_zvol']
+    'send_hole_birth', 'send_mixed_raw', 'send-wDR_encrypted_zvol',
+    'send_partial_dataset']
 tags = ['functional', 'rsend']
 
 [tests/functional/scrub_mirror]

tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c

@@ -508,6 +508,7 @@ test_send_new(const char *snapshot, int fd)
 	fnvlist_add_string(optional, "fromsnap", from);
 	fnvlist_add_uint64(optional, "resume_object", resumeobj);
 	fnvlist_add_uint64(optional, "resume_offset", offset);
+	fnvlist_add_boolean(optional, "savedok");
 #endif
 	IOC_INPUT_TEST(ZFS_IOC_SEND_NEW, snapshot, required, optional, 0);
 

tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh

@@ -73,7 +73,7 @@ log_must eval "get_diff $send_mnt/f3 $recv_mnt/f3 >$tmpdir/get_diff.out"
 range=$(cat $tmpdir/get_diff.out)
 [[ "$RANGE10" = "$range" ]] || log_fail "Unexpected range: $range"
 
-# Test recv -A works properly
+# Test recv -A works properly and verify saved sends are not allowed
 log_mustnot zfs recv -A $recvfs
 log_must zfs destroy -R $recvfs
 log_mustnot zfs recv -A $recvfs
@@ -81,6 +81,7 @@ log_must eval "zfs send --redact book1 $sendfs@snap >$stream"
 dd if=$stream bs=64k count=1 | log_mustnot zfs receive -s $recvfs
 [[ "-" = $(get_prop receive_resume_token $recvfs) ]] && \
     log_fail "Receive token not found."
+log_mustnot eval "zfs send --saved --redact book1 $recvfs > /dev/null"
 log_must zfs recv -A $recvfs
 log_must datasetnonexists $recvfs
 

tests/zfs-tests/tests/functional/rsend/Makefile.am

@@ -41,6 +41,7 @@ dist_pkgdata_SCRIPTS = \
 	send-c_zstreamdump.ksh \
 	send-cpL_varied_recsize.ksh \
 	send_freeobjects.ksh \
+	send_partial_dataset.ksh \
 	send_realloc_dnode_size.ksh \
 	send_realloc_files.ksh \
 	send_realloc_encrypted_files.ksh \

tests/zfs-tests/tests/functional/rsend/rsend.kshlib

@@ -563,17 +563,31 @@ function churn_files
 }
 
 #
-# Mess up file contents
+# Mess up a send file's contents
 #
-# $1 The file path
+# $1 The send file path
 #
-function mess_file
+function mess_send_file
 {
 	file=$1
 
 	filesize=$(stat_size $file)
 
 	offset=$(($RANDOM * $RANDOM % $filesize))
+
+	# The random offset might truncate the send stream to be
+	# smaller than the DRR_BEGIN record. If this happens, then
+	# the receiving system won't have enough info to create the
+	# partial dataset at all. We use zstreamdump to check for
+	# this and retry in this case.
+	nr_begins=$(head -c $offset $file | zstreamdump | \
+	    grep DRR_BEGIN | awk '{ print $5 }')
+	while [ "$nr_begins" -eq 0 ]; do
+		offset=$(($RANDOM * $RANDOM % $filesize))
+		nr_begins=$(head -c $offset $file | zstreamdump | \
+		    grep DRR_BEGIN | awk '{ print $5 }')
+	done
+
 	if (($RANDOM % 7 <= 1)); then
 		#
 		# We corrupt 2 bytes to minimize the chance that we
@@ -626,7 +640,7 @@ function resume_test
 	log_must eval "$sendcmd >/$streamfs/$stream_num"
 
 	for ((i=0; i<2; i=i+1)); do
-		mess_file /$streamfs/$stream_num
+		mess_send_file /$streamfs/$stream_num
 		log_mustnot zfs recv -suv $recvfs </$streamfs/$stream_num
 		stream_num=$((stream_num+1))
 

tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh

@@ -0,0 +1,110 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version a.0.
+# You may only use this file in accordance with the terms of version
+# a.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2019 Datto Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify that a partially received dataset can be sent with
+# 'zfs send --saved'.
+#
+# Strategy:
+# 1. Setup a pool with partially received filesystem
+# 2. Perform saved send without incremental
+# 3. Perform saved send with incremental
+# 4. Perform saved send with incremental, resuming from a token
+# 5. Perform negative tests for invalid command inputs
+#
+
+verify_runnable "both"
+
+log_assert "Verify that a partially received dataset can be sent with " \
+	"'zfs send --saved'."
+
+function cleanup
+{
+	destroy_dataset $POOL/testfs2 "-r"
+	destroy_dataset $POOL/stream "-r"
+	destroy_dataset $POOL/recvfs "-r"
+	destroy_dataset $POOL/partialfs "-r"
+}
+log_onexit cleanup
+
+log_must zfs create $POOL/testfs2
+log_must zfs create $POOL/stream
+mntpnt=$(get_prop mountpoint $POOL/testfs2)
+
+# Setup a pool with partially received filesystems
+log_must mkfile 1m $mntpnt/filea
+log_must zfs snap $POOL/testfs2@a
+log_must mkfile 1m $mntpnt/fileb
+log_must zfs snap $POOL/testfs2@b
+log_must eval "zfs send $POOL/testfs2@a | zfs recv $POOL/recvfs"
+log_must eval "zfs send -i $POOL/testfs2@a $POOL/testfs2@b > " \
+	"/$POOL/stream/inc.send"
+log_must eval "zfs send $POOL/testfs2@b > /$POOL/stream/full.send"
+mess_send_file /$POOL/stream/full.send
+mess_send_file /$POOL/stream/inc.send
+log_mustnot zfs recv -s $POOL/recvfullfs < /$POOL/stream/full.send
+log_mustnot zfs recv -s $POOL/recvfs < /$POOL/stream/inc.send
+
+# Perform saved send without incremental
+log_mustnot eval "zfs send --saved $POOL/recvfullfs | zfs recv -s " \
+	"$POOL/partialfs"
+token=$(zfs get -Hp -o value receive_resume_token $POOL/partialfs)
+log_must eval "zfs send -t $token | zfs recv -s $POOL/partialfs"
+file_check $POOL/recvfullfs $POOL/partialfs
+log_must zfs destroy -r $POOL/partialfs
+
+# Perform saved send with incremental
+log_must eval "zfs send $POOL/recvfs@a | zfs recv $POOL/partialfs"
+log_mustnot eval "zfs send --saved $POOL/recvfs | " \
+	"zfs recv -s $POOL/partialfs"
+token=$(zfs get -Hp -o value receive_resume_token $POOL/partialfs)
+log_must eval "zfs send -t $token | zfs recv -s $POOL/partialfs"
+file_check $POOL/recvfs $POOL/partialfs
+log_must zfs destroy -r $POOL/partialfs
+
+# Perform saved send with incremental, resuming from token
+log_must eval "zfs send $POOL/recvfs@a | zfs recv $POOL/partialfs"
+log_must eval "zfs send --saved $POOL/recvfs > " \
+	"/$POOL/stream/partial.send"
+mess_send_file /$POOL/stream/partial.send
+log_mustnot zfs recv -s $POOL/partialfs < /$POOL/stream/partial.send
+token=$(zfs get -Hp -o value receive_resume_token $POOL/partialfs)
+log_must eval "zfs send -t $token | zfs recv -s $POOL/partialfs"
+file_check $POOL/recvfs $POOL/partialfs
+
+# Perform negative tests for invalid command inputs
+set -A badargs \
+	"" \
+	"$POOL/recvfs@a" \
+	"-i $POOL/recvfs@a $POOL/recvfs@b" \
+	"-R $POOL/recvfs" \
+	"-p $POOL/recvfs" \
+	"-I $POOL/recvfs" \
+	"-D $POOL/recvfs" \
+	"-h $POOL/recvfs"
+
+while (( i < ${#badargs[*]} ))
+do
+	log_mustnot eval "zfs send --saved ${badargs[i]} >/dev/null"
+	(( i = i + 1 ))
+done
+
+log_pass "A partially received dataset can be sent with 'zfs send --saved'."