Allow zfs send to exclude datasets

Add support for a -exclude/-X option to `zfs send` to allow dataset 
hierarchies to be excluded.

Snapshots can be excluded using a channel program; however,
this can result in failures with 'zfs send -R'; this option allows 
them to be excluded.  Fortunately, this required a change only to 
cmd/zfs/zfs_main.c, using the already-existing callback argument 
to zfs_send() that is currently unused.

Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: Christian Schwarz <christian.schwarz@nutanix.com>
Reviewed-by: Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Co-authored-by: Sean Eric Fagan <kithrup@mac.com>
Signed-off-by: Sean Eric Fagan <kithrup@mac.com>
Closes #13158

cmd/zfs/zfs_main.c

@@ -315,8 +315,9 @@ get_usage(zfs_help_t idx)
 	case HELP_ROLLBACK:
 		return (gettext("\trollback [-rRf] <snapshot>\n"));
 	case HELP_SEND:
-		return (gettext("\tsend [-DnPpRvLecwhb] [-[i|I] snapshot] "
-		    "<snapshot>\n"
+		return (gettext("\tsend [-DnPpRvLecwhb] "
+		    "[-X dataset[,dataset]...] "
+		    "[-[i|I] snapshot] <snapshot>\n"
 		    "\tsend [-DnvPLecw] [-i snapshot|bookmark] "
 		    "<filesystem|volume|snapshot>\n"
 		    "\tsend [-DnPpvLec] [-i bookmark|snapshot] "
@@ -4317,6 +4318,77 @@ usage:
 	return (-1);
 }
 
+typedef struct zfs_send_exclude_arg {
+	size_t count;
+	char **list;
+} zfs_send_exclude_arg_t;
+
+/*
+ * This function creates the zfs_send_exclude_arg_t
+ * object described above; it can be called multiple
+ * times, and the input can be comma-separated.
+ * This is NOT the most efficient data layout; however,
+ * I couldn't think of a non-pathological case where
+ * it should have more than a couple dozen instances
+ * of excludes. If that turns out to be used in
+ * practice, we might want to instead use a tree.
+ */
+static void
+add_dataset_excludes(char *exclude, zfs_send_exclude_arg_t *context)
+{
+	char *tok;
+	while ((tok = strsep(&exclude, ",")) != NULL) {
+		if (!zfs_name_valid(tok, ZFS_TYPE_DATASET) ||
+		    strchr(tok, '/') == NULL) {
+			(void) fprintf(stderr, gettext("-X %s: "
+			    "not a valid non-root dataset name.\n"), tok);
+			usage(B_FALSE);
+		}
+		context->list = safe_realloc(context->list,
+		    (sizeof (char *)) * (context->count + 1));
+		context->list[context->count++] = tok;
+	}
+}
+
+static void
+free_dataset_excludes(zfs_send_exclude_arg_t *exclude_list)
+{
+	free(exclude_list->list);
+}
+
+/*
+ * This is the call back used by zfs_send to
+ * determine if a dataset should be skipped.
+ * As stated above, this is not the most efficient
+ * data structure to use, but as long as the
+ * number of excluded datasets is relatively
+ * small (a couple of dozen or so), it won't
+ * have a big impact on performance on modern
+ * processors. Since it's excluding hierarchies,
+ * we'd probably want to move to a more complex
+ * tree structure in that case.
+ */
+static boolean_t
+zfs_do_send_exclude(zfs_handle_t *zhp, void *context)
+{
+	zfs_send_exclude_arg_t *exclude = context;
+	const char *name = zfs_get_name(zhp);
+
+	for (size_t indx = 0; indx < exclude->count; indx++) {
+		char *exclude_name = exclude->list[indx];
+		size_t len = strlen(exclude_name);
+		/* If it's shorter, it can't possibly match */
+		if (strlen(name) < len)
+			continue;
+		if (strncmp(name, exclude_name, len) == 0 &&
+		    (name[len] == '/' || name[len] == '\0' ||
+		    name[len] == '@')) {
+			return (B_FALSE);
+		}
+	}
+
+	return (B_TRUE);
+}
 
 /*
  * Send a backup stream to stdout.
@@ -4333,6 +4405,7 @@ zfs_do_send(int argc, char **argv)
 	int c, err;
 	nvlist_t *dbgnv = NULL;
 	char *redactbook = NULL;
+	zfs_send_exclude_arg_t exclude_context = { 0 };
 
 	struct option long_options[] = {
 		{"replicate",	no_argument,		NULL, 'R'},
@@ -4351,13 +4424,17 @@ zfs_do_send(int argc, char **argv)
 		{"backup",	no_argument,		NULL, 'b'},
 		{"holds",	no_argument,		NULL, 'h'},
 		{"saved",	no_argument,		NULL, 'S'},
+		{"exclude",	required_argument,	NULL, 'X'},
 		{0, 0, 0, 0}
 	};
 
 	/* check options */
-	while ((c = getopt_long(argc, argv, ":i:I:RsDpvnPLeht:cwbd:S",
+	while ((c = getopt_long(argc, argv, ":i:I:RsDpvnPLeht:cwbd:SX:",
 	    long_options, NULL)) != -1) {
 		switch (c) {
+		case 'X':
+			add_dataset_excludes(optarg, &exclude_context);
+			break;
 		case 'i':
 			if (fromname)
 				usage(B_FALSE);
@@ -4467,6 +4544,13 @@ zfs_do_send(int argc, char **argv)
 	if (flags.parsable && flags.verbosity == 0)
 		flags.verbosity = 1;
 
+	if (exclude_context.count > 0 && !flags.replicate) {
+		(void) fprintf(stderr, gettext("Cannot specify "
+		    "dataset exclusion (-X) on a non-recursive "
+		    "send.\n"));
+		return (1);
+	}
+
 	argc -= optind;
 	argv += optind;
 
@@ -4648,8 +4732,11 @@ zfs_do_send(int argc, char **argv)
 	if (flags.replicate && fromname == NULL)
 		flags.doall = B_TRUE;
 
-	err = zfs_send(zhp, fromname, toname, &flags, STDOUT_FILENO, NULL, 0,
-	    flags.verbosity >= 3 ? &dbgnv : NULL);
+	err = zfs_send(zhp, fromname, toname, &flags, STDOUT_FILENO,
+	    exclude_context.count > 0 ? zfs_do_send_exclude : NULL,
+	    &exclude_context, flags.verbosity >= 3 ? &dbgnv : NULL);
+
+	free_dataset_excludes(&exclude_context);
 
 	if (flags.verbosity >= 3 && dbgnv != NULL) {
 		/*

man/man8/zfs-send.8

@@ -40,6 +40,7 @@
 .Nm zfs
 .Cm send
 .Op Fl DLPRbcehnpsvw
+.Op Fl X Ar dataset Ns Oo , Ns Ar dataset Oc Ns ...
 .Op Oo Fl I Ns | Ns Fl i Oc Ar snapshot
 .Ar snapshot
 .Nm zfs
@@ -73,6 +74,7 @@
 .Nm zfs
 .Cm send
 .Op Fl DLPRbcehnpvw
+.Op Fl X Ar dataset Ns Oo , Ns Ar dataset Oc Ns ...
 .Op Oo Fl I Ns | Ns Fl i Oc Ar snapshot
 .Ar snapshot
 .Xc
@@ -140,6 +142,23 @@ If the
 flag is used to send encrypted datasets, then
 .Fl w
 must also be specified.
+.It Fl X , -exclude Ar dataset Ns Oo , Ns Ar dataset Oc Ns ...
+When the
+.Fl R
+flag is given,
+.Fl X
+can be used to specify a list of datasets to be excluded from the
+data stream.
+The
+.Fl X
+option can be used multiple times, or the list of datasets can be
+specified as a comma-separated list, or both.
+.Ar dataset
+must not be the pool's root dataset, and all descendant datasets of
+.Ar dataset
+will be excluded from the send stream.
+Requires
+.Fl R .
 .It Fl e , -embed
 Generate a more compact stream by using
 .Sy WRITE_EMBEDDED

tests/runfiles/common.run

@@ -824,7 +824,8 @@ tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
     'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', 'rsend_009_pos',
     'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', 'rsend_013_pos',
     'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos',
-    'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos',
+    'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos', 'rsend_025_pos',
+    'rsend_026_neg', 'rsend_027_pos', 'rsend_028_neg', 'rsend_029_neg',
     'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props',
     'send-c_incremental', 'send-c_volume', 'send-c_zstreamdump',
     'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',

tests/zfs-tests/tests/functional/rsend/Makefile.am

@@ -24,6 +24,11 @@ dist_pkgdata_SCRIPTS = \
 	rsend_021_pos.ksh \
 	rsend_022_pos.ksh \
 	rsend_024_pos.ksh \
+	rsend_025_pos.ksh \
+	rsend_026_neg.ksh \
+	rsend_027_pos.ksh \
+	rsend_028_neg.ksh \
+	rsend_029_neg.ksh \
 	send_encrypted_files.ksh \
 	send_encrypted_hierarchy.ksh \
 	send_encrypted_props.ksh \

tests/zfs-tests/tests/functional/rsend/rsend_025_pos.ksh

@@ -0,0 +1,90 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+# DESCRIPTION:
+#	zfs send -exclude will exclude the given hierarchy
+#	and only that given hierarchy.
+#
+# STRATEGY:
+#	1. Setup test model
+#	2. Create several datasets on pool.
+#	3. Send -R -X pool/dataset
+#	4. Verify receive does not have the excluded dataset(s).
+
+verify_runnable "both"
+
+function cleanup
+{
+    cleanup_pool $POOL2
+    cleanup_pool $POOL
+    log_must setup_test_model $POOL
+}
+
+log_assert "zfs send -R -X will skip excluded dataset(s)"
+log_onexit cleanup
+
+cleanup
+
+#
+# Create some datasets
+log_must zfs create -p $POOL/ds1/second/third
+log_must zfs create -p $POOL/ds2/second
+
+log_must zfs snapshot -r $POOL@presend
+
+log_must eval "zfs send -R $POOL@presend > $BACKDIR/presend"
+log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/presend"
+
+for ds in ds1 ds1/second ds1/second/third \
+	      ds2 ds2/second
+do
+    log_must datasetexists $POOL2/$ds
+done
+
+log_must_busy zfs destroy -r $POOL2
+
+log_must eval "zfs send -R -X $POOL/ds1/second $POOL@presend > $BACKDIR/presend"
+log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/presend"
+
+for ds in ds1 ds2 ds2/second
+do
+    log_must datasetexists $POOL2/$ds
+done
+
+for ds in ds1/second ds1/second/third
+do
+    log_must datasetnonexists $POOL2/$ds
+done
+
+log_pass "zfs send -X excluded datasets"
+

tests/zfs-tests/tests/functional/rsend/rsend_026_neg.ksh

@@ -0,0 +1,58 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+# DESCRIPTION:
+#	zfs send -X without -R will fail.
+#
+# STRATEGY:
+#	1. Setup test model
+#	2. Run "zfs send -X random $POOL" and check for failure.
+
+verify_runnable "both"
+
+function cleanup
+{
+    cleanup_pool $POOL2
+    cleanup_pool $POOL
+    log_must setup_test_model $POOL
+}
+
+log_assert "zfs send -X without -R will fail"
+log_onexit cleanup
+
+cleanup
+
+log_mustnot eval "zfs send -X $POOL/foobar $POOL@final"
+
+log_pass "Ensure that zfs send -X without -R will fail"

tests/zfs-tests/tests/functional/rsend/rsend_027_pos.ksh

@@ -0,0 +1,92 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+# DESCRIPTION:
+#	zfs send with multiple -X/--exclude options will
+#	exclude all of them.
+#
+# STRATEGY:
+#	1. Setup test model
+#	2. Create several datasets on pool.
+#	3. Send -R -X pool/dataset
+#	4. Verify receive does not have the excluded dataset(s).
+
+verify_runnable "both"
+
+function cleanup
+{
+    cleanup_pool $POOL2
+    cleanup_pool $POOL
+    log_must setup_test_model $POOL
+}
+
+log_assert "zfs send with multiple -X options will skip excluded dataset"
+log_onexit cleanup
+
+cleanup
+
+#
+# Create some datasets
+log_must zfs create -p $POOL/ds1/second/third
+log_must zfs create -p $POOL/ds2/second
+log_must zfs create -p $POOL/ds3/first/second/third
+
+log_must zfs snapshot -r $POOL@presend
+
+log_must eval "zfs send -R $POOL@presend > $BACKDIR/presend"
+log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/presend"
+
+for ds in ds1 ds1/second ds1/second/third \
+	      ds2 ds2/second \
+	      ds3 ds3/first ds3/first/second ds3/first/second/third
+do
+    log_must datasetexists $POOL2/$ds
+done
+
+log_must_busy zfs destroy -r $POOL2
+
+log_must eval "zfs send -R -X $POOL/ds1/second --exclude $POOL/ds3/first/second $POOL@presend > $BACKDIR/presend"
+log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/presend"
+
+for ds in ds1 ds2 ds2/second ds3 ds3/first
+do
+    log_must datasetexists $POOL2/$ds
+done
+
+for ds in ds1/second ds1/second/third ds3/first/second ds3/first/second/third
+do
+    log_must datasetnonexists $POOL2/$ds
+done
+
+log_pass "zfs send with multiple -X options  excluded datasets"
+

tests/zfs-tests/tests/functional/rsend/rsend_028_neg.ksh

@@ -0,0 +1,58 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+# DESCRIPTION:
+#	zfs send -X with invalid dataset name will fail.
+#
+# STRATEGY:
+#	1. Setup test model
+#	2. Run "zfs send -X $POOL $POOL" and check for failure.
+
+verify_runnable "both"
+
+function cleanup
+{
+    cleanup_pool $POOL2
+    cleanup_pool $POOL
+    log_must setup_test_model $POOL
+}
+
+log_assert "zfs send -X $POOL will fail"
+log_onexit cleanup
+
+cleanup
+
+log_mustnot eval "zfs send -X $POOL $POOL@final"
+
+log_pass "Ensure that zfs send -X $POOL will fail"

tests/zfs-tests/tests/functional/rsend/rsend_029_neg.ksh

@@ -0,0 +1,58 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+# DESCRIPTION:
+#	zfs send -X with invalid dataset name will fail.
+#
+# STRATEGY:
+#	1. Setup test model
+#	2. Run "zfs send -X $POOL/da%set $POOL" and check for failure.
+
+verify_runnable "both"
+
+function cleanup
+{
+    cleanup_pool $POOL2
+    cleanup_pool $POOL
+    log_must setup_test_model $POOL
+}
+
+log_assert "zfs send -X $POOL/da%set will fail"
+log_onexit cleanup
+
+cleanup
+
+log_mustnot eval "zfs send -X $POOL/da%set $POOL@final"
+
+log_pass "Ensure that zfs send -X with invalid dataset name will fail"