Fix the send --exclude option to work with encryption

When using --exclude, filtering needs to take place in two places:
in zfs_main.c via the callback previously added to support the
options, and in libzfs_sendrecv.c because it generates the nvlist
during a first pass, and that results in it complaining if the
excluded dataset is not available for sending. (eg, excluding an
encrypted dataset so you don't have to use --raw wouldn't work,
because the first pass would look at the dataset and decide you
couldn't use it.) Add send --exclude tests, including one that tests
excluding an encrypted hierarchy.

Reviewed-by: Allan Jude <allan@klarasystems.com>
Reviewed-by: Alexander Motin <alexander.motin@TrueNAS.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Sean Eric Fagan <sef@kithrup.ie>
Closes #18278

tests/runfiles/common.run

@@ -987,7 +987,8 @@ tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
     'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos',
     'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos', 'rsend_025_pos',
     'rsend_026_neg', 'rsend_027_pos', 'rsend_028_neg', 'rsend_029_neg',
-    'rsend_030_pos', 'rsend_031_pos', 'send-c_verify_ratio',
+    'rsend_030_pos', 'rsend_031_pos', 'rsend-exclude_001_pos',
+    'rsend-exclude_002_pos', 'send-c_verify_ratio',
     'send-c_verify_contents', 'send-c_props', 'send-c_incremental',
     'send-c_volume', 'send-c_zstream_recompress', 'send-c_zstreamdump',
     'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',

tests/runfiles/sanity.run

@@ -557,7 +557,8 @@ tags = ['functional', 'reservation']
 tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
     'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos',
     'rsend_006_pos', 'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos',
-    'rsend_014_pos', 'rsend_016_neg', 'send-c_verify_contents',
+    'rsend_014_pos', 'rsend_016_neg', 'rsend-exclude_001_pos',
+    'rsend-exclude_002_pos', 'send-c_verify_contents',
     'send-c_volume', 'send-c_zstreamdump', 'send-c_recv_dedup',
     'send-L_toggle', 'send_encrypted_hierarchy', 'send_encrypted_props',
     'send_encrypted_freeobjects',

tests/zfs-tests/tests/Makefile.am

@@ -2051,6 +2051,8 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/rsend/rsend_029_neg.ksh \
 	functional/rsend/rsend_030_pos.ksh \
 	functional/rsend/rsend_031_pos.ksh \
+	functional/rsend/rsend-exclude_001_pos.ksh \
+	functional/rsend/rsend-exclude_002_pos.ksh \
 	functional/rsend/send-c_embedded_blocks.ksh \
 	functional/rsend/send-c_incremental.ksh \
 	functional/rsend/send-c_longname.ksh \

tests/zfs-tests/tests/functional/rsend/rsend-exclude_001_pos.ksh

@@ -0,0 +1,68 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2023 by Delphix. All rights reserved.
+# Copyright (c) 2026 by Sean Eric Fagan. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify recursive incremental with -X properly excludes
+# specified datasets.
+#
+# Strategy:
+# 1. Create multiple datasets on source pool.
+# 2. Create snapshots on source pool.
+# 3. Recursively send snapshots excluding datasets
+# 4. Confirm destination pool does not include excluded datasets.
+#
+
+verify_runnable "both"
+
+sendfs=$POOL/sendfs
+recvfs=$POOL2/recvfs
+
+function cleanup {
+	rm -f $BACKDIR/list
+	rm -f $BACKDIR/stream1
+	zfs destroy -rf $sendfs
+	zfs destroy -rf $recvfs
+}
+
+log_assert "Verify recursive sends excluding datasets behave properly."
+log_onexit cleanup
+
+log_must zfs create $sendfs
+log_must zfs create $sendfs/ds1
+log_must zfs create $sendfs/ds1/sub1
+log_must zfs create $sendfs/ds1/sub1/sub2
+log_must zfs create $sendfs/ds2
+log_must zfs create $sendfs/ds2/sub1
+log_must zfs create $sendfs/ds2/sub1/sub3
+log_must zfs create $recvfs
+
+log_must zfs snapshot -r $sendfs@A
+
+# Now we'll send $sendfs@A, but exclude ds1/sub1
+log_must zfs send -R --exclude ds1/sub1 $sendfs@A > $BACKDIR/stream1
+log_must zfs recv -dFu $recvfs < $BACKDIR/stream1
+log_must zfs list -r $recvfs > $BACKDIR/list
+lost_mustnot grep -q ds1/sub1/sub2 $BACKDIR/list
+lost_mustnot grep -q ds1/sub1 $BACKDIR/list
+log_must grep -q ds2/sub1 $BACKDIR/list
+
+log_pass "Verify recursive incremental excluding datasets behave properly."

tests/zfs-tests/tests/functional/rsend/rsend-exclude_002_pos.ksh

@@ -0,0 +1,73 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2023 by Delphix. All rights reserved.
+# Copyright (c) 2026 by Sean Eric Fagan. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# Description:
+# Verify recursive incremental with -X properly excludes
+# encrypted specified datasets.
+#
+# Strategy:
+# 1. Create multiple datasets on source pool.
+# 2. Create snapshots on source pool.
+# 3. Recursively send snapshots excluding datasets
+# 4. Confirm destination pool does not include excluded datasets.
+#
+
+verify_runnable "both"
+
+sendfs=$POOL/sendfs
+recvfs=$POOL2/recvfs
+PASSPHRASE=${PASSPHRASE:-password}
+
+function cleanup {
+	rm -f $BACKDIR/list
+	rm -f $BACKDIR/stream1
+	zfs destroy -rf $sendfs
+	zfs destroy -rf $recvfs
+}
+
+log_assert "Verify recursive sends excluding datasets behave properly."
+log_onexit cleanup
+
+log_must zfs create $sendfs
+log_must zfs create $sendfs/ds1
+log_must zfs create $sendfs/ds1/sub1
+log_must zfs create $sendfs/ds1/sub1/sub2
+log_must zfs create $sendfs/ds2
+log_must zfs create $sendfs/ds2/sub1
+log_must zfs create $sendfs/ds2/sub1/sub3
+log_must eval "echo $PASSPHRASE | zfs create -o encryption=on" \
+	 "-o keyformat=passphrase $sendfs/enc"
+log_must zfs create $sendfs/enc/enc2
+
+log_must zfs create $recvfs
+
+log_must zfs snapshot -r $sendfs@A
+
+# Now we'll send $sendfs@A, but exclude enc
+log_must zfs send -R --exclude $sendfs/enc  $sendfs@A > $BACKDIR/stream1
+log_must zfs recv -dFu $recvfs < $BACKDIR/stream1
+log_must zfs list -r $recvfs > $BACKDIR/list
+lost_mustnot grep -q enc/enc2 $BACKDIR/list
+lost_mustnot grep -q enc $BACKDIR/list
+log_must grep -q ds2/sub1 $BACKDIR/list
+
+log_pass "Verify recursive incremental excluding encrypted datasets behave properly."