Receive checks should allow unencrypted child datasets

dmu_recv_begin_check() unconditionally sets the DS_HOLD_FLAG_DECRYPT
flag before calling dsl_dataset_hold_flags(). If the key on the
receiving side isn't loaded or the send stream contains embedded
blocks, the receive check fails for a stream which is perfectly
valid and could be received without any problem. This seems like
a remnant of the initial design, where unencrypted datasets below
encrypted ones weren't allowed.

Add a condition to set `DS_HOLD_FLAG_DECRYPT` only for encrypted
datasets, modify an existing test to detect this regression and add
a test for raw replication streams.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: George Amanakis <gamanakis@gmail.com>
Co-authored-by: George Amanakis <gamanakis@gmail.com>
Signed-off-by: Attila Fülöp <attila@fueloep.org>
Closes #13033 
Closes #13076

module/zfs/dmu_recv.c

@@ -597,7 +597,15 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
 		if (!(flags & DRR_FLAG_SPILL_BLOCK))
 			return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING));
 	} else {
-		dsflags |= DS_HOLD_FLAG_DECRYPT;
+		/*
+		 * We support unencrypted datasets below encrypted ones now,
+		 * so add the DS_HOLD_FLAG_DECRYPT flag only if we are dealing
+		 * with a dataset we may encrypt.
+		 */
+		if (drba->drba_dcp != NULL &&
+		    drba->drba_dcp->cp_crypt != ZIO_CRYPT_OFF) {
+			dsflags |= DS_HOLD_FLAG_DECRYPT;
+		}
 	}
 
 	error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds);

tests/runfiles/common.run

@@ -236,7 +236,8 @@ tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos',
     'receive-o-x_props_aliases',
     'zfs_receive_from_encrypted', 'zfs_receive_to_encrypted',
     'zfs_receive_raw', 'zfs_receive_raw_incremental', 'zfs_receive_-e',
-    'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props']
+    'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props',
+    'zfs_receive_-wR-encrypted-mix']
 tags = ['functional', 'cli_root', 'zfs_receive']
 
 [tests/functional/cli_root/zfs_rename]

tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am

@@ -27,7 +27,8 @@ dist_pkgdata_SCRIPTS = \
 	zfs_receive_raw.ksh \
 	zfs_receive_raw_incremental.ksh \
 	zfs_receive_raw_-d.ksh \
-	zfs_receive_-e.ksh
+	zfs_receive_-e.ksh \
+	zfs_receive_-wR-encrypted-mix.ksh
 
 dist_pkgdata_DATA = \
 	zstd_test_data.txt

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh

@@ -0,0 +1,75 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2022 by Attila Fülöp <attila@fueloep.org>
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+#      ZFS should receive a raw send of a mix of unencrypted and encrypted
+#      child datasets
+#
+#      The layout of the datasets is:  enc/unenc/enc/unenc
+#
+# STRATEGY:
+# 1. Create the dataset hierarchy
+# 2. Snapshot the dataset hierarchy
+# 3. Send -Rw the dataset hierarchy and receive into a top-level dataset
+# 4. Check the encryption property of the received datasets
+
+verify_runnable "both"
+
+function cleanup
+{
+	datasetexists "$TESTPOOL/$TESTFS1" && \
+		destroy_dataset "$TESTPOOL/$TESTFS1" -r
+
+	datasetexists "$TESTPOOL/$TESTFS2" && \
+		destroy_dataset "$TESTPOOL/$TESTFS2" -r
+}
+
+log_onexit cleanup
+
+log_assert "ZFS should receive a mix of un/encrypted childs"
+
+typeset src="$TESTPOOL/$TESTFS1"
+typeset dst="$TESTPOOL/$TESTFS2"
+typeset snap="snap"
+
+echo "password" | \
+	create_dataset "$src" -o encryption=on -o keyformat=passphrase
+create_dataset "$src/u" "-o encryption=off"
+echo "password" | \
+	create_dataset "$src/u/e" -o encryption=on -o keyformat=passphrase
+create_dataset "$src/u/e/u" -o encryption=off
+
+log_must zfs snapshot -r "$src@$snap"
+log_must eval "zfs send -Rw $src@$snap | zfs receive -u $dst"
+log_must test "$(get_prop 'encryption' $dst)" != "off"
+log_must test "$(get_prop 'encryption' $dst/u)" == "off"
+log_must test "$(get_prop 'encryption' $dst/u/e)" != "off"
+log_must test "$(get_prop 'encryption' $dst/u/e/u)" == "off"
+
+log_pass "ZFS can receive a mix of un/encrypted childs"

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh

@@ -59,19 +59,29 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
 
 log_note "Verifying ZFS will receive to an encrypted child"
 log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c1"
+log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c1)" != "off"
 
-log_note "Verifying 'send -p' will receive to an encrypted child"
+# Unload the key, the following tests won't require it and we will test
+# the receive checks as well.
+log_must zfs unmount $TESTPOOL/$TESTFS1
+log_must zfs unload-key $TESTPOOL/$TESTFS1
+
+log_note "Verifying 'send -p' will receive to an unencrypted child"
 log_must eval "zfs send -p $snap | zfs receive $TESTPOOL/$TESTFS1/c2"
 log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c2)" == "off"
 
-log_note "Verifying 'send -R' will receive to an encrypted child"
+# For completeness add the property override case.
+log_note "Verifying recv -o encyption=off' will receive to an unencrypted child"
+log_must eval "zfs send $snap | \
+	zfs receive -o encryption=off $TESTPOOL/$TESTFS1/c2o"
+log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c2o)" == "off"
+
+log_note "Verifying 'send -R' will receive to an unencrypted child"
 log_must eval "zfs send -R $snap | zfs receive $TESTPOOL/$TESTFS1/c3"
 log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c3)" == "off"
 
 log_note "Verifying ZFS will not receive to an encrypted child when the" \
 	"parent key is unloaded"
-log_must zfs unmount $TESTPOOL/$TESTFS1
-log_must zfs unload-key $TESTPOOL/$TESTFS1
 log_mustnot eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c4"
 
 log_pass "ZFS can receive encrypted filesystems into child dataset"