libzfs: add keylocation=https://, backed by fetch(3) or libcurl

Add support for http and https to the keylocation properly to
allow encryption keys to be fetched from the specified URL.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Ryan Moeller <ryan@ixsystems.com>
Signed-off-by: Ahelenia Ziemiańska <nabijaczleweli@nabijaczleweli.xyz>
Issue #9543
Closes #9947 
Closes #11956

.github/workflows/zfs-tests-functional.yml

@@ -26,7 +26,7 @@ jobs:
           xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
           libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
           libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi
+          python3 python3-dev python3-setuptools python3-cffi libcurl4-openssl-dev
     - name: Autogen.sh
       run: |
         sh autogen.sh

.github/workflows/zfs-tests-sanity.yml

@@ -22,7 +22,7 @@ jobs:
           xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
           libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
           libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi
+          python3 python3-dev python3-setuptools python3-cffi libcurl4-openssl-dev
     - name: Autogen.sh
       run: |
         sh autogen.sh

config/Substfiles.am

@@ -15,7 +15,9 @@ subst_sed_cmd = \
 	-e 's|@PYTHON[@]|$(PYTHON)|g' \
 	-e 's|@PYTHON_SHEBANG[@]|$(PYTHON_SHEBANG)|g' \
 	-e 's|@DEFAULT_INIT_NFS_SERVER[@]|$(DEFAULT_INIT_NFS_SERVER)|g' \
-	-e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g'
+	-e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g' \
+	-e 's|@LIBFETCH_DYNAMIC[@]|$(LIBFETCH_DYNAMIC)|g' \
+	-e 's|@LIBFETCH_SONAME[@]|$(LIBFETCH_SONAME)|g'
 
 SUBSTFILES =
 CLEANFILES = $(SUBSTFILES)

config/user-libfetch.m4

@@ -0,0 +1,71 @@
+dnl #
+dnl # Check for a libfetch - either fetch(3) or libcurl.
+dnl #
+dnl # There are two configuration dimensions:
+dnl #   * fetch(3) vs libcurl
+dnl #   * static vs dynamic
+dnl #
+dnl # fetch(3) is only dynamic.
+dnl # We use sover 6, which first appeared in FreeBSD 8.0-RELEASE.
+dnl #
+dnl # libcurl development packages include curl-config(1) – we want:
+dnl #   * HTTPS support
+dnl #   * version at least 7.16 (October 2006), for sover 4
+dnl #   * to decide if it's static or not
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_USER_LIBFETCH], [
+	AC_MSG_CHECKING([for libfetch])
+	LIBFETCH_LIBS=
+	LIBFETCH_IS_FETCH=0
+	LIBFETCH_IS_LIBCURL=0
+	LIBFETCH_DYNAMIC=0
+	LIBFETCH_SONAME=
+	have_libfetch=
+
+	saved_libs="$LIBS"
+	LIBS="$LIBS -lfetch"
+	AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+		#include <sys/param.h>
+		#include <stdio.h>
+		#include <fetch.h>
+	]], [fetchGetURL("", "");])], [
+		have_libfetch=1
+		LIBFETCH_IS_FETCH=1
+		LIBFETCH_DYNAMIC=1
+		LIBFETCH_SONAME='"libfetch.so.6"'
+		LIBFETCH_LIBS="-ldl"
+		AC_MSG_RESULT([fetch(3)])
+	], [])
+	LIBS="$saved_libs"
+
+	if test -z "$have_libfetch"; then
+		if curl-config --protocols 2>/dev/null | grep -q HTTPS &&
+		    test "$(printf "%u" "0x$(curl-config --vernum)")" -ge "$(printf "%u" "0x071000")"; then
+			have_libfetch=1
+			LIBFETCH_IS_LIBCURL=1
+			if test "$(curl-config --built-shared)" = "yes"; then
+				LIBFETCH_DYNAMIC=1
+				LIBFETCH_SONAME='"libcurl.so.4"'
+				LIBFETCH_LIBS="-ldl"
+				AC_MSG_RESULT([libcurl])
+			else
+				LIBFETCH_LIBS="$(curl-config --libs)"
+				AC_MSG_RESULT([libcurl (static)])
+			fi
+
+			CCFLAGS="$CCFLAGS $(curl-config --cflags)"
+		fi
+	fi
+
+	if test -z "$have_libfetch"; then
+		AC_MSG_RESULT([none])
+	fi
+
+	AC_SUBST([LIBFETCH_LIBS])
+	AC_SUBST([LIBFETCH_DYNAMIC])
+	AC_SUBST([LIBFETCH_SONAME])
+	AC_DEFINE_UNQUOTED([LIBFETCH_IS_FETCH], [$LIBFETCH_IS_FETCH], [libfetch is fetch(3)])
+	AC_DEFINE_UNQUOTED([LIBFETCH_IS_LIBCURL], [$LIBFETCH_IS_LIBCURL], [libfetch is libcurl])
+	AC_DEFINE_UNQUOTED([LIBFETCH_DYNAMIC], [$LIBFETCH_DYNAMIC], [whether the chosen libfetch is to be loaded at run-time])
+	AC_DEFINE_UNQUOTED([LIBFETCH_SONAME], [$LIBFETCH_SONAME], [soname of chosen libfetch])
+])

config/user.m4

@@ -22,6 +22,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_LIBCRYPTO
 	ZFS_AC_CONFIG_USER_LIBAIO
 	ZFS_AC_CONFIG_USER_LIBATOMIC
+	ZFS_AC_CONFIG_USER_LIBFETCH
 	ZFS_AC_CONFIG_USER_CLOCK_GETTIME
 	ZFS_AC_CONFIG_USER_PAM
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR

contrib/dracut/90zfs/module-setup.sh.in

@@ -56,6 +56,11 @@ install() {
 		# Fallback: Guess the path and include all matches
 		dracut_install /usr/lib/gcc/*/*/libgcc_s.so*
 	fi
+	if [ @LIBFETCH_DYNAMIC@ != 0 ]; then
+		for d in $libdirs; do
+			[ -e "$d"/@LIBFETCH_SONAME@ ] && dracut_install "$d"/@LIBFETCH_SONAME@
+		done
+	fi
 	dracut_install @mounthelperdir@/mount.zfs
 	dracut_install @udevdir@/vdev_id
 	dracut_install awk

contrib/dracut/90zfs/zfs-env-bootfs.service.in

@@ -8,7 +8,7 @@ Before=zfs-import.target
 
 [Service]
 Type=oneshot
-ExecStart=/bin/sh -c "systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -v '^-$')"
+ExecStart=/bin/sh -c "exec systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -v '^-$')"
 
 [Install]
 WantedBy=zfs-import.target

contrib/dracut/90zfs/zfs-load-key.sh.in

@@ -43,13 +43,14 @@ if [ "$(zpool list -H -o feature@encryption "$(echo "${BOOTFS}" | awk -F/ '{prin
         [ "$KEYSTATUS" = "unavailable" ] || exit 0
         # if key is stored in a file, do not prompt
         if ! [ "${KEYLOCATION}" = "prompt" ]; then
+            if ! [ "${KEYLOCATION#http}" = "${KEYLOCATION}" ]; then
+                systemctl start network-online.target
+            fi
             zfs load-key "${ENCRYPTIONROOT}"
         else
             # decrypt them
-            TRY_COUNT=5
+            for _ in 1 2 3 4 5; do
-            while [ $TRY_COUNT -gt 0 ]; do
                 systemd-ask-password "Encrypted ZFS password for ${BOOTFS}" --no-tty | zfs load-key "${ENCRYPTIONROOT}" && break
-                TRY_COUNT=$((TRY_COUNT - 1))
             done
         fi
     fi

contrib/dracut/90zfs/zfs-rollback-bootfs.service.in

@@ -10,5 +10,5 @@ ConditionKernelCommandLine=bootfs.rollback
 # ${BOOTFS} should have been set by zfs-env-bootfs.service
 Type=oneshot
 ExecStartPre=/bin/sh -c 'test -n "${BOOTFS}"'
-ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; @sbindir@/zfs rollback -Rf "${BOOTFS}@${SNAPNAME:-%v}"'
+ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; exec @sbindir@/zfs rollback -Rf "${BOOTFS}@${SNAPNAME:-%v}"'
 RemainAfterExit=yes

contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in

@@ -10,5 +10,5 @@ ConditionKernelCommandLine=bootfs.snapshot
 # ${BOOTFS} should have been set by zfs-env-bootfs.service
 Type=oneshot
 ExecStartPre=/bin/sh -c 'test -n "${BOOTFS}"'
-ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; @sbindir@/zfs snapshot "${BOOTFS}@${SNAPNAME:-%v}"'
+ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; exec @sbindir@/zfs snapshot "${BOOTFS}@${SNAPNAME:-%v}"'
 RemainAfterExit=yes

contrib/initramfs/hooks/zfs.in

@@ -63,6 +63,14 @@ mkdir -p "$DESTDIR/etc/"
 # multi-arch installations.
 cp --target-directory="$DESTDIR" --parents $(find /lib/ -type f -name libgcc_s.so.1)
 
+if [ @LIBFETCH_DYNAMIC@ != 0 ]
+then
+	for l in $(find /lib/ -name @LIBFETCH_SONAME@)
+	do
+		copy_exec "$l"
+	done
+fi
+
 for ii in $COPY_EXEC_LIST
 do
 	copy_exec "$ii"

contrib/initramfs/scripts/zfs

@@ -406,28 +406,25 @@ decrypt_fs()
 			KEYSTATUS="$(get_fs_value "${ENCRYPTIONROOT}" keystatus)"
 			# Continue only if the key needs to be loaded
 			[ "$KEYSTATUS" = "unavailable" ] || return 0
-			TRY_COUNT=3
 
-			# If key is stored in a file, do not prompt
+			# Do not prompt if key is stored noninteractively,
 			if ! [ "${KEYLOCATION}" = "prompt" ]; then
 				$ZFS load-key "${ENCRYPTIONROOT}"
 
 			# Prompt with plymouth, if active
-			elif [ -e /bin/plymouth ] && /bin/plymouth --ping 2>/dev/null; then
+			elif /bin/plymouth --ping 2>/dev/null; then
 				echo "plymouth" > /run/zfs_console_askpwd_cmd
-				while [ $TRY_COUNT -gt 0 ]; do
+				for _ in 1 2 3; do
 					plymouth ask-for-password --prompt "Encrypted ZFS password for ${ENCRYPTIONROOT}" | \
 						$ZFS load-key "${ENCRYPTIONROOT}" && break
-					TRY_COUNT=$((TRY_COUNT - 1))
 				done
 
 			# Prompt with systemd, if active
 			elif [ -e /run/systemd/system ]; then
 				echo "systemd-ask-password" > /run/zfs_console_askpwd_cmd
-				while [ $TRY_COUNT -gt 0 ]; do
+				for _ in 1 2 3; do
 					systemd-ask-password "Encrypted ZFS password for ${ENCRYPTIONROOT}" --no-tty | \
 						$ZFS load-key "${ENCRYPTIONROOT}" && break
-					TRY_COUNT=$((TRY_COUNT - 1))
 				done
 
 			# Prompt with ZFS tty, otherwise

include/libzfs_impl.h

@@ -69,6 +69,8 @@ struct libzfs_handle {
 	boolean_t libzfs_prop_debug;
 	regex_t libzfs_urire;
 	uint64_t libzfs_max_nvlist;
+	void *libfetch;
+	char *libfetch_load_error;
 };
 
 struct zfs_handle {

lib/libzfs/Makefile.am

@@ -75,7 +75,7 @@ libzfs_la_LIBADD = \
 	$(abs_top_builddir)/lib/libnvpair/libnvpair.la \
 	$(abs_top_builddir)/lib/libuutil/libuutil.la
 
-libzfs_la_LIBADD += -lm $(LIBCRYPTO_LIBS) $(ZLIB_LIBS) $(LTLIBINTL)
+libzfs_la_LIBADD += -lm $(LIBCRYPTO_LIBS) $(ZLIB_LIBS) $(LIBFETCH_LIBS) $(LTLIBINTL)
 
 libzfs_la_LDFLAGS = -pthread
 

lib/libzfs/libzfs_crypto.c

@@ -26,6 +26,16 @@
 #include <signal.h>
 #include <errno.h>
 #include <openssl/evp.h>
+#if LIBFETCH_DYNAMIC
+#include <dlfcn.h>
+#endif
+#if LIBFETCH_IS_FETCH
+#include <sys/param.h>
+#include <stdio.h>
+#include <fetch.h>
+#elif LIBFETCH_IS_LIBCURL
+#include <curl/curl.h>
+#endif
 #include <libzfs.h>
 #include "libzfs_impl.h"
 #include "zfeature_common.h"
@@ -59,9 +69,13 @@ static int caught_interrupt;
 
 static int get_key_material_file(libzfs_handle_t *, const char *, const char *,
     zfs_keyformat_t, boolean_t, uint8_t **, size_t *);
+static int get_key_material_https(libzfs_handle_t *, const char *, const char *,
+    zfs_keyformat_t, boolean_t, uint8_t **, size_t *);
 
 static zfs_uri_handler_t uri_handlers[] = {
 	{ "file", get_key_material_file },
+	{ "https", get_key_material_https },
+	{ "http", get_key_material_https },
 	{ NULL, NULL }
 };
 
@@ -483,6 +497,178 @@ get_key_material_file(libzfs_handle_t *hdl, const char *uri,
 	return (ret);
 }
 
+static int
+get_key_material_https(libzfs_handle_t *hdl, const char *uri,
+    const char *fsname, zfs_keyformat_t keyformat, boolean_t newkey,
+    uint8_t **restrict buf, size_t *restrict len_out)
+{
+	int ret = 0;
+	FILE *key = NULL;
+	boolean_t is_http = strncmp(uri, "http:", strlen("http:")) == 0;
+
+	if (strlen(uri) < (is_http ? 7 : 8)) {
+		ret = EINVAL;
+		goto end;
+	}
+
+#if LIBFETCH_DYNAMIC
+#define	LOAD_FUNCTION(func) \
+	__typeof__(func) *func = dlsym(hdl->libfetch, #func);
+
+	if (hdl->libfetch == NULL)
+		hdl->libfetch = dlopen(LIBFETCH_SONAME, RTLD_LAZY);
+
+	if (hdl->libfetch == NULL) {
+		hdl->libfetch = (void *)-1;
+		char *err = dlerror();
+		if (err)
+			hdl->libfetch_load_error = strdup(err);
+	}
+
+	if (hdl->libfetch == (void *)-1) {
+		ret = ENOSYS;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Couldn't load %s: %s"),
+		    LIBFETCH_SONAME, hdl->libfetch_load_error ?: "(?)");
+		goto end;
+	}
+
+	boolean_t ok;
+#if LIBFETCH_IS_FETCH
+	LOAD_FUNCTION(fetchGetURL);
+	char *fetchLastErrString = dlsym(hdl->libfetch, "fetchLastErrString");
+
+	ok = fetchGetURL && fetchLastErrString;
+#elif LIBFETCH_IS_LIBCURL
+	LOAD_FUNCTION(curl_easy_init);
+	LOAD_FUNCTION(curl_easy_setopt);
+	LOAD_FUNCTION(curl_easy_perform);
+	LOAD_FUNCTION(curl_easy_cleanup);
+	LOAD_FUNCTION(curl_easy_strerror);
+	LOAD_FUNCTION(curl_easy_getinfo);
+
+	ok = curl_easy_init && curl_easy_setopt && curl_easy_perform &&
+	    curl_easy_cleanup && curl_easy_strerror && curl_easy_getinfo;
+#endif
+	if (!ok) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "keylocation=%s back-end %s missing symbols."),
+		    is_http ? "http://" : "https://", LIBFETCH_SONAME);
+		ret = ENOSYS;
+		goto end;
+	}
+#endif
+
+#if LIBFETCH_IS_FETCH
+	key = fetchGetURL(uri, "");
+	if (key == NULL) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Couldn't GET %s: %s"),
+		    uri, fetchLastErrString);
+		ret = ENETDOWN;
+	}
+#elif LIBFETCH_IS_LIBCURL
+	CURL *curl = curl_easy_init();
+	if (curl == NULL) {
+		ret = ENOTSUP;
+		goto end;
+	}
+
+	int kfd = -1;
+#ifdef O_TMPFILE
+	kfd = open(getenv("TMPDIR") ?: "/tmp",
+	    O_RDWR | O_TMPFILE | O_EXCL | O_CLOEXEC, 0600);
+	if (kfd != -1)
+		goto kfdok;
+#endif
+
+	char *path;
+	if (asprintf(&path,
+	    "%s/libzfs-XXXXXXXX.https", getenv("TMPDIR") ?: "/tmp") == -1) {
+		ret = ENOMEM;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "%s"),
+		    strerror(ret));
+		goto end;
+	}
+
+	kfd = mkostemps(path, strlen(".https"), O_CLOEXEC);
+	if (kfd == -1) {
+		ret = errno;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Couldn't create temporary file %s: %s"),
+		    path, strerror(ret));
+		free(path);
+		goto end;
+	}
+	(void) unlink(path);
+	free(path);
+
+kfdok:
+	if ((key = fdopen(kfd, "r+")) == NULL) {
+		ret = errno;
+		free(path);
+		(void) close(kfd);
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Couldn't reopen temporary file: %s"), strerror(ret));
+		goto end;
+	}
+
+	char errbuf[CURL_ERROR_SIZE] = "";
+	char *cainfo = getenv("SSL_CA_CERT_FILE"); /* matches fetch(3) */
+	char *capath = getenv("SSL_CA_CERT_PATH"); /* matches fetch(3) */
+	char *clcert = getenv("SSL_CLIENT_CERT_FILE"); /* matches fetch(3) */
+	char *clkey  = getenv("SSL_CLIENT_KEY_FILE"); /* matches fetch(3) */
+	(void) curl_easy_setopt(curl, CURLOPT_URL, uri);
+	(void) curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+	(void) curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 30000L);
+	(void) curl_easy_setopt(curl, CURLOPT_WRITEDATA, key);
+	(void) curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf);
+	if (cainfo != NULL)
+		(void) curl_easy_setopt(curl, CURLOPT_CAINFO, cainfo);
+	if (capath != NULL)
+		(void) curl_easy_setopt(curl, CURLOPT_CAPATH, capath);
+	if (clcert != NULL)
+		(void) curl_easy_setopt(curl, CURLOPT_SSLCERT, clcert);
+	if (clkey != NULL)
+		(void) curl_easy_setopt(curl, CURLOPT_SSLKEY, clkey);
+
+	CURLcode res = curl_easy_perform(curl);
+
+	if (res != CURLE_OK) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Failed to connect to %s: %s"),
+		    uri, strlen(errbuf) ? errbuf : curl_easy_strerror(res));
+		ret = ENETDOWN;
+	} else {
+		long resp = 200;
+		(void) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp);
+
+		if (resp < 200 || resp >= 300) {
+			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+			    "Couldn't GET %s: %ld"),
+			    uri, resp);
+			ret = ENOENT;
+		} else
+			rewind(key);
+	}
+
+	curl_easy_cleanup(curl);
+#else
+	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+	    "No keylocation=%s back-end."), is_http ? "http://" : "https://");
+	ret = ENOSYS;
+#endif
+
+end:
+	if (ret == 0)
+		ret = get_key_material_raw(key, keyformat, buf, len_out);
+
+	if (key != NULL)
+		fclose(key);
+
+	return (ret);
+}
+
 /*
  * Attempts to fetch key material, no matter where it might live. The key
  * material is allocated and returned in km_out. *can_retry_out will be set

lib/libzfs/libzfs_util.c

@@ -44,6 +44,9 @@
 #include <strings.h>
 #include <unistd.h>
 #include <math.h>
+#if LIBFETCH_DYNAMIC
+#include <dlfcn.h>
+#endif
 #include <sys/stat.h>
 #include <sys/mnttab.h>
 #include <sys/mntent.h>
@@ -1083,6 +1086,11 @@ libzfs_fini(libzfs_handle_t *hdl)
 	libzfs_core_fini();
 	regfree(&hdl->libzfs_urire);
 	fletcher_4_fini();
+#if LIBFETCH_DYNAMIC
+	if (hdl->libfetch != (void *)-1 && hdl->libfetch != NULL)
+		(void) dlclose(hdl->libfetch);
+	free(hdl->libfetch_load_error);
+#endif
 	free(hdl);
 }
 

man/man8/zfsprops.8

@@ -1085,7 +1085,7 @@ encryption suite cannot be changed after dataset creation, the keyformat can be
 with
 .Nm zfs Cm change-key .
 .It Xo
-.Sy keylocation Ns = Ns Sy prompt Ns | Ns Sy file:// Ns Em </absolute/file/path>
+.Sy keylocation Ns = Ns Sy prompt Ns | Ns Sy file:// Ns Em </absolute/file/path> Ns | Ns Sy https:// Ns Em <address> | Ns Sy http:// Ns Em <address>
 .Xc
 Controls where the user's encryption key will be loaded from by default for
 commands such as
@@ -1109,7 +1109,22 @@ to access the encrypted data (see
 for details). This setting will also allow the key to be passed in via STDIN,
 but users should be careful not to place keys which should be kept secret on
 the command line. If a file URI is selected, the key will be loaded from the
-specified absolute file path.
+specified absolute file path. If an HTTPS or HTTP URL is selected,
+it will be GETted using
+.Xr fetch 3 ,
+libcurl, or nothing, depending on compile-time configuration and run-time
+availability. The
+.Ev SSL_CA_CERT_FILE
+environment variable can be set to set the location
+of the concatenated certificate store. The
+.Ev SSL_CA_CERT_PATH
+environment variable can be set to override the location
+of the directory containing the certificate authority bundle. The
+.Ev SSL_CLIENT_CERT_FILE
+and
+.Ev SSL_CLIENT_KEY_FILE
+environment variables can be set to configure the path
+to the client certificate and its key.
 .It Sy pbkdf2iters Ns = Ns Ar iterations
 Controls the number of PBKDF2 iterations that a
 .Sy passphrase

module/zcommon/zfs_prop.c

@@ -583,7 +583,7 @@ zfs_prop_init(void)
 	    "ENCROOT");
 	zprop_register_string(ZFS_PROP_KEYLOCATION, "keylocation",
 	    "none", PROP_DEFAULT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
-	    "prompt | <file URI>", "KEYLOCATION");
+	    "prompt | <file URI> | <https URL> | <http URL>", "KEYLOCATION");
 	zprop_register_string(ZFS_PROP_REDACT_SNAPS,
 	    "redact_snaps", NULL, PROP_READONLY,
 	    ZFS_TYPE_DATASET | ZFS_TYPE_BOOKMARK, "<snapshot>[,...]",
@@ -936,6 +936,10 @@ zfs_prop_valid_keylocation(const char *str, boolean_t encrypted)
 		return (B_TRUE);
 	else if (strlen(str) > 8 && strncmp("file:///", str, 8) == 0)
 		return (B_TRUE);
+	else if (strlen(str) > 8 && strncmp("https://", str, 8) == 0)
+		return (B_TRUE);
+	else if (strlen(str) > 7 && strncmp("http://", str, 7) == 0)
+		return (B_TRUE);
 
 	return (B_FALSE);
 }

tests/runfiles/common.run

@@ -198,7 +198,8 @@ tags = ['functional', 'cli_root', 'zfs_inherit']
 
 [tests/functional/cli_root/zfs_load-key]
 tests = ['zfs_load-key', 'zfs_load-key_all', 'zfs_load-key_file',
-    'zfs_load-key_location', 'zfs_load-key_noop', 'zfs_load-key_recursive']
+    'zfs_load-key_https', 'zfs_load-key_location', 'zfs_load-key_noop',
+    'zfs_load-key_recursive']
 tags = ['functional', 'cli_root', 'zfs_load-key']
 
 [tests/functional/cli_root/zfs_mount]

tests/runfiles/sanity.run

@@ -146,7 +146,8 @@ tags = ['functional', 'cli_root', 'zfs_inherit']
 
 [tests/functional/cli_root/zfs_load-key]
 tests = ['zfs_load-key', 'zfs_load-key_all', 'zfs_load-key_file',
-    'zfs_load-key_location', 'zfs_load-key_noop', 'zfs_load-key_recursive']
+    'zfs_load-key_https', 'zfs_load-key_location', 'zfs_load-key_noop',
+    'zfs_load-key_recursive']
 tags = ['functional', 'cli_root', 'zfs_load-key']
 
 [tests/functional/cli_root/zfs_mount]

tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am

@@ -5,6 +5,7 @@ dist_pkgdata_SCRIPTS = \
 	zfs_load-key.ksh \
 	zfs_load-key_all.ksh \
 	zfs_load-key_file.ksh \
+	zfs_load-key_https.ksh \
 	zfs_load-key_location.ksh \
 	zfs_load-key_noop.ksh \
 	zfs_load-key_recursive.ksh

tests/zfs-tests/tests/functional/cli_root/zfs_load-key/cleanup.ksh

@@ -26,5 +26,7 @@
 #
 
 . $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
 
+cleanup_https
 default_cleanup

tests/zfs-tests/tests/functional/cli_root/zfs_load-key/setup.ksh

@@ -26,7 +26,10 @@
 #
 
 . $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
 
 DISK=${DISKS%% *}
 
-default_setup $DISK
+default_setup_noexit $DISK
+setup_https
+log_pass

tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg

@@ -27,3 +27,31 @@ export HEXKEY="000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F"
 export HEXKEY1="201F1E1D1C1B1A191817161514131211100F0E0D0C0B0A090807060504030201"
 export RAWKEY="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 export RAWKEY1="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+
+export SSL_CA_CERT_FILE="/$TESTPOOL/snakeoil.crt"
+export HTTPS_PORT_FILE="/$TESTPOOL/snakeoil.port"
+export HTTPS_HOSTNAME="localhost"
+export HTTPS_PORT=
+export HTTPS_BASE_URL=
+
+function get_https_port
+{
+	if [ -z "$HTTPS_PORT" ]; then
+		read -r HTTPS_PORT < "$HTTPS_PORT_FILE" || return
+	fi
+
+	echo "$HTTPS_PORT"
+}
+
+function get_https_base_url
+{
+	if [ -z "$HTTPS_BASE_URL" ]; then
+		HTTPS_BASE_URL="https://$HTTPS_HOSTNAME:$(get_https_port)" || {
+			typeset ret=$?
+			HTTPS_BASE_URL=
+			return $ret
+		}
+	fi
+
+	echo "$HTTPS_BASE_URL"
+}

tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_all.ksh

@@ -39,6 +39,8 @@ function cleanup
 {
 	datasetexists $TESTPOOL/$TESTFS1 && \
 		log_must zfs destroy $TESTPOOL/$TESTFS1
+	datasetexists $TESTPOOL/$TESTFS2 && \
+		log_must zfs destroy $TESTPOOL/$TESTFS2
 	datasetexists $TESTPOOL/zvol && log_must zfs destroy $TESTPOOL/zvol
 	poolexists $TESTPOOL1 && log_must destroy_pool $TESTPOOL1
 }
@@ -50,6 +52,9 @@ log_must eval "echo $PASSPHRASE1 > /$TESTPOOL/pkey"
 log_must zfs create -o encryption=on -o keyformat=passphrase \
 	-o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1
 
+log_must zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS2
+
 log_must zfs create -V 64M -o encryption=on -o keyformat=passphrase \
 	-o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/zvol
 
@@ -60,6 +65,9 @@ log_must zpool create -O encryption=on -O keyformat=passphrase \
 log_must zfs unmount $TESTPOOL/$TESTFS1
 log_must zfs unload-key $TESTPOOL/$TESTFS1
 
+log_must zfs unmount $TESTPOOL/$TESTFS2
+log_must zfs unload-key $TESTPOOL/$TESTFS2
+
 log_must zfs unload-key $TESTPOOL/zvol
 
 log_must zfs unmount $TESTPOOL1
@@ -70,8 +78,10 @@ log_must zfs load-key -a
 log_must key_available $TESTPOOL1
 log_must key_available $TESTPOOL/zvol
 log_must key_available $TESTPOOL/$TESTFS1
+log_must key_available $TESTPOOL/$TESTFS2
 
 log_must zfs mount $TESTPOOL1
 log_must zfs mount $TESTPOOL/$TESTFS1
+log_must zfs mount $TESTPOOL/$TESTFS2
 
 log_pass "'zfs load-key -a' loads keys for all datasets"

tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib

@@ -99,3 +99,66 @@ function verify_origin
 
 	return 0
 }
+
+function setup_https
+{
+	log_must openssl req -x509 -newkey rsa:4096 -sha256 -days 1 -nodes -keyout "/$TESTPOOL/snakeoil.key" -out "$SSL_CA_CERT_FILE" -subj "/CN=$HTTPS_HOSTNAME"
+
+	python3 -uc "
+import http.server, ssl, sys, os, time, random
+
+sys.stdin.close()
+
+httpd, err, port = None, None, None
+for i in range(1, 100):
+	port = random.randint(0xC000, 0xFFFF) # ephemeral range
+	try:
+		httpd = http.server.HTTPServer(('$HTTPS_HOSTNAME', port), http.server.SimpleHTTPRequestHandler)
+		break
+	except:
+		err = sys.exc_info()[1]
+		time.sleep(i / 100)
+if not httpd:
+	raise err
+
+with open('$HTTPS_PORT_FILE', 'w') as portf:
+	print(port, file=portf)
+
+httpd.socket = ssl.wrap_socket(httpd.socket, server_side=True, keyfile='/$TESTPOOL/snakeoil.key', certfile='$SSL_CA_CERT_FILE', ssl_version=ssl.PROTOCOL_TLS)
+
+os.chdir('$STF_SUITE/tests/functional/cli_root/zfs_load-key')
+
+with open('/$TESTPOOL/snakeoil.pid', 'w') as pidf:
+	if os.fork() != 0:
+	  os._exit(0)
+	print(os.getpid(), file=pidf)
+
+sys.stdout.close()
+sys.stderr.close()
+try:
+	sys.stdout = sys.stderr = open('/tmp/ZTS-snakeoil.log', 'w', buffering=1) # line
+except:
+	sys.stdout = sys.stderr = open('/dev/null', 'w')
+
+print('{} start on {}'.format(os.getpid(), port))
+httpd.serve_forever()
+" || log_fail
+
+	typeset https_pid=
+	for d in $(seq 0 0.1 5); do
+		read -r https_pid 2>/dev/null < "/$TESTPOOL/snakeoil.pid" && [ -n "$https_pid" ] && break
+		sleep "$d"
+	done
+	[ -z "$https_pid" ] && log_fail "Couldn't start HTTPS server"
+	log_note "Started HTTPS server as $https_pid on port $(get_https_port)"
+}
+
+function cleanup_https
+{
+	typeset https_pid=
+	read -r https_pid 2>/dev/null < "/$TESTPOOL/snakeoil.pid" || return 0
+
+	log_must kill "$https_pid"
+	cat /tmp/ZTS-snakeoil.log
+	rm -f "/$TESTPOOL/snakeoil.pid" "/tmp/ZTS-snakeoil.log"
+}

tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_https.ksh

@@ -0,0 +1,78 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
+
+#
+# DESCRIPTION:
+# 'zfs load-key' should load a dataset's key from an https:// URL,
+# but fail to do so if the domain doesn't exist or the file 404s.
+#
+# STRATEGY:
+# 1. Try to create a dataset pointing to an RFC6761-guaranteed unresolvable domain,
+#    one to the sshd port (which will be either unoccupied (ECONNREFUSED)
+#                          or have sshd on it ("wrong version number")).
+#    and one pointing to an URL that will always 404.
+# 2. Create encrypted datasets with keylocation=https://address
+# 3. Unmount the datasets and unload their keys
+# 4. Attempt to load the keys
+# 5. Verify the keys are loaded
+# 6. Attempt to mount the datasets
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	for fs in "$TESTFS1" "$TESTFS2" "$TESTFS3"; do
+		datasetexists $TESTPOOL/$fs && \
+			log_must zfs destroy $TESTPOOL/$fs
+	done
+}
+log_onexit cleanup
+
+log_assert "'zfs load-key' should load a key from a file"
+
+log_mustnot zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=https://invalid./where-ever $TESTPOOL/$TESTFS1
+
+log_mustnot zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=https://$HTTPS_HOSTNAME:22 $TESTPOOL/$TESTFS1
+
+log_mustnot zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=$(get_https_base_url)/ENOENT $TESTPOOL/$TESTFS1
+
+log_must zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1
+
+log_must zfs create -o encryption=on -o keyformat=hex \
+	-o keylocation=$(get_https_base_url)/HEXKEY $TESTPOOL/$TESTFS2
+
+log_must zfs create -o encryption=on -o keyformat=raw \
+	-o keylocation=$(get_https_base_url)/RAWKEY $TESTPOOL/$TESTFS3
+
+for fs in "$TESTFS1" "$TESTFS2" "$TESTFS3"; do
+	log_must zfs unmount $TESTPOOL/$fs
+	log_must zfs unload-key $TESTPOOL/$fs
+done
+for fs in "$TESTFS1" "$TESTFS2" "$TESTFS3"; do
+	log_must zfs load-key $TESTPOOL/$fs
+	log_must key_available $TESTPOOL/$fs
+	log_must zfs mount $TESTPOOL/$fs
+done
+
+log_pass "'zfs load-key' loads a key from a file"

tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_location.ksh

@@ -70,4 +70,9 @@ log_must eval "echo $PASSPHRASE | zfs load-key -L prompt $TESTPOOL/$TESTFS1"
 log_must key_available $TESTPOOL/$TESTFS1
 log_must verify_keylocation $TESTPOOL/$TESTFS1 "file://$key_location"
 
+log_must zfs unload-key $TESTPOOL/$TESTFS1
+log_must zfs load-key -L $(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1
+log_must key_available $TESTPOOL/$TESTFS1
+log_must verify_keylocation $TESTPOOL/$TESTFS1 "file://$key_location"
+
 log_pass "'zfs load-key -L' overrides keylocation with provided value"

tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh

@@ -52,15 +52,21 @@ log_must zfs create -o encryption=on -o keyformat=passphrase \
 log_must zfs create -o keyformat=passphrase \
 	-o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1/child
 
+log_must zfs create -o keyformat=passphrase \
+	-o keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1/child/child
+
 log_must zfs unmount $TESTPOOL/$TESTFS1
+log_must zfs unload-key $TESTPOOL/$TESTFS1/child/child
 log_must zfs unload-key $TESTPOOL/$TESTFS1/child
 log_must zfs unload-key $TESTPOOL/$TESTFS1
 
 log_must zfs load-key -r $TESTPOOL
 log_must key_available $TESTPOOL/$TESTFS1
 log_must key_available $TESTPOOL/$TESTFS1/child
+log_must key_available $TESTPOOL/$TESTFS1/child/child
 
 log_must zfs mount $TESTPOOL/$TESTFS1
 log_must zfs mount $TESTPOOL/$TESTFS1/child
+log_must zfs mount $TESTPOOL/$TESTFS1/child/child
 
 log_pass "'zfs load-key -r' recursively loads keys"

tests/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_keylocation.ksh

@@ -46,11 +46,12 @@ function cleanup
 {
 	datasetexists $TESTPOOL/$TESTFS1 && \
 		log_must zfs destroy -r $TESTPOOL/$TESTFS1
+	cleanup_https
 }
 log_onexit cleanup
 
-log_assert "Key location can only be 'prompt' or a file path for encryption" \
+log_assert "Key location can only be 'prompt', 'file://', or 'https://'" \
-	"roots, and 'none' for unencrypted volumes"
+	"for encryption roots, and 'none' for unencrypted volumes"
 
 log_must eval "echo $PASSPHRASE > /$TESTPOOL/pkey"
 
@@ -64,19 +65,15 @@ log_must zfs create -o encryption=on -o keyformat=passphrase \
 	-o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1
 
 log_mustnot zfs set keylocation=none $TESTPOOL/$TESTFS1
-if true; then
 log_mustnot zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1
-else
-	### SOON: ###
-	# file:///$TESTPOOL/pkey and /$TESTPOOL/pkey are equivalent on FreeBSD
-	# thanks to libfetch. Eventually we want to make the other platforms
-	# work this way as well, either by porting libfetch or by other means.
-	log_must zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1
-fi
 
 log_must zfs set keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1
 log_must verify_keylocation $TESTPOOL/$TESTFS1 "file:///$TESTPOOL/pkey"
 
+setup_https
+log_must zfs set keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1
+log_must verify_keylocation $TESTPOOL/$TESTFS1 "$(get_https_base_url)/PASSPHRASE"
+
 log_must zfs set keylocation=prompt $TESTPOOL/$TESTFS1
 log_must verify_keylocation $TESTPOOL/$TESTFS1 "prompt"
 
@@ -97,5 +94,5 @@ log_mustnot zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1/child
 
 log_must verify_keylocation $TESTPOOL/$TESTFS1/child "none"
 
-log_pass "Key location can only be 'prompt' or a file path for encryption" \
+log_pass "Key location can only be 'prompt', 'file://', or 'https://'" \
-	"roots, and 'none' for unencrypted volumes"
+	"for encryption roots, and 'none' for unencrypted volumes"