Tag zfs-2.1.4

META file and changelog updated.

Signed-off-by: Tony Hutter <hutter2@llnl.gov>

.github/workflows/zfs-tests-functional.yml

@@ -26,7 +26,8 @@ jobs:
           xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
           libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
           libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi python3-packaging
+          python3 python3-dev python3-setuptools python3-cffi python3-packaging \
+          libcurl4-openssl-dev
     - name: Autogen.sh
       run: |
         sh autogen.sh
@@ -64,6 +65,7 @@ jobs:
     - name: Tests
       run: |
         /usr/share/zfs/zfs-tests.sh -vR -s 3G
+      timeout-minutes: 330
     - name: Prepare artifacts
       if: failure()
       run: |

.github/workflows/zfs-tests-sanity.yml

@@ -22,7 +22,8 @@ jobs:
           xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
           libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
           libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi python3-packaging
+          python3 python3-dev python3-setuptools python3-cffi python3-packaging \
+          libcurl4-openssl-dev
     - name: Autogen.sh
       run: |
         sh autogen.sh
@@ -60,6 +61,7 @@ jobs:
     - name: Tests
       run: |
         /usr/share/zfs/zfs-tests.sh -vR -s 3G -r sanity
+      timeout-minutes: 330
     - name: Prepare artifacts
       if: failure()
       run: |

META

@@ -1,10 +1,10 @@
 Meta:          1
 Name:          zfs
 Branch:        1.0
-Version:       2.1.2
+Version:       2.1.4
 Release:       1
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS
-Linux-Maximum: 5.15
+Linux-Maximum: 5.17
 Linux-Minimum: 3.10

README.md

@@ -12,7 +12,7 @@ This repository contains the code for running OpenZFS on Linux and FreeBSD.
   * [Documentation](https://openzfs.github.io/openzfs-docs/) - for using and developing this repo
   * [ZoL Site](https://zfsonlinux.org) - Linux release info & links
   * [Mailing lists](https://openzfs.github.io/openzfs-docs/Project%20and%20Community/Mailing%20Lists.html)
-  * [OpenZFS site](http://open-zfs.org/) - for conference videos and info on other platforms (illumos, OSX, Windows, etc)
+  * [OpenZFS site](https://openzfs.org/) - for conference videos and info on other platforms (illumos, OSX, Windows, etc)
 
 # Installation
 

cmd/mount_zfs/mount_zfs.c

@@ -246,13 +246,6 @@ main(int argc, char **argv)
 		}
 	}
 
-	if (verbose)
-		(void) fprintf(stdout, gettext("mount.zfs:\n"
-		    "  dataset:    \"%s\"\n  mountpoint: \"%s\"\n"
-		    "  mountflags: 0x%lx\n  zfsflags:   0x%lx\n"
-		    "  mountopts:  \"%s\"\n  mtabopts:   \"%s\"\n"),
-		    dataset, mntpoint, mntflags, zfsflags, mntopts, mtabopt);
-
 	if (mntflags & MS_REMOUNT) {
 		nomtab = 1;
 		remount = 1;
@@ -275,7 +268,10 @@ main(int argc, char **argv)
 		return (MOUNT_USAGE);
 	}
 
-	zfs_adjust_mount_options(zhp, mntpoint, mntopts, mtabopt);
+	if (!zfsutil || sloppy ||
+	    libzfs_envvar_is_set("ZFS_MOUNT_HELPER")) {
+		zfs_adjust_mount_options(zhp, mntpoint, mntopts, mtabopt);
+	}
 
 	/* treat all snapshots as legacy mount points */
 	if (zfs_get_type(zhp) == ZFS_TYPE_SNAPSHOT)
@@ -293,12 +289,11 @@ main(int argc, char **argv)
 	if (zfs_version == 0) {
 		fprintf(stderr, gettext("unable to fetch "
 		    "ZFS version for filesystem '%s'\n"), dataset);
+		zfs_close(zhp);
+		libzfs_fini(g_zfs);
 		return (MOUNT_SYSERR);
 	}
 
-	zfs_close(zhp);
-	libzfs_fini(g_zfs);
-
 	/*
 	 * Legacy mount points may only be mounted using 'mount', never using
 	 * 'zfs mount'.  However, since 'zfs mount' actually invokes 'mount'
@@ -316,6 +311,8 @@ main(int argc, char **argv)
 		    "Use 'zfs set mountpoint=%s' or 'mount -t zfs %s %s'.\n"
 		    "See zfs(8) for more information.\n"),
 		    dataset, mntpoint, dataset, mntpoint);
+		zfs_close(zhp);
+		libzfs_fini(g_zfs);
 		return (MOUNT_USAGE);
 	}
 
@@ -326,14 +323,38 @@ main(int argc, char **argv)
 		    "Use 'zfs set mountpoint=%s' or 'zfs mount %s'.\n"
 		    "See zfs(8) for more information.\n"),
 		    dataset, "legacy", dataset);
+		zfs_close(zhp);
+		libzfs_fini(g_zfs);
 		return (MOUNT_USAGE);
 	}
 
+	if (verbose)
+		(void) fprintf(stdout, gettext("mount.zfs:\n"
+		    "  dataset:    \"%s\"\n  mountpoint: \"%s\"\n"
+		    "  mountflags: 0x%lx\n  zfsflags:   0x%lx\n"
+		    "  mountopts:  \"%s\"\n  mtabopts:   \"%s\"\n"),
+		    dataset, mntpoint, mntflags, zfsflags, mntopts, mtabopt);
+
 	if (!fake) {
-		error = mount(dataset, mntpoint, MNTTYPE_ZFS,
+		if (zfsutil && !sloppy &&
-		    mntflags, mntopts);
+		    !libzfs_envvar_is_set("ZFS_MOUNT_HELPER")) {
+			error = zfs_mount_at(zhp, mntopts, mntflags, mntpoint);
+			if (error) {
+				(void) fprintf(stderr, "zfs_mount_at() failed: "
+				    "%s", libzfs_error_description(g_zfs));
+				zfs_close(zhp);
+				libzfs_fini(g_zfs);
+				return (MOUNT_SYSERR);
+			}
+		} else {
+			error = mount(dataset, mntpoint, MNTTYPE_ZFS,
+			    mntflags, mntopts);
+		}
 	}
 
+	zfs_close(zhp);
+	libzfs_fini(g_zfs);
+
 	if (error) {
 		switch (errno) {
 		case ENOENT:

cmd/vdev_id/vdev_id

@@ -596,7 +596,9 @@ enclosure_handler () {
 	# DEVPATH=/sys/devices/pci0000:00/0000:00:03.0/0000:05:00.0/host0/subsystem/devices/0:0:0:0/scsi_generic/sg0
 
 	# Get the enclosure ID ("0:0:0:0")
-	ENC=$(basename $(readlink -m "/sys/$DEVPATH/../.."))
+	ENC="${DEVPATH%/*}"
+	ENC="${ENC%/*}"
+	ENC="${ENC##*/}"
 	if [ ! -d "/sys/class/enclosure/$ENC" ] ; then
 		# Not an enclosure, bail out
 		return
@@ -616,10 +618,11 @@ enclosure_handler () {
 
 	# The PCI directory is two directories up from the port directory
 	# /sys/devices/pci0000:00/0000:00:03.0/0000:05:00.0
-	PCI_ID_LONG=$(basename $(readlink -m "/sys/$PORT_DIR/../.."))
+	PCI_ID_LONG="$(readlink -m "/sys/$PORT_DIR/../..")"
+	PCI_ID_LONG="${PCI_ID_LONG##*/}"
 
 	# Strip down the PCI address from 0000:05:00.0 to 05:00.0
-	PCI_ID=$(echo "$PCI_ID_LONG" | sed -r 's/^[0-9]+://g')
+	PCI_ID="${PCI_ID_LONG#[0-9]*:}"
 
 	# Name our device according to vdev_id.conf (like "L0" or "U1").
 	NAME=$(awk "/channel/{if (\$1 == \"channel\" && \$2 == \"$PCI_ID\" && \
@@ -674,7 +677,7 @@ alias_handler () {
 			link=$(echo "$link" | sed 's/p[0-9][0-9]*$//')
 		fi
 		# Check both the fully qualified and the base name of link.
-		for l in $link $(basename "$link") ; do
+		for l in $link ${link##*/} ; do
 			if [ ! -z "$l" ]; then
 				alias=$(awk -v var="$l" '($1 == "alias") && \
 					($3 == var) \

cmd/zed/agents/zfs_mod.c

@@ -183,14 +183,14 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	nvlist_t *nvroot, *newvd;
 	pendingdev_t *device;
 	uint64_t wholedisk = 0ULL;
-	uint64_t offline = 0ULL;
+	uint64_t offline = 0ULL, faulted = 0ULL;
 	uint64_t guid = 0ULL;
 	char *physpath = NULL, *new_devid = NULL, *enc_sysfs_path = NULL;
 	char rawpath[PATH_MAX], fullpath[PATH_MAX];
 	char devpath[PATH_MAX];
 	int ret;
-	boolean_t is_dm = B_FALSE;
 	boolean_t is_sd = B_FALSE;
+	boolean_t is_mpath_wholedisk = B_FALSE;
 	uint_t c;
 	vdev_stat_t *vs;
 
@@ -211,15 +211,73 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	    &enc_sysfs_path);
 	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_WHOLE_DISK, &wholedisk);
 	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_OFFLINE, &offline);
+	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_FAULTED, &faulted);
+
 	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_GUID, &guid);
 
-	if (offline)
+	/*
-		return;  /* don't intervene if it was taken offline */
+	 * Special case:
+	 *
+	 * We've seen times where a disk won't have a ZPOOL_CONFIG_PHYS_PATH
+	 * entry in their config. For example, on this force-faulted disk:
+	 *
+	 *	children[0]:
+	 *	   type: 'disk'
+	 *	   id: 0
+	 *	   guid: 14309659774640089719
+	 *        path: '/dev/disk/by-vdev/L28'
+	 *        whole_disk: 0
+	 *        DTL: 654
+	 *        create_txg: 4
+	 *        com.delphix:vdev_zap_leaf: 1161
+	 *        faulted: 1
+	 *        aux_state: 'external'
+	 *	children[1]:
+	 *        type: 'disk'
+	 *        id: 1
+	 *        guid: 16002508084177980912
+	 *        path: '/dev/disk/by-vdev/L29'
+	 *        devid: 'dm-uuid-mpath-35000c500a61d68a3'
+	 *        phys_path: 'L29'
+	 *        vdev_enc_sysfs_path: '/sys/class/enclosure/0:0:1:0/SLOT 30 32'
+	 *        whole_disk: 0
+	 *        DTL: 1028
+	 *        create_txg: 4
+	 *        com.delphix:vdev_zap_leaf: 131
+	 *
+	 * If the disk's path is a /dev/disk/by-vdev/ path, then we can infer
+	 * the ZPOOL_CONFIG_PHYS_PATH from the by-vdev disk name.
+	 */
+	if (physpath == NULL && path != NULL) {
+		/* If path begins with "/dev/disk/by-vdev/" ... */
+		if (strncmp(path, DEV_BYVDEV_PATH,
+		    strlen(DEV_BYVDEV_PATH)) == 0) {
+			/* Set physpath to the char after "/dev/disk/by-vdev" */
+			physpath = &path[strlen(DEV_BYVDEV_PATH)];
+		}
+	}
 
-	is_dm = zfs_dev_is_dm(path);
+	/*
+	 * We don't want to autoreplace offlined disks.  However, we do want to
+	 * replace force-faulted disks (`zpool offline -f`).  Force-faulted
+	 * disks have both offline=1 and faulted=1 in the nvlist.
+	 */
+	if (offline && !faulted) {
+		zed_log_msg(LOG_INFO, "%s: %s is offline, skip autoreplace",
+		    __func__, path);
+		return;
+	}
+
+	is_mpath_wholedisk = is_mpath_whole_disk(path);
 	zed_log_msg(LOG_INFO, "zfs_process_add: pool '%s' vdev '%s', phys '%s'"
-	    " wholedisk %d, %s dm (guid %llu)", zpool_get_name(zhp), path,
+	    " %s blank disk, %s mpath blank disk, %s labeled, enc sysfs '%s', "
-	    physpath ? physpath : "NULL", wholedisk, is_dm ? "is" : "not",
+	    "(guid %llu)",
+	    zpool_get_name(zhp), path,
+	    physpath ? physpath : "NULL",
+	    wholedisk ? "is" : "not",
+	    is_mpath_wholedisk? "is" : "not",
+	    labeled ? "is" : "not",
+	    enc_sysfs_path,
 	    (long long unsigned int)guid);
 
 	/*
@@ -253,8 +311,9 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	    ZFS_ONLINE_CHECKREMOVE | ZFS_ONLINE_UNSPARE, &newstate) == 0 &&
 	    (newstate == VDEV_STATE_HEALTHY ||
 	    newstate == VDEV_STATE_DEGRADED)) {
-		zed_log_msg(LOG_INFO, "  zpool_vdev_online: vdev %s is %s",
+		zed_log_msg(LOG_INFO,
-		    fullpath, (newstate == VDEV_STATE_HEALTHY) ?
+		    "  zpool_vdev_online: vdev '%s' ('%s') is "
+		    "%s", fullpath, physpath, (newstate == VDEV_STATE_HEALTHY) ?
 		    "HEALTHY" : "DEGRADED");
 		return;
 	}
@@ -271,11 +330,12 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	 * vdev online to trigger a FMA fault by posting an ereport.
 	 */
 	if (!zpool_get_prop_int(zhp, ZPOOL_PROP_AUTOREPLACE, NULL) ||
-	    !(wholedisk || is_dm) || (physpath == NULL)) {
+	    !(wholedisk || is_mpath_wholedisk) || (physpath == NULL)) {
 		(void) zpool_vdev_online(zhp, fullpath, ZFS_ONLINE_FORCEFAULT,
 		    &newstate);
 		zed_log_msg(LOG_INFO, "Pool's autoreplace is not enabled or "
-		    "not a whole disk for '%s'", fullpath);
+		    "not a blank disk for '%s' ('%s')", fullpath,
+		    physpath);
 		return;
 	}
 
@@ -287,7 +347,7 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	(void) snprintf(rawpath, sizeof (rawpath), "%s%s",
 	    is_sd ? DEV_BYVDEV_PATH : DEV_BYPATH_PATH, physpath);
 
-	if (realpath(rawpath, devpath) == NULL && !is_dm) {
+	if (realpath(rawpath, devpath) == NULL && !is_mpath_wholedisk) {
 		zed_log_msg(LOG_INFO, "  realpath: %s failed (%s)",
 		    rawpath, strerror(errno));
 
@@ -303,12 +363,14 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	if ((vs->vs_state != VDEV_STATE_DEGRADED) &&
 	    (vs->vs_state != VDEV_STATE_FAULTED) &&
 	    (vs->vs_state != VDEV_STATE_CANT_OPEN)) {
+		zed_log_msg(LOG_INFO, "  not autoreplacing since disk isn't in "
+		    "a bad state (currently %d)", vs->vs_state);
 		return;
 	}
 
 	nvlist_lookup_string(vdev, "new_devid", &new_devid);
 
-	if (is_dm) {
+	if (is_mpath_wholedisk) {
 		/* Don't label device mapper or multipath disks. */
 	} else if (!labeled) {
 		/*
@@ -522,8 +584,11 @@ zfs_iter_vdev(zpool_handle_t *zhp, nvlist_t *nvl, void *data)
 		 * the dp->dd_compare value.
 		 */
 		if (nvlist_lookup_string(nvl, dp->dd_prop, &path) != 0 ||
-		    strcmp(dp->dd_compare, path) != 0)
+		    strcmp(dp->dd_compare, path) != 0) {
+			zed_log_msg(LOG_INFO, "  %s: no match (%s != vdev %s)",
+			    __func__, dp->dd_compare, path);
 			return;
+		}
 
 		zed_log_msg(LOG_INFO, "  zfs_iter_vdev: matched %s on %s",
 		    dp->dd_prop, path);
@@ -571,6 +636,8 @@ zfs_iter_pool(zpool_handle_t *zhp, void *data)
 			    ZPOOL_CONFIG_VDEV_TREE, &nvl);
 			zfs_iter_vdev(zhp, nvl, data);
 		}
+	} else {
+		zed_log_msg(LOG_INFO, "%s: no config\n", __func__);
 	}
 
 	/*
@@ -619,6 +686,72 @@ devphys_iter(const char *physical, const char *devid, zfs_process_func_t func,
 	return (data.dd_found);
 }
 
+/*
+ * Given a device identifier, find any vdevs with a matching by-vdev
+ * path.  Normally we shouldn't need this as the comparison would be
+ * made earlier in the devphys_iter().  For example, if we were replacing
+ * /dev/disk/by-vdev/L28, normally devphys_iter() would match the
+ * ZPOOL_CONFIG_PHYS_PATH of "L28" from the old disk config to "L28"
+ * of the new disk config.  However, we've seen cases where
+ * ZPOOL_CONFIG_PHYS_PATH was not in the config for the old disk.  Here's
+ * an example of a real 2-disk mirror pool where one disk was force
+ * faulted:
+ *
+ *       com.delphix:vdev_zap_top: 129
+ *           children[0]:
+ *               type: 'disk'
+ *               id: 0
+ *               guid: 14309659774640089719
+ *               path: '/dev/disk/by-vdev/L28'
+ *               whole_disk: 0
+ *               DTL: 654
+ *               create_txg: 4
+ *               com.delphix:vdev_zap_leaf: 1161
+ *               faulted: 1
+ *               aux_state: 'external'
+ *           children[1]:
+ *               type: 'disk'
+ *               id: 1
+ *               guid: 16002508084177980912
+ *               path: '/dev/disk/by-vdev/L29'
+ *               devid: 'dm-uuid-mpath-35000c500a61d68a3'
+ *               phys_path: 'L29'
+ *               vdev_enc_sysfs_path: '/sys/class/enclosure/0:0:1:0/SLOT 30 32'
+ *               whole_disk: 0
+ *               DTL: 1028
+ *               create_txg: 4
+ *               com.delphix:vdev_zap_leaf: 131
+ *
+ * So in the case above, the only thing we could compare is the path.
+ *
+ * We can do this because we assume by-vdev paths are authoritative as physical
+ * paths.  We could not assume this for normal paths like /dev/sda since the
+ * physical location /dev/sda points to could change over time.
+ */
+static boolean_t
+by_vdev_path_iter(const char *by_vdev_path, const char *devid,
+    zfs_process_func_t func, boolean_t is_slice)
+{
+	dev_data_t data = { 0 };
+
+	data.dd_compare = by_vdev_path;
+	data.dd_func = func;
+	data.dd_prop = ZPOOL_CONFIG_PATH;
+	data.dd_found = B_FALSE;
+	data.dd_islabeled = is_slice;
+	data.dd_new_devid = devid;
+
+	if (strncmp(by_vdev_path, DEV_BYVDEV_PATH,
+	    strlen(DEV_BYVDEV_PATH)) != 0) {
+		/* by_vdev_path doesn't start with "/dev/disk/by-vdev/" */
+		return (B_FALSE);
+	}
+
+	(void) zpool_iter(g_zfshdl, zfs_iter_pool, &data);
+
+	return (data.dd_found);
+}
+
 /*
  * Given a device identifier, find any vdevs with a matching devid.
  * On Linux we can match devid directly which is always a whole disk.
@@ -683,15 +816,17 @@ guid_iter(uint64_t pool_guid, uint64_t vdev_guid, const char *devid,
 static int
 zfs_deliver_add(nvlist_t *nvl, boolean_t is_lofi)
 {
-	char *devpath = NULL, *devid;
+	char *devpath = NULL, *devid = NULL;
 	uint64_t pool_guid = 0, vdev_guid = 0;
 	boolean_t is_slice;
 
 	/*
 	 * Expecting a devid string and an optional physical location and guid
 	 */
-	if (nvlist_lookup_string(nvl, DEV_IDENTIFIER, &devid) != 0)
+	if (nvlist_lookup_string(nvl, DEV_IDENTIFIER, &devid) != 0) {
+		zed_log_msg(LOG_INFO, "%s: no dev identifier\n", __func__);
 		return (-1);
+	}
 
 	(void) nvlist_lookup_string(nvl, DEV_PHYS_PATH, &devpath);
 	(void) nvlist_lookup_uint64(nvl, ZFS_EV_POOL_GUID, &pool_guid);
@@ -707,6 +842,8 @@ zfs_deliver_add(nvlist_t *nvl, boolean_t is_lofi)
 	 * 1. ZPOOL_CONFIG_DEVID (identifies the unique disk)
 	 * 2. ZPOOL_CONFIG_PHYS_PATH (identifies disk physical location).
 	 * 3. ZPOOL_CONFIG_GUID (identifies unique vdev).
+	 * 4. ZPOOL_CONFIG_PATH for /dev/disk/by-vdev devices only (since
+	 *    by-vdev paths represent physical paths).
 	 */
 	if (devid_iter(devid, zfs_process_add, is_slice))
 		return (0);
@@ -717,6 +854,16 @@ zfs_deliver_add(nvlist_t *nvl, boolean_t is_lofi)
 		(void) guid_iter(pool_guid, vdev_guid, devid, zfs_process_add,
 		    is_slice);
 
+	if (devpath != NULL) {
+		/* Can we match a /dev/disk/by-vdev/ path? */
+		char by_vdev_path[MAXPATHLEN];
+		snprintf(by_vdev_path, sizeof (by_vdev_path),
+		    "/dev/disk/by-vdev/%s", devpath);
+		if (by_vdev_path_iter(by_vdev_path, devid, zfs_process_add,
+		    is_slice))
+			return (0);
+	}
+
 	return (0);
 }
 

cmd/zed/agents/zfs_retire.c

@@ -40,6 +40,7 @@
 #include <sys/fm/fs/zfs.h>
 #include <libzfs.h>
 #include <string.h>
+#include <libgen.h>
 
 #include "zfs_agents.h"
 #include "fmd_api.h"

cmd/zed/zed.c

@@ -291,7 +291,7 @@ idle:
 		rv = zed_event_service(&zcp);
 
 		/* ENODEV: When kernel module is unloaded (osx) */
-		if (rv == ENODEV)
+		if (rv != 0)
 			break;
 	}
 

cmd/zed/zed.d/all-syslog.sh

@@ -21,7 +21,7 @@ if [ "${ZED_SYSLOG_DISPLAY_GUIDS}" = "1" ]; then
     [ -n "${ZEVENT_VDEV_GUID}" ] && msg="${msg} vdev_guid=${ZEVENT_VDEV_GUID}"
 else
     [ -n "${ZEVENT_POOL}" ] && msg="${msg} pool='${ZEVENT_POOL}'"
-    [ -n "${ZEVENT_VDEV_PATH}" ] && msg="${msg} vdev=$(basename "${ZEVENT_VDEV_PATH}")"
+    [ -n "${ZEVENT_VDEV_PATH}" ] && msg="${msg} vdev=${ZEVENT_VDEV_PATH##*/}"
 fi
 
 # log pool state if state is anything other than 'ACTIVE'

cmd/zed/zed.d/generic-notify.sh

@@ -23,7 +23,7 @@
 
 # Rate-limit the notification based in part on the filename.
 #
-rate_limit_tag="${ZEVENT_POOL};${ZEVENT_SUBCLASS};$(basename -- "$0")"
+rate_limit_tag="${ZEVENT_POOL};${ZEVENT_SUBCLASS};${0##*/}"
 rate_limit_interval="${ZED_NOTIFY_INTERVAL_SECS}"
 zed_rate_limit "${rate_limit_tag}" "${rate_limit_interval}" || exit 3
 

cmd/zed/zed.d/statechange-notify.sh

@@ -15,7 +15,7 @@
 # Send notification in response to a fault induced statechange
 #
 # ZEVENT_SUBCLASS: 'statechange'
-# ZEVENT_VDEV_STATE_STR: 'DEGRADED', 'FAULTED' or 'REMOVED'
+# ZEVENT_VDEV_STATE_STR: 'DEGRADED', 'FAULTED', 'REMOVED', or 'UNAVAIL'
 #
 # Exit codes:
 #   0: notification sent
@@ -31,7 +31,8 @@
 
 if [ "${ZEVENT_VDEV_STATE_STR}" != "FAULTED" ] \
         && [ "${ZEVENT_VDEV_STATE_STR}" != "DEGRADED" ] \
-        && [ "${ZEVENT_VDEV_STATE_STR}" != "REMOVED" ]; then
+        && [ "${ZEVENT_VDEV_STATE_STR}" != "REMOVED" ] \
+        && [ "${ZEVENT_VDEV_STATE_STR}" != "UNAVAIL" ]; then
     exit 3
 fi
 

cmd/zed/zed.d/zed-functions.sh

@@ -77,7 +77,7 @@ zed_log_msg()
 zed_log_err()
 {
     logger -p "${ZED_SYSLOG_PRIORITY}" -t "${ZED_SYSLOG_TAG}" -- "error:" \
-        "$(basename -- "$0"):""${ZEVENT_EID:+" eid=${ZEVENT_EID}:"}" "$@"
+        "${0##*/}:""${ZEVENT_EID:+" eid=${ZEVENT_EID}:"}" "$@"
 }
 
 
@@ -202,6 +202,10 @@ zed_notify()
     [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
     [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
 
+    zed_notify_pushover "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
     [ "${num_success}" -gt 0 ] && return 0
     [ "${num_failure}" -gt 0 ] && return 1
     return 2
@@ -254,7 +258,7 @@ zed_notify_email()
     [ -n "${subject}" ] || return 1
     if [ ! -r "${pathname}" ]; then
         zed_log_err \
-                "$(basename "${ZED_EMAIL_PROG}") cannot read \"${pathname}\""
+                "${ZED_EMAIL_PROG##*/} cannot read \"${pathname}\""
         return 1
     fi
 
@@ -266,7 +270,7 @@ zed_notify_email()
     eval ${ZED_EMAIL_PROG} ${ZED_EMAIL_OPTS} < "${pathname}" >/dev/null 2>&1
     rv=$?
     if [ "${rv}" -ne 0 ]; then
-        zed_log_err "$(basename "${ZED_EMAIL_PROG}") exit=${rv}"
+        zed_log_err "${ZED_EMAIL_PROG##*/} exit=${rv}"
         return 1
     fi
     return 0
@@ -413,7 +417,7 @@ zed_notify_slack_webhook()
 
     # Construct the JSON message for posting.
     #
-    msg_json="$(printf '{"text": "*%s*\n%s"}' "${subject}" "${msg_body}" )"
+    msg_json="$(printf '{"text": "*%s*\\n%s"}' "${subject}" "${msg_body}" )"
 
     # Send the POST request and check for errors.
     #
@@ -433,6 +437,84 @@ zed_notify_slack_webhook()
     return 0
 }
 
+# zed_notify_pushover (subject, pathname)
+#
+# Send a notification via Pushover <https://pushover.net/>.
+# The access token (ZED_PUSHOVER_TOKEN) identifies this client to the
+# Pushover server. The user token (ZED_PUSHOVER_USER) defines the user or
+# group to which the notification will be sent.
+#
+# Requires curl and sed executables to be installed in the standard PATH.
+#
+# References
+#   https://pushover.net/api
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_PUSHOVER_TOKEN
+#   ZED_PUSHOVER_USER
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_pushover()
+{
+    local subject="$1"
+    local pathname="${2:-"/dev/null"}"
+    local msg_body
+    local msg_out
+    local msg_err
+    local url="https://api.pushover.net/1/messages.json"
+
+    [ -n "${ZED_PUSHOVER_TOKEN}" ] && [ -n "${ZED_PUSHOVER_USER}" ] || return 2
+
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err "pushover cannot read \"${pathname}\""
+        return 1
+    fi
+
+    zed_check_cmd "curl" "sed" || return 1
+
+    # Read the message body in.
+    #
+    msg_body="$(cat "${pathname}")"
+
+    if [ -z "${msg_body}" ]
+    then
+        msg_body=$subject
+        subject=""
+    fi
+
+    # Send the POST request and check for errors.
+    #
+    msg_out="$( \
+        curl \
+        --form-string "token=${ZED_PUSHOVER_TOKEN}" \
+        --form-string "user=${ZED_PUSHOVER_USER}" \
+        --form-string "message=${msg_body}" \
+        --form-string "title=${subject}" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "curl exit=${rv}"
+        return 1
+    fi
+    msg_err="$(echo "${msg_out}" \
+        | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')"
+    if [ -n "${msg_err}" ]; then
+        zed_log_err "pushover \"${msg_err}"\"
+        return 1
+    fi
+    return 0
+}
+
+
 # zed_rate_limit (tag, [interval])
 #
 # Check whether an event of a given type [tag] has already occurred within the

cmd/zed/zed.d/zed.rc

@@ -13,9 +13,9 @@
 # Email address of the zpool administrator for receipt of notifications;
 #   multiple addresses can be specified if they are delimited by whitespace.
 # Email will only be sent if ZED_EMAIL_ADDR is defined.
-# Disabled by default; uncomment to enable.
+# Enabled by default; comment to disable.
 #
-#ZED_EMAIL_ADDR="root"
+ZED_EMAIL_ADDR="root"
 
 ##
 # Name or path of executable responsible for sending notifications via email;
@@ -82,6 +82,23 @@
 #
 #ZED_SLACK_WEBHOOK_URL=""
 
+##
+# Pushover token.
+# This defines the application from which the notification will be sent.
+#   <https://pushover.net/api#registration>
+# Disabled by default; uncomment to enable.
+# ZED_PUSHOVER_USER, below, must also be configured.
+#
+#ZED_PUSHOVER_TOKEN=""
+
+##
+# Pushover user key.
+# This defines which user or group will receive Pushover notifications.
+#  <https://pushover.net/api#identifiers>
+# Disabled by default; uncomment to enable.
+# ZED_PUSHOVER_TOKEN, above, must also be configured.
+#ZED_PUSHOVER_USER=""
+
 ##
 # Default directory for zed state files.
 #

cmd/zed/zed_conf.c

@@ -22,6 +22,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/uio.h>
 #include <unistd.h>

cmd/zed/zed_disk_event.c

@@ -215,6 +215,11 @@ zed_udev_monitor(void *arg)
 		if (type != NULL && type[0] != '\0' &&
 		    strcmp(type, "disk") == 0 &&
 		    part != NULL && part[0] != '\0') {
+			zed_log_msg(LOG_INFO,
+			    "%s: skip %s since it has a %s partition already",
+			    __func__,
+			    udev_device_get_property_value(dev, "DEVNAME"),
+			    part);
 			/* skip and wait for partition event */
 			udev_device_unref(dev);
 			continue;
@@ -229,6 +234,11 @@ zed_udev_monitor(void *arg)
 			sectors = udev_device_get_sysattr_value(dev, "size");
 		if (sectors != NULL &&
 		    strtoull(sectors, NULL, 10) < MINIMUM_SECTORS) {
+			zed_log_msg(LOG_INFO,
+			    "%s: %s sectors %s < %llu (minimum)",
+			    __func__,
+			    udev_device_get_property_value(dev, "DEVNAME"),
+			    sectors, MINIMUM_SECTORS);
 			udev_device_unref(dev);
 			continue;
 		}

cmd/zed/zed_exec.c

@@ -26,6 +26,8 @@
 #include <time.h>
 #include <unistd.h>
 #include <pthread.h>
+#include <signal.h>
+
 #include "zed_exec.h"
 #include "zed_log.h"
 #include "zed_strings.h"

cmd/zpool/zpool.d/dm-deps

@@ -16,14 +16,12 @@ if [ -L "$dev" ] ; then
 	dev=$(readlink "$dev")
 fi
 
-dev=$(basename "$dev")
+dev="${dev##*/}"
 val=""
 if [ -d "/sys/class/block/$dev/slaves" ] ; then
-	# ls -C: output in columns, no newlines
+	# ls -C: output in columns, no newlines, two spaces (change to one)
-	val=$(ls -C "/sys/class/block/$dev/slaves")
+	# shellcheck disable=SC2012
-
+	val=$(ls -C "/sys/class/block/$dev/slaves" | tr -s '[:space:]' ' ')
-	# ls -C will print two spaces between files; change to one space.
-	val=$(echo "$val" | sed -r 's/[[:blank:]]+/ /g')
 fi
 
 echo "dm-deps=$val"

cmd/zpool/zpool.d/iostat

@@ -9,7 +9,7 @@ iostat:		Show iostat values since boot (summary page).
 iostat-1s:	Do a single 1-second iostat sample and show values.
 iostat-10s:	Do a single 10-second iostat sample and show values."
 
-script=$(basename "$0")
+script="${0##*/}"
 if [ "$1" = "-h" ] ; then
 	echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
 	exit
@@ -42,7 +42,7 @@ else
 		${brief:+"-y"} \
 		${interval:+"$interval"} \
 		${interval:+"1"} \
-		"$VDEV_UPATH" | awk NF | tail -n 2)
+		"$VDEV_UPATH" | grep -v '^$' | tail -n 2)
 fi
 
 
@@ -61,7 +61,7 @@ fi
 cols=$(echo "$out" | head -n 1)
 
 # Get the values and tab separate them to make them cut-able.
-vals=$(echo "$out" | tail -n 1 | sed -r 's/[[:blank:]]+/\t/g')
+vals=$(echo "$out" | tail -n 1 | tr -s '[:space:]' '\t')
 
 i=0
 for col in $cols ; do

cmd/zpool/zpool.d/lsblk

@@ -48,7 +48,7 @@ size:	Show the disk capacity.
 vendor:	Show the disk vendor.
 lsblk:	Show the disk size, vendor, and model number."
 
-script=$(basename "$0")
+script="${0##*/}"
 
 if [ "$1" = "-h" ] ; then
         echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-

cmd/zpool/zpool.d/media

@@ -9,15 +9,12 @@ if [ "$1" = "-h" ] ; then
 fi
 
 if [ -b "$VDEV_UPATH" ]; then
-	device=$(basename "$VDEV_UPATH")
+	device="${VDEV_UPATH##*/}"
-	val=$(cat "/sys/block/$device/queue/rotational" 2>/dev/null)
+	read -r val 2>/dev/null < "/sys/block/$device/queue/rotational"
-	if [ "$val" = "0" ]; then
+	case "$val" in
-		MEDIA="ssd"
+		0) MEDIA="ssd" ;;
-	fi
+		1) MEDIA="hdd" ;;
-
+	esac
-	if [ "$val" = "1" ]; then
-		MEDIA="hdd"
-	fi
 
 	vpd_pg83="/sys/block/$device/device/vpd_pg83"
 	if [ -f "$vpd_pg83" ]; then

cmd/zpool/zpool.d/ses

@@ -11,7 +11,7 @@ fault_led:	Show value of the disk enclosure slot fault LED.
 locate_led:	Show value of the disk enclosure slot locate LED.
 ses:		Show disk's enc, enc device, slot, and fault/locate LED values."
 
-script=$(basename "$0")
+script="${0##*/}"
 if [ "$1" = "-h" ] ; then
 	echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
 	exit

cmd/zpool/zpool_main.c

@@ -4825,7 +4825,7 @@ children:
 			continue;
 
 		vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
-		    cb->cb_name_flags);
+		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
 		ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
 		    newchild[c], cb, depth + 2);
 		free(vname);
@@ -4868,7 +4868,7 @@ children:
 			}
 
 			vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
-			    cb->cb_name_flags);
+			    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
 			ret += print_vdev_stats(zhp, vname, oldnv ?
 			    oldchild[c] : NULL, newchild[c], cb, depth + 2);
 			free(vname);
@@ -6182,7 +6182,7 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
 			continue;
 
 		vname = zpool_vdev_name(g_zfs, zhp, child[c],
-		    cb->cb_name_flags);
+		    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
 		print_list_stats(zhp, vname, child[c], cb, depth + 2, B_FALSE);
 		free(vname);
 	}
@@ -6216,7 +6216,7 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
 				printed = B_TRUE;
 			}
 			vname = zpool_vdev_name(g_zfs, zhp, child[c],
-			    cb->cb_name_flags);
+			    cb->cb_name_flags | VDEV_NAME_TYPE_ID);
 			print_list_stats(zhp, vname, child[c], cb, depth + 2,
 			    B_FALSE);
 			free(vname);

cmd/zstream/zstream_dump.c

@@ -297,6 +297,7 @@ zstream_do_dump(int argc, char *argv[])
 
 	fletcher_4_init();
 	while (read_hdr(drr, &zc)) {
+		uint64_t featureflags = 0;
 
 		/*
 		 * If this is the first DMU record being processed, check for
@@ -362,6 +363,9 @@ zstream_do_dump(int argc, char *argv[])
 				    BSWAP_64(drrb->drr_fromguid);
 			}
 
+			featureflags =
+			    DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
+
 			(void) printf("BEGIN record\n");
 			(void) printf("\thdrtype = %lld\n",
 			    DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo));
@@ -461,6 +465,15 @@ zstream_do_dump(int argc, char *argv[])
 				    BSWAP_64(drro->drr_maxblkid);
 			}
 
+			if (featureflags & DMU_BACKUP_FEATURE_RAW &&
+			    drro->drr_bonuslen > drro->drr_raw_bonuslen) {
+				(void) fprintf(stderr,
+				    "Warning: Object %llu has bonuslen = "
+				    "%u > raw_bonuslen = %u\n\n",
+				    (u_longlong_t)drro->drr_object,
+				    drro->drr_bonuslen, drro->drr_raw_bonuslen);
+			}
+
 			payload_size = DRR_OBJECT_PAYLOAD_SIZE(drro);
 
 			if (verbose) {

config/Substfiles.am

@@ -15,7 +15,9 @@ subst_sed_cmd = \
 	-e 's|@PYTHON[@]|$(PYTHON)|g' \
 	-e 's|@PYTHON_SHEBANG[@]|$(PYTHON_SHEBANG)|g' \
 	-e 's|@DEFAULT_INIT_NFS_SERVER[@]|$(DEFAULT_INIT_NFS_SERVER)|g' \
-	-e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g'
+	-e 's|@DEFAULT_INIT_SHELL[@]|$(DEFAULT_INIT_SHELL)|g' \
+	-e 's|@LIBFETCH_DYNAMIC[@]|$(LIBFETCH_DYNAMIC)|g' \
+	-e 's|@LIBFETCH_SONAME[@]|$(LIBFETCH_SONAME)|g'
 
 SUBSTFILES =
 CLEANFILES = $(SUBSTFILES)

config/always-python.m4

@@ -28,7 +28,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYTHON], [
 	dnl #
 	AM_PATH_PYTHON([], [], [:])
 	AS_IF([test -z "$PYTHON_VERSION"], [
-		PYTHON_VERSION=$(basename $PYTHON | tr -cd 0-9.)
+		PYTHON_VERSION=$(echo ${PYTHON##*/} | tr -cd 0-9.)
 	])
 	PYTHON_MINOR=${PYTHON_VERSION#*\.}
 

config/always-pyzfs.m4

@@ -6,7 +6,7 @@ dnl # https://www.gnu.org/software/autoconf-archive/ax_python_module.html
 dnl # Required by ZFS_AC_CONFIG_ALWAYS_PYZFS.
 dnl #
 AC_DEFUN([ZFS_AC_PYTHON_MODULE], [
-	PYTHON_NAME=$(basename $PYTHON)
+	PYTHON_NAME=${PYTHON##*/}
 	AC_MSG_CHECKING([for $PYTHON_NAME module: $1])
 	AS_IF([$PYTHON -c "import $1" 2>/dev/null], [
 		AC_MSG_RESULT(yes)

config/kernel-add-disk.m4

@@ -0,0 +1,26 @@
+dnl #
+dnl # 5.16 API change
+dnl # add_disk grew a must-check return code
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_ADD_DISK], [
+
+	ZFS_LINUX_TEST_SRC([add_disk_ret], [
+		#include <linux/genhd.h>
+	], [
+		struct gendisk *disk = NULL;
+		int err = add_disk(disk);
+		err = err;
+	])
+
+])
+AC_DEFUN([ZFS_AC_KERNEL_ADD_DISK], [
+	AC_MSG_CHECKING([whether add_disk() returns int])
+	ZFS_LINUX_TEST_RESULT([add_disk_ret],
+	[
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_ADD_DISK_RET, 1,
+		    [add_disk() returns int])
+	], [
+		AC_MSG_RESULT(no)
+	])
+])

config/kernel-fallocate.m4

@@ -3,6 +3,10 @@ dnl # Linux 2.6.38 - 3.x API
 dnl # The fallocate callback was moved from the inode_operations
 dnl # structure to the file_operations structure.
 dnl #
+dnl #
+dnl # Linux 3.15+
+dnl # fallocate learned a new flag, FALLOC_FL_ZERO_RANGE
+dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_FALLOCATE], [
 	ZFS_LINUX_TEST_SRC([file_fallocate], [
 		#include <linux/fs.h>
@@ -15,12 +19,25 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FALLOCATE], [
 			.fallocate = test_fallocate,
 		};
 	], [])
+	ZFS_LINUX_TEST_SRC([falloc_fl_zero_range], [
+		#include <linux/falloc.h>
+	],[
+		int flags __attribute__ ((unused));
+		flags = FALLOC_FL_ZERO_RANGE;
+	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_FALLOCATE], [
 	AC_MSG_CHECKING([whether fops->fallocate() exists])
 	ZFS_LINUX_TEST_RESULT([file_fallocate], [
 		AC_MSG_RESULT(yes)
+		AC_MSG_CHECKING([whether FALLOC_FL_ZERO_RANGE exists])
+		ZFS_LINUX_TEST_RESULT([falloc_fl_zero_range], [
+			AC_MSG_RESULT(yes)
+			AC_DEFINE(HAVE_FALLOC_FL_ZERO_RANGE, 1, [FALLOC_FL_ZERO_RANGE is defined])
+		],[
+			AC_MSG_RESULT(no)
+		])
 	],[
 		ZFS_LINUX_TEST_ERROR([file_fallocate])
 	])

config/kernel-fpu.m4

@@ -1,9 +1,15 @@
-dnl # 
+dnl #
 dnl # Handle differences in kernel FPU code.
 dnl #
 dnl # Kernel
 dnl # 5.16:	XCR code put into asm/fpu/xcr.h
-dnl # 		HAVE_KERNEL_FPU_XCR_HEADER
+dnl #		HAVE_KERNEL_FPU_XCR_HEADER
+dnl #
+dnl #		XSTATE_XSAVE and XSTATE_XRESTORE aren't accessible any more
+dnl #		HAVE_KERNEL_FPU_XSAVE_INTERNAL
+dnl #
+dnl # 5.11:	kernel_fpu_begin() is an inlined function now, so don't check
+dnl #		for it inside the kernel symbols.
 dnl #
 dnl # 5.0:	Wrappers have been introduced to save/restore the FPU state.
 dnl #		This change was made to the 4.19.38 and 4.14.120 LTS kernels.
@@ -107,6 +113,36 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [
 		struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave;
 		struct xregs_state *xr __attribute__ ((unused)) = &st->xsave;
 	])
+
+	ZFS_LINUX_TEST_SRC([fpu_xsave_internal], [
+		#include <linux/sched.h>
+		#if defined(__x86_64) || defined(__x86_64__) || \
+		    defined(__i386) || defined(__i386__)
+		#if !defined(__x86)
+		#define __x86
+		#endif
+		#endif
+
+		#if !defined(__x86)
+		#error Unsupported architecture
+		#endif
+
+		#include <linux/types.h>
+		#ifdef HAVE_KERNEL_FPU_API_HEADER
+		#include <asm/fpu/api.h>
+		#include <asm/fpu/internal.h>
+		#else
+		#include <asm/i387.h>
+		#include <asm/xcr.h>
+		#endif
+
+	],[
+		struct fpu *fpu = &current->thread.fpu;
+		union fpregs_state *st = &fpu->fpstate->regs;
+		struct fregs_state *fr __attribute__ ((unused)) = &st->fsave;
+		struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave;
+		struct xregs_state *xr __attribute__ ((unused)) = &st->xsave;
+	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_FPU], [
@@ -114,8 +150,7 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU], [
 	dnl # Legacy kernel
 	dnl #
 	AC_MSG_CHECKING([whether kernel fpu is available])
-	ZFS_LINUX_TEST_RESULT_SYMBOL([kernel_fpu_license],
+	ZFS_LINUX_TEST_RESULT([kernel_fpu_license], [
-	    [kernel_fpu_begin], [arch/x86/kernel/fpu/core.c], [
 		AC_MSG_RESULT(kernel_fpu_*)
 		AC_DEFINE(HAVE_KERNEL_FPU, 1,
 		    [kernel has kernel_fpu_* functions])
@@ -139,7 +174,13 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU], [
 				AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1,
 				    [kernel fpu internal])
 			],[
+				ZFS_LINUX_TEST_RESULT([fpu_xsave_internal], [
+				    AC_MSG_RESULT(internal with internal XSAVE)
+				    AC_DEFINE(HAVE_KERNEL_FPU_XSAVE_INTERNAL, 1,
+					[kernel fpu and XSAVE internal])
+			    ],[
 				AC_MSG_RESULT(unavailable)
+			    ])
 			])
 		])
 	])

config/kernel-kmem.m4

@@ -64,6 +64,7 @@ dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_KVMALLOC], [
 	ZFS_LINUX_TEST_SRC([kvmalloc], [
 		#include <linux/mm.h>
+		#include <linux/slab.h>
 	],[
 		void *p __attribute__ ((unused));
 

config/kernel-kthread.m4

@@ -0,0 +1,68 @@
+AC_DEFUN([ZFS_AC_KERNEL_KTHREAD_COMPLETE_AND_EXIT], [
+	dnl #
+	dnl # 5.17 API,
+	dnl # cead18552660702a4a46f58e65188fe5f36e9dfe ("exit: Rename complete_and_exit to kthread_complete_and_exit")
+	dnl #
+	dnl # Also moves the definition from include/linux/kernel.h to include/linux/kthread.h
+	dnl #
+	AC_MSG_CHECKING([whether kthread_complete_and_exit() is available])
+	ZFS_LINUX_TEST_RESULT([kthread_complete_and_exit], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(SPL_KTHREAD_COMPLETE_AND_EXIT, kthread_complete_and_exit, [kthread_complete_and_exit() available])
+	], [
+		AC_MSG_RESULT(no)
+		AC_DEFINE(SPL_KTHREAD_COMPLETE_AND_EXIT, complete_and_exit, [using complete_and_exit() instead])
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_KTHREAD_DEQUEUE_SIGNAL_4ARG], [
+	dnl #
+	dnl # 5.17 API: enum pid_type * as new 4th dequeue_signal() argument,
+	dnl # 5768d8906bc23d512b1a736c1e198aa833a6daa4 ("signal: Requeue signals in the appropriate queue")
+	dnl #
+	dnl # int dequeue_signal(struct task_struct *task, sigset_t *mask, kernel_siginfo_t *info);
+	dnl # int dequeue_signal(struct task_struct *task, sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type);
+	dnl #
+	AC_MSG_CHECKING([whether dequeue_signal() takes 4 arguments])
+	ZFS_LINUX_TEST_RESULT([kthread_dequeue_signal], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_DEQUEUE_SIGNAL_4ARG, 1, [dequeue_signal() takes 4 arguments])
+	], [
+		AC_MSG_RESULT(no)
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD_COMPLETE_AND_EXIT], [
+	ZFS_LINUX_TEST_SRC([kthread_complete_and_exit], [
+		#include <linux/kthread.h>
+	], [
+		struct completion *completion = NULL;
+		long code = 0;
+
+		kthread_complete_and_exit(completion, code);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD_DEQUEUE_SIGNAL_4ARG], [
+	ZFS_LINUX_TEST_SRC([kthread_dequeue_signal], [
+		#include <linux/sched/signal.h>
+	], [
+		struct task_struct *task = NULL;
+		sigset_t *mask = NULL;
+		kernel_siginfo_t *info = NULL;
+		enum pid_type *type = NULL;
+		int error __attribute__ ((unused));
+
+		error = dequeue_signal(task, mask, info, type);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_KTHREAD], [
+	ZFS_AC_KERNEL_KTHREAD_COMPLETE_AND_EXIT
+	ZFS_AC_KERNEL_KTHREAD_DEQUEUE_SIGNAL_4ARG
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD], [
+	ZFS_AC_KERNEL_SRC_KTHREAD_COMPLETE_AND_EXIT
+	ZFS_AC_KERNEL_SRC_KTHREAD_DEQUEUE_SIGNAL_4ARG
+])

config/kernel-pde-data.m4

@@ -1,20 +1,22 @@
 dnl #
-dnl # 3.10 API change,
+dnl # 5.17 API: PDE_DATA() renamed to pde_data(),
-dnl # PDE is replaced by PDE_DATA
+dnl # 359745d78351c6f5442435f81549f0207ece28aa ("proc: remove PDE_DATA() completely")
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_PDE_DATA], [
 	ZFS_LINUX_TEST_SRC([pde_data], [
 		#include <linux/proc_fs.h>
 	], [
-		PDE_DATA(NULL);
+		pde_data(NULL);
 	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_PDE_DATA], [
-	AC_MSG_CHECKING([whether PDE_DATA() is available])
+	AC_MSG_CHECKING([whether pde_data() is lowercase])
-	ZFS_LINUX_TEST_RESULT_SYMBOL([pde_data], [PDE_DATA], [], [
+	ZFS_LINUX_TEST_RESULT([pde_data], [
 		AC_MSG_RESULT(yes)
-	],[
+		AC_DEFINE(SPL_PDE_DATA, pde_data, [pde_data() is pde_data()])
-		ZFS_LINUX_TEST_ERROR([PDE_DATA])
+	], [
+		AC_MSG_RESULT(no)
+		AC_DEFINE(SPL_PDE_DATA, PDE_DATA, [pde_data() is PDE_DATA()])
 	])
 ])

config/kernel-vfs-iov_iter.m4

@@ -41,6 +41,17 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_IOV_ITER], [
 		error = iov_iter_fault_in_readable(&iter, size);
 	])
 
+	ZFS_LINUX_TEST_SRC([fault_in_iov_iter_readable], [
+		#include <linux/fs.h>
+		#include <linux/uio.h>
+	],[
+		struct iov_iter iter = { 0 };
+		size_t size = 512;
+		int error __attribute__ ((unused));
+
+		error = fault_in_iov_iter_readable(&iter, size);
+	])
+
 	ZFS_LINUX_TEST_SRC([iov_iter_count], [
 		#include <linux/fs.h>
 		#include <linux/uio.h>
@@ -123,8 +134,15 @@ AC_DEFUN([ZFS_AC_KERNEL_VFS_IOV_ITER], [
 		AC_DEFINE(HAVE_IOV_ITER_FAULT_IN_READABLE, 1,
 		    [iov_iter_fault_in_readable() is available])
 	],[
-		AC_MSG_RESULT(no)
+		AC_MSG_CHECKING([whether fault_in_iov_iter_readable() is available])
-		enable_vfs_iov_iter="no"
+		ZFS_LINUX_TEST_RESULT([fault_in_iov_iter_readable], [
+			AC_MSG_RESULT(yes)
+			AC_DEFINE(HAVE_FAULT_IN_IOV_ITER_READABLE, 1,
+			    [fault_in_iov_iter_readable() is available])
+		],[
+			AC_MSG_RESULT(no)
+			enable_vfs_iov_iter="no"
+		])
 	])
 
 	AC_MSG_CHECKING([whether iov_iter_count() is available])

config/kernel.m4

@@ -135,6 +135,8 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC_VFS_SET_PAGE_DIRTY_NOBUFFERS
 	ZFS_AC_KERNEL_SRC_STANDALONE_LINUX_STDARG
 	ZFS_AC_KERNEL_SRC_PAGEMAP_FOLIO_WAIT_BIT
+	ZFS_AC_KERNEL_SRC_ADD_DISK
+	ZFS_AC_KERNEL_SRC_KTHREAD
 
 	AC_MSG_CHECKING([for available kernel interfaces])
 	ZFS_LINUX_TEST_COMPILE_ALL([kabi])
@@ -243,6 +245,8 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL_VFS_SET_PAGE_DIRTY_NOBUFFERS
 	ZFS_AC_KERNEL_STANDALONE_LINUX_STDARG
 	ZFS_AC_KERNEL_PAGEMAP_FOLIO_WAIT_BIT
+	ZFS_AC_KERNEL_ADD_DISK
+	ZFS_AC_KERNEL_KTHREAD
 ])
 
 dnl #
@@ -276,6 +280,35 @@ AC_DEFUN([ZFS_AC_MODULE_SYMVERS], [
 dnl #
 dnl # Detect the kernel to be built against
 dnl #
+dnl # Most modern Linux distributions have separate locations for bare
+dnl # source (source) and prebuilt (build) files. Additionally, there are
+dnl # `source` and `build` symlinks in `/lib/modules/$(KERNEL_VERSION)`
+dnl # pointing to them. The directory search order is now:
+dnl # 
+dnl # - `configure` command line values if both `--with-linux` and
+dnl #   `--with-linux-obj` were defined
+dnl # 
+dnl # - If only `--with-linux` was defined, `--with-linux-obj` is assumed
+dnl #   to have the same value as `--with-linux`
+dnl # 
+dnl # - If neither `--with-linux` nor `--with-linux-obj` were defined
+dnl #   autodetection is used:
+dnl # 
+dnl #   - `/lib/modules/$(uname -r)/{source,build}` respectively, if exist.
+dnl # 
+dnl #   - If only `/lib/modules/$(uname -r)/build` exists, it is assumed
+dnl #     to be both source and build directory.
+dnl # 
+dnl #   - The first directory in `/lib/modules` with the highest version
+dnl #     number according to `sort -V` which contains both `source` and
+dnl #     `build` symlinks/directories. If module directory contains only
+dnl #     `build` component, it is assumed to be both source and build
+dnl #     directory.
+dnl # 
+dnl #   - Last resort: the first directory matching `/usr/src/kernels/*`
+dnl #     and `/usr/src/linux-*` with the highest version number according
+dnl #     to `sort -V` is assumed to be both source and build directory.
+dnl #
 AC_DEFUN([ZFS_AC_KERNEL], [
 	AC_ARG_WITH([linux],
 		AS_HELP_STRING([--with-linux=PATH],
@@ -287,25 +320,52 @@ AC_DEFUN([ZFS_AC_KERNEL], [
 		[Path to kernel build objects]),
 		[kernelbuild="$withval"])
 
-	AC_MSG_CHECKING([kernel source directory])
+	AC_MSG_CHECKING([kernel source and build directories])
-	AS_IF([test -z "$kernelsrc"], [
+	AS_IF([test -n "$kernelsrc" && test -z "$kernelbuild"], [
-		AS_IF([test -e "/lib/modules/$(uname -r)/source"], [
+		kernelbuild="$kernelsrc"
-			headersdir="/lib/modules/$(uname -r)/source"
+	], [test -z "$kernelsrc"], [
-			sourcelink=$(readlink -f "$headersdir")
+		AS_IF([test -e "/lib/modules/$(uname -r)/source" && \
+		       test -e "/lib/modules/$(uname -r)/build"], [
+			src="/lib/modules/$(uname -r)/source"
+			build="/lib/modules/$(uname -r)/build"
 		], [test -e "/lib/modules/$(uname -r)/build"], [
-			headersdir="/lib/modules/$(uname -r)/build"
+			build="/lib/modules/$(uname -r)/build"
-			sourcelink=$(readlink -f "$headersdir")
+			src="$build"
 		], [
-			sourcelink=$(ls -1d /usr/src/kernels/* \
+			src=
-			             /usr/src/linux-* \
+
-			             2>/dev/null | grep -v obj | tail -1)
+			for d in $(ls -1d /lib/modules/* 2>/dev/null | sort -Vr); do
+				if test -e "$d/source" && test -e "$d/build"; then
+					src="$d/source"
+					build="$d/build"
+					break
+				fi
+
+				if test -e "$d/build"; then
+					src="$d/build"
+					build="$d/build"
+					break
+				fi
+			done
+
+			# the least reliable method
+			if test -z "$src"; then
+				src=$(ls -1d /usr/src/kernels/* /usr/src/linux-* \
+				      2>/dev/null | grep -v obj | sort -Vr | head -1)
+				build="$src"
+			fi
 		])
 
-		AS_IF([test -n "$sourcelink" && test -e ${sourcelink}], [
+		AS_IF([test -n "$src" && test -e "$src"], [
-			kernelsrc=`readlink -f ${sourcelink}`
+			kernelsrc=$(readlink -e "$src")
 		], [
 			kernelsrc="[Not found]"
 		])
+		AS_IF([test -n "$build" && test -e "$build"], [
+			kernelbuild=$(readlink -e "$build")
+		], [
+			kernelbuild="[Not found]"
+		])
 	], [
 		AS_IF([test "$kernelsrc" = "NONE"], [
 			kernsrcver=NONE
@@ -313,30 +373,19 @@ AC_DEFUN([ZFS_AC_KERNEL], [
 		withlinux=yes
 	])
 
+	AC_MSG_RESULT([done])
+	AC_MSG_CHECKING([kernel source directory])
 	AC_MSG_RESULT([$kernelsrc])
-	AS_IF([test ! -d "$kernelsrc"], [
+	AC_MSG_CHECKING([kernel build directory])
+	AC_MSG_RESULT([$kernelbuild])
+	AS_IF([test ! -d "$kernelsrc" || test ! -d "$kernelbuild"], [
 		AC_MSG_ERROR([
 	*** Please make sure the kernel devel package for your distribution
 	*** is installed and then try again.  If that fails, you can specify the
-	*** location of the kernel source with the '--with-linux=PATH' option.])
+	*** location of the kernel source and build with the '--with-linux=PATH' and
+	*** '--with-linux-obj=PATH' options respectively.])
 	])
 
-	AC_MSG_CHECKING([kernel build directory])
-	AS_IF([test -z "$kernelbuild"], [
-		AS_IF([test x$withlinux != xyes -a -e "/lib/modules/$(uname -r)/build"], [
-			kernelbuild=`readlink -f /lib/modules/$(uname -r)/build`
-		], [test -d ${kernelsrc}-obj/${target_cpu}/${target_cpu}], [
-			kernelbuild=${kernelsrc}-obj/${target_cpu}/${target_cpu}
-		], [test -d ${kernelsrc}-obj/${target_cpu}/default], [
-			kernelbuild=${kernelsrc}-obj/${target_cpu}/default
-		], [test -d `dirname ${kernelsrc}`/build-${target_cpu}], [
-			kernelbuild=`dirname ${kernelsrc}`/build-${target_cpu}
-		], [
-			kernelbuild=${kernelsrc}
-		])
-	])
-	AC_MSG_RESULT([$kernelbuild])
-
 	AC_MSG_CHECKING([kernel source version])
 	utsrelease1=$kernelbuild/include/linux/version.h
 	utsrelease2=$kernelbuild/include/linux/utsrelease.h
@@ -597,9 +646,15 @@ dnl #
 dnl # Used internally by ZFS_LINUX_TEST_{COMPILE,MODPOST}
 dnl #
 AC_DEFUN([ZFS_LINUX_COMPILE], [
+	AC_ARG_VAR([KERNEL_CC], [C compiler for
+		building kernel modules])
+	AC_ARG_VAR([KERNEL_LD], [Linker for
+		building kernel modules])
+	AC_ARG_VAR([KERNEL_LLVM], [Binary option to
+		build kernel modules with LLVM/CLANG toolchain])
 	AC_TRY_COMMAND([
 	    KBUILD_MODPOST_NOFINAL="$5" KBUILD_MODPOST_WARN="$6"
-	    make modules -k -j$TEST_JOBS -C $LINUX_OBJ $ARCH_UM
+	    make modules -k -j$TEST_JOBS ${KERNEL_CC:+CC=$KERNEL_CC} ${KERNEL_LD:+LD=$KERNEL_LD} ${KERNEL_LLVM:+LLVM=$KERNEL_LLVM} -C $LINUX_OBJ $ARCH_UM
 	    M=$PWD/$1 >$1/build.log 2>&1])
 	AS_IF([AC_TRY_COMMAND([$2])], [$3], [$4])
 ])

config/toolchain-simd.m4

@@ -24,6 +24,9 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES
 			;;
 	esac
 ])
@@ -422,3 +425,66 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE], [
 		AC_MSG_RESULT([no])
 	])
 ])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE], [
+	AC_MSG_CHECKING([whether host toolchain supports XSAVE])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		void main()
+		{
+		  char b[4096] __attribute__ ((aligned (64)));
+		  __asm__ __volatile__("xsave %[b]\n" : : [b] "m" (*b) : "memory");
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_XSAVE], 1, [Define if host toolchain supports XSAVE])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT], [
+	AC_MSG_CHECKING([whether host toolchain supports XSAVEOPT])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		void main()
+		{
+		  char b[4096] __attribute__ ((aligned (64)));
+		  __asm__ __volatile__("xsaveopt %[b]\n" : : [b] "m" (*b) : "memory");
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_XSAVEOPT], 1, [Define if host toolchain supports XSAVEOPT])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES], [
+	AC_MSG_CHECKING([whether host toolchain supports XSAVES])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		void main()
+		{
+		  char b[4096] __attribute__ ((aligned (64)));
+		  __asm__ __volatile__("xsaves %[b]\n" : : [b] "m" (*b) : "memory");
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_XSAVES], 1, [Define if host toolchain supports XSAVES])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])

config/user-libfetch.m4

@@ -0,0 +1,71 @@
+dnl #
+dnl # Check for a libfetch - either fetch(3) or libcurl.
+dnl #
+dnl # There are two configuration dimensions:
+dnl #   * fetch(3) vs libcurl
+dnl #   * static vs dynamic
+dnl #
+dnl # fetch(3) is only dynamic.
+dnl # We use sover 6, which first appeared in FreeBSD 8.0-RELEASE.
+dnl #
+dnl # libcurl development packages include curl-config(1) – we want:
+dnl #   * HTTPS support
+dnl #   * version at least 7.16 (October 2006), for sover 4
+dnl #   * to decide if it's static or not
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_USER_LIBFETCH], [
+	AC_MSG_CHECKING([for libfetch])
+	LIBFETCH_LIBS=
+	LIBFETCH_IS_FETCH=0
+	LIBFETCH_IS_LIBCURL=0
+	LIBFETCH_DYNAMIC=0
+	LIBFETCH_SONAME=
+	have_libfetch=
+
+	saved_libs="$LIBS"
+	LIBS="$LIBS -lfetch"
+	AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+		#include <sys/param.h>
+		#include <stdio.h>
+		#include <fetch.h>
+	]], [fetchGetURL("", "");])], [
+		have_libfetch=1
+		LIBFETCH_IS_FETCH=1
+		LIBFETCH_DYNAMIC=1
+		LIBFETCH_SONAME="libfetch.so.6"
+		LIBFETCH_LIBS="-ldl"
+		AC_MSG_RESULT([fetch(3)])
+	], [])
+	LIBS="$saved_libs"
+
+	if test -z "$have_libfetch"; then
+		if curl-config --protocols 2>/dev/null | grep -q HTTPS &&
+		    test "$(printf "%u" "0x$(curl-config --vernum)")" -ge "$(printf "%u" "0x071000")"; then
+			have_libfetch=1
+			LIBFETCH_IS_LIBCURL=1
+			if test "$(curl-config --built-shared)" = "yes"; then
+				LIBFETCH_DYNAMIC=1
+				LIBFETCH_SONAME="libcurl.so.4"
+				LIBFETCH_LIBS="-ldl"
+				AC_MSG_RESULT([libcurl])
+			else
+				LIBFETCH_LIBS="$(curl-config --libs)"
+				AC_MSG_RESULT([libcurl (static)])
+			fi
+
+			CCFLAGS="$CCFLAGS $(curl-config --cflags)"
+		fi
+	fi
+
+	if test -z "$have_libfetch"; then
+		AC_MSG_RESULT([none])
+	fi
+
+	AC_SUBST([LIBFETCH_LIBS])
+	AC_SUBST([LIBFETCH_DYNAMIC])
+	AC_SUBST([LIBFETCH_SONAME])
+	AC_DEFINE_UNQUOTED([LIBFETCH_IS_FETCH], [$LIBFETCH_IS_FETCH], [libfetch is fetch(3)])
+	AC_DEFINE_UNQUOTED([LIBFETCH_IS_LIBCURL], [$LIBFETCH_IS_LIBCURL], [libfetch is libcurl])
+	AC_DEFINE_UNQUOTED([LIBFETCH_DYNAMIC], [$LIBFETCH_DYNAMIC], [whether the chosen libfetch is to be loaded at run-time])
+	AC_DEFINE_UNQUOTED([LIBFETCH_SONAME], ["$LIBFETCH_SONAME"], [soname of chosen libfetch])
+])

config/user.m4

@@ -22,6 +22,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_LIBCRYPTO
 	ZFS_AC_CONFIG_USER_LIBAIO
 	ZFS_AC_CONFIG_USER_LIBATOMIC
+	ZFS_AC_CONFIG_USER_LIBFETCH
 	ZFS_AC_CONFIG_USER_CLOCK_GETTIME
 	ZFS_AC_CONFIG_USER_PAM
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR

config/zfs-build.m4

@@ -368,6 +368,9 @@ AC_DEFUN([ZFS_AC_RPM], [
 		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernels $(LINUX_VERSION)"'
 		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "ksrc $(LINUX)"'
 		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kobj $(LINUX_OBJ)"'
+		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_cc KERNEL_CC=$(KERNEL_CC)"'
+		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_ld KERNEL_LD=$(KERNEL_LD)"'
+		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_llvm KERNEL_LLVM=$(KERNEL_LLVM)"'
 	])
 
 	RPM_DEFINE_DKMS=''

config/zfs-meta.m4

@@ -73,14 +73,14 @@ AC_DEFUN([ZFS_AC_META], [
 		if test ! -f ".nogitrelease" && git rev-parse --git-dir > /dev/null 2>&1; then
 			_match="${ZFS_META_NAME}-${ZFS_META_VERSION}"
 			_alias=$(git describe --match=${_match} 2>/dev/null)
-			_release=$(echo ${_alias}|cut -f3- -d'-'|sed 's/-/_/g')
+			_release=$(echo ${_alias}|sed "s/${ZFS_META_NAME}//"|cut -f3- -d'-'|tr - _)
 			if test -n "${_release}"; then
 				ZFS_META_RELEASE=${_release}
 				_zfs_ac_meta_type="git describe"
 			else
 				_match="${ZFS_META_NAME}-${ZFS_META_VERSION}-${ZFS_META_RELEASE}"
 	                        _alias=$(git describe --match=${_match} 2>/dev/null)
-	                        _release=$(echo ${_alias}|cut -f3- -d'-'|sed 's/-/_/g')
+				_release=$(echo ${_alias}|sed 's/${ZFS_META_NAME}//'|cut -f3- -d'-'|tr - _)
 				if test -n "${_release}"; then
 					ZFS_META_RELEASE=${_release}
 					_zfs_ac_meta_type="git describe"

configure.ac

@@ -383,6 +383,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/rootpool/Makefile
 	tests/zfs-tests/tests/functional/rsend/Makefile
 	tests/zfs-tests/tests/functional/scrub_mirror/Makefile
+	tests/zfs-tests/tests/functional/simd/Makefile
 	tests/zfs-tests/tests/functional/slog/Makefile
 	tests/zfs-tests/tests/functional/snapshot/Makefile
 	tests/zfs-tests/tests/functional/snapused/Makefile

contrib/bpftrace/zfs-trace.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 
-ZVER=$(cut -f 1 -d '-' /sys/module/zfs/version)
+read -r ZVER < /sys/module/zfs/version
+ZVER="${ZVER%%-*}"
 KVER=$(uname -r)
 
 exec bpftrace \

contrib/dracut/02zfsexpandknowledge/module-setup.sh.in

@@ -2,8 +2,8 @@
 
 get_devtype() {
   local typ
-  typ=$(udevadm info --query=property --name="$1" | grep "^ID_FS_TYPE=" | sed 's|^ID_FS_TYPE=||')
+  typ=$(udevadm info --query=property --name="$1" | sed -n 's|^ID_FS_TYPE=||p')
-  if [ "$typ" = "" ] ; then
+  if [ -z "$typ" ] ; then
      typ=$(blkid -c /dev/null "$1" -o value -s TYPE)
   fi
   echo "$typ"
@@ -36,7 +36,6 @@ find_zfs_block_devices() {
     local dev
     local mp
     local fstype
-    local pool
     local _
     numfields="$(awk '{print NF; exit}' /proc/self/mountinfo)"
     if [ "$numfields" = "10" ] ; then
@@ -47,10 +46,7 @@ find_zfs_block_devices() {
     # shellcheck disable=SC2086
     while read -r ${fields?} ; do
        [ "$fstype" = "zfs" ] || continue
-       if [ "$mp" = "$1" ]; then
+       [ "$mp" = "$1" ] && get_pool_devices "${dev%%/*}"
-           pool=$(echo "$dev" | cut -d / -f 1)
-           get_pool_devices "$pool"
-       fi
     done < /proc/self/mountinfo
 }
 
@@ -100,9 +96,9 @@ if [ -n "$hostonly" ]; then
             majmin=$(get_maj_min "$dev")
             if [ -d "/sys/dev/block/$majmin/slaves" ] ; then
                 for _depdev in "/sys/dev/block/$majmin/slaves"/*; do
-                    [[ -f $_depdev/dev ]] || continue
+                    [ -f "$_depdev/dev" ] || continue
-                    _depdev=/dev/$(basename "$_depdev")
+                    _depdev="/dev/${_depdev##*/}"
-                    _depdevname=$(udevadm info --query=property --name="$_depdev" | grep "^DEVNAME=" | sed 's|^DEVNAME=||')
+                    _depdevname=$(udevadm info --query=property --name="$_depdev" | sed -n 's|^DEVNAME=||p')
                     _depdevtype=$(get_devtype "$_depdevname")
                     dinfo "zfsexpandknowledge: underlying block device backing ZFS dataset $mp: ${_depdevname//$'\n'/ }"
                     array_contains "$_depdevname" "${host_devs[@]}" || host_devs+=("$_depdevname")

contrib/dracut/90zfs/module-setup.sh.in

@@ -60,11 +60,17 @@ install() {
 		# Fallback: Guess the path and include all matches
 		dracut_install /usr/lib*/gcc/**/libgcc_s.so*
 	fi
+	# shellcheck disable=SC2050
+	if [ @LIBFETCH_DYNAMIC@ -gt 0 ]; then
+		for d in $libdirs; do
+			[ -e "$d/@LIBFETCH_SONAME@" ] && dracut_install "$d/@LIBFETCH_SONAME@"
+		done
+	fi
 	dracut_install @mounthelperdir@/mount.zfs
 	dracut_install @udevdir@/vdev_id
 	dracut_install awk
-	dracut_install basename
 	dracut_install cut
+	dracut_install tr
 	dracut_install head
 	dracut_install @udevdir@/zvol_id
 	inst_hook cmdline 95 "${moddir}/parse-zfs.sh"

contrib/dracut/90zfs/parse-zfs.sh.in

@@ -43,7 +43,7 @@ case "${root}" in
 		root="${root#FILESYSTEM=}"
 		root="zfs:${root#ZFS=}"
 		# switch + with spaces because kernel cmdline does not allow us to quote parameters
-		root=$(printf '%s\n' "$root" | sed "s/+/ /g")
+		root=$(echo "$root" | tr '+' ' ')
 		rootok=1
 		wait_for_zfs=1
 

contrib/dracut/90zfs/zfs-env-bootfs.service.in

@@ -8,7 +8,7 @@ Before=zfs-import.target
 
 [Service]
 Type=oneshot
-ExecStart=/bin/sh -c "systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -v '^-$')"
+ExecStart=/bin/sh -c "exec systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -v '^-$')"
 
 [Install]
 WantedBy=zfs-import.target

contrib/dracut/90zfs/zfs-generator.sh.in

@@ -89,7 +89,7 @@ else
   _zfs_generator_cb() {
       dset="${1}"
       mpnt="${2}"
-      unit="sysroot$(echo "$mpnt" | sed 's;/;-;g').mount"
+      unit="sysroot$(echo "$mpnt" | tr '/' '-').mount"
 
       {
           echo "[Unit]"

contrib/dracut/90zfs/zfs-load-key.sh.in

@@ -42,15 +42,32 @@ if [ "$(zpool list -H -o feature@encryption "${BOOTFS%%/*}")" = 'active' ]; then
         [ "$KEYSTATUS" = "unavailable" ] || exit 0
 
         KEYLOCATION="$(zfs get -H -o value keylocation "${ENCRYPTIONROOT}")"
-        if ! [ "${KEYLOCATION}" = "prompt" ]; then
+        case "${KEYLOCATION%%://*}" in
-            zfs load-key "${ENCRYPTIONROOT}"
+            prompt)
-        else
+                for _ in 1 2 3; do
-            # decrypt them
+                    systemd-ask-password --no-tty "Encrypted ZFS password for ${BOOTFS}" | zfs load-key "${ENCRYPTIONROOT}" && break
-            TRY_COUNT=5
+                done
-            while [ $TRY_COUNT -gt 0 ]; do
+                ;;
-                systemd-ask-password "Encrypted ZFS password for ${BOOTFS}" --no-tty | zfs load-key "${ENCRYPTIONROOT}" && break
+            http*)
-                TRY_COUNT=$((TRY_COUNT - 1))
+                systemctl start network-online.target
-            done
+                zfs load-key "${ENCRYPTIONROOT}"
-        fi
+                ;;
+            file)
+                KEYFILE="${KEYLOCATION#file://}"
+                [ -r "${KEYFILE}" ] || udevadm settle
+                [ -r "${KEYFILE}" ] || {
+                    info "Waiting for key ${KEYFILE} for ${ENCRYPTIONROOT}..."
+                    for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do
+                        sleep 0.5s
+                        [ -r "${KEYFILE}" ] && break
+                    done
+                }
+                [ -r "${KEYFILE}" ] || warn "Key ${KEYFILE} for ${ENCRYPTIONROOT} hasn't appeared. Trying anyway."
+                zfs load-key "${ENCRYPTIONROOT}"
+                ;;
+            *)
+                zfs load-key "${ENCRYPTIONROOT}"
+                ;;
+        esac
     fi
 fi

contrib/dracut/90zfs/zfs-rollback-bootfs.service.in

@@ -10,5 +10,5 @@ ConditionKernelCommandLine=bootfs.rollback
 # ${BOOTFS} should have been set by zfs-env-bootfs.service
 Type=oneshot
 ExecStartPre=/bin/sh -c 'test -n "${BOOTFS}"'
-ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; @sbindir@/zfs rollback -Rf "${BOOTFS}@${SNAPNAME:-%v}"'
+ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; exec @sbindir@/zfs rollback -Rf "${BOOTFS}@${SNAPNAME:-%v}"'
 RemainAfterExit=yes

contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in

@@ -10,5 +10,5 @@ ConditionKernelCommandLine=bootfs.snapshot
 # ${BOOTFS} should have been set by zfs-env-bootfs.service
 Type=oneshot
 ExecStartPre=/bin/sh -c 'test -n "${BOOTFS}"'
-ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; @sbindir@/zfs snapshot "${BOOTFS}@${SNAPNAME:-%v}"'
+ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; exec @sbindir@/zfs snapshot "${BOOTFS}@${SNAPNAME:-%v}"'
 RemainAfterExit=yes

contrib/initramfs/hooks/zfs.in

@@ -30,6 +30,13 @@ find /lib/ -type f -name "libgcc_s.so.[1-9]" | while read -r libgcc; do
 	copy_exec "$libgcc"
 done
 
+# shellcheck disable=SC2050
+if [ @LIBFETCH_DYNAMIC@ -gt 0 ]; then
+	find /lib/ -name "@LIBFETCH_SONAME@" | while read -r libfetch; do
+		copy_exec "$libfetch"
+	done
+fi
+
 copy_file config "/etc/hostid"
 copy_file cache  "@sysconfdir@/zfs/zpool.cache"
 copy_file config "@initconfdir@/zfs"

contrib/initramfs/scripts/zfs

@@ -105,8 +105,7 @@ find_rootfs()
 find_pools()
 {
 	pools=$("$@" 2> /dev/null | \
-		grep -E "pool:|^[a-zA-Z0-9]" | \
+		sed -Ee '/pool:|^[a-zA-Z0-9]/!d' -e 's@.*: @@' | \
-		sed 's@.*: @@' | \
 		tr '\n' ';')
 
 	echo "${pools%%;}" # Return without the last ';'.
@@ -403,35 +402,32 @@ decrypt_fs()
 			KEYSTATUS="$(get_fs_value "${ENCRYPTIONROOT}" keystatus)"
 			# Continue only if the key needs to be loaded
 			[ "$KEYSTATUS" = "unavailable" ] || return 0
-			TRY_COUNT=3
 
-			# If key is stored in a file, do not prompt
+			# Do not prompt if key is stored noninteractively,
 			if ! [ "${KEYLOCATION}" = "prompt" ]; then
 				$ZFS load-key "${ENCRYPTIONROOT}"
 
 			# Prompt with plymouth, if active
-			elif [ -e /bin/plymouth ] && /bin/plymouth --ping 2>/dev/null; then
+			elif /bin/plymouth --ping 2>/dev/null; then
 				echo "plymouth" > /run/zfs_console_askpwd_cmd
-				while [ $TRY_COUNT -gt 0 ]; do
+				for _ in 1 2 3; do
 					plymouth ask-for-password --prompt "Encrypted ZFS password for ${ENCRYPTIONROOT}" | \
 						$ZFS load-key "${ENCRYPTIONROOT}" && break
-					TRY_COUNT=$((TRY_COUNT - 1))
 				done
 
 			# Prompt with systemd, if active
 			elif [ -e /run/systemd/system ]; then
 				echo "systemd-ask-password" > /run/zfs_console_askpwd_cmd
-				while [ $TRY_COUNT -gt 0 ]; do
+				for _ in 1 2 3; do
-					systemd-ask-password "Encrypted ZFS password for ${ENCRYPTIONROOT}" --no-tty | \
+					systemd-ask-password --no-tty "Encrypted ZFS password for ${ENCRYPTIONROOT}" | \
 						$ZFS load-key "${ENCRYPTIONROOT}" && break
-					TRY_COUNT=$((TRY_COUNT - 1))
 				done
 
 			# Prompt with ZFS tty, otherwise
 			else
 				# Temporarily setting "printk" to "7" allows the prompt to appear even when the "quiet" kernel option has been used
 				echo "load-key" > /run/zfs_console_askpwd_cmd
-				storeprintk="$(awk '{print $1}' /proc/sys/kernel/printk)"
+				read -r storeprintk _ < /proc/sys/kernel/printk
 				echo 7 > /proc/sys/kernel/printk
 				$ZFS load-key "${ENCRYPTIONROOT}"
 				echo "$storeprintk" > /proc/sys/kernel/printk

etc/default/zfs.in

@@ -1,4 +1,4 @@
-# ZoL userland configuration.
+# OpenZFS userland configuration.
 
 # NOTE: This file is intended for sysv init and initramfs.
 # Changing some of these settings may not make any difference on
@@ -9,6 +9,12 @@
 # To enable a boolean setting, set it to yes, on, true, or 1.
 # Anything else will be interpreted as unset.
 
+# Run `zfs load-key` during system start?
+ZFS_LOAD_KEY='yes'
+
+# Run `zfs unload-key` during system stop?
+ZFS_UNLOAD_KEY='no'
+
 # Run `zfs mount -a` during system start?
 ZFS_MOUNT='yes'
 

etc/init.d/.gitignore

@@ -1,4 +1,5 @@
 zfs-import
+zfs-load-key
 zfs-mount
 zfs-share
 zfs-zed

etc/init.d/Makefile.am

@@ -3,7 +3,7 @@ include $(top_srcdir)/config/Shellcheck.am
 
 EXTRA_DIST += README.md
 
-init_SCRIPTS = zfs-import zfs-mount zfs-share zfs-zed
+init_SCRIPTS = zfs-import zfs-load-key zfs-mount zfs-share zfs-zed
 
 SUBSTFILES += $(init_SCRIPTS)
 

etc/init.d/README.md

@@ -42,14 +42,16 @@ INSTALLING INIT SCRIPT LINKS
   To setup the init script links in /etc/rc?.d manually on a Debian GNU/Linux
   (or derived) system, run the following commands (the order is important!):
 
-    update-rc.d zfs-import start 07 S .       stop 07 0 1 6 .
+    update-rc.d zfs-import   start 07 S .       stop 07 0 1 6 .
-    update-rc.d zfs-mount  start 02 2 3 4 5 . stop 06 0 1 6 .
+    update-rc.d zfs-load-key start 02 2 3 4 5 . stop 06 0 1 6 .
-    update-rc.d zfs-zed    start 07 2 3 4 5 . stop 08 0 1 6 .
+    update-rc.d zfs-mount    start 02 2 3 4 5 . stop 06 0 1 6 .
-    update-rc.d zfs-share  start 27 2 3 4 5 . stop 05 0 1 6 .
+    update-rc.d zfs-zed      start 07 2 3 4 5 . stop 08 0 1 6 .
+    update-rc.d zfs-share    start 27 2 3 4 5 . stop 05 0 1 6 .
 
   To do the same on RedHat, Fedora and/or CentOS:
 
     chkconfig zfs-import
+    chkconfig zfs-load-key
     chkconfig zfs-mount
     chkconfig zfs-zed
     chkconfig zfs-share
@@ -57,6 +59,7 @@ INSTALLING INIT SCRIPT LINKS
   On Gentoo:
 
     rc-update add zfs-import boot
+    rc-update add zfs-load-key boot
     rc-update add zfs-mount boot
     rc-update add zfs-zed default
     rc-update add zfs-share default

etc/init.d/zfs-import.in

@@ -57,8 +57,7 @@ find_pools()
 	local pools
 
 	pools=$("$@" 2> /dev/null | \
-		grep -E "pool:|^[a-zA-Z0-9]" | \
+		sed -Ee '/pool:|^[a-zA-Z0-9]/!d' -e 's@.*: @@' | \
-		sed 's@.*: @@' | \
 		sort | \
 		tr '\n' ';')
 

etc/init.d/zfs-load-key.in

@@ -0,0 +1,131 @@
+#!@DEFAULT_INIT_SHELL@
+#
+# zfs-load-key  This script will load/unload the zfs filesystems keys.
+#
+# chkconfig:    2345 06 99
+# description:  This script will load or unload the zfs filesystems keys during
+#               system boot/shutdown. Only filesystems with key path set
+#               in keylocation property. See the zfs(8) man page for details.
+# probe: true
+#
+### BEGIN INIT INFO
+# Provides:          zfs-load-key
+# Required-Start:    $local_fs zfs-import
+# Required-Stop:     $local_fs zfs-import
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# X-Start-Before:    zfs-mount
+# X-Stop-After:      zfs-zed
+# Short-Description: Load ZFS keys for filesystems and volumes
+# Description: Run the `zfs load-key` or `zfs unload-key` commands.
+### END INIT INFO
+#
+# Released under the 2-clause BSD license.
+#
+# This script is based on debian/zfsutils.zfs.init from the
+# Debian GNU/kFreeBSD zfsutils 8.1-3 package, written by Aurelien Jarno.
+
+# Source the common init script
+. @sysconfdir@/zfs/zfs-functions
+
+# ----------------------------------------------------
+
+do_depend()
+{
+	# bootmisc will log to /var which may be a different zfs than root.
+	before bootmisc logger zfs-mount
+
+	after zfs-import sysfs
+	keyword -lxc -openvz -prefix -vserver
+}
+
+# Load keys for all datasets/filesystems
+do_load_keys()
+{
+	zfs_log_begin_msg "Load ZFS filesystem(s) keys"
+
+	"$ZFS" list -Ho name,encryptionroot,keystatus,keylocation |
+	    while IFS="	" read -r name encryptionroot keystatus keylocation; do
+		if [ "$encryptionroot" != "-" ] &&
+			[ "$name" = "$encryptionroot" ] &&
+			[ "$keystatus" = "unavailable" ] &&
+			[ "$keylocation" != "prompt" ] &&
+			[ "$keylocation" != "none" ]
+		then
+			zfs_action "Load key for $encryptionroot" \
+			    "$ZFS" load-key "$encryptionroot"
+		fi
+	done
+
+	zfs_log_end_msg 0
+
+	return 0
+}
+
+# Unload keys for all datasets/filesystems
+do_unload_keys()
+{
+	zfs_log_begin_msg "Unload ZFS filesystem(s) key"
+
+	"$ZFS" list -Ho name,encryptionroot,keystatus | sed '1!G;h;$!d' |
+	    while IFS="	" read -r name encryptionroot keystatus; do
+		if [ "$encryptionroot" != "-" ] &&
+			[ "$name" = "$encryptionroot" ] &&
+			[ "$keystatus" = "available" ]
+		then
+			zfs_action "Unload key for $encryptionroot" \
+			    "$ZFS" unload-key "$encryptionroot"
+		fi
+	done
+
+	zfs_log_end_msg 0
+
+	return 0
+}
+
+do_start()
+{
+	check_boolean "$ZFS_LOAD_KEY" || exit 0
+
+	check_module_loaded "zfs" || exit 0
+
+	do_load_keys
+}
+
+do_stop()
+{
+	check_boolean "$ZFS_UNLOAD_KEY" || exit 0
+
+	check_module_loaded "zfs" || exit 0
+
+	do_unload_keys
+}
+
+# ----------------------------------------------------
+
+if [ ! -e /sbin/openrc-run ]
+then
+	case "$1" in
+		start)
+			do_start
+			;;
+		stop)
+			do_stop
+			;;
+		force-reload|condrestart|reload|restart|status)
+			# no-op
+			;;
+		*)
+			[ -n "$1" ] && echo "Error: Unknown command $1."
+			echo "Usage: $0 {start|stop}"
+			exit 3
+			;;
+	esac
+
+	exit $?
+else
+	# Create wrapper functions since Gentoo don't use the case part.
+	depend() { do_depend; }
+	start() { do_start; }
+	stop() { do_stop; }
+fi

etc/systemd/system/.gitignore

@@ -1,3 +1,4 @@
 *.service
 *.target
 *.preset
+*.timer

etc/systemd/system/Makefile.am

@@ -12,7 +12,10 @@ systemdunit_DATA = \
 	zfs-volume-wait.service \
 	zfs-import.target \
 	zfs-volumes.target \
-	zfs.target
+	zfs.target \
+	zfs-scrub-monthly@.timer \
+	zfs-scrub-weekly@.timer \
+	zfs-scrub@.service
 
 SUBSTFILES += $(systemdpreset_DATA) $(systemdunit_DATA)
 

etc/systemd/system/zfs-scrub-monthly@.timer.in

@@ -0,0 +1,12 @@
+[Unit]
+Description=Monthly zpool scrub timer for %i
+Documentation=man:zpool-scrub(8)
+
+[Timer]
+OnCalendar=monthly
+Persistent=true
+RandomizedDelaySec=1h
+Unit=zfs-scrub@%i.service
+
+[Install]
+WantedBy=timers.target

etc/systemd/system/zfs-scrub-weekly@.timer.in

@@ -0,0 +1,12 @@
+[Unit]
+Description=Weekly zpool scrub timer for %i
+Documentation=man:zpool-scrub(8)
+
+[Timer]
+OnCalendar=weekly
+Persistent=true
+RandomizedDelaySec=1h
+Unit=zfs-scrub@%i.service
+
+[Install]
+WantedBy=timers.target

etc/systemd/system/zfs-scrub@.service.in

@@ -0,0 +1,14 @@
+[Unit]
+Description=zpool scrub on %i
+Documentation=man:zpool-scrub(8)
+Requires=zfs.target
+After=zfs.target
+ConditionACPower=true
+ConditionPathIsDirectory=/sys/module/zfs
+
+[Service]
+ExecStart=/bin/sh -c '\
+if @sbindir@/zpool status %i | grep "scrub in progress"; then\
+exec @sbindir@/zpool wait -t scrub %i;\
+else exec @sbindir@/zpool scrub -w %i; fi'
+ExecStop=-/bin/sh -c '@sbindir@/zpool scrub -p %i 2>/dev/null || true'

etc/zfs/zfs-functions.in

@@ -1,5 +1,5 @@
-# This is a script with common functions etc used by zfs-import, zfs-mount,
+# This is a script with common functions etc used by zfs-import, zfs-load-key,
-# zfs-share and zfs-zed.
+# zfs-mount, zfs-share and zfs-zed.
 #
 # It is _NOT_ to be called independently
 #
@@ -92,6 +92,8 @@ ZPOOL="@sbindir@/zpool"
 ZPOOL_CACHE="@sysconfdir@/zfs/zpool.cache"
 
 # Sensible defaults
+ZFS_LOAD_KEY='yes'
+ZFS_UNLOAD_KEY='no'
 ZFS_MOUNT='yes'
 ZFS_UNMOUNT='yes'
 ZFS_SHARE='yes'
@@ -104,7 +106,8 @@ fi
 
 # ----------------------------------------------------
 
-export ZFS ZED ZPOOL ZPOOL_CACHE ZFS_MOUNT ZFS_UNMOUNT ZFS_SHARE ZFS_UNSHARE
+export ZFS ZED ZPOOL ZPOOL_CACHE ZFS_LOAD_KEY ZFS_UNLOAD_KEY ZFS_MOUNT ZFS_UNMOUNT \
+    ZFS_SHARE ZFS_UNSHARE
 
 zfs_action()
 {
@@ -345,7 +348,7 @@ read_mtab()
 
 	# Unset all MTAB_* variables
 	# shellcheck disable=SC2046
-	unset $(env | grep ^MTAB_ | sed 's,=.*,,')
+	unset $(env | sed -e '/^MTAB_/!d' -e 's,=.*,,')
 
 	while read -r fs mntpnt fstype opts rest; do
 		if echo "$fs $mntpnt $fstype $opts" | grep -qE "$match"; then
@@ -360,9 +363,8 @@ read_mtab()
 			fs=$(/bin/echo "$fs" | sed 's,\\0,\\00,')
 
 			# Remove 'unwanted' characters.
-			mntpnt=$(printf '%b\n' "$mntpnt" | sed -e 's,/,,g' \
+			mntpnt=$(printf '%b' "$mntpnt" | tr -d '/. -')
-			    -e 's,-,,g' -e 's,\.,,g' -e 's, ,,g')
+			fs=$(printf '%b' "$fs")
-			fs=$(printf '%b\n' "$fs")
 
 			# Set the variable.
 			eval export "MTAB_$mntpnt=\"$fs\""
@@ -374,8 +376,7 @@ in_mtab()
 {
 	local mntpnt="$1"
 	# Remove 'unwanted' characters.
-	mntpnt=$(printf '%b\n' "$mntpnt" | sed -e 's,/,,g' \
+	mntpnt=$(printf '%b' "$mntpnt" | tr -d '/. -')
-	    -e 's,-,,g' -e 's,\.,,g' -e 's, ,,g')
 	local var
 
 	var="$(eval echo "MTAB_$mntpnt")"
@@ -391,7 +392,7 @@ read_fstab()
 
 	# Unset all FSTAB_* variables
 	# shellcheck disable=SC2046
-	unset $(env | grep ^FSTAB_ | sed 's,=.*,,')
+	unset $(env | sed -e '/^FSTAB_/!d' -e 's,=.*,,')
 
 	i=0
 	while read -r fs mntpnt fstype opts; do
@@ -401,7 +402,7 @@ read_fstab()
 
 		if echo "$fs $mntpnt $fstype $opts" | grep -qE "$match"; then
 			eval export "FSTAB_dev_$i=$fs"
-			fs=$(printf '%b\n' "$fs" | sed 's,/,_,g')
+			fs=$(printf '%b' "$fs" | tr '/' '_')
 			eval export "FSTAB_$i=$mntpnt"
 
 			i=$((i + 1))

include/libzfs_impl.h

@@ -72,6 +72,8 @@ struct libzfs_handle {
 	boolean_t libzfs_prop_debug;
 	regex_t libzfs_urire;
 	uint64_t libzfs_max_nvlist;
+	void *libfetch;
+	char *libfetch_load_error;
 };
 
 struct zfs_handle {

include/os/freebsd/spl/sys/Makefile.am

@@ -22,7 +22,6 @@ KERNEL_H = \
 	inttypes.h \
 	isa_defs.h \
 	kmem_cache.h \
-	kidmap.h \
 	kmem.h \
 	kstat.h \
 	list_impl.h \

include/os/freebsd/spl/sys/cred.h

@@ -48,138 +48,20 @@ extern "C" {
 typedef struct ucred cred_t;
 
 #define	CRED()		curthread->td_ucred
+
+/*
+ * kcred is used when you need all privileges.
+ */
 #define	kcred	(thread0.td_ucred)
 
 #define	KUID_TO_SUID(x)		(x)
 #define	KGID_TO_SGID(x)		(x)
-#define	crgetuid(cred)		((cred)->cr_uid)
+#define	crgetuid(cr)		((cr)->cr_uid)
-#define	crgetruid(cred)		((cred)->cr_ruid)
+#define	crgetruid(cr)		((cr)->cr_ruid)
-#define	crgetgid(cred)		((cred)->cr_gid)
+#define	crgetgid(cr)		((cr)->cr_gid)
-#define	crgetgroups(cred)	((cred)->cr_groups)
+#define	crgetgroups(cr)		((cr)->cr_groups)
-#define	crgetngroups(cred)	((cred)->cr_ngroups)
+#define	crgetngroups(cr)	((cr)->cr_ngroups)
-#define	crgetsid(cred, i)	(NULL)
+#define	crgetzoneid(cr) 	((cr)->cr_prison->pr_id)
-
-struct proc;				/* cred.h is included in proc.h */
-struct prcred;
-struct ksid;
-struct ksidlist;
-struct credklpd;
-struct credgrp;
-
-struct auditinfo_addr;			/* cred.h is included in audit.h */
-
-extern int ngroups_max;
-/*
- * kcred is used when you need all privileges.
- */
-
-extern void cred_init(void);
-extern void crfree(cred_t *);
-extern cred_t *cralloc(void);		/* all but ref uninitialized */
-extern cred_t *cralloc_ksid(void);	/* cralloc() + ksid alloc'ed */
-extern cred_t *crget(void);		/* initialized */
-extern void crcopy_to(cred_t *, cred_t *);
-extern cred_t *crdup(cred_t *);
-extern void crdup_to(cred_t *, cred_t *);
-extern cred_t *crgetcred(void);
-extern void crset(struct proc *, cred_t *);
-extern void crset_zone_privall(cred_t *);
-extern int supgroupmember(gid_t, const cred_t *);
-extern int hasprocperm(const cred_t *, const cred_t *);
-extern int prochasprocperm(struct proc *, struct proc *, const cred_t *);
-extern int crcmp(const cred_t *, const cred_t *);
-extern cred_t *zone_kcred(void);
-
-extern gid_t crgetrgid(const cred_t *);
-extern gid_t crgetsgid(const cred_t *);
-
-#define	crgetzoneid(cr) ((cr)->cr_prison->pr_id)
-extern projid_t crgetprojid(const cred_t *);
-
-extern cred_t *crgetmapped(const cred_t *);
-
-
-extern const struct auditinfo_addr *crgetauinfo(const cred_t *);
-extern struct auditinfo_addr *crgetauinfo_modifiable(cred_t *);
-
-extern uint_t crgetref(const cred_t *);
-
-extern const gid_t *crgetggroups(const struct credgrp *);
-
-
-/*
- * Sets real, effective and/or saved uid/gid;
- * -1 argument accepted as "no change".
- */
-extern int crsetresuid(cred_t *, uid_t, uid_t, uid_t);
-extern int crsetresgid(cred_t *, gid_t, gid_t, gid_t);
-
-/*
- * Sets real, effective and saved uids/gids all to the same
- * values.  Both values must be non-negative and <= MAXUID
- */
-extern int crsetugid(cred_t *, uid_t, gid_t);
-
-/*
- * Functions to handle the supplemental group list.
- */
-extern struct credgrp *crgrpcopyin(int, gid_t *);
-extern void crgrprele(struct credgrp *);
-extern void crsetcredgrp(cred_t *, struct credgrp *);
-
-/*
- * Private interface for setting zone association of credential.
- */
-struct zone;
-extern void crsetzone(cred_t *, struct zone *);
-extern struct zone *crgetzone(const cred_t *);
-
-/*
- * Private interface for setting project id in credential.
- */
-extern void crsetprojid(cred_t *, projid_t);
-
-/*
- * Private interface for nfs.
- */
-extern cred_t *crnetadjust(cred_t *);
-
-/*
- * Private interface for procfs.
- */
-extern void cred2prcred(const cred_t *, struct prcred *);
-
-/*
- * Private interfaces for Rampart Trusted Solaris.
- */
-struct ts_label_s;
-extern struct ts_label_s *crgetlabel(const cred_t *);
-extern boolean_t crisremote(const cred_t *);
-
-/*
- * Private interfaces for ephemeral uids.
- */
-#define	VALID_UID(id, zn)					\
-	((id) <= MAXUID || valid_ephemeral_uid((zn), (id)))
-
-#define	VALID_GID(id, zn)					\
-	((id) <= MAXUID || valid_ephemeral_gid((zn), (id)))
-
-extern boolean_t valid_ephemeral_uid(struct zone *, uid_t);
-extern boolean_t valid_ephemeral_gid(struct zone *, gid_t);
-
-extern int eph_uid_alloc(struct zone *, int, uid_t *, int);
-extern int eph_gid_alloc(struct zone *, int, gid_t *, int);
-
-extern void crsetsid(cred_t *, struct ksid *, int);
-extern void crsetsidlist(cred_t *, struct ksidlist *);
-
-extern struct ksidlist *crgetsidlist(const cred_t *);
-
-extern int crsetpriv(cred_t *, ...);
-
-extern struct credklpd *crgetcrklpd(const cred_t *);
-extern void crsetcrklpd(cred_t *, struct credklpd *);
 
 #ifdef	__cplusplus
 }

include/os/freebsd/spl/sys/kidmap.h

@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2007 Pawel Jakub Dawidek <pjd@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#ifndef _OPENSOLARIS_SYS_KIDMAP_H_
-#define	_OPENSOLARIS_SYS_KIDMAP_H_
-
-#include <sys/idmap.h>
-
-typedef int32_t	idmap_stat;
-typedef void	idmap_get_handle_t;
-
-#define	kidmap_get_create()		(NULL)
-#define	kidmap_get_destroy(hdl)		do { } while (0)
-#define	kidmap_get_mappings(hdl)	(NULL)
-
-#endif	/* _OPENSOLARIS_SYS_KIDMAP_H_ */

include/os/freebsd/spl/sys/sid.h

@@ -29,7 +29,6 @@
 #ifndef _OPENSOLARIS_SYS_SID_H_
 #define	_OPENSOLARIS_SYS_SID_H_
 #include <sys/idmap.h>
-#include <sys/kidmap.h>
 
 typedef struct ksiddomain {
 	char	*kd_name;	/* Domain part of SID */
@@ -59,28 +58,4 @@ ksiddomain_rele(ksiddomain_t *kd)
 	kmem_free(kd, sizeof (*kd));
 }
 
-static __inline uint_t
-ksid_getid(ksid_t *ks)
-{
-
-	panic("%s has been unexpectedly called", __func__);
-}
-
-static __inline const char *
-ksid_getdomain(ksid_t *ks)
-{
-
-	panic("%s has been unexpectedly called", __func__);
-}
-
-static __inline uint_t
-ksid_getrid(ksid_t *ks)
-{
-
-	panic("%s has been unexpectedly called", __func__);
-}
-
-#define	kidmap_getsidbyuid(zone, uid, sid_prefix, rid)	(1)
-#define	kidmap_getsidbygid(zone, gid, sid_prefix, rid)	(1)
-
 #endif	/* _OPENSOLARIS_SYS_SID_H_ */

include/os/freebsd/spl/sys/vnode.h

@@ -132,7 +132,6 @@ vn_flush_cached_data(vnode_t *vp, boolean_t sync)
 /* TODO: This field needs conversion! */
 #define	va_nblocks	va_bytes
 #define	va_blksize	va_blocksize
-#define	va_seq		va_gen
 
 #define	MAXOFFSET_T	OFF_MAX
 #define	EXCL		0

include/os/linux/kernel/linux/simd_x86.h

@@ -136,9 +136,27 @@
  * When the kernel_fpu_* symbols are unavailable then provide our own
  * versions which allow the FPU to be safely used.
  */
+#if defined(HAVE_KERNEL_FPU_INTERNAL) || defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+/*
+ * Some sanity checks.
+ * HAVE_KERNEL_FPU_INTERNAL and HAVE_KERNEL_FPU_XSAVE_INTERNAL are exclusive.
+ */
 #if defined(HAVE_KERNEL_FPU_INTERNAL)
+#error "HAVE_KERNEL_FPU_INTERNAL and HAVE_KERNEL_FPU_XSAVE_INTERNAL defined"
+#endif
+/*
+ * For kernels >= 5.16 we have to use inline assembly with the XSAVE{,OPT,S}
+ * instructions, so we need the toolchain to support at least XSAVE.
+ */
+#if !defined(HAVE_XSAVE)
+#error "Toolchain needs to support the XSAVE assembler instruction"
+#endif
+#endif
 
 #include <linux/mm.h>
+#include <linux/slab.h>
 
 extern union fpregs_state **zfs_kfpu_fpregs;
 
@@ -191,7 +209,9 @@ kfpu_init(void)
 }
 
 #define	kfpu_allowed()		1
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 #define	ex_handler_fprestore	ex_handler_default
+#endif
 
 /*
  * FPU save and restore instructions.
@@ -206,6 +226,7 @@ kfpu_init(void)
 #define	kfpu_fxsr_clean(rval)	__asm("fnclex; emms; fildl %P[addr]" \
 				    : : [addr] "m" (rval));
 
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 static inline void
 kfpu_save_xsave(struct xregs_state *addr, uint64_t mask)
 {
@@ -217,6 +238,21 @@ kfpu_save_xsave(struct xregs_state *addr, uint64_t mask)
 	XSTATE_XSAVE(addr, low, hi, err);
 	WARN_ON_ONCE(err);
 }
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
+
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+#define	kfpu_do_xsave(instruction, addr, mask)			\
+{								\
+	uint32_t low, hi;					\
+								\
+	low = mask;						\
+	hi = (uint64_t)(mask) >> 32;				\
+	__asm(instruction " %[dst]\n\t"				\
+	    :							\
+	    : [dst] "m" (*(addr)), "a" (low), "d" (hi)		\
+	    : "memory");					\
+}
+#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
 
 static inline void
 kfpu_save_fxsr(struct fxregs_state *addr)
@@ -233,6 +269,7 @@ kfpu_save_fsave(struct fregs_state *addr)
 	kfpu_fnsave(addr);
 }
 
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 static inline void
 kfpu_begin(void)
 {
@@ -250,7 +287,6 @@ kfpu_begin(void)
 	 * FPU state to be correctly preserved and restored.
 	 */
 	union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()];
-
 	if (static_cpu_has(X86_FEATURE_XSAVE)) {
 		kfpu_save_xsave(&state->xsave, ~0);
 	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
@@ -259,7 +295,49 @@ kfpu_begin(void)
 		kfpu_save_fsave(&state->fsave);
 	}
 }
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
 
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+static inline void
+kfpu_begin(void)
+{
+	/*
+	 * Preemption and interrupts must be disabled for the critical
+	 * region where the FPU state is being modified.
+	 */
+	preempt_disable();
+	local_irq_disable();
+
+	/*
+	 * The current FPU registers need to be preserved by kfpu_begin()
+	 * and restored by kfpu_end().  They are stored in a dedicated
+	 * per-cpu variable, not in the task struct, this allows any user
+	 * FPU state to be correctly preserved and restored.
+	 */
+	union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()];
+#if defined(HAVE_XSAVES)
+	if (static_cpu_has(X86_FEATURE_XSAVES)) {
+		kfpu_do_xsave("xsaves", &state->xsave, ~0);
+		return;
+	}
+#endif
+#if defined(HAVE_XSAVEOPT)
+	if (static_cpu_has(X86_FEATURE_XSAVEOPT)) {
+		kfpu_do_xsave("xsaveopt", &state->xsave, ~0);
+		return;
+	}
+#endif
+	if (static_cpu_has(X86_FEATURE_XSAVE)) {
+		kfpu_do_xsave("xsave", &state->xsave, ~0);
+	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
+		kfpu_save_fxsr(&state->fxsave);
+	} else {
+		kfpu_save_fsave(&state->fsave);
+	}
+}
+#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
+
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 static inline void
 kfpu_restore_xsave(struct xregs_state *addr, uint64_t mask)
 {
@@ -269,6 +347,21 @@ kfpu_restore_xsave(struct xregs_state *addr, uint64_t mask)
 	hi = mask >> 32;
 	XSTATE_XRESTORE(addr, low, hi);
 }
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
+
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+#define	kfpu_do_xrstor(instruction, addr, mask)			\
+{								\
+	uint32_t low, hi;					\
+								\
+	low = mask;						\
+	hi = (uint64_t)(mask) >> 32;				\
+	__asm(instruction " %[src]"				\
+	    :							\
+	    : [src] "m" (*(addr)), "a" (low), "d" (hi)		\
+	    : "memory");					\
+}
+#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
 
 static inline void
 kfpu_restore_fxsr(struct fxregs_state *addr)
@@ -294,6 +387,7 @@ kfpu_restore_fsave(struct fregs_state *addr)
 	kfpu_frstor(addr);
 }
 
+#if defined(HAVE_KERNEL_FPU_INTERNAL)
 static inline void
 kfpu_end(void)
 {
@@ -310,6 +404,32 @@ kfpu_end(void)
 	local_irq_enable();
 	preempt_enable();
 }
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
+
+#if defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL)
+static inline void
+kfpu_end(void)
+{
+	union fpregs_state *state = zfs_kfpu_fpregs[smp_processor_id()];
+#if defined(HAVE_XSAVES)
+	if (static_cpu_has(X86_FEATURE_XSAVES)) {
+		kfpu_do_xrstor("xrstors", &state->xsave, ~0);
+		goto out;
+	}
+#endif
+	if (static_cpu_has(X86_FEATURE_XSAVE)) {
+		kfpu_do_xrstor("xrstor", &state->xsave, ~0);
+	} else if (static_cpu_has(X86_FEATURE_FXSR)) {
+		kfpu_restore_fxsr(&state->fxsave);
+	} else {
+		kfpu_restore_fsave(&state->fsave);
+	}
+out:
+	local_irq_enable();
+	preempt_enable();
+
+}
+#endif /* defined(HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
 
 #else
 
@@ -322,7 +442,7 @@ kfpu_end(void)
 #define	kfpu_init()		0
 #define	kfpu_fini()		((void) 0)
 
-#endif /* defined(HAVE_KERNEL_FPU_INTERNAL) */
+#endif /* defined(HAVE_KERNEL_FPU_INTERNAL || HAVE_KERNEL_FPU_XSAVE_INTERNAL) */
 #endif /* defined(KERNEL_EXPORTS_X86_FPU) */
 
 /*

include/os/linux/spl/sys/cred.h

@@ -49,12 +49,7 @@ extern void crhold(cred_t *cr);
 extern void crfree(cred_t *cr);
 extern uid_t crgetuid(const cred_t *cr);
 extern uid_t crgetruid(const cred_t *cr);
-extern uid_t crgetsuid(const cred_t *cr);
-extern uid_t crgetfsuid(const cred_t *cr);
 extern gid_t crgetgid(const cred_t *cr);
-extern gid_t crgetrgid(const cred_t *cr);
-extern gid_t crgetsgid(const cred_t *cr);
-extern gid_t crgetfsgid(const cred_t *cr);
 extern int crgetngroups(const cred_t *cr);
 extern gid_t *crgetgroups(const cred_t *cr);
 extern int groupmember(gid_t gid, const cred_t *cr);

include/os/linux/spl/sys/uio.h

@@ -34,6 +34,10 @@
 #include <asm/uaccess.h>
 #include <sys/types.h>
 
+#if defined(HAVE_VFS_IOV_ITER) && defined(HAVE_FAULT_IN_IOV_ITER_READABLE)
+#define	iov_iter_fault_in_readable(a, b)	fault_in_iov_iter_readable(a, b)
+#endif
+
 typedef struct iovec iovec_t;
 
 typedef enum zfs_uio_rw {

include/sys/dsl_pool.h

@@ -158,6 +158,7 @@ int dsl_pool_sync_context(dsl_pool_t *dp);
 uint64_t dsl_pool_adjustedsize(dsl_pool_t *dp, zfs_space_check_t slop_policy);
 uint64_t dsl_pool_unreserved_space(dsl_pool_t *dp,
     zfs_space_check_t slop_policy);
+uint64_t dsl_pool_deferred_space(dsl_pool_t *dp);
 void dsl_pool_dirty_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx);
 void dsl_pool_undirty_space(dsl_pool_t *dp, int64_t space, uint64_t txg);
 void dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp);

include/sys/fm/fs/zfs.h

@@ -110,6 +110,10 @@ extern "C" {
 #define	FM_EREPORT_PAYLOAD_ZFS_BAD_CLEARED_BITS	"bad_cleared_bits"
 #define	FM_EREPORT_PAYLOAD_ZFS_BAD_SET_HISTOGRAM "bad_set_histogram"
 #define	FM_EREPORT_PAYLOAD_ZFS_BAD_CLEARED_HISTOGRAM "bad_cleared_histogram"
+#define	FM_EREPORT_PAYLOAD_ZFS_SNAPSHOT_NAME	"snapshot_name"
+#define	FM_EREPORT_PAYLOAD_ZFS_DEVICE_NAME	"device_name"
+#define	FM_EREPORT_PAYLOAD_ZFS_RAW_DEVICE_NAME	"raw_name"
+#define	FM_EREPORT_PAYLOAD_ZFS_VOLUME	"volume"
 
 #define	FM_EREPORT_FAILMODE_WAIT		"wait"
 #define	FM_EREPORT_FAILMODE_CONTINUE		"continue"
@@ -119,6 +123,11 @@ extern "C" {
 #define	FM_RESOURCE_AUTOREPLACE			"autoreplace"
 #define	FM_RESOURCE_STATECHANGE			"statechange"
 
+#define	FM_RESOURCE_ZFS_SNAPSHOT_MOUNT		"snapshot_mount"
+#define	FM_RESOURCE_ZFS_SNAPSHOT_UNMOUNT		"snapshot_unmount"
+#define	FM_RESOURCE_ZVOL_CREATE_SYMLINK		"zvol_create"
+#define	FM_RESOURCE_ZVOL_REMOVE_SYMLINK		"zvol_remove"
+
 #ifdef	__cplusplus
 }
 #endif

include/sys/spa.h

@@ -1189,6 +1189,8 @@ extern void spa_configfile_set(spa_t *, nvlist_t *, boolean_t);
 /* asynchronous event notification */
 extern void spa_event_notify(spa_t *spa, vdev_t *vdev, nvlist_t *hist_nvl,
     const char *name);
+extern void zfs_ereport_zvol_post(const char *subclass, const char *name,
+    const char *device_name, const char *raw_name);
 
 /* waiting for pool activities to complete */
 extern int spa_wait(const char *pool, zpool_wait_activity_t activity,

include/sys/vdev_draid.h

@@ -96,6 +96,7 @@ extern boolean_t vdev_draid_readable(vdev_t *, uint64_t);
 extern boolean_t vdev_draid_missing(vdev_t *, uint64_t, uint64_t, uint64_t);
 extern uint64_t vdev_draid_asize_to_psize(vdev_t *, uint64_t);
 extern void vdev_draid_map_alloc_empty(zio_t *, struct raidz_row *);
+extern int vdev_draid_map_verify_empty(zio_t *, struct raidz_row *);
 extern nvlist_t *vdev_draid_read_config_spare(vdev_t *);
 
 /* Functions for dRAID distributed spares. */

include/sys/vdev_raidz.h

@@ -32,6 +32,7 @@ extern "C" {
 #endif
 
 struct zio;
+struct raidz_col;
 struct raidz_row;
 struct raidz_map;
 #if !defined(_KERNEL)
@@ -49,6 +50,7 @@ void vdev_raidz_generate_parity(struct raidz_map *);
 void vdev_raidz_reconstruct(struct raidz_map *, const int *, int);
 void vdev_raidz_child_done(zio_t *);
 void vdev_raidz_io_done(zio_t *);
+void vdev_raidz_checksum_error(zio_t *, struct raidz_col *, abd_t *);
 
 extern const zio_vsd_ops_t vdev_raidz_vsd_ops;
 

include/sys/zio.h

@@ -690,6 +690,8 @@ extern int zfs_ereport_post_checksum(spa_t *spa, vdev_t *vd,
     struct zio_bad_cksum *info);
 
 void zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr);
+extern void zfs_ereport_snapshot_post(const char *subclass, spa_t *spa,
+    const char *name);
 
 /* Called from spa_sync(), but primarily an injection handler */
 extern void spa_handle_ignored_writes(spa_t *spa);

include/sys/zthr.h

@@ -38,6 +38,7 @@ extern void zthr_resume(zthr_t *t);
 extern void zthr_wait_cycle_done(zthr_t *t);
 
 extern boolean_t zthr_iscancelled(zthr_t *t);
+extern boolean_t zthr_iscurthread(zthr_t *t);
 extern boolean_t zthr_has_waiters(zthr_t *t);
 
 #endif /* _SYS_ZTHR_H */

lib/libshare/os/freebsd/nfs.c

@@ -423,9 +423,10 @@ nfs_commit_shares(void)
 	struct pidfh *pfh;
 	pid_t mountdpid;
 
+start:
 	pfh = pidfile_open(_PATH_MOUNTDPID, 0600, &mountdpid);
 	if (pfh != NULL) {
-		/* Mountd is not running. */
+		/* mountd(8) is not running. */
 		pidfile_remove(pfh);
 		return (SA_OK);
 	}
@@ -433,6 +434,11 @@ nfs_commit_shares(void)
 		/* Cannot open pidfile for some reason. */
 		return (SA_SYSTEM_ERR);
 	}
+	if (mountdpid == -1) {
+		/* mountd(8) exists, but didn't write the PID yet */
+		usleep(500);
+		goto start;
+	}
 	/* We have mountd(8) PID in mountdpid variable. */
 	kill(mountdpid, SIGHUP);
 	return (SA_OK);

lib/libspl/include/sys/feature_tests.h

@@ -30,7 +30,7 @@
 #define	____cacheline_aligned
 #define	__NORETURN		__attribute__((__noreturn__))
 
-#if !defined(fallthrough)
+#if !defined(fallthrough) && !defined(_LIBCPP_VERSION)
 #if defined(HAVE_IMPLICIT_FALLTHROUGH)
 #define	fallthrough		__attribute__((__fallthrough__))
 #else

lib/libspl/os/freebsd/mnttab.c

@@ -91,16 +91,28 @@ optadd(char *mntopts, size_t size, const char *opt)
 	strlcat(mntopts, opt, size);
 }
 
+static __thread char gfstypename[MFSNAMELEN];
+static __thread char gmntfromname[MNAMELEN];
+static __thread char gmntonname[MNAMELEN];
+static __thread char gmntopts[MNTMAXSTR];
+
 void
 statfs2mnttab(struct statfs *sfs, struct mnttab *mp)
 {
-	static char mntopts[MNTMAXSTR];
 	long flags;
 
-	mntopts[0] = '\0';
+	strlcpy(gfstypename, sfs->f_fstypename, sizeof (gfstypename));
+	mp->mnt_fstype = gfstypename;
+
+	strlcpy(gmntfromname, sfs->f_mntfromname, sizeof (gmntfromname));
+	mp->mnt_special = gmntfromname;
+
+	strlcpy(gmntonname, sfs->f_mntonname, sizeof (gmntonname));
+	mp->mnt_mountp = gmntonname;
 
 	flags = sfs->f_flags;
-#define	OPTADD(opt)	optadd(mntopts, sizeof (mntopts), (opt))
+	gmntopts[0] = '\0';
+#define	OPTADD(opt)	optadd(gmntopts, sizeof (gmntopts), (opt))
 	if (flags & MNT_RDONLY)
 		OPTADD(MNTOPT_RO);
 	else
@@ -121,10 +133,7 @@ statfs2mnttab(struct statfs *sfs, struct mnttab *mp)
 	else
 		OPTADD(MNTOPT_EXEC);
 #undef	OPTADD
-	mp->mnt_special = strdup(sfs->f_mntfromname);
+	mp->mnt_mntopts = gmntopts;
-	mp->mnt_mountp = strdup(sfs->f_mntonname);
-	mp->mnt_fstype = strdup(sfs->f_fstypename);
-	mp->mnt_mntopts = strdup(mntopts);
 }
 
 static struct statfs *gsfs = NULL;
@@ -166,7 +175,6 @@ fail:
 int
 getmntany(FILE *fd __unused, struct mnttab *mgetp, struct mnttab *mrefp)
 {
-	//	struct statfs *sfs;
 	int i, error;
 
 	error = statfs_init();
@@ -195,7 +203,6 @@ getmntany(FILE *fd __unused, struct mnttab *mgetp, struct mnttab *mrefp)
 int
 getmntent(FILE *fp, struct mnttab *mp)
 {
-	//	struct statfs *sfs;
 	int error, nfs;
 
 	nfs = (int)lseek(fileno(fp), 0, SEEK_CUR);

lib/libzfs/Makefile.am

@@ -75,7 +75,7 @@ libzfs_la_LIBADD = \
 	$(abs_top_builddir)/lib/libnvpair/libnvpair.la \
 	$(abs_top_builddir)/lib/libuutil/libuutil.la
 
-libzfs_la_LIBADD += -lm $(LIBCRYPTO_LIBS) $(ZLIB_LIBS) $(LTLIBINTL)
+libzfs_la_LIBADD += -lm $(LIBCRYPTO_LIBS) $(ZLIB_LIBS) $(LIBFETCH_LIBS) $(LTLIBINTL)
 
 libzfs_la_LDFLAGS = -pthread
 

lib/libzfs/libzfs.abi

@@ -551,10 +551,6 @@
       <parameter type-id='e1c52942'/>
       <return type-id='95e97e5e'/>
     </function-decl>
-    <function-decl name='unlink' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='80f4b756'/>
-      <return type-id='95e97e5e'/>
-    </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='os/linux/smb.c' language='LANG_C99'>
     <array-type-def dimensions='1' type-id='a84c031d' size-in-bits='2040' id='11641789'>
@@ -1373,7 +1369,7 @@
     <typedef-decl name='zpool_handle_t' type-id='67002a8a' id='b1efc708'/>
     <typedef-decl name='libzfs_handle_t' type-id='c8a9d9d8' id='95942d0c'/>
     <typedef-decl name='zfs_iter_f' type-id='5571cde4' id='d8e49ab9'/>
-    <class-decl name='libzfs_handle' size-in-bits='20224' is-struct='yes' visibility='default' id='c8a9d9d8'>
+    <class-decl name='libzfs_handle' size-in-bits='20352' is-struct='yes' visibility='default' id='c8a9d9d8'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='libzfs_error' type-id='95e97e5e' visibility='default'/>
       </data-member>
@@ -1434,6 +1430,12 @@
       <data-member access='public' layout-offset-in-bits='20160'>
         <var-decl name='libzfs_max_nvlist' type-id='9c313c2d' visibility='default'/>
       </data-member>
+      <data-member access='public' layout-offset-in-bits='20224'>
+        <var-decl name='libfetch' type-id='eaa32e2f' visibility='default'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='20288'>
+        <var-decl name='libfetch_load_error' type-id='26a90f95' visibility='default'/>
+      </data-member>
     </class-decl>
     <class-decl name='zfs_handle' size-in-bits='4928' is-struct='yes' visibility='default' id='f6ee4445'>
       <data-member access='public' layout-offset-in-bits='0'>
@@ -3190,6 +3192,19 @@
     <function-decl name='__ctype_b_loc' visibility='default' binding='global' size-in-bits='64'>
       <return type-id='c59e1ef0'/>
     </function-decl>
+    <function-decl name='dlopen' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <parameter type-id='95e97e5e'/>
+      <return type-id='eaa32e2f'/>
+    </function-decl>
+    <function-decl name='dlsym' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='1b7446cd'/>
+      <parameter type-id='9d26089a'/>
+      <return type-id='eaa32e2f'/>
+    </function-decl>
+    <function-decl name='dlerror' visibility='default' binding='global' size-in-bits='64'>
+      <return type-id='26a90f95'/>
+    </function-decl>
     <function-decl name='PKCS5_PBKDF2_HMAC_SHA1' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='80f4b756'/>
       <parameter type-id='95e97e5e'/>
@@ -3231,6 +3246,11 @@
       <parameter type-id='822cd80b'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='fdopen' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='95e97e5e'/>
+      <parameter type-id='80f4b756'/>
+      <return type-id='822cd80b'/>
+    </function-decl>
     <function-decl name='printf' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='80f4b756'/>
       <parameter is-variadic='yes'/>
@@ -3243,6 +3263,12 @@
       <parameter is-variadic='yes'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='asprintf' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='8c85230f'/>
+      <parameter type-id='9d26089a'/>
+      <parameter is-variadic='yes'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='fputc' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='95e97e5e'/>
       <parameter type-id='822cd80b'/>
@@ -3262,6 +3288,10 @@
       <parameter type-id='e75a27e9'/>
       <return type-id='b59d7dce'/>
     </function-decl>
+    <function-decl name='rewind' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='822cd80b'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='ferror' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='822cd80b'/>
       <return type-id='95e97e5e'/>
@@ -3285,6 +3315,10 @@
       <parameter type-id='b59d7dce'/>
       <return type-id='eaa32e2f'/>
     </function-decl>
+    <function-decl name='strdup' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <return type-id='26a90f95'/>
+    </function-decl>
     <function-decl name='strerror' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='95e97e5e'/>
       <return type-id='26a90f95'/>
@@ -3317,6 +3351,10 @@
       <parameter type-id='95e97e5e'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='unlink' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-type size-in-bits='64' id='ee076206'>
       <return type-id='48b5725f'/>
     </function-type>
@@ -4425,12 +4463,6 @@
       <parameter is-variadic='yes'/>
       <return type-id='95e97e5e'/>
     </function-decl>
-    <function-decl name='asprintf' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='8c85230f'/>
-      <parameter type-id='9d26089a'/>
-      <parameter is-variadic='yes'/>
-      <return type-id='95e97e5e'/>
-    </function-decl>
     <function-decl name='strtol' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='9d26089a'/>
       <parameter type-id='8c85230f'/>
@@ -4452,10 +4484,6 @@
       <parameter type-id='b59d7dce'/>
       <return type-id='26a90f95'/>
     </function-decl>
-    <function-decl name='strdup' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='80f4b756'/>
-      <return type-id='26a90f95'/>
-    </function-decl>
     <function-decl name='strrchr' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='80f4b756'/>
       <parameter type-id='95e97e5e'/>
@@ -4620,11 +4648,6 @@
       <parameter type-id='4051f5e7'/>
       <return type-id='95e97e5e'/>
     </function-decl>
-    <function-decl name='fdopen' visibility='default' binding='global' size-in-bits='64'>
-      <parameter type-id='95e97e5e'/>
-      <parameter type-id='80f4b756'/>
-      <return type-id='822cd80b'/>
-    </function-decl>
     <function-decl name='pipe2' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='7292109c'/>
       <parameter type-id='95e97e5e'/>
@@ -6705,6 +6728,12 @@
       <parameter type-id='95e97e5e'/>
       <return type-id='48b5725f'/>
     </function-decl>
+    <function-decl name='avl_insert' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='a3681dea'/>
+      <parameter type-id='eaa32e2f'/>
+      <parameter type-id='fba6cb51'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='nvlist_lookup_boolean' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='5ce45b60'/>
       <parameter type-id='80f4b756'/>
@@ -7235,6 +7264,10 @@
     <function-decl name='__ctype_toupper_loc' visibility='default' binding='global' size-in-bits='64'>
       <return type-id='24f95ba5'/>
     </function-decl>
+    <function-decl name='dlclose' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='eaa32e2f'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='regcomp' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='5c53ba29'/>
       <parameter type-id='9d26089a'/>

lib/libzfs/libzfs_crypto.c

@@ -26,6 +26,16 @@
 #include <signal.h>
 #include <errno.h>
 #include <openssl/evp.h>
+#if LIBFETCH_DYNAMIC
+#include <dlfcn.h>
+#endif
+#if LIBFETCH_IS_FETCH
+#include <sys/param.h>
+#include <stdio.h>
+#include <fetch.h>
+#elif LIBFETCH_IS_LIBCURL
+#include <curl/curl.h>
+#endif
 #include <libzfs.h>
 #include "libzfs_impl.h"
 #include "zfeature_common.h"
@@ -59,9 +69,13 @@ static int caught_interrupt;
 
 static int get_key_material_file(libzfs_handle_t *, const char *, const char *,
     zfs_keyformat_t, boolean_t, uint8_t **, size_t *);
+static int get_key_material_https(libzfs_handle_t *, const char *, const char *,
+    zfs_keyformat_t, boolean_t, uint8_t **, size_t *);
 
 static zfs_uri_handler_t uri_handlers[] = {
 	{ "file", get_key_material_file },
+	{ "https", get_key_material_https },
+	{ "http", get_key_material_https },
 	{ NULL, NULL }
 };
 
@@ -483,6 +497,177 @@ get_key_material_file(libzfs_handle_t *hdl, const char *uri,
 	return (ret);
 }
 
+static int
+get_key_material_https(libzfs_handle_t *hdl, const char *uri,
+    const char *fsname, zfs_keyformat_t keyformat, boolean_t newkey,
+    uint8_t **restrict buf, size_t *restrict len_out)
+{
+	int ret = 0;
+	FILE *key = NULL;
+	boolean_t is_http = strncmp(uri, "http:", strlen("http:")) == 0;
+
+	if (strlen(uri) < (is_http ? 7 : 8)) {
+		ret = EINVAL;
+		goto end;
+	}
+
+#if LIBFETCH_DYNAMIC
+#define	LOAD_FUNCTION(func) \
+	__typeof__(func) *func = dlsym(hdl->libfetch, #func);
+
+	if (hdl->libfetch == NULL)
+		hdl->libfetch = dlopen(LIBFETCH_SONAME, RTLD_LAZY);
+
+	if (hdl->libfetch == NULL) {
+		hdl->libfetch = (void *)-1;
+		char *err = dlerror();
+		if (err)
+			hdl->libfetch_load_error = strdup(err);
+	}
+
+	if (hdl->libfetch == (void *)-1) {
+		ret = ENOSYS;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Couldn't load %s: %s"),
+		    LIBFETCH_SONAME, hdl->libfetch_load_error ?: "(?)");
+		goto end;
+	}
+
+	boolean_t ok;
+#if LIBFETCH_IS_FETCH
+	LOAD_FUNCTION(fetchGetURL);
+	char *fetchLastErrString = dlsym(hdl->libfetch, "fetchLastErrString");
+
+	ok = fetchGetURL && fetchLastErrString;
+#elif LIBFETCH_IS_LIBCURL
+	LOAD_FUNCTION(curl_easy_init);
+	LOAD_FUNCTION(curl_easy_setopt);
+	LOAD_FUNCTION(curl_easy_perform);
+	LOAD_FUNCTION(curl_easy_cleanup);
+	LOAD_FUNCTION(curl_easy_strerror);
+	LOAD_FUNCTION(curl_easy_getinfo);
+
+	ok = curl_easy_init && curl_easy_setopt && curl_easy_perform &&
+	    curl_easy_cleanup && curl_easy_strerror && curl_easy_getinfo;
+#endif
+	if (!ok) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "keylocation=%s back-end %s missing symbols."),
+		    is_http ? "http://" : "https://", LIBFETCH_SONAME);
+		ret = ENOSYS;
+		goto end;
+	}
+#endif
+
+#if LIBFETCH_IS_FETCH
+	key = fetchGetURL(uri, "");
+	if (key == NULL) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Couldn't GET %s: %s"),
+		    uri, fetchLastErrString);
+		ret = ENETDOWN;
+	}
+#elif LIBFETCH_IS_LIBCURL
+	CURL *curl = curl_easy_init();
+	if (curl == NULL) {
+		ret = ENOTSUP;
+		goto end;
+	}
+
+	int kfd = -1;
+#ifdef O_TMPFILE
+	kfd = open(getenv("TMPDIR") ?: "/tmp",
+	    O_RDWR | O_TMPFILE | O_EXCL | O_CLOEXEC, 0600);
+	if (kfd != -1)
+		goto kfdok;
+#endif
+
+	char *path;
+	if (asprintf(&path,
+	    "%s/libzfs-XXXXXXXX.https", getenv("TMPDIR") ?: "/tmp") == -1) {
+		ret = ENOMEM;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "%s"),
+		    strerror(ret));
+		goto end;
+	}
+
+	kfd = mkostemps(path, strlen(".https"), O_CLOEXEC);
+	if (kfd == -1) {
+		ret = errno;
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Couldn't create temporary file %s: %s"),
+		    path, strerror(ret));
+		free(path);
+		goto end;
+	}
+	(void) unlink(path);
+	free(path);
+
+kfdok:
+	if ((key = fdopen(kfd, "r+")) == NULL) {
+		ret = errno;
+		(void) close(kfd);
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Couldn't reopen temporary file: %s"), strerror(ret));
+		goto end;
+	}
+
+	char errbuf[CURL_ERROR_SIZE] = "";
+	char *cainfo = getenv("SSL_CA_CERT_FILE"); /* matches fetch(3) */
+	char *capath = getenv("SSL_CA_CERT_PATH"); /* matches fetch(3) */
+	char *clcert = getenv("SSL_CLIENT_CERT_FILE"); /* matches fetch(3) */
+	char *clkey  = getenv("SSL_CLIENT_KEY_FILE"); /* matches fetch(3) */
+	(void) curl_easy_setopt(curl, CURLOPT_URL, uri);
+	(void) curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+	(void) curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 30000L);
+	(void) curl_easy_setopt(curl, CURLOPT_WRITEDATA, key);
+	(void) curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf);
+	if (cainfo != NULL)
+		(void) curl_easy_setopt(curl, CURLOPT_CAINFO, cainfo);
+	if (capath != NULL)
+		(void) curl_easy_setopt(curl, CURLOPT_CAPATH, capath);
+	if (clcert != NULL)
+		(void) curl_easy_setopt(curl, CURLOPT_SSLCERT, clcert);
+	if (clkey != NULL)
+		(void) curl_easy_setopt(curl, CURLOPT_SSLKEY, clkey);
+
+	CURLcode res = curl_easy_perform(curl);
+
+	if (res != CURLE_OK) {
+		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+		    "Failed to connect to %s: %s"),
+		    uri, strlen(errbuf) ? errbuf : curl_easy_strerror(res));
+		ret = ENETDOWN;
+	} else {
+		long resp = 200;
+		(void) curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &resp);
+
+		if (resp < 200 || resp >= 300) {
+			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+			    "Couldn't GET %s: %ld"),
+			    uri, resp);
+			ret = ENOENT;
+		} else
+			rewind(key);
+	}
+
+	curl_easy_cleanup(curl);
+#else
+	zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+	    "No keylocation=%s back-end."), is_http ? "http://" : "https://");
+	ret = ENOSYS;
+#endif
+
+end:
+	if (ret == 0)
+		ret = get_key_material_raw(key, keyformat, buf, len_out);
+
+	if (key != NULL)
+		fclose(key);
+
+	return (ret);
+}
+
 /*
  * Attempts to fetch key material, no matter where it might live. The key
  * material is allocated and returned in km_out. *can_retry_out will be set

lib/libzfs/libzfs_sendrecv.c

@@ -206,8 +206,9 @@ fsavl_create(nvlist_t *fss)
 			 * Note: if there are multiple snaps with the
 			 * same GUID, we ignore all but one.
 			 */
-			if (avl_find(fsavl, fn, NULL) == NULL)
+			avl_index_t where = 0;
-				avl_add(fsavl, fn);
+			if (avl_find(fsavl, fn, &where) == NULL)
+				avl_insert(fsavl, fn, where);
 			else
 				free(fn);
 		}
@@ -350,12 +351,11 @@ send_iterate_snap(zfs_handle_t *zhp, void *arg)
 	fnvlist_add_nvlist(sd->snapprops, snapname, nv);
 	fnvlist_free(nv);
 	if (sd->holds) {
-		nvlist_t *holds = fnvlist_alloc();
+		nvlist_t *holds;
-		int err = lzc_get_holds(zhp->zfs_name, &holds);
+		if (lzc_get_holds(zhp->zfs_name, &holds) == 0) {
-		if (err == 0) {
 			fnvlist_add_nvlist(sd->snapholds, snapname, holds);
+			fnvlist_free(holds);
 		}
-		fnvlist_free(holds);
 	}
 
 	zfs_close(zhp);
@@ -2532,7 +2532,7 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 			    "progress thread exited nonzero")));
 	}
 
-	if (flags->props || flags->holds || flags->backup) {
+	if (err == 0 && (flags->props || flags->holds || flags->backup)) {
 		/* Write the final end record. */
 		err = send_conclusion_record(fd, NULL);
 		if (err != 0)

lib/libzfs/libzfs_util.c

@@ -44,6 +44,9 @@
 #include <strings.h>
 #include <unistd.h>
 #include <math.h>
+#if LIBFETCH_DYNAMIC
+#include <dlfcn.h>
+#endif
 #include <sys/stat.h>
 #include <sys/mnttab.h>
 #include <sys/mntent.h>
@@ -1101,6 +1104,11 @@ libzfs_fini(libzfs_handle_t *hdl)
 	libzfs_core_fini();
 	regfree(&hdl->libzfs_urire);
 	fletcher_4_fini();
+#if LIBFETCH_DYNAMIC
+	if (hdl->libfetch != (void *)-1 && hdl->libfetch != NULL)
+		(void) dlclose(hdl->libfetch);
+	free(hdl->libfetch_load_error);
+#endif
 	free(hdl);
 }
 

lib/libzfs/os/linux/libzfs_mount_os.c

@@ -327,7 +327,7 @@ do_mount(zfs_handle_t *zhp, const char *mntpt, char *opts, int flags)
 
 	if (!libzfs_envvar_is_set("ZFS_MOUNT_HELPER")) {
 		char badopt[MNT_LINE_MAX] = {0};
-		unsigned long mntflags = flags, zfsflags;
+		unsigned long mntflags = flags, zfsflags = 0;
 		char myopts[MNT_LINE_MAX] = {0};
 
 		if (zfs_parse_mount_options(opts, &mntflags,

lib/libzutil/os/linux/zutil_device_path_os.c

@@ -527,7 +527,7 @@ zfs_dev_is_dm(const char *dev_name)
 boolean_t
 zfs_dev_is_whole_disk(const char *dev_name)
 {
-	struct dk_gpt *label;
+	struct dk_gpt *label = NULL;
 	int fd;
 
 	if ((fd = open(dev_name, O_RDONLY | O_DIRECT | O_CLOEXEC)) < 0)
@@ -613,22 +613,24 @@ zfs_get_underlying_path(const char *dev_name)
 /*
  * A disk is considered a multipath whole disk when:
  *	DEVNAME key value has "dm-"
- *	DM_NAME key value has "mpath" prefix
+ *	DM_UUID key exists and starts with 'mpath-'
- *	DM_UUID key exists
  *	ID_PART_TABLE_TYPE key does not exist or is not gpt
+ *	ID_FS_LABEL key does not exist (disk isn't labeled)
  */
 static boolean_t
-udev_mpath_whole_disk(struct udev_device *dev)
+is_mpath_udev_sane(struct udev_device *dev)
 {
-	const char *devname, *type, *uuid;
+	const char *devname, *type, *uuid, *label;
 
 	devname = udev_device_get_property_value(dev, "DEVNAME");
 	type = udev_device_get_property_value(dev, "ID_PART_TABLE_TYPE");
 	uuid = udev_device_get_property_value(dev, "DM_UUID");
+	label = udev_device_get_property_value(dev, "ID_FS_LABEL");
 
 	if ((devname != NULL && strncmp(devname, "/dev/dm-", 8) == 0) &&
 	    ((type == NULL) || (strcmp(type, "gpt") != 0)) &&
-	    (uuid != NULL)) {
+	    ((uuid != NULL) && (strncmp(uuid, "mpath-", 6) == 0)) &&
+	    (label == NULL)) {
 		return (B_TRUE);
 	}
 
@@ -636,7 +638,11 @@ udev_mpath_whole_disk(struct udev_device *dev)
 }
 
 /*
- * Check if a disk is effectively a multipath whole disk
+ * Check if a disk is a multipath "blank" disk:
+ *
+ * 1. The disk has udev values that suggest it's a multipath disk
+ * 2. The disk is not currently labeled with a filesystem of any type
+ * 3. There are no partitions on the disk
  */
 boolean_t
 is_mpath_whole_disk(const char *path)
@@ -645,7 +651,6 @@ is_mpath_whole_disk(const char *path)
 	struct udev_device *dev = NULL;
 	char nodepath[MAXPATHLEN];
 	char *sysname;
-	boolean_t wholedisk = B_FALSE;
 
 	if (realpath(path, nodepath) == NULL)
 		return (B_FALSE);
@@ -660,10 +665,11 @@ is_mpath_whole_disk(const char *path)
 		return (B_FALSE);
 	}
 
-	wholedisk = udev_mpath_whole_disk(dev);
+	/* Sanity check some udev values */
-
+	boolean_t is_sane = is_mpath_udev_sane(dev);
 	udev_device_unref(dev);
-	return (wholedisk);
+
+	return (is_sane);
 }
 
 #else /* HAVE_LIBUDEV */

man/man4/zfs.4

@@ -671,6 +671,13 @@ Minimum time "prescient prefetched" blocks are locked in the ARC.
 These blocks are meant to be prefetched fairly aggressively ahead of
 the code that may use them.
 .
+.It Sy zfs_arc_prune_task_threads Ns = Ns Sy 1 Pq int
+Number of arc_prune threads.
+.Fx
+does not need more than one.
+Linux may theoretically use one per mount point up to number of CPUs,
+but that was not proven to be useful.
+.
 .It Sy zfs_max_missing_tvds Ns = Ns Sy 0 Pq int
 Number of missing top-level vdevs which will be allowed during
 pool import (only in read-only mode).

man/man7/zfsprops.7

@@ -909,7 +909,7 @@ ratio; for example, 8kB blocks on disks with 4kB disk sectors must compress to 1
 or less of their original size.
 .It Xo
 .Sy context Ns = Ns Sy none Ns | Ns
-.Ar SELinux-User : Ns Ar SElinux-Role : Ns Ar Selinux-Type : Ns Ar Sensitivity-Level
+.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level
 .Xc
 This flag sets the SELinux context for all files in the file system under
 a mount point for that file system.
@@ -918,7 +918,7 @@ See
 for more information.
 .It Xo
 .Sy fscontext Ns = Ns Sy none Ns | Ns
-.Ar SELinux-User : Ns Ar SElinux-Role : Ns Ar Selinux-Type : Ns Ar Sensitivity-Level
+.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level
 .Xc
 This flag sets the SELinux context for the file system file system being
 mounted.
@@ -927,7 +927,7 @@ See
 for more information.
 .It Xo
 .Sy defcontext Ns = Ns Sy none Ns | Ns
-.Ar SELinux-User : Ns Ar SElinux-Role : Ns Ar Selinux-Type : Ns Ar Sensitivity-Level
+.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level
 .Xc
 This flag sets the SELinux default context for unlabeled files.
 See
@@ -935,7 +935,7 @@ See
 for more information.
 .It Xo
 .Sy rootcontext Ns = Ns Sy none Ns | Ns
-.Ar SELinux-User : Ns Ar SElinux-Role : Ns Ar Selinux-Type : Ns Ar Sensitivity-Level
+.Ar SELinux-User : Ns Ar SELinux-Role : Ns Ar SELinux-Type : Ns Ar Sensitivity-Level
 .Xc
 This flag sets the SELinux context for the root inode of the file system.
 See
@@ -1099,7 +1099,7 @@ Even though the encryption suite cannot be changed after dataset creation,
 the keyformat can be with
 .Nm zfs Cm change-key .
 .It Xo
-.Sy keylocation Ns = Ns Sy prompt Ns | Ns Sy file:// Ns Em </absolute/file/path>
+.Sy keylocation Ns = Ns Sy prompt Ns | Ns Sy file:// Ns Em </absolute/file/path> Ns | Ns Sy https:// Ns Em <address> | Ns Sy http:// Ns Em <address>
 .Xc
 Controls where the user's encryption key will be loaded from by default for
 commands such as
@@ -1126,6 +1126,24 @@ but users should be careful not to place keys which should be kept secret on
 the command line.
 If a file URI is selected, the key will be loaded from the
 specified absolute file path.
+If an HTTPS or HTTP URL is selected, it will be GETted using
+.Xr fetch 3 ,
+libcurl, or nothing, depending on compile-time configuration and run-time
+availability.
+The
+.Ev SSL_CA_CERT_FILE
+environment variable can be set to set the location
+of the concatenated certificate store.
+The
+.Ev SSL_CA_CERT_PATH
+environment variable can be set to override the location
+of the directory containing the certificate authority bundle.
+The
+.Ev SSL_CLIENT_CERT_FILE
+and
+.Ev SSL_CLIENT_KEY_FILE
+environment variables can be set to configure the path
+to the client certificate and its key.
 .It Sy pbkdf2iters Ns = Ns Ar iterations
 Controls the number of PBKDF2 iterations that a
 .Sy passphrase
@@ -1623,6 +1641,11 @@ If the property is set to
 the dataset is shared using the default options:
 .Dl sec=sys,rw,crossmnt,no_subtree_check
 .Pp
+Please note that the options are comma-separated, unlike those found in
+.Xr exports 5 .
+This is done to negate the need for quoting, as well as to make parsing
+with scripts easier.
+.Pp
 See
 .Xr exports 5
 for the meaning of the default options.

man/man7/zpoolprops.7

@@ -101,6 +101,10 @@ Over time
 will decrease while
 .Sy free
 increases.
+.It Sy leaked
+Space not released while
+.Sy freeing
+due to corruption, now permanently leaked into the pool.
 .It Sy health
 The current health of the pool.
 Health can be one of
@@ -337,7 +341,8 @@ The behavior of such an event is determined as follows:
 .Bl -tag -width "continue"
 .It Sy wait
 Blocks all I/O access until the device connectivity is recovered and the errors
-are cleared.
+are cleared with
+.Nm zpool Cm clear .
 This is the default behavior.
 .It Sy continue
 Returns

man/man8/zfs-load-key.8

@@ -200,7 +200,7 @@ if supported by your hardware, otherwise
 .Bl -tag -width "-r"
 .It Fl l
 Ensures the key is loaded before attempting to change the key.
-This is effectively equivalent to runnin
+This is effectively equivalent to running
 .Nm zfs Cm load-key Ar filesystem ; Nm zfs Cm change-key Ar filesystem
 .It Fl o Ar property Ns = Ns Ar value
 Allows the user to set encryption key properties

man/man8/zfs-receive.8

@@ -319,7 +319,7 @@ received as an encryption root, specify encryption properties in the same
 manner as is required for
 .Nm zfs Cm create .
 For instance:
-.Dl # Nm zfs Cm send Pa tank/test@snap1 | Nm zfs Cm recv Fl o Sy encryption Ns = Ns Sy on Fl o keyformat=passphrase Fl o Sy keylocation Ns = Ns Pa file:///path/to/keyfile
+.Dl # Nm zfs Cm send Pa tank/test@snap1 | Nm zfs Cm recv Fl o Sy encryption Ns = Ns Sy on Fl o Sy keyformat Ns = Ns Sy passphrase Fl o Sy keylocation Ns = Ns Pa file:///path/to/keyfile
 .Pp
 Note that
 .Fl o Sy keylocation Ns = Ns Sy prompt
@@ -329,7 +329,7 @@ Once the receive has completed, you can use
 .Nm zfs Cm set
 to change this setting after the fact.
 Similarly, you can receive a dataset as an encrypted child by specifying
-.Op Fl x Ar encryption
+.Fl x Sy encryption
 to force the property to be inherited.
 Overriding encryption properties (except for
 .Sy keylocation )

man/man8/zfs-share.8

@@ -29,7 +29,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd June 30, 2019
+.Dd May 17, 2021
 .Dt ZFS-SHARE 8
 .Os
 .
@@ -39,6 +39,7 @@
 .Sh SYNOPSIS
 .Nm zfs
 .Cm share
+.Op Fl l
 .Fl a Ns | Ns Ar filesystem
 .Nm zfs
 .Cm unshare
@@ -49,10 +50,19 @@
 .It Xo
 .Nm zfs
 .Cm share
+.Op Fl l
 .Fl a Ns | Ns Ar filesystem
 .Xc
 Shares available ZFS file systems.
 .Bl -tag -width "-a"
+.It Fl l
+Load keys for encrypted filesystems as they are being mounted.
+This is equivalent to executing
+.Nm zfs Cm load-key
+on each encryption root before mounting it.
+Note that if a filesystem has
+.Sy keylocation Ns = Ns Sy prompt ,
+this will cause the terminal to interactively block after asking for the key.
 .It Fl a
 Share all available ZFS file systems.
 Invoked automatically as part of the boot process.

man/man8/zgenhostid.8

@@ -80,7 +80,7 @@ be an 8-digit-long hexadecimal number, optionally prefixed by
 .Dl # Nm Qq $ Ns Pq Nm hostid
 .It Record a custom hostid Po Ar 0xdeadbeef Pc in Pa /etc/hostid
 .Dl # Nm Ar deadbeef
-.It Record a custom hostid Po Ar 0x01234567 Pc in Pa /tmp/hostid No and ovewrite the file if it exists
+.It Record a custom hostid Po Ar 0x01234567 Pc in Pa /tmp/hostid No and overwrite the file if it exists
 .Dl # Nm Fl f o Ar /tmp/hostid 0x01234567
 .El
 .

man/man8/zpool-attach.8

@@ -79,7 +79,7 @@ The only property supported at the moment is
 The
 .Ar new_device
 is reconstructed sequentially to restore redundancy as quickly as possible.
-Checksums are not verfied during sequential reconstruction so a scrub is
+Checksums are not verified during sequential reconstruction so a scrub is
 started when the resilver completes.
 Sequential reconstruction is not supported for raidz configurations.
 .It Fl w

man/man8/zpool-clear.8

@@ -44,9 +44,12 @@ Clears device errors in a pool.
 If no arguments are specified, all device errors within the pool are cleared.
 If one or more devices is specified, only those errors associated with the
 specified device or devices are cleared.
-If
+.Pp
+If the pool was suspended it will be brought back online provided the
+devices can be accessed.
+Pools with
 .Sy multihost
-is enabled and the pool has been suspended, this will not resume I/O.
+enabled which have been suspended cannot be resumed.
 While the pool was suspended, it may have been imported on
 another host, and resuming I/O could result in pool damage.
 .

man/man8/zpool-events.8

@@ -68,7 +68,7 @@ Print the entire payload for each event.
 .El
 .
 .Sh EVENTS
-Theese are the different event subclasses.
+These are the different event subclasses.
 The full event name would be
 .Sy ereport.fs.zfs.\& Ns Em SUBCLASS ,
 but only the last part is listed here.
@@ -136,7 +136,7 @@ Issued when the ashift alignment requirement has increased.
 .It Sy vdev.remove
 Issued when a vdev is detached from a mirror (or a spare detached from a
 vdev where it have been used to replace a failed drive - only works if
-the original drive have been readded).
+the original drive have been re-added).
 .It Sy vdev.clear
 Issued when clearing device errors in a pool.
 Such as running