Tag 2.1.0

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>

.github/ISSUE_TEMPLATE/config.yml

@@ -10,5 +10,5 @@ contact_links:
     url: https://lists.freebsd.org/mailman/listinfo/freebsd-fs
     about: Get community support for OpenZFS on FreeBSD
   - name: OpenZFS on IRC
-    url: https://webchat.freenode.net/#openzfs
+    url: https://web.libera.chat/#openzfs
     about: Use IRC to get community support for OpenZFS

.github/workflows/zfs-tests-functional.yml

@@ -44,6 +44,12 @@ jobs:
         sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
         sudo depmod
         sudo modprobe zfs
+        # Workaround to provide additional free space for testing.
+        #   https://github.com/actions/virtual-environments/issues/2840
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
     - name: Tests
       run: |
         /usr/share/zfs/zfs-tests.sh -v -s 3G

.github/workflows/zfs-tests-sanity.yml

@@ -40,6 +40,12 @@ jobs:
         sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
         sudo depmod
         sudo modprobe zfs
+        # Workaround to provide additional free space for testing.
+        #   https://github.com/actions/virtual-environments/issues/2840
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
     - name: Tests
       run: |
         /usr/share/zfs/zfs-tests.sh -v -s 3G -r sanity

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "scripts/zfs-images"]
 	path = scripts/zfs-images
-	url = https://github.com/zfsonlinux/zfs-images
+	url = https://github.com/openzfs/zfs-images

META

@@ -2,9 +2,9 @@ Meta:          1
 Name:          zfs
 Branch:        1.0
 Version:       2.1.0
-Release:       rc1
+Release:       1
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS
-Linux-Maximum: 5.11
+Linux-Maximum: 5.13
 Linux-Minimum: 3.10

Makefile.am

@@ -1,3 +1,5 @@
+include $(top_srcdir)/config/Shellcheck.am
+
 ACLOCAL_AMFLAGS = -I config
 
 SUBDIRS = include
@@ -6,7 +8,7 @@ SUBDIRS += rpm
 endif
 
 if CONFIG_USER
-SUBDIRS += etc man scripts lib tests cmd contrib
+SUBDIRS += man scripts lib tests cmd etc contrib
 if BUILD_LINUX
 SUBDIRS += udev
 endif
@@ -26,8 +28,8 @@ endif
 AUTOMAKE_OPTIONS = foreign
 EXTRA_DIST  = autogen.sh copy-builtin
 EXTRA_DIST += config/config.awk config/rpm.am config/deb.am config/tgz.am
-EXTRA_DIST += META AUTHORS COPYRIGHT LICENSE NEWS NOTICE README.md
+EXTRA_DIST += AUTHORS CODE_OF_CONDUCT.md COPYRIGHT LICENSE META NEWS NOTICE
-EXTRA_DIST += CODE_OF_CONDUCT.md
+EXTRA_DIST += README.md RELEASES.md
 EXTRA_DIST += module/lua/README.zfs module/os/linux/spl/README.md
 
 # Include all the extra licensing information for modules
@@ -123,17 +125,8 @@ cstyle:
 
 filter_executable = -exec test -x '{}' \; -print
 
-PHONY += shellcheck
+SHELLCHECKDIRS = cmd contrib etc scripts tests
-shellcheck:
+SHELLCHECKSCRIPTS = autogen.sh
-	@if type shellcheck > /dev/null 2>&1; then \
-		shellcheck --exclude=SC1090 --exclude=SC1117 --format=gcc \
-			$$(find ${top_srcdir}/scripts/*.sh -type f) \
-			$$(find ${top_srcdir}/cmd/zed/zed.d/*.sh -type f) \
-			$$(find ${top_srcdir}/cmd/zpool/zpool.d/* \
-			-type f ${filter_executable}); \
-	else \
-		echo "skipping shellcheck because shellcheck is not installed"; \
-	fi
 
 PHONY += checkabi storeabi
 checkabi: lib
@@ -142,40 +135,9 @@ checkabi: lib
 storeabi: lib
 	$(MAKE) -C lib storeabi
 
-PHONY += checkbashisms
-checkbashisms:
-	@if type checkbashisms > /dev/null 2>&1; then \
-		checkbashisms -n -p -x \
-			$$(find ${top_srcdir} \
-				-name '.git' -prune \
-				-o -name 'build' -prune \
-				-o -name 'tests' -prune \
-				-o -name 'config' -prune \
-				-o -name 'zed-functions.sh*' -prune \
-				-o -name 'zfs-import*' -prune \
-				-o -name 'zfs-mount*' -prune \
-				-o -name 'zfs-zed*' -prune \
-				-o -name 'smart' -prune \
-				-o -name 'paxcheck.sh' -prune \
-				-o -name 'make_gitrev.sh' -prune \
-				-o -name '90zfs' -prune \
-				-o -type f ! -name 'config*' \
-				! -name 'libtool' \
-			-exec sh -c 'awk "NR==1 && /\#\!.*bin\/sh.*/ {print FILENAME;}" "{}"' \;); \
-	else \
-		echo "skipping checkbashisms because checkbashisms is not installed"; \
-	fi
-
 PHONY += mancheck
 mancheck:
-	@if type mandoc > /dev/null 2>&1; then \
+	${top_srcdir}/scripts/mancheck.sh ${top_srcdir}/man ${top_srcdir}/tests/test-runner/man
-		find ${top_srcdir}/man/man8 -type f -name 'zfs.8' \
-			-o -name 'zpool.8' -o -name 'zdb.8' \
-			-o -name 'zgenhostid.8' | \
-			xargs mandoc -Tlint -Werror; \
-	else \
-		echo "skipping mancheck because mandoc is not installed"; \
-	fi
 
 if BUILD_LINUX
 stat_fmt = -c '%A %n'

RELEASES.md

@@ -0,0 +1,37 @@
+OpenZFS uses the MAJOR.MINOR.PATCH versioning scheme described here:
+
+  * MAJOR - Incremented at the discretion of the OpenZFS developers to indicate
+    a particularly noteworthy feature or change. An increase in MAJOR number
+    does not indicate any incompatible on-disk format change. The ability
+    to import a ZFS pool is controlled by the feature flags enabled on the
+    pool and the feature flags supported by the installed OpenZFS version.
+    Increasing the MAJOR version is expected to be an infrequent occurrence.
+
+  * MINOR - Incremented to indicate new functionality such as a new feature
+    flag, pool/dataset property, zfs/zpool sub-command, new user/kernel
+    interface, etc. MINOR releases may introduce incompatible changes to the
+    user space library APIs (libzfs.so). Existing user/kernel interfaces are
+    considered to be stable to maximize compatibility between OpenZFS releases.
+    Additions to the user/kernel interface are backwards compatible.
+
+  * PATCH - Incremented when applying documentation updates, important bug
+    fixes, minor performance improvements, and kernel compatibility patches.
+    The user space library APIs and user/kernel interface are considered to
+    be stable. PATCH releases for a MAJOR.MINOR are published as needed.
+
+Two release branches are maintained for OpenZFS, they are:
+
+  * OpenZFS LTS - A designated MAJOR.MINOR release with periodic PATCH
+    releases that incorporate important changes backported from newer OpenZFS
+    releases. This branch is intended for use in environments using an
+    LTS, enterprise, or similarly managed kernel (RHEL, Ubuntu LTS, Debian).
+    Minor changes to support these distribution kernels will be applied as
+    needed. New kernel versions released after the OpenZFS LTS release are
+    not supported. LTS releases will receive patches for at least 2 years.
+    The current LTS release is OpenZFS 2.1.
+
+  * OpenZFS current - Tracks the newest MAJOR.MINOR release. This branch
+    includes support for the latest OpenZFS features and recently releases
+    kernels.  When a new MINOR release is tagged the previous MINOR release
+    will no longer be maintained (unless it is an LTS release). New MINOR
+    releases are planned to occur roughly annually.

cmd/Makefile.am

@@ -1,10 +1,15 @@
-SUBDIRS  = zfs zpool zdb zhack zinject zstream zstreamdump ztest
+include $(top_srcdir)/config/Shellcheck.am
+
+SUBDIRS  = zfs zpool zdb zhack zinject zstream ztest
 SUBDIRS += fsck_zfs vdev_id raidz_test zfs_ids_to_path
 SUBDIRS += zpool_influxdb
 
 CPPCHECKDIRS  = zfs zpool zdb zhack zinject zstream ztest
 CPPCHECKDIRS += raidz_test zfs_ids_to_path zpool_influxdb
 
+# TODO: #12084: SHELLCHECKDIRS = fsck_zfs vdev_id zpool
+SHELLCHECKDIRS = fsck_zfs zpool
+
 if USING_PYTHON
 SUBDIRS += arcstat arc_summary dbufstat
 endif
@@ -12,6 +17,7 @@ endif
 if BUILD_LINUX
 SUBDIRS += mount_zfs zed zgenhostid zvol_id zvol_wait
 CPPCHECKDIRS += mount_zfs zed zgenhostid zvol_id
+SHELLCHECKDIRS += zed
 endif
 
 PHONY = cppcheck

cmd/arc_summary/arc_summary2

@@ -102,18 +102,6 @@ show_tunable_descriptions = False
 alternate_tunable_layout = False
 
 
-def handle_Exception(ex_cls, ex, tb):
-    if ex is IOError:
-        if ex.errno == errno.EPIPE:
-            sys.exit()
-
-    if ex is KeyboardInterrupt:
-        sys.exit()
-
-
-sys.excepthook = handle_Exception
-
-
 def get_Kstat():
     """Collect information on the ZFS subsystem from the /proc virtual
     file system. The name "kstat" is a holdover from the Solaris utility
@@ -1137,48 +1125,55 @@ def main():
     global alternate_tunable_layout
 
     try:
-        opts, args = getopt.getopt(
-            sys.argv[1:],
-            "adp:h", ["alternate", "description", "page=", "help"]
-        )
-    except getopt.error as e:
-        sys.stderr.write("Error: %s\n" % e.msg)
-        usage()
-        sys.exit(1)
-
-    args = {}
-    for opt, arg in opts:
-        if opt in ('-a', '--alternate'):
-            args['a'] = True
-        if opt in ('-d', '--description'):
-            args['d'] = True
-        if opt in ('-p', '--page'):
-            args['p'] = arg
-        if opt in ('-h', '--help'):
-            usage()
-            sys.exit(0)
-
-    Kstat = get_Kstat()
-
-    alternate_tunable_layout = 'a' in args
-    show_tunable_descriptions = 'd' in args
-
-    pages = []
-
-    if 'p' in args:
         try:
-            pages.append(unSub[int(args['p']) - 1])
+            opts, args = getopt.getopt(
-        except IndexError:
+                sys.argv[1:],
-            sys.stderr.write('the argument to -p must be between 1 and ' +
+                "adp:h", ["alternate", "description", "page=", "help"]
-                             str(len(unSub)) + '\n')
+            )
+        except getopt.error as e:
+            sys.stderr.write("Error: %s\n" % e.msg)
+            usage()
             sys.exit(1)
-    else:
-        pages = unSub
 
-    zfs_header()
+        args = {}
-    for page in pages:
+        for opt, arg in opts:
-        page(Kstat)
+            if opt in ('-a', '--alternate'):
-        sys.stdout.write("\n")
+                args['a'] = True
+            if opt in ('-d', '--description'):
+                args['d'] = True
+            if opt in ('-p', '--page'):
+                args['p'] = arg
+            if opt in ('-h', '--help'):
+                usage()
+                sys.exit(0)
+
+        Kstat = get_Kstat()
+
+        alternate_tunable_layout = 'a' in args
+        show_tunable_descriptions = 'd' in args
+
+        pages = []
+
+        if 'p' in args:
+            try:
+                pages.append(unSub[int(args['p']) - 1])
+            except IndexError:
+                sys.stderr.write('the argument to -p must be between 1 and ' +
+                                 str(len(unSub)) + '\n')
+                sys.exit(1)
+        else:
+            pages = unSub
+
+        zfs_header()
+        for page in pages:
+            page(Kstat)
+            sys.stdout.write("\n")
+    except IOError as ex:
+        if (ex.errno == errno.EPIPE):
+            sys.exit(0)
+        raise
+    except KeyboardInterrupt:
+        sys.exit(0)
 
 
 if __name__ == '__main__':

cmd/arc_summary/arc_summary3

@@ -42,6 +42,13 @@ import os
 import subprocess
 import sys
 import time
+import errno
+
+# We can't use env -S portably, and we need python3 -u to handle pipes in
+# the shell abruptly closing the way we want to, so...
+import io
+if isinstance(sys.__stderr__.buffer, io.BufferedWriter):
+    os.execv(sys.executable, [sys.executable, "-u"] + sys.argv)
 
 DESCRIPTION = 'Print ARC and other statistics for OpenZFS'
 INDENT = ' '*8
@@ -161,21 +168,11 @@ elif sys.platform.startswith('linux'):
         # The original arc_summary called /sbin/modinfo/{spl,zfs} to get
         # the version information. We switch to /sys/module/{spl,zfs}/version
         # to make sure we get what is really loaded in the kernel
-        command = ["cat", "/sys/module/{0}/version".format(request)]
+        try:
-        req = request.upper()
+            with open("/sys/module/{}/version".format(request)) as f:
-
+                return f.read().strip()
-        # The recommended way to do this is with subprocess.run(). However,
+        except:
-        # some installed versions of Python are < 3.5, so we offer them
+            return "(unknown)"
-        # the option of doing it the old way (for now)
-        if 'run' in dir(subprocess):
-            info = subprocess.run(command, stdout=subprocess.PIPE,
-                                  universal_newlines=True)
-            version = info.stdout.strip()
-        else:
-            info = subprocess.check_output(command, universal_newlines=True)
-            version = info.strip()
-
-        return version
 
     def get_descriptions(request):
         """Get the descriptions of the Solaris Porting Layer (SPL) or the
@@ -231,6 +228,29 @@ elif sys.platform.startswith('linux'):
 
         return descs
 
+def handle_unraisableException(exc_type, exc_value=None, exc_traceback=None,
+                               err_msg=None, object=None):
+   handle_Exception(exc_type, object, exc_traceback)
+
+def handle_Exception(ex_cls, ex, tb):
+    if ex_cls is KeyboardInterrupt:
+        sys.exit()
+
+    if ex_cls is BrokenPipeError:
+        # It turns out that while sys.exit() triggers an exception
+        # not handled message on Python 3.8+, os._exit() does not.
+        os._exit(0)
+
+    if ex_cls is OSError:
+      if ex.errno == errno.ENOTCONN:
+        sys.exit()
+
+    raise ex
+
+if hasattr(sys,'unraisablehook'): # Python 3.8+
+    sys.unraisablehook = handle_unraisableException
+sys.excepthook = handle_Exception
+
 
 def cleanup_line(single_line):
     """Format a raw line of data from /proc and isolate the name value

cmd/fsck_zfs/.gitignore

@@ -0,0 +1 @@
+/fsck.zfs

cmd/fsck_zfs/Makefile.am

@@ -1 +1,6 @@
+include $(top_srcdir)/config/Substfiles.am
+include $(top_srcdir)/config/Shellcheck.am
+
 dist_sbin_SCRIPTS = fsck.zfs
+
+SUBSTFILES += $(dist_sbin_SCRIPTS)

cmd/fsck_zfs/fsck.zfs

@@ -1,9 +0,0 @@
-#!/bin/sh
-#
-# fsck.zfs: A fsck helper to accommodate distributions that expect
-# to be able to execute a fsck on all filesystem types.  Currently
-# this script does nothing but it could be extended to act as a
-# compatibility wrapper for 'zpool scrub'.
-#
-
-exit 0

cmd/fsck_zfs/fsck.zfs.in

@@ -0,0 +1,44 @@
+#!/bin/sh
+#
+# fsck.zfs: A fsck helper to accommodate distributions that expect
+# to be able to execute a fsck on all filesystem types.
+#
+# This script simply bubbles up some already-known-about errors,
+# see fsck.zfs(8)
+#
+
+if [ "$#" = "0" ]; then
+	echo "Usage: $0 [options] dataset…" >&2
+	exit 16
+fi
+
+ret=0
+for dataset in "$@"; do
+	case "$dataset" in
+		-*)
+			continue
+			;;
+		*)
+			;;
+	esac
+
+	pool="${dataset%%/*}"
+
+	case "$(@sbindir@/zpool list -Ho health "$pool")" in
+		DEGRADED)
+			ret=$(( ret | 4 ))
+			;;
+		FAULTED)
+			awk '!/^([[:space:]]*#.*)?$/ && $1 == "'"$dataset"'" && $3 == "zfs" {exit 1}' /etc/fstab || \
+				ret=$(( ret | 8 ))
+			;;
+		"")
+			# Pool not found, error printed by zpool(8)
+			ret=$(( ret | 8 ))
+			;;
+		*)
+			;;
+	esac
+done
+
+exit "$ret"

cmd/mount_zfs/mount_zfs.c

@@ -185,10 +185,11 @@ main(int argc, char **argv)
 			break;
 		case 'h':
 		case '?':
-			(void) fprintf(stderr, gettext("Invalid option '%c'\n"),
+			if (optopt)
-			    optopt);
+				(void) fprintf(stderr,
+				    gettext("Invalid option '%c'\n"), optopt);
 			(void) fprintf(stderr, gettext("Usage: mount.zfs "
-			    "[-sfnv] [-o options] <dataset> <mountpoint>\n"));
+			    "[-sfnvh] [-o options] <dataset> <mountpoint>\n"));
 			return (MOUNT_USAGE);
 		}
 	}

cmd/vdev_id/Makefile.am

@@ -1 +1,3 @@
+include $(top_srcdir)/config/Shellcheck.am
+
 dist_udev_SCRIPTS = vdev_id

cmd/vdev_id/vdev_id

@@ -147,8 +147,9 @@ map_slot() {
 	LINUX_SLOT=$1
 	CHANNEL=$2
 
-	MAPPED_SLOT=$(awk '$1 == "slot" && $2 == "${LINUX_SLOT}" && \
+	MAPPED_SLOT=$(awk -v linux_slot="$LINUX_SLOT" -v channel="$CHANNEL" \
-			$4 ~ /^${CHANNEL}$|^$/ { print $3; exit}' $CONFIG)
+			'$1 == "slot" && $2 == linux_slot && \
+			($4 ~ "^"channel"$" || $4 ~ /^$/) { print $3; exit}' $CONFIG)
 	if [ -z "$MAPPED_SLOT" ] ; then
 		MAPPED_SLOT=$LINUX_SLOT
 	fi
@@ -163,7 +164,7 @@ map_channel() {
 	case $TOPOLOGY in
 		"sas_switch")
 		MAPPED_CHAN=$(awk -v port="$PORT" \
-			'$1 == "channel" && $2 == ${PORT} \
+			'$1 == "channel" && $2 == port \
 			{ print $3; exit }' $CONFIG)
 		;;
 		"sas_direct"|"scsi")

cmd/zdb/zdb.c

@@ -162,12 +162,6 @@ static int dump_bpobj_cb(void *arg, const blkptr_t *bp, boolean_t free,
     dmu_tx_t *tx);
 
 typedef struct sublivelist_verify {
-	/* all ALLOC'd blkptr_t in one sub-livelist */
-	zfs_btree_t sv_all_allocs;
-
-	/* all FREE'd blkptr_t in one sub-livelist */
-	zfs_btree_t sv_all_frees;
-
 	/* FREE's that haven't yet matched to an ALLOC, in one sub-livelist */
 	zfs_btree_t sv_pair;
 
@@ -226,29 +220,68 @@ typedef struct sublivelist_verify_block {
 
 static void zdb_print_blkptr(const blkptr_t *bp, int flags);
 
+typedef struct sublivelist_verify_block_refcnt {
+	/* block pointer entry in livelist being verified */
+	blkptr_t svbr_blk;
+
+	/*
+	 * Refcount gets incremented to 1 when we encounter the first
+	 * FREE entry for the svfbr block pointer and a node for it
+	 * is created in our ZDB verification/tracking metadata.
+	 *
+	 * As we encounter more FREE entries we increment this counter
+	 * and similarly decrement it whenever we find the respective
+	 * ALLOC entries for this block.
+	 *
+	 * When the refcount gets to 0 it means that all the FREE and
+	 * ALLOC entries of this block have paired up and we no longer
+	 * need to track it in our verification logic (e.g. the node
+	 * containing this struct in our verification data structure
+	 * should be freed).
+	 *
+	 * [refer to sublivelist_verify_blkptr() for the actual code]
+	 */
+	uint32_t svbr_refcnt;
+} sublivelist_verify_block_refcnt_t;
+
+static int
+sublivelist_block_refcnt_compare(const void *larg, const void *rarg)
+{
+	const sublivelist_verify_block_refcnt_t *l = larg;
+	const sublivelist_verify_block_refcnt_t *r = rarg;
+	return (livelist_compare(&l->svbr_blk, &r->svbr_blk));
+}
+
 static int
 sublivelist_verify_blkptr(void *arg, const blkptr_t *bp, boolean_t free,
     dmu_tx_t *tx)
 {
 	ASSERT3P(tx, ==, NULL);
 	struct sublivelist_verify *sv = arg;
-	char blkbuf[BP_SPRINTF_LEN];
+	sublivelist_verify_block_refcnt_t current = {
+			.svbr_blk = *bp,
+
+			/*
+			 * Start with 1 in case this is the first free entry.
+			 * This field is not used for our B-Tree comparisons
+			 * anyway.
+			 */
+			.svbr_refcnt = 1,
+	};
+
 	zfs_btree_index_t where;
+	sublivelist_verify_block_refcnt_t *pair =
+	    zfs_btree_find(&sv->sv_pair, &current, &where);
 	if (free) {
-		zfs_btree_add(&sv->sv_pair, bp);
+		if (pair == NULL) {
-		/* Check if the FREE is a duplicate */
+			/* first free entry for this block pointer */
-		if (zfs_btree_find(&sv->sv_all_frees, bp, &where) != NULL) {
+			zfs_btree_add(&sv->sv_pair, &current);
-			snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), bp,
-			    free);
-			(void) printf("\tERROR: Duplicate FREE: %s\n", blkbuf);
 		} else {
-			zfs_btree_add_idx(&sv->sv_all_frees, bp, &where);
+			pair->svbr_refcnt++;
 		}
 	} else {
-		/* Check if the ALLOC has been freed */
+		if (pair == NULL) {
-		if (zfs_btree_find(&sv->sv_pair, bp, &where) != NULL) {
+			/* block that is currently marked as allocated */
-			zfs_btree_remove_idx(&sv->sv_pair, &where);
-		} else {
 			for (int i = 0; i < SPA_DVAS_PER_BP; i++) {
 				if (DVA_IS_EMPTY(&bp->blk_dva[i]))
 					break;
@@ -263,16 +296,16 @@ sublivelist_verify_blkptr(void *arg, const blkptr_t *bp, boolean_t free,
 					    &svb, &where);
 				}
 			}
-		}
-		/* Check if the ALLOC is a duplicate */
-		if (zfs_btree_find(&sv->sv_all_allocs, bp, &where) != NULL) {
-			snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), bp,
-			    free);
-			(void) printf("\tERROR: Duplicate ALLOC: %s\n", blkbuf);
 		} else {
-			zfs_btree_add_idx(&sv->sv_all_allocs, bp, &where);
+			/* alloc matches a free entry */
+			pair->svbr_refcnt--;
+			if (pair->svbr_refcnt == 0) {
+				/* all allocs and frees have been matched */
+				zfs_btree_remove_idx(&sv->sv_pair, &where);
+			}
 		}
 	}
+
 	return (0);
 }
 
@@ -280,32 +313,22 @@ static int
 sublivelist_verify_func(void *args, dsl_deadlist_entry_t *dle)
 {
 	int err;
-	char blkbuf[BP_SPRINTF_LEN];
 	struct sublivelist_verify *sv = args;
 
-	zfs_btree_create(&sv->sv_all_allocs, livelist_compare,
+	zfs_btree_create(&sv->sv_pair, sublivelist_block_refcnt_compare,
-	    sizeof (blkptr_t));
+	    sizeof (sublivelist_verify_block_refcnt_t));
-
-	zfs_btree_create(&sv->sv_all_frees, livelist_compare,
-	    sizeof (blkptr_t));
-
-	zfs_btree_create(&sv->sv_pair, livelist_compare,
-	    sizeof (blkptr_t));
 
 	err = bpobj_iterate_nofree(&dle->dle_bpobj, sublivelist_verify_blkptr,
 	    sv, NULL);
 
-	zfs_btree_clear(&sv->sv_all_allocs);
+	sublivelist_verify_block_refcnt_t *e;
-	zfs_btree_destroy(&sv->sv_all_allocs);
-
-	zfs_btree_clear(&sv->sv_all_frees);
-	zfs_btree_destroy(&sv->sv_all_frees);
-
-	blkptr_t *e;
 	zfs_btree_index_t *cookie = NULL;
 	while ((e = zfs_btree_destroy_nodes(&sv->sv_pair, &cookie)) != NULL) {
-		snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), e, B_TRUE);
+		char blkbuf[BP_SPRINTF_LEN];
-		(void) printf("\tERROR: Unmatched FREE: %s\n", blkbuf);
+		snprintf_blkptr_compact(blkbuf, sizeof (blkbuf),
+		    &e->svbr_blk, B_TRUE);
+		(void) printf("\tERROR: %d unmatched FREE(s): %s\n",
+		    e->svbr_refcnt, blkbuf);
 	}
 	zfs_btree_destroy(&sv->sv_pair);
 
@@ -614,10 +637,14 @@ mv_populate_livelist_allocs(metaslab_verify_t *mv, sublivelist_verify_t *sv)
 /*
  * [Livelist Check]
  * Iterate through all the sublivelists and:
- * - report leftover frees
+ * - report leftover frees (**)
- * - report double ALLOCs/FREEs
  * - record leftover ALLOCs together with their TXG [see Cross Check]
  *
+ * (**) Note: Double ALLOCs are valid in datasets that have dedup
+ *      enabled. Similarly double FREEs are allowed as well but
+ *      only if they pair up with a corresponding ALLOC entry once
+ *      we our done with our sublivelist iteration.
+ *
  * [Spacemap Check]
  * for each metaslab:
  * - iterate over spacemap and then the metaslab's entries in the
@@ -5932,7 +5959,8 @@ zdb_leak_init_prepare_indirect_vdevs(spa_t *spa, zdb_cb_t *zcb)
 		vdev_metaslab_group_create(vd);
 		VERIFY0(vdev_metaslab_init(vd, 0));
 
-		vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping;
+		vdev_indirect_mapping_t *vim __maybe_unused =
+		    vd->vdev_indirect_mapping;
 		uint64_t vim_idx = 0;
 		for (uint64_t m = 0; m < vd->vdev_ms_count; m++) {
 
@@ -7041,7 +7069,7 @@ verify_checkpoint_vdev_spacemaps(spa_t *checkpoint, spa_t *current)
 		for (uint64_t c = ckpoint_rvd->vdev_children;
 		    c < current_rvd->vdev_children; c++) {
 			vdev_t *current_vd = current_rvd->vdev_child[c];
-			ASSERT3P(current_vd->vdev_checkpoint_sm, ==, NULL);
+			VERIFY3P(current_vd->vdev_checkpoint_sm, ==, NULL);
 		}
 	}
 

cmd/zed/Makefile.am

@@ -1,8 +1,10 @@
 include $(top_srcdir)/config/Rules.am
+include $(top_srcdir)/config/Shellcheck.am
 
 AM_CFLAGS += $(LIBUDEV_CFLAGS) $(LIBUUID_CFLAGS)
 
 SUBDIRS = zed.d
+SHELLCHECKDIRS = $(SUBDIRS)
 
 sbin_PROGRAMS = zed
 
@@ -43,7 +45,7 @@ zed_LDADD = \
 	$(abs_top_builddir)/lib/libnvpair/libnvpair.la \
 	$(abs_top_builddir)/lib/libuutil/libuutil.la
 
-zed_LDADD += -lrt $(LIBUDEV_LIBS) $(LIBUUID_LIBS)
+zed_LDADD += -lrt $(LIBATOMIC_LIBS) $(LIBUDEV_LIBS) $(LIBUUID_LIBS)
 zed_LDFLAGS = -pthread
 
 EXTRA_DIST = agents/README.md

cmd/zed/agents/zfs_agents.c

@@ -392,6 +392,7 @@ zfs_agent_init(libzfs_handle_t *zfs_hdl)
 		list_destroy(&agent_events);
 		zed_log_die("Failed to initialize agents");
 	}
+	pthread_setname_np(g_agents_tid, "agents");
 }
 
 void

cmd/zed/agents/zfs_mod.c

@@ -918,6 +918,7 @@ zfs_slm_init()
 		return (-1);
 	}
 
+	pthread_setname_np(g_zfs_tid, "enum-pools");
 	list_create(&g_device_list, sizeof (struct pendingdev),
 	    offsetof(struct pendingdev, pd_node));
 

cmd/zed/zed.c

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).
@@ -60,8 +60,8 @@ _setup_sig_handlers(void)
 		zed_log_die("Failed to initialize sigset");
 
 	sa.sa_flags = SA_RESTART;
-	sa.sa_handler = SIG_IGN;
 
+	sa.sa_handler = SIG_IGN;
 	if (sigaction(SIGPIPE, &sa, NULL) < 0)
 		zed_log_die("Failed to ignore SIGPIPE");
 
@@ -75,6 +75,10 @@ _setup_sig_handlers(void)
 	sa.sa_handler = _hup_handler;
 	if (sigaction(SIGHUP, &sa, NULL) < 0)
 		zed_log_die("Failed to register SIGHUP handler");
+
+	(void) sigaddset(&sa.sa_mask, SIGCHLD);
+	if (pthread_sigmask(SIG_BLOCK, &sa.sa_mask, NULL) < 0)
+		zed_log_die("Failed to block SIGCHLD");
 }
 
 /*
@@ -212,22 +216,20 @@ _finish_daemonize(void)
 int
 main(int argc, char *argv[])
 {
-	struct zed_conf *zcp;
+	struct zed_conf zcp;
 	uint64_t saved_eid;
 	int64_t saved_etime[2];
 
 	zed_log_init(argv[0]);
 	zed_log_stderr_open(LOG_NOTICE);
-	zcp = zed_conf_create();
+	zed_conf_init(&zcp);
-	zed_conf_parse_opts(zcp, argc, argv);
+	zed_conf_parse_opts(&zcp, argc, argv);
-	if (zcp->do_verbose)
+	if (zcp.do_verbose)
 		zed_log_stderr_open(LOG_INFO);
 
 	if (geteuid() != 0)
 		zed_log_die("Must be run as root");
 
-	zed_conf_parse_file(zcp);
-
 	zed_file_close_from(STDERR_FILENO + 1);
 
 	(void) umask(0);
@@ -235,32 +237,32 @@ main(int argc, char *argv[])
 	if (chdir("/") < 0)
 		zed_log_die("Failed to change to root directory");
 
-	if (zed_conf_scan_dir(zcp) < 0)
+	if (zed_conf_scan_dir(&zcp) < 0)
 		exit(EXIT_FAILURE);
 
-	if (!zcp->do_foreground) {
+	if (!zcp.do_foreground) {
 		_start_daemonize();
 		zed_log_syslog_open(LOG_DAEMON);
 	}
 	_setup_sig_handlers();
 
-	if (zcp->do_memlock)
+	if (zcp.do_memlock)
 		_lock_memory();
 
-	if ((zed_conf_write_pid(zcp) < 0) && (!zcp->do_force))
+	if ((zed_conf_write_pid(&zcp) < 0) && (!zcp.do_force))
 		exit(EXIT_FAILURE);
 
-	if (!zcp->do_foreground)
+	if (!zcp.do_foreground)
 		_finish_daemonize();
 
 	zed_log_msg(LOG_NOTICE,
 	    "ZFS Event Daemon %s-%s (PID %d)",
 	    ZFS_META_VERSION, ZFS_META_RELEASE, (int)getpid());
 
-	if (zed_conf_open_state(zcp) < 0)
+	if (zed_conf_open_state(&zcp) < 0)
 		exit(EXIT_FAILURE);
 
-	if (zed_conf_read_state(zcp, &saved_eid, saved_etime) < 0)
+	if (zed_conf_read_state(&zcp, &saved_eid, saved_etime) < 0)
 		exit(EXIT_FAILURE);
 
 idle:
@@ -269,24 +271,24 @@ idle:
 	 * successful.
 	 */
 	do {
-		if (!zed_event_init(zcp))
+		if (!zed_event_init(&zcp))
 			break;
 		/* Wait for some time and try again. tunable? */
 		sleep(30);
-	} while (!_got_exit && zcp->do_idle);
+	} while (!_got_exit && zcp.do_idle);
 
 	if (_got_exit)
 		goto out;
 
-	zed_event_seek(zcp, saved_eid, saved_etime);
+	zed_event_seek(&zcp, saved_eid, saved_etime);
 
 	while (!_got_exit) {
 		int rv;
 		if (_got_hup) {
 			_got_hup = 0;
-			(void) zed_conf_scan_dir(zcp);
+			(void) zed_conf_scan_dir(&zcp);
 		}
-		rv = zed_event_service(zcp);
+		rv = zed_event_service(&zcp);
 
 		/* ENODEV: When kernel module is unloaded (osx) */
 		if (rv == ENODEV)
@@ -294,13 +296,13 @@ idle:
 	}
 
 	zed_log_msg(LOG_NOTICE, "Exiting");
-	zed_event_fini(zcp);
+	zed_event_fini(&zcp);
 
-	if (zcp->do_idle && !_got_exit)
+	if (zcp.do_idle && !_got_exit)
 		goto idle;
 
 out:
-	zed_conf_destroy(zcp);
+	zed_conf_destroy(&zcp);
 	zed_log_fini();
 	exit(EXIT_SUCCESS);
 }

cmd/zed/zed.d/Makefile.am

@@ -1,5 +1,6 @@
 include $(top_srcdir)/config/Rules.am
 include $(top_srcdir)/config/Substfiles.am
+include $(top_srcdir)/config/Shellcheck.am
 
 EXTRA_DIST += README
 
@@ -51,3 +52,6 @@ install-data-hook:
 	    ln -s "$(zedexecdir)/$${f}" "$(DESTDIR)$(zedconfdir)"; \
 	done
 	chmod 0600 "$(DESTDIR)$(zedconfdir)/zed.rc"
+
+# False positive: 1>&"${ZED_FLOCK_FD}" looks suspiciously similar to a >&filename bash extension
+CHECKBASHISMS_IGNORE = -e 'should be >word 2>&1' -e '&"$${ZED_FLOCK_FD}"'

cmd/zed/zed.d/all-debug.sh

@@ -12,15 +12,11 @@
 
 zed_exit_if_ignoring_this_event
 
-lockfile="$(basename -- "${ZED_DEBUG_LOG}").lock"
+zed_lock "${ZED_DEBUG_LOG}"
+{
+	printenv | sort
+	echo
+} 1>&"${ZED_FLOCK_FD}"
+zed_unlock "${ZED_DEBUG_LOG}"
 
-umask 077
-zed_lock "${lockfile}"
-exec >> "${ZED_DEBUG_LOG}"
-
-printenv | sort
-echo
-
-exec >&-
-zed_unlock "${lockfile}"
 exit 0

cmd/zed/zed.d/all-syslog.sh

@@ -42,6 +42,7 @@ fi
     msg="${msg} delay=$((ZEVENT_ZIO_DELAY / 1000000))ms"
 
 # list the bookmark data together
+# shellcheck disable=SC2153
 [ -n "${ZEVENT_ZIO_OBJSET}" ] && \
     msg="${msg} bookmark=${ZEVENT_ZIO_OBJSET}:${ZEVENT_ZIO_OBJECT}:${ZEVENT_ZIO_LEVEL}:${ZEVENT_ZIO_BLKID}"
 

cmd/zed/zed.d/data-notify.sh

@@ -25,7 +25,7 @@ zed_rate_limit "${rate_limit_tag}" || exit 3
 
 umask 077
 note_subject="ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on $(hostname)"
-note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
+note_pathname="$(mktemp)"
 {
     echo "ZFS has detected a data error:"
     echo

cmd/zed/zed.d/generic-notify.sh

@@ -31,7 +31,7 @@ umask 077
 pool_str="${ZEVENT_POOL:+" for ${ZEVENT_POOL}"}"
 host_str=" on $(hostname)"
 note_subject="ZFS ${ZEVENT_SUBCLASS} event${pool_str}${host_str}"
-note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
+note_pathname="$(mktemp)"
 {
     echo "ZFS has posted the following event:"
     echo

cmd/zed/zed.d/history_event-zfs-list-cacher.sh.in

@@ -3,9 +3,8 @@
 # Track changes to enumerated pools for use in early-boot
 set -ef
 
-FSLIST_DIR="@sysconfdir@/zfs/zfs-list.cache"
+FSLIST="@sysconfdir@/zfs/zfs-list.cache/${ZEVENT_POOL}"
-FSLIST_TMP="@runstatedir@/zfs-list.cache.new"
+FSLIST_TMP="@runstatedir@/zfs-list.cache@${ZEVENT_POOL}"
-FSLIST="${FSLIST_DIR}/${ZEVENT_POOL}"
 
 # If the pool specific cache file is not writeable, abort
 [ -w "${FSLIST}" ] || exit 0
@@ -14,20 +13,20 @@ FSLIST="${FSLIST_DIR}/${ZEVENT_POOL}"
 . "${ZED_ZEDLET_DIR}/zed-functions.sh"
 
 [ "$ZEVENT_SUBCLASS" != "history_event" ] && exit 0
-zed_check_cmd "${ZFS}" sort diff grep
+zed_check_cmd "${ZFS}" sort diff
 
 # If we are acting on a snapshot, we have nothing to do
-printf '%s' "${ZEVENT_HISTORY_DSNAME}" | grep '@' && exit 0
+[ "${ZEVENT_HISTORY_DSNAME%@*}" = "${ZEVENT_HISTORY_DSNAME}" ] || exit 0
 
-# We obtain a lock on zfs-list to avoid any simultaneous writes.
+# We lock the output file to avoid simultaneous writes.
 # If we run into trouble, log and drop the lock
 abort_alter() {
-  zed_log_msg "Error updating zfs-list.cache!"
+  zed_log_msg "Error updating zfs-list.cache for ${ZEVENT_POOL}!"
-  zed_unlock zfs-list
+  zed_unlock "${FSLIST}"
 }
 
 finished() {
-  zed_unlock zfs-list
+  zed_unlock "${FSLIST}"
   trap - EXIT
   exit 0
 }
@@ -37,7 +36,7 @@ case "${ZEVENT_HISTORY_INTERNAL_NAME}" in
       ;;
 
     export)
-        zed_lock zfs-list
+        zed_lock "${FSLIST}"
         trap abort_alter EXIT
         echo > "${FSLIST}"
         finished
@@ -63,7 +62,7 @@ case "${ZEVENT_HISTORY_INTERNAL_NAME}" in
       ;;
 esac
 
-zed_lock zfs-list
+zed_lock "${FSLIST}"
 trap abort_alter EXIT
 
 PROPS="name,mountpoint,canmount,atime,relatime,devices,exec\
@@ -79,7 +78,7 @@ PROPS="name,mountpoint,canmount,atime,relatime,devices,exec\
 sort "${FSLIST_TMP}" -o "${FSLIST_TMP}"
 
 # Don't modify the file if it hasn't changed
-diff -q "${FSLIST_TMP}" "${FSLIST}" || mv "${FSLIST_TMP}" "${FSLIST}"
+diff -q "${FSLIST_TMP}" "${FSLIST}" || cat "${FSLIST_TMP}" > "${FSLIST}"
 rm -f "${FSLIST_TMP}"
 
 finished

cmd/zed/zed.d/scrub_finish-notify.sh

@@ -41,7 +41,7 @@ fi
 
 umask 077
 note_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)"
-note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
+note_pathname="$(mktemp)"
 {
     echo "ZFS has finished a ${action}:"
     echo

cmd/zed/zed.d/statechange-led.sh

@@ -1,21 +1,21 @@
 #!/bin/sh
 #
-# Turn off/on the VDEV's enclosure fault LEDs when the pool's state changes.
+# Turn off/on vdevs' enclosure fault LEDs when their pool's state changes.
 #
-# Turn the VDEV's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL.
+# Turn a vdev's fault LED on if it becomes FAULTED, DEGRADED or UNAVAIL.
-# Turn the LED off when it's back ONLINE again.
+# Turn its LED off when it's back ONLINE again.
 #
 # This script run in two basic modes:
 #
 # 1. If $ZEVENT_VDEV_ENC_SYSFS_PATH and $ZEVENT_VDEV_STATE_STR are set, then
-# only set the LED for that particular VDEV. This is the case for statechange
+# only set the LED for that particular vdev. This is the case for statechange
 # events and some vdev_* events.
 #
-# 2. If those vars are not set, then check the state of all VDEVs in the pool
+# 2. If those vars are not set, then check the state of all vdevs in the pool
 # and set the LEDs accordingly.  This is the case for pool_import events.
 #
 # Note that this script requires that your enclosure be supported by the
-# Linux SCSI enclosure services (ses) driver.  The script will do nothing
+# Linux SCSI Enclosure services (SES) driver.  The script will do nothing
 # if you have no enclosure, or if your enclosure isn't supported.
 #
 # Exit codes:
@@ -59,6 +59,10 @@ check_and_set_led()
 	file="$1"
 	val="$2"
 
+	if [ -z "$val" ]; then
+		return 0
+	fi
+
 	if [ ! -e "$file" ] ; then
 		return 3
 	fi
@@ -66,11 +70,11 @@ check_and_set_led()
 	# If another process is accessing the LED when we attempt to update it,
 	# the update will be lost so retry until the LED actually changes or we
 	# timeout.
-	for _ in $(seq 1 5); do
+	for _ in 1 2 3 4 5; do
 		# We want to check the current state first, since writing to the
 		# 'fault' entry always causes a SES command, even if the
 		# current state is already what you want.
-		current=$(cat "${file}")
+		read -r current < "${file}"
 
 		# On some enclosures if you write 1 to fault, and read it back,
 		# it will return 2.  Treat all non-zero values as 1 for
@@ -85,27 +89,29 @@ check_and_set_led()
 		else
 			break
 		fi
-        done
+	done
 }
 
 state_to_val()
 {
 	state="$1"
-	if [ "$state" = "FAULTED" ] || [ "$state" = "DEGRADED" ] || \
+	case "$state" in
-	   [ "$state" = "UNAVAIL" ] ; then
+		FAULTED|DEGRADED|UNAVAIL)
-		echo 1
+			echo 1
-	elif [ "$state" = "ONLINE" ] ; then
+			;;
-		echo 0
+		ONLINE)
-	fi
+			echo 0
+			;;
+	esac
 }
 
-# process_pool ([pool])
+# process_pool (pool)
 #
-# Iterate through a pool (or pools) and set the VDEV's enclosure slot LEDs to
+# Iterate through a pool and set the vdevs' enclosure slot LEDs to
-# the VDEV's state.
+# those vdevs' state.
 #
 # Arguments
-#   pool:	Optional pool name.  If not specified, iterate though all pools.
+#   pool:	Pool name.
 #
 # Return
 #  0 on success, 3 on missing sysfs path
@@ -113,19 +119,22 @@ state_to_val()
 process_pool()
 {
 	pool="$1"
+
+	# The output will be the vdevs only (from "grep '/dev/'"):
+	#
+	#    U45     ONLINE       0     0     0   /dev/sdk          0
+	#    U46     ONLINE       0     0     0   /dev/sdm          0
+	#    U47     ONLINE       0     0     0   /dev/sdn          0
+	#    U50     ONLINE       0     0     0  /dev/sdbn          0
+	#
+	ZPOOL_SCRIPTS_AS_ROOT=1 $ZPOOL status -c upath,fault_led "$pool" | grep '/dev/' | (
 	rc=0
-
+	while read -r vdev state _ _ _ therest; do
-	# Lookup all the current LED values and paths in parallel
-	#shellcheck disable=SC2016
-	cmd='echo led_token=$(cat "$VDEV_ENC_SYSFS_PATH/fault"),"$VDEV_ENC_SYSFS_PATH",'
-	out=$($ZPOOL status -vc "$cmd" "$pool" | grep 'led_token=')
-
-	#shellcheck disable=SC2034
-	echo "$out" | while read -r vdev state read write chksum therest; do
 		# Read out current LED value and path
-		tmp=$(echo "$therest" | sed 's/^.*led_token=//g')
+		# Get dev name (like 'sda')
-		vdev_enc_sysfs_path=$(echo "$tmp" | awk -F ',' '{print $2}')
+		dev=$(basename "$(echo "$therest" | awk '{print $(NF-1)}')")
-		current_val=$(echo "$tmp" | awk -F ',' '{print $1}')
+		vdev_enc_sysfs_path=$(realpath "/sys/class/block/$dev/device/enclosure_device"*)
+		current_val=$(echo "$therest" | awk '{print $NF}')
 
 		if [ "$current_val" != "0" ] ; then
 			current_val=1
@@ -137,36 +146,27 @@ process_pool()
 		fi
 
 		if [ ! -e "$vdev_enc_sysfs_path/fault" ] ; then
-			#shellcheck disable=SC2030
+			rc=3
-			rc=1
 			zed_log_msg "vdev $vdev '$file/fault' doesn't exist"
-			continue;
+			continue
 		fi
 
 		val=$(state_to_val "$state")
 
 		if [ "$current_val" = "$val" ] ; then
 			# LED is already set correctly
-			continue;
+			continue
 		fi
 
 		if ! check_and_set_led "$vdev_enc_sysfs_path/fault" "$val"; then
-			rc=1
+			rc=3
 		fi
-
 	done
-
+	exit "$rc"; )
-	#shellcheck disable=SC2031
-	if [ "$rc" = "0" ] ; then
-		return 0
-	else
-		# We didn't see a sysfs entry that we wanted to set
-		return 3
-	fi
 }
 
 if [ -n "$ZEVENT_VDEV_ENC_SYSFS_PATH" ] && [ -n "$ZEVENT_VDEV_STATE_STR" ] ; then
-	# Got a statechange for an individual VDEV
+	# Got a statechange for an individual vdev
 	val=$(state_to_val "$ZEVENT_VDEV_STATE_STR")
 	vdev=$(basename "$ZEVENT_VDEV_PATH")
 	check_and_set_led "$ZEVENT_VDEV_ENC_SYSFS_PATH/fault" "$val"

cmd/zed/zed.d/statechange-notify.sh

@@ -37,7 +37,7 @@ fi
 
 umask 077
 note_subject="ZFS device fault for pool ${ZEVENT_POOL_GUID} on $(hostname)"
-note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
+note_pathname="$(mktemp)"
 {
     if [ "${ZEVENT_VDEV_STATE_STR}" = "FAULTED" ] ; then
         echo "The number of I/O errors associated with a ZFS device exceeded"

cmd/zed/zed.d/trim_finish-notify.sh

@@ -19,7 +19,7 @@ zed_check_cmd "${ZPOOL}" || exit 9
 
 umask 077
 note_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)"
-note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
+note_pathname="$(mktemp)"
 {
     echo "ZFS has finished a trim:"
     echo

cmd/zed/zed.d/zed-functions.sh

@@ -126,10 +126,8 @@ zed_lock()
 
     # Obtain a lock on the file bound to the given file descriptor.
     #
-    eval "exec ${fd}> '${lockfile}'"
+    eval "exec ${fd}>> '${lockfile}'"
-    err="$(flock --exclusive "${fd}" 2>&1)"
+    if ! err="$(flock --exclusive "${fd}" 2>&1)"; then
-    # shellcheck disable=SC2181
-    if [ $? -ne 0 ]; then
         zed_log_err "failed to lock \"${lockfile}\": ${err}"
     fi
 
@@ -165,9 +163,7 @@ zed_unlock()
     fi
 
     # Release the lock and close the file descriptor.
-    err="$(flock --unlock "${fd}" 2>&1)"
+    if ! err="$(flock --unlock "${fd}" 2>&1)"; then
-    # shellcheck disable=SC2181
-    if [ $? -ne 0 ]; then
         zed_log_err "failed to unlock \"${lockfile}\": ${err}"
     fi
     eval "exec ${fd}>&-"
@@ -267,7 +263,7 @@ zed_notify_email()
                 -e "s/@SUBJECT@/${subject}/g")"
 
     # shellcheck disable=SC2086
-    eval "${ZED_EMAIL_PROG}" ${ZED_EMAIL_OPTS} < "${pathname}" >/dev/null 2>&1
+    eval ${ZED_EMAIL_PROG} ${ZED_EMAIL_OPTS} < "${pathname}" >/dev/null 2>&1
     rv=$?
     if [ "${rv}" -ne 0 ]; then
         zed_log_err "$(basename "${ZED_EMAIL_PROG}") exit=${rv}"
@@ -367,7 +363,7 @@ zed_notify_pushbullet()
 #
 # Notification via Slack Webhook <https://api.slack.com/incoming-webhooks>.
 # The Webhook URL (ZED_SLACK_WEBHOOK_URL) identifies this client to the
-# Slack channel. 
+# Slack channel.
 #
 # Requires awk, curl, and sed executables to be installed in the standard PATH.
 #
@@ -511,10 +507,8 @@ zed_guid_to_pool()
 		return
 	fi
 
-	guid=$(printf "%llu" "$1")
+	guid="$(printf "%u" "$1")"
-	if [ -n "$guid" ] ; then
+	$ZPOOL get -H -ovalue,name guid | awk '$1 == '"$guid"' {print $2; exit}'
-		$ZPOOL get -H -ovalue,name guid | awk '$1=='"$guid"' {print $2}'
-	fi
 }
 
 # zed_exit_if_ignoring_this_event

cmd/zed/zed.h

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).
@@ -15,11 +15,6 @@
 #ifndef	ZED_H
 #define	ZED_H
 
-/*
- * Absolute path for the default zed configuration file.
- */
-#define	ZED_CONF_FILE		SYSCONFDIR "/zfs/zed.conf"
-
 /*
  * Absolute path for the default zed pid file.
  */
@@ -35,16 +30,6 @@
  */
 #define	ZED_ZEDLET_DIR		SYSCONFDIR "/zfs/zed.d"
 
-/*
- * Reserved for future use.
- */
-#define	ZED_MAX_EVENTS		0
-
-/*
- * Reserved for future use.
- */
-#define	ZED_MIN_EVENTS		0
-
 /*
  * String prefix for ZED variables passed via environment variables.
  */

cmd/zed/zed_conf.c

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).
@@ -32,43 +32,26 @@
 #include "zed_strings.h"
 
 /*
- * Return a new configuration with default values.
+ * Initialise the configuration with default values.
  */
-struct zed_conf *
+void
-zed_conf_create(void)
+zed_conf_init(struct zed_conf *zcp)
 {
-	struct zed_conf *zcp;
+	memset(zcp, 0, sizeof (*zcp));
 
-	zcp = calloc(1, sizeof (*zcp));
+	/* zcp->zfs_hdl opened in zed_event_init() */
-	if (!zcp)
+	/* zcp->zedlets created in zed_conf_scan_dir() */
-		goto nomem;
 
-	zcp->syslog_facility = LOG_DAEMON;
+	zcp->pid_fd = -1;		/* opened in zed_conf_write_pid() */
-	zcp->min_events = ZED_MIN_EVENTS;
+	zcp->state_fd = -1;		/* opened in zed_conf_open_state() */
-	zcp->max_events = ZED_MAX_EVENTS;
+	zcp->zevent_fd = -1;		/* opened in zed_event_init() */
-	zcp->pid_fd = -1;
-	zcp->zedlets = NULL;		/* created via zed_conf_scan_dir() */
-	zcp->state_fd = -1;		/* opened via zed_conf_open_state() */
-	zcp->zfs_hdl = NULL;		/* opened via zed_event_init() */
-	zcp->zevent_fd = -1;		/* opened via zed_event_init() */
 
-	if (!(zcp->conf_file = strdup(ZED_CONF_FILE)))
+	zcp->max_jobs = 16;
-		goto nomem;
 
-	if (!(zcp->pid_file = strdup(ZED_PID_FILE)))
+	if (!(zcp->pid_file = strdup(ZED_PID_FILE)) ||
-		goto nomem;
+	    !(zcp->zedlet_dir = strdup(ZED_ZEDLET_DIR)) ||
-
+	    !(zcp->state_file = strdup(ZED_STATE_FILE)))
-	if (!(zcp->zedlet_dir = strdup(ZED_ZEDLET_DIR)))
+		zed_log_die("Failed to create conf: %s", strerror(errno));
-		goto nomem;
-
-	if (!(zcp->state_file = strdup(ZED_STATE_FILE)))
-		goto nomem;
-
-	return (zcp);
-
-nomem:
-	zed_log_die("Failed to create conf: %s", strerror(errno));
-	return (NULL);
 }
 
 /*
@@ -79,9 +62,6 @@ nomem:
 void
 zed_conf_destroy(struct zed_conf *zcp)
 {
-	if (!zcp)
-		return;
-
 	if (zcp->state_fd >= 0) {
 		if (close(zcp->state_fd) < 0)
 			zed_log_msg(LOG_WARNING,
@@ -102,10 +82,6 @@ zed_conf_destroy(struct zed_conf *zcp)
 			    zcp->pid_file, strerror(errno));
 		zcp->pid_fd = -1;
 	}
-	if (zcp->conf_file) {
-		free(zcp->conf_file);
-		zcp->conf_file = NULL;
-	}
 	if (zcp->pid_file) {
 		free(zcp->pid_file);
 		zcp->pid_file = NULL;
@@ -122,7 +98,6 @@ zed_conf_destroy(struct zed_conf *zcp)
 		zed_strings_destroy(zcp->zedlets);
 		zcp->zedlets = NULL;
 	}
-	free(zcp);
 }
 
 /*
@@ -132,46 +107,52 @@ zed_conf_destroy(struct zed_conf *zcp)
  * otherwise, output to stderr and exit with a failure status.
  */
 static void
-_zed_conf_display_help(const char *prog, int got_err)
+_zed_conf_display_help(const char *prog, boolean_t got_err)
 {
+	struct opt { const char *o, *d, *v; };
+
 	FILE *fp = got_err ? stderr : stdout;
-	int w1 = 4;			/* width of leading whitespace */
+
-	int w2 = 8;			/* width of L-justified option field */
+	struct opt *oo;
+	struct opt iopts[] = {
+		{ .o = "-h", .d = "Display help" },
+		{ .o = "-L", .d = "Display license information" },
+		{ .o = "-V", .d = "Display version information" },
+		{},
+	};
+	struct opt nopts[] = {
+		{ .o = "-v", .d = "Be verbose" },
+		{ .o = "-f", .d = "Force daemon to run" },
+		{ .o = "-F", .d = "Run daemon in the foreground" },
+		{ .o = "-I",
+		    .d = "Idle daemon until kernel module is (re)loaded" },
+		{ .o = "-M", .d = "Lock all pages in memory" },
+		{ .o = "-P", .d = "$PATH for ZED to use (only used by ZTS)" },
+		{ .o = "-Z", .d = "Zero state file" },
+		{},
+	};
+	struct opt vopts[] = {
+		{ .o = "-d DIR", .d = "Read enabled ZEDLETs from DIR.",
+		    .v = ZED_ZEDLET_DIR },
+		{ .o = "-p FILE", .d = "Write daemon's PID to FILE.",
+		    .v = ZED_PID_FILE },
+		{ .o = "-s FILE", .d = "Write daemon's state to FILE.",
+		    .v = ZED_STATE_FILE },
+		{ .o = "-j JOBS", .d = "Start at most JOBS at once.",
+		    .v = "16" },
+		{},
+	};
 
 	fprintf(fp, "Usage: %s [OPTION]...\n", (prog ? prog : "zed"));
 	fprintf(fp, "\n");
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-h",
+	for (oo = iopts; oo->o; ++oo)
-	    "Display help.");
+		fprintf(fp, "    %*s %s\n", -8, oo->o, oo->d);
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-L",
-	    "Display license information.");
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-V",
-	    "Display version information.");
 	fprintf(fp, "\n");
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-v",
+	for (oo = nopts; oo->o; ++oo)
-	    "Be verbose.");
+		fprintf(fp, "    %*s %s\n", -8, oo->o, oo->d);
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-f",
-	    "Force daemon to run.");
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-F",
-	    "Run daemon in the foreground.");
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-I",
-	    "Idle daemon until kernel module is (re)loaded.");
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-M",
-	    "Lock all pages in memory.");
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-P",
-	    "$PATH for ZED to use (only used by ZTS).");
-	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-Z",
-	    "Zero state file.");
 	fprintf(fp, "\n");
-#if 0
+	for (oo = vopts; oo->o; ++oo)
-	fprintf(fp, "%*c%*s %s [%s]\n", w1, 0x20, -w2, "-c FILE",
+		fprintf(fp, "    %*s %s [%s]\n", -8, oo->o, oo->d, oo->v);
-	    "Read configuration from FILE.", ZED_CONF_FILE);
-#endif
-	fprintf(fp, "%*c%*s %s [%s]\n", w1, 0x20, -w2, "-d DIR",
-	    "Read enabled ZEDLETs from DIR.", ZED_ZEDLET_DIR);
-	fprintf(fp, "%*c%*s %s [%s]\n", w1, 0x20, -w2, "-p FILE",
-	    "Write daemon's PID to FILE.", ZED_PID_FILE);
-	fprintf(fp, "%*c%*s %s [%s]\n", w1, 0x20, -w2, "-s FILE",
-	    "Write daemon's state to FILE.", ZED_STATE_FILE);
 	fprintf(fp, "\n");
 
 	exit(got_err ? EXIT_FAILURE : EXIT_SUCCESS);
@@ -183,20 +164,14 @@ _zed_conf_display_help(const char *prog, int got_err)
 static void
 _zed_conf_display_license(void)
 {
-	const char **pp;
+	printf(
-	const char *text[] = {
+	    "The ZFS Event Daemon (ZED) is distributed under the terms of the\n"
-	    "The ZFS Event Daemon (ZED) is distributed under the terms of the",
+	    "  Common Development and Distribution License (CDDL-1.0)\n"
-	    "  Common Development and Distribution License (CDDL-1.0)",
+	    "  <http://opensource.org/licenses/CDDL-1.0>.\n"
-	    "  <http://opensource.org/licenses/CDDL-1.0>.",
+	    "\n"
-	    "",
 	    "Developed at Lawrence Livermore National Laboratory"
-	    " (LLNL-CODE-403049).",
+	    " (LLNL-CODE-403049).\n"
-	    "",
+	    "\n");
-	    NULL
-	};
-
-	for (pp = text; *pp; pp++)
-		printf("%s\n", *pp);
 
 	exit(EXIT_SUCCESS);
 }
@@ -231,16 +206,19 @@ _zed_conf_parse_path(char **resultp, const char *path)
 
 	if (path[0] == '/') {
 		*resultp = strdup(path);
-	} else if (!getcwd(buf, sizeof (buf))) {
-		zed_log_die("Failed to get current working dir: %s",
-		    strerror(errno));
-	} else if (strlcat(buf, "/", sizeof (buf)) >= sizeof (buf)) {
-		zed_log_die("Failed to copy path: %s", strerror(ENAMETOOLONG));
-	} else if (strlcat(buf, path, sizeof (buf)) >= sizeof (buf)) {
-		zed_log_die("Failed to copy path: %s", strerror(ENAMETOOLONG));
 	} else {
+		if (!getcwd(buf, sizeof (buf)))
+			zed_log_die("Failed to get current working dir: %s",
+			    strerror(errno));
+
+		if (strlcat(buf, "/", sizeof (buf)) >= sizeof (buf) ||
+		    strlcat(buf, path, sizeof (buf)) >= sizeof (buf))
+			zed_log_die("Failed to copy path: %s",
+			    strerror(ENAMETOOLONG));
+
 		*resultp = strdup(buf);
 	}
+
 	if (!*resultp)
 		zed_log_die("Failed to copy path: %s", strerror(ENOMEM));
 }
@@ -251,8 +229,9 @@ _zed_conf_parse_path(char **resultp, const char *path)
 void
 zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 {
-	const char * const opts = ":hLVc:d:p:P:s:vfFMZI";
+	const char * const opts = ":hLVd:p:P:s:vfFMZIj:";
 	int opt;
+	unsigned long raw;
 
 	if (!zcp || !argv || !argv[0])
 		zed_log_die("Failed to parse options: Internal error");
@@ -262,7 +241,7 @@ zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 	while ((opt = getopt(argc, argv, opts)) != -1) {
 		switch (opt) {
 		case 'h':
-			_zed_conf_display_help(argv[0], EXIT_SUCCESS);
+			_zed_conf_display_help(argv[0], B_FALSE);
 			break;
 		case 'L':
 			_zed_conf_display_license();
@@ -270,9 +249,6 @@ zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 		case 'V':
 			_zed_conf_display_version();
 			break;
-		case 'c':
-			_zed_conf_parse_path(&zcp->conf_file, optarg);
-			break;
 		case 'd':
 			_zed_conf_parse_path(&zcp->zedlet_dir, optarg);
 			break;
@@ -303,31 +279,30 @@ zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 		case 'Z':
 			zcp->do_zero = 1;
 			break;
+		case 'j':
+			errno = 0;
+			raw = strtoul(optarg, NULL, 0);
+			if (errno == ERANGE || raw > INT16_MAX) {
+				zed_log_die("%lu is too many jobs", raw);
+			} if (raw == 0) {
+				zed_log_die("0 jobs makes no sense");
+			} else {
+				zcp->max_jobs = raw;
+			}
+			break;
 		case '?':
 		default:
 			if (optopt == '?')
-				_zed_conf_display_help(argv[0], EXIT_SUCCESS);
+				_zed_conf_display_help(argv[0], B_FALSE);
 
-			fprintf(stderr, "%s: %s '-%c'\n\n", argv[0],
+			fprintf(stderr, "%s: Invalid option '-%c'\n\n",
-			    "Invalid option", optopt);
+			    argv[0], optopt);
-			_zed_conf_display_help(argv[0], EXIT_FAILURE);
+			_zed_conf_display_help(argv[0], B_TRUE);
 			break;
 		}
 	}
 }
 
-/*
- * Parse the configuration file into the configuration [zcp].
- *
- * FIXME: Not yet implemented.
- */
-void
-zed_conf_parse_file(struct zed_conf *zcp)
-{
-	if (!zcp)
-		zed_log_die("Failed to parse config: %s", strerror(EINVAL));
-}
-
 /*
  * Scan the [zcp] zedlet_dir for files to exec based on the event class.
  * Files must be executable by user, but not writable by group or other.
@@ -335,8 +310,6 @@ zed_conf_parse_file(struct zed_conf *zcp)
  *
  * Return 0 on success with an updated set of zedlets,
  * or -1 on error with errno set.
- *
- * FIXME: Check if zedlet_dir and all parent dirs are secure.
  */
 int
 zed_conf_scan_dir(struct zed_conf *zcp)
@@ -452,8 +425,6 @@ zed_conf_scan_dir(struct zed_conf *zcp)
 int
 zed_conf_write_pid(struct zed_conf *zcp)
 {
-	const mode_t dirmode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
-	const mode_t filemode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
 	char buf[PATH_MAX];
 	int n;
 	char *p;
@@ -481,7 +452,7 @@ zed_conf_write_pid(struct zed_conf *zcp)
 	if (p)
 		*p = '\0';
 
-	if ((mkdirp(buf, dirmode) < 0) && (errno != EEXIST)) {
+	if ((mkdirp(buf, 0755) < 0) && (errno != EEXIST)) {
 		zed_log_msg(LOG_ERR, "Failed to create directory \"%s\": %s",
 		    buf, strerror(errno));
 		goto err;
@@ -491,7 +462,7 @@ zed_conf_write_pid(struct zed_conf *zcp)
 	 */
 	mask = umask(0);
 	umask(mask | 022);
-	zcp->pid_fd = open(zcp->pid_file, (O_RDWR | O_CREAT), filemode);
+	zcp->pid_fd = open(zcp->pid_file, O_RDWR | O_CREAT | O_CLOEXEC, 0644);
 	umask(mask);
 	if (zcp->pid_fd < 0) {
 		zed_log_msg(LOG_ERR, "Failed to open PID file \"%s\": %s",
@@ -528,7 +499,7 @@ zed_conf_write_pid(struct zed_conf *zcp)
 		errno = ERANGE;
 		zed_log_msg(LOG_ERR, "Failed to write PID file \"%s\": %s",
 		    zcp->pid_file, strerror(errno));
-	} else if (zed_file_write_n(zcp->pid_fd, buf, n) != n) {
+	} else if (write(zcp->pid_fd, buf, n) != n) {
 		zed_log_msg(LOG_ERR, "Failed to write PID file \"%s\": %s",
 		    zcp->pid_file, strerror(errno));
 	} else if (fdatasync(zcp->pid_fd) < 0) {
@@ -556,7 +527,6 @@ int
 zed_conf_open_state(struct zed_conf *zcp)
 {
 	char dirbuf[PATH_MAX];
-	mode_t dirmode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
 	int n;
 	char *p;
 	int rv;
@@ -578,7 +548,7 @@ zed_conf_open_state(struct zed_conf *zcp)
 	if (p)
 		*p = '\0';
 
-	if ((mkdirp(dirbuf, dirmode) < 0) && (errno != EEXIST)) {
+	if ((mkdirp(dirbuf, 0755) < 0) && (errno != EEXIST)) {
 		zed_log_msg(LOG_WARNING,
 		    "Failed to create directory \"%s\": %s",
 		    dirbuf, strerror(errno));
@@ -596,7 +566,7 @@ zed_conf_open_state(struct zed_conf *zcp)
 		(void) unlink(zcp->state_file);
 
 	zcp->state_fd = open(zcp->state_file,
-	    (O_RDWR | O_CREAT), (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH));
+	    O_RDWR | O_CREAT | O_CLOEXEC, 0644);
 	if (zcp->state_fd < 0) {
 		zed_log_msg(LOG_WARNING, "Failed to open state file \"%s\": %s",
 		    zcp->state_file, strerror(errno));

cmd/zed/zed_conf.h

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).
@@ -20,43 +20,39 @@
 #include "zed_strings.h"
 
 struct zed_conf {
-	unsigned	do_force:1;		/* true if force enabled */
-	unsigned	do_foreground:1;	/* true if run in foreground */
-	unsigned	do_memlock:1;		/* true if locking memory */
-	unsigned	do_verbose:1;		/* true if verbosity enabled */
-	unsigned	do_zero:1;		/* true if zeroing state */
-	unsigned	do_idle:1;		/* true if idle enabled */
-	int		syslog_facility;	/* syslog facility value */
-	int		min_events;		/* RESERVED FOR FUTURE USE */
-	int		max_events;		/* RESERVED FOR FUTURE USE */
-	char		*conf_file;		/* abs path to config file */
 	char		*pid_file;		/* abs path to pid file */
-	int		pid_fd;			/* fd to pid file for lock */
 	char		*zedlet_dir;		/* abs path to zedlet dir */
-	zed_strings_t	*zedlets;		/* names of enabled zedlets */
 	char		*state_file;		/* abs path to state file */
-	int		state_fd;		/* fd to state file */
+
 	libzfs_handle_t	*zfs_hdl;		/* handle to libzfs */
-	int		zevent_fd;		/* fd for access to zevents */
+	zed_strings_t	*zedlets;		/* names of enabled zedlets */
 	char		*path;		/* custom $PATH for zedlets to use */
+
+	int		pid_fd;			/* fd to pid file for lock */
+	int		state_fd;		/* fd to state file */
+	int		zevent_fd;		/* fd for access to zevents */
+
+	int16_t max_jobs;		/* max zedlets to run at one time */
+
+	boolean_t	do_force:1;		/* true if force enabled */
+	boolean_t	do_foreground:1;	/* true if run in foreground */
+	boolean_t	do_memlock:1;		/* true if locking memory */
+	boolean_t	do_verbose:1;		/* true if verbosity enabled */
+	boolean_t	do_zero:1;		/* true if zeroing state */
+	boolean_t	do_idle:1;		/* true if idle enabled */
 };
 
-struct zed_conf *zed_conf_create(void);
+void zed_conf_init(struct zed_conf *zcp);
-
 void zed_conf_destroy(struct zed_conf *zcp);
 
 void zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv);
 
-void zed_conf_parse_file(struct zed_conf *zcp);
-
 int zed_conf_scan_dir(struct zed_conf *zcp);
 
 int zed_conf_write_pid(struct zed_conf *zcp);
 
 int zed_conf_open_state(struct zed_conf *zcp);
-
 int zed_conf_read_state(struct zed_conf *zcp, uint64_t *eidp, int64_t etime[]);
-
 int zed_conf_write_state(struct zed_conf *zcp, uint64_t eid, int64_t etime[]);
 
 #endif	/* !ZED_CONF_H */

cmd/zed/zed_disk_event.c

@@ -379,6 +379,7 @@ zed_disk_event_init()
 		return (-1);
 	}
 
+	pthread_setname_np(g_mon_tid, "udev monitor");
 	zed_log_msg(LOG_INFO, "zed_disk_event_init");
 
 	return (0);

cmd/zed/zed_event.c

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).
@@ -15,7 +15,7 @@
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <libzfs.h>			/* FIXME: Replace with libzfs_core. */
+#include <libzfs_core.h>
 #include <paths.h>
 #include <stdarg.h>
 #include <stdio.h>
@@ -54,7 +54,7 @@ zed_event_init(struct zed_conf *zcp)
 		zed_log_die("Failed to initialize libzfs");
 	}
 
-	zcp->zevent_fd = open(ZFS_DEV, O_RDWR);
+	zcp->zevent_fd = open(ZFS_DEV, O_RDWR | O_CLOEXEC);
 	if (zcp->zevent_fd < 0) {
 		if (zcp->do_idle)
 			return (-1);
@@ -96,6 +96,47 @@ zed_event_fini(struct zed_conf *zcp)
 		libzfs_fini(zcp->zfs_hdl);
 		zcp->zfs_hdl = NULL;
 	}
+
+	zed_exec_fini();
+}
+
+static void
+_bump_event_queue_length(void)
+{
+	int zzlm = -1, wr;
+	char qlen_buf[12] = {0}; /* parameter is int => max "-2147483647\n" */
+	long int qlen;
+
+	zzlm = open("/sys/module/zfs/parameters/zfs_zevent_len_max", O_RDWR);
+	if (zzlm < 0)
+		goto done;
+
+	if (read(zzlm, qlen_buf, sizeof (qlen_buf)) < 0)
+		goto done;
+	qlen_buf[sizeof (qlen_buf) - 1] = '\0';
+
+	errno = 0;
+	qlen = strtol(qlen_buf, NULL, 10);
+	if (errno == ERANGE)
+		goto done;
+
+	if (qlen <= 0)
+		qlen = 512; /* default zfs_zevent_len_max value */
+	else
+		qlen *= 2;
+
+	if (qlen > INT_MAX)
+		qlen = INT_MAX;
+	wr = snprintf(qlen_buf, sizeof (qlen_buf), "%ld", qlen);
+
+	if (pwrite(zzlm, qlen_buf, wr, 0) < 0)
+		goto done;
+
+	zed_log_msg(LOG_WARNING, "Bumping queue length to %ld", qlen);
+
+done:
+	if (zzlm > -1)
+		(void) close(zzlm);
 }
 
 /*
@@ -136,10 +177,7 @@ zed_event_seek(struct zed_conf *zcp, uint64_t saved_eid, int64_t saved_etime[])
 
 		if (n_dropped > 0) {
 			zed_log_msg(LOG_WARNING, "Missed %d events", n_dropped);
-			/*
+			_bump_event_queue_length();
-			 * FIXME: Increase max size of event nvlist in
-			 *   /sys/module/zfs/parameters/zfs_zevent_len_max ?
-			 */
 		}
 		if (nvlist_lookup_uint64(nvl, "eid", &eid) != 0) {
 			zed_log_msg(LOG_WARNING, "Failed to lookup zevent eid");
@@ -211,7 +249,7 @@ _zed_event_value_is_hex(const char *name)
  *
  * All environment variables in [zsp] should be added through this function.
  */
-static int
+static __attribute__((format(printf, 5, 6))) int
 _zed_event_add_var(uint64_t eid, zed_strings_t *zsp,
     const char *prefix, const char *name, const char *fmt, ...)
 {
@@ -586,8 +624,6 @@ _zed_event_add_string_array(uint64_t eid, zed_strings_t *zsp,
  * Convert the nvpair [nvp] to a string which is added to the environment
  * of the child process.
  * Return 0 on success, -1 on error.
- *
- * FIXME: Refactor with cmd/zpool/zpool_main.c:zpool_do_events_nvprint()?
  */
 static void
 _zed_event_add_nvpair(uint64_t eid, zed_strings_t *zsp, nvpair_t *nvp)
@@ -686,23 +722,11 @@ _zed_event_add_nvpair(uint64_t eid, zed_strings_t *zsp, nvpair_t *nvp)
 		_zed_event_add_var(eid, zsp, prefix, name,
 		    "%llu", (u_longlong_t)i64);
 		break;
-	case DATA_TYPE_NVLIST:
-		_zed_event_add_var(eid, zsp, prefix, name,
-		    "%s", "_NOT_IMPLEMENTED_");			/* FIXME */
-		break;
 	case DATA_TYPE_STRING:
 		(void) nvpair_value_string(nvp, &str);
 		_zed_event_add_var(eid, zsp, prefix, name,
 		    "%s", (str ? str : "<NULL>"));
 		break;
-	case DATA_TYPE_BOOLEAN_ARRAY:
-		_zed_event_add_var(eid, zsp, prefix, name,
-		    "%s", "_NOT_IMPLEMENTED_");			/* FIXME */
-		break;
-	case DATA_TYPE_BYTE_ARRAY:
-		_zed_event_add_var(eid, zsp, prefix, name,
-		    "%s", "_NOT_IMPLEMENTED_");			/* FIXME */
-		break;
 	case DATA_TYPE_INT8_ARRAY:
 		_zed_event_add_int8_array(eid, zsp, prefix, nvp);
 		break;
@@ -730,9 +754,11 @@ _zed_event_add_nvpair(uint64_t eid, zed_strings_t *zsp, nvpair_t *nvp)
 	case DATA_TYPE_STRING_ARRAY:
 		_zed_event_add_string_array(eid, zsp, prefix, nvp);
 		break;
+	case DATA_TYPE_NVLIST:
+	case DATA_TYPE_BOOLEAN_ARRAY:
+	case DATA_TYPE_BYTE_ARRAY:
 	case DATA_TYPE_NVLIST_ARRAY:
-		_zed_event_add_var(eid, zsp, prefix, name,
+		_zed_event_add_var(eid, zsp, prefix, name, "_NOT_IMPLEMENTED_");
-		    "%s", "_NOT_IMPLEMENTED_");			/* FIXME */
 		break;
 	default:
 		errno = EINVAL;
@@ -912,10 +938,7 @@ zed_event_service(struct zed_conf *zcp)
 
 	if (n_dropped > 0) {
 		zed_log_msg(LOG_WARNING, "Missed %d events", n_dropped);
-		/*
+		_bump_event_queue_length();
-		 * FIXME: Increase max size of event nvlist in
-		 * /sys/module/zfs/parameters/zfs_zevent_len_max ?
-		 */
 	}
 	if (nvlist_lookup_uint64(nvl, "eid", &eid) != 0) {
 		zed_log_msg(LOG_WARNING, "Failed to lookup zevent eid");
@@ -953,8 +976,7 @@ zed_event_service(struct zed_conf *zcp)
 
 		_zed_event_add_time_strings(eid, zsp, etime);
 
-		zed_exec_process(eid, class, subclass,
+		zed_exec_process(eid, class, subclass, zcp, zsp);
-		    zcp->zedlet_dir, zcp->zedlets, zsp, zcp->zevent_fd);
 
 		zed_conf_write_state(zcp, eid, etime);
 

cmd/zed/zed_event.h

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).

cmd/zed/zed_exec.c

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).
@@ -18,17 +18,53 @@
 #include <fcntl.h>
 #include <stdlib.h>
 #include <string.h>
+#include <stddef.h>
+#include <sys/avl.h>
+#include <sys/resource.h>
 #include <sys/stat.h>
 #include <sys/wait.h>
 #include <time.h>
 #include <unistd.h>
+#include <pthread.h>
 #include "zed_exec.h"
-#include "zed_file.h"
 #include "zed_log.h"
 #include "zed_strings.h"
 
 #define	ZEVENT_FILENO	3
 
+struct launched_process_node {
+	avl_node_t node;
+	pid_t pid;
+	uint64_t eid;
+	char *name;
+};
+
+static int
+_launched_process_node_compare(const void *x1, const void *x2)
+{
+	pid_t p1;
+	pid_t p2;
+
+	assert(x1 != NULL);
+	assert(x2 != NULL);
+
+	p1 = ((const struct launched_process_node *) x1)->pid;
+	p2 = ((const struct launched_process_node *) x2)->pid;
+
+	if (p1 < p2)
+		return (-1);
+	else if (p1 == p2)
+		return (0);
+	else
+		return (1);
+}
+
+static pthread_t _reap_children_tid = (pthread_t)-1;
+static volatile boolean_t _reap_children_stop;
+static avl_tree_t _launched_processes;
+static pthread_mutex_t _launched_processes_lock = PTHREAD_MUTEX_INITIALIZER;
+static int16_t _launched_processes_limit;
+
 /*
  * Create an environment string array for passing to execve() using the
  * NAME=VALUE strings in container [zsp].
@@ -79,20 +115,26 @@ _zed_exec_create_env(zed_strings_t *zsp)
  */
 static void
 _zed_exec_fork_child(uint64_t eid, const char *dir, const char *prog,
-    char *env[], int zfd)
+    char *env[], int zfd, boolean_t in_foreground)
 {
 	char path[PATH_MAX];
 	int n;
 	pid_t pid;
 	int fd;
-	pid_t wpid;
+	struct launched_process_node *node;
-	int status;
+	sigset_t mask;
+	struct timespec launch_timeout =
+		{ .tv_sec = 0, .tv_nsec = 200 * 1000 * 1000, };
 
 	assert(dir != NULL);
 	assert(prog != NULL);
 	assert(env != NULL);
 	assert(zfd >= 0);
 
+	while (__atomic_load_n(&_launched_processes_limit,
+	    __ATOMIC_SEQ_CST) <= 0)
+		(void) nanosleep(&launch_timeout, NULL);
+
 	n = snprintf(path, sizeof (path), "%s/%s", dir, prog);
 	if ((n < 0) || (n >= sizeof (path))) {
 		zed_log_msg(LOG_WARNING,
@@ -100,101 +142,179 @@ _zed_exec_fork_child(uint64_t eid, const char *dir, const char *prog,
 		    prog, eid, strerror(ENAMETOOLONG));
 		return;
 	}
+	(void) pthread_mutex_lock(&_launched_processes_lock);
 	pid = fork();
 	if (pid < 0) {
+		(void) pthread_mutex_unlock(&_launched_processes_lock);
 		zed_log_msg(LOG_WARNING,
 		    "Failed to fork \"%s\" for eid=%llu: %s",
 		    prog, eid, strerror(errno));
 		return;
 	} else if (pid == 0) {
+		(void) sigemptyset(&mask);
+		(void) sigprocmask(SIG_SETMASK, &mask, NULL);
+
 		(void) umask(022);
-		if ((fd = open("/dev/null", O_RDWR)) != -1) {
+		if (in_foreground && /* we're already devnulled if daemonised */
+		    (fd = open("/dev/null", O_RDWR | O_CLOEXEC)) != -1) {
 			(void) dup2(fd, STDIN_FILENO);
 			(void) dup2(fd, STDOUT_FILENO);
 			(void) dup2(fd, STDERR_FILENO);
 		}
 		(void) dup2(zfd, ZEVENT_FILENO);
-		zed_file_close_from(ZEVENT_FILENO + 1);
 		execle(path, prog, NULL, env);
 		_exit(127);
 	}
 
 	/* parent process */
 
+	node = calloc(1, sizeof (*node));
+	if (node) {
+		node->pid = pid;
+		node->eid = eid;
+		node->name = strdup(prog);
+
+		avl_add(&_launched_processes, node);
+	}
+	(void) pthread_mutex_unlock(&_launched_processes_lock);
+
+	__atomic_sub_fetch(&_launched_processes_limit, 1, __ATOMIC_SEQ_CST);
 	zed_log_msg(LOG_INFO, "Invoking \"%s\" eid=%llu pid=%d",
 	    prog, eid, pid);
+}
 
-	/* FIXME: Timeout rogue child processes with sigalarm? */
+static void
+_nop(int sig)
+{}
 
-	/*
+static void *
-	 * Wait for child process using WNOHANG to limit
+_reap_children(void *arg)
-	 * the time spent waiting to 10 seconds (10,000ms).
+{
-	 */
+	struct launched_process_node node, *pnode;
-	for (n = 0; n < 1000; n++) {
+	pid_t pid;
-		wpid = waitpid(pid, &status, WNOHANG);
+	int status;
-		if (wpid == (pid_t)-1) {
+	struct rusage usage;
-			if (errno == EINTR)
+	struct sigaction sa = {};
-				continue;
-			zed_log_msg(LOG_WARNING,
-			    "Failed to wait for \"%s\" eid=%llu pid=%d",
-			    prog, eid, pid);
-			break;
-		} else if (wpid == 0) {
-			struct timespec t;
 
-			/* child still running */
+	(void) sigfillset(&sa.sa_mask);
-			t.tv_sec = 0;
+	(void) sigdelset(&sa.sa_mask, SIGCHLD);
-			t.tv_nsec = 10000000;	/* 10ms */
+	(void) pthread_sigmask(SIG_SETMASK, &sa.sa_mask, NULL);
-			(void) nanosleep(&t, NULL);
-			continue;
-		}
 
-		if (WIFEXITED(status)) {
+	(void) sigemptyset(&sa.sa_mask);
-			zed_log_msg(LOG_INFO,
+	sa.sa_handler = _nop;
-			    "Finished \"%s\" eid=%llu pid=%d exit=%d",
+	sa.sa_flags = SA_NOCLDSTOP;
-			    prog, eid, pid, WEXITSTATUS(status));
+	(void) sigaction(SIGCHLD, &sa, NULL);
-		} else if (WIFSIGNALED(status)) {
+
-			zed_log_msg(LOG_INFO,
+	for (_reap_children_stop = B_FALSE; !_reap_children_stop; ) {
-			    "Finished \"%s\" eid=%llu pid=%d sig=%d/%s",
+		(void) pthread_mutex_lock(&_launched_processes_lock);
-			    prog, eid, pid, WTERMSIG(status),
+		pid = wait4(0, &status, WNOHANG, &usage);
-			    strsignal(WTERMSIG(status)));
+
+		if (pid == 0 || pid == (pid_t)-1) {
+			(void) pthread_mutex_unlock(&_launched_processes_lock);
+			if (pid == 0 || errno == ECHILD)
+				pause();
+			else if (errno != EINTR)
+				zed_log_msg(LOG_WARNING,
+				    "Failed to wait for children: %s",
+				    strerror(errno));
 		} else {
-			zed_log_msg(LOG_INFO,
+			memset(&node, 0, sizeof (node));
-			    "Finished \"%s\" eid=%llu pid=%d status=0x%X",
+			node.pid = pid;
-			    prog, eid, (unsigned int) status);
+			pnode = avl_find(&_launched_processes, &node, NULL);
+			if (pnode) {
+				memcpy(&node, pnode, sizeof (node));
+
+				avl_remove(&_launched_processes, pnode);
+				free(pnode);
+			}
+			(void) pthread_mutex_unlock(&_launched_processes_lock);
+			__atomic_add_fetch(&_launched_processes_limit, 1,
+			    __ATOMIC_SEQ_CST);
+
+			usage.ru_utime.tv_sec += usage.ru_stime.tv_sec;
+			usage.ru_utime.tv_usec += usage.ru_stime.tv_usec;
+			usage.ru_utime.tv_sec +=
+			    usage.ru_utime.tv_usec / (1000 * 1000);
+			usage.ru_utime.tv_usec %= 1000 * 1000;
+
+			if (WIFEXITED(status)) {
+				zed_log_msg(LOG_INFO,
+				    "Finished \"%s\" eid=%llu pid=%d "
+				    "time=%llu.%06us exit=%d",
+				    node.name, node.eid, pid,
+				    (unsigned long long) usage.ru_utime.tv_sec,
+				    (unsigned int) usage.ru_utime.tv_usec,
+				    WEXITSTATUS(status));
+			} else if (WIFSIGNALED(status)) {
+				zed_log_msg(LOG_INFO,
+				    "Finished \"%s\" eid=%llu pid=%d "
+				    "time=%llu.%06us sig=%d/%s",
+				    node.name, node.eid, pid,
+				    (unsigned long long) usage.ru_utime.tv_sec,
+				    (unsigned int) usage.ru_utime.tv_usec,
+				    WTERMSIG(status),
+				    strsignal(WTERMSIG(status)));
+			} else {
+				zed_log_msg(LOG_INFO,
+				    "Finished \"%s\" eid=%llu pid=%d "
+				    "time=%llu.%06us status=0x%X",
+				    node.name, node.eid,
+				    (unsigned long long) usage.ru_utime.tv_sec,
+				    (unsigned int) usage.ru_utime.tv_usec,
+				    (unsigned int) status);
+			}
+
+			free(node.name);
 		}
-		break;
 	}
 
-	/*
+	return (NULL);
-	 * kill child process after 10 seconds
+}
-	 */
+
-	if (wpid == 0) {
+void
-		zed_log_msg(LOG_WARNING, "Killing hung \"%s\" pid=%d",
+zed_exec_fini(void)
-		    prog, pid);
+{
-		(void) kill(pid, SIGKILL);
+	struct launched_process_node *node;
-		(void) waitpid(pid, &status, 0);
+	void *ck = NULL;
+
+	if (_reap_children_tid == (pthread_t)-1)
+		return;
+
+	_reap_children_stop = B_TRUE;
+	(void) pthread_kill(_reap_children_tid, SIGCHLD);
+	(void) pthread_join(_reap_children_tid, NULL);
+
+	while ((node = avl_destroy_nodes(&_launched_processes, &ck)) != NULL) {
+		free(node->name);
+		free(node);
 	}
+	avl_destroy(&_launched_processes);
+
+	(void) pthread_mutex_destroy(&_launched_processes_lock);
+	(void) pthread_mutex_init(&_launched_processes_lock, NULL);
+
+	_reap_children_tid = (pthread_t)-1;
 }
 
 /*
  * Process the event [eid] by synchronously invoking all zedlets with a
  * matching class prefix.
  *
- * Each executable in [zedlets] from the directory [dir] is matched against
+ * Each executable in [zcp->zedlets] from the directory [zcp->zedlet_dir]
- * the event's [class], [subclass], and the "all" class (which matches
+ * is matched against the event's [class], [subclass], and the "all" class
- * all events).  Every zedlet with a matching class prefix is invoked.
+ * (which matches all events).
+ * Every zedlet with a matching class prefix is invoked.
  * The NAME=VALUE strings in [envs] will be passed to the zedlet as
  * environment variables.
  *
- * The file descriptor [zfd] is the zevent_fd used to track the
+ * The file descriptor [zcp->zevent_fd] is the zevent_fd used to track the
  * current cursor location within the zevent nvlist.
  *
  * Return 0 on success, -1 on error.
  */
 int
 zed_exec_process(uint64_t eid, const char *class, const char *subclass,
-    const char *dir, zed_strings_t *zedlets, zed_strings_t *envs, int zfd)
+    struct zed_conf *zcp, zed_strings_t *envs)
 {
 	const char *class_strings[4];
 	const char *allclass = "all";
@@ -203,9 +323,22 @@ zed_exec_process(uint64_t eid, const char *class, const char *subclass,
 	char **e;
 	int n;
 
-	if (!dir || !zedlets || !envs || zfd < 0)
+	if (!zcp->zedlet_dir || !zcp->zedlets || !envs || zcp->zevent_fd < 0)
 		return (-1);
 
+	if (_reap_children_tid == (pthread_t)-1) {
+		_launched_processes_limit = zcp->max_jobs;
+
+		if (pthread_create(&_reap_children_tid, NULL,
+		    _reap_children, NULL) != 0)
+			return (-1);
+		pthread_setname_np(_reap_children_tid, "reap ZEDLETs");
+
+		avl_create(&_launched_processes, _launched_process_node_compare,
+		    sizeof (struct launched_process_node),
+		    offsetof(struct launched_process_node, node));
+	}
+
 	csp = class_strings;
 
 	if (class)
@@ -221,11 +354,13 @@ zed_exec_process(uint64_t eid, const char *class, const char *subclass,
 
 	e = _zed_exec_create_env(envs);
 
-	for (z = zed_strings_first(zedlets); z; z = zed_strings_next(zedlets)) {
+	for (z = zed_strings_first(zcp->zedlets); z;
+	    z = zed_strings_next(zcp->zedlets)) {
 		for (csp = class_strings; *csp; csp++) {
 			n = strlen(*csp);
 			if ((strncmp(z, *csp, n) == 0) && !isalpha(z[n]))
-				_zed_exec_fork_child(eid, dir, z, e, zfd);
+				_zed_exec_fork_child(eid, zcp->zedlet_dir,
+				    z, e, zcp->zevent_fd, zcp->do_foreground);
 		}
 	}
 	free(e);

cmd/zed/zed_exec.h

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).
@@ -17,9 +17,11 @@
 
 #include <stdint.h>
 #include "zed_strings.h"
+#include "zed_conf.h"
+
+void zed_exec_fini(void);
 
 int zed_exec_process(uint64_t eid, const char *class, const char *subclass,
-    const char *dir, zed_strings_t *zedlets, zed_strings_t *envs,
+    struct zed_conf *zcp, zed_strings_t *envs);
-    int zevent_fd);
 
 #endif	/* !ZED_EXEC_H */

cmd/zed/zed_file.c

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).
@@ -12,73 +12,17 @@
  * You may not use this file except in compliance with the license.
  */
 
+#include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
 #include <string.h>
-#include <sys/resource.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include "zed_file.h"
 #include "zed_log.h"
 
-/*
- * Read up to [n] bytes from [fd] into [buf].
- * Return the number of bytes read, 0 on EOF, or -1 on error.
- */
-ssize_t
-zed_file_read_n(int fd, void *buf, size_t n)
-{
-	unsigned char *p;
-	size_t n_left;
-	ssize_t n_read;
-
-	p = buf;
-	n_left = n;
-	while (n_left > 0) {
-		if ((n_read = read(fd, p, n_left)) < 0) {
-			if (errno == EINTR)
-				continue;
-			else
-				return (-1);
-
-		} else if (n_read == 0) {
-			break;
-		}
-		n_left -= n_read;
-		p += n_read;
-	}
-	return (n - n_left);
-}
-
-/*
- * Write [n] bytes from [buf] out to [fd].
- * Return the number of bytes written, or -1 on error.
- */
-ssize_t
-zed_file_write_n(int fd, void *buf, size_t n)
-{
-	const unsigned char *p;
-	size_t n_left;
-	ssize_t n_written;
-
-	p = buf;
-	n_left = n;
-	while (n_left > 0) {
-		if ((n_written = write(fd, p, n_left)) < 0) {
-			if (errno == EINTR)
-				continue;
-			else
-				return (-1);
-
-		}
-		n_left -= n_written;
-		p += n_written;
-	}
-	return (n);
-}
-
 /*
  * Set an exclusive advisory lock on the open file descriptor [fd].
  * Return 0 on success, 1 if a conflicting lock is held by another process,
@@ -160,6 +104,13 @@ zed_file_is_locked(int fd)
 	return (lock.l_pid);
 }
 
+
+#if __APPLE__
+#define	PROC_SELF_FD "/dev/fd"
+#else /* Linux-compatible layout */
+#define	PROC_SELF_FD "/proc/self/fd"
+#endif
+
 /*
  * Close all open file descriptors greater than or equal to [lowfd].
  * Any errors encountered while closing file descriptors are ignored.
@@ -167,51 +118,24 @@ zed_file_is_locked(int fd)
 void
 zed_file_close_from(int lowfd)
 {
-	const int maxfd_def = 256;
+	int errno_bak = errno;
-	int errno_bak;
+	int maxfd = 0;
-	struct rlimit rl;
-	int maxfd;
 	int fd;
+	DIR *fddir;
+	struct dirent *fdent;
 
-	errno_bak = errno;
+	if ((fddir = opendir(PROC_SELF_FD)) != NULL) {
-
+		while ((fdent = readdir(fddir)) != NULL) {
-	if (getrlimit(RLIMIT_NOFILE, &rl) < 0) {
+			fd = atoi(fdent->d_name);
-		maxfd = maxfd_def;
+			if (fd > maxfd && fd != dirfd(fddir))
-	} else if (rl.rlim_max == RLIM_INFINITY) {
+				maxfd = fd;
-		maxfd = maxfd_def;
+		}
+		(void) closedir(fddir);
 	} else {
-		maxfd = rl.rlim_max;
+		maxfd = sysconf(_SC_OPEN_MAX);
 	}
 	for (fd = lowfd; fd < maxfd; fd++)
 		(void) close(fd);
 
 	errno = errno_bak;
 }
-
-/*
- * Set the CLOEXEC flag on file descriptor [fd] so it will be automatically
- * closed upon successful execution of one of the exec functions.
- * Return 0 on success, or -1 on error.
- *
- * FIXME: No longer needed?
- */
-int
-zed_file_close_on_exec(int fd)
-{
-	int flags;
-
-	if (fd < 0) {
-		errno = EBADF;
-		return (-1);
-	}
-	flags = fcntl(fd, F_GETFD);
-	if (flags == -1)
-		return (-1);
-
-	flags |= FD_CLOEXEC;
-
-	if (fcntl(fd, F_SETFD, flags) == -1)
-		return (-1);
-
-	return (0);
-}

cmd/zed/zed_file.h

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).
@@ -18,10 +18,6 @@
 #include <sys/types.h>
 #include <unistd.h>
 
-ssize_t zed_file_read_n(int fd, void *buf, size_t n);
-
-ssize_t zed_file_write_n(int fd, void *buf, size_t n);
-
 int zed_file_lock(int fd);
 
 int zed_file_unlock(int fd);
@@ -30,6 +26,4 @@ pid_t zed_file_is_locked(int fd);
 
 void zed_file_close_from(int fd);
 
-int zed_file_close_on_exec(int fd);
-
 #endif	/* !ZED_FILE_H */

cmd/zed/zed_log.c

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).

cmd/zed/zed_log.h

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).

cmd/zed/zed_strings.c

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).

cmd/zed/zed_strings.h

@@ -3,7 +3,7 @@
  *
  * Developed at Lawrence Livermore National Laboratory (LLNL-CODE-403049).
  * Copyright (C) 2013-2014 Lawrence Livermore National Security, LLC.
- * Refer to the ZoL git commit log for authoritative copyright attribution.
+ * Refer to the OpenZFS git commit log for authoritative copyright attribution.
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License Version 1.0 (CDDL-1.0).

cmd/zfs/zfs_main.c

@@ -728,6 +728,32 @@ finish_progress(char *done)
 	pt_header = NULL;
 }
 
+/* This function checks if the passed fd refers to /dev/null or /dev/zero */
+#ifdef __linux__
+static boolean_t
+is_dev_nullzero(int fd)
+{
+	struct stat st;
+	fstat(fd, &st);
+	return (major(st.st_rdev) == 1 && (minor(st.st_rdev) == 3 /* null */ ||
+	    minor(st.st_rdev) == 5 /* zero */));
+}
+#endif
+
+static void
+note_dev_error(int err, int fd)
+{
+#ifdef __linux__
+	if (err == EINVAL && is_dev_nullzero(fd)) {
+		(void) fprintf(stderr,
+		    gettext("Error: Writing directly to /dev/{null,zero} files"
+		    " on certain kernels is not currently implemented.\n"
+		    "(As a workaround, "
+		    "try \"zfs send [...] | cat > /dev/null\")\n"));
+	}
+#endif
+}
+
 static int
 zfs_mount_and_share(libzfs_handle_t *hdl, const char *dataset, zfs_type_t type)
 {
@@ -4376,6 +4402,7 @@ zfs_do_send(int argc, char **argv)
 
 	struct option long_options[] = {
 		{"replicate",	no_argument,		NULL, 'R'},
+		{"skip-missing",	no_argument,		NULL, 's'},
 		{"redact",	required_argument,	NULL, 'd'},
 		{"props",	no_argument,		NULL, 'p'},
 		{"parsable",	no_argument,		NULL, 'P'},
@@ -4394,7 +4421,7 @@ zfs_do_send(int argc, char **argv)
 	};
 
 	/* check options */
-	while ((c = getopt_long(argc, argv, ":i:I:RDpvnPLeht:cwbd:S",
+	while ((c = getopt_long(argc, argv, ":i:I:RsDpvnPLeht:cwbd:S",
 	    long_options, NULL)) != -1) {
 		switch (c) {
 		case 'i':
@@ -4411,6 +4438,9 @@ zfs_do_send(int argc, char **argv)
 		case 'R':
 			flags.replicate = B_TRUE;
 			break;
+		case 's':
+			flags.skipmissing = B_TRUE;
+			break;
 		case 'd':
 			redactbook = optarg;
 			break;
@@ -4568,11 +4598,23 @@ zfs_do_send(int argc, char **argv)
 
 		err = zfs_send_saved(zhp, &flags, STDOUT_FILENO,
 		    resume_token);
+		if (err != 0)
+			note_dev_error(errno, STDOUT_FILENO);
 		zfs_close(zhp);
 		return (err != 0);
 	} else if (resume_token != NULL) {
-		return (zfs_send_resume(g_zfs, &flags, STDOUT_FILENO,
+		err = zfs_send_resume(g_zfs, &flags, STDOUT_FILENO,
-		    resume_token));
+		    resume_token);
+		if (err != 0)
+			note_dev_error(errno, STDOUT_FILENO);
+		return (err);
+	}
+
+	if (flags.skipmissing && !flags.replicate) {
+		(void) fprintf(stderr,
+		    gettext("skip-missing flag can only be used in "
+		    "conjunction with replicate\n"));
+		usage(B_FALSE);
 	}
 
 	/*
@@ -4616,6 +4658,8 @@ zfs_do_send(int argc, char **argv)
 		err = zfs_send_one(zhp, fromname, STDOUT_FILENO, &flags,
 		    redactbook);
 		zfs_close(zhp);
+		if (err != 0)
+			note_dev_error(errno, STDOUT_FILENO);
 		return (err != 0);
 	}
 
@@ -4692,6 +4736,7 @@ zfs_do_send(int argc, char **argv)
 		nvlist_free(dbgnv);
 	}
 	zfs_close(zhp);
+	note_dev_error(errno, STDOUT_FILENO);
 
 	return (err != 0);
 }

cmd/zfs_ids_to_path/zfs_ids_to_path.c

@@ -35,7 +35,7 @@ libzfs_handle_t *g_zfs;
 static void
 usage(int err)
 {
-	fprintf(stderr, "Usage: [-v] zfs_ids_to_path <pool> <objset id> "
+	fprintf(stderr, "Usage: zfs_ids_to_path [-v] <pool> <objset id> "
 	    "<object id>\n");
 	exit(err);
 }
@@ -63,11 +63,11 @@ main(int argc, char **argv)
 
 	uint64_t objset, object;
 	if (sscanf(argv[1], "%llu", (u_longlong_t *)&objset) != 1) {
-		(void) fprintf(stderr, "Invalid objset id: %s\n", argv[2]);
+		(void) fprintf(stderr, "Invalid objset id: %s\n", argv[1]);
 		usage(2);
 	}
 	if (sscanf(argv[2], "%llu", (u_longlong_t *)&object) != 1) {
-		(void) fprintf(stderr, "Invalid object id: %s\n", argv[3]);
+		(void) fprintf(stderr, "Invalid object id: %s\n", argv[2]);
 		usage(3);
 	}
 	if ((g_zfs = libzfs_init()) == NULL) {
@@ -76,7 +76,7 @@ main(int argc, char **argv)
 	}
 	zpool_handle_t *pool = zpool_open(g_zfs, argv[0]);
 	if (pool == NULL) {
-		fprintf(stderr, "Could not open pool %s\n", argv[1]);
+		fprintf(stderr, "Could not open pool %s\n", argv[0]);
 		libzfs_fini(g_zfs);
 		return (5);
 	}

cmd/zgenhostid/zgenhostid.c

@@ -36,8 +36,6 @@
 #include <time.h>
 #include <unistd.h>
 
-static void usage(void);
-
 static void
 usage(void)
 {
@@ -60,12 +58,11 @@ int
 main(int argc, char **argv)
 {
 	/* default file path, can be optionally set by user */
-	char path[PATH_MAX] = "/etc/hostid";
+	const char *path = "/etc/hostid";
 	/* holds converted user input or lrand48() generated value */
 	unsigned long input_i = 0;
 
 	int opt;
-	int pathlen;
 	int force_fwrite = 0;
 	while ((opt = getopt_long(argc, argv, "fo:h?", 0, 0)) != -1) {
 		switch (opt) {
@@ -73,14 +70,7 @@ main(int argc, char **argv)
 			force_fwrite = 1;
 			break;
 		case 'o':
-			pathlen = snprintf(path, sizeof (path), "%s", optarg);
+			path = optarg;
-			if (pathlen >= sizeof (path)) {
-				fprintf(stderr, "%s\n", strerror(EOVERFLOW));
-				exit(EXIT_FAILURE);
-			} else if (pathlen < 1) {
-				fprintf(stderr, "%s\n", strerror(EINVAL));
-				exit(EXIT_FAILURE);
-			}
 			break;
 		case 'h':
 		case '?':
@@ -118,7 +108,7 @@ main(int argc, char **argv)
 	if (force_fwrite == 0 && stat(path, &fstat) == 0 &&
 	    S_ISREG(fstat.st_mode)) {
 		fprintf(stderr, "%s: %s\n", path, strerror(EEXIST));
-			exit(EXIT_FAILURE);
+		exit(EXIT_FAILURE);
 	}
 
 	/*
@@ -137,7 +127,7 @@ main(int argc, char **argv)
 	}
 
 	/*
-	 * we need just 4 bytes in native endianess
+	 * we need just 4 bytes in native endianness
 	 * not using sethostid() because it may be missing or just a stub
 	 */
 	uint32_t hostid = input_i;

cmd/zpool/Makefile.am

@@ -1,4 +1,5 @@
 include $(top_srcdir)/config/Rules.am
+include $(top_srcdir)/config/Shellcheck.am
 
 AM_CFLAGS += $(LIBBLKID_CFLAGS) $(LIBUUID_CFLAGS)
 
@@ -146,6 +147,10 @@ dist_zpoolcompat_DATA = \
 	compatibility.d/openzfsonosx-1.9.3 \
 	compatibility.d/openzfs-2.0-freebsd \
 	compatibility.d/openzfs-2.0-linux \
+	compatibility.d/openzfs-2.1-freebsd \
+	compatibility.d/openzfs-2.1-linux \
+	compatibility.d/zol-0.6.1 \
+	compatibility.d/zol-0.6.4 \
 	compatibility.d/zol-0.6.5 \
 	compatibility.d/zol-0.7 \
 	compatibility.d/zol-0.8

cmd/zpool/compatibility.d/openzfs-2.1-freebsd

@@ -0,0 +1,34 @@
+# Features supported by OpenZFS 2.1 on FreeBSD
+allocation_classes
+async_destroy
+bookmark_v2
+bookmark_written
+bookmarks
+device_rebuild
+device_removal
+draid
+embedded_data
+empty_bpobj
+enabled_txg
+encryption
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+large_dnode
+livelist
+log_spacemap
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+project_quota
+redacted_datasets
+redaction_bookmarks
+resilver_defer
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+userobj_accounting
+zpool_checkpoint
+zstd_compress

cmd/zpool/compatibility.d/openzfs-2.1-linux

@@ -0,0 +1,35 @@
+# Features supported by OpenZFS 2.1 on Linux
+allocation_classes
+async_destroy
+bookmark_v2
+bookmark_written
+bookmarks
+device_rebuild
+device_removal
+draid
+edonr
+embedded_data
+empty_bpobj
+enabled_txg
+encryption
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+large_dnode
+livelist
+log_spacemap
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+project_quota
+redacted_datasets
+redaction_bookmarks
+resilver_defer
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+userobj_accounting
+zpool_checkpoint
+zstd_compress

cmd/zpool/compatibility.d/zol-0.6.1

@@ -0,0 +1,4 @@
+# Features supported by ZFSonLinux v0.6.1
+async_destroy
+empty_bpobj
+lz4_compress

cmd/zpool/compatibility.d/zol-0.6.4

@@ -0,0 +1,10 @@
+# Features supported by ZFSonLinux v0.6.4
+async_destroy
+bookmarks
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+hole_birth
+lz4_compress
+spacemap_histogram

cmd/zpool/os/freebsd/zpool_vdev_os.c

@@ -101,3 +101,18 @@ check_sector_size_database(char *path, int *sector_size)
 {
 	return (0);
 }
+
+void
+after_zpool_upgrade(zpool_handle_t *zhp)
+{
+	char bootfs[ZPOOL_MAXPROPLEN];
+
+	if (zpool_get_prop(zhp, ZPOOL_PROP_BOOTFS, bootfs,
+	    sizeof (bootfs), NULL, B_FALSE) == 0 &&
+	    strcmp(bootfs, "-") != 0) {
+		(void) printf(gettext("Pool '%s' has the bootfs "
+		    "property set, you might need to update\nthe boot "
+		    "code. See gptzfsboot(8) and loader.efi(8) for "
+		    "details.\n"), zpool_get_name(zhp));
+	}
+}

cmd/zpool/os/linux/zpool_vdev_os.c

@@ -405,3 +405,8 @@ check_device(const char *path, boolean_t force,
 
 	return (error);
 }
+
+void
+after_zpool_upgrade(zpool_handle_t *zhp)
+{
+}

cmd/zpool/zpool.d/smart

@@ -53,7 +53,7 @@ get_filename_from_dir()
 	num_files=$(find "$dir" -maxdepth 1 -type f | wc -l)
 	mod=$((pid % num_files))
 	i=0
-	find "$dir" -type f -printf "%f\n" | while read -r file ; do
+	find "$dir" -type f -printf '%f\n' | while read -r file ; do
 		if [ "$mod" = "$i" ] ; then
 			echo "$file"
 			break
@@ -62,17 +62,14 @@ get_filename_from_dir()
 	done
 }
 
-script=$(basename "$0")
+script="${0##*/}"
 
 if [ "$1" = "-h" ] ; then
         echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
         exit
 fi
 
-smartctl_path=$(command -v smartctl)
+if [ -b "$VDEV_UPATH" ] && PATH="/usr/sbin:$PATH" command -v smartctl > /dev/null || [ -n "$samples" ] ; then
-
-# shellcheck disable=SC2015
-if [ -b "$VDEV_UPATH" ] && [ -x "$smartctl_path" ] || [ -n "$samples" ] ; then
 	if [ -n "$samples" ] ; then
 		# cat a smartctl output text file instead of running smartctl
 		# on a vdev (only used for developer testing).
@@ -80,7 +77,7 @@ if [ -b "$VDEV_UPATH" ] && [ -x "$smartctl_path" ] || [ -n "$samples" ] ; then
 		echo "file=$file"
 		raw_out=$(cat "$samples/$file")
 	else
-		raw_out=$(eval "sudo $smartctl_path -a $VDEV_UPATH")
+		raw_out=$(sudo smartctl -a "$VDEV_UPATH")
 	fi
 
 	# What kind of drive are we?  Look for the right line in smartctl:
@@ -231,11 +228,11 @@ esac
 with_vals=$(echo "$out" | grep -E "$scripts")
 if [ -n "$with_vals" ]; then
 	echo "$with_vals"
-	without_vals=$(echo "$scripts" | tr "|" "\n" |
+	without_vals=$(echo "$scripts" | tr '|' '\n' |
 		grep -v -E "$(echo "$with_vals" |
 		awk -F "=" '{print $1}')" | awk '{print $0"="}')
 else
-	without_vals=$(echo "$scripts" | tr "|" "\n" | awk '{print $0"="}')
+	without_vals=$(echo "$scripts" | tr '|' '\n' | awk '{print $0"="}')
 fi
 
 if [ -n "$without_vals" ]; then

cmd/zpool/zpool_iter.c

@@ -494,19 +494,25 @@ vdev_run_cmd(vdev_cmd_data_t *data, char *cmd)
 	/* Setup our custom environment variables */
 	rc = asprintf(&env[1], "VDEV_PATH=%s",
 	    data->path ? data->path : "");
-	if (rc == -1)
+	if (rc == -1) {
+		env[1] = NULL;
 		goto out;
+	}
 
 	rc = asprintf(&env[2], "VDEV_UPATH=%s",
 	    data->upath ? data->upath : "");
-	if (rc == -1)
+	if (rc == -1) {
+		env[2] = NULL;
 		goto out;
+	}
 
 	rc = asprintf(&env[3], "VDEV_ENC_SYSFS_PATH=%s",
 	    data->vdev_enc_sysfs_path ?
 	    data->vdev_enc_sysfs_path : "");
-	if (rc == -1)
+	if (rc == -1) {
+		env[3] = NULL;
 		goto out;
+	}
 
 	/* Run the command */
 	rc = libzfs_run_process_get_stdout_nopath(cmd, argv, env, &lines,
@@ -525,8 +531,7 @@ out:
 
 	/* Start with i = 1 since env[0] was statically allocated */
 	for (i = 1; i < ARRAY_SIZE(env); i++)
-		if (env[i] != NULL)
+		free(env[i]);
-			free(env[i]);
 }
 
 /*

cmd/zpool/zpool_main.c

@@ -32,6 +32,7 @@
  * Copyright (c) 2017, Intel Corporation.
  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
+ * Copyright [2021] Hewlett Packard Enterprise Development LP
  */
 
 #include <assert.h>
@@ -532,7 +533,7 @@ usage(boolean_t requested)
 		(void) fprintf(fp, "YES   disabled | enabled | active\n");
 
 		(void) fprintf(fp, gettext("\nThe feature@ properties must be "
-		    "appended with a feature name.\nSee zpool-features(5).\n"));
+		    "appended with a feature name.\nSee zpool-features(7).\n"));
 	}
 
 	/*
@@ -786,7 +787,7 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
 
 	if (poolprop) {
 		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
-		const char *fname =
+		const char *cname =
 		    zpool_prop_to_name(ZPOOL_PROP_COMPATIBILITY);
 
 		if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
@@ -811,16 +812,19 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
 		}
 
 		/*
-		 * compatibility property and version should not be specified
+		 * if version is specified, only "legacy" compatibility
-		 * at the same time.
+		 * may be requested
 		 */
 		if ((prop == ZPOOL_PROP_COMPATIBILITY &&
+		    strcmp(propval, ZPOOL_COMPAT_LEGACY) != 0 &&
 		    nvlist_exists(proplist, vname)) ||
 		    (prop == ZPOOL_PROP_VERSION &&
-		    nvlist_exists(proplist, fname))) {
+		    nvlist_exists(proplist, cname) &&
-			(void) fprintf(stderr, gettext("'compatibility' and "
+		    strcmp(fnvlist_lookup_string(proplist, cname),
-			    "'version' properties cannot be specified "
+		    ZPOOL_COMPAT_LEGACY) != 0)) {
-			    "together\n"));
+			(void) fprintf(stderr, gettext("when 'version' is "
+			    "specified, the 'compatibility' feature may only "
+			    "be set to '" ZPOOL_COMPAT_LEGACY "'\n"));
 			return (2);
 		}
 
@@ -1065,7 +1069,7 @@ zpool_do_add(int argc, char **argv)
 				free(vname);
 			}
 		}
-		/* And finaly the spares */
+		/* And finally the spares */
 		if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_SPARES,
 		    &sparechild, &sparechildren) == 0 && sparechildren > 0) {
 			hadspare = B_TRUE;
@@ -1674,6 +1678,7 @@ zpool_do_create(int argc, char **argv)
 		 *    - enable_pool_features (ie: no '-d' or '-o version')
 		 *    - it's supported by the kernel module
 		 *    - it's in the requested feature set
+		 *  - warn if it's enabled but not in compat
 		 */
 		for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
 			char propname[MAXPATHLEN];
@@ -1687,6 +1692,14 @@ zpool_do_create(int argc, char **argv)
 				if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
 					(void) nvlist_remove_all(props,
 					    propname);
+				if (strcmp(propval,
+				    ZFS_FEATURE_ENABLED) == 0 &&
+				    !requested_features[i])
+					(void) fprintf(stderr, gettext(
+					    "Warning: feature \"%s\" enabled "
+					    "but is not in specified "
+					    "'compatibility' feature set.\n"),
+					    feat->fi_uname);
 			} else if (
 			    enable_pool_features &&
 			    feat->fi_zfs_mod_supported &&
@@ -2367,6 +2380,10 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
 			(void) printf(gettext("all children offline"));
 			break;
 
+		case VDEV_AUX_BAD_LABEL:
+			(void) printf(gettext("invalid label"));
+			break;
+
 		default:
 			(void) printf(gettext("corrupted data"));
 			break;
@@ -2509,6 +2526,10 @@ print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
 			(void) printf(gettext("all children offline"));
 			break;
 
+		case VDEV_AUX_BAD_LABEL:
+			(void) printf(gettext("invalid label"));
+			break;
+
 		default:
 			(void) printf(gettext("corrupted data"));
 			break;
@@ -2717,8 +2738,10 @@ show_import(nvlist_t *config, boolean_t report_error)
 
 	case ZPOOL_STATUS_FEAT_DISABLED:
 		printf_color(ANSI_BOLD, gettext("status: "));
-		printf_color(ANSI_YELLOW, gettext("Some supported and "
+		printf_color(ANSI_YELLOW, gettext("Some supported "
-		    "requested features are not enabled on the pool.\n"));
+		    "features are not enabled on the pool.\n\t"
+		    "(Note that they may be intentionally disabled "
+		    "if the\n\t'compatibility' property is set.)\n"));
 		break;
 
 	case ZPOOL_STATUS_COMPATIBILITY_ERR:
@@ -2728,6 +2751,13 @@ show_import(nvlist_t *config, boolean_t report_error)
 		    "property.\n"));
 		break;
 
+	case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
+		printf_color(ANSI_BOLD, gettext("status: "));
+		printf_color(ANSI_YELLOW, gettext("One or more features "
+		    "are enabled on the pool despite not being\n"
+		    "requested by the 'compatibility' property.\n"));
+		break;
+
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
 		printf_color(ANSI_BOLD, gettext("status: "));
 		printf_color(ANSI_YELLOW, gettext("The pool uses the following "
@@ -4965,8 +4995,8 @@ get_interval_count(int *argcp, char **argv, float *iv,
 
 		if (*end == '\0' && errno == 0) {
 			if (interval == 0) {
-				(void) fprintf(stderr, gettext("interval "
+				(void) fprintf(stderr, gettext(
-				    "cannot be zero\n"));
+				    "interval cannot be zero\n"));
 				usage(B_FALSE);
 			}
 			/*
@@ -4996,8 +5026,8 @@ get_interval_count(int *argcp, char **argv, float *iv,
 
 		if (*end == '\0' && errno == 0) {
 			if (interval == 0) {
-				(void) fprintf(stderr, gettext("interval "
+				(void) fprintf(stderr, gettext(
-				    "cannot be zero\n"));
+				    "interval cannot be zero\n"));
 				usage(B_FALSE);
 			}
 
@@ -6000,7 +6030,7 @@ print_one_column(zpool_prop_t prop, uint64_t value, const char *str,
 		break;
 	case ZPOOL_PROP_HEALTH:
 		width = 8;
-		snprintf(propval, sizeof (propval), "%-*s", (int)width, str);
+		(void) strlcpy(propval, str, sizeof (propval));
 		break;
 	default:
 		zfs_nicenum_format(value, propval, sizeof (propval), format);
@@ -8055,7 +8085,8 @@ status_callback(zpool_handle_t *zhp, void *data)
 	    (reason == ZPOOL_STATUS_OK ||
 	    reason == ZPOOL_STATUS_VERSION_OLDER ||
 	    reason == ZPOOL_STATUS_FEAT_DISABLED ||
-	    reason == ZPOOL_STATUS_COMPATIBILITY_ERR)) {
+	    reason == ZPOOL_STATUS_COMPATIBILITY_ERR ||
+	    reason == ZPOOL_STATUS_INCOMPATIBLE_FEAT)) {
 		if (!cbp->cb_allpools) {
 			(void) printf(gettext("pool '%s' is healthy\n"),
 			    zpool_get_name(zhp));
@@ -8238,7 +8269,7 @@ status_callback(zpool_handle_t *zhp, void *data)
 		printf_color(ANSI_YELLOW, gettext("Enable all features using "
 		    "'zpool upgrade'. Once this is done,\n\tthe pool may no "
 		    "longer be accessible by software that does not support\n\t"
-		    "the features. See zpool-features(5) for details.\n"));
+		    "the features. See zpool-features(7) for details.\n"));
 		break;
 
 	case ZPOOL_STATUS_COMPATIBILITY_ERR:
@@ -8254,6 +8285,18 @@ status_callback(zpool_handle_t *zhp, void *data)
 		    ZPOOL_DATA_COMPAT_D ".\n"));
 		break;
 
+	case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
+		printf_color(ANSI_BOLD, gettext("status: "));
+		printf_color(ANSI_YELLOW, gettext("One or more features "
+		    "are enabled on the pool despite not being\n\t"
+		    "requested by the 'compatibility' property.\n"));
+		printf_color(ANSI_BOLD, gettext("action: "));
+		printf_color(ANSI_YELLOW, gettext("Consider setting "
+		    "'compatibility' to an appropriate value, or\n\t"
+		    "adding needed features to the relevant file in\n\t"
+		    ZPOOL_SYSCONF_COMPAT_D " or " ZPOOL_DATA_COMPAT_D ".\n"));
+		break;
+
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
 		printf_color(ANSI_BOLD, gettext("status: "));
 		printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
@@ -8713,6 +8756,11 @@ upgrade_version(zpool_handle_t *zhp, uint64_t version)
 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
 	    &oldversion) == 0);
 
+	char compat[ZFS_MAXPROPLEN];
+	if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
+	    ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
+		compat[0] = '\0';
+
 	assert(SPA_VERSION_IS_SUPPORTED(oldversion));
 	assert(oldversion < version);
 
@@ -8727,6 +8775,13 @@ upgrade_version(zpool_handle_t *zhp, uint64_t version)
 		return (1);
 	}
 
+	if (strcmp(compat, ZPOOL_COMPAT_LEGACY) == 0) {
+		(void) fprintf(stderr, gettext("Upgrade not performed because "
+		    "'compatibility' property set to '"
+		    ZPOOL_COMPAT_LEGACY "'.\n"));
+		return (1);
+	}
+
 	ret = zpool_upgrade(zhp, version);
 	if (ret != 0)
 		return (ret);
@@ -8803,7 +8858,7 @@ upgrade_cb(zpool_handle_t *zhp, void *arg)
 	upgrade_cbdata_t *cbp = arg;
 	nvlist_t *config;
 	uint64_t version;
-	boolean_t printnl = B_FALSE;
+	boolean_t modified_pool = B_FALSE;
 	int ret;
 
 	config = zpool_get_config(zhp, NULL);
@@ -8817,7 +8872,7 @@ upgrade_cb(zpool_handle_t *zhp, void *arg)
 		ret = upgrade_version(zhp, cbp->cb_version);
 		if (ret != 0)
 			return (ret);
-		printnl = B_TRUE;
+		modified_pool = B_TRUE;
 
 		/*
 		 * If they did "zpool upgrade -a", then we could
@@ -8837,12 +8892,13 @@ upgrade_cb(zpool_handle_t *zhp, void *arg)
 
 		if (count > 0) {
 			cbp->cb_first = B_FALSE;
-			printnl = B_TRUE;
+			modified_pool = B_TRUE;
 		}
 	}
 
-	if (printnl) {
+	if (modified_pool) {
-		(void) printf(gettext("\n"));
+		(void) printf("\n");
+		(void) after_zpool_upgrade(zhp);
 	}
 
 	return (0);
@@ -8868,7 +8924,10 @@ upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
 			    "be upgraded to use feature flags.  After "
 			    "being upgraded, these pools\nwill no "
 			    "longer be accessible by software that does not "
-			    "support feature\nflags.\n\n"));
+			    "support feature\nflags.\n\n"
+			    "Note that setting a pool's 'compatibility' "
+			    "feature to '" ZPOOL_COMPAT_LEGACY "' will\n"
+			    "inhibit upgrades.\n\n"));
 			(void) printf(gettext("VER  POOL\n"));
 			(void) printf(gettext("---  ------------\n"));
 			cbp->cb_first = B_FALSE;
@@ -8913,8 +8972,12 @@ upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
 					    "pool may become incompatible with "
 					    "software\nthat does not support "
 					    "the feature. See "
-					    "zpool-features(5) for "
+					    "zpool-features(7) for "
-					    "details.\n\n"));
+					    "details.\n\n"
+					    "Note that the pool "
+					    "'compatibility' feature can be "
+					    "used to inhibit\nfeature "
+					    "upgrades.\n\n"));
 					(void) printf(gettext("POOL  "
 					    "FEATURE\n"));
 					(void) printf(gettext("------"
@@ -8948,7 +9011,7 @@ upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
 static int
 upgrade_one(zpool_handle_t *zhp, void *data)
 {
-	boolean_t printnl = B_FALSE;
+	boolean_t modified_pool = B_FALSE;
 	upgrade_cbdata_t *cbp = data;
 	uint64_t cur_version;
 	int ret;
@@ -8976,7 +9039,7 @@ upgrade_one(zpool_handle_t *zhp, void *data)
 	}
 
 	if (cur_version != cbp->cb_version) {
-		printnl = B_TRUE;
+		modified_pool = B_TRUE;
 		ret = upgrade_version(zhp, cbp->cb_version);
 		if (ret != 0)
 			return (ret);
@@ -8989,7 +9052,7 @@ upgrade_one(zpool_handle_t *zhp, void *data)
 			return (ret);
 
 		if (count != 0) {
-			printnl = B_TRUE;
+			modified_pool = B_TRUE;
 		} else if (cur_version == SPA_VERSION) {
 			(void) printf(gettext("Pool '%s' already has all "
 			    "supported and requested features enabled.\n"),
@@ -8997,8 +9060,9 @@ upgrade_one(zpool_handle_t *zhp, void *data)
 		}
 	}
 
-	if (printnl) {
+	if (modified_pool) {
-		(void) printf(gettext("\n"));
+		(void) printf("\n");
+		(void) after_zpool_upgrade(zhp);
 	}
 
 	return (0);
@@ -9970,6 +10034,63 @@ set_callback(zpool_handle_t *zhp, void *data)
 	int error;
 	set_cbdata_t *cb = (set_cbdata_t *)data;
 
+	/* Check if we have out-of-bounds features */
+	if (strcmp(cb->cb_propname, ZPOOL_CONFIG_COMPATIBILITY) == 0) {
+		boolean_t features[SPA_FEATURES];
+		if (zpool_do_load_compat(cb->cb_value, features) !=
+		    ZPOOL_COMPATIBILITY_OK)
+			return (-1);
+
+		nvlist_t *enabled = zpool_get_features(zhp);
+		spa_feature_t i;
+		for (i = 0; i < SPA_FEATURES; i++) {
+			const char *fguid = spa_feature_table[i].fi_guid;
+			if (nvlist_exists(enabled, fguid) && !features[i])
+				break;
+		}
+		if (i < SPA_FEATURES)
+			(void) fprintf(stderr, gettext("Warning: one or "
+			    "more features already enabled on pool '%s'\n"
+			    "are not present in this compatibility set.\n"),
+			    zpool_get_name(zhp));
+	}
+
+	/* if we're setting a feature, check it's in compatibility set */
+	if (zpool_prop_feature(cb->cb_propname) &&
+	    strcmp(cb->cb_value, ZFS_FEATURE_ENABLED) == 0) {
+		char *fname = strchr(cb->cb_propname, '@') + 1;
+		spa_feature_t f;
+
+		if (zfeature_lookup_name(fname, &f) == 0) {
+			char compat[ZFS_MAXPROPLEN];
+			if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY,
+			    compat, ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
+				compat[0] = '\0';
+
+			boolean_t features[SPA_FEATURES];
+			if (zpool_do_load_compat(compat, features) !=
+			    ZPOOL_COMPATIBILITY_OK) {
+				(void) fprintf(stderr, gettext("Error: "
+				    "cannot enable feature '%s' on pool '%s'\n"
+				    "because the pool's 'compatibility' "
+				    "property cannot be parsed.\n"),
+				    fname, zpool_get_name(zhp));
+				return (-1);
+			}
+
+			if (!features[f]) {
+				(void) fprintf(stderr, gettext("Error: "
+				    "cannot enable feature '%s' on pool '%s'\n"
+				    "as it is not specified in this pool's "
+				    "current compatibility set.\n"
+				    "Consider setting 'compatibility' to a "
+				    "less restrictive set, or to 'off'.\n"),
+				    fname, zpool_get_name(zhp));
+				return (-1);
+			}
+		}
+	}
+
 	error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
 
 	if (!error)
@@ -10492,28 +10613,25 @@ zpool_do_version(int argc, char **argv)
 static zpool_compat_status_t
 zpool_do_load_compat(const char *compat, boolean_t *list)
 {
-	char badword[ZFS_MAXPROPLEN];
+	char report[1024];
-	char badfile[MAXPATHLEN];
+
 	zpool_compat_status_t ret;
 
-	switch (ret = zpool_load_compat(compat, list, badword, badfile)) {
+	ret = zpool_load_compat(compat, list, report, 1024);
+	switch (ret) {
+
 	case ZPOOL_COMPATIBILITY_OK:
 		break;
-	case ZPOOL_COMPATIBILITY_READERR:
+
-		(void) fprintf(stderr, gettext("error reading compatibility "
-		    "file '%s'\n"), badfile);
-		break;
-	case ZPOOL_COMPATIBILITY_BADFILE:
-		(void) fprintf(stderr, gettext("compatibility file '%s' "
-		    "too large or not newline-terminated\n"), badfile);
-		break;
-	case ZPOOL_COMPATIBILITY_BADWORD:
-		(void) fprintf(stderr, gettext("unknown feature '%s' in "
-		    "compatibility file '%s'\n"), badword, badfile);
-		break;
 	case ZPOOL_COMPATIBILITY_NOFILES:
-		(void) fprintf(stderr, gettext("no compatibility files "
+	case ZPOOL_COMPATIBILITY_BADFILE:
-		    "specified\n"));
+	case ZPOOL_COMPATIBILITY_BADTOKEN:
+		(void) fprintf(stderr, "Error: %s\n", report);
+		break;
+
+	case ZPOOL_COMPATIBILITY_WARNTOKEN:
+		(void) fprintf(stderr, "Warning: %s\n", report);
+		ret = ZPOOL_COMPATIBILITY_OK;
 		break;
 	}
 	return (ret);

cmd/zpool/zpool_util.h

@@ -129,6 +129,7 @@ int check_device(const char *path, boolean_t force,
 boolean_t check_sector_size_database(char *path, int *sector_size);
 void vdev_error(const char *fmt, ...);
 int check_file(const char *file, boolean_t force, boolean_t isspare);
+void after_zpool_upgrade(zpool_handle_t *zhp);
 
 #ifdef	__cplusplus
 }

cmd/zpool/zpool_vdev.c

@@ -445,7 +445,7 @@ typedef struct replication_level {
 
 /*
  * N.B. For the purposes of comparing replication levels dRAID can be
- * considered functionally equivilant to raidz.
+ * considered functionally equivalent to raidz.
  */
 static boolean_t
 is_raidz_mirror(replication_level_t *a, replication_level_t *b,

cmd/zpool_influxdb/dashboards/grafana/ZFS-pool-latency-heatmaps-influxdb.json

@@ -1360,7 +1360,7 @@
       "type": "row"
     },
     {
-      "content": "I/O requests that are satisfied by accessing pool devices are managed by the ZIO scheduler.\nThe total latency is measured from the start of the I/O to completion by the disk.\nLatency through each queue is shown prior to its submission to the disk queue.\n\nThis view is useful for observing the effects of tuning the ZIO scheduler min and max values\n(see zfs-module-parameters(5) and [ZFS on Linux Module Parameters](https://openzfs.github.io/openzfs-docs/Performance%20and%20tuning/ZFS%20on%20Linux%20Module%20Parameters.html)):\n+ *zfs_vdev_max_active* controls the ZIO scheduler's disk queue depth (do not confuse with the block device's nr_requests)\n+ *zfs_vdev_sync_read_min_active* and *zfs_vdev_sync_read_max_active* control the synchronous queue for reads: most reads are sync\n+ *zfs_vdev_sync_write_min_active* and *zfs_vdev_sync_write_max_active* control the synchronous queue for writes: \nusually metadata or user data depending on the \"sync\" property setting or I/Os that are requested to be flushed\n+ *zfs_vdev_async_read_min_active* and *zfs_vdev_async_read_max_active* control the asynchronous queue for reads: usually prefetches\n+ *zfs_vdev_async_write_min_active* and *zfs_vdev_async_write_max_active* control the asynchronous queue for writes: \nusually the bulk of all writes at transaction group (txg) commit\n+ *zfs_vdev_scrub_min_active* and *zfs_vdev_scrub_max_active* controls the scan reads: usually scrub or resilver\n\n",
+      "content": "I/O requests that are satisfied by accessing pool devices are managed by the ZIO scheduler.\nThe total latency is measured from the start of the I/O to completion by the disk.\nLatency through each queue is shown prior to its submission to the disk queue.\n\nThis view is useful for observing the effects of tuning the ZIO scheduler min and max values\n(see zfs(4) and [ZFS on Linux Module Parameters](https://openzfs.github.io/openzfs-docs/Performance%20and%20tuning/ZFS%20on%20Linux%20Module%20Parameters.html)):\n+ *zfs_vdev_max_active* controls the ZIO scheduler's disk queue depth (do not confuse with the block device's nr_requests)\n+ *zfs_vdev_sync_read_min_active* and *zfs_vdev_sync_read_max_active* control the synchronous queue for reads: most reads are sync\n+ *zfs_vdev_sync_write_min_active* and *zfs_vdev_sync_write_max_active* control the synchronous queue for writes: \nusually metadata or user data depending on the \"sync\" property setting or I/Os that are requested to be flushed\n+ *zfs_vdev_async_read_min_active* and *zfs_vdev_async_read_max_active* control the asynchronous queue for reads: usually prefetches\n+ *zfs_vdev_async_write_min_active* and *zfs_vdev_async_write_max_active* control the asynchronous queue for writes: \nusually the bulk of all writes at transaction group (txg) commit\n+ *zfs_vdev_scrub_min_active* and *zfs_vdev_scrub_max_active* controls the scan reads: usually scrub or resilver\n\n",
       "datasource": "${DS_MACBOOK-INFLUX}",
       "fieldConfig": {
         "defaults": {
@@ -1664,4 +1664,4 @@
     "list": []
   },
   "version": 2
-}
+}

cmd/zstream/Makefile.am

@@ -15,3 +15,6 @@ zstream_LDADD = \
 	$(abs_top_builddir)/lib/libnvpair/libnvpair.la
 
 include $(top_srcdir)/config/CppCheck.am
+
+install-exec-hook:
+	cd $(DESTDIR)$(sbindir) && $(LN_S) -f zstream zstreamdump

cmd/zstream/zstream.c

@@ -49,6 +49,11 @@ zstream_usage(void)
 int
 main(int argc, char *argv[])
 {
+	char *basename = strrchr(argv[0], '/');
+	basename = basename ? (basename + 1) : argv[0];
+	if (argc >= 1 && strcmp(basename, "zstreamdump") == 0)
+		return (zstream_do_dump(argc, argv));
+
 	if (argc < 2)
 		zstream_usage();
 

cmd/zstreamdump/Makefile.am

@@ -1 +0,0 @@
-dist_sbin_SCRIPTS = zstreamdump

cmd/zstreamdump/zstreamdump

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-zstream dump "$@"

cmd/ztest/ztest.c

@@ -124,6 +124,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
+#include <getopt.h>
 #include <signal.h>
 #include <umem.h>
 #include <ctype.h>
@@ -192,30 +193,58 @@ typedef struct ztest_shared_opts {
 	char zo_gvars[ZO_GVARS_MAX_COUNT][ZO_GVARS_MAX_ARGLEN];
 } ztest_shared_opts_t;
 
+/* Default values for command line options. */
+#define	DEFAULT_POOL "ztest"
+#define	DEFAULT_VDEV_DIR "/tmp"
+#define	DEFAULT_VDEV_COUNT 5
+#define	DEFAULT_VDEV_SIZE (SPA_MINDEVSIZE * 4)	/* 256m default size */
+#define	DEFAULT_VDEV_SIZE_STR "256M"
+#define	DEFAULT_ASHIFT SPA_MINBLOCKSHIFT
+#define	DEFAULT_MIRRORS 2
+#define	DEFAULT_RAID_CHILDREN 4
+#define	DEFAULT_RAID_PARITY 1
+#define	DEFAULT_DRAID_DATA 4
+#define	DEFAULT_DRAID_SPARES 1
+#define	DEFAULT_DATASETS_COUNT 7
+#define	DEFAULT_THREADS 23
+#define	DEFAULT_RUN_TIME 300 /* 300 seconds */
+#define	DEFAULT_RUN_TIME_STR "300 sec"
+#define	DEFAULT_PASS_TIME 60 /* 60 seconds */
+#define	DEFAULT_PASS_TIME_STR "60 sec"
+#define	DEFAULT_KILL_RATE 70 /* 70% kill rate */
+#define	DEFAULT_KILLRATE_STR "70%"
+#define	DEFAULT_INITS 1
+#define	DEFAULT_MAX_LOOPS 50 /* 5 minutes */
+#define	DEFAULT_FORCE_GANGING (64 << 10)
+#define	DEFAULT_FORCE_GANGING_STR "64K"
+
+/* Simplifying assumption: -1 is not a valid default. */
+#define	NO_DEFAULT -1
+
 static const ztest_shared_opts_t ztest_opts_defaults = {
-	.zo_pool = "ztest",
+	.zo_pool = DEFAULT_POOL,
-	.zo_dir = "/tmp",
+	.zo_dir = DEFAULT_VDEV_DIR,
 	.zo_alt_ztest = { '\0' },
 	.zo_alt_libpath = { '\0' },
-	.zo_vdevs = 5,
+	.zo_vdevs = DEFAULT_VDEV_COUNT,
-	.zo_ashift = SPA_MINBLOCKSHIFT,
+	.zo_ashift = DEFAULT_ASHIFT,
-	.zo_mirrors = 2,
+	.zo_mirrors = DEFAULT_MIRRORS,
-	.zo_raid_children = 4,
+	.zo_raid_children = DEFAULT_RAID_CHILDREN,
-	.zo_raid_parity = 1,
+	.zo_raid_parity = DEFAULT_RAID_PARITY,
 	.zo_raid_type = VDEV_TYPE_RAIDZ,
-	.zo_vdev_size = SPA_MINDEVSIZE * 4,	/* 256m default size */
+	.zo_vdev_size = DEFAULT_VDEV_SIZE,
-	.zo_draid_data = 4,		/* data drives */
+	.zo_draid_data = DEFAULT_DRAID_DATA,	/* data drives */
-	.zo_draid_spares = 1,		/* distributed spares */
+	.zo_draid_spares = DEFAULT_DRAID_SPARES, /* distributed spares */
-	.zo_datasets = 7,
+	.zo_datasets = DEFAULT_DATASETS_COUNT,
-	.zo_threads = 23,
+	.zo_threads = DEFAULT_THREADS,
-	.zo_passtime = 60,		/* 60 seconds */
+	.zo_passtime = DEFAULT_PASS_TIME,
-	.zo_killrate = 70,		/* 70% kill rate */
+	.zo_killrate = DEFAULT_KILL_RATE,
 	.zo_verbose = 0,
 	.zo_mmp_test = 0,
-	.zo_init = 1,
+	.zo_init = DEFAULT_INITS,
-	.zo_time = 300,			/* 5 minutes */
+	.zo_time = DEFAULT_RUN_TIME,
-	.zo_maxloops = 50,		/* max loops during spa_freeze() */
+	.zo_maxloops = DEFAULT_MAX_LOOPS, /* max loops during spa_freeze() */
-	.zo_metaslab_force_ganging = 64 << 10,
+	.zo_metaslab_force_ganging = DEFAULT_FORCE_GANGING,
 	.zo_special_vdevs = ZTEST_VDEV_CLASS_RND,
 	.zo_gvars_count = 0,
 };
@@ -684,68 +713,154 @@ nicenumtoull(const char *buf)
 	return (val);
 }
 
+typedef struct ztest_option {
+	const char	short_opt;
+	const char	*long_opt;
+	const char	*long_opt_param;
+	const char	*comment;
+	unsigned int	default_int;
+	char		*default_str;
+} ztest_option_t;
+
+/*
+ * The following option_table is used for generating the usage info as well as
+ * the long and short option information for calling getopt_long().
+ */
+static ztest_option_t option_table[] = {
+	{ 'v',	"vdevs", "INTEGER", "Number of vdevs", DEFAULT_VDEV_COUNT,
+	    NULL},
+	{ 's',	"vdev-size", "INTEGER", "Size of each vdev",
+	    NO_DEFAULT, DEFAULT_VDEV_SIZE_STR},
+	{ 'a',	"alignment-shift", "INTEGER",
+	    "Alignment shift; use 0 for random", DEFAULT_ASHIFT, NULL},
+	{ 'm',	"mirror-copies", "INTEGER", "Number of mirror copies",
+	    DEFAULT_MIRRORS, NULL},
+	{ 'r',	"raid-disks", "INTEGER", "Number of raidz/draid disks",
+	    DEFAULT_RAID_CHILDREN, NULL},
+	{ 'R',	"raid-parity", "INTEGER", "Raid parity",
+	    DEFAULT_RAID_PARITY, NULL},
+	{ 'K',	"raid-kind", "raidz|draid|random", "Raid kind",
+	    NO_DEFAULT, "random"},
+	{ 'D',	"draid-data", "INTEGER", "Number of draid data drives",
+	    DEFAULT_DRAID_DATA, NULL},
+	{ 'S',	"draid-spares", "INTEGER", "Number of draid spares",
+	    DEFAULT_DRAID_SPARES, NULL},
+	{ 'd',	"datasets", "INTEGER", "Number of datasets",
+	    DEFAULT_DATASETS_COUNT, NULL},
+	{ 't',	"threads", "INTEGER", "Number of ztest threads",
+	    DEFAULT_THREADS, NULL},
+	{ 'g',	"gang-block-threshold", "INTEGER",
+	    "Metaslab gang block threshold",
+	    NO_DEFAULT, DEFAULT_FORCE_GANGING_STR},
+	{ 'i',	"init-count", "INTEGER", "Number of times to initialize pool",
+	    DEFAULT_INITS, NULL},
+	{ 'k',	"kill-percentage", "INTEGER", "Kill percentage",
+	    NO_DEFAULT, DEFAULT_KILLRATE_STR},
+	{ 'p',	"pool-name", "STRING", "Pool name",
+	    NO_DEFAULT, DEFAULT_POOL},
+	{ 'f',	"vdev-file-directory", "PATH", "File directory for vdev files",
+	    NO_DEFAULT, DEFAULT_VDEV_DIR},
+	{ 'M',	"multi-host", NULL,
+	    "Multi-host; simulate pool imported on remote host",
+	    NO_DEFAULT, NULL},
+	{ 'E',	"use-existing-pool", NULL,
+	    "Use existing pool instead of creating new one", NO_DEFAULT, NULL},
+	{ 'T',	"run-time", "INTEGER", "Total run time",
+	    NO_DEFAULT, DEFAULT_RUN_TIME_STR},
+	{ 'P',	"pass-time", "INTEGER", "Time per pass",
+	    NO_DEFAULT, DEFAULT_PASS_TIME_STR},
+	{ 'F',	"freeze-loops", "INTEGER", "Max loops in spa_freeze()",
+	    DEFAULT_MAX_LOOPS, NULL},
+	{ 'B',	"alt-ztest", "PATH", "Alternate ztest path",
+	    NO_DEFAULT, NULL},
+	{ 'C',	"vdev-class-state", "on|off|random", "vdev class state",
+	    NO_DEFAULT, "random"},
+	{ 'o',	"option", "\"OPTION=INTEGER\"",
+	    "Set global variable to an unsigned 32-bit integer value",
+	    NO_DEFAULT, NULL},
+	{ 'G',	"dump-debug-msg", NULL,
+	    "Dump zfs_dbgmsg buffer before exiting due to an error",
+	    NO_DEFAULT, NULL},
+	{ 'V',	"verbose", NULL,
+	    "Verbose (use multiple times for ever more verbosity)",
+	    NO_DEFAULT, NULL},
+	{ 'h',	"help",	NULL, "Show this help",
+	    NO_DEFAULT, NULL},
+	{0, 0, 0, 0, 0, 0}
+};
+
+static struct option *long_opts = NULL;
+static char *short_opts = NULL;
+
+static void
+init_options(void)
+{
+	ASSERT3P(long_opts, ==, NULL);
+	ASSERT3P(short_opts, ==, NULL);
+
+	int count = sizeof (option_table) / sizeof (option_table[0]);
+	long_opts = umem_alloc(sizeof (struct option) * count, UMEM_NOFAIL);
+
+	short_opts = umem_alloc(sizeof (char) * 2 * count, UMEM_NOFAIL);
+	int short_opt_index = 0;
+
+	for (int i = 0; i < count; i++) {
+		long_opts[i].val = option_table[i].short_opt;
+		long_opts[i].name = option_table[i].long_opt;
+		long_opts[i].has_arg = option_table[i].long_opt_param != NULL
+		    ? required_argument : no_argument;
+		long_opts[i].flag = NULL;
+		short_opts[short_opt_index++] = option_table[i].short_opt;
+		if (option_table[i].long_opt_param != NULL) {
+			short_opts[short_opt_index++] = ':';
+		}
+	}
+}
+
+static void
+fini_options(void)
+{
+	int count = sizeof (option_table) / sizeof (option_table[0]);
+
+	umem_free(long_opts, sizeof (struct option) * count);
+	umem_free(short_opts, sizeof (char) * 2 * count);
+
+	long_opts = NULL;
+	short_opts = NULL;
+}
+
 static void
 usage(boolean_t requested)
 {
-	const ztest_shared_opts_t *zo = &ztest_opts_defaults;
+	char option[80];
-
-	char nice_vdev_size[NN_NUMBUF_SZ];
-	char nice_force_ganging[NN_NUMBUF_SZ];
 	FILE *fp = requested ? stdout : stderr;
 
-	nicenum(zo->zo_vdev_size, nice_vdev_size, sizeof (nice_vdev_size));
+	(void) fprintf(fp, "Usage: %s [OPTIONS...]\n", DEFAULT_POOL);
-	nicenum(zo->zo_metaslab_force_ganging, nice_force_ganging,
+	for (int i = 0; option_table[i].short_opt != 0; i++) {
-	    sizeof (nice_force_ganging));
+		if (option_table[i].long_opt_param != NULL) {
+			(void) sprintf(option, "  -%c --%s=%s",
+			    option_table[i].short_opt,
+			    option_table[i].long_opt,
+			    option_table[i].long_opt_param);
+		} else {
+			(void) sprintf(option, "  -%c --%s",
+			    option_table[i].short_opt,
+			    option_table[i].long_opt);
+		}
+		(void) fprintf(fp, "  %-40s%s", option,
+		    option_table[i].comment);
 
-	(void) fprintf(fp, "Usage: %s\n"
+		if (option_table[i].long_opt_param != NULL) {
-	    "\t[-v vdevs (default: %llu)]\n"
+			if (option_table[i].default_str != NULL) {
-	    "\t[-s size_of_each_vdev (default: %s)]\n"
+				(void) fprintf(fp, " (default: %s)",
-	    "\t[-a alignment_shift (default: %d)] use 0 for random\n"
+				    option_table[i].default_str);
-	    "\t[-m mirror_copies (default: %d)]\n"
+			} else if (option_table[i].default_int != NO_DEFAULT) {
-	    "\t[-r raidz_disks / draid_disks (default: %d)]\n"
+				(void) fprintf(fp, " (default: %u)",
-	    "\t[-R raid_parity (default: %d)]\n"
+				    option_table[i].default_int);
-	    "\t[-K raid_kind (default: random)] raidz|draid|random\n"
+			}
-	    "\t[-D draid_data (default: %d)] in config\n"
+		}
-	    "\t[-S draid_spares (default: %d)]\n"
+		(void) fprintf(fp, "\n");
-	    "\t[-d datasets (default: %d)]\n"
+	}
-	    "\t[-t threads (default: %d)]\n"
-	    "\t[-g gang_block_threshold (default: %s)]\n"
-	    "\t[-i init_count (default: %d)] initialize pool i times\n"
-	    "\t[-k kill_percentage (default: %llu%%)]\n"
-	    "\t[-p pool_name (default: %s)]\n"
-	    "\t[-f dir (default: %s)] file directory for vdev files\n"
-	    "\t[-M] Multi-host simulate pool imported on remote host\n"
-	    "\t[-V] verbose (use multiple times for ever more blather)\n"
-	    "\t[-E] use existing pool instead of creating new one\n"
-	    "\t[-T time (default: %llu sec)] total run time\n"
-	    "\t[-F freezeloops (default: %llu)] max loops in spa_freeze()\n"
-	    "\t[-P passtime (default: %llu sec)] time per pass\n"
-	    "\t[-B alt_ztest (default: <none>)] alternate ztest path\n"
-	    "\t[-C vdev class state (default: random)] special=on|off|random\n"
-	    "\t[-o variable=value] ... set global variable to an unsigned\n"
-	    "\t    32-bit integer value\n"
-	    "\t[-G dump zfs_dbgmsg buffer before exiting due to an error\n"
-	    "\t[-h] (print help)\n"
-	    "",
-	    zo->zo_pool,
-	    (u_longlong_t)zo->zo_vdevs,			/* -v */
-	    nice_vdev_size,				/* -s */
-	    zo->zo_ashift,				/* -a */
-	    zo->zo_mirrors,				/* -m */
-	    zo->zo_raid_children,			/* -r */
-	    zo->zo_raid_parity,				/* -R */
-	    zo->zo_draid_data,				/* -D */
-	    zo->zo_draid_spares,			/* -S */
-	    zo->zo_datasets,				/* -d */
-	    zo->zo_threads,				/* -t */
-	    nice_force_ganging,				/* -g */
-	    zo->zo_init,				/* -i */
-	    (u_longlong_t)zo->zo_killrate,		/* -k */
-	    zo->zo_pool,				/* -p */
-	    zo->zo_dir,					/* -f */
-	    (u_longlong_t)zo->zo_time,			/* -T */
-	    (u_longlong_t)zo->zo_maxloops,		/* -F */
-	    (u_longlong_t)zo->zo_passtime);
 	exit(requested ? 0 : 1);
 }
 
@@ -817,8 +932,10 @@ process_options(int argc, char **argv)
 
 	bcopy(&ztest_opts_defaults, zo, sizeof (*zo));
 
-	while ((opt = getopt(argc, argv,
+	init_options();
-	    "v:s:a:m:r:R:K:D:S:d:t:g:i:k:p:f:MVET:P:hF:B:C:o:G")) != EOF) {
+
+	while ((opt = getopt_long(argc, argv, short_opts, long_opts,
+	    NULL)) != EOF) {
 		value = 0;
 		switch (opt) {
 		case 'v':
@@ -953,6 +1070,8 @@ process_options(int argc, char **argv)
 		}
 	}
 
+	fini_options();
+
 	/* When raid choice is 'random' add a draid pool 50% of the time */
 	if (strcmp(raid_kind, "random") == 0) {
 		(void) strlcpy(raid_kind, (ztest_random(2) == 0) ?
@@ -5979,7 +6098,7 @@ ztest_fault_inject(ztest_ds_t *zd, uint64_t id)
 		    vd0->vdev_resilver_txg != 0)) {
 			/*
 			 * Make vd0 explicitly claim to be unreadable,
-			 * or unwriteable, or reach behind its back
+			 * or unwritable, or reach behind its back
 			 * and close the underlying fd.  We can do this if
 			 * maxfaults == 0 because we'll fail and reexecute,
 			 * and we can do it if maxfaults >= 2 because we'll
@@ -6550,7 +6669,7 @@ ztest_initialize(ztest_ds_t *zd, uint64_t id)
 	char *path = strdup(rand_vd->vdev_path);
 	boolean_t active = rand_vd->vdev_initialize_thread != NULL;
 
-	zfs_dbgmsg("vd %px, guid %llu", rand_vd, guid);
+	zfs_dbgmsg("vd %px, guid %llu", rand_vd, (u_longlong_t)guid);
 	spa_config_exit(spa, SCL_VDEV, FTAG);
 
 	uint64_t cmd = ztest_random(POOL_INITIALIZE_FUNCS);
@@ -6622,7 +6741,7 @@ ztest_trim(ztest_ds_t *zd, uint64_t id)
 	char *path = strdup(rand_vd->vdev_path);
 	boolean_t active = rand_vd->vdev_trim_thread != NULL;
 
-	zfs_dbgmsg("vd %p, guid %llu", rand_vd, guid);
+	zfs_dbgmsg("vd %p, guid %llu", rand_vd, (u_longlong_t)guid);
 	spa_config_exit(spa, SCL_VDEV, FTAG);
 
 	uint64_t cmd = ztest_random(POOL_TRIM_FUNCS);

cmd/zvol_wait/Makefile.am

@@ -1 +1,3 @@
+include $(top_srcdir)/config/Shellcheck.am
+
 dist_bin_SCRIPTS = zvol_wait

cmd/zvol_wait/zvol_wait

@@ -9,45 +9,42 @@ count_zvols() {
 }
 
 filter_out_zvols_with_links() {
-	while read -r zvol; do
+	echo "$zvols" | tr ' ' '+' | while read -r zvol; do
-		if [ ! -L "/dev/zvol/$zvol" ]; then
+		if ! [ -L "/dev/zvol/$zvol" ]; then
 			echo "$zvol"
 		fi
-	done
+	done | tr '+' ' '
 }
 
 filter_out_deleted_zvols() {
-	while read -r zvol; do
+	OIFS="$IFS"
-		if zfs list "$zvol" >/dev/null 2>&1; then
+	IFS="
-			echo "$zvol"
+"
-		fi
+	# shellcheck disable=SC2086
-	done
+	zfs list -H -o name $zvols 2>/dev/null
+	IFS="$OIFS"
 }
 
 list_zvols() {
+	read -r default_volmode < /sys/module/zfs/parameters/zvol_volmode
 	zfs list -t volume -H -o \
-		name,volmode,receive_resume_token,redact_snaps |
+	    name,volmode,receive_resume_token,redact_snaps |
-		while read -r zvol_line; do
+	    while IFS="	" read -r name volmode token redacted; do # IFS=\t here!
-		name=$(echo "$zvol_line" | awk '{print $1}')
+
-		volmode=$(echo "$zvol_line" | awk '{print $2}')
-		token=$(echo "$zvol_line" | awk '{print $3}')
-		redacted=$(echo "$zvol_line" | awk '{print $4}')
-		#
 		# /dev links are not created for zvols with volmode = "none"
 		# or for redacted zvols.
-		#
 		[ "$volmode" = "none" ] && continue
+		[ "$volmode" = "default" ] && [ "$default_volmode" = "3" ] &&
+		    continue
 		[ "$redacted" = "-" ] || continue
-		#
+
-		# We also also ignore partially received zvols if it is
+		# We also ignore partially received zvols if it is
 		# not an incremental receive, as those won't even have a block
 		# device minor node created yet.
-		#
 		if [ "$token" != "-" ]; then
-			#
+
 			# Incremental receives create an invisible clone that
 			# is not automatically displayed by zfs list.
-			#
 			if ! zfs list "$name/%recv" >/dev/null 2>&1; then
 				continue
 			fi
@@ -75,7 +72,7 @@ while [ "$outer_loop" -lt 20 ]; do
 	while [ "$inner_loop" -lt 30 ]; do
 		inner_loop=$((inner_loop + 1))
 
-		zvols="$(echo "$zvols" | filter_out_zvols_with_links)"
+		zvols="$(filter_out_zvols_with_links)"
 
 		zvols_count=$(count_zvols)
 		if [ "$zvols_count" -eq 0 ]; then
@@ -95,7 +92,7 @@ while [ "$outer_loop" -lt 20 ]; do
 		echo "No progress since last loop."
 		echo "Checking if any zvols were deleted."
 
-		zvols=$(echo "$zvols" | filter_out_deleted_zvols)
+		zvols=$(filter_out_deleted_zvols)
 		zvols_count=$(count_zvols)
 
 		if [ "$old_zvols_count" -ne "$zvols_count" ]; then

config/CppCheck.am

@@ -1,5 +1,5 @@
 #
-# Default rules for running cppcheck against the the user space components.
+# Default rules for running cppcheck against the user space components.
 #
 
 PHONY += cppcheck

config/Shellcheck.am

@@ -0,0 +1,22 @@
+.PHONY: shellcheck
+shellcheck: $(SCRIPTS) $(SHELLCHECKSCRIPTS)
+if HAVE_SHELLCHECK
+	[ -z "$(SCRIPTS)$(SHELLCHECKSCRIPTS)" ] && exit; shellcheck $$([ -n "$(SHELLCHECK_SHELL)" ] && echo "--shell=$(SHELLCHECK_SHELL)") --exclude=SC1090,SC1091$(SHELLCHECK_IGNORE) --format=gcc $(SCRIPTS) $(SHELLCHECKSCRIPTS)
+else
+	@[ -z "$(SCRIPTS)$(SHELLCHECKSCRIPTS)" ] && exit; echo "skipping shellcheck of" $(SCRIPTS) $(SHELLCHECKSCRIPTS) "because shellcheck is not installed"
+endif
+	@set -e; for dir in $(SHELLCHECKDIRS); do $(MAKE) -C $$dir shellcheck; done
+
+
+# command -v *is* specified by POSIX and every shell in existence supports it
+.PHONY: checkbashisms
+checkbashisms: $(SCRIPTS) $(SHELLCHECKSCRIPTS)
+if HAVE_CHECKBASHISMS
+	[ -z "$(SCRIPTS)$(SHELLCHECKSCRIPTS)" ] && exit; ! if [ -z "$(SHELLCHECK_SHELL)" ]; then \
+	    checkbashisms -npx $(SCRIPTS) $(SHELLCHECKSCRIPTS); else \
+	    for f in $(SCRIPTS) $(SHELLCHECKSCRIPTS); do echo $$f >&3; { echo '#!/bin/$(SHELLCHECK_SHELL)'; cat $$f; } | checkbashisms -npx; done; \
+	fi 3>&2 2>&1 | grep -vFe "'command' with option other than -p" -e 'command -v' $(CHECKBASHISMS_IGNORE) >&2
+else
+	@[ -z "$(SCRIPTS)$(SHELLCHECKSCRIPTS)" ] && exit; echo "skipping checkbashisms of" $(SCRIPTS) $(SHELLCHECKSCRIPTS) "because checkbashisms is not installed"
+endif
+	@set -e; for dir in $(SHELLCHECKDIRS); do $(MAKE) -C $$dir checkbashisms; done

config/always-shellcheck.m4

@@ -0,0 +1,10 @@
+dnl #
+dnl # Check if shellcheck and/or checkbashisms are available.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_SHELLCHECK], [
+	AC_CHECK_PROG([SHELLCHECK], [shellcheck], [yes])
+	AC_CHECK_PROG([CHECKBASHISMS], [checkbashisms], [yes])
+
+	AM_CONDITIONAL([HAVE_SHELLCHECK], [test "x$SHELLCHECK" = "xyes"])
+	AM_CONDITIONAL([HAVE_CHECKBASHISMS], [test "x$CHECKBASHISMS" = "xyes"])
+])

config/ax_python_devel.m4

@@ -148,8 +148,7 @@ variable to configure. See ``configure --help'' for reference.
 	# Check if you have distutils, else fail
 	#
 	AC_MSG_CHECKING([for the distutils Python package])
-	ac_distutils_result=`$PYTHON -c "import distutils" 2>&1`
+	if ac_distutils_result=`$PYTHON -c "import distutils" 2>&1`; then
-	if test $? -eq 0; then
 		AC_MSG_RESULT([yes])
 	else
 		AC_MSG_RESULT([no])

config/deb.am

@@ -14,7 +14,19 @@ deb-local:
 	"*** package for your distribution which provides ${ALIEN},\n" \
 	"*** re-run configure, and try again.\n"; \
 		exit 1; \
-	fi)
+	fi; \
+        if test "${ALIEN_MAJOR}" = "8" && \
+           test "${ALIEN_MINOR}" = "95"; then \
+        if test "${ALIEN_POINT}" = "1" || \
+           test "${ALIEN_POINT}" = "2" || \
+           test "${ALIEN_POINT}" = "3"; then \
+                /bin/echo -e "\n" \
+        "*** Installed version of ${ALIEN} is known to be broken;\n" \
+        "*** attempting to generate debs will fail! See\n" \
+        "*** https://github.com/openzfs/zfs/issues/11650 for details.\n"; \
+                exit 1; \
+        fi; \
+        fi)
 
 deb-kmod: deb-local rpm-kmod
 	name=${PACKAGE}; \
@@ -43,9 +55,9 @@ deb-utils: deb-local rpm-utils-initramfs
 	pkg1=$${name}-$${version}.$${arch}.rpm; \
 	pkg2=libnvpair3-$${version}.$${arch}.rpm; \
 	pkg3=libuutil3-$${version}.$${arch}.rpm; \
-	pkg4=libzfs4-$${version}.$${arch}.rpm; \
+	pkg4=libzfs5-$${version}.$${arch}.rpm; \
-	pkg5=libzpool4-$${version}.$${arch}.rpm; \
+	pkg5=libzpool5-$${version}.$${arch}.rpm; \
-	pkg6=libzfs4-devel-$${version}.$${arch}.rpm; \
+	pkg6=libzfs5-devel-$${version}.$${arch}.rpm; \
 	pkg7=$${name}-test-$${version}.$${arch}.rpm; \
 	pkg8=$${name}-dracut-$${version}.noarch.rpm; \
 	pkg9=$${name}-initramfs-$${version}.$${arch}.rpm; \
@@ -56,7 +68,7 @@ deb-utils: deb-local rpm-utils-initramfs
 	path_prepend=`mktemp -d /tmp/intercept.XXXXXX`; \
 	echo "#$(SHELL)" > $${path_prepend}/dh_shlibdeps; \
 	echo "`which dh_shlibdeps` -- \
-	 -xlibuutil3linux -xlibnvpair3linux -xlibzfs4linux -xlibzpool4linux" \
+	 -xlibuutil3linux -xlibnvpair3linux -xlibzfs5linux -xlibzpool5linux" \
 	 >> $${path_prepend}/dh_shlibdeps; \
 ## These -x arguments are passed to dpkg-shlibdeps, which exclude the
 ## Debianized packages from the auto-generated dependencies of the new debs,

config/kernel-acl.m4

@@ -189,7 +189,22 @@ dnl #
 dnl # 3.14 API change,
 dnl # Check if inode_operations contains the function set_acl
 dnl #
+dnl # 5.12 API change,
+dnl # set_acl() added a user_namespace* parameter first
+dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_SET_ACL], [
+	ZFS_LINUX_TEST_SRC([inode_operations_set_acl_userns], [
+		#include <linux/fs.h>
+
+		int set_acl_fn(struct user_namespace *userns,
+		    struct inode *inode, struct posix_acl *acl,
+		    int type) { return 0; }
+
+		static const struct inode_operations
+		    iops __attribute__ ((unused)) = {
+			.set_acl = set_acl_fn,
+		};
+	],[])
 	ZFS_LINUX_TEST_SRC([inode_operations_set_acl], [
 		#include <linux/fs.h>
 
@@ -205,11 +220,17 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_SET_ACL], [
 
 AC_DEFUN([ZFS_AC_KERNEL_INODE_OPERATIONS_SET_ACL], [
 	AC_MSG_CHECKING([whether iops->set_acl() exists])
-	ZFS_LINUX_TEST_RESULT([inode_operations_set_acl], [
+	ZFS_LINUX_TEST_RESULT([inode_operations_set_acl_userns], [
 		AC_MSG_RESULT(yes)
 		AC_DEFINE(HAVE_SET_ACL, 1, [iops->set_acl() exists])
+		AC_DEFINE(HAVE_SET_ACL_USERNS, 1, [iops->set_acl() takes 4 args])
 	],[
-		AC_MSG_RESULT(no)
+		ZFS_LINUX_TEST_RESULT([inode_operations_set_acl], [
+			AC_MSG_RESULT(yes)
+			AC_DEFINE(HAVE_SET_ACL, 1, [iops->set_acl() exists, takes 3 args])
+		],[
+			AC_MSG_RESULT(no)
+		])
 	])
 ])
 

config/kernel-bdi.m4

@@ -8,7 +8,9 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BDI], [
 	], [
 		char *name = "bdi";
 		atomic_long_t zfs_bdi_seq;
-		int error __attribute__((unused)) =
+		int error __attribute__((unused));
+		atomic_long_set(&zfs_bdi_seq, 0);
+		error =
 		    super_setup_bdi_name(&sb, "%.28s-%ld", name,
 		    atomic_long_inc_return(&zfs_bdi_seq));
 	])

config/kernel-blk-queue.m4

@@ -23,8 +23,8 @@ AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_PLUG], [
 ])
 
 dnl #
-dnl # 2.6.32 - 4.11, statically allocated bdi in request_queue
+dnl # 2.6.32 - 4.11: statically allocated bdi in request_queue
-dnl # 4.12 - x.y, dynamically allocated bdi in request_queue
+dnl # 4.12: dynamically allocated bdi in request_queue
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_BDI], [
 	ZFS_LINUX_TEST_SRC([blk_queue_bdi], [
@@ -48,7 +48,7 @@ AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_BDI], [
 ])
 
 dnl #
-dnl # 2.6.32 - 4.x API,
+dnl # 2.6.32 API,
 dnl #   blk_queue_discard()
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_DISCARD], [
@@ -71,7 +71,7 @@ AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_DISCARD], [
 ])
 
 dnl #
-dnl # 4.8 - 4.x API,
+dnl # 4.8 API,
 dnl #   blk_queue_secure_erase()
 dnl #
 dnl # 2.6.36 - 4.7 API,

config/kernel-block-device-operations.m4

@@ -52,12 +52,44 @@ AC_DEFUN([ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_RELEASE_VOID], [
 	])
 ])
 
+dnl #
+dnl # 5.13 API change
+dnl # block_device_operations->revalidate_disk() was removed
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK], [
+	ZFS_LINUX_TEST_SRC([block_device_operations_revalidate_disk], [
+		#include <linux/blkdev.h>
+
+		int blk_revalidate_disk(struct gendisk *disk) {
+			return(0);
+		}
+
+		static const struct block_device_operations
+		    bops __attribute__ ((unused)) = {
+			.revalidate_disk	= blk_revalidate_disk,
+		};
+	], [], [$NO_UNUSED_BUT_SET_VARIABLE])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK], [
+	AC_MSG_CHECKING([whether bops->revalidate_disk() exists])
+	ZFS_LINUX_TEST_RESULT([block_device_operations_revalidate_disk], [
+		AC_DEFINE([HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK], [1],
+			[Define if revalidate_disk() in block_device_operations])
+		AC_MSG_RESULT(yes)
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
 AC_DEFUN([ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS], [
 	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_CHECK_EVENTS
 	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_RELEASE_VOID
+	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS], [
 	ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_CHECK_EVENTS
 	ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_RELEASE_VOID
+	ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
 ])

config/kernel-generic_fillattr.m4

@@ -16,7 +16,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_GENERIC_FILLATTR_USERNS], [
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_GENERIC_FILLATTR_USERNS], [
-	AC_MSG_CHECKING([whether generic_fillattr requres struct user_namespace*])
+	AC_MSG_CHECKING([whether generic_fillattr requires struct user_namespace*])
 	ZFS_LINUX_TEST_RESULT([generic_fillattr_userns], [
 		AC_MSG_RESULT([yes])
 		AC_DEFINE(HAVE_GENERIC_FILLATTR_USERNS, 1,

config/kernel-is_owner_or_cap.m4

@@ -4,6 +4,10 @@ dnl # The is_owner_or_cap() macro was renamed to inode_owner_or_capable(),
 dnl # This is used for permission checks in the xattr and file attribute call
 dnl # paths.
 dnl #
+dnl # 5.12 API change,
+dnl # inode_owner_or_capable() now takes struct user_namespace *
+dnl # to support idmapped mounts
+dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OWNER_OR_CAPABLE], [
 	ZFS_LINUX_TEST_SRC([inode_owner_or_capable], [
 		#include <linux/fs.h>

config/kernel-percpu.m4

@@ -25,6 +25,31 @@ AC_DEFUN([ZFS_AC_KERNEL_PERCPU_COUNTER_INIT], [
 	])
 ])
 
+dnl #
+dnl # 4.13 API change,
+dnl # __percpu_counter_add() was renamed to percpu_counter_add_batch().
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_PERCPU_COUNTER_ADD_BATCH], [
+	ZFS_LINUX_TEST_SRC([percpu_counter_add_batch], [
+		#include <linux/percpu_counter.h>
+	],[
+		struct percpu_counter counter;
+
+		percpu_counter_add_batch(&counter, 1, 1);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_PERCPU_COUNTER_ADD_BATCH], [
+	AC_MSG_CHECKING([whether percpu_counter_add_batch() is defined])
+	ZFS_LINUX_TEST_RESULT([percpu_counter_add_batch], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_PERCPU_COUNTER_ADD_BATCH, 1,
+		    [percpu_counter_add_batch() is defined])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
 dnl #
 dnl # 5.10 API change,
 dnl # The "count" was moved into ref->data, from ref
@@ -51,10 +76,12 @@ AC_DEFUN([ZFS_AC_KERNEL_PERCPU_REF_COUNT_IN_DATA], [
 ])
 AC_DEFUN([ZFS_AC_KERNEL_SRC_PERCPU], [
 	ZFS_AC_KERNEL_SRC_PERCPU_COUNTER_INIT
+	ZFS_AC_KERNEL_SRC_PERCPU_COUNTER_ADD_BATCH
 	ZFS_AC_KERNEL_SRC_PERCPU_REF_COUNT_IN_DATA
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_PERCPU], [
 	ZFS_AC_KERNEL_PERCPU_COUNTER_INIT
+	ZFS_AC_KERNEL_PERCPU_COUNTER_ADD_BATCH
 	ZFS_AC_KERNEL_PERCPU_REF_COUNT_IN_DATA
 ])

config/kernel-rename.m4

@@ -44,6 +44,7 @@ AC_DEFUN([ZFS_AC_KERNEL_RENAME], [
 	],[
 		AC_MSG_RESULT(no)
 
+		AC_MSG_CHECKING([whether iop->rename() wants flags])
 		ZFS_LINUX_TEST_RESULT([inode_operations_rename_flags], [
 			AC_MSG_RESULT(yes)
 			AC_DEFINE(HAVE_RENAME_WANTS_FLAGS, 1,

config/kernel-siginfo.m4

@@ -0,0 +1,21 @@
+dnl #
+dnl # 4.20 API change
+dnl # Added kernel_siginfo_t
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_SIGINFO], [
+	ZFS_LINUX_TEST_SRC([siginfo], [
+		#include <linux/signal_types.h>
+	],[
+		kernel_siginfo_t info __attribute__ ((unused));
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SIGINFO], [
+	AC_MSG_CHECKING([whether kernel_siginfo_t tyepedef exists])
+	ZFS_LINUX_TEST_RESULT([siginfo], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_SIGINFO, 1, [kernel_siginfo_t exists])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])

config/kernel-signal-stop.m4

@@ -0,0 +1,21 @@
+dnl #
+dnl # 4.4 API change
+dnl # Added kernel_signal_stop
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_SIGNAL_STOP], [
+	ZFS_LINUX_TEST_SRC([signal_stop], [
+		#include <linux/sched/signal.h>
+	],[
+		kernel_signal_stop();
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SIGNAL_STOP], [
+	AC_MSG_CHECKING([whether signal_stop() exists])
+	ZFS_LINUX_TEST_RESULT([signal_stop], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_SIGNAL_STOP, 1, [signal_stop() exists])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])

config/kernel-special-state.m4

@@ -0,0 +1,21 @@
+dnl #
+dnl # 4.17 API change
+dnl # Added set_special_state() function
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_SET_SPECIAL_STATE], [
+	ZFS_LINUX_TEST_SRC([set_special_state], [
+		#include <linux/sched.h>
+	],[
+		set_special_state(TASK_STOPPED);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SET_SPECIAL_STATE], [
+	AC_MSG_CHECKING([whether set_special_state() exists])
+	ZFS_LINUX_TEST_RESULT([set_special_state], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_SET_SPECIAL_STATE, 1, [set_special_state() exists])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])

config/kernel-tmpfile.m4

@@ -3,23 +3,43 @@ dnl # 3.11 API change
 dnl # Add support for i_op->tmpfile
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_TMPFILE], [
-	ZFS_LINUX_TEST_SRC([inode_operations_tmpfile], [
+	dnl #
+	dnl # 5.11 API change
+	dnl # add support for userns parameter to tmpfile
+	dnl #
+	ZFS_LINUX_TEST_SRC([inode_operations_tmpfile_userns], [
 		#include <linux/fs.h>
-		int tmpfile(struct inode *inode, struct dentry *dentry,
+		int tmpfile(struct user_namespace *userns,
+		    struct inode *inode, struct dentry *dentry,
 		    umode_t mode) { return 0; }
 		static struct inode_operations
 		    iops __attribute__ ((unused)) = {
 			.tmpfile = tmpfile,
 		};
 	],[])
+	ZFS_LINUX_TEST_SRC([inode_operations_tmpfile], [
+			#include <linux/fs.h>
+			int tmpfile(struct inode *inode, struct dentry *dentry,
+			    umode_t mode) { return 0; }
+			static struct inode_operations
+			    iops __attribute__ ((unused)) = {
+				.tmpfile = tmpfile,
+			};
+	],[])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_TMPFILE], [
 	AC_MSG_CHECKING([whether i_op->tmpfile() exists])
-	ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile], [
+	ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile_userns], [
 		AC_MSG_RESULT(yes)
 		AC_DEFINE(HAVE_TMPFILE, 1, [i_op->tmpfile() exists])
+		AC_DEFINE(HAVE_TMPFILE_USERNS, 1, [i_op->tmpfile() has userns])
 	],[
-		AC_MSG_RESULT(no)
+		ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile], [
+			AC_MSG_RESULT(yes)
+			AC_DEFINE(HAVE_TMPFILE, 1, [i_op->tmpfile() exists])
+		],[
+			AC_MSG_RESULT(no)
+		])
 	])
 ])

config/kernel.m4

@@ -129,6 +129,9 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC_MKNOD
 	ZFS_AC_KERNEL_SRC_SYMLINK
 	ZFS_AC_KERNEL_SRC_BIO_MAX_SEGS
+	ZFS_AC_KERNEL_SRC_SIGNAL_STOP
+	ZFS_AC_KERNEL_SRC_SIGINFO
+	ZFS_AC_KERNEL_SRC_SET_SPECIAL_STATE
 
 	AC_MSG_CHECKING([for available kernel interfaces])
 	ZFS_LINUX_TEST_COMPILE_ALL([kabi])
@@ -231,6 +234,9 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL_MKNOD
 	ZFS_AC_KERNEL_SYMLINK
 	ZFS_AC_KERNEL_BIO_MAX_SEGS
+	ZFS_AC_KERNEL_SIGNAL_STOP
+	ZFS_AC_KERNEL_SIGINFO
+	ZFS_AC_KERNEL_SET_SPECIAL_STATE
 ])
 
 dnl #

config/user-libatomic.m4

@@ -0,0 +1,34 @@
+dnl #
+dnl # If -latomic exists, it's needed for __atomic intrinsics.
+dnl #
+dnl # Some systems (like FreeBSD 13) don't have a libatomic at all because
+dnl # their toolchain doesn't ship it – they obviously don't need it.
+dnl #
+dnl # Others (like sufficiently ancient CentOS) have one,
+dnl # but terminally broken or unlinkable (e.g. it's a dangling symlink,
+dnl # or a linker script that points to a nonexistent file) –
+dnl # most arches affected by this don't actually need -latomic (and if they do,
+dnl # then they should have libatomic that actually exists and links,
+dnl # so don't fall into this category).
+dnl #
+dnl # Technically, we could check if the platform *actually* needs -latomic,
+dnl # or if it has native support for all the intrinsics we use,
+dnl # but it /really/ doesn't matter, and C11 recommends to always link it.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_USER_LIBATOMIC], [
+	AC_MSG_CHECKING([whether -latomic is present])
+
+	saved_libs="$LIBS"
+	LIBS="$LIBS -latomic"
+
+	AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], [
+		LIBATOMIC_LIBS="-latomic"
+		AC_MSG_RESULT([yes])
+	], [
+		LIBATOMIC_LIBS=""
+		AC_MSG_RESULT([no])
+	])
+
+	LIBS="$saved_libs"
+	AC_SUBST([LIBATOMIC_LIBS])
+])

config/user.m4

@@ -21,6 +21,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [
 	ZFS_AC_CONFIG_USER_LIBUDEV
 	ZFS_AC_CONFIG_USER_LIBCRYPTO
 	ZFS_AC_CONFIG_USER_LIBAIO
+	ZFS_AC_CONFIG_USER_LIBATOMIC
 	ZFS_AC_CONFIG_USER_CLOCK_GETTIME
 	ZFS_AC_CONFIG_USER_PAM
 	ZFS_AC_CONFIG_USER_RUNSTATEDIR

config/zfs-build.m4

@@ -34,6 +34,9 @@ dnl # When debugging is enabled:
 dnl # - Enable all ASSERTs (-DDEBUG)
 dnl # - Promote all compiler warnings to errors (-Werror)
 dnl #
+dnl # (If INVARIANTS is detected, we need to force DEBUG, or strange panics
+dnl # can ensue.)
+dnl #
 AC_DEFUN([ZFS_AC_DEBUG], [
 	AC_MSG_CHECKING([whether assertion support will be enabled])
 	AC_ARG_ENABLE([debug],
@@ -49,6 +52,20 @@ AC_DEFUN([ZFS_AC_DEBUG], [
 		[ZFS_AC_DEBUG_DISABLE],
 		[AC_MSG_ERROR([Unknown option $enable_debug])])
 
+	AS_CASE(["x$enable_invariants"],
+		["xyes"],
+		[],
+		["xno"],
+		[],
+		[ZFS_AC_DEBUG_INVARIANTS_DETECT])
+
+	AS_CASE(["x$enable_invariants"],
+		["xyes"],
+		[ZFS_AC_DEBUG_ENABLE],
+		["xno"],
+		[],
+		[AC_MSG_ERROR([Unknown option $enable_invariants])])
+
 	AC_SUBST(DEBUG_CFLAGS)
 	AC_SUBST(DEBUG_CPPFLAGS)
 	AC_SUBST(DEBUG_LDFLAGS)
@@ -207,6 +224,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	ZFS_AC_CONFIG_ALWAYS_PYZFS
 	ZFS_AC_CONFIG_ALWAYS_SED
 	ZFS_AC_CONFIG_ALWAYS_CPPCHECK
+	ZFS_AC_CONFIG_ALWAYS_SHELLCHECK
 ])
 
 AC_DEFUN([ZFS_AC_CONFIG], [
@@ -437,6 +455,9 @@ AC_DEFUN([ZFS_AC_ALIEN], [
 	AC_MSG_CHECKING([whether $ALIEN is available])
 	AS_IF([tmp=$($ALIEN --version 2>/dev/null)], [
 		ALIEN_VERSION=$(echo $tmp | $AWK '{ print $[3] }')
+		ALIEN_MAJOR=$(echo ${ALIEN_VERSION} | $AWK -F'.' '{ print $[1] }')
+		ALIEN_MINOR=$(echo ${ALIEN_VERSION} | $AWK -F'.' '{ print $[2] }')
+		ALIEN_POINT=$(echo ${ALIEN_VERSION} | $AWK -F'.' '{ print $[3] }')
 		HAVE_ALIEN=yes
 		AC_MSG_RESULT([$HAVE_ALIEN ($ALIEN_VERSION)])
 	],[
@@ -447,6 +468,9 @@ AC_DEFUN([ZFS_AC_ALIEN], [
 	AC_SUBST(HAVE_ALIEN)
 	AC_SUBST(ALIEN)
 	AC_SUBST(ALIEN_VERSION)
+	AC_SUBST(ALIEN_MAJOR)
+	AC_SUBST(ALIEN_MINOR)
+	AC_SUBST(ALIEN_POINT)
 ])
 
 dnl #

configure.ac

@@ -48,6 +48,7 @@ AC_CONFIG_HEADERS([zfs_config.h], [
 LT_INIT
 AC_PROG_INSTALL
 AC_PROG_CC
+AC_PROG_LN_S
 PKG_PROG_PKG_CONFIG
 AM_PROG_AS
 AM_PROG_CC_C_O
@@ -83,7 +84,6 @@ AC_CONFIG_FILES([
 	cmd/zinject/Makefile
 	cmd/zpool/Makefile
 	cmd/zstream/Makefile
-	cmd/zstreamdump/Makefile
 	cmd/ztest/Makefile
 	cmd/zvol_id/Makefile
 	cmd/zvol_wait/Makefile
@@ -171,9 +171,6 @@ AC_CONFIG_FILES([
 	lib/libzstd/Makefile
 	lib/libzutil/Makefile
 	man/Makefile
-	man/man1/Makefile
-	man/man5/Makefile
-	man/man8/Makefile
 	module/Kbuild
 	module/Makefile
 	module/avl/Makefile

contrib/Makefile.am

@@ -1,3 +1,5 @@
+include $(top_srcdir)/config/Shellcheck.am
+
 SUBDIRS = bash_completion.d pyzfs zcp
 if BUILD_LINUX
 SUBDIRS += bpftrace dracut initramfs
@@ -6,3 +8,5 @@ if PAM_ZFS_ENABLED
 SUBDIRS += pam_zfs_key
 endif
 DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pam_zfs_key pyzfs zcp
+
+SHELLCHECKDIRS = bash_completion.d bpftrace dracut initramfs

contrib/bash_completion.d/.gitignore

@@ -0,0 +1 @@
+/zfs

contrib/bash_completion.d/Makefile.am

@@ -1,5 +1,13 @@
+include $(top_srcdir)/config/Substfiles.am
+include $(top_srcdir)/config/Shellcheck.am
+
 bashcompletiondir = $(sysconfdir)/bash_completion.d
 
 noinst_DATA = zfs
 
-EXTRA_DIST = $(noinst_DATA)
+EXTRA_DIST += $(noinst_DATA)
+SUBSTFILES += $(noinst_DATA)
+
+SHELLCHECKSCRIPTS = $(noinst_DATA)
+SHELLCHECK_SHELL = bash
+SHELLCHECK_IGNORE = ,SC2207

contrib/bash_completion.d/zfs.in

@@ -21,13 +21,8 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 # OTHER DEALINGS IN THE SOFTWARE.
 
-if [[ -w /dev/zfs ]]; then
+__ZFS_CMD="@sbindir@/zfs"
-    __ZFS_CMD="zfs"
+__ZPOOL_CMD="@sbindir@/zpool"
-    __ZPOOL_CMD="zpool"
-else
-    __ZFS_CMD="sudo zfs"
-    __ZPOOL_CMD="sudo zpool"
-fi
 
 # Disable bash's built-in hostname completion, as this makes it impossible to
 # provide completions containing an @-sign, which is necessary for completing
@@ -67,24 +62,25 @@ __zfs_list_filesystems()
 
 __zfs_match_snapshot()
 {
-    local base_dataset=${cur%@*}
+    local base_dataset="${cur%@*}"
-    if [[ $base_dataset != $cur ]]
+    if [ "$base_dataset" != "$cur" ]
     then
-        $__ZFS_CMD list -H -o name -s name -t snapshot -d 1 $base_dataset
+        $__ZFS_CMD list -H -o name -s name -t snapshot -d 1 "$base_dataset"
     else
-        if [[ $cur != "" ]] && __zfs_list_datasets $cur &> /dev/null
+        if [ "$cur" != "" ] && __zfs_list_datasets "$cur" &> /dev/null
         then
-            $__ZFS_CMD list -H -o name -s name -t filesystem -r $cur | tail -n +2
+            $__ZFS_CMD list -H -o name -s name -t filesystem -r "$cur" | tail -n +2
             # We output the base dataset name even though we might be
             # completing a command that can only take a snapshot, because it
             # prevents bash from considering the completion finished when it
             # ends in the bare @.
-            echo $cur
+            echo "$cur"
-            echo $cur@
+            echo "$cur@"
         else
-            local datasets=$(__zfs_list_datasets)
+            local datasets
+            datasets="$(__zfs_list_datasets)"
             # As above
-            echo $datasets
+            echo "$datasets"
             if [[ "$cur" == */ ]]
             then
                 # If the current command ends with a slash, then the only way
@@ -94,54 +90,57 @@ __zfs_match_snapshot()
                 local num_children
                 # This is actually off by one as zfs list includes the named
                 # dataset in addition to its children
-                num_children=$(__zfs_list_datasets -d 1 ${cur%/} 2> /dev/null | wc -l)
+                num_children=$(__zfs_list_datasets -d 1 "${cur%/}" 2> /dev/null | wc -l)
                 if [[ $num_children != 2 ]]
                 then
                     return 0
                 fi
             fi
-            echo "$datasets" | awk '{print $1"@"}'
+            echo "$datasets" | awk '{print $1 "@"}'
         fi
     fi
 }
 
 __zfs_match_snapshot_or_bookmark()
 {
-    local base_dataset=${cur%[#@]*}
+    local base_dataset="${cur%[#@]*}"
-    if [[ $base_dataset != $cur ]]
+    if [ "$base_dataset" != "$cur" ]
     then
         if [[ $cur == *@* ]]
         then
-            $__ZFS_CMD list -H -o name -s name -t snapshot -d 1 $base_dataset
+            $__ZFS_CMD list -H -o name -s name -t snapshot -d 1 "$base_dataset"
         else
-            $__ZFS_CMD list -H -o name -s name -t bookmark -d 1 $base_dataset
+            $__ZFS_CMD list -H -o name -s name -t bookmark -d 1 "$base_dataset"
         fi
     else
         $__ZFS_CMD list -H -o name -s name -t filesystem,volume
-        if [[ $cur != "" ]] && $__ZFS_CMD list -H -o name -s name -t filesystem,volume $cur &> /dev/null
+        if [ -e "$cur" ] && $__ZFS_CMD list -H -o name -s name -t filesystem,volume "$cur" &> /dev/null
         then
-            echo $cur@
+            echo "$cur@"
-            echo $cur#
+            echo "$cur#"
         fi
     fi
 }
 
 __zfs_match_multiple_snapshots()
 {
-    local existing_opts=$(expr "$cur" : '\(.*\)[%,]')
+    local existing_opts
-    if [[ $existing_opts ]]
+    existing_opts="$(expr "$cur" : '\(.*\)[%,]')"
+    if [ -e "$existing_opts" ]
     then
-        local base_dataset=${cur%@*}
+        local base_dataset="${cur%@*}"
-        if [[ $base_dataset != $cur ]]
+        if [ "$base_dataset" != "$cur" ]
         then
-            local cur=${cur##*,}
+            local cur="${cur##*,}"
             if [[ $cur =~ ^%|%.*% ]]
             then
                 # correct range syntax is start%end
                 return 1
             fi
-            local range_start=$(expr "$cur" : '\(.*%\)')
+            local range_start
-            $__ZFS_CMD list -H -o name -s name -t snapshot -d 1 $base_dataset | sed 's$.*@$'$range_start'$g'
+            range_start="$(expr "$cur" : '\(.*%\)')"
+            # shellcheck disable=SC2016
+            $__ZFS_CMD list -H -o name -s name -t snapshot -d 1 "$base_dataset" | sed 's$.*@$'"$range_start"'$g'
         fi
     else
         __zfs_match_snapshot_or_bookmark
@@ -165,7 +164,7 @@ __zfs_argument_chosen()
             then
                 return 0
             fi
-            for property in $@
+            for property in "$@"
             do
                 if [[ $prev == "$property"* ]]
                 then
@@ -183,6 +182,7 @@ __zfs_complete_ordered_arguments()
     local list2=$2
     local cur=$3
     local extra=$4
+    # shellcheck disable=SC2086
     if __zfs_argument_chosen $list1
     then
         COMPREPLY=($(compgen -W "$list2 $extra" -- "$cur"))
@@ -195,10 +195,11 @@ __zfs_complete_multiple_options()
 {
     local options=$1
     local cur=$2
+    local existing_opts
 
     COMPREPLY=($(compgen -W "$options" -- "${cur##*,}"))
-    local existing_opts=$(expr "$cur" : '\(.*,\)')
+    existing_opts=$(expr "$cur" : '\(.*,\)')
-    if [[ $existing_opts ]] 
+    if [[ $existing_opts ]]
     then
         COMPREPLY=( "${COMPREPLY[@]/#/${existing_opts}}" )
     fi
@@ -292,6 +293,7 @@ __zfs_complete()
                 *)
                     if ! __zfs_complete_switch "H,r,p,d,o,t,s"
                     then
+                        # shellcheck disable=SC2046
                         if __zfs_argument_chosen $(__zfs_get_properties)
                         then
                             COMPREPLY=($(compgen -W "$(__zfs_match_snapshot)" -- "$cur"))
@@ -305,7 +307,7 @@ __zfs_complete()
         inherit)
             if ! __zfs_complete_switch "r"
             then
-                __zfs_complete_ordered_arguments "$(__zfs_get_inheritable_properties)" "$(__zfs_match_snapshot)" $cur
+                __zfs_complete_ordered_arguments "$(__zfs_get_inheritable_properties)" "$(__zfs_match_snapshot)" "$cur"
             fi
             ;;
         list)
@@ -371,7 +373,7 @@ __zfs_complete()
             esac
             ;;
         set)
-            __zfs_complete_ordered_arguments "$(__zfs_get_editable_properties)" "$(__zfs_match_snapshot)" $cur
+            __zfs_complete_ordered_arguments "$(__zfs_get_editable_properties)" "$(__zfs_match_snapshot)" "$cur"
             __zfs_complete_nospace
             ;;
         upgrade)
@@ -390,7 +392,7 @@ __zfs_complete()
         destroy)
             if ! __zfs_complete_switch "d,f,n,p,R,r,v"
             then
-                __zfs_complete_multiple_options "$(__zfs_match_multiple_snapshots)" $cur
+                __zfs_complete_multiple_options "$(__zfs_match_multiple_snapshots)" "$cur"
                 __zfs_complete_nospace
             fi
             ;;
@@ -427,7 +429,7 @@ __zpool_list_pools()
 
 __zpool_complete()
 {
-    local cur prev cmd cmds
+    local cur prev cmd cmds pools
     COMPREPLY=()
     cur="${COMP_WORDS[COMP_CWORD]}"
     prev="${COMP_WORDS[COMP_CWORD-1]}"
@@ -442,7 +444,7 @@ __zpool_complete()
 
     case "${cmd}" in
         get)
-            __zfs_complete_ordered_arguments "$(__zpool_get_properties)" "$(__zpool_list_pools)" $cur
+            __zfs_complete_ordered_arguments "$(__zpool_get_properties)" "$(__zpool_list_pools)" "$cur"
             return 0
             ;;
         import)
@@ -455,12 +457,13 @@ __zpool_complete()
             return 0
             ;;
         set)
-            __zfs_complete_ordered_arguments "$(__zpool_get_editable_properties)" "$(__zpool_list_pools)" $cur
+            __zfs_complete_ordered_arguments "$(__zpool_get_editable_properties)" "$(__zpool_list_pools)" "$cur"
             __zfs_complete_nospace
             return 0
             ;;
         add|attach|clear|create|detach|offline|online|remove|replace)
-            local pools="$(__zpool_list_pools)"
+            pools="$(__zpool_list_pools)"
+            # shellcheck disable=SC2086
             if __zfs_argument_chosen $pools
             then
                 _filedir

contrib/bpftrace/Makefile.am

@@ -1,3 +1,7 @@
+include $(top_srcdir)/config/Shellcheck.am
+
 EXTRA_DIST = \
 	taskqlatency.bt \
 	zfs-trace.sh
+
+SHELLCHECKSCRIPTS = zfs-trace.sh

contrib/bpftrace/zfs-trace.sh

@@ -1,9 +1,9 @@
 #!/bin/sh
 
-ZVER=$(cat /sys/module/zfs/version | cut -f 1 -d '-')
+ZVER=$(cut -f 1 -d '-' /sys/module/zfs/version)
 KVER=$(uname -r)
 
-bpftrace \
+exec bpftrace \
 	--include "/usr/src/zfs-$ZVER/$KVER/zfs_config.h" \
 	-I "/usr/src/zfs-$ZVER/include" \
 	-I "/usr/src/zfs-$ZVER/include/spl" \

contrib/dracut/02zfsexpandknowledge/Makefile.am

@@ -1,4 +1,5 @@
 include $(top_srcdir)/config/Substfiles.am
+include $(top_srcdir)/config/Shellcheck.am
 
 pkgdracutdir = $(dracutdir)/modules.d/02zfsexpandknowledge
 pkgdracut_SCRIPTS = \

contrib/dracut/02zfsexpandknowledge/module-setup.sh.in

@@ -14,22 +14,16 @@ get_pool_devices() {
   local poolconfigtemp
   local poolconfigoutput
   local pooldev
-  local prefix
   local resolved
-  poolconfigtemp=`mktemp`
+  poolconfigtemp="$(mktemp)"
-  @sbindir@/zpool list -v -H -P "$1" > "$poolconfigtemp" 2>&1
+  if ! @sbindir@/zpool list -v -H -P "$1" > "$poolconfigtemp" 2>&1 ; then
-  if [ "$?" != "0" ] ; then
+    poolconfigoutput="$(cat "$poolconfigtemp")"
-    poolconfigoutput=$(cat "$poolconfigtemp")
     dinfo "zfsexpandknowledge: pool $1 cannot be listed: $poolconfigoutput"
   else
-    cat "$poolconfigtemp" |  awk -F '\t' '/\t\/dev/ { print $2 }' | \
+    awk -F '\t' '/\t\/dev/ { print $2 }' "$poolconfigtemp" | \
-    while read pooldev ; do
+    while read -r pooldev ; do
-        if [ -n "$pooldev" -a -e "$pooldev" ] ; then
+        if [ -e "$pooldev" ] ; then
-          if [ -h "$pooldev" ] ; then
+          resolved="$(readlink -f "$pooldev")"
-              resolved=`readlink -f "$pooldev"`
-          else
-              resolved="$pooldev"
-          fi
           dinfo "zfsexpandknowledge: pool $1 has device $pooldev (which resolves to $resolved)"
           echo "$resolved"
         fi
@@ -40,22 +34,20 @@ get_pool_devices() {
 
 find_zfs_block_devices() {
     local dev
-    local blockdev
     local mp
     local fstype
     local pool
-    local key
+    local _
-    local n
+    numfields="$(awk '{print NF; exit}' /proc/self/mountinfo)"
-    local poolconfigoutput
+    if [ "$numfields" = "10" ] ; then
-    numfields=`head -1 /proc/self/mountinfo | awk '{print NF}'`
+        fields="_ _ _ _ mp _ _ fstype dev _"
-    if [ "$numfields" == "10" ] ; then
-        fields="n n n n mp n n fstype dev n"
     else
-        fields="n n n n mp n n n fstype dev n"
+        fields="_ _ _ _ mp _ _ _ fstype dev _"
     fi
-    while read $fields ; do
+    # shellcheck disable=SC2086
-       if [ "$fstype" != "zfs" ]; then continue ; fi
+    while read -r ${fields?} ; do
-       if [ "$mp" == "$1" ]; then
+       [ "$fstype" = "zfs" ] || continue
+       if [ "$mp" = "$1" ]; then
            pool=$(echo "$dev" | cut -d / -f 1)
            get_pool_devices "$pool"
        fi
@@ -77,10 +69,9 @@ check() {
     local _depdev
     local _depdevname
     local _depdevtype
-    local _depmajmin
-    local _dev
 
-if [[ $hostonly ]]; then
+# shellcheck disable=SC2154
+if [ -n "$hostonly" ]; then
 
     for mp in \
         "/" \
@@ -100,21 +91,20 @@ if [[ $hostonly ]]; then
         mountpoint "$mp" >/dev/null 2>&1 || continue
         blockdevs=$(find_zfs_block_devices "$mp")
         if [ -z "$blockdevs" ] ; then continue ; fi
-        dinfo "zfsexpandknowledge: block devices backing ZFS dataset $mp: $blockdevs"
+        dinfo "zfsexpandknowledge: block devices backing ZFS dataset $mp: ${blockdevs//$'\n'/ }"
         for dev in $blockdevs
         do
             array_contains "$dev" "${host_devs[@]}" || host_devs+=("$dev")
             fstype=$(get_devtype "$dev")
             host_fs_types["$dev"]="$fstype"
             majmin=$(get_maj_min "$dev")
-            if [[ -d /sys/dev/block/$majmin/slaves ]] ; then
+            if [ -d "/sys/dev/block/$majmin/slaves" ] ; then
-                for _depdev in /sys/dev/block/$majmin/slaves/*; do
+                for _depdev in "/sys/dev/block/$majmin/slaves"/*; do
                     [[ -f $_depdev/dev ]] || continue
                     _depdev=/dev/$(basename "$_depdev")
                     _depdevname=$(udevadm info --query=property --name="$_depdev" | grep "^DEVNAME=" | sed 's|^DEVNAME=||')
                     _depdevtype=$(get_devtype "$_depdevname")
-                    _depmajmin=$(get_maj_min "$_depdevname")
+                    dinfo "zfsexpandknowledge: underlying block device backing ZFS dataset $mp: ${_depdevname//$'\n'/ }"
-                    dinfo "zfsexpandknowledge: underlying block device backing ZFS dataset $mp: $_depdevname"
                     array_contains "$_depdevname" "${host_devs[@]}" || host_devs+=("$_depdevname")
                     host_fs_types["$_depdevname"]="$_depdevtype"
                 done