Tag zfs-2.1.11

META file and changelog updated. Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Values printed by zpool-iostat(8) should be right-aligned
2026-05-23 02:44:41 +03:00 · 2023-04-18 11:44:34 -07:00 · 2023-04-18 11:44:07 -07:00 · 2023-04-18 11:44:07 -07:00 · 2023-04-18 10:10:45 -07:00 · 2023-04-13 15:59:45 -07:00
116 changed files with 2452 additions and 1976 deletions
@@ -0,0 +1,57 @@
+acl
+alien
+attr
+autoconf
+bc
+build-essential
+curl
+dbench
+debhelper-compat
+dh-python
+dkms
+fakeroot
+fio
+gdb
+gdebi
+git
+ksh
+lcov
+libacl1-dev
+libaio-dev
+libattr1-dev
+libblkid-dev
+libcurl4-openssl-dev
+libdevmapper-dev
+libelf-dev
+libffi-dev
+libmount-dev
+libpam0g-dev
+libselinux1-dev
+libssl-dev
+libtool
+libudev-dev
+linux-headers-generic
+lsscsi
+mdadm
+nfs-kernel-server
+pamtester
+parted
+po-debconf
+python3
+python3-all-dev
+python3-cffi
+python3-dev
+python3-packaging
+python3-pip
+python3-setuptools
+python3-sphinx
+rng-tools-debian
+rsync
+samba
+sysstat
+uuid-dev
+watchdog
+wget
+xfslibs-dev
+xz-utils
+zlib1g-dev
@@ -0,0 +1,2 @@
+pax-utils
+shellcheck
@@ -6,20 +6,27 @@ on:

 jobs:
  checkstyle:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
    steps:
    - uses: actions/checkout@v3
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Install dependencies
      run: |
-        sudo apt-get update
-        sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot linux-headers-$(uname -r)
-        sudo apt-get install --yes -qq zlib1g-dev uuid-dev libattr1-dev libblkid-dev libselinux-dev libudev-dev libssl-dev python-dev python-setuptools python-cffi python3 python3-dev python3-setuptools python3-cffi
-        # packages for tests
-        sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio
-        sudo apt-get install --yes -qq mandoc cppcheck pax-utils devscripts
-        sudo -E pip --quiet install flake8
+        # https://github.com/orgs/community/discussions/47863
+        sudo apt-mark hold grub-efi-amd64-signed
+        sudo apt-get update --fix-missing
+        sudo apt-get upgrade
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/checkstyle-dependencies.txt apt-get install -qq
+        sudo python3 -m pip install --quiet flake8
+        sudo apt-get clean
+
+        # confirm that the tools are installed
+        # the build system doesn't fail when they are not
+        flake8 --version
+        scanelf --version
+        shellcheck --version
    - name: Prepare
      run: |
        sh ./autogen.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -eu
+
+# remove 4GiB of images
+sudo systemd-run docker system prune --force --all --volumes
+
+# remove unused software
+sudo systemd-run --wait rm -rf \
+  "$AGENT_TOOLSDIRECTORY" \
+  /opt/* \
+  /usr/local/* \
+  /usr/share/az* \
+  /usr/share/dotnet \
+  /usr/share/gradle* \
+  /usr/share/miniconda \
+  /usr/share/swift \
+  /var/lib/gems \
+  /var/lib/mysql \
+  /var/lib/snapd
+
+# trim the cleaned space
+sudo fstrim /
@@ -9,7 +9,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        os: [20.04]
+        os: [20.04, 22.04]
    runs-on: ubuntu-${{ matrix.os }}
    steps:
    - uses: actions/checkout@v3
@@ -17,17 +17,12 @@ jobs:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Install dependencies
      run: |
-        sudo apt-get update
-        sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \
-          git alien fakeroot wget curl bc fio acl \
-          sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \
-          nfs-kernel-server samba rng-tools xz-utils \
-          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
-          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
-          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
-          libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi python3-packaging \
-          libcurl4-openssl-dev
+        # https://github.com/orgs/community/discussions/47863
+        sudo apt-mark hold grub-efi-amd64-signed
+        sudo apt-get update --fix-missing
+        sudo apt-get upgrade
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
+        sudo apt-get clean
    - name: Autogen.sh
      run: |
        sh autogen.sh
@@ -56,12 +51,13 @@ jobs:
            sudo udevadm control --reload-rules
          fi
        fi
-        # Workaround to provide additional free space for testing.
-        #   https://github.com/actions/virtual-environments/issues/2840
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf /opt/ghc
-        sudo rm -rf "/usr/local/share/boost"
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+    - name: Clear the kernel ring buffer
+      run: |
+        sudo dmesg -c >/var/tmp/dmesg-prerun
+    - name: Reclaim and report disk space
+      run: |
+        ${{ github.workspace }}/.github/workflows/scripts/reclaim_disk_space.sh
+        df -h /
    - name: Tests
      run: |
        /usr/share/zfs/zfs-tests.sh -vR -s 3G
@@ -71,7 +67,7 @@ jobs:
      run: |
        RESULTS_PATH=$(readlink -f /var/tmp/test_results/current)
        sudo dmesg > $RESULTS_PATH/dmesg
-        sudo cp /var/log/syslog $RESULTS_PATH/
+        sudo cp /var/log/syslog /var/tmp/dmesg-prerun $RESULTS_PATH/
        sudo chmod +r $RESULTS_PATH/*
        # Replace ':' in dir names, actions/upload-artifact doesn't support it
        for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done
@@ -79,5 +75,7 @@ jobs:
      if: failure()
      with:
        name: Test logs Ubuntu-${{ matrix.os }}
-        path: /var/tmp/test_results/20*/
+        path: |
+          /var/tmp/test_results/*
+          !/var/tmp/test_results/current
        if-no-files-found: ignore
@@ -6,24 +6,19 @@ on:

 jobs:
  tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
    steps:
    - uses: actions/checkout@v3
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Install dependencies
      run: |
-        sudo apt-get update
-        sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \
-          git alien fakeroot wget curl bc fio acl \
-          sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \
-          nfs-kernel-server samba rng-tools xz-utils \
-          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
-          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
-          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
-          libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi python3-packaging \
-          libcurl4-openssl-dev
+        # https://github.com/orgs/community/discussions/47863
+        sudo apt-mark hold grub-efi-amd64-signed
+        sudo apt-get update --fix-missing
+        sudo apt-get upgrade
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
+        sudo apt-get clean
    - name: Autogen.sh
      run: |
        sh autogen.sh
@@ -52,12 +47,13 @@ jobs:
            sudo udevadm control --reload-rules
          fi
        fi
-        # Workaround to provide additional free space for testing.
-        #   https://github.com/actions/virtual-environments/issues/2840
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf /opt/ghc
-        sudo rm -rf "/usr/local/share/boost"
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+    - name: Clear the kernel ring buffer
+      run: |
+        sudo dmesg -c >/var/tmp/dmesg-prerun
+    - name: Reclaim and report disk space
+      run: |
+        ${{ github.workspace }}/.github/workflows/scripts/reclaim_disk_space.sh
+        df -h /
    - name: Tests
      run: |
        /usr/share/zfs/zfs-tests.sh -vR -s 3G -r sanity
@@ -67,13 +63,15 @@ jobs:
      run: |
        RESULTS_PATH=$(readlink -f /var/tmp/test_results/current)
        sudo dmesg > $RESULTS_PATH/dmesg
-        sudo cp /var/log/syslog $RESULTS_PATH/
+        sudo cp /var/log/syslog /var/tmp/dmesg-prerun $RESULTS_PATH/
        sudo chmod +r $RESULTS_PATH/*
        # Replace ':' in dir names, actions/upload-artifact doesn't support it
        for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done
    - uses: actions/upload-artifact@v3
      if: failure()
      with:
-        name: Test logs
-        path: /var/tmp/test_results/20*/
+        name: Test logs Ubuntu-${{ matrix.os }}
+        path: |
+          /var/tmp/test_results/*
+          !/var/tmp/test_results/current
        if-no-files-found: ignore
@@ -6,7 +6,7 @@ on:

 jobs:
  tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
    env:
      TEST_DIR: /var/tmp/zloop
    steps:
@@ -15,15 +15,12 @@ jobs:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Install dependencies
      run: |
-        sudo apt-get update
-        sudo apt-get install --yes -qq build-essential autoconf libtool gdb \
-          git alien fakeroot \
-          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
-          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
-          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
-          libpam0g-dev \
-          python-dev python-setuptools python-cffi python-packaging \
-          python3 python3-dev python3-setuptools python3-cffi python3-packaging
+        # https://github.com/orgs/community/discussions/47863
+        sudo apt-mark hold grub-efi-amd64-signed
+        sudo apt-get update --fix-missing
+        sudo apt-get upgrade
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
+        sudo apt-get clean
    - name: Autogen.sh
      run: |
        sh autogen.sh
@@ -44,8 +41,9 @@ jobs:
    - name: Tests
      run: |
        sudo mkdir -p $TEST_DIR
-        # run for 20 minutes to have a total runner time of 30 minutes
-        sudo /usr/share/zfs/zloop.sh -t 1200 -l -m1 -- -T 120 -P 60
+        # run for 10 minutes or at most 2 iterations for a maximum runner
+        #   time of 20 minutes.
+        sudo /usr/share/zfs/zloop.sh -t 600 -I 2 -l -m1 -- -T 120 -P 60
    - name: Prepare artifacts
      if: failure()
      run: |
@@ -1,10 +1,10 @@
 Meta:          1
 Name:          zfs
 Branch:        1.0
-Version:       2.1.9
+Version:       2.1.11
 Release:       1
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS
-Linux-Maximum: 6.1
+Linux-Maximum: 6.2
 Linux-Minimum: 3.10
@@ -1,13 +1,8 @@
 bin_SCRIPTS = arc_summary

 CLEANFILES = arc_summary
-EXTRA_DIST = arc_summary2 arc_summary3
-
-if USING_PYTHON_2
-SCRIPT = arc_summary2
-else
+EXTRA_DIST = arc_summary3
 SCRIPT = arc_summary3
-endif

 arc_summary: $(SCRIPT)
 	cp $< $@
@@ -191,21 +191,13 @@ elif sys.platform.startswith('linux'):
        # there, so we fall back on modinfo
        command = ["/sbin/modinfo", request, "-0"]

-        # The recommended way to do this is with subprocess.run(). However,
-        # some installed versions of Python are < 3.5, so we offer them
-        # the option of doing it the old way (for now)
        info = ''

        try:

-            if 'run' in dir(subprocess):
-                info = subprocess.run(command, stdout=subprocess.PIPE,
-                                      universal_newlines=True)
-                raw_output = info.stdout.split('\0')
-            else:
-                info = subprocess.check_output(command,
-                                               universal_newlines=True)
-                raw_output = info.split('\0')
+            info = subprocess.run(command, stdout=subprocess.PIPE,
+                                  check=True, universal_newlines=True)
+            raw_output = info.stdout.split('\0')

        except subprocess.CalledProcessError:
            print("Error: Descriptions not available",
@@ -47,7 +47,7 @@
 # @hdr is the array of fields that needs to be printed, so we
 # just iterate over this array and print the values using our pretty printer.
 #
-# This script must remain compatible with Python 2.6+ and Python 3.4+.
+# This script must remain compatible with Python 3.6+.
 #

 import sys
@@ -27,7 +27,7 @@
 # Copyright (C) 2013 Lawrence Livermore National Security, LLC.
 # Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 #
-# This script must remain compatible with Python 2.6+ and Python 3.4+.
+# This script must remain compatible with and Python 3.6+.
 #

 import sys
@@ -80,6 +80,7 @@ zfs_agent_iter_vdev(zpool_handle_t *zhp, nvlist_t *nvl, void *arg)
 	char *path = NULL;
 	uint_t c, children;
 	nvlist_t **child;
+	uint64_t vdev_guid;

 	/*
 	 * First iterate over any children.
@@ -100,7 +101,7 @@ zfs_agent_iter_vdev(zpool_handle_t *zhp, nvlist_t *nvl, void *arg)
 	    &child, &children) == 0) {
 		for (c = 0; c < children; c++) {
 			if (zfs_agent_iter_vdev(zhp, child[c], gsp)) {
-				gsp->gs_vdev_type = DEVICE_TYPE_L2ARC;
+				gsp->gs_vdev_type = DEVICE_TYPE_SPARE;
 				return (B_TRUE);
 			}
 		}
@@ -109,7 +110,7 @@ zfs_agent_iter_vdev(zpool_handle_t *zhp, nvlist_t *nvl, void *arg)
 	    &child, &children) == 0) {
 		for (c = 0; c < children; c++) {
 			if (zfs_agent_iter_vdev(zhp, child[c], gsp)) {
-				gsp->gs_vdev_type = DEVICE_TYPE_SPARE;
+				gsp->gs_vdev_type = DEVICE_TYPE_L2ARC;
 				return (B_TRUE);
 			}
 		}
@@ -126,6 +127,21 @@ zfs_agent_iter_vdev(zpool_handle_t *zhp, nvlist_t *nvl, void *arg)
 		    &gsp->gs_vdev_expandtime);
 		return (B_TRUE);
 	}
+	/*
+	 * Otherwise, on a vdev guid match, grab the devid and expansion
+	 * time. The devid might be missing on removal since its not part
+	 * of blkid cache and L2ARC VDEV does not contain pool guid in its
+	 * blkid, so this is a special case for L2ARC VDEV.
+	 */
+	else if (gsp->gs_vdev_guid != 0 && gsp->gs_devid == NULL &&
+	    nvlist_lookup_uint64(nvl, ZPOOL_CONFIG_GUID, &vdev_guid) == 0 &&
+	    gsp->gs_vdev_guid == vdev_guid) {
+		(void) nvlist_lookup_string(nvl, ZPOOL_CONFIG_DEVID,
+		    &gsp->gs_devid);
+		(void) nvlist_lookup_uint64(nvl, ZPOOL_CONFIG_EXPANSION_TIME,
+		    &gsp->gs_vdev_expandtime);
+		return (B_TRUE);
+	}

 	return (B_FALSE);
 }
@@ -148,13 +164,13 @@ zfs_agent_iter_pool(zpool_handle_t *zhp, void *arg)
 	/*
 	 * if a match was found then grab the pool guid
 	 */
-	if (gsp->gs_vdev_guid) {
+	if (gsp->gs_vdev_guid && gsp->gs_devid) {
 		(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
 		    &gsp->gs_pool_guid);
 	}

 	zpool_close(zhp);
-	return (gsp->gs_vdev_guid != 0);
+	return (gsp->gs_devid != NULL && gsp->gs_vdev_guid != 0);
 }

 void
@@ -195,11 +211,13 @@ zfs_agent_post_event(const char *class, const char *subclass, nvlist_t *nvl)
 		uint64_t pool_guid = 0, vdev_guid = 0;
 		guid_search_t search = { 0 };
 		device_type_t devtype = DEVICE_TYPE_PRIMARY;
+		char *devid = NULL;

 		class = "resource.fs.zfs.removed";
 		subclass = "";

 		(void) nvlist_add_string(payload, FM_CLASS, class);
+		(void) nvlist_lookup_string(nvl, DEV_IDENTIFIER, &devid);
 		(void) nvlist_lookup_uint64(nvl, ZFS_EV_POOL_GUID, &pool_guid);
 		(void) nvlist_lookup_uint64(nvl, ZFS_EV_VDEV_GUID, &vdev_guid);

@@ -209,20 +227,24 @@ zfs_agent_post_event(const char *class, const char *subclass, nvlist_t *nvl)
 		(void) nvlist_add_int64_array(payload, FM_EREPORT_TIME, tod, 2);

 		/*
+		 * If devid is missing but vdev_guid is available, find devid
+		 * and pool_guid from vdev_guid.
 		 * For multipath, spare and l2arc devices ZFS_EV_VDEV_GUID or
 		 * ZFS_EV_POOL_GUID may be missing so find them.
 		 */
-		if (pool_guid == 0 || vdev_guid == 0) {
-			if ((nvlist_lookup_string(nvl, DEV_IDENTIFIER,
-			    &search.gs_devid) == 0) &&
-			    (zpool_iter(g_zfs_hdl, zfs_agent_iter_pool, &search)
-			    == 1)) {
-				if (pool_guid == 0)
-					pool_guid = search.gs_pool_guid;
-				if (vdev_guid == 0)
-					vdev_guid = search.gs_vdev_guid;
-				devtype = search.gs_vdev_type;
-			}
+		if (devid == NULL || pool_guid == 0 || vdev_guid == 0) {
+			if (devid == NULL)
+				search.gs_vdev_guid = vdev_guid;
+			else
+				search.gs_devid = devid;
+			zpool_iter(g_zfs_hdl, zfs_agent_iter_pool, &search);
+			if (devid == NULL)
+				devid = search.gs_devid;
+			if (pool_guid == 0)
+				pool_guid = search.gs_pool_guid;
+			if (vdev_guid == 0)
+				vdev_guid = search.gs_vdev_guid;
+			devtype = search.gs_vdev_type;
 		}

 		/*
@@ -235,7 +257,9 @@ zfs_agent_post_event(const char *class, const char *subclass, nvlist_t *nvl)
 		    search.gs_vdev_expandtime + 10 > tv.tv_sec) {
 			zed_log_msg(LOG_INFO, "agent post event: ignoring '%s' "
 			    "for recently expanded device '%s'", EC_DEV_REMOVE,
-			    search.gs_devid);
+			    devid);
+			fnvlist_free(payload);
+			free(event);
 			goto out;
 		}

@@ -185,10 +185,12 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	uint64_t wholedisk = 0ULL;
 	uint64_t offline = 0ULL, faulted = 0ULL;
 	uint64_t guid = 0ULL;
+	uint64_t is_spare = 0;
 	char *physpath = NULL, *new_devid = NULL, *enc_sysfs_path = NULL;
 	char rawpath[PATH_MAX], fullpath[PATH_MAX];
 	char devpath[PATH_MAX];
 	int ret;
+	int online_flag = ZFS_ONLINE_CHECKREMOVE | ZFS_ONLINE_UNSPARE;
 	boolean_t is_sd = B_FALSE;
 	boolean_t is_mpath_wholedisk = B_FALSE;
 	uint_t c;
@@ -214,6 +216,7 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_FAULTED, &faulted);

 	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_GUID, &guid);
+	(void) nvlist_lookup_uint64(vdev, ZPOOL_CONFIG_IS_SPARE, &is_spare);

 	/*
 	 * Special case:
@@ -304,11 +307,13 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 		}
 	}

+	if (is_spare)
+		online_flag |= ZFS_ONLINE_SPARE;
+
 	/*
 	 * Attempt to online the device.
 	 */
-	if (zpool_vdev_online(zhp, fullpath,
-	    ZFS_ONLINE_CHECKREMOVE | ZFS_ONLINE_UNSPARE, &newstate) == 0 &&
+	if (zpool_vdev_online(zhp, fullpath, online_flag, &newstate) == 0 &&
 	    (newstate == VDEV_STATE_HEALTHY ||
 	    newstate == VDEV_STATE_DEGRADED)) {
 		zed_log_msg(LOG_INFO,
@@ -527,6 +532,7 @@ typedef struct dev_data {
 	uint64_t		dd_vdev_guid;
 	uint64_t		dd_new_vdev_guid;
 	const char		*dd_new_devid;
+	uint64_t		dd_num_spares;
 } dev_data_t;

 static void
@@ -537,6 +543,7 @@ zfs_iter_vdev(zpool_handle_t *zhp, nvlist_t *nvl, void *data)
 	uint_t c, children;
 	nvlist_t **child;
 	uint64_t guid = 0;
+	uint64_t isspare = 0;

 	/*
 	 * First iterate over any children.
@@ -562,7 +569,7 @@ zfs_iter_vdev(zpool_handle_t *zhp, nvlist_t *nvl, void *data)
 	}

 	/* once a vdev was matched and processed there is nothing left to do */
-	if (dp->dd_found)
+	if (dp->dd_found && dp->dd_num_spares == 0)
 		return;
 	(void) nvlist_lookup_uint64(nvl, ZPOOL_CONFIG_GUID, &guid);

@@ -612,6 +619,10 @@ zfs_iter_vdev(zpool_handle_t *zhp, nvlist_t *nvl, void *data)
 		}
 	}

+	if (dp->dd_found == B_TRUE && nvlist_lookup_uint64(nvl,
+	    ZPOOL_CONFIG_IS_SPARE, &isspare) == 0 && isspare)
+		dp->dd_num_spares++;
+
 	(dp->dd_func)(zhp, nvl, dp->dd_islabeled);
 }

@@ -672,7 +683,9 @@ zfs_iter_pool(zpool_handle_t *zhp, void *data)
 	}

 	zpool_close(zhp);
-	return (dp->dd_found);	/* cease iteration after a match */
+
+	/* cease iteration after a match */
+	return (dp->dd_found && dp->dd_num_spares == 0);
 }

 /*
@@ -75,6 +75,8 @@ typedef struct find_cbdata {
 	uint64_t	cb_guid;
 	zpool_handle_t	*cb_zhp;
 	nvlist_t	*cb_vdev;
+	uint64_t	cb_vdev_guid;
+	uint64_t	cb_num_spares;
 } find_cbdata_t;

 static int
@@ -140,6 +142,64 @@ find_vdev(libzfs_handle_t *zhdl, nvlist_t *nv, uint64_t search_guid)
 	return (NULL);
 }

+static int
+remove_spares(zpool_handle_t *zhp, void *data)
+{
+	nvlist_t *config, *nvroot;
+	nvlist_t **spares;
+	uint_t nspares;
+	char *devname;
+	find_cbdata_t *cbp = data;
+	uint64_t spareguid = 0;
+	vdev_stat_t *vs;
+	unsigned int c;
+
+	config = zpool_get_config(zhp, NULL);
+	if (nvlist_lookup_nvlist(config,
+	    ZPOOL_CONFIG_VDEV_TREE, &nvroot) != 0) {
+		zpool_close(zhp);
+		return (0);
+	}
+
+	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
+	    &spares, &nspares) != 0) {
+		zpool_close(zhp);
+		return (0);
+	}
+
+	for (int i = 0; i < nspares; i++) {
+		if (nvlist_lookup_uint64(spares[i], ZPOOL_CONFIG_GUID,
+		    &spareguid) == 0 && spareguid == cbp->cb_vdev_guid) {
+			devname = zpool_vdev_name(NULL, zhp, spares[i],
+			    B_FALSE);
+			nvlist_lookup_uint64_array(spares[i],
+			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &c);
+			if (vs->vs_state != VDEV_STATE_REMOVED &&
+			    zpool_vdev_remove_wanted(zhp, devname) == 0)
+				cbp->cb_num_spares++;
+			break;
+		}
+	}
+
+	zpool_close(zhp);
+	return (0);
+}
+
+/*
+ * Given a vdev guid, find and remove all spares associated with it.
+ */
+static int
+find_and_remove_spares(libzfs_handle_t *zhdl, uint64_t vdev_guid)
+{
+	find_cbdata_t cb;
+
+	cb.cb_num_spares = 0;
+	cb.cb_vdev_guid = vdev_guid;
+	zpool_iter(zhdl, remove_spares, &cb);
+
+	return (cb.cb_num_spares);
+}
+
 /*
 * Given a (pool, vdev) GUID pair, find the matching pool and vdev.
 */
@@ -315,6 +375,8 @@ zfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
 	libzfs_handle_t *zhdl = zdp->zrd_hdl;
 	boolean_t fault_device, degrade_device;
 	boolean_t is_repair;
+	boolean_t l2arc = B_FALSE;
+	boolean_t spare = B_FALSE;
 	char *scheme;
 	nvlist_t *vdev = NULL;
 	char *uuid;
@@ -323,6 +385,8 @@ zfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
 	boolean_t is_disk;
 	vdev_aux_t aux;
 	uint64_t state = 0;
+	vdev_stat_t *vs;
+	unsigned int c;

 	fmd_hdl_debug(hdl, "zfs_retire_recv: '%s'", class);

@@ -339,10 +403,26 @@ zfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
 		char *devtype;
 		char *devname;

+		if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
+		    &devtype) == 0) {
+			if (strcmp(devtype, VDEV_TYPE_SPARE) == 0)
+				spare = B_TRUE;
+			else if (strcmp(devtype, VDEV_TYPE_L2CACHE) == 0)
+				l2arc = B_TRUE;
+		}
+
+		if (nvlist_lookup_uint64(nvl,
+		    FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
+			return;
+
+		if (spare) {
+			int nspares = find_and_remove_spares(zhdl, vdev_guid);
+			fmd_hdl_debug(hdl, "%d spares removed", nspares);
+			return;
+		}
+
 		if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
-		    &pool_guid) != 0 ||
-		    nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
-		    &vdev_guid) != 0)
+		    &pool_guid) != 0)
 			return;

 		if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
@@ -351,13 +431,28 @@ zfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,

 		devname = zpool_vdev_name(NULL, zhp, vdev, B_FALSE);

-		/* Can't replace l2arc with a spare: offline the device */
-		if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
-		    &devtype) == 0 && strcmp(devtype, VDEV_TYPE_L2CACHE) == 0) {
-			fmd_hdl_debug(hdl, "zpool_vdev_offline '%s'", devname);
-			zpool_vdev_offline(zhp, devname, B_TRUE);
-		} else if (!fmd_prop_get_int32(hdl, "spare_on_remove") ||
-		    replace_with_spare(hdl, zhp, vdev) == B_FALSE) {
+		nvlist_lookup_uint64_array(vdev, ZPOOL_CONFIG_VDEV_STATS,
+		    (uint64_t **)&vs, &c);
+
+		/*
+		 * If state removed is requested for already removed vdev,
+		 * its a loopback event from spa_async_remove(). Just
+		 * ignore it.
+		 */
+		if (vs->vs_state == VDEV_STATE_REMOVED &&
+		    state == VDEV_STATE_REMOVED)
+			return;
+
+		/* Remove the vdev since device is unplugged */
+		if (l2arc || (strcmp(class, "resource.fs.zfs.removed") == 0)) {
+			int status = zpool_vdev_remove_wanted(zhp, devname);
+			fmd_hdl_debug(hdl, "zpool_vdev_remove_wanted '%s'"
+			    ", ret:%d", devname, status);
+		}
+
+		/* Replace the vdev with a spare if its not a l2arc */
+		if (!l2arc && (!fmd_prop_get_int32(hdl, "spare_on_remove") ||
+		    replace_with_spare(hdl, zhp, vdev) == B_FALSE)) {
 			/* Could not handle with spare */
 			fmd_hdl_debug(hdl, "no spare for '%s'", devname);
 		}
@@ -315,14 +315,14 @@ get_usage(zfs_help_t idx)
 	case HELP_ROLLBACK:
 		return (gettext("\trollback [-rRf] <snapshot>\n"));
 	case HELP_SEND:
-		return (gettext("\tsend [-DnPpRvLecwhb] [-[i|I] snapshot] "
+		return (gettext("\tsend [-DnPpRVvLecwhb] [-[i|I] snapshot] "
 		    "<snapshot>\n"
-		    "\tsend [-DnvPLecw] [-i snapshot|bookmark] "
+		    "\tsend [-DnVvPLecw] [-i snapshot|bookmark] "
 		    "<filesystem|volume|snapshot>\n"
-		    "\tsend [-DnPpvLec] [-i bookmark|snapshot] "
+		    "\tsend [-DnPpVvLec] [-i bookmark|snapshot] "
 		    "--redact <bookmark> <snapshot>\n"
-		    "\tsend [-nvPe] -t <receive_resume_token>\n"
-		    "\tsend [-Pnv] --saved filesystem\n"));
+		    "\tsend [-nVvPe] -t <receive_resume_token>\n"
+		    "\tsend [-PnVv] --saved filesystem\n"));
 	case HELP_SET:
 		return (gettext("\tset <property=value> ... "
 		    "<filesystem|volume|snapshot> ...\n"));
@@ -3474,6 +3474,8 @@ print_header(list_cbdata_t *cb)
 	boolean_t first = B_TRUE;
 	boolean_t right_justify;

+	color_start(ANSI_BOLD);
+
 	for (; pl != NULL; pl = pl->pl_next) {
 		if (!first) {
 			(void) printf("  ");
@@ -3500,9 +3502,31 @@ print_header(list_cbdata_t *cb)
 			(void) printf("%-*s", (int)pl->pl_width, header);
 	}

+	color_end();
+
 	(void) printf("\n");
 }

+/*
+ * Decides on the color that the avail value should be printed in.
+ * > 80% used = yellow
+ * > 90% used = red
+ */
+static const char *
+zfs_list_avail_color(zfs_handle_t *zhp)
+{
+	uint64_t used = zfs_prop_get_int(zhp, ZFS_PROP_USED);
+	uint64_t avail = zfs_prop_get_int(zhp, ZFS_PROP_AVAILABLE);
+	int percentage = (int)((double)avail / MAX(avail + used, 1) * 100);
+
+	if (percentage > 20)
+		return (NULL);
+	else if (percentage > 10)
+		return (ANSI_YELLOW);
+	else
+		return (ANSI_RED);
+}
+
 /*
 * Given a dataset and a list of fields, print out all the properties according
 * to the described layout.
@@ -3565,6 +3589,22 @@ print_dataset(zfs_handle_t *zhp, list_cbdata_t *cb)
 			right_justify = B_FALSE;
 		}

+		/*
+		 * zfs_list_avail_color() needs ZFS_PROP_AVAILABLE + USED
+		 * - so we need another for() search for the USED part
+		 * - when no colors wanted, we can skip the whole thing
+		 */
+		if (use_color() && pl->pl_prop == ZFS_PROP_AVAILABLE) {
+			zprop_list_t *pl2 = cb->cb_proplist;
+			for (; pl2 != NULL; pl2 = pl2->pl_next) {
+				if (pl2->pl_prop == ZFS_PROP_USED) {
+					color_start(zfs_list_avail_color(zhp));
+					/* found it, no need for more loops */
+					break;
+				}
+			}
+		}
+
 		/*
 		 * If this is being called in scripted mode, or if this is the
 		 * last column and it is left-justified, don't include a width
@@ -3576,6 +3616,9 @@ print_dataset(zfs_handle_t *zhp, list_cbdata_t *cb)
 			(void) printf("%*s", (int)pl->pl_width, propstr);
 		else
 			(void) printf("%-*s", (int)pl->pl_width, propstr);
+
+		if (pl->pl_prop == ZFS_PROP_AVAILABLE)
+			color_end();
 	}

 	(void) printf("\n");
@@ -4407,6 +4450,7 @@ zfs_do_send(int argc, char **argv)
 		{"props",	no_argument,		NULL, 'p'},
 		{"parsable",	no_argument,		NULL, 'P'},
 		{"dedup",	no_argument,		NULL, 'D'},
+		{"proctitle",	no_argument,		NULL, 'V'},
 		{"verbose",	no_argument,		NULL, 'v'},
 		{"dryrun",	no_argument,		NULL, 'n'},
 		{"large-block",	no_argument,		NULL, 'L'},
@@ -4421,7 +4465,7 @@ zfs_do_send(int argc, char **argv)
 	};

 	/* check options */
-	while ((c = getopt_long(argc, argv, ":i:I:RsDpvnPLeht:cwbd:S",
+	while ((c = getopt_long(argc, argv, ":i:I:RsDpVvnPLeht:cwbd:S",
 	    long_options, NULL)) != -1) {
 		switch (c) {
 		case 'i':
@@ -4456,6 +4500,9 @@ zfs_do_send(int argc, char **argv)
 		case 'P':
 			flags.parsable = B_TRUE;
 			break;
+		case 'V':
+			flags.progressastitle = B_TRUE;
+			break;
 		case 'v':
 			flags.verbosity++;
 			flags.progress = B_TRUE;
@@ -4531,7 +4578,7 @@ zfs_do_send(int argc, char **argv)
 		}
 	}

-	if (flags.parsable && flags.verbosity == 0)
+	if ((flags.parsable || flags.progressastitle) && flags.verbosity == 0)
 		flags.verbosity = 1;

 	argc -= optind;
@@ -8693,6 +8740,8 @@ main(int argc, char **argv)

 	libzfs_print_on_error(g_zfs, B_TRUE);

+	zfs_setproctitle_init(argc, argv, environ);
+
 	/*
 	 * Many commands modify input strings for string parsing reasons.
 	 * We create a copy to protect the original argv.
@@ -4205,6 +4205,8 @@ print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
 	unsigned int namewidth;
 	const char *title;

+	color_start(ANSI_BOLD);
+
 	if (cb->cb_flags & IOS_ANYHISTO_M) {
 		title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
 	} else if (cb->cb_vdev_names_count) {
@@ -4238,6 +4240,8 @@ print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
 	if (cb->vcdl != NULL)
 		print_cmd_columns(cb->vcdl, 1);

+	color_end();
+
 	printf("\n");
 }

@@ -4247,6 +4251,37 @@ print_iostat_header(iostat_cbdata_t *cb)
 	print_iostat_header_impl(cb, 0, NULL);
 }

+/*
+ * Prints a size string (i.e. 120M) with the suffix ("M") colored
+ * by order of magnitude. Uses column_size to add padding.
+ */
+static void
+print_stat_color(const char *statbuf, unsigned int column_size)
+{
+	fputs("  ", stdout);
+	size_t len = strlen(statbuf);
+	while (len < column_size) {
+		fputc(' ', stdout);
+		column_size--;
+	}
+	if (*statbuf == '0') {
+		color_start(ANSI_GRAY);
+		fputc('0', stdout);
+	} else {
+		for (; *statbuf; statbuf++) {
+			if (*statbuf == 'K') color_start(ANSI_GREEN);
+			else if (*statbuf == 'M') color_start(ANSI_YELLOW);
+			else if (*statbuf == 'G') color_start(ANSI_RED);
+			else if (*statbuf == 'T') color_start(ANSI_BOLD_BLUE);
+			else if (*statbuf == 'P') color_start(ANSI_MAGENTA);
+			else if (*statbuf == 'E') color_start(ANSI_CYAN);
+			fputc(*statbuf, stdout);
+			if (--column_size <= 0)
+				break;
+		}
+	}
+	color_end();
+}

 /*
 * Display a single statistic.
@@ -4262,7 +4297,7 @@ print_one_stat(uint64_t value, enum zfs_nicenum_format format,
 	if (scripted)
 		printf("\t%s", buf);
 	else
-		printf("  %*s", column_size, buf);
+		print_stat_color(buf, column_size);
 }

 /*
@@ -1184,7 +1184,7 @@ ztest_kill(ztest_shared_t *zs)
 	 * See comment above spa_write_cachefile().
 	 */
 	mutex_enter(&spa_namespace_lock);
-	spa_write_cachefile(ztest_spa, B_FALSE, B_FALSE);
+	spa_write_cachefile(ztest_spa, B_FALSE, B_FALSE, B_FALSE);
 	mutex_exit(&spa_namespace_lock);

 	(void) kill(getpid(), SIGKILL);
@@ -1,7 +1,6 @@
 dnl #
 dnl # The majority of the python scripts are written to be compatible
-dnl # with Python 2.6 and Python 3.4.  Therefore, they may be installed
-dnl # and used with either interpreter.  This option is intended to
+dnl # with Python 3.6. This option is intended to
 dnl # to provide a method to specify the default system version, and
 dnl # set the PYTHON environment variable accordingly.
 dnl #
@@ -13,9 +12,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYTHON], [
 		[with_python=check])

 	AS_CASE([$with_python],
-		[check], [AC_CHECK_PROGS([PYTHON], [python3 python2], [:])],
-		[2*], [PYTHON="python${with_python}"],
-		[*python2*], [PYTHON="${with_python}"],
+		[check], [AC_CHECK_PROGS([PYTHON], [python3], [:])],
 		[3*], [PYTHON="python${with_python}"],
 		[*python3*], [PYTHON="${with_python}"],
 		[no], [PYTHON=":"],
@@ -23,8 +20,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYTHON], [
 	)

 	dnl #
-	dnl # Minimum supported Python versions for utilities:
-	dnl # Python 2.6 or Python 3.4
+	dnl # Minimum supported Python versions for utilities: Python 3.6
 	dnl #
 	AM_PATH_PYTHON([], [], [:])
 	AS_IF([test -z "$PYTHON_VERSION"], [
@@ -33,25 +29,16 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYTHON], [
 	PYTHON_MINOR=${PYTHON_VERSION#*\.}

 	AS_CASE([$PYTHON_VERSION],
-		[2.*], [
-			AS_IF([test $PYTHON_MINOR -lt 6],
-				[AC_MSG_ERROR("Python >= 2.6 is required")])
-		],
 		[3.*], [
-			AS_IF([test $PYTHON_MINOR -lt 4],
-				[AC_MSG_ERROR("Python >= 3.4 is required")])
+			AS_IF([test $PYTHON_MINOR -lt 6],
+				[AC_MSG_ERROR("Python >= 3.6 is required")])
 		],
 		[:|2|3], [],
 		[PYTHON_VERSION=3]
 	)

 	AM_CONDITIONAL([USING_PYTHON], [test "$PYTHON" != :])
-	AM_CONDITIONAL([USING_PYTHON_2], [test "x${PYTHON_VERSION%%\.*}" = x2])
-	AM_CONDITIONAL([USING_PYTHON_3], [test "x${PYTHON_VERSION%%\.*}" = x3])
-
-	AM_COND_IF([USING_PYTHON_2],
-		[AC_SUBST([PYTHON_SHEBANG], [python2])],
-		[AC_SUBST([PYTHON_SHEBANG], [python3])])
+	AC_SUBST([PYTHON_SHEBANG], [python3])

 	dnl #
 	dnl # Request that packages be built for a specific Python version.
@@ -18,7 +18,7 @@ AC_DEFUN([ZFS_AC_PYTHON_MODULE], [
 ])

 dnl #
-dnl # Determines if pyzfs can be built, requires Python 2.7 or later.
+dnl # Determines if pyzfs can be built, requires Python 3.6 or later.
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 	AC_ARG_ENABLE([pyzfs],
@@ -72,12 +72,11 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_PYZFS], [
 	])

 	dnl #
-	dnl # Require python-devel libraries
+	dnl # Require python3-devel libraries
 	dnl #
 	AS_IF([test "x$enable_pyzfs" = xcheck  -o "x$enable_pyzfs" = xyes], [
 		AS_CASE([$PYTHON_VERSION],
-			[3.*], [PYTHON_REQUIRED_VERSION=">= '3.4.0'"],
-			[2.*], [PYTHON_REQUIRED_VERSION=">= '2.7.0'"],
+			[3.*], [PYTHON_REQUIRED_VERSION=">= '3.6.0'"],
 			[AC_MSG_ERROR("Python $PYTHON_VERSION unknown")]
 		)

@@ -103,6 +103,57 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_CHECK_DISK_CHANGE], [
 	])
 ])

+dnl #
+dnl # bdev_kobj() is introduced from 5.12
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_KOBJ], [
+	ZFS_LINUX_TEST_SRC([bdev_kobj], [
+		#include <linux/fs.h>
+		#include <linux/blkdev.h>
+		#include <linux/kobject.h>
+	], [
+		struct block_device *bdev = NULL;
+		struct kobject *disk_kobj;
+		disk_kobj = bdev_kobj(bdev);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_BDEV_KOBJ], [
+	AC_MSG_CHECKING([whether bdev_kobj() exists])
+	ZFS_LINUX_TEST_RESULT([bdev_kobj], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_BDEV_KOBJ, 1,
+		    [bdev_kobj() exists])
+	], [
+		AC_MSG_RESULT(no)
+	])
+])
+
+dnl #
+dnl # part_to_dev() was removed in 5.12
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV_PART_TO_DEV], [
+	ZFS_LINUX_TEST_SRC([part_to_dev], [
+		#include <linux/fs.h>
+		#include <linux/blkdev.h>
+	], [
+		struct hd_struct *p = NULL;
+		struct device *pdev;
+		pdev = part_to_dev(p);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_BLKDEV_PART_TO_DEV], [
+	AC_MSG_CHECKING([whether part_to_dev() exists])
+	ZFS_LINUX_TEST_RESULT([part_to_dev], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_PART_TO_DEV, 1,
+		    [part_to_dev() exists])
+	], [
+		AC_MSG_RESULT(no)
+	])
+])
+
 dnl #
 dnl # 5.10 API, check_disk_change() is removed, in favor of
 dnl # bdev_check_media_change(), which doesn't force revalidation
@@ -405,6 +456,8 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLKDEV], [
 	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_WHOLE
 	ZFS_AC_KERNEL_SRC_BLKDEV_BDEVNAME
 	ZFS_AC_KERNEL_SRC_BLKDEV_ISSUE_SECURE_ERASE
+	ZFS_AC_KERNEL_SRC_BLKDEV_BDEV_KOBJ
+	ZFS_AC_KERNEL_SRC_BLKDEV_PART_TO_DEV
 ])

 AC_DEFUN([ZFS_AC_KERNEL_BLKDEV], [
@@ -421,4 +474,6 @@ AC_DEFUN([ZFS_AC_KERNEL_BLKDEV], [
 	ZFS_AC_KERNEL_BLKDEV_BDEVNAME
 	ZFS_AC_KERNEL_BLKDEV_GET_ERESTARTSYS
 	ZFS_AC_KERNEL_BLKDEV_ISSUE_SECURE_ERASE
+	ZFS_AC_KERNEL_BLKDEV_BDEV_KOBJ
+	ZFS_AC_KERNEL_BLKDEV_PART_TO_DEV
 ])
@@ -2,7 +2,20 @@ dnl #
 dnl # Check for generic io accounting interface.
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_GENERIC_IO_ACCT], [
-	ZFS_LINUX_TEST_SRC([bdev_io_acct], [
+	ZFS_LINUX_TEST_SRC([bdev_io_acct_63], [
+		#include <linux/blkdev.h>
+	], [
+		struct block_device *bdev = NULL;
+		struct bio *bio = NULL;
+		unsigned long passed_time = 0;
+		unsigned long start_time;
+
+		start_time = bdev_start_io_acct(bdev, bio_op(bio),
+		    passed_time);
+		bdev_end_io_acct(bdev, bio_op(bio), bio_sectors(bio), start_time);
+	])
+
+	ZFS_LINUX_TEST_SRC([bdev_io_acct_old], [
 		#include <linux/blkdev.h>
 	], [
 		struct block_device *bdev = NULL;
@@ -63,74 +76,85 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_GENERIC_IO_ACCT], [

 AC_DEFUN([ZFS_AC_KERNEL_GENERIC_IO_ACCT], [
 	dnl #
-	dnl # 5.19 API,
+	dnl # Linux 6.3, and then backports thereof, changed
+	dnl # the signatures on bdev_start_io_acct/bdev_end_io_acct
 	dnl #
-	dnl # disk_start_io_acct() and disk_end_io_acct() have been replaced by
-	dnl # bdev_start_io_acct() and bdev_end_io_acct().
-	dnl #
-	AC_MSG_CHECKING([whether generic bdev_*_io_acct() are available])
-	ZFS_LINUX_TEST_RESULT([bdev_io_acct], [
+	AC_MSG_CHECKING([whether 6.3+ bdev_*_io_acct() are available])
+	ZFS_LINUX_TEST_RESULT([bdev_io_acct_63], [
 		AC_MSG_RESULT(yes)
-		AC_DEFINE(HAVE_BDEV_IO_ACCT, 1, [bdev_*_io_acct() available])
+		AC_DEFINE(HAVE_BDEV_IO_ACCT_63, 1, [bdev_*_io_acct() available])
 	], [
 		AC_MSG_RESULT(no)

 		dnl #
-		dnl # 5.12 API,
+		dnl # 5.19 API,
 		dnl #
-		dnl # bio_start_io_acct() and bio_end_io_acct() became GPL-exported
-		dnl # so use disk_start_io_acct() and disk_end_io_acct() instead
+		dnl # disk_start_io_acct() and disk_end_io_acct() have been replaced by
+		dnl # bdev_start_io_acct() and bdev_end_io_acct().
 		dnl #
-		AC_MSG_CHECKING([whether generic disk_*_io_acct() are available])
-		ZFS_LINUX_TEST_RESULT([disk_io_acct], [
+		AC_MSG_CHECKING([whether pre-6.3 bdev_*_io_acct() are available])
+		ZFS_LINUX_TEST_RESULT([bdev_io_acct_old], [
 			AC_MSG_RESULT(yes)
-			AC_DEFINE(HAVE_DISK_IO_ACCT, 1, [disk_*_io_acct() available])
+			AC_DEFINE(HAVE_BDEV_IO_ACCT_OLD, 1, [bdev_*_io_acct() available])
 		], [
 			AC_MSG_RESULT(no)
-
 			dnl #
-			dnl # 5.7 API,
+			dnl # 5.12 API,
 			dnl #
-			dnl # Added bio_start_io_acct() and bio_end_io_acct() helpers.
+			dnl # bio_start_io_acct() and bio_end_io_acct() became GPL-exported
+			dnl # so use disk_start_io_acct() and disk_end_io_acct() instead
 			dnl #
-			AC_MSG_CHECKING([whether generic bio_*_io_acct() are available])
-			ZFS_LINUX_TEST_RESULT([bio_io_acct], [
+			AC_MSG_CHECKING([whether generic disk_*_io_acct() are available])
+			ZFS_LINUX_TEST_RESULT([disk_io_acct], [
 				AC_MSG_RESULT(yes)
-				AC_DEFINE(HAVE_BIO_IO_ACCT, 1, [bio_*_io_acct() available])
+				AC_DEFINE(HAVE_DISK_IO_ACCT, 1, [disk_*_io_acct() available])
 			], [
 				AC_MSG_RESULT(no)

 				dnl #
-				dnl # 4.14 API,
+				dnl # 5.7 API,
 				dnl #
-				dnl # generic_start_io_acct/generic_end_io_acct now require
-				dnl # request_queue to be provided. No functional changes,
-				dnl # but preparation for inflight accounting.
+				dnl # Added bio_start_io_acct() and bio_end_io_acct() helpers.
 				dnl #
-				AC_MSG_CHECKING([whether generic_*_io_acct wants 4 args])
-				ZFS_LINUX_TEST_RESULT_SYMBOL([generic_acct_4args],
-				    [generic_start_io_acct], [block/bio.c], [
+				AC_MSG_CHECKING([whether generic bio_*_io_acct() are available])
+				ZFS_LINUX_TEST_RESULT([bio_io_acct], [
 					AC_MSG_RESULT(yes)
-					AC_DEFINE(HAVE_GENERIC_IO_ACCT_4ARG, 1,
-					    [generic_*_io_acct() 4 arg available])
+					AC_DEFINE(HAVE_BIO_IO_ACCT, 1, [bio_*_io_acct() available])
 				], [
 					AC_MSG_RESULT(no)

 					dnl #
-					dnl # 3.19 API addition
+					dnl # 4.14 API,
 					dnl #
-					dnl # torvalds/linux@394ffa50 allows us to increment
-					dnl # iostat counters without generic_make_request().
+					dnl # generic_start_io_acct/generic_end_io_acct now require
+					dnl # request_queue to be provided. No functional changes,
+					dnl # but preparation for inflight accounting.
 					dnl #
-					AC_MSG_CHECKING(
-					    [whether generic_*_io_acct wants 3 args])
-					ZFS_LINUX_TEST_RESULT_SYMBOL([generic_acct_3args],
+					AC_MSG_CHECKING([whether generic_*_io_acct wants 4 args])
+					ZFS_LINUX_TEST_RESULT_SYMBOL([generic_acct_4args],
 					    [generic_start_io_acct], [block/bio.c], [
 						AC_MSG_RESULT(yes)
-						AC_DEFINE(HAVE_GENERIC_IO_ACCT_3ARG, 1,
-						    [generic_*_io_acct() 3 arg available])
+						AC_DEFINE(HAVE_GENERIC_IO_ACCT_4ARG, 1,
+						    [generic_*_io_acct() 4 arg available])
 					], [
 						AC_MSG_RESULT(no)
+
+						dnl #
+						dnl # 3.19 API addition
+						dnl #
+						dnl # torvalds/linux@394ffa50 allows us to increment
+						dnl # iostat counters without generic_make_request().
+						dnl #
+						AC_MSG_CHECKING(
+						    [whether generic_*_io_acct wants 3 args])
+						ZFS_LINUX_TEST_RESULT_SYMBOL([generic_acct_3args],
+						    [generic_start_io_acct], [block/bio.c], [
+							AC_MSG_RESULT(yes)
+							AC_DEFINE(HAVE_GENERIC_IO_ACCT_3ARG, 1,
+							    [generic_*_io_acct() 3 arg available])
+						], [
+							AC_MSG_RESULT(no)
+						])
 					])
 				])
 			])
@@ -518,6 +518,8 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [
 			VENDOR=alpine ;
 		elif test -f /bin/freebsd-version ; then
 			VENDOR=freebsd ;
+		elif test -f /etc/openEuler-release ; then
+			VENDOR=openeuler ;
 		else
 			VENDOR= ;
 		fi],
@@ -542,6 +544,7 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [
 		ubuntu)     DEFAULT_PACKAGE=deb  ;;
 		debian)     DEFAULT_PACKAGE=deb  ;;
 		freebsd)    DEFAULT_PACKAGE=pkg  ;;
+		openeuler)  DEFAULT_PACKAGE=rpm  ;;
 		*)          DEFAULT_PACKAGE=rpm  ;;
 	esac
 	AC_MSG_RESULT([$DEFAULT_PACKAGE])
@@ -555,35 +558,14 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [
 	AC_MSG_RESULT([$initdir])
 	AC_SUBST(initdir)

-	AC_MSG_CHECKING([default init script type and shell])
+	AC_MSG_CHECKING([default shell])
 	case "$VENDOR" in
-		toss)       DEFAULT_INIT_SCRIPT=redhat ;;
-		redhat)     DEFAULT_INIT_SCRIPT=redhat ;;
-		fedora)     DEFAULT_INIT_SCRIPT=fedora ;;
-		gentoo)     DEFAULT_INIT_SCRIPT=openrc ;;
-		alpine)     DEFAULT_INIT_SCRIPT=openrc ;;
-		arch)       DEFAULT_INIT_SCRIPT=lsb    ;;
-		sles)       DEFAULT_INIT_SCRIPT=lsb    ;;
-		slackware)  DEFAULT_INIT_SCRIPT=lsb    ;;
-		lunar)      DEFAULT_INIT_SCRIPT=lunar  ;;
-		ubuntu)     DEFAULT_INIT_SCRIPT=lsb    ;;
-		debian)     DEFAULT_INIT_SCRIPT=lsb    ;;
-		freebsd)    DEFAULT_INIT_SCRIPT=freebsd;;
-		*)          DEFAULT_INIT_SCRIPT=lsb    ;;
+		gentoo)     DEFAULT_INIT_SHELL="/sbin/openrc-run";;
+		alpine)     DEFAULT_INIT_SHELL="/sbin/openrc-run";;
+		*)          DEFAULT_INIT_SHELL="/bin/sh"         ;;
 	esac

-	# On gentoo, it's possible that OpenRC isn't installed.  Check if
-	# /sbin/openrc-run exists, and if not, fall back to generic defaults.
-
-	DEFAULT_INIT_SHELL="/bin/sh"
-	AS_IF([test "$DEFAULT_INIT_SCRIPT" = "openrc"], [
-		AS_IF([test -x "/sbin/openrc-run"],
-			[DEFAULT_INIT_SHELL="/sbin/openrc-run"],
-			[DEFAULT_INIT_SCRIPT=lsb])
-	])
-
-	AC_MSG_RESULT([$DEFAULT_INIT_SCRIPT:$DEFAULT_INIT_SHELL])
-	AC_SUBST(DEFAULT_INIT_SCRIPT)
+	AC_MSG_RESULT([$DEFAULT_INIT_SHELL])
 	AC_SUBST(DEFAULT_INIT_SHELL)

 	AC_MSG_CHECKING([default nfs server init script])
@@ -602,6 +584,7 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [
 		redhat)     initconfdir=/etc/sysconfig ;;
 		fedora)     initconfdir=/etc/sysconfig ;;
 		sles)       initconfdir=/etc/sysconfig ;;
+		openeuler)  initconfdir=/etc/sysconfig ;;
 		ubuntu)     initconfdir=/etc/default   ;;
 		debian)     initconfdir=/etc/default   ;;
 		freebsd)    initconfdir=$sysconfdir/rc.conf.d;;
@@ -15,6 +15,7 @@ pkgdracut_SCRIPTS = \

 pkgdracut_DATA = \
 	zfs-env-bootfs.service \
+	zfs-nonroot-necessities.service \
 	zfs-snapshot-bootfs.service \
 	zfs-rollback-bootfs.service

@@ -84,6 +84,9 @@ install() {
 		inst_simple "${moddir}/zfs-env-bootfs.service" "${systemdsystemunitdir}/zfs-env-bootfs.service"
 		systemctl -q --root "${initdir}" add-wants zfs-import.target zfs-env-bootfs.service

+		inst_simple "${moddir}/zfs-nonroot-necessities.service" "${systemdsystemunitdir}/zfs-nonroot-necessities.service"
+		systemctl -q --root "${initdir}" add-requires initrd-root-fs.target zfs-nonroot-necessities.service
+
 		for _service in \
 			"zfs-import-scan.service" \
 			"zfs-import-cache.service"; do
@@ -1,6 +1,5 @@
 [Unit]
-Description=Set BOOTFS environment for dracut
-Documentation=man:zpool(8)
+Description=Set BOOTFS and BOOTFSFLAGS environment variables for dracut
 DefaultDependencies=no
 After=zfs-import-cache.service
 After=zfs-import-scan.service
@@ -8,7 +7,17 @@ Before=zfs-import.target

 [Service]
 Type=oneshot
-ExecStart=/bin/sh -c "exec systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -vFx -)"
+ExecStart=/bin/sh -c '                                                                         \
+    . /lib/dracut-zfs-lib.sh;                                                                  \
+    decode_root_args || exit 0;                                                                \
+    [ "$root" = "zfs:AUTO" ] && root="$(@sbindir@/zpool list -H -o bootfs | grep -m1 -vFx -)"; \
+    rootflags="$(getarg rootflags=)";                                                          \
+    case ",$rootflags," in                                                                     \
+        *,zfsutil,*) ;;                                                                        \
+        ,,) rootflags=zfsutil ;;                                                               \
+        *)  rootflags="zfsutil,$rootflags" ;;                                                  \
+    esac;                                                                                      \
+    exec systemctl set-environment BOOTFS="$root" BOOTFSFLAGS="$rootflags"'

 [Install]
 WantedBy=zfs-import.target
@@ -14,81 +14,24 @@ GENERATOR_DIR="$1"
 . /lib/dracut-zfs-lib.sh
 decode_root_args || exit 0

-[ -z "${rootflags}" ] && rootflags=$(getarg rootflags=)
-case ",${rootflags}," in
-	*,zfsutil,*) ;;
-	,,)	rootflags=zfsutil ;;
-	*)	rootflags="zfsutil,${rootflags}" ;;
-esac
-
 [ -n "$debug" ] && echo "zfs-generator: writing extension for sysroot.mount to $GENERATOR_DIR/sysroot.mount.d/zfs-enhancement.conf" >> /dev/kmsg


-mkdir -p "$GENERATOR_DIR"/sysroot.mount.d "$GENERATOR_DIR"/initrd-root-fs.target.requires "$GENERATOR_DIR"/dracut-pre-mount.service.d
+mkdir -p "$GENERATOR_DIR"/sysroot.mount.d "$GENERATOR_DIR"/dracut-pre-mount.service.d
+
 {
    echo "[Unit]"
    echo "Before=initrd-root-fs.target"
    echo "After=zfs-import.target"
    echo
    echo "[Mount]"
-    if [ "${root}" = "zfs:AUTO" ]; then
-      echo "PassEnvironment=BOOTFS"
-      echo 'What=${BOOTFS}'
-    else
-      echo "What=${root}"
-    fi
+    echo "PassEnvironment=BOOTFS BOOTFSFLAGS"
+    echo 'What=${BOOTFS}'
    echo "Type=zfs"
-    echo "Options=${rootflags}"
+    echo 'Options=${BOOTFSFLAGS}'
 } > "$GENERATOR_DIR"/sysroot.mount.d/zfs-enhancement.conf
 ln -fs ../sysroot.mount "$GENERATOR_DIR"/initrd-root-fs.target.requires/sysroot.mount

-
-if [ "${root}" = "zfs:AUTO" ]; then
-  {
-      echo "[Unit]"
-      echo "Before=initrd-root-fs.target"
-      echo "After=sysroot.mount"
-      echo "DefaultDependencies=no"
-      echo
-      echo "[Service]"
-      echo "Type=oneshot"
-      echo "PassEnvironment=BOOTFS"
-      echo "ExecStart=/bin/sh -c '" '                                        \
-        . /lib/dracut-zfs-lib.sh;                                            \
-        _zfs_nonroot_necessities_cb() {                                      \
-            zfs mount | grep -m1 -q "^$1 " && return 0;                      \
-            echo "Mounting $1 on /sysroot$2";                                \
-            mount -o zfsutil -t zfs "$1" "/sysroot$2";                       \
-        };                                                                   \
-        for_relevant_root_children "${BOOTFS}" _zfs_nonroot_necessities_cb;' \
-      "'"
-  } > "$GENERATOR_DIR"/zfs-nonroot-necessities.service
-  ln -fs ../zfs-nonroot-necessities.service "$GENERATOR_DIR"/initrd-root-fs.target.requires/zfs-nonroot-necessities.service
-else
-  # We can solve this statically at generation time, so do!
-  _zfs_generator_cb() {
-      dset="${1}"
-      mpnt="${2}"
-      unit="$(systemd-escape --suffix=mount -p "/sysroot${mpnt}")"
-
-      {
-          echo "[Unit]"
-          echo "Before=initrd-root-fs.target"
-          echo "After=sysroot.mount"
-          echo
-          echo "[Mount]"
-          echo "Where=/sysroot${mpnt}"
-          echo "What=${dset}"
-          echo "Type=zfs"
-          echo "Options=zfsutil"
-      } > "$GENERATOR_DIR/${unit}"
-      ln -fs ../"${unit}" "$GENERATOR_DIR"/initrd-root-fs.target.requires/"${unit}"
-  }
-
-  for_relevant_root_children "${root}" _zfs_generator_cb
-fi
-
-
 {
    echo "[Unit]"
    echo "After=zfs-import.target"
@@ -38,7 +38,7 @@ mount_dataset() {

 # for_relevant_root_children DATASET EXEC
 #   Runs "EXEC dataset mountpoint" for all children of DATASET that are needed for system bringup
-#   Used by zfs-generator.sh and friends, too!
+#   Used by zfs-nonroot-necessities.service and friends, too!
 for_relevant_root_children() {
    dataset="${1}"
    exec="${2}"
@@ -0,0 +1,20 @@
+[Unit]
+Before=initrd-root-fs.target
+After=sysroot.mount
+DefaultDependencies=no
+ConditionEnvironment=BOOTFS
+
+[Service]
+Type=oneshot
+PassEnvironment=BOOTFS
+ExecStart=/bin/sh -c '                                                \
+    . /lib/dracut-zfs-lib.sh;                                         \
+    _zfs_nonroot_necessities_cb() {                                   \
+        @sbindir@/zfs mount | grep -m1 -q "^$1 " && return 0;         \
+        echo "Mounting $1 on /sysroot$2";                             \
+        mount -o zfsutil -t zfs "$1" "/sysroot$2";                    \
+    };                                                                \
+    for_relevant_root_children "${BOOTFS}" _zfs_nonroot_necessities_cb'
+
+[Install]
+RequiredBy=initrd-root-fs.target
@@ -5,8 +5,9 @@ After=zfs-import.target dracut-pre-mount.service zfs-snapshot-bootfs.service
 Before=dracut-mount.service
 DefaultDependencies=no
 ConditionKernelCommandLine=bootfs.rollback
+ConditionEnvironment=BOOTFS

 [Service]
 Type=oneshot
-ExecStart=/bin/sh -c '. /lib/dracut-zfs-lib.sh; decode_root_args || exit; [ "$root" = "zfs:AUTO" ] && root="$BOOTFS"; SNAPNAME="$(getarg bootfs.rollback)"; exec @sbindir@/zfs rollback -Rf "$root@${SNAPNAME:-%v}"'
+ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; exec @sbindir@/zfs rollback -Rf "$BOOTFS@${SNAPNAME:-%v}"'
 RemainAfterExit=yes
@@ -5,8 +5,9 @@ After=zfs-import.target dracut-pre-mount.service
 Before=dracut-mount.service
 DefaultDependencies=no
 ConditionKernelCommandLine=bootfs.snapshot
+ConditionEnvironment=BOOTFS

 [Service]
 Type=oneshot
-ExecStart=-/bin/sh -c '. /lib/dracut-zfs-lib.sh; decode_root_args || exit; [ "$root" = "zfs:AUTO" ] && root="$BOOTFS"; SNAPNAME="$(getarg bootfs.snapshot)"; exec @sbindir@/zfs snapshot "$root@${SNAPNAME:-%v}"'
+ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; exec @sbindir@/zfs snapshot "$BOOTFS@${SNAPNAME:-%v}"'
 RemainAfterExit=yes
@@ -7,5 +7,6 @@ dist_scripts_SCRIPTS = \

 SUBDIRS = local-top

+SHELLCHECK_IGNORE = ,SC2295
 SHELLCHECKDIRS = $(SUBDIRS)
 SHELLCHECK_SHELL = sh
@@ -192,7 +192,7 @@ import_pool()

 	# Verify that the pool isn't already imported
 	# Make as sure as we can to not require '-f' to import.
-	"${ZPOOL}" get name,guid -o value -H 2>/dev/null | grep -Fxq "$pool" && return 0
+	"${ZPOOL}" get -H -o value name,guid 2>/dev/null | grep -Fxq "$pool" && return 0

 	# For backwards compatibility, make sure that ZPOOL_IMPORT_PATH is set
 	# to something we can use later with the real import(s). We want to
@@ -341,7 +341,10 @@ mount_fs()
 				# isn't the root fs.
 				return 0
 			fi
-			ZFS_CMD="mount.zfs"
+			# Don't use mount.zfs -o zfsutils for legacy mountpoint
+			if [ "$mountpoint" = "legacy" ]; then
+				ZFS_CMD="mount.zfs"
+			fi
 			# Last hail-mary: Hope 'rootmnt' is set!
 			mountpoint=""
 		else
@@ -880,12 +883,12 @@ mountroot()
 	fi

 	# In case the pool was specified as guid, resolve guid to name
-	pool="$("${ZPOOL}" get name,guid -o name,value -H | \
+	pool="$("${ZPOOL}" get -H -o name,value name,guid | \
 	    awk -v pool="${ZFS_RPOOL}" '$2 == pool { print $1 }')"
 	if [ -n "$pool" ]; then
 		# If $ZFS_BOOTFS contains guid, replace the guid portion with $pool
 		ZFS_BOOTFS=$(echo "$ZFS_BOOTFS" | \
-			sed -e "s/$("${ZPOOL}" get guid -o value "$pool" -H)/$pool/g")
+			sed -e "s/$("${ZPOOL}" get -H -o value guid "$pool")/$pool/g")
 		ZFS_RPOOL="${pool}"
 	fi

@@ -29,13 +29,12 @@ setup(
        "Development Status :: 4 - Beta",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.4",
-        "Programming Language :: Python :: 3.5",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
        "Topic :: System :: Filesystems",
        "Topic :: Software Development :: Libraries",
    ],
@@ -53,7 +52,7 @@ setup(
    setup_requires=[
        "cffi",
    ],
-    python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,<4',
+    python_requires='>=3.6,<4',
    zip_safe=False,
    test_suite="libzfs_core.test",
 )
@@ -307,6 +307,7 @@ extern int zpool_vdev_remove_cancel(zpool_handle_t *);
 extern int zpool_vdev_indirect_size(zpool_handle_t *, const char *, uint64_t *);
 extern int zpool_vdev_split(zpool_handle_t *, char *, nvlist_t **, nvlist_t *,
    splitflags_t);
+_LIBZFS_H int zpool_vdev_remove_wanted(zpool_handle_t *, const char *);

 extern int zpool_vdev_fault(zpool_handle_t *, uint64_t, vdev_aux_t);
 extern int zpool_vdev_degrade(zpool_handle_t *, uint64_t, vdev_aux_t);
@@ -692,6 +693,9 @@ typedef struct sendflags {
 	/* show progress (ie. -v) */
 	boolean_t progress;

+	/* show progress as process title (ie. -V) */
+	boolean_t progressastitle;
+
 	/* large blocks (>128K) are permitted */
 	boolean_t largeblock;

@@ -150,17 +150,33 @@ int zfs_ioctl_fd(int fd, unsigned long request, struct zfs_cmd *zc);
 /*
 * List of colors to use
 */
+#define	ANSI_BLACK	"\033[0;30m"
 #define	ANSI_RED	"\033[0;31m"
 #define	ANSI_GREEN	"\033[0;32m"
 #define	ANSI_YELLOW	"\033[0;33m"
 #define	ANSI_BLUE	"\033[0;34m"
+#define	ANSI_BOLD_BLUE	"\033[1;34m" /* light blue */
+#define	ANSI_MAGENTA	"\033[0;35m"
+#define	ANSI_CYAN	"\033[0;36m"
+#define	ANSI_GRAY	"\033[0;37m"
+
 #define	ANSI_RESET	"\033[0m"
 #define	ANSI_BOLD	"\033[1m"

+int use_color(void);
 void color_start(const char *color);
 void color_end(void);
 int printf_color(const char *color, char *format, ...);

+#ifdef __linux__
+extern char **environ;
+_LIBZUTIL_H void zfs_setproctitle_init(int argc, char *argv[], char *envp[]);
+_LIBZUTIL_H void zfs_setproctitle(const char *fmt, ...);
+#else
+#define	zfs_setproctitle(fmt, ...)	setproctitle(fmt, ##__VA_ARGS__)
+#define	zfs_setproctitle_init(x, y, z)	((void)0)
+#endif
+
 /*
 * These functions are used by the ZFS libraries and cmd/zpool code, but are
 * not exported in the ABI.
@@ -256,6 +256,32 @@ bio_set_bi_error(struct bio *bio, int error)
 #define	BIO_END_IO(bio, error)		bio_endio(bio, error);
 #endif /* HAVE_1ARG_BIO_END_IO_T */

+/*
+ * 5.15 MACRO,
+ *   GD_DEAD
+ *
+ * 2.6.36 - 5.14 MACRO,
+ *   GENHD_FL_UP
+ *
+ * Check the disk status and return B_TRUE if alive
+ * otherwise B_FALSE
+ */
+static inline boolean_t
+zfs_check_disk_status(struct block_device *bdev)
+{
+#if defined(GENHD_FL_UP)
+	return (!!(bdev->bd_disk->flags & GENHD_FL_UP));
+#elif defined(GD_DEAD)
+	return (!test_bit(GD_DEAD, &bdev->bd_disk->state));
+#else
+/*
+ * This is encountered if neither GENHD_FL_UP nor GD_DEAD is available in
+ * the kernel - likely due to an MACRO change that needs to be chased down.
+ */
+#error "Unsupported kernel: no usable disk status check"
+#endif
+}
+
 /*
 * 4.1 API,
 * 3.10.0 CentOS 7.x API,
@@ -389,7 +415,7 @@ static inline void
 bio_set_flush(struct bio *bio)
 {
 #if defined(HAVE_REQ_PREFLUSH)	/* >= 4.10 */
-	bio_set_op_attrs(bio, 0, REQ_PREFLUSH);
+	bio_set_op_attrs(bio, 0, REQ_PREFLUSH | REQ_OP_WRITE);
 #elif defined(WRITE_FLUSH_FUA)	/* >= 2.6.37 and <= 4.9 */
 	bio_set_op_attrs(bio, 0, WRITE_FLUSH_FUA);
 #else
@@ -555,7 +581,10 @@ blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)),
    struct gendisk *disk __attribute__((unused)),
    int rw __attribute__((unused)), struct bio *bio)
 {
-#if defined(HAVE_BDEV_IO_ACCT)
+#if defined(HAVE_BDEV_IO_ACCT_63)
+	return (bdev_start_io_acct(bio->bi_bdev, bio_op(bio),
+	    jiffies));
+#elif defined(HAVE_BDEV_IO_ACCT_OLD)
 	return (bdev_start_io_acct(bio->bi_bdev, bio_sectors(bio),
 	    bio_op(bio), jiffies));
 #elif defined(HAVE_DISK_IO_ACCT)
@@ -581,7 +610,10 @@ blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)),
    struct gendisk *disk __attribute__((unused)),
    int rw __attribute__((unused)), struct bio *bio, unsigned long start_time)
 {
-#if defined(HAVE_BDEV_IO_ACCT)
+#if defined(HAVE_BDEV_IO_ACCT_63)
+	bdev_end_io_acct(bio->bi_bdev, bio_op(bio), bio_sectors(bio),
+	    start_time);
+#elif defined(HAVE_BDEV_IO_ACCT_OLD)
 	bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time);
 #elif defined(HAVE_DISK_IO_ACCT)
 	disk_end_io_acct(disk, bio_op(bio), start_time);
@@ -20,6 +20,7 @@ KERNEL_H = \
 	kmem.h \
 	kstat.h \
 	list.h \
+	misc.h \
 	mod_os.h \
 	mutex.h \
 	param.h \
@@ -0,0 +1,29 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#ifndef _OS_LINUX_SPL_MISC_H
+#define	_OS_LINUX_SPL_MISC_H
+
+#include <linux/kobject.h>
+
+extern void spl_signal_kobj_evt(struct block_device *bdev);
+
+#endif
@@ -30,6 +30,7 @@
 #define	_SYS_ARC_IMPL_H

 #include <sys/arc.h>
+#include <sys/multilist.h>
 #include <sys/zio_crypt.h>
 #include <sys/zthr.h>
 #include <sys/aggsum.h>
@@ -87,6 +87,7 @@ int livelist_bpobj_iterate_from_nofree(bpobj_t *bpo, bpobj_itor_t func,
    void *arg, int64_t start);

 void bpobj_enqueue_subobj(bpobj_t *bpo, uint64_t subobj, dmu_tx_t *tx);
+void bpobj_prefetch_subobj(bpobj_t *bpo, uint64_t subobj);
 void bpobj_enqueue(bpobj_t *bpo, const blkptr_t *bp, boolean_t bp_freed,
    dmu_tx_t *tx);

@@ -65,7 +65,7 @@ extern "C" {
 * them, and increased memory overhead. Increasing these values results in
 * higher variance in operation time, and reduces memory overhead.
 */
-#define	BTREE_CORE_ELEMS	128
+#define	BTREE_CORE_ELEMS	126
 #define	BTREE_LEAF_SIZE		4096

 extern kmem_cache_t *zfs_btree_leaf_cache;
@@ -95,9 +95,6 @@ typedef struct zfs_btree_leaf {
 	uint8_t		btl_elems[];
 } zfs_btree_leaf_t;

-#define	BTREE_LEAF_ESIZE	(BTREE_LEAF_SIZE - \
-    offsetof(zfs_btree_leaf_t, btl_elems))
-
 typedef struct zfs_btree_index {
 	zfs_btree_hdr_t	*bti_node;
 	uint32_t	bti_offset;
@@ -109,14 +106,15 @@ typedef struct zfs_btree_index {
 } zfs_btree_index_t;

 typedef struct btree {
-	zfs_btree_hdr_t		*bt_root;
-	int64_t			bt_height;
+	int (*bt_compar) (const void *, const void *);
 	size_t			bt_elem_size;
+	size_t			bt_leaf_size;
 	uint32_t		bt_leaf_cap;
+	int32_t			bt_height;
 	uint64_t		bt_num_elems;
 	uint64_t		bt_num_nodes;
+	zfs_btree_hdr_t		*bt_root;
 	zfs_btree_leaf_t	*bt_bulk; // non-null if bulk loading
-	int (*bt_compar) (const void *, const void *);
 } zfs_btree_t;

 /*
@@ -132,9 +130,12 @@ void zfs_btree_fini(void);
 * compar - function to compare two nodes, it must return exactly: -1, 0, or +1
 *          -1 for <, 0 for ==, and +1 for >
 * size   - the value of sizeof(struct my_type)
+ * lsize  - custom leaf size
 */
 void zfs_btree_create(zfs_btree_t *, int (*) (const void *, const void *),
    size_t);
+void zfs_btree_create_custom(zfs_btree_t *, int (*)(const void *, const void *),
+    size_t, size_t);

 /*
 * Find a node with a matching value in the tree. Returns the matching node
@@ -36,6 +36,7 @@
 #include <sys/dmu_zfetch.h>
 #include <sys/zrlock.h>
 #include <sys/multilist.h>
+#include <sys/wmsum.h>

 #ifdef	__cplusplus
 extern "C" {
@@ -587,10 +588,42 @@ typedef struct dnode_stats {
 	kstat_named_t dnode_move_active;
 } dnode_stats_t;

+typedef struct dnode_sums {
+	wmsum_t dnode_hold_dbuf_hold;
+	wmsum_t dnode_hold_dbuf_read;
+	wmsum_t dnode_hold_alloc_hits;
+	wmsum_t dnode_hold_alloc_misses;
+	wmsum_t dnode_hold_alloc_interior;
+	wmsum_t dnode_hold_alloc_lock_retry;
+	wmsum_t dnode_hold_alloc_lock_misses;
+	wmsum_t dnode_hold_alloc_type_none;
+	wmsum_t dnode_hold_free_hits;
+	wmsum_t dnode_hold_free_misses;
+	wmsum_t dnode_hold_free_lock_misses;
+	wmsum_t dnode_hold_free_lock_retry;
+	wmsum_t dnode_hold_free_refcount;
+	wmsum_t dnode_hold_free_overflow;
+	wmsum_t dnode_free_interior_lock_retry;
+	wmsum_t dnode_allocate;
+	wmsum_t dnode_reallocate;
+	wmsum_t dnode_buf_evict;
+	wmsum_t dnode_alloc_next_chunk;
+	wmsum_t dnode_alloc_race;
+	wmsum_t dnode_alloc_next_block;
+	wmsum_t dnode_move_invalid;
+	wmsum_t dnode_move_recheck1;
+	wmsum_t dnode_move_recheck2;
+	wmsum_t dnode_move_special;
+	wmsum_t dnode_move_handle;
+	wmsum_t dnode_move_rwlock;
+	wmsum_t dnode_move_active;
+} dnode_sums_t;
+
 extern dnode_stats_t dnode_stats;
+extern dnode_sums_t dnode_sums;

 #define	DNODE_STAT_INCR(stat, val) \
-    atomic_add_64(&dnode_stats.stat.value.ui64, (val));
+    wmsum_add(&dnode_sums.stat, (val))
 #define	DNODE_STAT_BUMP(stat) \
    DNODE_STAT_INCR(stat, 1);

@@ -1518,6 +1518,7 @@ typedef enum {
 #define	ZFS_ONLINE_UNSPARE	0x2
 #define	ZFS_ONLINE_FORCEFAULT	0x4
 #define	ZFS_ONLINE_EXPAND	0x8
+#define	ZFS_ONLINE_SPARE	0x10
 #define	ZFS_OFFLINE_TEMPORARY	0x1

 /*
@@ -837,7 +837,7 @@ extern kmutex_t spa_namespace_lock;
 #define	SPA_CONFIG_UPDATE_POOL	0
 #define	SPA_CONFIG_UPDATE_VDEVS	1

-extern void spa_write_cachefile(spa_t *, boolean_t, boolean_t);
+extern void spa_write_cachefile(spa_t *, boolean_t, boolean_t, boolean_t);
 extern void spa_config_load(void);
 extern nvlist_t *spa_all_configs(uint64_t *);
 extern void spa_config_set(spa_t *spa, nvlist_t *config);
@@ -147,6 +147,7 @@ extern int vdev_degrade(spa_t *spa, uint64_t guid, vdev_aux_t aux);
 extern int vdev_online(spa_t *spa, uint64_t guid, uint64_t flags,
    vdev_state_t *);
 extern int vdev_offline(spa_t *spa, uint64_t guid, uint64_t flags);
+extern int vdev_remove_wanted(spa_t *spa, uint64_t guid);
 extern void vdev_clear(spa_t *spa, vdev_t *vd);

 extern boolean_t vdev_is_dead(vdev_t *vd);
@@ -189,6 +190,8 @@ typedef enum vdev_config_flag {
 	VDEV_CONFIG_MISSING = 1 << 4
 } vdev_config_flag_t;

+extern void vdev_post_kobj_evt(vdev_t *vd);
+extern void vdev_clear_kobj_evt(vdev_t *vd);
 extern void vdev_top_config_generate(spa_t *spa, nvlist_t *config);
 extern nvlist_t *vdev_config_generate(spa_t *spa, vdev_t *vd,
    boolean_t getstats, vdev_config_flag_t flags);
@@ -69,6 +69,7 @@ extern uint32_t zfs_vdev_async_write_max_active;
 * Virtual device operations
 */
 typedef int	vdev_init_func_t(spa_t *spa, nvlist_t *nv, void **tsd);
+typedef void	vdev_kobj_post_evt_func_t(vdev_t *vd);
 typedef void	vdev_fini_func_t(vdev_t *vd);
 typedef int	vdev_open_func_t(vdev_t *vd, uint64_t *size, uint64_t *max_size,
    uint64_t *ashift, uint64_t *pshift);
@@ -123,6 +124,7 @@ typedef const struct vdev_ops {
 	vdev_config_generate_func_t	*vdev_op_config_generate;
 	vdev_nparity_func_t		*vdev_op_nparity;
 	vdev_ndisks_func_t		*vdev_op_ndisks;
+	vdev_kobj_post_evt_func_t	*vdev_op_kobj_evt_post;
 	char				vdev_op_type[16];
 	boolean_t			vdev_op_leaf;
 } vdev_ops_t;
@@ -435,6 +437,7 @@ struct vdev {
 	boolean_t	vdev_isl2cache;	/* was a l2cache device		*/
 	boolean_t	vdev_copy_uberblocks;  /* post expand copy uberblocks */
 	boolean_t	vdev_resilver_deferred;  /* resilver deferred */
+	boolean_t	vdev_kobj_flag; /* kobj event record */
 	vdev_queue_t	vdev_queue;	/* I/O deadline schedule queue	*/
 	vdev_cache_t	vdev_cache;	/* physical block cache		*/
 	spa_aux_vdev_t	*vdev_aux;	/* for l2cache and spares vdevs	*/
@@ -66,10 +66,9 @@ typedef struct mzap_phys {
 } mzap_phys_t;

 typedef struct mzap_ent {
-	avl_node_t mze_node;
-	int mze_chunkid;
-	uint64_t mze_hash;
-	uint32_t mze_cd; /* copy from mze_phys->mze_cd */
+	uint32_t mze_hash;
+	uint16_t mze_cd; /* copy from mze_phys->mze_cd */
+	uint16_t mze_chunkid;
 } mzap_ent_t;

 #define	MZE_PHYS(zap, mze) \
@@ -164,7 +163,7 @@ typedef struct zap {
 			int16_t zap_num_entries;
 			int16_t zap_num_chunks;
 			int16_t zap_alloc_next;
-			avl_tree_t zap_avl;
+			zfs_btree_t zap_tree;
 		} zap_micro;
 	} zap_u;
 } zap_t;
@@ -202,7 +201,7 @@ int zap_lockdir(objset_t *os, uint64_t obj, dmu_tx_t *tx,
    krw_t lti, boolean_t fatreader, boolean_t adding, void *tag, zap_t **zapp);
 void zap_unlockdir(zap_t *zap, void *tag);
 void zap_evict_sync(void *dbu);
-zap_name_t *zap_name_alloc(zap_t *zap, const char *key, matchtype_t mt);
+zap_name_t *zap_name_alloc_str(zap_t *zap, const char *key, matchtype_t mt);
 void zap_name_free(zap_name_t *zn);
 int zap_hashbits(zap_t *zap);
 uint32_t zap_maxcd(zap_t *zap);
@@ -51,6 +51,7 @@ extern "C" {
 #include <sys/kmem.h>
 #include <sys/kmem_cache.h>
 #include <sys/vmem.h>
+#include <sys/misc.h>
 #include <sys/taskq.h>
 #include <sys/param.h>
 #include <sys/disp.h>
@@ -217,9 +217,9 @@ typedef struct znode {

 typedef struct znode_hold {
 	uint64_t	zh_obj;		/* object id */
-	kmutex_t	zh_lock;	/* lock serializing object access */
 	avl_node_t	zh_node;	/* avl tree linkage */
-	zfs_refcount_t	zh_refcount;	/* active consumer reference count */
+	kmutex_t	zh_lock;	/* lock serializing object access */
+	int		zh_refcount;	/* active consumer reference count */
 } znode_hold_t;

 static inline uint64_t
@@ -34,9 +34,8 @@ extern "C" {

 typedef struct zrlock {
 	kmutex_t zr_mtx;
-	volatile int32_t zr_refcount;
 	kcondvar_t zr_cv;
-	uint16_t zr_pad;
+	volatile int32_t zr_refcount;
 #ifdef	ZFS_DEBUG
 	kthread_t *zr_owner;
 	const char *zr_caller;
@@ -99,6 +99,7 @@
    <elf-symbol name='sa_validate_shareopts' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='snapshot_namecheck' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='unshare_one' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='use_color' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zcmd_alloc_dst_nvlist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zcmd_expand_dst_nvlist' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zcmd_free_nvlists' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -378,6 +379,7 @@
    <elf-symbol name='zpool_vdev_path_to_guid' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zpool_vdev_remove' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zpool_vdev_remove_cancel' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zpool_vdev_remove_wanted' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zpool_vdev_split' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zpool_wait' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zpool_wait_status' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -5069,6 +5071,9 @@
      <parameter type-id='9cf59a50'/>
      <return type-id='48b5725f'/>
    </function-decl>
+    <function-decl name='use_color' mangled-name='use_color' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='use_color'>
+      <return type-id='95e97e5e'/>
+    </function-decl>
    <function-decl name='mkdirp' visibility='default' binding='global' size-in-bits='64'>
      <parameter type-id='80f4b756'/>
      <parameter type-id='d50d396c'/>
@@ -5858,6 +5863,11 @@
      <parameter type-id='c19b74c3' name='istmp'/>
      <return type-id='95e97e5e'/>
    </function-decl>
+    <function-decl name='zpool_vdev_remove_wanted' mangled-name='zpool_vdev_remove_wanted' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_remove_wanted'>
+      <parameter type-id='4c81de99' name='zhp'/>
+      <parameter type-id='80f4b756' name='path'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
    <function-decl name='zpool_vdev_fault' mangled-name='zpool_vdev_fault' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_vdev_fault'>
      <parameter type-id='4c81de99' name='zhp'/>
      <parameter type-id='9c313c2d' name='guid'/>
@@ -6098,7 +6108,7 @@
    <array-type-def dimensions='1' type-id='b96825af' size-in-bits='64' id='13339fda'>
      <subrange length='8' type-id='7359adad' id='56e0c0b1'/>
    </array-type-def>
-    <class-decl name='sendflags' size-in-bits='544' is-struct='yes' visibility='default' id='f6aa15be'>
+    <class-decl name='sendflags' size-in-bits='576' is-struct='yes' visibility='default' id='f6aa15be'>
      <data-member access='public' layout-offset-in-bits='0'>
        <var-decl name='verbosity' type-id='95e97e5e' visibility='default'/>
      </data-member>
@@ -6130,24 +6140,27 @@
        <var-decl name='progress' type-id='c19b74c3' visibility='default'/>
      </data-member>
      <data-member access='public' layout-offset-in-bits='320'>
-        <var-decl name='largeblock' type-id='c19b74c3' visibility='default'/>
+        <var-decl name='progressastitle' type-id='c19b74c3' visibility='default'/>
      </data-member>
      <data-member access='public' layout-offset-in-bits='352'>
-        <var-decl name='embed_data' type-id='c19b74c3' visibility='default'/>
+        <var-decl name='largeblock' type-id='c19b74c3' visibility='default'/>
      </data-member>
      <data-member access='public' layout-offset-in-bits='384'>
-        <var-decl name='compress' type-id='c19b74c3' visibility='default'/>
+        <var-decl name='embed_data' type-id='c19b74c3' visibility='default'/>
      </data-member>
      <data-member access='public' layout-offset-in-bits='416'>
-        <var-decl name='raw' type-id='c19b74c3' visibility='default'/>
+        <var-decl name='compress' type-id='c19b74c3' visibility='default'/>
      </data-member>
      <data-member access='public' layout-offset-in-bits='448'>
-        <var-decl name='backup' type-id='c19b74c3' visibility='default'/>
+        <var-decl name='raw' type-id='c19b74c3' visibility='default'/>
      </data-member>
      <data-member access='public' layout-offset-in-bits='480'>
-        <var-decl name='holds' type-id='c19b74c3' visibility='default'/>
+        <var-decl name='backup' type-id='c19b74c3' visibility='default'/>
      </data-member>
      <data-member access='public' layout-offset-in-bits='512'>
+        <var-decl name='holds' type-id='c19b74c3' visibility='default'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='544'>
        <var-decl name='saved' type-id='c19b74c3' visibility='default'/>
      </data-member>
    </class-decl>
@@ -6728,6 +6741,11 @@
      <parameter type-id='95e97e5e'/>
      <return type-id='48b5725f'/>
    </function-decl>
+    <function-decl name='zfs_setproctitle' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <parameter is-variadic='yes'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
    <function-decl name='avl_insert' visibility='default' binding='global' size-in-bits='64'>
      <parameter type-id='a3681dea'/>
      <parameter type-id='eaa32e2f'/>
@@ -56,10 +56,10 @@
 #define	ZDIFF_REMOVED	'-'
 #define	ZDIFF_RENAMED	"R"

-#define	ZDIFF_ADDED_COLOR ANSI_GREEN
+#define	ZDIFF_ADDED_COLOR    ANSI_GREEN
 #define	ZDIFF_MODIFIED_COLOR ANSI_YELLOW
-#define	ZDIFF_REMOVED_COLOR ANSI_RED
-#define	ZDIFF_RENAMED_COLOR ANSI_BLUE
+#define	ZDIFF_REMOVED_COLOR  ANSI_RED
+#define	ZDIFF_RENAMED_COLOR  ANSI_BOLD_BLUE

 /*
 * Given a {dsname, object id}, get the object path
@@ -3051,7 +3051,7 @@ zpool_vdev_online(zpool_handle_t *zhp, const char *path, int flags,

 	verify(nvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID, &zc.zc_guid) == 0);

-	if (avail_spare)
+	if (!(flags & ZFS_ONLINE_SPARE) && avail_spare)
 		return (zfs_error(hdl, EZFS_ISSPARE, msg));

 	if ((flags & ZFS_ONLINE_EXPAND ||
@@ -3158,6 +3158,40 @@ zpool_vdev_offline(zpool_handle_t *zhp, const char *path, boolean_t istmp)
 	}
 }

+/*
+ * Remove the specified vdev asynchronously from the configuration, so
+ * that it may come ONLINE if reinserted. This is called from zed on
+ * Udev remove event.
+ * Note: We also have a similar function zpool_vdev_remove() that
+ * removes the vdev from the pool.
+ */
+int
+zpool_vdev_remove_wanted(zpool_handle_t *zhp, const char *path)
+{
+	zfs_cmd_t zc = {"\0"};
+	char errbuf[1024];
+	nvlist_t *tgt;
+	boolean_t avail_spare, l2cache;
+	libzfs_handle_t *hdl = zhp->zpool_hdl;
+
+	(void) snprintf(errbuf, sizeof (errbuf),
+	    dgettext(TEXT_DOMAIN, "cannot remove %s"), path);
+
+	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
+	if ((tgt = zpool_find_vdev(zhp, path, &avail_spare, &l2cache,
+	    NULL)) == NULL)
+		return (zfs_error(hdl, EZFS_NODEVICE, errbuf));
+
+	zc.zc_guid = fnvlist_lookup_uint64(tgt, ZPOOL_CONFIG_GUID);
+
+	zc.zc_cookie = VDEV_STATE_REMOVED;
+
+	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
+		return (0);
+
+	return (zpool_standard_error(hdl, errno, errbuf));
+}
+
 /*
 * Mark the given vdev faulted.
 */
@@ -83,6 +83,9 @@ typedef struct progress_arg {
 	boolean_t pa_parsable;
 	boolean_t pa_estimate;
 	int pa_verbosity;
+	boolean_t pa_astitle;
+	boolean_t pa_progress;
+	uint64_t pa_size;
 } progress_arg_t;

 static int
@@ -712,6 +715,7 @@ typedef struct send_dump_data {
 	boolean_t seenfrom, seento, replicate, doall, fromorigin;
 	boolean_t dryrun, parsable, progress, embed_data, std_out;
 	boolean_t large_block, compress, raw, holds;
+	boolean_t progressastitle;
 	int outfd;
 	boolean_t err;
 	nvlist_t *fss;
@@ -904,6 +908,7 @@ send_progress_thread(void *arg)
 	zfs_handle_t *zhp = pa->pa_zhp;
 	uint64_t bytes;
 	uint64_t blocks;
+	uint64_t total = pa->pa_size / 100;
 	char buf[16];
 	time_t t;
 	struct tm *tm;
@@ -922,7 +927,7 @@ send_progress_thread(void *arg)
 			return ((void *)(uintptr_t)err);
 		}

-		if (firstloop && !pa->pa_parsable) {
+		if (firstloop && !pa->pa_parsable && pa->pa_progress) {
 			(void) fprintf(stderr,
 			    "TIME       %s   %sSNAPSHOT %s\n",
 			    pa->pa_estimate ? "BYTES" : " SENT",
@@ -934,6 +939,17 @@ send_progress_thread(void *arg)
 		(void) time(&t);
 		tm = localtime(&t);

+		if (pa->pa_astitle) {
+			char buf_bytes[16];
+			char buf_size[16];
+			int pct;
+			zfs_nicenum(bytes, buf_bytes, sizeof (buf_bytes));
+			zfs_nicenum(pa->pa_size, buf_size, sizeof (buf_size));
+			pct = (total > 0) ? bytes / total : 100;
+			zfs_setproctitle("sending %s (%d%%: %s/%s)",
+			    zhp->zfs_name, MIN(pct, 100), buf_bytes, buf_size);
+		}
+
 		if (pa->pa_verbosity >= 2 && pa->pa_parsable) {
 			(void) fprintf(stderr,
 			    "%02d:%02d:%02d\t%llu\t%llu\t%s\n",
@@ -950,7 +966,7 @@ send_progress_thread(void *arg)
 			(void) fprintf(stderr, "%02d:%02d:%02d\t%llu\t%s\n",
 			    tm->tm_hour, tm->tm_min, tm->tm_sec,
 			    (u_longlong_t)bytes, zhp->zfs_name);
-		} else {
+		} else if (pa->pa_progress) {
 			zfs_nicebytes(bytes, buf, sizeof (buf));
 			(void) fprintf(stderr, "%02d:%02d:%02d   %5s   %s\n",
 			    tm->tm_hour, tm->tm_min, tm->tm_sec,
@@ -1114,12 +1130,15 @@ dump_snapshot(zfs_handle_t *zhp, void *arg)
 		 * If progress reporting is requested, spawn a new thread to
 		 * poll ZFS_IOC_SEND_PROGRESS at a regular interval.
 		 */
-		if (sdd->progress) {
+		if (sdd->progress || sdd->progressastitle) {
 			pa.pa_zhp = zhp;
 			pa.pa_fd = sdd->outfd;
 			pa.pa_parsable = sdd->parsable;
 			pa.pa_estimate = B_FALSE;
 			pa.pa_verbosity = sdd->verbosity;
+			pa.pa_size = sdd->size;
+			pa.pa_astitle = sdd->progressastitle;
+			pa.pa_progress = sdd->progress;

 			if ((err = pthread_create(&tid, NULL,
 			    send_progress_thread, &pa)) != 0) {
@@ -1131,7 +1150,7 @@ dump_snapshot(zfs_handle_t *zhp, void *arg)
 		err = dump_ioctl(zhp, sdd->prevsnap, sdd->prevsnap_obj,
 		    fromorigin, sdd->outfd, flags, sdd->debugnv);

-		if (sdd->progress) {
+		if (sdd->progress || sdd->progressastitle) {
 			void *status = NULL;
 			(void) pthread_cancel(tid);
 			(void) pthread_join(tid, &status);
@@ -1462,7 +1481,7 @@ lzc_flags_from_sendflags(const sendflags_t *flags)
 static int
 estimate_size(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
    uint64_t resumeobj, uint64_t resumeoff, uint64_t bytes,
-    const char *redactbook, char *errbuf)
+    const char *redactbook, char *errbuf, uint64_t *sizep)
 {
 	uint64_t size;
 	FILE *fout = flags->dryrun ? stdout : stderr;
@@ -1470,7 +1489,7 @@ estimate_size(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 	int err = 0;
 	pthread_t ptid;

-	if (flags->progress) {
+	if (flags->progress || flags->progressastitle) {
 		pa.pa_zhp = zhp;
 		pa.pa_fd = fd;
 		pa.pa_parsable = flags->parsable;
@@ -1489,8 +1508,9 @@ estimate_size(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 	err = lzc_send_space_resume_redacted(zhp->zfs_name, from,
 	    lzc_flags_from_sendflags(flags), resumeobj, resumeoff, bytes,
 	    redactbook, fd, &size);
+	*sizep = size;

-	if (flags->progress) {
+	if (flags->progress || flags->progressastitle) {
 		void *status = NULL;
 		(void) pthread_cancel(ptid);
 		(void) pthread_join(ptid, &status);
@@ -1505,6 +1525,9 @@ estimate_size(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 		}
 	}

+	if (!flags->progress && !flags->parsable)
+		return (err);
+
 	if (err != 0) {
 		zfs_error_aux(zhp->zfs_hdl, "%s", strerror(err));
 		return (zfs_error(zhp->zfs_hdl, EZFS_BADBACKUP,
@@ -1638,6 +1661,7 @@ zfs_send_resume_impl(libzfs_handle_t *hdl, sendflags_t *flags, int outfd,
 	uint64_t *redact_snap_guids = NULL;
 	int num_redact_snaps = 0;
 	char *redact_book = NULL;
+	uint64_t size = 0;

 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
 	    "cannot resume send"));
@@ -1731,7 +1755,7 @@ zfs_send_resume_impl(libzfs_handle_t *hdl, sendflags_t *flags, int outfd,
 		}
 	}

-	if (flags->verbosity != 0) {
+	if (flags->verbosity != 0 || flags->progressastitle) {
 		/*
 		 * Some of these may have come from the resume token, set them
 		 * here for size estimate purposes.
@@ -1748,7 +1772,7 @@ zfs_send_resume_impl(libzfs_handle_t *hdl, sendflags_t *flags, int outfd,
 		if (lzc_flags & LZC_SEND_FLAG_SAVED)
 			tmpflags.saved = B_TRUE;
 		error = estimate_size(zhp, fromname, outfd, &tmpflags,
-		    resumeobj, resumeoff, bytes, redact_book, errbuf);
+		    resumeobj, resumeoff, bytes, redact_book, errbuf, &size);
 	}

 	if (!flags->dryrun) {
@@ -1758,12 +1782,15 @@ zfs_send_resume_impl(libzfs_handle_t *hdl, sendflags_t *flags, int outfd,
 		 * If progress reporting is requested, spawn a new thread to
 		 * poll ZFS_IOC_SEND_PROGRESS at a regular interval.
 		 */
-		if (flags->progress) {
+		if (flags->progress || flags->progressastitle) {
 			pa.pa_zhp = zhp;
 			pa.pa_fd = outfd;
 			pa.pa_parsable = flags->parsable;
 			pa.pa_estimate = B_FALSE;
 			pa.pa_verbosity = flags->verbosity;
+			pa.pa_size = size;
+			pa.pa_astitle = flags->progressastitle;
+			pa.pa_progress = flags->progress;

 			error = pthread_create(&tid, NULL,
 			    send_progress_thread, &pa);
@@ -1780,7 +1807,7 @@ zfs_send_resume_impl(libzfs_handle_t *hdl, sendflags_t *flags, int outfd,
 		if (redact_book != NULL)
 			free(redact_book);

-		if (flags->progress) {
+		if (flags->progress || flags->progress) {
 			void *status = NULL;
 			(void) pthread_cancel(tid);
 			(void) pthread_join(tid, &status);
@@ -1790,6 +1817,7 @@ zfs_send_resume_impl(libzfs_handle_t *hdl, sendflags_t *flags, int outfd,
 				(void) snprintf(errbuf, sizeof (errbuf),
 				    dgettext(TEXT_DOMAIN,
 				    "progress thread exited nonzero"));
+				zfs_close(zhp);
 				return (zfs_standard_error(hdl, error, errbuf));
 			}
 		}
@@ -2199,6 +2227,7 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 	sdd.verbosity = flags->verbosity;
 	sdd.parsable = flags->parsable;
 	sdd.progress = flags->progress;
+	sdd.progressastitle = flags->progressastitle;
 	sdd.dryrun = flags->dryrun;
 	sdd.large_block = flags->largeblock;
 	sdd.embed_data = flags->embed_data;
@@ -2410,6 +2439,7 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 	char *name = zhp->zfs_name;
 	pthread_t ptid;
 	progress_arg_t pa = { 0 };
+	uint64_t size = 0;

 	char errbuf[1024];
 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
@@ -2492,9 +2522,9 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 	/*
 	 * Perform size estimate if verbose was specified.
 	 */
-	if (flags->verbosity != 0) {
+	if (flags->verbosity != 0 || flags->progressastitle) {
 		err = estimate_size(zhp, from, fd, flags, 0, 0, 0, redactbook,
-		    errbuf);
+		    errbuf, &size);
 		if (err != 0)
 			return (err);
 	}
@@ -2506,12 +2536,15 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 	 * If progress reporting is requested, spawn a new thread to poll
 	 * ZFS_IOC_SEND_PROGRESS at a regular interval.
 	 */
-	if (flags->progress) {
+	if (flags->progress || flags->progressastitle) {
 		pa.pa_zhp = zhp;
 		pa.pa_fd = fd;
 		pa.pa_parsable = flags->parsable;
 		pa.pa_estimate = B_FALSE;
 		pa.pa_verbosity = flags->verbosity;
+		pa.pa_size = size;
+		pa.pa_astitle = flags->progressastitle;
+		pa.pa_progress = flags->progress;

 		err = pthread_create(&ptid, NULL,
 		    send_progress_thread, &pa);
@@ -2525,7 +2558,7 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
 	err = lzc_send_redacted(name, from, fd,
 	    lzc_flags_from_sendflags(flags), redactbook);

-	if (flags->progress) {
+	if (flags->progress || flags->progressastitle) {
 		void *status = NULL;
 		(void) pthread_cancel(ptid);
 		(void) pthread_join(ptid, &status);
@@ -2027,7 +2027,7 @@ zfs_version_print(void)
 * Return 1 if the user requested ANSI color output, and our terminal supports
 * it.  Return 0 for no color.
 */
-static int
+int
 use_color(void)
 {
 	static int use_color = -1;
@@ -2073,10 +2073,11 @@ use_color(void)
 }

 /*
- * color_start() and color_end() are used for when you want to colorize a block
- * of text.  For example:
+ * The functions color_start() and color_end() are used for when you want
+ * to colorize a block of text.
 *
- * color_start(ANSI_RED_FG)
+ * For example:
+ * color_start(ANSI_RED)
 * printf("hello");
 * printf("world");
 * color_end();
@@ -2084,7 +2085,7 @@ use_color(void)
 void
 color_start(const char *color)
 {
-	if (use_color()) {
+	if (color && use_color()) {
 		fputs(color, stdout);
 		fflush(stdout);
 	}
@@ -2099,7 +2100,9 @@ color_end(void)
 	}
 }

-/* printf() with a color.  If color is NULL, then do a normal printf. */
+/*
+ * printf() with a color. If color is NULL, then do a normal printf.
+ */
 int
 printf_color(const char *color, char *format, ...)
 {
@@ -272,6 +272,8 @@
    <elf-symbol name='zfs_niceraw' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zfs_nicetime' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zfs_resolve_shortname' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zfs_setproctitle' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zfs_setproctitle_init' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zfs_strcmp_pathname' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zfs_strip_partition' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
    <elf-symbol name='zfs_strip_path' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -3340,6 +3342,30 @@
      <return type-id='95e97e5e'/>
    </function-decl>
  </abi-instr>
+  <abi-instr address-size='64' path='os/linux/zutil_setproctitle.c' language='LANG_C99'>
+    <function-decl name='warnx' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <parameter is-variadic='yes'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
+    <function-decl name='setenv' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <parameter type-id='80f4b756'/>
+      <parameter type-id='95e97e5e'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
+    <function-decl name='zfs_setproctitle_init' mangled-name='zfs_setproctitle_init' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_setproctitle_init'>
+      <parameter type-id='95e97e5e' name='argc'/>
+      <parameter type-id='9b23c9ad' name='argv'/>
+      <parameter type-id='9b23c9ad' name='envp'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
+    <function-decl name='zfs_setproctitle' mangled-name='zfs_setproctitle' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zfs_setproctitle'>
+      <parameter type-id='80f4b756' name='fmt'/>
+      <parameter is-variadic='yes'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
+  </abi-instr>
  <abi-instr address-size='64' path='zutil_device_path.c' language='LANG_C99'>
    <qualified-type-def type-id='26a90f95' restrict='yes' id='266fe297'/>
    <qualified-type-def type-id='80f4b756' const='yes' id='b99c00c9'/>
@@ -18,6 +18,7 @@ USER_C = \

 if BUILD_LINUX
 USER_C += \
+	os/linux/zutil_setproctitle.c \
 	os/linux/zutil_device_path_os.c \
 	os/linux/zutil_import_os.c \
 	os/linux/zutil_compat.c
@@ -0,0 +1,299 @@
+/*
+ * Copyright © 2013 Guillem Jover <guillem@hadrons.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL
+ * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <errno.h>
+#include <stddef.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/param.h>
+#include <libzutil.h>
+
+static struct {
+	/* Original value. */
+	const char *arg0;
+
+	/* Title space available. */
+	char *base, *end;
+
+	/* Pointer to original nul character within base. */
+	char *nul;
+
+	boolean_t warned;
+	boolean_t reset;
+	int error;
+} SPT;
+
+#define	LIBBSD_IS_PATHNAME_SEPARATOR(c) ((c) == '/')
+#define	SPT_MAXTITLE 255
+
+extern const char *__progname;
+
+static const char *
+getprogname(void)
+{
+	return (__progname);
+}
+
+static void
+setprogname(const char *progname)
+{
+	size_t i;
+
+	for (i = strlen(progname); i > 0; i--) {
+		if (LIBBSD_IS_PATHNAME_SEPARATOR(progname[i - 1])) {
+			__progname = progname + i;
+			return;
+		}
+	}
+	__progname = progname;
+}
+
+
+static inline size_t
+spt_min(size_t a, size_t b)
+{
+	return ((a < b) ? a : b);
+}
+
+/*
+ * For discussion on the portability of the various methods, see
+ * https://lists.freebsd.org/pipermail/freebsd-stable/2008-June/043136.html
+ */
+static int
+spt_clearenv(void)
+{
+	char **tmp;
+
+	tmp = malloc(sizeof (*tmp));
+	if (tmp == NULL)
+		return (errno);
+
+	tmp[0] = NULL;
+	environ = tmp;
+
+	return (0);
+}
+
+static int
+spt_copyenv(int envc, char *envp[])
+{
+	char **envcopy;
+	char *eq;
+	int envsize;
+	int i, error;
+
+	if (environ != envp)
+		return (0);
+
+	/*
+	 * Make a copy of the old environ array of pointers, in case
+	 * clearenv() or setenv() is implemented to free the internal
+	 * environ array, because we will need to access the old environ
+	 * contents to make the new copy.
+	 */
+	envsize = (envc + 1) * sizeof (char *);
+	envcopy = malloc(envsize);
+	if (envcopy == NULL)
+		return (errno);
+	memcpy(envcopy, envp, envsize);
+
+	error = spt_clearenv();
+	if (error) {
+		environ = envp;
+		free(envcopy);
+		return (error);
+	}
+
+	for (i = 0; envcopy[i]; i++) {
+		eq = strchr(envcopy[i], '=');
+		if (eq == NULL)
+			continue;
+
+		*eq = '\0';
+		if (setenv(envcopy[i], eq + 1, 1) < 0)
+			error = errno;
+		*eq = '=';
+
+		if (error) {
+			environ = envp;
+			free(envcopy);
+			return (error);
+		}
+	}
+
+	/*
+	 * Dispose of the shallow copy, now that we've finished transfering
+	 * the old environment.
+	 */
+	free(envcopy);
+
+	return (0);
+}
+
+static int
+spt_copyargs(int argc, char *argv[])
+{
+	char *tmp;
+	int i;
+
+	for (i = 1; i < argc || (i >= argc && argv[i]); i++) {
+		if (argv[i] == NULL)
+			continue;
+
+		tmp = strdup(argv[i]);
+		if (tmp == NULL)
+			return (errno);
+
+		argv[i] = tmp;
+	}
+
+	return (0);
+}
+
+void
+zfs_setproctitle_init(int argc, char *argv[], char *envp[])
+{
+	char *base, *end, *nul, *tmp;
+	int i, envc, error;
+
+	/* Try to make sure we got called with main() arguments. */
+	if (argc < 0)
+		return;
+
+	base = argv[0];
+	if (base == NULL)
+		return;
+
+	nul = base + strlen(base);
+	end = nul + 1;
+
+	for (i = 0; i < argc || (i >= argc && argv[i]); i++) {
+		if (argv[i] == NULL || argv[i] != end)
+			continue;
+
+		end = argv[i] + strlen(argv[i]) + 1;
+	}
+
+	for (i = 0; envp[i]; i++) {
+		if (envp[i] != end)
+			continue;
+
+		end = envp[i] + strlen(envp[i]) + 1;
+	}
+	envc = i;
+
+	SPT.arg0 = strdup(argv[0]);
+	if (SPT.arg0 == NULL) {
+		SPT.error = errno;
+		return;
+	}
+
+	tmp = strdup(getprogname());
+	if (tmp == NULL) {
+		SPT.error = errno;
+		return;
+	}
+	setprogname(tmp);
+
+	error = spt_copyenv(envc, envp);
+	if (error) {
+		SPT.error = error;
+		return;
+	}
+
+	error = spt_copyargs(argc, argv);
+	if (error) {
+		SPT.error = error;
+		return;
+	}
+
+	SPT.nul  = nul;
+	SPT.base = base;
+	SPT.end  = end;
+}
+
+void
+zfs_setproctitle(const char *fmt, ...)
+{
+	/* Use buffer in case argv[0] is passed. */
+	char buf[SPT_MAXTITLE + 1];
+	va_list ap;
+	char *nul;
+	int len;
+	if (SPT.base == NULL) {
+		if (!SPT.warned) {
+			warnx("setproctitle not initialized, please"
+			    "call zfs_setproctitle_init()");
+			SPT.warned = B_TRUE;
+		}
+		return;
+	}
+
+	if (fmt) {
+		if (fmt[0] == '-') {
+			/* Skip program name prefix. */
+			fmt++;
+			len = 0;
+		} else {
+			/* Print program name heading for grep. */
+			snprintf(buf, sizeof (buf), "%s: ", getprogname());
+			len = strlen(buf);
+		}
+
+		va_start(ap, fmt);
+		len += vsnprintf(buf + len, sizeof (buf) - len, fmt, ap);
+		va_end(ap);
+	} else {
+		len = snprintf(buf, sizeof (buf), "%s", SPT.arg0);
+	}
+
+	if (len <= 0) {
+		SPT.error = errno;
+		return;
+	}
+
+	if (!SPT.reset) {
+		memset(SPT.base, 0, SPT.end - SPT.base);
+		SPT.reset = B_TRUE;
+	} else {
+		memset(SPT.base, 0, spt_min(sizeof (buf), SPT.end - SPT.base));
+	}
+
+	len = spt_min(len, spt_min(sizeof (buf), SPT.end - SPT.base) - 1);
+	memcpy(SPT.base, buf, len);
+	nul = SPT.base + len;
+
+	if (nul < SPT.nul) {
+		*SPT.nul = '.';
+	} else if (nul == SPT.nul && nul + 1 < SPT.end) {
+		*SPT.nul = ' ';
+		*++nul = '\0';
+	}
+}
@@ -15,7 +15,7 @@
 .\" own identifying information:
 .\" Portions Copyright [yyyy] [name of copyright owner]
 .\"
-.Dd June 1, 2021
+.Dd January 10, 2023
 .Dt ZFS 4
 .Os
 .
@@ -233,6 +233,12 @@ relative to the pool.
 Make some blocks above a certain size be gang blocks.
 This option is used by the test suite to facilitate testing.
 .
+.It Sy zfs_default_bs Ns = Ns Sy 9 Po 512 B Pc Pq int
+Default dnode block size as a power of 2.
+.
+.It Sy zfs_default_ibs Ns = Ns Sy 17 Po 128 KiB Pc Pq int
+Default dnode indirect block size as a power of 2.
+.
 .It Sy zfs_history_output_max Ns = Ns Sy 1048576 Ns B Ns B Po 1MB Pc Pq int
 When attempting to log an output nvlist of an ioctl in the on-disk history,
 the output will not be stored if it is larger than this size (in bytes).
@@ -2126,6 +2132,13 @@ On very fragmented pools, lowering this
 .Pq typically to Sy 36kB
 can improve performance.
 .
+.It Sy zil_min_commit_timeout Ns = Ns Sy 5000 Pq u64
+This sets the minimum delay in nanoseconds ZIL care to delay block commit,
+waiting for more records.
+If ZIL writes are too fast, kernel may not be able sleep for so short interval,
+increasing log latency above allowed by
+.Sy zfs_commit_timeout_pct .
+.
 .It Sy zil_nocacheflush Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Disable the cache flush commands that are normally sent to disk by
 the ZIL after an LWB write has completed.
@@ -1,6 +1,6 @@
 .\" SPDX-License-Identifier: 0BSD
 .\"
-.Dd April 4, 2022
+.Dd March 28, 2023
 .Dt DRACUT.ZFS 7
 .Os
 .
@@ -28,13 +28,13 @@ zfs-import-scan.service \(da                       \(da           | zfs-import-c
                 zfs-import.target \(-> dracut-pre-mount.service
                        |          \(ua            |
                        | dracut-zfs-generator  |
-                        |  ____________________/|
+                        | _____________________/|
                        |/                      \(da
-                        |                   sysroot.mount \(<-\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em dracut-zfs-generator
-                        |                       |                                        \(da   |
-                        |                       \(da            sysroot-{usr,etc,lib,&c.}.mount |
-                        |             initrd-root-fs.target \(<-\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em or                 \(da
-                        |                       |              zfs-nonroot-necessities.service
+                        |                   sysroot.mount \(<-\(em\(em\(em dracut-zfs-generator
+                        |                       |
+                        |                       \(da
+                        |             initrd-root-fs.target \(<-\(em zfs-nonroot-necessities.service
+                        |                       |                                 |
                        |                       \(da                                 |
                        \(da             dracut-mount.service                        |
       zfs-snapshot-bootfs.service              |                                 |
@@ -42,7 +42,7 @@ zfs-import-scan.service \(da                       \(da           | zfs-import-c
                        \(da                       …                                 |
       zfs-rollback-bootfs.service              |                                 |
                        |                       \(da                                 |
-                        |               sysroot-usr.mount \(<-\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em/
+                        |          /sysroot/{usr,etc,lib,&c.} \(<-\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em/
                        |                       |
                        |                       \(da
                        |                initrd-fs.target
@@ -29,7 +29,7 @@
 .\" Copyright 2018 Nexenta Systems, Inc.
 .\" Copyright 2019 Joyent, Inc.
 .\"
-.Dd April 15, 2021
+.Dd January 12, 2023
 .Dt ZFS-SEND 8
 .Os
 .
@@ -39,28 +39,28 @@
 .Sh SYNOPSIS
 .Nm zfs
 .Cm send
-.Op Fl DLPRbcehnpsvw
+.Op Fl DLPVRbcehnpsvw
 .Op Oo Fl I Ns | Ns Fl i Oc Ar snapshot
 .Ar snapshot
 .Nm zfs
 .Cm send
-.Op Fl DLPcensvw
+.Op Fl DLPVcensvw
 .Op Fl i Ar snapshot Ns | Ns Ar bookmark
 .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot
 .Nm zfs
 .Cm send
 .Fl -redact Ar redaction_bookmark
-.Op Fl DLPcenpv
+.Op Fl DLPVcenpv
 .Op Fl i Ar snapshot Ns | Ns Ar bookmark
 .Ar snapshot
 .Nm zfs
 .Cm send
-.Op Fl Penv
+.Op Fl PVenv
 .Fl t
 .Ar receive_resume_token
 .Nm zfs
 .Cm send
-.Op Fl Pnv
+.Op Fl PVnv
 .Fl S Ar filesystem
 .Nm zfs
 .Cm redact
@@ -72,7 +72,7 @@
 .It Xo
 .Nm zfs
 .Cm send
-.Op Fl DLPRbcehnpvw
+.Op Fl DLPVRbcehnpvw
 .Op Oo Fl I Ns | Ns Fl i Oc Ar snapshot
 .Ar snapshot
 .Xc
@@ -140,6 +140,8 @@ If the
 flag is used to send encrypted datasets, then
 .Fl w
 must also be specified.
+.It Fl V , -proctitle
+Set the process title to a per-second report of how much data has been sent.
 .It Fl e , -embed
 Generate a more compact stream by using
 .Sy WRITE_EMBEDDED
@@ -285,7 +287,7 @@ You will be able to receive your streams on future versions of ZFS.
 .It Xo
 .Nm zfs
 .Cm send
-.Op Fl DLPcenvw
+.Op Fl DLPVcenvw
 .Op Fl i Ar snapshot Ns | Ns Ar bookmark
 .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot
 .Xc
@@ -417,7 +419,7 @@ This information includes a per-second report of how much data has been sent.
 .Nm zfs
 .Cm send
 .Fl -redact Ar redaction_bookmark
-.Op Fl DLPcenpv
+.Op Fl DLPVcenpv
 .Op Fl i Ar snapshot Ns | Ns Ar bookmark
 .Ar snapshot
 .Xc
@@ -511,7 +513,7 @@ raw sends and redacted sends cannot be combined at this time.
 .It Xo
 .Nm zfs
 .Cm send
-.Op Fl Penv
+.Op Fl PVenv
 .Fl t
 .Ar receive_resume_token
 .Xc
@@ -526,7 +528,7 @@ for more details.
 .It Xo
 .Nm zfs
 .Cm send
-.Op Fl Pnv
+.Op Fl PVnv
 .Op Fl i Ar snapshot Ns | Ns Ar bookmark
 .Fl S
 .Ar filesystem
@@ -708,6 +708,8 @@ command will be undone if the share is ever unshared (like via a reboot).
 .It Sy ZFS_COLOR
 Use ANSI color in
 .Nm zfs Cm diff
+and
+.Nm zfs Cm list
 output.
 .El
 .Bl -tag -width "ZFS_MOUNT_HELPER"
@@ -433,6 +433,8 @@ to dump core on exit for the purposes of running
 .It Sy ZFS_COLOR
 Use ANSI color in
 .Nm zpool status
+and
+.Nm zpool iostat
 output.
 .It Sy ZPOOL_IMPORT_PATH
 The search path for devices or files to use with the pool.
@@ -125,7 +125,6 @@ mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, char *fspath,
 	struct vfsconf *vfsp;
 	struct mount *mp;
 	vnode_t *vp, *mvp;
-	struct ucred *cr;
 	int error;

 	ASSERT_VOP_ELOCKED(*vpp, "mount_snapshot");
@@ -194,15 +193,8 @@ mount_snapshot(kthread_t *td, vnode_t **vpp, const char *fstype, char *fspath,
 	 * mount(8) and df(1) output.
 	 */
 	mp->mnt_flag |= MNT_IGNORE;
-	/*
-	 * XXX: This is evil, but we can't mount a snapshot as a regular user.
-	 * XXX: Is is safe when snapshot is mounted from within a jail?
-	 */
-	cr = td->td_ucred;
-	td->td_ucred = kcred;
-	error = VFS_MOUNT(mp);
-	td->td_ucred = cr;

+	error = VFS_MOUNT(mp);
 	if (error != 0) {
 		/*
 		 * Clear VI_MOUNT and decrement the use count "atomically",
@@ -160,8 +160,6 @@ arc_prune_task(void *arg)
 {
 	int64_t nr_scan = (intptr_t)arg;

-	arc_reduce_target_size(ptob(nr_scan));
-
 #ifndef __ILP32__
 	if (nr_scan > INT_MAX)
 		nr_scan = INT_MAX;
@@ -356,14 +356,6 @@ SYSCTL_UINT(_vfs_zfs_zfetch, OID_AUTO, max_idistance, CTLFLAG_RWTUN,
 /* dsl_pool.c */

 /* dnode.c */
-extern int zfs_default_bs;
-SYSCTL_INT(_vfs_zfs, OID_AUTO, default_bs, CTLFLAG_RWTUN,
-    &zfs_default_bs, 0, "Default dnode block shift");
-
-extern int zfs_default_ibs;
-SYSCTL_INT(_vfs_zfs, OID_AUTO, default_ibs, CTLFLAG_RWTUN,
-    &zfs_default_ibs, 0, "Default dnode indirect block shift");
-

 /* dsl_scan.c */

@@ -64,6 +64,7 @@
 #include <sys/dsl_dir.h>
 #include <sys/spa_boot.h>
 #include <sys/jail.h>
+#include <sys/osd.h>
 #include <ufs/ufs/quota.h>
 #include <sys/zfs_quota.h>

@@ -90,6 +91,20 @@ int zfs_debug_level;
 SYSCTL_INT(_vfs_zfs, OID_AUTO, debug, CTLFLAG_RWTUN, &zfs_debug_level, 0,
 	"Debug level");

+struct zfs_jailparam {
+	int mount_snapshot;
+};
+
+static struct zfs_jailparam zfs_jailparam0 = {
+	.mount_snapshot = 0,
+};
+
+static int zfs_jailparam_slot;
+
+SYSCTL_JAIL_PARAM_SYS_NODE(zfs, CTLFLAG_RW, "Jail ZFS parameters");
+SYSCTL_JAIL_PARAM(_zfs, mount_snapshot, CTLTYPE_INT | CTLFLAG_RW, "I",
+	"Allow mounting snapshots in the .zfs directory for unjailed datasets");
+
 SYSCTL_NODE(_vfs_zfs, OID_AUTO, version, CTLFLAG_RD, 0, "ZFS versions");
 static int zfs_version_acl = ZFS_ACL_VERSION;
 SYSCTL_INT(_vfs_zfs_version, OID_AUTO, acl, CTLFLAG_RD, &zfs_version_acl, 0,
@@ -1332,7 +1347,7 @@ zfs_mount(vfs_t *vfsp)
 	char		*osname;
 	int		error = 0;
 	int		canwrite;
-	bool		checkpointrewind;
+	bool		checkpointrewind, isctlsnap = false;

 	if (vfs_getopt(vfsp->mnt_optnew, "from", (void **)&osname, NULL))
 		return (SET_ERROR(EINVAL));
@@ -1347,6 +1362,8 @@ zfs_mount(vfs_t *vfsp)
 	}

 	fetch_osname_options(osname, &checkpointrewind);
+	isctlsnap = (mvp != NULL && zfsctl_is_node(mvp) &&
+	    strchr(osname, '@') != NULL);

 	/*
 	 * Check for mount privilege?
@@ -1355,7 +1372,9 @@ zfs_mount(vfs_t *vfsp)
 	 * we have local permission to allow it
 	 */
 	error = secpolicy_fs_mount(cr, mvp, vfsp);
-	if (error) {
+	if (error && isctlsnap) {
+		secpolicy_fs_mount_clearopts(cr, vfsp);
+	} else if (error) {
 		if (dsl_deleg_access(osname, ZFS_DELEG_PERM_MOUNT, cr) != 0)
 			goto out;

@@ -1392,8 +1411,27 @@ zfs_mount(vfs_t *vfsp)
 	 */
 	if (!INGLOBALZONE(curproc) &&
 	    (!zone_dataset_visible(osname, &canwrite) || !canwrite)) {
-		error = SET_ERROR(EPERM);
-		goto out;
+		boolean_t mount_snapshot = B_FALSE;
+
+		/*
+		 * Snapshots may be mounted in .zfs for unjailed datasets
+		 * if allowed by the jail param zfs.mount_snapshot.
+		 */
+		if (isctlsnap) {
+			struct prison *pr;
+			struct zfs_jailparam *zjp;
+
+			pr = curthread->td_ucred->cr_prison;
+			mtx_lock(&pr->pr_mtx);
+			zjp = osd_jail_get(pr, zfs_jailparam_slot);
+			mtx_unlock(&pr->pr_mtx);
+			if (zjp && zjp->mount_snapshot)
+				mount_snapshot = B_TRUE;
+		}
+		if (!mount_snapshot) {
+			error = SET_ERROR(EPERM);
+			goto out;
+		}
 	}

 	vfsp->vfs_flag |= MNT_NFS4ACLS;
@@ -2343,3 +2381,236 @@ zfsvfs_update_fromname(const char *oldname, const char *newname)
 	mtx_unlock(&mountlist_mtx);
 }
 #endif
+
+/*
+ * Find a prison with ZFS info.
+ * Return the ZFS info and the (locked) prison.
+ */
+static struct zfs_jailparam *
+zfs_jailparam_find(struct prison *spr, struct prison **prp)
+{
+	struct prison *pr;
+	struct zfs_jailparam *zjp;
+
+	for (pr = spr; ; pr = pr->pr_parent) {
+		mtx_lock(&pr->pr_mtx);
+		if (pr == &prison0) {
+			zjp = &zfs_jailparam0;
+			break;
+		}
+		zjp = osd_jail_get(pr, zfs_jailparam_slot);
+		if (zjp != NULL)
+			break;
+		mtx_unlock(&pr->pr_mtx);
+	}
+	*prp = pr;
+
+	return (zjp);
+}
+
+/*
+ * Ensure a prison has its own ZFS info.  If zjpp is non-null, point it to the
+ * ZFS info and lock the prison.
+ */
+static void
+zfs_jailparam_alloc(struct prison *pr, struct zfs_jailparam **zjpp)
+{
+	struct prison *ppr;
+	struct zfs_jailparam *zjp, *nzjp;
+	void **rsv;
+
+	/* If this prison already has ZFS info, return that. */
+	zjp = zfs_jailparam_find(pr, &ppr);
+	if (ppr == pr)
+		goto done;
+
+	/*
+	 * Allocate a new info record.  Then check again, in case something
+	 * changed during the allocation.
+	 */
+	mtx_unlock(&ppr->pr_mtx);
+	nzjp = malloc(sizeof (struct zfs_jailparam), M_PRISON, M_WAITOK);
+	rsv = osd_reserve(zfs_jailparam_slot);
+	zjp = zfs_jailparam_find(pr, &ppr);
+	if (ppr == pr) {
+		free(nzjp, M_PRISON);
+		osd_free_reserved(rsv);
+		goto done;
+	}
+	/* Inherit the initial values from the ancestor. */
+	mtx_lock(&pr->pr_mtx);
+	(void) osd_jail_set_reserved(pr, zfs_jailparam_slot, rsv, nzjp);
+	(void) memcpy(nzjp, zjp, sizeof (*zjp));
+	zjp = nzjp;
+	mtx_unlock(&ppr->pr_mtx);
+done:
+	if (zjpp != NULL)
+		*zjpp = zjp;
+	else
+		mtx_unlock(&pr->pr_mtx);
+}
+
+/*
+ * Jail OSD methods for ZFS VFS info.
+ */
+static int
+zfs_jailparam_create(void *obj, void *data)
+{
+	struct prison *pr = obj;
+	struct vfsoptlist *opts = data;
+	int jsys;
+
+	if (vfs_copyopt(opts, "zfs", &jsys, sizeof (jsys)) == 0 &&
+	    jsys == JAIL_SYS_INHERIT)
+		return (0);
+	/*
+	 * Inherit a prison's initial values from its parent
+	 * (different from JAIL_SYS_INHERIT which also inherits changes).
+	 */
+	zfs_jailparam_alloc(pr, NULL);
+	return (0);
+}
+
+static int
+zfs_jailparam_get(void *obj, void *data)
+{
+	struct prison *ppr, *pr = obj;
+	struct vfsoptlist *opts = data;
+	struct zfs_jailparam *zjp;
+	int jsys, error;
+
+	zjp = zfs_jailparam_find(pr, &ppr);
+	jsys = (ppr == pr) ? JAIL_SYS_NEW : JAIL_SYS_INHERIT;
+	error = vfs_setopt(opts, "zfs", &jsys, sizeof (jsys));
+	if (error != 0 && error != ENOENT)
+		goto done;
+	if (jsys == JAIL_SYS_NEW) {
+		error = vfs_setopt(opts, "zfs.mount_snapshot",
+		    &zjp->mount_snapshot, sizeof (zjp->mount_snapshot));
+		if (error != 0 && error != ENOENT)
+			goto done;
+	} else {
+		/*
+		 * If this prison is inheriting its ZFS info, report
+		 * empty/zero parameters.
+		 */
+		static int mount_snapshot = 0;
+
+		error = vfs_setopt(opts, "zfs.mount_snapshot",
+		    &mount_snapshot, sizeof (mount_snapshot));
+		if (error != 0 && error != ENOENT)
+			goto done;
+	}
+	error = 0;
+done:
+	mtx_unlock(&ppr->pr_mtx);
+	return (error);
+}
+
+static int
+zfs_jailparam_set(void *obj, void *data)
+{
+	struct prison *pr = obj;
+	struct prison *ppr;
+	struct vfsoptlist *opts = data;
+	int error, jsys, mount_snapshot;
+
+	/* Set the parameters, which should be correct. */
+	error = vfs_copyopt(opts, "zfs", &jsys, sizeof (jsys));
+	if (error == ENOENT)
+		jsys = -1;
+	error = vfs_copyopt(opts, "zfs.mount_snapshot", &mount_snapshot,
+	    sizeof (mount_snapshot));
+	if (error == ENOENT)
+		mount_snapshot = -1;
+	else
+		jsys = JAIL_SYS_NEW;
+	if (jsys == JAIL_SYS_NEW) {
+		/* "zfs=new" or "zfs.*": the prison gets its own ZFS info. */
+		struct zfs_jailparam *zjp;
+
+		/*
+		 * A child jail cannot have more permissions than its parent
+		 */
+		if (pr->pr_parent != &prison0) {
+			zjp = zfs_jailparam_find(pr->pr_parent, &ppr);
+			mtx_unlock(&ppr->pr_mtx);
+			if (zjp->mount_snapshot < mount_snapshot) {
+				return (EPERM);
+			}
+		}
+		zfs_jailparam_alloc(pr, &zjp);
+		if (mount_snapshot != -1)
+			zjp->mount_snapshot = mount_snapshot;
+		mtx_unlock(&pr->pr_mtx);
+	} else {
+		/* "zfs=inherit": inherit the parent's ZFS info. */
+		mtx_lock(&pr->pr_mtx);
+		osd_jail_del(pr, zfs_jailparam_slot);
+		mtx_unlock(&pr->pr_mtx);
+	}
+	return (0);
+}
+
+static int
+zfs_jailparam_check(void *obj __unused, void *data)
+{
+	struct vfsoptlist *opts = data;
+	int error, jsys, mount_snapshot;
+
+	/* Check that the parameters are correct. */
+	error = vfs_copyopt(opts, "zfs", &jsys, sizeof (jsys));
+	if (error != ENOENT) {
+		if (error != 0)
+			return (error);
+		if (jsys != JAIL_SYS_NEW && jsys != JAIL_SYS_INHERIT)
+			return (EINVAL);
+	}
+	error = vfs_copyopt(opts, "zfs.mount_snapshot", &mount_snapshot,
+	    sizeof (mount_snapshot));
+	if (error != ENOENT) {
+		if (error != 0)
+			return (error);
+		if (mount_snapshot != 0 && mount_snapshot != 1)
+			return (EINVAL);
+	}
+	return (0);
+}
+
+static void
+zfs_jailparam_destroy(void *data)
+{
+
+	free(data, M_PRISON);
+}
+
+static void
+zfs_jailparam_sysinit(void *arg __unused)
+{
+	struct prison *pr;
+	osd_method_t  methods[PR_MAXMETHOD] = {
+		[PR_METHOD_CREATE] = zfs_jailparam_create,
+		[PR_METHOD_GET] = zfs_jailparam_get,
+		[PR_METHOD_SET] = zfs_jailparam_set,
+		[PR_METHOD_CHECK] = zfs_jailparam_check,
+	};
+
+	zfs_jailparam_slot = osd_jail_register(zfs_jailparam_destroy, methods);
+	/* Copy the defaults to any existing prisons. */
+	sx_slock(&allprison_lock);
+	TAILQ_FOREACH(pr, &allprison, pr_list)
+		zfs_jailparam_alloc(pr, NULL);
+	sx_sunlock(&allprison_lock);
+}
+
+static void
+zfs_jailparam_sysuninit(void *arg __unused)
+{
+
+	osd_jail_deregister(zfs_jailparam_slot);
+}
+
+SYSINIT(zfs_jailparam_sysinit, SI_SUB_DRIVERS, SI_ORDER_ANY,
+	zfs_jailparam_sysinit, NULL);
+SYSUNINIT(zfs_jailparam_sysuninit, SI_SUB_DRIVERS, SI_ORDER_ANY,
+	zfs_jailparam_sysuninit, NULL);
@@ -1161,7 +1161,10 @@ zvol_cdev_ioctl(struct cdev *dev, ulong_t cmd, caddr_t data,

 		hole = (cmd == FIOSEEKHOLE);
 		noff = *off;
+		lr = zfs_rangelock_enter(&zv->zv_rangelock, 0, UINT64_MAX,
+		    RL_READER);
 		error = dmu_offset_next(zv->zv_objset, ZVOL_OBJ, hole, &noff);
+		zfs_rangelock_exit(lr);
 		*off = noff;
 		break;
 	}
@@ -48,6 +48,7 @@
 #include <linux/mod_compat.h>
 #include <sys/cred.h>
 #include <sys/vnode.h>
+#include <sys/misc.h>

 char spl_gitrev[64] = ZFS_META_GITREV;

@@ -540,6 +541,38 @@ ddi_copyin(const void *from, void *to, size_t len, int flags)
 }
 EXPORT_SYMBOL(ddi_copyin);

+/*
+ * Post a uevent to userspace whenever a new vdev adds to the pool. It is
+ * necessary to sync blkid information with udev, which zed daemon uses
+ * during device hotplug to identify the vdev.
+ */
+void
+spl_signal_kobj_evt(struct block_device *bdev)
+{
+#if defined(HAVE_BDEV_KOBJ) || defined(HAVE_PART_TO_DEV)
+#ifdef HAVE_BDEV_KOBJ
+	struct kobject *disk_kobj = bdev_kobj(bdev);
+#else
+	struct kobject *disk_kobj = &part_to_dev(bdev->bd_part)->kobj;
+#endif
+	if (disk_kobj) {
+		int ret = kobject_uevent(disk_kobj, KOBJ_CHANGE);
+		if (ret) {
+			pr_warn("ZFS: Sending event '%d' to kobject: '%s'"
+			    " (%p): failed(ret:%d)\n", KOBJ_CHANGE,
+			    kobject_name(disk_kobj), disk_kobj, ret);
+		}
+	}
+#else
+/*
+ * This is encountered if neither bdev_kobj() nor part_to_dev() is available
+ * in the kernel - likely due to an API change that needs to be chased down.
+ */
+#error "Unsupported kernel: unable to get struct kobj from bdev"
+#endif
+}
+EXPORT_SYMBOL(spl_signal_kobj_evt);
+
 int
 ddi_copyout(const void *from, void *to, size_t len, int flags)
 {
@@ -247,7 +247,7 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
 	Cpa8U *buffer_meta_src = NULL;
 	Cpa8U *buffer_meta_dst = NULL;
 	Cpa32U buffer_meta_size = 0;
-	CpaDcRqResults dc_results;
+	CpaDcRqResults dc_results = {.checksum = 1};
 	CpaStatus status = CPA_STATUS_FAIL;
 	Cpa32U hdr_sz = 0;
 	Cpa32U compressed_sz;
@@ -179,6 +179,18 @@ vdev_disk_error(zio_t *zio)
 	    zio->io_flags);
 }

+static void
+vdev_disk_kobj_evt_post(vdev_t *v)
+{
+	vdev_disk_t *vd = v->vdev_tsd;
+	if (vd && vd->vd_bdev) {
+		spl_signal_kobj_evt(vd->vd_bdev);
+	} else {
+		vdev_dbgmsg(v, "vdev_disk_t is NULL for VDEV:%s\n",
+		    v->vdev_path);
+	}
+}
+
 static int
 vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
    uint64_t *logical_ashift, uint64_t *physical_ashift)
@@ -290,6 +302,13 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
 		bdev = blkdev_get_by_path(v->vdev_path, mode | FMODE_EXCL,
 		    zfs_vdev_holder);
 		if (unlikely(PTR_ERR(bdev) == -ENOENT)) {
+			/*
+			 * There is no point of waiting since device is removed
+			 * explicitly
+			 */
+			if (v->vdev_removed)
+				break;
+
 			schedule_timeout(MSEC_TO_TICK(10));
 		} else if (unlikely(PTR_ERR(bdev) == -ERESTARTSYS)) {
 			timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms * 10);
@@ -899,7 +918,7 @@ vdev_disk_io_done(zio_t *zio)
 		vdev_t *v = zio->io_vd;
 		vdev_disk_t *vd = v->vdev_tsd;

-		if (zfs_check_media_change(vd->vd_bdev)) {
+		if (!zfs_check_disk_status(vd->vd_bdev)) {
 			invalidate_bdev(vd->vd_bdev);
 			v->vdev_remove_wanted = B_TRUE;
 			spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE);
@@ -955,7 +974,8 @@ vdev_ops_t vdev_disk_ops = {
 	.vdev_op_nparity = NULL,
 	.vdev_op_ndisks = NULL,
 	.vdev_op_type = VDEV_TYPE_DISK,		/* name of this vdev type */
-	.vdev_op_leaf = B_TRUE			/* leaf vdev */
+	.vdev_op_leaf = B_TRUE,			/* leaf vdev */
+	.vdev_op_kobj_evt_post = vdev_disk_kobj_evt_post
 };

 /*
@@ -162,8 +162,7 @@ zfs_znode_hold_cache_constructor(void *buf, void *arg, int kmflags)
 	znode_hold_t *zh = buf;

 	mutex_init(&zh->zh_lock, NULL, MUTEX_DEFAULT, NULL);
-	zfs_refcount_create(&zh->zh_refcount);
-	zh->zh_obj = ZFS_NO_OBJECT;
+	zh->zh_refcount = 0;

 	return (0);
 }
@@ -174,7 +173,6 @@ zfs_znode_hold_cache_destructor(void *buf, void *arg)
 	znode_hold_t *zh = buf;

 	mutex_destroy(&zh->zh_lock);
-	zfs_refcount_destroy(&zh->zh_refcount);
 }

 void
@@ -273,26 +271,26 @@ zfs_znode_hold_enter(zfsvfs_t *zfsvfs, uint64_t obj)
 	boolean_t found = B_FALSE;

 	zh_new = kmem_cache_alloc(znode_hold_cache, KM_SLEEP);
-	zh_new->zh_obj = obj;
 	search.zh_obj = obj;

 	mutex_enter(&zfsvfs->z_hold_locks[i]);
 	zh = avl_find(&zfsvfs->z_hold_trees[i], &search, NULL);
 	if (likely(zh == NULL)) {
 		zh = zh_new;
+		zh->zh_obj = obj;
 		avl_add(&zfsvfs->z_hold_trees[i], zh);
 	} else {
 		ASSERT3U(zh->zh_obj, ==, obj);
 		found = B_TRUE;
 	}
-	zfs_refcount_add(&zh->zh_refcount, NULL);
+	zh->zh_refcount++;
+	ASSERT3S(zh->zh_refcount, >, 0);
 	mutex_exit(&zfsvfs->z_hold_locks[i]);

 	if (found == B_TRUE)
 		kmem_cache_free(znode_hold_cache, zh_new);

 	ASSERT(MUTEX_NOT_HELD(&zh->zh_lock));
-	ASSERT3S(zfs_refcount_count(&zh->zh_refcount), >, 0);
 	mutex_enter(&zh->zh_lock);

 	return (zh);
@@ -305,11 +303,11 @@ zfs_znode_hold_exit(zfsvfs_t *zfsvfs, znode_hold_t *zh)
 	boolean_t remove = B_FALSE;

 	ASSERT(zfs_znode_held(zfsvfs, zh->zh_obj));
-	ASSERT3S(zfs_refcount_count(&zh->zh_refcount), >, 0);
 	mutex_exit(&zh->zh_lock);

 	mutex_enter(&zfsvfs->z_hold_locks[i]);
-	if (zfs_refcount_remove(&zh->zh_refcount, NULL) == 0) {
+	ASSERT3S(zh->zh_refcount, >, 0);
+	if (--zh->zh_refcount == 0) {
 		avl_remove(&zfsvfs->z_hold_trees[i], zh);
 		remove = B_TRUE;
 	}
@@ -5943,6 +5943,7 @@ arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
 	    (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
 	boolean_t embedded_bp = !!BP_IS_EMBEDDED(bp);
 	boolean_t no_buf = *arc_flags & ARC_FLAG_NO_BUF;
+	arc_buf_t *buf = NULL;
 	int rc = 0;

 	ASSERT(!embedded_bp ||
@@ -5972,7 +5973,7 @@ top:
 	if (!zfs_blkptr_verify(spa, bp, zio_flags & ZIO_FLAG_CONFIG_WRITER,
 	    BLK_VERIFY_LOG)) {
 		rc = SET_ERROR(ECKSUM);
-		goto out;
+		goto done;
 	}

 	if (!embedded_bp) {
@@ -5992,7 +5993,6 @@ top:
 	 */
 	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) ||
 	    (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) {
-		arc_buf_t *buf = NULL;
 		*arc_flags |= ARC_FLAG_CACHED;

 		if (HDR_IO_IN_PROGRESS(hdr)) {
@@ -6002,7 +6002,7 @@ top:
 				mutex_exit(hash_lock);
 				ARCSTAT_BUMP(arcstat_cached_only_in_progress);
 				rc = SET_ERROR(ENOENT);
-				goto out;
+				goto done;
 			}

 			ASSERT3P(head_zio, !=, NULL);
@@ -6130,9 +6130,7 @@ top:
 		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
 		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
 		    data, metadata, hits);
-
-		if (done)
-			done(NULL, zb, bp, buf, private);
+		goto done;
 	} else {
 		uint64_t lsize = BP_GET_LSIZE(bp);
 		uint64_t psize = BP_GET_PSIZE(bp);
@@ -6145,10 +6143,10 @@ top:
 		int alloc_flags = encrypted_read ? ARC_HDR_ALLOC_RDATA : 0;

 		if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
-			rc = SET_ERROR(ENOENT);
 			if (hash_lock != NULL)
 				mutex_exit(hash_lock);
-			goto out;
+			rc = SET_ERROR(ENOENT);
+			goto done;
 		}

 		if (hdr == NULL) {
@@ -6474,6 +6472,16 @@ out:
 		spa_read_history_add(spa, zb, *arc_flags);
 	spl_fstrans_unmark(cookie);
 	return (rc);
+
+done:
+	if (done)
+		done(NULL, zb, bp, buf, private);
+	if (pio && rc != 0) {
+		zio_t *zio = zio_null(pio, spa, NULL, NULL, NULL, zio_flags);
+		zio->io_error = rc;
+		zio_nowait(zio);
+	}
+	goto out;
 }

 arc_prune_t *
@@ -663,14 +663,13 @@ bpobj_enqueue_subobj(bpobj_t *bpo, uint64_t subobj, dmu_tx_t *tx)
 	}

 	VERIFY3U(0, ==, bpobj_open(&subbpo, bpo->bpo_os, subobj));
-	VERIFY3U(0, ==, bpobj_space(&subbpo, &used, &comp, &uncomp));
-
 	if (bpobj_is_empty(&subbpo)) {
 		/* No point in having an empty subobj. */
 		bpobj_close(&subbpo);
 		bpobj_free(bpo->bpo_os, subobj, tx);
 		return;
 	}
+	VERIFY3U(0, ==, bpobj_space(&subbpo, &used, &comp, &uncomp));

 	mutex_enter(&bpo->bpo_lock);
 	dmu_buf_will_dirty(bpo->bpo_dbuf, tx);
@@ -780,6 +779,68 @@ bpobj_enqueue_subobj(bpobj_t *bpo, uint64_t subobj, dmu_tx_t *tx)

 }

+/*
+ * Prefetch metadata required for bpobj_enqueue_subobj().
+ */
+void
+bpobj_prefetch_subobj(bpobj_t *bpo, uint64_t subobj)
+{
+	dmu_object_info_t doi;
+	bpobj_t subbpo;
+	uint64_t subsubobjs;
+	boolean_t copy_subsub = B_TRUE;
+	boolean_t copy_bps = B_TRUE;
+
+	ASSERT(bpobj_is_open(bpo));
+	ASSERT(subobj != 0);
+
+	if (subobj == dmu_objset_pool(bpo->bpo_os)->dp_empty_bpobj)
+		return;
+
+	if (bpobj_open(&subbpo, bpo->bpo_os, subobj) != 0)
+		return;
+	if (bpobj_is_empty(&subbpo)) {
+		bpobj_close(&subbpo);
+		return;
+	}
+	subsubobjs = subbpo.bpo_phys->bpo_subobjs;
+	bpobj_close(&subbpo);
+
+	if (subsubobjs != 0) {
+		if (dmu_object_info(bpo->bpo_os, subsubobjs, &doi) != 0)
+			return;
+		if (doi.doi_max_offset > doi.doi_data_block_size)
+			copy_subsub = B_FALSE;
+	}
+
+	if (dmu_object_info(bpo->bpo_os, subobj, &doi) != 0)
+		return;
+	if (doi.doi_max_offset > doi.doi_data_block_size || !copy_subsub)
+		copy_bps = B_FALSE;
+
+	if (copy_subsub && subsubobjs != 0) {
+		if (bpo->bpo_phys->bpo_subobjs) {
+			dmu_prefetch(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs, 0,
+			    bpo->bpo_phys->bpo_num_subobjs * sizeof (subobj), 1,
+			    ZIO_PRIORITY_ASYNC_READ);
+		}
+		dmu_prefetch(bpo->bpo_os, subsubobjs, 0, 0, 1,
+		    ZIO_PRIORITY_ASYNC_READ);
+	}
+
+	if (copy_bps) {
+		dmu_prefetch(bpo->bpo_os, bpo->bpo_object, 0,
+		    bpo->bpo_phys->bpo_num_blkptrs * sizeof (blkptr_t), 1,
+		    ZIO_PRIORITY_ASYNC_READ);
+		dmu_prefetch(bpo->bpo_os, subobj, 0, 0, 1,
+		    ZIO_PRIORITY_ASYNC_READ);
+	} else if (bpo->bpo_phys->bpo_subobjs) {
+		dmu_prefetch(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs, 0,
+		    bpo->bpo_phys->bpo_num_subobjs * sizeof (subobj), 1,
+		    ZIO_PRIORITY_ASYNC_READ);
+	}
+}
+
 void
 bpobj_enqueue(bpobj_t *bpo, const blkptr_t *bp, boolean_t bp_freed,
    dmu_tx_t *tx)
@@ -102,7 +102,7 @@ zfs_btree_poison_node(zfs_btree_t *tree, zfs_btree_hdr_t *hdr)
 		(void) memset(leaf->btl_elems, 0x0f, hdr->bth_first * size);
 		(void) memset(leaf->btl_elems +
 		    (hdr->bth_first + hdr->bth_count) * size, 0x0f,
-		    BTREE_LEAF_ESIZE -
+		    tree->bt_leaf_size - offsetof(zfs_btree_leaf_t, btl_elems) -
 		    (hdr->bth_first + hdr->bth_count) * size);
 	}
 #endif
@@ -173,16 +173,44 @@ zfs_btree_fini(void)
 	kmem_cache_destroy(zfs_btree_leaf_cache);
 }

+static void *
+zfs_btree_leaf_alloc(zfs_btree_t *tree)
+{
+	if (tree->bt_leaf_size == BTREE_LEAF_SIZE)
+		return (kmem_cache_alloc(zfs_btree_leaf_cache, KM_SLEEP));
+	else
+		return (kmem_alloc(tree->bt_leaf_size, KM_SLEEP));
+}
+
+static void
+zfs_btree_leaf_free(zfs_btree_t *tree, void *ptr)
+{
+	if (tree->bt_leaf_size == BTREE_LEAF_SIZE)
+		return (kmem_cache_free(zfs_btree_leaf_cache, ptr));
+	else
+		return (kmem_free(ptr, tree->bt_leaf_size));
+}
+
 void
 zfs_btree_create(zfs_btree_t *tree, int (*compar) (const void *, const void *),
    size_t size)
 {
-	ASSERT3U(size, <=, BTREE_LEAF_ESIZE / 2);
+	zfs_btree_create_custom(tree, compar, size, BTREE_LEAF_SIZE);
+}

-	bzero(tree, sizeof (*tree));
+void
+zfs_btree_create_custom(zfs_btree_t *tree,
+    int (*compar) (const void *, const void *),
+    size_t size, size_t lsize)
+{
+	size_t esize = lsize - offsetof(zfs_btree_leaf_t, btl_elems);
+
+	ASSERT3U(size, <=, esize / 2);
+	memset(tree, 0, sizeof (*tree));
 	tree->bt_compar = compar;
 	tree->bt_elem_size = size;
-	tree->bt_leaf_cap = P2ALIGN(BTREE_LEAF_ESIZE / size, 2);
+	tree->bt_leaf_size = lsize;
+	tree->bt_leaf_cap = P2ALIGN(esize / size, 2);
 	tree->bt_height = -1;
 	tree->bt_bulk = NULL;
 }
@@ -290,7 +318,7 @@ zfs_btree_find(zfs_btree_t *tree, const void *value, zfs_btree_index_t *where)

 	zfs_btree_core_t *node = NULL;
 	uint32_t child = 0;
-	uint64_t depth = 0;
+	uint32_t depth = 0;

 	/*
 	 * Iterate down the tree, finding which child the value should be in
@@ -811,8 +839,7 @@ zfs_btree_insert_into_leaf(zfs_btree_t *tree, zfs_btree_leaf_t *leaf,
 		move_count++;
 	}
 	tree->bt_num_nodes++;
-	zfs_btree_leaf_t *new_leaf = kmem_cache_alloc(zfs_btree_leaf_cache,
-	    KM_SLEEP);
+	zfs_btree_leaf_t *new_leaf = zfs_btree_leaf_alloc(tree);
 	zfs_btree_hdr_t *new_hdr = &new_leaf->btl_hdr;
 	new_hdr->bth_parent = leaf->btl_hdr.bth_parent;
 	new_hdr->bth_first = (tree->bt_bulk ? 0 : capacity / 4) +
@@ -1078,8 +1105,7 @@ zfs_btree_add_idx(zfs_btree_t *tree, const void *value,
 		ASSERT0(where->bti_offset);

 		tree->bt_num_nodes++;
-		zfs_btree_leaf_t *leaf = kmem_cache_alloc(zfs_btree_leaf_cache,
-		    KM_SLEEP);
+		zfs_btree_leaf_t *leaf = zfs_btree_leaf_alloc(tree);
 		tree->bt_root = &leaf->btl_hdr;
 		tree->bt_height++;

@@ -1378,7 +1404,7 @@ zfs_btree_node_destroy(zfs_btree_t *tree, zfs_btree_hdr_t *node)
 {
 	tree->bt_num_nodes--;
 	if (!zfs_btree_is_core(node)) {
-		kmem_cache_free(zfs_btree_leaf_cache, node);
+		zfs_btree_leaf_free(tree, node);
 	} else {
 		kmem_free(node, sizeof (zfs_btree_core_t) +
 		    BTREE_CORE_ELEMS * tree->bt_elem_size);
@@ -1991,7 +2017,7 @@ zfs_btree_verify_counts(zfs_btree_t *tree)
 */
 static uint64_t
 zfs_btree_verify_height_helper(zfs_btree_t *tree, zfs_btree_hdr_t *hdr,
-    int64_t height)
+    int32_t height)
 {
 	if (!zfs_btree_is_core(hdr)) {
 		VERIFY0(height);
@@ -2117,8 +2143,10 @@ zfs_btree_verify_poison_helper(zfs_btree_t *tree, zfs_btree_hdr_t *hdr)
 		zfs_btree_leaf_t *leaf = (zfs_btree_leaf_t *)hdr;
 		for (size_t i = 0; i < hdr->bth_first * size; i++)
 			VERIFY3U(leaf->btl_elems[i], ==, 0x0f);
+		size_t esize = tree->bt_leaf_size -
+		    offsetof(zfs_btree_leaf_t, btl_elems);
 		for (size_t i = (hdr->bth_first + hdr->bth_count) * size;
-		    i < BTREE_LEAF_ESIZE; i++)
+		    i < esize; i++)
 			VERIFY3U(leaf->btl_elems[i], ==, 0x0f);
 	} else {
 		zfs_btree_core_t *node = (zfs_btree_core_t *)hdr;
@@ -602,58 +602,58 @@ dbuf_is_metadata(dmu_buf_impl_t *db)
 boolean_t
 dbuf_is_l2cacheable(dmu_buf_impl_t *db)
 {
-	vdev_t *vd = NULL;
-	zfs_cache_type_t cache = db->db_objset->os_secondary_cache;
-	blkptr_t *bp = db->db_blkptr;
+	if (db->db_objset->os_secondary_cache == ZFS_CACHE_ALL ||
+	    (db->db_objset->os_secondary_cache ==
+	    ZFS_CACHE_METADATA && dbuf_is_metadata(db))) {
+		if (l2arc_exclude_special == 0)
+			return (B_TRUE);

-	if (bp != NULL && !BP_IS_HOLE(bp)) {
+		blkptr_t *bp = db->db_blkptr;
+		if (bp == NULL || BP_IS_HOLE(bp))
+			return (B_FALSE);
 		uint64_t vdev = DVA_GET_VDEV(bp->blk_dva);
 		vdev_t *rvd = db->db_objset->os_spa->spa_root_vdev;
+		vdev_t *vd = NULL;

 		if (vdev < rvd->vdev_children)
 			vd = rvd->vdev_child[vdev];

-		if (cache == ZFS_CACHE_ALL ||
-		    (dbuf_is_metadata(db) && cache == ZFS_CACHE_METADATA)) {
-			if (vd == NULL)
-				return (B_TRUE);
+		if (vd == NULL)
+			return (B_TRUE);

-			if ((vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
-			    vd->vdev_alloc_bias != VDEV_BIAS_DEDUP) ||
-			    l2arc_exclude_special == 0)
-				return (B_TRUE);
-		}
+		if (vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
+		    vd->vdev_alloc_bias != VDEV_BIAS_DEDUP)
+			return (B_TRUE);
 	}
-
 	return (B_FALSE);
 }

 static inline boolean_t
 dnode_level_is_l2cacheable(blkptr_t *bp, dnode_t *dn, int64_t level)
 {
-	vdev_t *vd = NULL;
-	zfs_cache_type_t cache = dn->dn_objset->os_secondary_cache;
+	if (dn->dn_objset->os_secondary_cache == ZFS_CACHE_ALL ||
+	    (dn->dn_objset->os_secondary_cache == ZFS_CACHE_METADATA &&
+	    (level > 0 ||
+	    DMU_OT_IS_METADATA(dn->dn_handle->dnh_dnode->dn_type)))) {
+		if (l2arc_exclude_special == 0)
+			return (B_TRUE);

-	if (bp != NULL && !BP_IS_HOLE(bp)) {
+		if (bp == NULL || BP_IS_HOLE(bp))
+			return (B_FALSE);
 		uint64_t vdev = DVA_GET_VDEV(bp->blk_dva);
 		vdev_t *rvd = dn->dn_objset->os_spa->spa_root_vdev;
+		vdev_t *vd = NULL;

 		if (vdev < rvd->vdev_children)
 			vd = rvd->vdev_child[vdev];

-		if (cache == ZFS_CACHE_ALL || ((level > 0 ||
-		    DMU_OT_IS_METADATA(dn->dn_handle->dnh_dnode->dn_type)) &&
-		    cache == ZFS_CACHE_METADATA)) {
-			if (vd == NULL)
-				return (B_TRUE);
+		if (vd == NULL)
+			return (B_TRUE);

-			if ((vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
-			    vd->vdev_alloc_bias != VDEV_BIAS_DEDUP) ||
-			    l2arc_exclude_special == 0)
-				return (B_TRUE);
-		}
+		if (vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
+		    vd->vdev_alloc_bias != VDEV_BIAS_DEDUP)
+			return (B_TRUE);
 	}
-
 	return (B_FALSE);
 }

@@ -1483,8 +1483,8 @@ dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, uint32_t flags)

 	ASSERT(MUTEX_HELD(&db->db_mtx));

-	if (!os->os_encrypted || os->os_raw_receive ||
-	    (flags & DB_RF_NO_DECRYPT) != 0)
+	if ((flags & DB_RF_NO_DECRYPT) != 0 ||
+	    !os->os_encrypted || os->os_raw_receive)
 		return (0);

 	DB_DNODE_ENTER(db);
@@ -1719,8 +1719,6 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)

 	mutex_enter(&db->db_mtx);
 	if (db->db_state == DB_CACHED) {
-		spa_t *spa = dn->dn_objset->os_spa;
-
 		/*
 		 * Ensure that this block's dnode has been decrypted if
 		 * the caller has requested decrypted data.
@@ -1739,6 +1737,7 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
 		    (arc_is_encrypted(db->db_buf) ||
 		    arc_is_unauthenticated(db->db_buf) ||
 		    arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) {
+			spa_t *spa = dn->dn_objset->os_spa;
 			zbookmark_phys_t zb;

 			SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset),
@@ -1755,13 +1754,13 @@ dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
 		DB_DNODE_EXIT(db);
 		DBUF_STAT_BUMP(hash_hits);
 	} else if (db->db_state == DB_UNCACHED) {
-		spa_t *spa = dn->dn_objset->os_spa;
 		boolean_t need_wait = B_FALSE;

 		db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);

 		if (zio == NULL &&
 		    db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) {
+			spa_t *spa = dn->dn_objset->os_spa;
 			zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
 			need_wait = B_TRUE;
 		}
@@ -2100,18 +2100,18 @@ dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
 }

 /*
- * This function is only called from zfs_holey_common() for zpl_llseek()
- * in order to determine the location of holes.  In order to accurately
- * report holes all dirty data must be synced to disk.  This causes extremely
- * poor performance when seeking for holes in a dirty file.  As a compromise,
- * only provide hole data when the dnode is clean.  When a dnode is dirty
- * report the dnode as having no holes which is always a safe thing to do.
+ * Reports the location of data and holes in an object.  In order to
+ * accurately report holes all dirty data must be synced to disk.  This
+ * causes extremely poor performance when seeking for holes in a dirty file.
+ * As a compromise, only provide hole data when the dnode is clean.  When
+ * a dnode is dirty report the dnode as having no holes by returning EBUSY
+ * which is always safe to do.
 */
 int
 dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
 {
 	dnode_t *dn;
-	int err;
+	int restarted = 0, err;

 restart:
 	err = dnode_hold(os, object, FTAG, &dn);
@@ -2123,19 +2123,23 @@ restart:
 	if (dnode_is_dirty(dn)) {
 		/*
 		 * If the zfs_dmu_offset_next_sync module option is enabled
-		 * then strict hole reporting has been requested.  Dirty
-		 * dnodes must be synced to disk to accurately report all
-		 * holes.  When disabled dirty dnodes are reported to not
-		 * have any holes which is always safe.
+		 * then hole reporting has been requested.  Dirty dnodes
+		 * must be synced to disk to accurately report holes.
 		 *
-		 * When called by zfs_holey_common() the zp->z_rangelock
-		 * is held to prevent zfs_write() and mmap writeback from
-		 * re-dirtying the dnode after txg_wait_synced().
+		 * Provided a RL_READER rangelock spanning 0-UINT64_MAX is
+		 * held by the caller only a single restart will be required.
+		 * We tolerate callers which do not hold the rangelock by
+		 * returning EBUSY and not reporting holes after one restart.
 		 */
 		if (zfs_dmu_offset_next_sync) {
 			rw_exit(&dn->dn_struct_rwlock);
 			dnode_rele(dn, FTAG);
+
+			if (restarted)
+				return (SET_ERROR(EBUSY));
+
 			txg_wait_synced(dmu_objset_pool(os), 0);
+			restarted = 1;
 			goto restart;
 		}

@@ -419,28 +419,28 @@ dnode_multilist_index_func(multilist_t *ml, void *obj)
 static inline boolean_t
 dmu_os_is_l2cacheable(objset_t *os)
 {
-	vdev_t *vd = NULL;
-	zfs_cache_type_t cache = os->os_secondary_cache;
-	blkptr_t *bp = os->os_rootbp;
+	if (os->os_secondary_cache == ZFS_CACHE_ALL ||
+	    os->os_secondary_cache == ZFS_CACHE_METADATA) {
+		if (l2arc_exclude_special == 0)
+			return (B_TRUE);

-	if (bp != NULL && !BP_IS_HOLE(bp)) {
+		blkptr_t *bp = os->os_rootbp;
+		if (bp == NULL || BP_IS_HOLE(bp))
+			return (B_FALSE);
 		uint64_t vdev = DVA_GET_VDEV(bp->blk_dva);
 		vdev_t *rvd = os->os_spa->spa_root_vdev;
+		vdev_t *vd = NULL;

 		if (vdev < rvd->vdev_children)
 			vd = rvd->vdev_child[vdev];

-		if (cache == ZFS_CACHE_ALL || cache == ZFS_CACHE_METADATA) {
-			if (vd == NULL)
-				return (B_TRUE);
+		if (vd == NULL)
+			return (B_TRUE);

-			if ((vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
-			    vd->vdev_alloc_bias != VDEV_BIAS_DEDUP) ||
-			    l2arc_exclude_special == 0)
-				return (B_TRUE);
-		}
+		if (vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
+		    vd->vdev_alloc_bias != VDEV_BIAS_DEDUP)
+			return (B_TRUE);
 	}
-
 	return (B_FALSE);
 }

@@ -1301,6 +1301,7 @@ dmu_objset_create_sync(void *arg, dmu_tx_t *tx)
 			ASSERT3P(ds->ds_key_mapping, !=, NULL);
 			key_mapping_rele(spa, ds->ds_key_mapping, ds);
 			dsl_dataset_sync_done(ds, tx);
+			dmu_buf_rele(ds->ds_dbuf, ds);
 		}

 		mutex_enter(&ds->ds_lock);
@@ -2408,6 +2409,13 @@ dmu_objset_id_quota_upgrade_cb(objset_t *os)
 	    dmu_objset_userobjspace_present(os))
 		return (SET_ERROR(ENOTSUP));

+	if (dmu_objset_userobjused_enabled(os))
+		dmu_objset_ds(os)->ds_feature_activation[
+		    SPA_FEATURE_USEROBJ_ACCOUNTING] = (void *)B_TRUE;
+	if (dmu_objset_projectquota_enabled(os))
+		dmu_objset_ds(os)->ds_feature_activation[
+		    SPA_FEATURE_PROJECT_QUOTA] = (void *)B_TRUE;
+
 	err = dmu_objset_space_upgrade(os);
 	if (err)
 		return (err);
@@ -493,6 +493,7 @@ dmu_dump_write(dmu_send_cookie_t *dscp, dmu_object_type_t type, uint64_t object,
 	    (bp != NULL ? BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF &&
 	    io_compressed : lsize != psize);
 	if (raw || compressed) {
+		ASSERT(bp != NULL);
 		ASSERT(raw || dscp->dsc_featureflags &
 		    DMU_BACKUP_FEATURE_COMPRESSED);
 		ASSERT(!BP_IS_EMBEDDED(bp));
@@ -28,6 +28,7 @@
 */

 #include <sys/zfs_context.h>
+#include <sys/arc_impl.h>
 #include <sys/dnode.h>
 #include <sys/dmu_objset.h>
 #include <sys/dmu_zfetch.h>
@@ -65,13 +66,15 @@ typedef struct zfetch_stats {
 	kstat_named_t zfetchstat_misses;
 	kstat_named_t zfetchstat_max_streams;
 	kstat_named_t zfetchstat_io_issued;
+	kstat_named_t zfetchstat_io_active;
 } zfetch_stats_t;

 static zfetch_stats_t zfetch_stats = {
 	{ "hits",			KSTAT_DATA_UINT64 },
 	{ "misses",			KSTAT_DATA_UINT64 },
 	{ "max_streams",		KSTAT_DATA_UINT64 },
-	{ "io_issued",		KSTAT_DATA_UINT64 },
+	{ "io_issued",			KSTAT_DATA_UINT64 },
+	{ "io_active",			KSTAT_DATA_UINT64 },
 };

 struct {
@@ -79,6 +82,7 @@ struct {
 	wmsum_t zfetchstat_misses;
 	wmsum_t zfetchstat_max_streams;
 	wmsum_t zfetchstat_io_issued;
+	aggsum_t zfetchstat_io_active;
 } zfetch_sums;

 #define	ZFETCHSTAT_BUMP(stat)					\
@@ -104,6 +108,8 @@ zfetch_kstats_update(kstat_t *ksp, int rw)
 	    wmsum_value(&zfetch_sums.zfetchstat_max_streams);
 	zs->zfetchstat_io_issued.value.ui64 =
 	    wmsum_value(&zfetch_sums.zfetchstat_io_issued);
+	zs->zfetchstat_io_active.value.ui64 =
+	    aggsum_value(&zfetch_sums.zfetchstat_io_active);
 	return (0);
 }

@@ -114,6 +120,7 @@ zfetch_init(void)
 	wmsum_init(&zfetch_sums.zfetchstat_misses, 0);
 	wmsum_init(&zfetch_sums.zfetchstat_max_streams, 0);
 	wmsum_init(&zfetch_sums.zfetchstat_io_issued, 0);
+	aggsum_init(&zfetch_sums.zfetchstat_io_active, 0);

 	zfetch_ksp = kstat_create("zfs", 0, "zfetchstats", "misc",
 	    KSTAT_TYPE_NAMED, sizeof (zfetch_stats) / sizeof (kstat_named_t),
@@ -138,6 +145,8 @@ zfetch_fini(void)
 	wmsum_fini(&zfetch_sums.zfetchstat_misses);
 	wmsum_fini(&zfetch_sums.zfetchstat_max_streams);
 	wmsum_fini(&zfetch_sums.zfetchstat_io_issued);
+	ASSERT0(aggsum_value(&zfetch_sums.zfetchstat_io_active));
+	aggsum_fini(&zfetch_sums.zfetchstat_io_active);
 }

 /*
@@ -294,6 +303,7 @@ dmu_zfetch_done(void *arg, uint64_t level, uint64_t blkid, boolean_t io_issued)
 		zs->zs_more = B_TRUE;
 	if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)
 		dmu_zfetch_stream_fini(zs);
+	aggsum_add(&zfetch_sums.zfetchstat_io_active, -1);
 }

 /*
@@ -407,20 +417,28 @@ dmu_zfetch_prepare(zfetch_t *zf, uint64_t blkid, uint64_t nblks,
 	 * Start prefetch from the demand access size (nblks).  Double the
 	 * distance every access up to zfetch_min_distance.  After that only
 	 * if needed increase the distance by 1/8 up to zfetch_max_distance.
+	 *
+	 * Don't double the distance beyond single block if we have more
+	 * than ~6% of ARC held by active prefetches.  It should help with
+	 * getting out of RAM on some badly mispredicted read patterns.
 	 */
-	unsigned int nbytes = nblks << zf->zf_dnode->dn_datablkshift;
+	unsigned int dbs = zf->zf_dnode->dn_datablkshift;
+	unsigned int nbytes = nblks << dbs;
 	unsigned int pf_nblks;
 	if (fetch_data) {
 		if (unlikely(zs->zs_pf_dist < nbytes))
 			zs->zs_pf_dist = nbytes;
-		else if (zs->zs_pf_dist < zfetch_min_distance)
+		else if (zs->zs_pf_dist < zfetch_min_distance &&
+		    (zs->zs_pf_dist < (1 << dbs) ||
+		    aggsum_compare(&zfetch_sums.zfetchstat_io_active,
+		    arc_c_max >> (4 + dbs)) < 0))
 			zs->zs_pf_dist *= 2;
 		else if (zs->zs_more)
 			zs->zs_pf_dist += zs->zs_pf_dist / 8;
 		zs->zs_more = B_FALSE;
 		if (zs->zs_pf_dist > zfetch_max_distance)
 			zs->zs_pf_dist = zfetch_max_distance;
-		pf_nblks = zs->zs_pf_dist >> zf->zf_dnode->dn_datablkshift;
+		pf_nblks = zs->zs_pf_dist >> dbs;
 	} else {
 		pf_nblks = 0;
 	}
@@ -439,7 +457,7 @@ dmu_zfetch_prepare(zfetch_t *zf, uint64_t blkid, uint64_t nblks,
 		zs->zs_ipf_dist *= 2;
 	if (zs->zs_ipf_dist > zfetch_max_idistance)
 		zs->zs_ipf_dist = zfetch_max_idistance;
-	pf_nblks = zs->zs_ipf_dist >> zf->zf_dnode->dn_datablkshift;
+	pf_nblks = zs->zs_ipf_dist >> dbs;
 	if (zs->zs_ipf_start < zs->zs_pf_end)
 		zs->zs_ipf_start = zs->zs_pf_end;
 	if (zs->zs_ipf_end < zs->zs_pf_end + pf_nblks)
@@ -509,6 +527,7 @@ dmu_zfetch_run(zstream_t *zs, boolean_t missed, boolean_t have_lock)
 			dmu_zfetch_stream_fini(zs);
 		return;
 	}
+	aggsum_add(&zfetch_sums.zfetchstat_io_active, issued);

 	if (!have_lock)
 		rw_enter(&zf->zf_dnode->dn_struct_rwlock, RW_READER);
@@ -71,6 +71,8 @@ dnode_stats_t dnode_stats = {
 	{ "dnode_move_active",			KSTAT_DATA_UINT64 },
 };

+dnode_sums_t dnode_sums;
+
 static kstat_t *dnode_ksp;
 static kmem_cache_t *dnode_cache;

@@ -225,6 +227,72 @@ dnode_dest(void *arg, void *unused)
 	avl_destroy(&dn->dn_dbufs);
 }

+static int
+dnode_kstats_update(kstat_t *ksp, int rw)
+{
+	dnode_stats_t *ds = ksp->ks_data;
+
+	if (rw == KSTAT_WRITE)
+		return (EACCES);
+	ds->dnode_hold_dbuf_hold.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_dbuf_hold);
+	ds->dnode_hold_dbuf_read.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_dbuf_read);
+	ds->dnode_hold_alloc_hits.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_alloc_hits);
+	ds->dnode_hold_alloc_misses.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_alloc_misses);
+	ds->dnode_hold_alloc_interior.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_alloc_interior);
+	ds->dnode_hold_alloc_lock_retry.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_alloc_lock_retry);
+	ds->dnode_hold_alloc_lock_misses.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_alloc_lock_misses);
+	ds->dnode_hold_alloc_type_none.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_alloc_type_none);
+	ds->dnode_hold_free_hits.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_free_hits);
+	ds->dnode_hold_free_misses.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_free_misses);
+	ds->dnode_hold_free_lock_misses.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_free_lock_misses);
+	ds->dnode_hold_free_lock_retry.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_free_lock_retry);
+	ds->dnode_hold_free_refcount.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_free_refcount);
+	ds->dnode_hold_free_overflow.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_hold_free_overflow);
+	ds->dnode_free_interior_lock_retry.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_free_interior_lock_retry);
+	ds->dnode_allocate.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_allocate);
+	ds->dnode_reallocate.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_reallocate);
+	ds->dnode_buf_evict.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_buf_evict);
+	ds->dnode_alloc_next_chunk.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_alloc_next_chunk);
+	ds->dnode_alloc_race.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_alloc_race);
+	ds->dnode_alloc_next_block.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_alloc_next_block);
+	ds->dnode_move_invalid.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_move_invalid);
+	ds->dnode_move_recheck1.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_move_recheck1);
+	ds->dnode_move_recheck2.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_move_recheck2);
+	ds->dnode_move_special.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_move_special);
+	ds->dnode_move_handle.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_move_handle);
+	ds->dnode_move_rwlock.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_move_rwlock);
+	ds->dnode_move_active.value.ui64 =
+	    wmsum_value(&dnode_sums.dnode_move_active);
+	return (0);
+}
+
 void
 dnode_init(void)
 {
@@ -233,11 +301,41 @@ dnode_init(void)
 	    0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0);
 	kmem_cache_set_move(dnode_cache, dnode_move);

+	wmsum_init(&dnode_sums.dnode_hold_dbuf_hold, 0);
+	wmsum_init(&dnode_sums.dnode_hold_dbuf_read, 0);
+	wmsum_init(&dnode_sums.dnode_hold_alloc_hits, 0);
+	wmsum_init(&dnode_sums.dnode_hold_alloc_misses, 0);
+	wmsum_init(&dnode_sums.dnode_hold_alloc_interior, 0);
+	wmsum_init(&dnode_sums.dnode_hold_alloc_lock_retry, 0);
+	wmsum_init(&dnode_sums.dnode_hold_alloc_lock_misses, 0);
+	wmsum_init(&dnode_sums.dnode_hold_alloc_type_none, 0);
+	wmsum_init(&dnode_sums.dnode_hold_free_hits, 0);
+	wmsum_init(&dnode_sums.dnode_hold_free_misses, 0);
+	wmsum_init(&dnode_sums.dnode_hold_free_lock_misses, 0);
+	wmsum_init(&dnode_sums.dnode_hold_free_lock_retry, 0);
+	wmsum_init(&dnode_sums.dnode_hold_free_refcount, 0);
+	wmsum_init(&dnode_sums.dnode_hold_free_overflow, 0);
+	wmsum_init(&dnode_sums.dnode_free_interior_lock_retry, 0);
+	wmsum_init(&dnode_sums.dnode_allocate, 0);
+	wmsum_init(&dnode_sums.dnode_reallocate, 0);
+	wmsum_init(&dnode_sums.dnode_buf_evict, 0);
+	wmsum_init(&dnode_sums.dnode_alloc_next_chunk, 0);
+	wmsum_init(&dnode_sums.dnode_alloc_race, 0);
+	wmsum_init(&dnode_sums.dnode_alloc_next_block, 0);
+	wmsum_init(&dnode_sums.dnode_move_invalid, 0);
+	wmsum_init(&dnode_sums.dnode_move_recheck1, 0);
+	wmsum_init(&dnode_sums.dnode_move_recheck2, 0);
+	wmsum_init(&dnode_sums.dnode_move_special, 0);
+	wmsum_init(&dnode_sums.dnode_move_handle, 0);
+	wmsum_init(&dnode_sums.dnode_move_rwlock, 0);
+	wmsum_init(&dnode_sums.dnode_move_active, 0);
+
 	dnode_ksp = kstat_create("zfs", 0, "dnodestats", "misc",
 	    KSTAT_TYPE_NAMED, sizeof (dnode_stats) / sizeof (kstat_named_t),
 	    KSTAT_FLAG_VIRTUAL);
 	if (dnode_ksp != NULL) {
 		dnode_ksp->ks_data = &dnode_stats;
+		dnode_ksp->ks_update = dnode_kstats_update;
 		kstat_install(dnode_ksp);
 	}
 }
@@ -250,6 +348,35 @@ dnode_fini(void)
 		dnode_ksp = NULL;
 	}

+	wmsum_fini(&dnode_sums.dnode_hold_dbuf_hold);
+	wmsum_fini(&dnode_sums.dnode_hold_dbuf_read);
+	wmsum_fini(&dnode_sums.dnode_hold_alloc_hits);
+	wmsum_fini(&dnode_sums.dnode_hold_alloc_misses);
+	wmsum_fini(&dnode_sums.dnode_hold_alloc_interior);
+	wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_retry);
+	wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_misses);
+	wmsum_fini(&dnode_sums.dnode_hold_alloc_type_none);
+	wmsum_fini(&dnode_sums.dnode_hold_free_hits);
+	wmsum_fini(&dnode_sums.dnode_hold_free_misses);
+	wmsum_fini(&dnode_sums.dnode_hold_free_lock_misses);
+	wmsum_fini(&dnode_sums.dnode_hold_free_lock_retry);
+	wmsum_fini(&dnode_sums.dnode_hold_free_refcount);
+	wmsum_fini(&dnode_sums.dnode_hold_free_overflow);
+	wmsum_fini(&dnode_sums.dnode_free_interior_lock_retry);
+	wmsum_fini(&dnode_sums.dnode_allocate);
+	wmsum_fini(&dnode_sums.dnode_reallocate);
+	wmsum_fini(&dnode_sums.dnode_buf_evict);
+	wmsum_fini(&dnode_sums.dnode_alloc_next_chunk);
+	wmsum_fini(&dnode_sums.dnode_alloc_race);
+	wmsum_fini(&dnode_sums.dnode_alloc_next_block);
+	wmsum_fini(&dnode_sums.dnode_move_invalid);
+	wmsum_fini(&dnode_sums.dnode_move_recheck1);
+	wmsum_fini(&dnode_sums.dnode_move_recheck2);
+	wmsum_fini(&dnode_sums.dnode_move_special);
+	wmsum_fini(&dnode_sums.dnode_move_handle);
+	wmsum_fini(&dnode_sums.dnode_move_rwlock);
+	wmsum_fini(&dnode_sums.dnode_move_active);
+
 	kmem_cache_destroy(dnode_cache);
 	dnode_cache = NULL;
 }
@@ -2298,19 +2425,11 @@ dnode_spill_freed(dnode_t *dn)
 uint64_t
 dnode_block_freed(dnode_t *dn, uint64_t blkid)
 {
-	void *dp = spa_get_dsl(dn->dn_objset->os_spa);
 	int i;

 	if (blkid == DMU_BONUS_BLKID)
 		return (FALSE);

-	/*
-	 * If we're in the process of opening the pool, dp will not be
-	 * set yet, but there shouldn't be anything dirty.
-	 */
-	if (dp == NULL)
-		return (FALSE);
-
 	if (dn->dn_free_txg)
 		return (TRUE);

@@ -2595,3 +2714,8 @@ EXPORT_SYMBOL(dnode_free_range);
 EXPORT_SYMBOL(dnode_evict_dbufs);
 EXPORT_SYMBOL(dnode_evict_bonus);
 #endif
+
+ZFS_MODULE_PARAM(zfs, zfs_, default_bs, INT, ZMOD_RW,
+	"Default dnode block shift");
+ZFS_MODULE_PARAM(zfs, zfs_, default_ibs, INT, ZMOD_RW,
+	"Default dnode indirect block shift");
@@ -2120,9 +2120,6 @@ dsl_crypto_recv_raw_objset_sync(dsl_dataset_t *ds, dmu_objset_type_t ostype,
 		zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
 		dsl_dataset_sync(ds, zio, tx);
 		VERIFY0(zio_wait(zio));
-
-		/* dsl_dataset_sync_done will drop this reference. */
-		dmu_buf_add_ref(ds->ds_dbuf, ds);
 		dsl_dataset_sync_done(ds, tx);
 	}
 }
@@ -1259,9 +1259,6 @@ dsl_dataset_zero_zil(dsl_dataset_t *ds, dmu_tx_t *tx)
 		zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
 		dsl_dataset_sync(ds, zio, tx);
 		VERIFY0(zio_wait(zio));
-
-		/* dsl_dataset_sync_done will drop this reference. */
-		dmu_buf_add_ref(ds->ds_dbuf, ds);
 		dsl_dataset_sync_done(ds, tx);
 	}
 }
@@ -1747,25 +1744,6 @@ dsl_dataset_snapshot_sync_impl(dsl_dataset_t *ds, const char *snapname,
 		}
 	}

-	/*
-	 * We are not allowed to dirty a filesystem when done receiving
-	 * a snapshot. In this case some flags such as SPA_FEATURE_LARGE_BLOCKS
-	 * will not be set and a subsequent encrypted raw send will fail. Hence
-	 * activate this feature if needed here. This needs to happen only in
-	 * syncing context.
-	 */
-	if (dmu_tx_is_syncing(tx)) {
-		for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
-			if (zfeature_active(f, ds->ds_feature_activation[f]) &&
-			    !(zfeature_active(f, ds->ds_feature[f]))) {
-				dsl_dataset_activate_feature(dsobj, f,
-				    ds->ds_feature_activation[f], tx);
-				ds->ds_feature[f] =
-				    ds->ds_feature_activation[f];
-			}
-		}
-	}
-
 	ASSERT3U(ds->ds_prev != 0, ==,
 	    dsl_dataset_phys(ds)->ds_prev_snap_obj != 0);
 	if (ds->ds_prev) {
@@ -2108,16 +2086,6 @@ dsl_dataset_sync(dsl_dataset_t *ds, zio_t *zio, dmu_tx_t *tx)
 	}

 	dmu_objset_sync(ds->ds_objset, zio, tx);
-
-	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
-		if (zfeature_active(f, ds->ds_feature_activation[f])) {
-			if (zfeature_active(f, ds->ds_feature[f]))
-				continue;
-			dsl_dataset_activate_feature(ds->ds_object, f,
-			    ds->ds_feature_activation[f], tx);
-			ds->ds_feature[f] = ds->ds_feature_activation[f];
-		}
-	}
 }

 /*
@@ -2289,9 +2257,18 @@ dsl_dataset_sync_done(dsl_dataset_t *ds, dmu_tx_t *tx)
 	else
 		ASSERT0(os->os_next_write_raw[tx->tx_txg & TXG_MASK]);

-	ASSERT(!dmu_objset_is_dirty(os, dmu_tx_get_txg(tx)));
+	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
+		if (zfeature_active(f,
+		    ds->ds_feature_activation[f])) {
+			if (zfeature_active(f, ds->ds_feature[f]))
+				continue;
+			dsl_dataset_activate_feature(ds->ds_object, f,
+			    ds->ds_feature_activation[f], tx);
+			ds->ds_feature[f] = ds->ds_feature_activation[f];
+		}
+	}

-	dmu_buf_rele(ds->ds_dbuf, ds);
+	ASSERT(!dmu_objset_is_dirty(os, dmu_tx_get_txg(tx)));
 }

 int
@@ -438,6 +438,18 @@ dle_enqueue_subobj(dsl_deadlist_t *dl, dsl_deadlist_entry_t *dle,
 	}
 }

+/*
+ * Prefetch metadata required for dle_enqueue_subobj().
+ */
+static void
+dle_prefetch_subobj(dsl_deadlist_t *dl, dsl_deadlist_entry_t *dle,
+    uint64_t obj)
+{
+	if (dle->dle_bpobj.bpo_object !=
+	    dmu_objset_pool(dl->dl_os)->dp_empty_bpobj)
+		bpobj_prefetch_subobj(&dle->dle_bpobj, obj);
+}
+
 void
 dsl_deadlist_insert(dsl_deadlist_t *dl, const blkptr_t *bp, boolean_t bp_freed,
    dmu_tx_t *tx)
@@ -809,6 +821,27 @@ dsl_deadlist_insert_bpobj(dsl_deadlist_t *dl, uint64_t obj, uint64_t birth,
 	dle_enqueue_subobj(dl, dle, obj, tx);
 }

+/*
+ * Prefetch metadata required for dsl_deadlist_insert_bpobj().
+ */
+static void
+dsl_deadlist_prefetch_bpobj(dsl_deadlist_t *dl, uint64_t obj, uint64_t birth)
+{
+	dsl_deadlist_entry_t dle_tofind;
+	dsl_deadlist_entry_t *dle;
+	avl_index_t where;
+
+	ASSERT(MUTEX_HELD(&dl->dl_lock));
+
+	dsl_deadlist_load_tree(dl);
+
+	dle_tofind.dle_mintxg = birth;
+	dle = avl_find(&dl->dl_tree, &dle_tofind, &where);
+	if (dle == NULL)
+		dle = avl_nearest(&dl->dl_tree, where, AVL_BEFORE);
+	dle_prefetch_subobj(dl, dle, obj);
+}
+
 static int
 dsl_deadlist_insert_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
    dmu_tx_t *tx)
@@ -825,12 +858,12 @@ dsl_deadlist_insert_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed,
 void
 dsl_deadlist_merge(dsl_deadlist_t *dl, uint64_t obj, dmu_tx_t *tx)
 {
-	zap_cursor_t zc;
-	zap_attribute_t za;
+	zap_cursor_t zc, pzc;
+	zap_attribute_t za, pza;
 	dmu_buf_t *bonus;
 	dsl_deadlist_phys_t *dlp;
 	dmu_object_info_t doi;
-	int error;
+	int error, perror, i;

 	VERIFY0(dmu_object_info(dl->dl_os, obj, &doi));
 	if (doi.doi_type == DMU_OT_BPOBJ) {
@@ -842,15 +875,32 @@ dsl_deadlist_merge(dsl_deadlist_t *dl, uint64_t obj, dmu_tx_t *tx)
 	}

 	mutex_enter(&dl->dl_lock);
+	/*
+	 * Prefetch up to 128 deadlists first and then more as we progress.
+	 * The limit is a balance between ARC use and diminishing returns.
+	 */
+	for (zap_cursor_init(&pzc, dl->dl_os, obj), i = 0;
+	    (perror = zap_cursor_retrieve(&pzc, &pza)) == 0 && i < 128;
+	    zap_cursor_advance(&pzc), i++) {
+		dsl_deadlist_prefetch_bpobj(dl, pza.za_first_integer,
+		    zfs_strtonum(pza.za_name, NULL));
+	}
 	for (zap_cursor_init(&zc, dl->dl_os, obj);
 	    (error = zap_cursor_retrieve(&zc, &za)) == 0;
 	    zap_cursor_advance(&zc)) {
 		uint64_t mintxg = zfs_strtonum(za.za_name, NULL);
 		dsl_deadlist_insert_bpobj(dl, za.za_first_integer, mintxg, tx);
 		VERIFY0(zap_remove_int(dl->dl_os, obj, mintxg, tx));
+		if (perror == 0) {
+			dsl_deadlist_prefetch_bpobj(dl, pza.za_first_integer,
+			    zfs_strtonum(pza.za_name, NULL));
+			zap_cursor_advance(&pzc);
+			perror = zap_cursor_retrieve(&pzc, &pza);
+		}
 	}
 	VERIFY3U(error, ==, ENOENT);
 	zap_cursor_fini(&zc);
+	zap_cursor_fini(&pzc);

 	VERIFY0(dmu_bonus_hold(dl->dl_os, obj, FTAG, &bonus));
 	dlp = bonus->db_data;
@@ -868,8 +918,9 @@ dsl_deadlist_move_bpobj(dsl_deadlist_t *dl, bpobj_t *bpo, uint64_t mintxg,
    dmu_tx_t *tx)
 {
 	dsl_deadlist_entry_t dle_tofind;
-	dsl_deadlist_entry_t *dle;
+	dsl_deadlist_entry_t *dle, *pdle;
 	avl_index_t where;
+	int i;

 	ASSERT(!dl->dl_oldfmt);

@@ -881,11 +932,23 @@ dsl_deadlist_move_bpobj(dsl_deadlist_t *dl, bpobj_t *bpo, uint64_t mintxg,
 	dle = avl_find(&dl->dl_tree, &dle_tofind, &where);
 	if (dle == NULL)
 		dle = avl_nearest(&dl->dl_tree, where, AVL_AFTER);
+	/*
+	 * Prefetch up to 128 deadlists first and then more as we progress.
+	 * The limit is a balance between ARC use and diminishing returns.
+	 */
+	for (pdle = dle, i = 0; pdle && i < 128; i++) {
+		bpobj_prefetch_subobj(bpo, pdle->dle_bpobj.bpo_object);
+		pdle = AVL_NEXT(&dl->dl_tree, pdle);
+	}
 	while (dle) {
 		uint64_t used, comp, uncomp;
 		dsl_deadlist_entry_t *dle_next;

 		bpobj_enqueue_subobj(bpo, dle->dle_bpobj.bpo_object, tx);
+		if (pdle) {
+			bpobj_prefetch_subobj(bpo, pdle->dle_bpobj.bpo_object);
+			pdle = AVL_NEXT(&dl->dl_tree, pdle);
+		}

 		VERIFY0(bpobj_space(&dle->dle_bpobj,
 		    &used, &comp, &uncomp));
@@ -786,6 +786,7 @@ dsl_pool_sync(dsl_pool_t *dp, uint64_t txg)
 		}

 		dsl_dataset_sync_done(ds, tx);
+		dmu_buf_rele(ds->ds_dbuf, ds);
 	}

 	while ((dd = txg_list_remove(&dp->dp_dirty_dirs, txg)) != NULL) {
@@ -2005,6 +2005,21 @@ dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb,
 		return;
 	}

+	/*
+	 * Check if this block contradicts any filesystem flags.
+	 */
+	spa_feature_t f = SPA_FEATURE_LARGE_BLOCKS;
+	if (BP_GET_LSIZE(bp) > SPA_OLD_MAXBLOCKSIZE)
+		ASSERT(dsl_dataset_feature_is_active(ds, f));
+
+	f = zio_checksum_to_feature(BP_GET_CHECKSUM(bp));
+	if (f != SPA_FEATURE_NONE)
+		ASSERT(dsl_dataset_feature_is_active(ds, f));
+
+	f = zio_compress_to_feature(BP_GET_COMPRESS(bp));
+	if (f != SPA_FEATURE_NONE)
+		ASSERT(dsl_dataset_feature_is_active(ds, f));
+
 	if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) {
 		scn->scn_lt_min_this_txg++;
 		return;
@@ -909,7 +909,16 @@ spa_change_guid(spa_t *spa)
 	    spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED);

 	if (error == 0) {
-		spa_write_cachefile(spa, B_FALSE, B_TRUE);
+		/*
+		 * Clear the kobj flag from all the vdevs to allow
+		 * vdev_cache_process_kobj_evt() to post events to all the
+		 * vdevs since GUID is updated.
+		 */
+		vdev_clear_kobj_evt(spa->spa_root_vdev);
+		for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
+			vdev_clear_kobj_evt(spa->spa_l2cache.sav_vdevs[i]);
+
+		spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);
 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID);
 	}

@@ -5192,7 +5201,7 @@ spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy,
 			 */
 			spa_unload(spa);
 			spa_deactivate(spa);
-			spa_write_cachefile(spa, B_TRUE, B_TRUE);
+			spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE);
 			spa_remove(spa);
 			if (locked)
 				mutex_exit(&spa_namespace_lock);
@@ -5327,13 +5336,15 @@ spa_add_spares(spa_t *spa, nvlist_t *config)
 		for (i = 0; i < nspares; i++) {
 			guid = fnvlist_lookup_uint64(spares[i],
 			    ZPOOL_CONFIG_GUID);
+			VERIFY0(nvlist_lookup_uint64_array(spares[i],
+			    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc));
 			if (spa_spare_exists(guid, &pool, NULL) &&
 			    pool != 0ULL) {
-				VERIFY0(nvlist_lookup_uint64_array(spares[i],
-				    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs,
-				    &vsc));
 				vs->vs_state = VDEV_STATE_CANT_OPEN;
 				vs->vs_aux = VDEV_AUX_SPARED;
+			} else {
+				vs->vs_state =
+				    spa->spa_spares.sav_vdevs[i]->vdev_state;
 			}
 		}
 	}
@@ -6012,7 +6023,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,

 	spa_spawn_aux_threads(spa);

-	spa_write_cachefile(spa, B_FALSE, B_TRUE);
+	spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);

 	/*
 	 * Don't count references from objsets that are already closed
@@ -6073,7 +6084,7 @@ spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
 		if (props != NULL)
 			spa_configfile_set(spa, props, B_FALSE);

-		spa_write_cachefile(spa, B_FALSE, B_TRUE);
+		spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE);
 		spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT);
 		zfs_dbgmsg("spa_import: verbatim import of %s", pool);
 		mutex_exit(&spa_namespace_lock);
@@ -6465,7 +6476,7 @@ export_spa:

 	if (new_state != POOL_STATE_UNINITIALIZED) {
 		if (!hardforce)
-			spa_write_cachefile(spa, B_TRUE, B_TRUE);
+			spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE);
 		spa_remove(spa);
 	} else {
 		/*
@@ -238,7 +238,8 @@ spa_config_write(spa_config_dirent_t *dp, nvlist_t *nvl)
 * would be required.
 */
 void
-spa_write_cachefile(spa_t *target, boolean_t removing, boolean_t postsysevent)
+spa_write_cachefile(spa_t *target, boolean_t removing, boolean_t postsysevent,
+    boolean_t postblkidevent)
 {
 	spa_config_dirent_t *dp, *tdp;
 	nvlist_t *nvl;
@@ -344,6 +345,18 @@ spa_write_cachefile(spa_t *target, boolean_t removing, boolean_t postsysevent)

 	if (postsysevent)
 		spa_event_notify(target, NULL, NULL, ESC_ZFS_CONFIG_SYNC);
+
+	/*
+	 * Post udev event to sync blkid information if the pool is created
+	 * or a new vdev is added to the pool.
+	 */
+	if ((target->spa_root_vdev) && postblkidevent) {
+		vdev_post_kobj_evt(target->spa_root_vdev);
+		for (int i = 0; i < target->spa_l2cache.sav_count; i++)
+			vdev_post_kobj_evt(target->spa_l2cache.sav_vdevs[i]);
+		for (int i = 0; i < target->spa_spares.sav_count; i++)
+			vdev_post_kobj_evt(target->spa_spares.sav_vdevs[i]);
+	}
 }

 /*
@@ -598,6 +611,7 @@ spa_config_update(spa_t *spa, int what)
 	 */
 	if (!spa->spa_is_root) {
 		spa_write_cachefile(spa, B_FALSE,
+		    what != SPA_CONFIG_UPDATE_POOL,
 		    what != SPA_CONFIG_UPDATE_POOL);
 	}

@@ -1291,7 +1291,7 @@ spa_vdev_config_exit(spa_t *spa, vdev_t *vd, uint64_t txg, int error, char *tag)
 	 * If the config changed, update the config cache.
 	 */
 	if (config_changed)
-		spa_write_cachefile(spa, B_FALSE, B_TRUE);
+		spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE);
 }

 /*
@@ -1386,7 +1386,7 @@ spa_vdev_state_exit(spa_t *spa, vdev_t *vd, int error)
 	 */
 	if (config_changed) {
 		mutex_enter(&spa_namespace_lock);
-		spa_write_cachefile(spa, B_FALSE, B_TRUE);
+		spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE);
 		mutex_exit(&spa_namespace_lock);
 	}

@@ -1938,6 +1938,14 @@ vdev_open(vdev_t *vd)

 	error = vd->vdev_ops->vdev_op_open(vd, &osize, &max_osize,
 	    &logical_ashift, &physical_ashift);
+
+	/* Keep the device in removed state if unplugged */
+	if (error == ENOENT && vd->vdev_removed) {
+		vdev_set_state(vd, B_TRUE, VDEV_STATE_REMOVED,
+		    VDEV_AUX_NONE);
+		return (error);
+	}
+
 	/*
 	 * Physical volume size should never be larger than its max size, unless
 	 * the disk has shrunk while we were reading it or the device is buggy
@@ -3156,6 +3164,34 @@ vdev_dtl_reassess(vdev_t *vd, uint64_t txg, uint64_t scrub_txg,
 	mutex_exit(&vd->vdev_dtl_lock);
 }

+/*
+ * Iterate over all the vdevs except spare, and post kobj events
+ */
+void
+vdev_post_kobj_evt(vdev_t *vd)
+{
+	if (vd->vdev_ops->vdev_op_kobj_evt_post &&
+	    vd->vdev_kobj_flag == B_FALSE) {
+		vd->vdev_kobj_flag = B_TRUE;
+		vd->vdev_ops->vdev_op_kobj_evt_post(vd);
+	}
+
+	for (int c = 0; c < vd->vdev_children; c++)
+		vdev_post_kobj_evt(vd->vdev_child[c]);
+}
+
+/*
+ * Iterate over all the vdevs except spare, and clear kobj events
+ */
+void
+vdev_clear_kobj_evt(vdev_t *vd)
+{
+	vd->vdev_kobj_flag = B_FALSE;
+
+	for (int c = 0; c < vd->vdev_children; c++)
+		vdev_clear_kobj_evt(vd->vdev_child[c]);
+}
+
 int
 vdev_dtl_load(vdev_t *vd)
 {
@@ -3936,6 +3972,29 @@ vdev_degrade(spa_t *spa, uint64_t guid, vdev_aux_t aux)
 	return (spa_vdev_state_exit(spa, vd, 0));
 }

+int
+vdev_remove_wanted(spa_t *spa, uint64_t guid)
+{
+	vdev_t *vd;
+
+	spa_vdev_state_enter(spa, SCL_NONE);
+
+	if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL)
+		return (spa_vdev_state_exit(spa, NULL, SET_ERROR(ENODEV)));
+
+	/*
+	 * If the vdev is already removed, then don't do anything.
+	 */
+	if (vd->vdev_removed)
+		return (spa_vdev_state_exit(spa, NULL, 0));
+
+	vd->vdev_remove_wanted = B_TRUE;
+	spa_async_request(spa, SPA_ASYNC_REMOVE);
+
+	return (spa_vdev_state_exit(spa, vd, 0));
+}
+
+
 /*
 * Online the given vdev.
 *
@@ -645,7 +645,7 @@ zap_entry_create(zap_leaf_t *l, zap_name_t *zn, uint32_t cd,
 * form of the name.  But all callers have one of these on hand anyway,
 * so might as well take advantage.  A cleaner but slower interface
 * would accept neither argument, and compute the normalized name as
- * needed (using zap_name_alloc(zap_entry_read_name(zeh))).
+ * needed (using zap_name_alloc_str(zap_entry_read_name(zeh))).
 */
 boolean_t
 zap_entry_normalization_conflict(zap_entry_handle_t *zeh, zap_name_t *zn,
@@ -666,7 +666,7 @@ zap_entry_normalization_conflict(zap_entry_handle_t *zeh, zap_name_t *zn,
 			continue;

 		if (zn == NULL) {
-			zn = zap_name_alloc(zap, name, MT_NORMALIZE);
+			zn = zap_name_alloc_str(zap, name, MT_NORMALIZE);
 			allocdzn = B_TRUE;
 		}
 		if (zap_leaf_array_match(zeh->zeh_leaf, zn,
@@ -33,7 +33,7 @@
 #include <sys/zap.h>
 #include <sys/zap_impl.h>
 #include <sys/zap_leaf.h>
-#include <sys/avl.h>
+#include <sys/btree.h>
 #include <sys/arc.h>
 #include <sys/dmu_objset.h>

@@ -92,7 +92,7 @@ zap_hash(zap_name_t *zn)
 			    wp++, i++) {
 				uint64_t word = *wp;

-				for (int j = 0; j < zn->zn_key_intlen; j++) {
+				for (int j = 0; j < 8; j++) {
 					h = (h >> 8) ^
 					    zfs_crc64_table[(h ^ word) & 0xFF];
 					word >>= NBBY;
@@ -162,18 +162,25 @@ zap_match(zap_name_t *zn, const char *matchname)
 	}
 }

+static zap_name_t *
+zap_name_alloc(zap_t *zap)
+{
+	zap_name_t *zn = kmem_alloc(sizeof (zap_name_t), KM_SLEEP);
+	zn->zn_zap = zap;
+	return (zn);
+}
+
 void
 zap_name_free(zap_name_t *zn)
 {
 	kmem_free(zn, sizeof (zap_name_t));
 }

-zap_name_t *
-zap_name_alloc(zap_t *zap, const char *key, matchtype_t mt)
+static int
+zap_name_init_str(zap_name_t *zn, const char *key, matchtype_t mt)
 {
-	zap_name_t *zn = kmem_alloc(sizeof (zap_name_t), KM_SLEEP);
+	zap_t *zap = zn->zn_zap;

-	zn->zn_zap = zap;
 	zn->zn_key_intlen = sizeof (*key);
 	zn->zn_key_orig = key;
 	zn->zn_key_orig_numints = strlen(zn->zn_key_orig) + 1;
@@ -194,17 +201,13 @@ zap_name_alloc(zap_t *zap, const char *key, matchtype_t mt)
 		 * what the hash is computed from.
 		 */
 		if (zap_normalize(zap, key, zn->zn_normbuf,
-		    zap->zap_normflags) != 0) {
-			zap_name_free(zn);
-			return (NULL);
-		}
+		    zap->zap_normflags) != 0)
+			return (SET_ERROR(ENOTSUP));
 		zn->zn_key_norm = zn->zn_normbuf;
 		zn->zn_key_norm_numints = strlen(zn->zn_key_norm) + 1;
 	} else {
-		if (mt != 0) {
-			zap_name_free(zn);
-			return (NULL);
-		}
+		if (mt != 0)
+			return (SET_ERROR(ENOTSUP));
 		zn->zn_key_norm = zn->zn_key_orig;
 		zn->zn_key_norm_numints = zn->zn_key_orig_numints;
 	}
@@ -217,13 +220,22 @@ zap_name_alloc(zap_t *zap, const char *key, matchtype_t mt)
 		 * what the matching is based on.  (Not the hash!)
 		 */
 		if (zap_normalize(zap, key, zn->zn_normbuf,
-		    zn->zn_normflags) != 0) {
-			zap_name_free(zn);
-			return (NULL);
-		}
+		    zn->zn_normflags) != 0)
+			return (SET_ERROR(ENOTSUP));
 		zn->zn_key_norm_numints = strlen(zn->zn_key_norm) + 1;
 	}

+	return (0);
+}
+
+zap_name_t *
+zap_name_alloc_str(zap_t *zap, const char *key, matchtype_t mt)
+{
+	zap_name_t *zn = zap_name_alloc(zap);
+	if (zap_name_init_str(zn, key, mt) != 0) {
+		zap_name_free(zn);
+		return (NULL);
+	}
 	return (zn);
 }

@@ -277,45 +289,46 @@ mze_compare(const void *arg1, const void *arg2)
 	const mzap_ent_t *mze1 = arg1;
 	const mzap_ent_t *mze2 = arg2;

-	int cmp = TREE_CMP(mze1->mze_hash, mze2->mze_hash);
-	if (likely(cmp))
-		return (cmp);
-
-	return (TREE_CMP(mze1->mze_cd, mze2->mze_cd));
+	return (TREE_CMP((uint64_t)(mze1->mze_hash) << 32 | mze1->mze_cd,
+	    (uint64_t)(mze2->mze_hash) << 32 | mze2->mze_cd));
 }

 static void
-mze_insert(zap_t *zap, int chunkid, uint64_t hash)
+mze_insert(zap_t *zap, uint16_t chunkid, uint64_t hash)
 {
+	mzap_ent_t mze;
+
 	ASSERT(zap->zap_ismicro);
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));

-	mzap_ent_t *mze = kmem_alloc(sizeof (mzap_ent_t), KM_SLEEP);
-	mze->mze_chunkid = chunkid;
-	mze->mze_hash = hash;
-	mze->mze_cd = MZE_PHYS(zap, mze)->mze_cd;
-	ASSERT(MZE_PHYS(zap, mze)->mze_name[0] != 0);
-	avl_add(&zap->zap_m.zap_avl, mze);
+	mze.mze_chunkid = chunkid;
+	ASSERT0(hash & 0xffffffff);
+	mze.mze_hash = hash >> 32;
+	ASSERT3U(MZE_PHYS(zap, &mze)->mze_cd, <=, 0xffff);
+	mze.mze_cd = (uint16_t)MZE_PHYS(zap, &mze)->mze_cd;
+	ASSERT(MZE_PHYS(zap, &mze)->mze_name[0] != 0);
+	zfs_btree_add(&zap->zap_m.zap_tree, &mze);
 }

 static mzap_ent_t *
-mze_find(zap_name_t *zn)
+mze_find(zap_name_t *zn, zfs_btree_index_t *idx)
 {
 	mzap_ent_t mze_tofind;
 	mzap_ent_t *mze;
-	avl_index_t idx;
-	avl_tree_t *avl = &zn->zn_zap->zap_m.zap_avl;
+	zfs_btree_t *tree = &zn->zn_zap->zap_m.zap_tree;

 	ASSERT(zn->zn_zap->zap_ismicro);
 	ASSERT(RW_LOCK_HELD(&zn->zn_zap->zap_rwlock));

-	mze_tofind.mze_hash = zn->zn_hash;
+	ASSERT0(zn->zn_hash & 0xffffffff);
+	mze_tofind.mze_hash = zn->zn_hash >> 32;
 	mze_tofind.mze_cd = 0;

-	mze = avl_find(avl, &mze_tofind, &idx);
+	mze = zfs_btree_find(tree, &mze_tofind, idx);
 	if (mze == NULL)
-		mze = avl_nearest(avl, idx, AVL_AFTER);
-	for (; mze && mze->mze_hash == zn->zn_hash; mze = AVL_NEXT(avl, mze)) {
+		mze = zfs_btree_next(tree, idx, idx);
+	for (; mze && mze->mze_hash == mze_tofind.mze_hash;
+	    mze = zfs_btree_next(tree, idx, idx)) {
 		ASSERT3U(mze->mze_cd, ==, MZE_PHYS(zn->zn_zap, mze)->mze_cd);
 		if (zap_match(zn, MZE_PHYS(zn->zn_zap, mze)->mze_name))
 			return (mze);
@@ -328,18 +341,21 @@ static uint32_t
 mze_find_unused_cd(zap_t *zap, uint64_t hash)
 {
 	mzap_ent_t mze_tofind;
-	avl_index_t idx;
-	avl_tree_t *avl = &zap->zap_m.zap_avl;
+	zfs_btree_index_t idx;
+	zfs_btree_t *tree = &zap->zap_m.zap_tree;

 	ASSERT(zap->zap_ismicro);
 	ASSERT(RW_LOCK_HELD(&zap->zap_rwlock));

+	ASSERT0(hash & 0xffffffff);
+	hash >>= 32;
 	mze_tofind.mze_hash = hash;
 	mze_tofind.mze_cd = 0;

 	uint32_t cd = 0;
-	for (mzap_ent_t *mze = avl_find(avl, &mze_tofind, &idx);
-	    mze && mze->mze_hash == hash; mze = AVL_NEXT(avl, mze)) {
+	for (mzap_ent_t *mze = zfs_btree_find(tree, &mze_tofind, &idx);
+	    mze && mze->mze_hash == hash;
+	    mze = zfs_btree_next(tree, &idx, &idx)) {
 		if (mze->mze_cd != cd)
 			break;
 		cd++;
@@ -364,16 +380,18 @@ mze_canfit_fzap_leaf(zap_name_t *zn, uint64_t hash)
 {
 	zap_t *zap = zn->zn_zap;
 	mzap_ent_t mze_tofind;
-	mzap_ent_t *mze;
-	avl_index_t idx;
-	avl_tree_t *avl = &zap->zap_m.zap_avl;
+	zfs_btree_index_t idx;
+	zfs_btree_t *tree = &zap->zap_m.zap_tree;
 	uint32_t mzap_ents = 0;

+	ASSERT0(hash & 0xffffffff);
+	hash >>= 32;
 	mze_tofind.mze_hash = hash;
 	mze_tofind.mze_cd = 0;

-	for (mze = avl_find(avl, &mze_tofind, &idx);
-	    mze && mze->mze_hash == hash; mze = AVL_NEXT(avl, mze)) {
+	for (mzap_ent_t *mze = zfs_btree_find(tree, &mze_tofind, &idx);
+	    mze && mze->mze_hash == hash;
+	    mze = zfs_btree_next(tree, &idx, &idx)) {
 		mzap_ents++;
 	}

@@ -383,25 +401,11 @@ mze_canfit_fzap_leaf(zap_name_t *zn, uint64_t hash)
 	return (ZAP_LEAF_NUMCHUNKS_DEF > (mzap_ents * MZAP_ENT_CHUNKS));
 }

-static void
-mze_remove(zap_t *zap, mzap_ent_t *mze)
-{
-	ASSERT(zap->zap_ismicro);
-	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
-
-	avl_remove(&zap->zap_m.zap_avl, mze);
-	kmem_free(mze, sizeof (mzap_ent_t));
-}
-
 static void
 mze_destroy(zap_t *zap)
 {
-	mzap_ent_t *mze;
-	void *avlcookie = NULL;
-
-	while ((mze = avl_destroy_nodes(&zap->zap_m.zap_avl, &avlcookie)))
-		kmem_free(mze, sizeof (mzap_ent_t));
-	avl_destroy(&zap->zap_m.zap_avl);
+	zfs_btree_clear(&zap->zap_m.zap_tree);
+	zfs_btree_destroy(&zap->zap_m.zap_tree);
 }

 static zap_t *
@@ -448,21 +452,26 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db)
 		zap->zap_salt = zap_m_phys(zap)->mz_salt;
 		zap->zap_normflags = zap_m_phys(zap)->mz_normflags;
 		zap->zap_m.zap_num_chunks = db->db_size / MZAP_ENT_LEN - 1;
-		avl_create(&zap->zap_m.zap_avl, mze_compare,
-		    sizeof (mzap_ent_t), offsetof(mzap_ent_t, mze_node));

-		for (int i = 0; i < zap->zap_m.zap_num_chunks; i++) {
+		/*
+		 * Reduce B-tree leaf from 4KB to 512 bytes to reduce memmove()
+		 * overhead on massive inserts below.  It still allows to store
+		 * 62 entries before we have to add 2KB B-tree core node.
+		 */
+		zfs_btree_create_custom(&zap->zap_m.zap_tree, mze_compare,
+		    sizeof (mzap_ent_t), 512);
+
+		zap_name_t *zn = zap_name_alloc(zap);
+		for (uint16_t i = 0; i < zap->zap_m.zap_num_chunks; i++) {
 			mzap_ent_phys_t *mze =
 			    &zap_m_phys(zap)->mz_chunk[i];
 			if (mze->mze_name[0]) {
-				zap_name_t *zn;
-
 				zap->zap_m.zap_num_entries++;
-				zn = zap_name_alloc(zap, mze->mze_name, 0);
+				zap_name_init_str(zn, mze->mze_name, 0);
 				mze_insert(zap, i, zn->zn_hash);
-				zap_name_free(zn);
 			}
 		}
+		zap_name_free(zn);
 	} else {
 		zap->zap_salt = zap_f_phys(zap)->zap_salt;
 		zap->zap_normflags = zap_f_phys(zap)->zap_normflags;
@@ -655,24 +664,25 @@ mzap_upgrade(zap_t **zapp, void *tag, dmu_tx_t *tx, zap_flags_t flags)

 	dprintf("upgrading obj=%llu with %u chunks\n",
 	    (u_longlong_t)zap->zap_object, nchunks);
-	/* XXX destroy the avl later, so we can use the stored hash value */
+	/* XXX destroy the tree later, so we can use the stored hash value */
 	mze_destroy(zap);

 	fzap_upgrade(zap, tx, flags);

+	zap_name_t *zn = zap_name_alloc(zap);
 	for (int i = 0; i < nchunks; i++) {
 		mzap_ent_phys_t *mze = &mzp->mz_chunk[i];
 		if (mze->mze_name[0] == 0)
 			continue;
 		dprintf("adding %s=%llu\n",
 		    mze->mze_name, (u_longlong_t)mze->mze_value);
-		zap_name_t *zn = zap_name_alloc(zap, mze->mze_name, 0);
+		zap_name_init_str(zn, mze->mze_name, 0);
 		/* If we fail here, we would end up losing entries */
 		VERIFY0(fzap_add_cd(zn, 8, 1, &mze->mze_value, mze->mze_cd,
 		    tag, tx));
 		zap = zn->zn_zap;	/* fzap_add_cd() may change zap */
-		zap_name_free(zn);
 	}
+	zap_name_free(zn);
 	vmem_free(mzp, sz);
 	*zapp = zap;
 	return (0);
@@ -914,22 +924,23 @@ zap_count(objset_t *os, uint64_t zapobj, uint64_t *count)
 * See also the comment above zap_entry_normalization_conflict().
 */
 static boolean_t
-mzap_normalization_conflict(zap_t *zap, zap_name_t *zn, mzap_ent_t *mze)
+mzap_normalization_conflict(zap_t *zap, zap_name_t *zn, mzap_ent_t *mze,
+    zfs_btree_index_t *idx)
 {
-	int direction = AVL_BEFORE;
 	boolean_t allocdzn = B_FALSE;
+	mzap_ent_t *other;
+	zfs_btree_index_t oidx;

 	if (zap->zap_normflags == 0)
 		return (B_FALSE);

-again:
-	for (mzap_ent_t *other = avl_walk(&zap->zap_m.zap_avl, mze, direction);
+	for (other = zfs_btree_prev(&zap->zap_m.zap_tree, idx, &oidx);
 	    other && other->mze_hash == mze->mze_hash;
-	    other = avl_walk(&zap->zap_m.zap_avl, other, direction)) {
+	    other = zfs_btree_prev(&zap->zap_m.zap_tree, &oidx, &oidx)) {

 		if (zn == NULL) {
-			zn = zap_name_alloc(zap, MZE_PHYS(zap, mze)->mze_name,
-			    MT_NORMALIZE);
+			zn = zap_name_alloc_str(zap,
+			    MZE_PHYS(zap, mze)->mze_name, MT_NORMALIZE);
 			allocdzn = B_TRUE;
 		}
 		if (zap_match(zn, MZE_PHYS(zap, other)->mze_name)) {
@@ -939,9 +950,20 @@ again:
 		}
 	}

-	if (direction == AVL_BEFORE) {
-		direction = AVL_AFTER;
-		goto again;
+	for (other = zfs_btree_next(&zap->zap_m.zap_tree, idx, &oidx);
+	    other && other->mze_hash == mze->mze_hash;
+	    other = zfs_btree_next(&zap->zap_m.zap_tree, &oidx, &oidx)) {
+
+		if (zn == NULL) {
+			zn = zap_name_alloc_str(zap,
+			    MZE_PHYS(zap, mze)->mze_name, MT_NORMALIZE);
+			allocdzn = B_TRUE;
+		}
+		if (zap_match(zn, MZE_PHYS(zap, other)->mze_name)) {
+			if (allocdzn)
+				zap_name_free(zn);
+			return (B_TRUE);
+		}
 	}

 	if (allocdzn)
@@ -969,7 +991,7 @@ zap_lookup_impl(zap_t *zap, const char *name,
 {
 	int err = 0;

-	zap_name_t *zn = zap_name_alloc(zap, name, mt);
+	zap_name_t *zn = zap_name_alloc_str(zap, name, mt);
 	if (zn == NULL)
 		return (SET_ERROR(ENOTSUP));

@@ -977,7 +999,8 @@ zap_lookup_impl(zap_t *zap, const char *name,
 		err = fzap_lookup(zn, integer_size, num_integers, buf,
 		    realname, rn_len, ncp);
 	} else {
-		mzap_ent_t *mze = mze_find(zn);
+		zfs_btree_index_t idx;
+		mzap_ent_t *mze = mze_find(zn, &idx);
 		if (mze == NULL) {
 			err = SET_ERROR(ENOENT);
 		} else {
@@ -994,7 +1017,7 @@ zap_lookup_impl(zap_t *zap, const char *name,
 					    rn_len);
 				if (ncp) {
 					*ncp = mzap_normalization_conflict(zap,
-					    zn, mze);
+					    zn, mze, &idx);
 				}
 			}
 		}
@@ -1031,7 +1054,7 @@ zap_prefetch(objset_t *os, uint64_t zapobj, const char *name)
 	err = zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
 	if (err)
 		return (err);
-	zn = zap_name_alloc(zap, name, 0);
+	zn = zap_name_alloc_str(zap, name, 0);
 	if (zn == NULL) {
 		zap_unlockdir(zap, FTAG);
 		return (SET_ERROR(ENOTSUP));
@@ -1134,7 +1157,7 @@ zap_length(objset_t *os, uint64_t zapobj, const char *name,
 	    zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap);
 	if (err != 0)
 		return (err);
-	zap_name_t *zn = zap_name_alloc(zap, name, 0);
+	zap_name_t *zn = zap_name_alloc_str(zap, name, 0);
 	if (zn == NULL) {
 		zap_unlockdir(zap, FTAG);
 		return (SET_ERROR(ENOTSUP));
@@ -1142,7 +1165,8 @@ zap_length(objset_t *os, uint64_t zapobj, const char *name,
 	if (!zap->zap_ismicro) {
 		err = fzap_length(zn, integer_size, num_integers);
 	} else {
-		mzap_ent_t *mze = mze_find(zn);
+		zfs_btree_index_t idx;
+		mzap_ent_t *mze = mze_find(zn, &idx);
 		if (mze == NULL) {
 			err = SET_ERROR(ENOENT);
 		} else {
@@ -1182,7 +1206,7 @@ static void
 mzap_addent(zap_name_t *zn, uint64_t value)
 {
 	zap_t *zap = zn->zn_zap;
-	int start = zap->zap_m.zap_alloc_next;
+	uint16_t start = zap->zap_m.zap_alloc_next;

 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));

@@ -1198,7 +1222,7 @@ mzap_addent(zap_name_t *zn, uint64_t value)
 	ASSERT(cd < zap_maxcd(zap));

 again:
-	for (int i = start; i < zap->zap_m.zap_num_chunks; i++) {
+	for (uint16_t i = start; i < zap->zap_m.zap_num_chunks; i++) {
 		mzap_ent_phys_t *mze = &zap_m_phys(zap)->mz_chunk[i];
 		if (mze->mze_name[0] == 0) {
 			mze->mze_value = value;
@@ -1229,7 +1253,7 @@ zap_add_impl(zap_t *zap, const char *key,
 	const uint64_t *intval = val;
 	int err = 0;

-	zap_name_t *zn = zap_name_alloc(zap, key, 0);
+	zap_name_t *zn = zap_name_alloc_str(zap, key, 0);
 	if (zn == NULL) {
 		zap_unlockdir(zap, tag);
 		return (SET_ERROR(ENOTSUP));
@@ -1247,7 +1271,8 @@ zap_add_impl(zap_t *zap, const char *key,
 		}
 		zap = zn->zn_zap;	/* fzap_add() may change zap */
 	} else {
-		if (mze_find(zn) != NULL) {
+		zfs_btree_index_t idx;
+		if (mze_find(zn, &idx) != NULL) {
 			err = SET_ERROR(EEXIST);
 		} else {
 			mzap_addent(zn, *intval);
@@ -1327,7 +1352,7 @@ zap_update(objset_t *os, uint64_t zapobj, const char *name,
 	    zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap);
 	if (err != 0)
 		return (err);
-	zap_name_t *zn = zap_name_alloc(zap, name, 0);
+	zap_name_t *zn = zap_name_alloc_str(zap, name, 0);
 	if (zn == NULL) {
 		zap_unlockdir(zap, FTAG);
 		return (SET_ERROR(ENOTSUP));
@@ -1348,7 +1373,8 @@ zap_update(objset_t *os, uint64_t zapobj, const char *name,
 		}
 		zap = zn->zn_zap;	/* fzap_update() may change zap */
 	} else {
-		mzap_ent_t *mze = mze_find(zn);
+		zfs_btree_index_t idx;
+		mzap_ent_t *mze = mze_find(zn, &idx);
 		if (mze != NULL) {
 			MZE_PHYS(zap, mze)->mze_value = *intval;
 		} else {
@@ -1398,20 +1424,20 @@ zap_remove_impl(zap_t *zap, const char *name,
 {
 	int err = 0;

-	zap_name_t *zn = zap_name_alloc(zap, name, mt);
+	zap_name_t *zn = zap_name_alloc_str(zap, name, mt);
 	if (zn == NULL)
 		return (SET_ERROR(ENOTSUP));
 	if (!zap->zap_ismicro) {
 		err = fzap_remove(zn, tx);
 	} else {
-		mzap_ent_t *mze = mze_find(zn);
+		zfs_btree_index_t idx;
+		mzap_ent_t *mze = mze_find(zn, &idx);
 		if (mze == NULL) {
 			err = SET_ERROR(ENOENT);
 		} else {
 			zap->zap_m.zap_num_entries--;
-			bzero(&zap_m_phys(zap)->mz_chunk[mze->mze_chunkid],
-			    sizeof (mzap_ent_phys_t));
-			mze_remove(zap, mze);
+			memset(MZE_PHYS(zap, mze), 0, sizeof (mzap_ent_phys_t));
+			zfs_btree_remove_idx(&zap->zap_m.zap_tree, &idx);
 		}
 	}
 	zap_name_free(zn);
@@ -1582,29 +1608,30 @@ zap_cursor_retrieve(zap_cursor_t *zc, zap_attribute_t *za)
 	if (!zc->zc_zap->zap_ismicro) {
 		err = fzap_cursor_retrieve(zc->zc_zap, zc, za);
 	} else {
-		avl_index_t idx;
+		zfs_btree_index_t idx;
 		mzap_ent_t mze_tofind;

-		mze_tofind.mze_hash = zc->zc_hash;
+		mze_tofind.mze_hash = zc->zc_hash >> 32;
 		mze_tofind.mze_cd = zc->zc_cd;

-		mzap_ent_t *mze =
-		    avl_find(&zc->zc_zap->zap_m.zap_avl, &mze_tofind, &idx);
+		mzap_ent_t *mze = zfs_btree_find(&zc->zc_zap->zap_m.zap_tree,
+		    &mze_tofind, &idx);
 		if (mze == NULL) {
-			mze = avl_nearest(&zc->zc_zap->zap_m.zap_avl,
-			    idx, AVL_AFTER);
+			mze = zfs_btree_next(&zc->zc_zap->zap_m.zap_tree,
+			    &idx, &idx);
 		}
 		if (mze) {
 			mzap_ent_phys_t *mzep = MZE_PHYS(zc->zc_zap, mze);
 			ASSERT3U(mze->mze_cd, ==, mzep->mze_cd);
 			za->za_normalization_conflict =
-			    mzap_normalization_conflict(zc->zc_zap, NULL, mze);
+			    mzap_normalization_conflict(zc->zc_zap, NULL,
+			    mze, &idx);
 			za->za_integer_length = 8;
 			za->za_num_integers = 1;
 			za->za_first_integer = mzep->mze_value;
 			(void) strlcpy(za->za_name, mzep->mze_name,
 			    sizeof (za->za_name));
-			zc->zc_hash = mze->mze_hash;
+			zc->zc_hash = (uint64_t)mze->mze_hash << 32;
 			zc->zc_cd = mze->mze_cd;
 			err = 0;
 		} else {
@@ -1921,6 +1921,10 @@ zfs_ioc_vdev_set_state(zfs_cmd_t *zc)
 		error = vdev_degrade(spa, zc->zc_guid, zc->zc_obj);
 		break;

+	case VDEV_STATE_REMOVED:
+		error = vdev_remove_wanted(spa, zc->zc_guid);
+		break;
+
 	default:
 		error = SET_ERROR(EINVAL);
 	}
@@ -2928,7 +2932,7 @@ zfs_ioc_pool_set_props(zfs_cmd_t *zc)
 		mutex_enter(&spa_namespace_lock);
 		if ((spa = spa_lookup(zc->zc_name)) != NULL) {
 			spa_configfile_set(spa, props, B_FALSE);
-			spa_write_cachefile(spa, B_FALSE, B_TRUE);
+			spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE);
 		}
 		mutex_exit(&spa_namespace_lock);
 		if (spa != NULL) {
@@ -105,7 +105,7 @@ zfs_holey_common(znode_t *zp, ulong_t cmd, loff_t *off)
 	if (zn_has_cached_data(zp))
 		zn_flush_cached_data(zp, B_FALSE);

-	lr = zfs_rangelock_enter(&zp->z_rangelock, 0, file_sz, RL_READER);
+	lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_READER);
 	error = dmu_offset_next(ZTOZSB(zp)->z_os, zp->z_id, hole, &noff);
 	zfs_rangelock_exit(lr);

@@ -91,6 +91,14 @@
 */
 int zfs_commit_timeout_pct = 5;

+/*
+ * Minimal time we care to delay commit waiting for more ZIL records.
+ * At least FreeBSD kernel can't sleep for less than 2us at its best.
+ * So requests to sleep for less then 5us is a waste of CPU time with
+ * a risk of significant log latency increase due to oversleep.
+ */
+static unsigned long zil_min_commit_timeout = 5000;
+
 /*
 * See zil.h for more information about these fields.
 */
@@ -1155,7 +1163,8 @@ zil_lwb_flush_vdevs_done(zio_t *zio)
 	lwb->lwb_tx = NULL;

 	ASSERT3U(lwb->lwb_issued_timestamp, >, 0);
-	zilog->zl_last_lwb_latency = gethrtime() - lwb->lwb_issued_timestamp;
+	zilog->zl_last_lwb_latency = (zilog->zl_last_lwb_latency * 3 +
+	    gethrtime() - lwb->lwb_issued_timestamp) / 4;

 	lwb->lwb_root_zio = NULL;

@@ -2283,8 +2292,9 @@ zil_process_commit_list(zilog_t *zilog)
 	spa_t *spa = zilog->zl_spa;
 	list_t nolwb_itxs;
 	list_t nolwb_waiters;
-	lwb_t *lwb;
+	lwb_t *lwb, *plwb;
 	itx_t *itx;
+	boolean_t first = B_TRUE;

 	ASSERT(MUTEX_HELD(&zilog->zl_issuer_lock));

@@ -2306,6 +2316,9 @@ zil_process_commit_list(zilog_t *zilog)
 		ASSERT3S(lwb->lwb_state, !=, LWB_STATE_ISSUED);
 		ASSERT3S(lwb->lwb_state, !=, LWB_STATE_WRITE_DONE);
 		ASSERT3S(lwb->lwb_state, !=, LWB_STATE_FLUSH_DONE);
+		first = (lwb->lwb_state != LWB_STATE_OPENED) &&
+		    ((plwb = list_prev(&zilog->zl_lwb_list, lwb)) == NULL ||
+		    plwb->lwb_state == LWB_STATE_FLUSH_DONE);
 	}

 	while ((itx = list_head(&zilog->zl_itx_commit_list)) != NULL) {
@@ -2476,7 +2489,23 @@ zil_process_commit_list(zilog_t *zilog)
 		 * try and pack as many itxs into as few lwbs as
 		 * possible, without significantly impacting the latency
 		 * of each individual itx.
+		 *
+		 * If we had no already running or open LWBs, it can be
+		 * the workload is single-threaded.  And if the ZIL write
+		 * latency is very small or if the LWB is almost full, it
+		 * may be cheaper to bypass the delay.
 		 */
+		if (lwb->lwb_state == LWB_STATE_OPENED && first) {
+			hrtime_t sleep = zilog->zl_last_lwb_latency *
+			    zfs_commit_timeout_pct / 100;
+			if (sleep < zil_min_commit_timeout ||
+			    lwb->lwb_sz - lwb->lwb_nused < lwb->lwb_sz / 8) {
+				lwb = zil_lwb_write_issue(zilog, lwb);
+				zilog->zl_cur_used = 0;
+				if (lwb == NULL)
+					zil_commit_writer_stall(zilog);
+			}
+		}
 	}
 }

@@ -3726,6 +3755,9 @@ EXPORT_SYMBOL(zil_set_logbias);
 ZFS_MODULE_PARAM(zfs, zfs_, commit_timeout_pct, INT, ZMOD_RW,
 	"ZIL block open timeout percentage");

+ZFS_MODULE_PARAM(zfs_zil, zil_, min_commit_timeout, ULONG, ZMOD_RW,
+	"Minimum delay we care for ZIL block commit");
+
 ZFS_MODULE_PARAM(zfs_zil, zil_, replay_disable, INT, ZMOD_RW,
 	"Disable intent logging replay");

@@ -3928,7 +3928,7 @@ zio_vdev_io_done(zio_t *zio)

 	ops->vdev_op_io_done(zio);

-	if (unexpected_error)
+	if (unexpected_error && vd->vdev_remove_wanted == B_FALSE)
 		VERIFY(vdev_probe(vd, zio) == NULL);

 	return (zio);
@@ -106,16 +106,16 @@ zrl_add_impl(zrlock_t *zrl, const char *zc)
 void
 zrl_remove(zrlock_t *zrl)
 {
-	uint32_t n;
-
 #ifdef	ZFS_DEBUG
 	if (zrl->zr_owner == curthread) {
 		zrl->zr_owner = NULL;
 		zrl->zr_caller = NULL;
 	}
+	int32_t n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount);
+	ASSERT3S(n, >=, 0);
+#else
+	atomic_dec_32((uint32_t *)&zrl->zr_refcount);
 #endif
-	n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount);
-	ASSERT3S((int32_t)n, >=, 0);
 }

 int
@@ -1,6 +1,6 @@
 %{?!packager: %define packager Brian Behlendorf <behlendorf1@llnl.gov>}

-%if ! 0%{?rhel}%{?fedora}%{?mageia}%{?suse_version}
+%if ! 0%{?rhel}%{?fedora}%{?mageia}%{?suse_version}%{?openEuler}
 %define not_rpm 1
 %endif

@@ -28,7 +28,7 @@ Requires(post): dkms >= 2.2.0.3
 Requires(preun): dkms >= 2.2.0.3
 Requires:       gcc, make, perl, diffutils
 Requires(post): gcc, make, perl, diffutils
-%if 0%{?rhel}%{?fedora}%{?mageia}%{?suse_version}
+%if 0%{?rhel}%{?fedora}%{?mageia}%{?suse_version}%{?openEuler}
 Requires:       kernel-devel >= @ZFS_META_KVER_MIN@, kernel-devel <= @ZFS_META_KVER_MAX@.999
 Requires(post): kernel-devel >= @ZFS_META_KVER_MIN@, kernel-devel <= @ZFS_META_KVER_MAX@.999
 Obsoletes:      spl-dkms <= %{version}
@@ -36,7 +36,7 @@ Obsoletes:      spl-dkms <= %{version}
 Provides:       %{module}-kmod = %{version}
 AutoReqProv:    no

-%if (0%{?fedora}%{?suse_version}) || (0%{?rhel} && 0%{?rhel} < 9)
+%if (0%{?fedora}%{?suse_version}%{?openEuler}) || (0%{?rhel} && 0%{?rhel} < 9)
 # We don't directly use it, but if this isn't installed, rpmbuild as root can
 # crash+corrupt rpmdb
 # See issue #12071
--- a/Show More
+++ b/Show More