Prevent unnecessary resilver restarts

If a device is participating in an active resilver, then it will have a
non-empty DTL. Operations like vdev_{open,reopen,probe}() can cause the
resilver to be restarted (or deferred to be restarted later), which is
unnecessary if the DTL is still covered by the current scan range. This
is similar to the logic in vdev_dtl_should_excise() where the DTL can
only be excised if it's max txg is in the resilvered range.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: John Gallagher <john.gallagher@delphix.com>
Reviewed-by: Kjeld Schouten <kjeld@schouten-lebbing.nl>
Signed-off-by: John Poduska <jpoduska@datto.com>
Issue #840
Closes #9155
Closes #9378
Closes #9551
Closes #9588

configure.ac

@@ -323,6 +323,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/rename_dirs/Makefile
 	tests/zfs-tests/tests/functional/replacement/Makefile
 	tests/zfs-tests/tests/functional/reservation/Makefile
+	tests/zfs-tests/tests/functional/resilver/Makefile
 	tests/zfs-tests/tests/functional/rootpool/Makefile
 	tests/zfs-tests/tests/functional/rsend/Makefile
 	tests/zfs-tests/tests/functional/scrub_mirror/Makefile

include/sys/dsl_scan.h

@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
- * Copyright (c) 2017 Datto Inc.
+ * Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
  */
 
 #ifndef	_SYS_DSL_SCAN_H
@@ -164,10 +164,12 @@ void dsl_scan_fini(struct dsl_pool *dp);
 void dsl_scan_sync(struct dsl_pool *, dmu_tx_t *);
 int dsl_scan_cancel(struct dsl_pool *);
 int dsl_scan(struct dsl_pool *, pool_scan_func_t);
+void dsl_scan_assess_vdev(struct dsl_pool *dp, vdev_t *vd);
 boolean_t dsl_scan_scrubbing(const struct dsl_pool *dp);
 int dsl_scrub_set_pause_resume(const struct dsl_pool *dp, pool_scrub_cmd_t cmd);
-void dsl_resilver_restart(struct dsl_pool *, uint64_t txg);
+void dsl_scan_restart_resilver(struct dsl_pool *, uint64_t txg);
 boolean_t dsl_scan_resilvering(struct dsl_pool *dp);
+boolean_t dsl_scan_resilver_scheduled(struct dsl_pool *dp);
 boolean_t dsl_dataset_unstable(struct dsl_dataset *ds);
 void dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum,
     ddt_entry_t *dde, dmu_tx_t *tx);

include/sys/spa.h

@@ -26,7 +26,7 @@
  * Copyright 2013 Saso Kiselkov. All rights reserved.
  * Copyright (c) 2014 Integros [integros.com]
  * Copyright 2017 Joyent, Inc.
- * Copyright (c) 2017 Datto Inc.
+ * Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
  * Copyright (c) 2017, Intel Corporation.
  */
 
@@ -777,6 +777,7 @@ extern void spa_async_request(spa_t *spa, int flag);
 extern void spa_async_unrequest(spa_t *spa, int flag);
 extern void spa_async_suspend(spa_t *spa);
 extern void spa_async_resume(spa_t *spa);
+extern int spa_async_tasks(spa_t *spa);
 extern spa_t *spa_inject_addref(char *pool);
 extern void spa_inject_delref(spa_t *spa);
 extern void spa_scan_stat_init(spa_t *spa);

include/sys/vdev.h

@@ -23,6 +23,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
  * Copyright (c) 2017, Intel Corporation.
+ * Copyright (c) 2019, Datto Inc. All rights reserved.
  */
 
 #ifndef _SYS_VDEV_H
@@ -151,7 +152,8 @@ extern int vdev_config_sync(vdev_t **svd, int svdcount, uint64_t txg);
 extern void vdev_state_dirty(vdev_t *vd);
 extern void vdev_state_clean(vdev_t *vd);
 
-extern void vdev_set_deferred_resilver(spa_t *spa, vdev_t *vd);
+extern void vdev_defer_resilver(vdev_t *vd);
+extern boolean_t vdev_clear_resilver_deferred(vdev_t *vd, dmu_tx_t *tx);
 
 typedef enum vdev_config_flag {
 	VDEV_CONFIG_SPARE = 1 << 0,

module/zfs/dsl_scan.c

@@ -22,7 +22,7 @@
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
  * Copyright 2016 Gary Mills
- * Copyright (c) 2017 Datto Inc.
+ * Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
  * Copyright 2019 Joyent, Inc.
  */
 
@@ -591,6 +591,13 @@ dsl_scan_restarting(dsl_scan_t *scn, dmu_tx_t *tx)
 	    scn->scn_restart_txg <= tx->tx_txg);
 }
 
+boolean_t
+dsl_scan_resilver_scheduled(dsl_pool_t *dp)
+{
+	return ((dp->dp_scan && dp->dp_scan->scn_restart_txg != 0) ||
+	    (spa_async_tasks(dp->dp_spa) & SPA_ASYNC_RESILVER));
+}
+
 boolean_t
 dsl_scan_scrubbing(const dsl_pool_t *dp)
 {
@@ -786,7 +793,7 @@ dsl_scan(dsl_pool_t *dp, pool_scan_func_t func)
 	(void) spa_vdev_state_exit(spa, NULL, 0);
 
 	if (func == POOL_SCAN_RESILVER) {
-		dsl_resilver_restart(spa->spa_dsl_pool, 0);
+		dsl_scan_restart_resilver(spa->spa_dsl_pool, 0);
 		return (0);
 	}
 
@@ -806,41 +813,6 @@ dsl_scan(dsl_pool_t *dp, pool_scan_func_t func)
 	    dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_EXTRA_RESERVED));
 }
 
-/*
- * Sets the resilver defer flag to B_FALSE on all leaf devs under vd. Returns
- * B_TRUE if we have devices that need to be resilvered and are available to
- * accept resilver I/Os.
- */
-static boolean_t
-dsl_scan_clear_deferred(vdev_t *vd, dmu_tx_t *tx)
-{
-	boolean_t resilver_needed = B_FALSE;
-	spa_t *spa = vd->vdev_spa;
-
-	for (int c = 0; c < vd->vdev_children; c++) {
-		resilver_needed |=
-		    dsl_scan_clear_deferred(vd->vdev_child[c], tx);
-	}
-
-	if (vd == spa->spa_root_vdev &&
-	    spa_feature_is_active(spa, SPA_FEATURE_RESILVER_DEFER)) {
-		spa_feature_decr(spa, SPA_FEATURE_RESILVER_DEFER, tx);
-		vdev_config_dirty(vd);
-		spa->spa_resilver_deferred = B_FALSE;
-		return (resilver_needed);
-	}
-
-	if (!vdev_is_concrete(vd) || vd->vdev_aux ||
-	    !vd->vdev_ops->vdev_op_leaf)
-		return (resilver_needed);
-
-	if (vd->vdev_resilver_deferred)
-		vd->vdev_resilver_deferred = B_FALSE;
-
-	return (!vdev_is_dead(vd) && !vd->vdev_offline &&
-	    vdev_resilver_needed(vd, NULL, NULL));
-}
-
 /* ARGSUSED */
 static void
 dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx)
@@ -943,25 +915,21 @@ dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx)
 		spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
 
 		/*
-		 * Clear any deferred_resilver flags in the config.
+		 * Clear any resilver_deferred flags in the config.
 		 * If there are drives that need resilvering, kick
 		 * off an asynchronous request to start resilver.
-		 * dsl_scan_clear_deferred() may update the config
+		 * vdev_clear_resilver_deferred() may update the config
 		 * before the resilver can restart. In the event of
 		 * a crash during this period, the spa loading code
 		 * will find the drives that need to be resilvered
-		 * when the machine reboots and start the resilver then.
+		 * and start the resilver then.
 		 */
-		if (spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) {
+		if (spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER) &&
-			boolean_t resilver_needed =
+		    vdev_clear_resilver_deferred(spa->spa_root_vdev, tx)) {
-			    dsl_scan_clear_deferred(spa->spa_root_vdev, tx);
+			spa_history_log_internal(spa,
-			if (resilver_needed) {
+			    "starting deferred resilver", tx, "errors=%llu",
-				spa_history_log_internal(spa,
+			    (u_longlong_t)spa_get_errlog_size(spa));
-				    "starting deferred resilver", tx,
+			spa_async_request(spa, SPA_ASYNC_RESILVER);
-				    "errors=%llu",
-				    (u_longlong_t)spa_get_errlog_size(spa));
-				spa_async_request(spa, SPA_ASYNC_RESILVER);
-			}
 		}
 	}
 
@@ -1071,7 +1039,7 @@ dsl_scrub_set_pause_resume(const dsl_pool_t *dp, pool_scrub_cmd_t cmd)
 
 /* start a new scan, or restart an existing one. */
 void
-dsl_resilver_restart(dsl_pool_t *dp, uint64_t txg)
+dsl_scan_restart_resilver(dsl_pool_t *dp, uint64_t txg)
 {
 	if (txg == 0) {
 		dmu_tx_t *tx;
@@ -4232,6 +4200,36 @@ dsl_scan_freed(spa_t *spa, const blkptr_t *bp)
 		dsl_scan_freed_dva(spa, bp, i);
 }
 
+/*
+ * Check if a vdev needs resilvering (non-empty DTL), if so, and resilver has
+ * not started, start it. Otherwise, only restart if max txg in DTL range is
+ * greater than the max txg in the current scan. If the DTL max is less than
+ * the scan max, then the vdev has not missed any new data since the resilver
+ * started, so a restart is not needed.
+ */
+void
+dsl_scan_assess_vdev(dsl_pool_t *dp, vdev_t *vd)
+{
+	uint64_t min, max;
+
+	if (!vdev_resilver_needed(vd, &min, &max))
+		return;
+
+	if (!dsl_scan_resilvering(dp)) {
+		spa_async_request(dp->dp_spa, SPA_ASYNC_RESILVER);
+		return;
+	}
+
+	if (max <= dp->dp_scan->scn_phys.scn_max_txg)
+		return;
+
+	/* restart is needed, check if it can be deferred */
+	if (spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))
+		vdev_defer_resilver(vd);
+	else
+		spa_async_request(dp->dp_spa, SPA_ASYNC_RESILVER);
+}
+
 #if defined(_KERNEL)
 /* CSTYLED */
 module_param(zfs_scan_vdev_limit, ulong, 0644);

module/zfs/spa.c

@@ -29,7 +29,7 @@
  * Copyright 2016 Toomas Soome <tsoome@me.com>
  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  * Copyright 2018 Joyent, Inc.
- * Copyright (c) 2017 Datto Inc.
+ * Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
  * Copyright 2017 Joyent, Inc.
  * Copyright (c) 2017, Intel Corporation.
  */
@@ -6242,9 +6242,9 @@ spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing)
 	 */
 	if (dsl_scan_resilvering(spa_get_dsl(spa)) &&
 	    spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER))
-		vdev_set_deferred_resilver(spa, newvd);
+		vdev_defer_resilver(newvd);
 	else
-		dsl_resilver_restart(spa->spa_dsl_pool, dtl_max_txg);
+		dsl_scan_restart_resilver(spa->spa_dsl_pool, dtl_max_txg);
 
 	if (spa->spa_bootfs)
 		spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH);
@@ -7479,7 +7479,7 @@ spa_async_thread(void *arg)
 	if (tasks & SPA_ASYNC_RESILVER &&
 	    (!dsl_scan_resilvering(dp) ||
 	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER)))
-		dsl_resilver_restart(dp, 0);
+		dsl_scan_restart_resilver(dp, 0);
 
 	if (tasks & SPA_ASYNC_INITIALIZE_RESTART) {
 		mutex_enter(&spa_namespace_lock);
@@ -7595,6 +7595,12 @@ spa_async_request(spa_t *spa, int task)
 	mutex_exit(&spa->spa_async_lock);
 }
 
+int
+spa_async_tasks(spa_t *spa)
+{
+	return (spa->spa_async_tasks);
+}
+
 /*
  * ==========================================================================
  * SPA syncing routines

module/zfs/vdev.c

@@ -27,6 +27,7 @@
  * Copyright 2016 Toomas Soome <tsoome@me.com>
  * Copyright 2017 Joyent, Inc.
  * Copyright (c) 2017, Intel Corporation.
+ * Copyright (c) 2019, Datto Inc. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -833,7 +834,7 @@ vdev_alloc(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, uint_t id,
 		    &vd->vdev_resilver_txg);
 
 		if (nvlist_exists(nv, ZPOOL_CONFIG_RESILVER_DEFER))
-			vdev_set_deferred_resilver(spa, vd);
+			vdev_defer_resilver(vd);
 
 		/*
 		 * In general, when importing a pool we want to ignore the
@@ -1863,18 +1864,12 @@ vdev_open(vdev_t *vd)
 	}
 
 	/*
-	 * If a leaf vdev has a DTL, and seems healthy, then kick off a
+	 * If this is a leaf vdev, assess whether a resilver is needed.
-	 * resilver.  But don't do this if we are doing a reopen for a scrub,
+	 * But don't do this if we are doing a reopen for a scrub, since
-	 * since this would just restart the scrub we are already doing.
+	 * this would just restart the scrub we are already doing.
 	 */
-	if (vd->vdev_ops->vdev_op_leaf && !spa->spa_scrub_reopen &&
+	if (vd->vdev_ops->vdev_op_leaf && !spa->spa_scrub_reopen)
-	    vdev_resilver_needed(vd, NULL, NULL)) {
+		dsl_scan_assess_vdev(spa->spa_dsl_pool, vd);
-		if (dsl_scan_resilvering(spa->spa_dsl_pool) &&
-		    spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER))
-			vdev_set_deferred_resilver(spa, vd);
-		else
-			spa_async_request(spa, SPA_ASYNC_RESILVER);
-	}
 
 	return (0);
 }
@@ -3693,14 +3688,11 @@ vdev_clear(spa_t *spa, vdev_t *vd)
 		if (vd != rvd && vdev_writeable(vd->vdev_top))
 			vdev_state_dirty(vd->vdev_top);
 
-		if (vd->vdev_aux == NULL && !vdev_is_dead(vd)) {
+		/* If a resilver isn't required, check if vdevs can be culled */
-			if (dsl_scan_resilvering(spa->spa_dsl_pool) &&
+		if (vd->vdev_aux == NULL && !vdev_is_dead(vd) &&
-			    spa_feature_is_enabled(spa,
+		    !dsl_scan_resilvering(spa->spa_dsl_pool) &&
-			    SPA_FEATURE_RESILVER_DEFER))
+		    !dsl_scan_resilver_scheduled(spa->spa_dsl_pool))
-				vdev_set_deferred_resilver(spa, vd);
+			spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
-			else
-				spa_async_request(spa, SPA_ASYNC_RESILVER);
-		}
 
 		spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_CLEAR);
 	}
@@ -4693,18 +4685,46 @@ vdev_deadman(vdev_t *vd, char *tag)
 }
 
 void
-vdev_set_deferred_resilver(spa_t *spa, vdev_t *vd)
+vdev_defer_resilver(vdev_t *vd)
 {
-	for (uint64_t i = 0; i < vd->vdev_children; i++)
+	ASSERT(vd->vdev_ops->vdev_op_leaf);
-		vdev_set_deferred_resilver(spa, vd->vdev_child[i]);
-
-	if (!vd->vdev_ops->vdev_op_leaf || !vdev_writeable(vd) ||
-	    range_tree_is_empty(vd->vdev_dtl[DTL_MISSING])) {
-		return;
-	}
 
 	vd->vdev_resilver_deferred = B_TRUE;
-	spa->spa_resilver_deferred = B_TRUE;
+	vd->vdev_spa->spa_resilver_deferred = B_TRUE;
+}
+
+/*
+ * Clears the resilver deferred flag on all leaf devs under vd. Returns
+ * B_TRUE if we have devices that need to be resilvered and are available to
+ * accept resilver I/Os.
+ */
+boolean_t
+vdev_clear_resilver_deferred(vdev_t *vd, dmu_tx_t *tx)
+{
+	boolean_t resilver_needed = B_FALSE;
+	spa_t *spa = vd->vdev_spa;
+
+	for (int c = 0; c < vd->vdev_children; c++) {
+		vdev_t *cvd = vd->vdev_child[c];
+		resilver_needed |= vdev_clear_resilver_deferred(cvd, tx);
+	}
+
+	if (vd == spa->spa_root_vdev &&
+	    spa_feature_is_active(spa, SPA_FEATURE_RESILVER_DEFER)) {
+		spa_feature_decr(spa, SPA_FEATURE_RESILVER_DEFER, tx);
+		vdev_config_dirty(vd);
+		spa->spa_resilver_deferred = B_FALSE;
+		return (resilver_needed);
+	}
+
+	if (!vdev_is_concrete(vd) || vd->vdev_aux ||
+	    !vd->vdev_ops->vdev_op_leaf)
+		return (resilver_needed);
+
+	vd->vdev_resilver_deferred = B_FALSE;
+
+	return (!vdev_is_dead(vd) && !vd->vdev_offline &&
+	    vdev_resilver_needed(vd, NULL, NULL));
 }
 
 /*

tests/runfiles/linux.run

@@ -796,6 +796,10 @@ tests = ['reservation_001_pos', 'reservation_002_pos', 'reservation_003_pos',
     'reservation_022_pos']
 tags = ['functional', 'reservation']
 
+[tests/functional/resilver]
+tests = ['resilver_restart_001']
+tags = ['functional', 'resilver']
+
 [tests/functional/rootpool]
 tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
 tags = ['functional', 'rootpool']

tests/zfs-tests/tests/functional/Makefile.am

@@ -59,6 +59,7 @@ SUBDIRS = \
 	rename_dirs \
 	replacement \
 	reservation \
+	resilver \
 	rootpool \
 	rsend \
 	scrub_mirror \

tests/zfs-tests/tests/functional/resilver/Makefile.am

@@ -0,0 +1,8 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/resilver
+dist_pkgdata_SCRIPTS = \
+	setup.ksh \
+	cleanup.ksh \
+	resilver_restart_001.ksh
+
+dist_pkgdata_DATA = \
+	resilver.cfg

tests/zfs-tests/tests/functional/resilver/cleanup.ksh

@@ -0,0 +1,31 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/resilver/resilver.cfg
+
+verify_runnable "global"
+
+log_pass

tests/zfs-tests/tests/functional/resilver/resilver.cfg

@@ -0,0 +1,32 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+set -A VDEV_FILES $TEST_BASE_DIR/file-{1..4}
+SPARE_VDEV_FILE=$TEST_BASE_DIR/spare-1
+
+VDEV_FILE_SIZE=$(( $SPA_MINDEVSIZE * 2 ))

tests/zfs-tests/tests/functional/resilver/resilver_restart_001.ksh

@@ -0,0 +1,185 @@
+#!/bin/ksh -p
+
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/resilver/resilver.cfg
+
+#
+# DESCRIPTION:
+# Testing resilver restart logic both with and without the deferred resilver
+# feature enabled, verifying that resilver is not restarted when it is
+# unecessary.
+#
+# STRATEGY:
+# 1. Create a pool
+# 2. Create four filesystems with the primary cache disable to force reads
+# 3. Write four files simultaneously, one to each filesystem
+# 4. Do with and without deferred resilvers enabled
+#    a. Replace a vdev with a spare & suspend resilver immediately
+#    b. Verify resilver starts properly
+#    c. Offline / online another vdev to introduce a new DTL range
+#    d. Verify resilver restart restart or defer
+#    e. Inject read errors on vdev that was offlined / onlned
+#    f. Verify that resilver did not restart
+#    g. Unsuspend resilver and wait for it to finish
+#    h. Verify that there are two resilvers and nothing is deferred
+#
+
+function cleanup
+{
+	echo $ORIG_RESILVER_MIN_TIME > $ZFS_PARAMS/zfs_resilver_min_time_ms
+	echo $ORIG_SCAN_SUSPEND_PROGRESS > $ZFS_PARAMS/zfs_scan_suspend_progress
+	log_must zinject -c all
+	destroy_pool $TESTPOOL
+	rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+}
+
+# Count resilver events in zpool and number of deferred rsilvers on vdevs
+function verify_restarts # <msg> <cnt> <defer>
+{
+	msg=$1
+	cnt=$2
+	defer=$3
+
+	# check the number of resilver start in events log
+	RESILVERS=$(zpool events | grep -c sysevent.fs.zfs.resilver_start)
+	log_note "expected $cnt resilver start(s)$msg, found $RESILVERS"
+	[[ "$RESILVERS" -ne "$cnt" ]] &&
+	    log_fail "expected $cnt resilver start(s)$msg, found $RESILVERS"
+
+	[[ -z "$defer" ]] && return
+
+	# use zdb to find which vdevs have the resilver defer flag
+	VDEV_DEFERS=$(zdb -C $TESTPOOL | \
+	    sed -n -e '/^ *children\[[0-9]\].*$/{h}' \
+	    -e '/ *com.datto:resilver_defer$/{g;p}')
+
+	if [[ "$defer" == "-" ]]
+	then
+		[[ -n $VDEV_DEFERS ]] &&
+		    log_fail "didn't expect any vdevs to have resilver deferred"
+		return
+	fi
+
+	[[ "x${VDEV_DEFERS}x" =~ "x +children[$defer]:x" ]] ||
+	    log_fail "resilver deferred set on unexpected vdev: $VDEV_DEFERS"
+}
+
+log_assert "Check for unnecessary resilver restarts"
+
+ZFS_PARAMS=/sys/module/zfs/parameters
+ORIG_RESILVER_MIN_TIME=$(cat $ZFS_PARAMS/zfs_resilver_min_time_ms)
+ORIG_SCAN_SUSPEND_PROGRESS=$(cat $ZFS_PARAMS/zfs_scan_suspend_progress)
+
+set -A RESTARTS -- '1' '2' '2' '2'
+set -A VDEVS -- '' '' '' ''
+set -A DEFER_RESTARTS -- '1' '1' '1' '2'
+set -A DEFER_VDEVS -- '-' '2' '2' '-'
+
+VDEV_REPLACE="${VDEV_FILES[1]} $SPARE_VDEV_FILE"
+
+log_onexit cleanup
+
+log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+
+log_must zpool create -f -o feature@resilver_defer=disabled $TESTPOOL \
+    raidz ${VDEV_FILES[@]}
+
+# Create 4 filesystems
+for fs in fs{0..3}
+do
+	log_must zfs create -o primarycache=none -o recordsize=1k $TESTPOOL/$fs
+done
+
+# simultaneously write 16M to each of them
+set -A DATAPATHS /$TESTPOOL/fs{0..3}/dat.0
+log_note "Writing data files"
+for path in ${DATAPATHS[@]}
+do
+	dd if=/dev/urandom of=$path bs=1M count=16 > /dev/null 2>&1 &
+done
+wait
+
+# Test without and with deferred resilve feature enabled
+for test in "without" "with"
+do
+	log_note "Testing $test deferred resilvers"
+
+	if [[ $test == "with" ]]
+	then
+		log_must zpool set feature@resilver_defer=enabled $TESTPOOL
+		RESTARTS=( "${DEFER_RESTARTS[@]}" )
+		VDEVS=( "${DEFER_VDEVS[@]}" )
+		VDEV_REPLACE="$SPARE_VDEV_FILE ${VDEV_FILES[1]}"
+	fi
+
+	# clear the events
+	log_must zpool events -c
+
+	# limit scanning time
+	echo 50 > $ZFS_PARAMS/zfs_resilver_min_time_ms
+
+	# initiate a resilver and suspend the scan as soon as possible
+	log_must zpool replace $TESTPOOL $VDEV_REPLACE
+	echo 1 > $ZFS_PARAMS/zfs_scan_suspend_progress
+
+	# there should only be 1 resilver start
+	verify_restarts '' "${RESTARTS[0]}" "${VDEVS[0]}"
+
+	# offline then online a vdev to introduce a new DTL range after current
+	# scan, which should restart (or defer) the resilver
+	log_must zpool offline $TESTPOOL ${VDEV_FILES[2]}
+	log_must zpool sync $TESTPOOL
+	log_must zpool online $TESTPOOL ${VDEV_FILES[2]}
+	log_must zpool sync $TESTPOOL
+
+	# there should now be 2 resilver starts w/o defer, 1 with defer
+	verify_restarts ' after offline/online' "${RESTARTS[1]}" "${VDEVS[1]}"
+
+	# inject read io errors on vdev and verify resilver does not restart
+	log_must zinject -a -d ${VDEV_FILES[2]} -e io -T read -f 0.25 $TESTPOOL
+	log_must cat ${DATAPATHS[1]} > /dev/null
+	log_must zinject -c all
+
+	# there should still be 2 resilver starts w/o defer, 1 with defer
+	verify_restarts ' after zinject' "${RESTARTS[2]}" "${VDEVS[2]}"
+
+	# unsuspend resilver
+	echo 0 > $ZFS_PARAMS/zfs_scan_suspend_progress
+	echo 3000 > $ZFS_PARAMS/zfs_resilver_min_time_ms
+
+	# wait for resilver to finish
+	for iter in {0..59}
+	do
+		is_pool_resilvered $TESTPOOL && break
+		sleep 1
+	done
+	is_pool_resilvered $TESTPOOL ||
+	    log_fail "resilver timed out"
+
+	# wait for a few txg's to see if a resilver happens
+	log_must zpool sync $TESTPOOL
+
+	# there should now be 2 resilver starts
+	verify_restarts ' after resilver' "${RESTARTS[3]}" "${VDEVS[3]}"
+done
+
+log_pass "Resilver did not restart unnecessarily"

tests/zfs-tests/tests/functional/resilver/setup.ksh

@@ -0,0 +1,31 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/resilver/resilver.cfg
+
+verify_runnable "global"
+
+log_pass