Add scrub after resilver zed script

* Add a zed script to kick off a scrub after a resilver.  The script is
disabled by default.

* Add a optional $PATH (-P) option to zed to allow it to use a custom
$PATH for its zedlets.  This is needed when you're running zed under
the ZTS in a local workspace.

* Update test scripts to not copy in all-debug.sh and all-syslog.sh by
default.  They can be optionally copied in as part of zed_setup().
These scripts slow down zed considerably under heavy events loads and
can cause events to be dropped or their delivery delayed. This was
causing some sporadic failures in the 'fault' tests.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Richard Laager <rlaager@wiktel.com>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Closes #4662
Closes #7086

cmd/zed/Makefile.am

@@ -69,7 +69,8 @@ dist_zedexec_SCRIPTS = \
 	zed.d/statechange-notify.sh \
 	zed.d/vdev_clear-led.sh \
 	zed.d/vdev_attach-led.sh \
-	zed.d/pool_import-led.sh
+	zed.d/pool_import-led.sh \
+	zed.d/resilver_finish-start-scrub.sh
 
 zedconfdefaults = \
 	all-syslog.sh \
@@ -80,7 +81,8 @@ zedconfdefaults = \
 	statechange-notify.sh \
 	vdev_clear-led.sh \
 	vdev_attach-led.sh \
-	pool_import-led.sh
+	pool_import-led.sh \
+	resilver_finish-start-scrub.sh
 
 install-data-hook:
 	$(MKDIR_P) "$(DESTDIR)$(zedconfdir)"

cmd/zed/zed.d/resilver_finish-start-scrub.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# resilver_finish-start-scrub.sh
+# Run a scrub after a resilver
+#
+# Exit codes:
+# 1: Internal error
+# 2: Script wasn't enabled in zed.rc
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ "${ZED_SCRUB_AFTER_RESILVER}" = "1" ] || exit 2
+[ -n "${ZEVENT_POOL}" ] || exit 1
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 1
+zed_check_cmd "${ZPOOL}" || exit 1
+
+zed_log_msg "Starting scrub after resilver on ${ZEVENT_POOL}"
+"${ZPOOL}" scrub "${ZEVENT_POOL}"

cmd/zed/zed.d/zed.rc

@@ -86,6 +86,9 @@
 #
 ZED_USE_ENCLOSURE_LEDS=1
 
+##
+# Run a scrub after every resilver
+#ZED_SCRUB_AFTER_RESILVER=1
 
 ##
 # The syslog priority (e.g., specified as a "facility.level" pair).

cmd/zed/zed_conf.c

@@ -155,6 +155,8 @@ _zed_conf_display_help(const char *prog, int got_err)
 	    "Run daemon in the foreground.");
 	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-M",
 	    "Lock all pages in memory.");
+	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-P",
+	    "$PATH for ZED to use (only used by ZTS).");
 	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-Z",
 	    "Zero state file.");
 	fprintf(fp, "\n");
@@ -247,7 +249,7 @@ _zed_conf_parse_path(char **resultp, const char *path)
 void
 zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 {
-	const char * const opts = ":hLVc:d:p:s:vfFMZ";
+	const char * const opts = ":hLVc:d:p:P:s:vfFMZ";
 	int opt;
 
 	if (!zcp || !argv || !argv[0])
@@ -275,6 +277,9 @@ zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 		case 'p':
 			_zed_conf_parse_path(&zcp->pid_file, optarg);
 			break;
+		case 'P':
+			_zed_conf_parse_path(&zcp->path, optarg);
+			break;
 		case 's':
 			_zed_conf_parse_path(&zcp->state_file, optarg);
 			break;

cmd/zed/zed_conf.h

@@ -37,6 +37,7 @@ struct zed_conf {
 	int		state_fd;		/* fd to state file */
 	libzfs_handle_t	*zfs_hdl;		/* handle to libzfs */
 	int		zevent_fd;		/* fd for access to zevents */
+	char		*path;		/* custom $PATH for zedlets to use */
 };
 
 struct zed_conf *zed_conf_create(void);

cmd/zed/zed_event.c

@@ -733,12 +733,14 @@ _zed_event_add_nvpair(uint64_t eid, zed_strings_t *zsp, nvpair_t *nvp)
 
 /*
  * Restrict various environment variables to safe and sane values
- * when constructing the environment for the child process.
+ * when constructing the environment for the child process, unless
+ * we're running with a custom $PATH (like under the ZFS test suite).
  *
  * Reference: Secure Programming Cookbook by Viega & Messier, Section 1.1.
  */
 static void
-_zed_event_add_env_restrict(uint64_t eid, zed_strings_t *zsp)
+_zed_event_add_env_restrict(uint64_t eid, zed_strings_t *zsp,
+    const char *path)
 {
 	const char *env_restrict[][2] = {
 		{ "IFS",		" \t\n" },
@@ -753,11 +755,35 @@ _zed_event_add_env_restrict(uint64_t eid, zed_strings_t *zsp)
 		{ "ZFS_RELEASE",	ZFS_META_RELEASE },
 		{ NULL,			NULL }
 	};
+
+	/*
+	 * If we have a custom $PATH, use the default ZFS binary locations
+	 * instead of the hard-coded ones.
+	 */
+	const char *env_path[][2] = {
+		{ "IFS",		" \t\n" },
+		{ "PATH",		NULL }, /* $PATH copied in later on */
+		{ "ZDB",		"zdb" },
+		{ "ZED",		"zed" },
+		{ "ZFS",		"zfs" },
+		{ "ZINJECT",		"zinject" },
+		{ "ZPOOL",		"zpool" },
+		{ "ZFS_ALIAS",		ZFS_META_ALIAS },
+		{ "ZFS_VERSION",	ZFS_META_VERSION },
+		{ "ZFS_RELEASE",	ZFS_META_RELEASE },
+		{ NULL,			NULL }
+	};
 	const char *(*pa)[2];
 
 	assert(zsp != NULL);
 
-	for (pa = env_restrict; *(*pa); pa++) {
+	pa = path != NULL ? env_path : env_restrict;
+
+	for (; *(*pa); pa++) {
+		/* Use our custom $PATH if we have one */
+		if (path != NULL && strcmp((*pa)[0], "PATH") == 0)
+			(*pa)[1] = path;
+
 		_zed_event_add_var(eid, zsp, NULL, (*pa)[0], "%s", (*pa)[1]);
 	}
 }
@@ -902,7 +928,7 @@ zed_event_service(struct zed_conf *zcp)
 		while ((nvp = nvlist_next_nvpair(nvl, nvp)))
 			_zed_event_add_nvpair(eid, zsp, nvp);
 
-		_zed_event_add_env_restrict(eid, zsp);
+		_zed_event_add_env_restrict(eid, zsp, zcp->path);
 		_zed_event_add_env_preserve(eid, zsp);
 
 		_zed_event_add_var(eid, zsp, ZED_VAR_PREFIX, "PID",