Add 'zfs wait' command

Add a mechanism to wait for delete queue to drain.

When doing redacted send/recv, many workflows involve deleting files 
that contain sensitive data. Because of the way zfs handles file 
deletions, snapshots taken quickly after a rm operation can sometimes 
still contain the file in question, especially if the file is very 
large. This can result in issues for redacted send/recv users who 
expect the deleted files to be redacted in the send streams, and not 
appear in their clones.

This change duplicates much of the zpool wait related logic into a 
zfs wait command, which can be used to wait until the internal
deleteq has been drained.  Additional wait activities may be added 
in the future. 

Reviewed-by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: John Gallagher <john.gallagher@delphix.com>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #9707

tests/runfiles/common.run

@@ -288,6 +288,10 @@ tests = ['zfs_upgrade_001_pos', 'zfs_upgrade_002_pos', 'zfs_upgrade_003_pos',
     'zfs_upgrade_007_neg']
 tags = ['functional', 'cli_root', 'zfs_upgrade']
 
+[tests/functional/cli_root/zfs_wait]
+tests = ['zfs_wait_deleteq']
+tags = ['functional', 'cli_root', 'zfs_wait']
+
 [tests/functional/cli_root/zpool]
 tests = ['zpool_001_neg', 'zpool_002_pos', 'zpool_003_pos', 'zpool_colors']
 tags = ['functional', 'cli_root', 'zpool']

tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c

@@ -739,6 +739,18 @@ test_wait(const char *pool)
 	nvlist_free(optional);
 }
 
+static void
+test_wait_fs(const char *dataset)
+{
+	nvlist_t *required = fnvlist_alloc();
+
+	fnvlist_add_int32(required, "wait_activity", 2);
+
+	IOC_INPUT_TEST(ZFS_IOC_WAIT_FS, dataset, required, NULL, EINVAL);
+
+	nvlist_free(required);
+}
+
 static void
 zfs_ioc_input_tests(const char *pool)
 {
@@ -826,6 +838,7 @@ zfs_ioc_input_tests(const char *pool)
 	test_vdev_trim(pool);
 
 	test_wait(pool);
+	test_wait_fs(dataset);
 
 	/*
 	 * cleanup
@@ -980,6 +993,7 @@ validate_ioc_values(void)
 	CHECK(ZFS_IOC_BASE + 81 == ZFS_IOC_REDACT);
 	CHECK(ZFS_IOC_BASE + 82 == ZFS_IOC_GET_BOOKMARK_PROPS);
 	CHECK(ZFS_IOC_BASE + 83 == ZFS_IOC_WAIT);
+	CHECK(ZFS_IOC_BASE + 84 == ZFS_IOC_WAIT_FS);
 	CHECK(ZFS_IOC_PLATFORM_BASE + 1 == ZFS_IOC_EVENTS_NEXT);
 	CHECK(ZFS_IOC_PLATFORM_BASE + 2 == ZFS_IOC_EVENTS_CLEAR);
 	CHECK(ZFS_IOC_PLATFORM_BASE + 3 == ZFS_IOC_EVENTS_SEEK);

tests/zfs-tests/tests/functional/cli_root/Makefile.am

@@ -32,6 +32,7 @@ SUBDIRS = \
 	zfs_unmount \
 	zfs_unshare \
 	zfs_upgrade \
+	zfs_wait \
 	zpool \
 	zpool_add \
 	zpool_attach \

tests/zfs-tests/tests/functional/cli_root/zfs_wait/Makefile.am

@@ -0,0 +1,8 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/cli_root/zfs_wait
+dist_pkgdata_SCRIPTS = \
+	setup.ksh \
+	cleanup.ksh \
+	zfs_wait_deleteq.ksh
+
+dist_pkgdata_DATA = \
+	zfs_wait.kshlib

tests/zfs-tests/tests/functional/cli_root/zfs_wait/cleanup.ksh

@@ -0,0 +1,20 @@
+#!/bin/ksh -p
+#
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2018 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+default_cleanup

tests/zfs-tests/tests/functional/cli_root/zfs_wait/setup.ksh

@@ -0,0 +1,21 @@
+#!/bin/ksh -p
+#
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2018 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+DISK=${DISKS%% *}
+
+default_setup $DISK

tests/zfs-tests/tests/functional/cli_root/zfs_wait/zfs_wait.kshlib

@@ -0,0 +1,80 @@
+#!/bin/ksh
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2018, 2019 by Delphix. All rights reserved.
+#
+
+typeset -a disk_array=($(find_disks $DISKS))
+
+typeset -r DISK1=${disk_array[0]}
+typeset -r DISK2=${disk_array[1]}
+typeset -r DISK3=${disk_array[2]}
+
+#
+# When the condition it is waiting for becomes true, 'zfs wait' should return
+# promptly. We want to enforce this, but any check will be racey because it will
+# take some small but indeterminate amount of time for the waiting thread to be
+# woken up and for the process to exit.
+#
+# To deal with this, we provide a grace period after the condition becomes true
+# during which 'zfs wait' can exit. If it hasn't exited by the time the grace
+# period expires we assume something is wrong and fail the test. While there is
+# no value that can really be correct, the idea is we choose something large
+# enough that it shouldn't cause issues in practice.
+#
+typeset -r WAIT_EXIT_GRACE=2.0
+
+function proc_exists # pid
+{
+	ps -p $1 >/dev/null
+}
+
+function proc_must_exist # pid
+{
+	proc_exists $1 || log_fail "zpool process exited too soon"
+}
+
+function proc_must_not_exist # pid
+{
+	proc_exists $1 && log_fail "zpool process took too long to exit"
+}
+
+function get_time
+{
+	date +'%H:%M:%S'
+}
+
+function kill_if_running
+{
+	typeset pid=$1
+	[[ $pid ]] && proc_exists $pid && log_must kill -s TERM $pid
+}
+
+# Log a command and then start it running in the background
+function log_bkgrnd
+{
+	log_note "$(get_time) Starting cmd in background '$@'"
+	"$@" &
+}
+
+# Check that a background process has completed and exited with a status of 0
+function bkgrnd_proc_succeeded
+{
+	typeset pid=$1
+
+	log_must sleep $WAIT_EXIT_GRACE
+
+	proc_must_not_exist $pid
+	wait $pid || log_fail "process exited with status $?"
+	log_note "$(get_time) wait completed successfully"
+}

tests/zfs-tests/tests/functional/cli_root/zfs_wait/zfs_wait_deleteq.ksh

@@ -0,0 +1,57 @@
+#!/bin/ksh -p
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2018 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_wait/zfs_wait.kshlib
+
+#
+# DESCRIPTION:
+# 'zfs wait' works when waiting for checkpoint discard to complete.
+#
+# STRATEGY:
+# 1. Create a file
+# 2. Open a file descriptor pointing to that file.
+# 3. Delete the file.
+# 4. Start a background process waiting for the delete queue to empty.
+# 5. Verify that the command doesn't return immediately.
+# 6. Close the open file descriptor.
+# 7. Verify that the command returns soon after the descriptor is closed.
+#
+
+function cleanup
+{
+	kill_if_running $pid
+	exec 3<&-
+}
+
+
+typeset -r TESTFILE="/$TESTPOOL/testfile"
+typeset pid
+
+log_onexit cleanup
+
+log_must touch $TESTFILE
+exec 3<> $TESTFILE
+log_must rm $TESTFILE
+log_bkgrnd zfs wait -t deleteq $TESTPOOL
+pid=$!
+proc_must_exist $pid
+
+exec 3<&-
+log_must sleep 0.5
+bkgrnd_proc_succeeded $pid
+
+log_pass "'zfs wait -t discard' works."