Add callback for zfs_multihost_interval

Add a callback to wake all running mmp threads when
zfs_multihost_interval is changed.

This is necessary when the interval is changed from a very large value
to a significantly lower one, while pools are imported that have the
multihost property enabled.

Without this commit, the mmp thread does not wake up and detect the new
interval until after it has waited the old multihost interval time.  A
user monitoring mmp writes via the provided kstat would be led to
believe that the changed setting did not work.

Added a test in the ZTS under mmp to verify the new functionality is
working.

Added a test to ztest which starts and stops mmp threads, and calls into
the code to signal sleeping mmp threads, to test for deadlocks or
similar locking issues.

Reviewed-by: Giuseppe Di Natale <dinatale2@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Olaf Faaland <faaland1@llnl.gov>
Closes #6387

tests/runfiles/linux.run

@@ -413,7 +413,7 @@ tests = ['mmap_write_001_pos', 'mmap_read_001_pos']
 [tests/functional/mmp]
 tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
     'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import',
-    'mmp_write_uberblocks']
+    'mmp_write_uberblocks', 'mmp_reset_interval']
 
 [tests/functional/mount]
 tests = ['umount_001', 'umountall_001']

tests/zfs-tests/tests/functional/mmp/Makefile.am

@@ -8,6 +8,7 @@ dist_pkgdata_SCRIPTS = \
 	mmp_inactive_import.ksh \
 	mmp_exported_import.ksh \
 	mmp_write_uberblocks.ksh \
+	mmp_reset_interval.ksh \
 	setup.ksh \
 	cleanup.ksh \
 	mmp.kshlib \

tests/zfs-tests/tests/functional/mmp/mmp.cfg

@@ -31,7 +31,9 @@ export TXG_TIMEOUT_DEFAULT=5
 export MMP_POOL=mmppool
 export MMP_DIR=$TEST_BASE_DIR/mmp
 export MMP_HISTORY=100
+export MMP_HISTORY_OFF=0
 
+export MMP_INTERVAL_HOUR=$((60*60*1000))
 export MMP_INTERVAL_DEFAULT=1000
 export MMP_INTERVAL_MIN=100
 

tests/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh

@@ -0,0 +1,67 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+# DESCRIPTION:
+#	Ensure that the MMP thread is notified when zfs_multihost_interval is
+#	reduced.
+#
+# STRATEGY:
+#	1. Set zfs_multihost_interval to much longer than the test duration
+#	2. Create a zpool and enable multihost
+#	3. Verify no MMP writes occurred
+#	4. Set zfs_multihost_interval to 1 second
+#	5. Sleep briefly
+#	6. Verify MMP writes began
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/mmp/mmp.cfg
+. $STF_SUITE/tests/functional/mmp/mmp.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+	default_cleanup_noexit
+	log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT
+	log_must mmp_clear_hostid
+}
+
+log_assert "mmp threads notified when zfs_multihost_interval reduced"
+log_onexit cleanup
+
+log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_HOUR
+log_must mmp_set_hostid $HOSTID1
+
+default_setup_noexit $DISK
+log_must zpool set multihost=on $TESTPOOL
+
+prev_count=$(wc -l /proc/spl/kstat/zfs/$TESTPOOL/multihost | cut -f1 -d' ')
+log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT
+
+# slight delay to allow time for the mmp write to complete
+sleep 1
+curr_count=$(wc -l /proc/spl/kstat/zfs/$TESTPOOL/multihost | cut -f1 -d' ')
+
+if [ $curr_count -eq $prev_count ]; then
+	log_fail "mmp writes did not start when zfs_multihost_interval reduced"
+fi
+
+log_pass "mmp threads notified when zfs_multihost_interval reduced"