From ae488e496f7d740fe78fc6a6d13a5b7caedce445 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 30 Jan 2026 15:05:40 -0800 Subject: [PATCH] ZTS: update the relevant mmp test cases - mmp_concurrent_import: added test case to verify that concurrent import correctness. The pool may only be imported once. - mmp_exported_import: an activity check is now required for pools which were cleanly exported if the system and pool hostids don't match. - mmp_inactive_import: an activity check is now required for any pool which wasn't cleanly exported, even if the system and pool hostids match. - mmp_on_uberblocks: updated expected uberblocks to take in to account the value MMP_INTERVAL_DEFAULT is set too. - mmp_reset_interval: reduce the number of iterations from 10 to 3. This is sufficient to verify functionality and significantly speeds up the test. - mmp_on_uberblocks: adjust the thresholds and increase the runtime to avoid false positives observed in CI. - Update tests to use 'zhack action idle' instead of ztest to improve the reliability of the tests. - Add additional log_note messages to test cases which have multiple verification steps to make it clear which portion of a test failed when reviewing the logs. - Replace default_setup/cleanup_noexit calls with 'zpool create' and 'zpool destroy' calls to avoid additional unnecessary dataset creation work. - Update activity/noactivity check helper functions to use the ZFS_LOAD_INFO_DEBUG information now available from 'zpool import' to determine if this activity check ran and why. This is more reliable in the CI than measuring the runtime. - Removed all mmp tests from the zts-report.py exceptions list. Signed-off-by: Brian Behlendorf Reviewed-by: Tony Hutter Reviewed-by: Olaf Faaland Reviewed-by: Akash B --- module/zfs/spa.c | 4 +- tests/runfiles/linux.run | 4 +- tests/test-runner/bin/zts-report.py.in | 4 - tests/zfs-tests/tests/Makefile.am | 1 + tests/zfs-tests/tests/functional/mmp/mmp.cfg | 6 +- .../zfs-tests/tests/functional/mmp/mmp.kshlib | 47 ++++--- .../functional/mmp/mmp_active_import.ksh | 42 +++--- .../functional/mmp/mmp_concurrent_import.ksh | 133 ++++++++++++++++++ .../functional/mmp/mmp_exported_import.ksh | 16 ++- .../tests/functional/mmp/mmp_hostid.ksh | 8 +- .../functional/mmp/mmp_inactive_import.ksh | 20 ++- .../tests/functional/mmp/mmp_on_off.ksh | 4 +- .../tests/functional/mmp/mmp_on_thread.ksh | 4 +- .../functional/mmp/mmp_on_uberblocks.ksh | 14 +- .../tests/functional/mmp/mmp_on_zdb.ksh | 3 +- .../functional/mmp/mmp_reset_interval.ksh | 8 +- .../functional/mmp/mmp_write_distribution.ksh | 2 +- .../functional/mmp/mmp_write_uberblocks.ksh | 4 +- .../functional/mmp/multihost_history.ksh | 2 + 19 files changed, 246 insertions(+), 80 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/mmp/mmp_concurrent_import.ksh diff --git a/module/zfs/spa.c b/module/zfs/spa.c index ef726733f..71974d4f3 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -4099,8 +4099,8 @@ spa_activity_check_tryimport(spa_t *spa, uberblock_t *spa_ub, hrtime_t extra_delay = MMP_IMPORT_VERIFY_ITERS * MSEC2NSEC(MMP_INTERVAL_VALID(spa_ub) ? MMP_INTERVAL(spa_ub) : MMP_MIN_INTERVAL); - cmn_err(CE_NOTE, "pool '%s' multihost activity check " - "required, %llu seconds remaining", spa_load_name(spa), + cmn_err(CE_NOTE, "pool '%s' activity check required, " + "%llu seconds remaining", spa_load_name(spa), (u_longlong_t)MAX(NSEC2SEC(import_delay + extra_delay), 1)); spa_import_progress_set_notes(spa, "Checking MMP activity, " "waiting %llu ms", (u_longlong_t)NSEC2MSEC(import_delay)); diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 339361cc2..a65633dd8 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -159,8 +159,10 @@ tags = ['functional', 'mmap'] tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval', 'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import', 'mmp_write_uberblocks', 'mmp_reset_interval', 'multihost_history', - 'mmp_on_zdb', 'mmp_write_distribution', 'mmp_hostid', 'mmp_write_slow_disk'] + 'mmp_on_zdb', 'mmp_write_distribution', 'mmp_hostid', 'mmp_write_slow_disk', + 'mmp_concurrent_import'] tags = ['functional', 'mmp'] +timeout = 1200 [tests/functional/mount:Linux] tests = ['umount_unlinked_drain', 'mount_loopback'] diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 5bc65f993..a3d367971 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -247,7 +247,6 @@ maybe = { 'l2arc/persist_l2arc_005_pos': ['FAIL', known_reason], 'largest_pool/largest_pool_001_pos': ['FAIL', known_reason], 'mmap/mmap_sync_001_pos': ['FAIL', known_reason], - 'mmp/mmp_on_uberblocks': ['FAIL', known_reason], 'pam/setup': ['SKIP', "pamtester might be not available"], 'pool_checkpoint/checkpoint_discard_busy': ['FAIL', 11946], 'projectquota/setup': ['SKIP', exec_reason], @@ -366,9 +365,6 @@ elif sys.platform.startswith('linux'): 'io/io_uring': ['SKIP', 'io_uring support required'], 'limits/filesystem_limit': ['SKIP', known_reason], 'limits/snapshot_limit': ['SKIP', known_reason], - 'mmp/mmp_active_import': ['FAIL', known_reason], - 'mmp/mmp_exported_import': ['FAIL', known_reason], - 'mmp/mmp_inactive_import': ['FAIL', known_reason], 'stat/statx_dioalign': ['SKIP', 'statx_reason'], }) diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index a9e08ae30..19b31d2ea 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -1719,6 +1719,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/mmap/setup.ksh \ functional/mmp/cleanup.ksh \ functional/mmp/mmp_active_import.ksh \ + functional/mmp/mmp_concurrent_import.ksh \ functional/mmp/mmp_exported_import.ksh \ functional/mmp/mmp_hostid.ksh \ functional/mmp/mmp_inactive_import.ksh \ diff --git a/tests/zfs-tests/tests/functional/mmp/mmp.cfg b/tests/zfs-tests/tests/functional/mmp/mmp.cfg index 18c43424e..c9356aeae 100644 --- a/tests/zfs-tests/tests/functional/mmp/mmp.cfg +++ b/tests/zfs-tests/tests/functional/mmp/mmp.cfg @@ -25,6 +25,8 @@ export DISK=${DISKS%% *} export HOSTID_FILE="/etc/hostid" export HOSTID1=01234567 export HOSTID2=89abcdef +export HOSTID3=aaaabbbb +export HOSTID4=ccccdddd export TXG_TIMEOUT_LONG=5000 export TXG_TIMEOUT_DEFAULT=5 @@ -32,7 +34,7 @@ export TXG_TIMEOUT_DEFAULT=5 export MMP_POOL=mmppool export MMP_DIR=$TEST_BASE_DIR/mmp export MMP_CACHE=$MMP_DIR/zpool.cache -export MMP_ZTEST_LOG=$MMP_DIR/ztest.log +export MMP_ZHACK_LOG=$MMP_DIR/zhack.log export MMP_HISTORY=100 export MMP_HISTORY_OFF=0 @@ -43,5 +45,3 @@ export MMP_INTERVAL_MIN=100 export MMP_IMPORT_INTERVALS=20 export MMP_FAIL_INTERVALS_DEFAULT=10 export MMP_FAIL_INTERVALS_MIN=2 - -export MMP_TEST_DURATION_DEFAULT=$((MMP_IMPORT_INTERVALS*MMP_INTERVAL_DEFAULT/1000)) diff --git a/tests/zfs-tests/tests/functional/mmp/mmp.kshlib b/tests/zfs-tests/tests/functional/mmp/mmp.kshlib index bd29c5080..8717b1912 100644 --- a/tests/zfs-tests/tests/functional/mmp/mmp.kshlib +++ b/tests/zfs-tests/tests/functional/mmp/mmp.kshlib @@ -99,11 +99,11 @@ function mmp_pool_create_simple # pool dir log_must zpool set multihost=on $pool } -function mmp_pool_create # pool dir +function mmp_pool_create_zhack # pool dir { typeset pool=${1:-$MMP_POOL} typeset dir=${2:-$MMP_DIR} - typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool" + typeset opts="-d $dir action idle -t120 $pool" mmp_pool_create_simple $pool $dir @@ -112,11 +112,11 @@ function mmp_pool_create # pool dir log_must mmp_clear_hostid log_must mmp_set_hostid $HOSTID2 - log_note "Starting ztest in the background as hostid $HOSTID1" - log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >$MMP_ZTEST_LOG 2>&1 &" + log_note "Starting zhack in the background as hostid $HOSTID1" + log_must eval "ZFS_HOSTID=$HOSTID1 zhack $opts >$MMP_ZHACK_LOG 2>&1 &" while ! is_pool_imported "$pool" "-d $dir"; do - log_must pgrep ztest + log_must pgrep zhack log_must sleep 5 done } @@ -126,10 +126,10 @@ function mmp_pool_destroy # pool dir typeset pool=${1:-$MMP_POOL} typeset dir=${2:-$MMP_DIR} - ZTESTPID=$(pgrep ztest) - if [ -n "$ZTESTPID" ]; then - log_must kill $ZTESTPID - wait $ZTESTPID + ZHACKPID=$(pgrep zhack) + if [ -n "$ZHACKPID" ]; then + log_must kill $ZHACKPID + wait $ZHACKPID fi if poolexists $pool; then @@ -158,33 +158,34 @@ function import_no_activity_check # pool opts typeset pool=$1 typeset opts=$2 - typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1)) - - SECONDS=0 - zpool import $opts $pool + RESULT=$(ZFS_LOAD_INFO_DEBUG=1 zpool import $opts $pool) typeset rc=$? - if [[ $SECONDS -gt $max_duration ]]; then - log_fail "ERROR: import_no_activity_check unexpected activity \ -check (${SECONDS}s gt $max_duration)" + # mmp_result: 3 (ESRCH) no activity check was run not required + # mmp_result: 6 (ENXIO) no activity check was run hostid not set + if ! echo "$RESULT" | grep -q "mmp_result: 3" && + ! echo "$RESULT" | grep -q "mmp_result: 6"; then + log_note "ERROR: $RESULT" + log_fail "ERROR: import_no_activity_check unexpected activity check" fi return $rc } -function import_activity_check # pool opts act_test_duration +function import_activity_check # pool opts { typeset pool=$1 typeset opts=$2 - typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT} - SECONDS=0 - zpool import $opts $pool + RESULT=$(ZFS_LOAD_INFO_DEBUG=1 zpool import $opts $pool) typeset rc=$? - if [[ $SECONDS -le $min_duration ]]; then - log_fail "ERROR: import_activity_check expected activity check \ -(${SECONDS}s le min_duration $min_duration)" + # mmp_result: 0 (Success) check was run no activity detected + # mmp_result: 121 (EREMOTEIO) check was run activity detected + # mmp_result: 4 (EINTR) check was run but interrupted by user + if ! echo "$RESULT" | grep -q "mmp_result: 0"; then + log_note "ERROR: $RESULT" + log_fail "ERROR: import_activity_check expected activity check" fi return $rc diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_active_import.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_active_import.ksh index 62a5262a6..48cfeaa31 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_active_import.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_active_import.ksh @@ -24,10 +24,10 @@ # with one hostid be importable by a host with a different hostid. # # STRATEGY: -# 1. Simulate an active pool on another host with ztest. +# 1. Simulate an active pool on another host with zhack. # 2. Verify 'zpool import' reports an active pool. # 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool. -# 4. Kill ztest to make pool eligible for import. +# 4. Kill zhack to make pool eligible for import. # 5. Verify 'zpool import' fails with the expected error message. # 6. Verify 'zpool import $MMP_POOL' fails with the expected message. # 7. Verify 'zpool import -f $MMP_POOL' can now import the pool. @@ -44,16 +44,16 @@ function cleanup { mmp_pool_destroy $MMP_POOL $MMP_DIR log_must mmp_clear_hostid - ZTESTPID=$(pgrep ztest) - if [ -n "$ZTESTPID" ]; then - for pid in $ZTESTPID; do + ZHACKPID=$(pgrep zhack) + if [ -n "$ZHACKPID" ]; then + for pid in $ZHACKPID; do log_must kill -9 $pid done else - # if ztest not running and log present, ztest crashed - if [ -f $MMP_ZTEST_LOG ]; then - log_note "ztest appears to have crashed. Tail of log:" - tail -n 50 $MMP_ZTEST_LOG + # if zhack is not running and log present, zhack crashed + if [ -f $MMP_ZHACK_LOG ]; then + log_note "zhack appears to have crashed. Tail of log:" + tail -n 50 $MMP_ZHACK_LOG fi fi } @@ -61,15 +61,18 @@ function cleanup log_assert "multihost=on|off active pool activity checks" log_onexit cleanup -# 1. Simulate an active pool on another host with ztest. +# 1. Simulate an active pool on another host with zhack. +log_note "Simulate an active pool on another host with zhack" mmp_pool_destroy $MMP_POOL $MMP_DIR -mmp_pool_create $MMP_POOL $MMP_DIR +mmp_pool_create_zhack $MMP_POOL $MMP_DIR # 2. Verify 'zpool import' reports an active pool. +log_note "Verify 'zpool import' reports an active pool" log_must mmp_set_hostid $HOSTID2 log_must is_pool_imported $MMP_POOL "-d $MMP_DIR" # 3. Verify 'zpool import [-f] $MMP_POOL' cannot import the pool. +log_note "Verify 'zpool import [-f] $MMP_POOL' cannot import the pool" MMP_IMPORTED_MSG="Cannot import '$MMP_POOL': pool is imported" log_must try_pool_import $MMP_POOL "-d $MMP_DIR" "$MMP_IMPORTED_MSG" @@ -84,14 +87,15 @@ for i in {1..10}; do "$MMP_IMPORTED_MSG" done -# 4. Kill ztest to make pool eligible for import. Poll with 'zpool status'. -ZTESTPID=$(pgrep ztest) -if [ -n "$ZTESTPID" ]; then - log_must kill -9 $ZTESTPID +# 4. Kill zhack to make pool eligible for import. Poll with 'zpool status'. +log_note "Kill zhack to make pool eligible for import. Poll with 'zpool status'" +ZHACKPID=$(pgrep zhack) +if [ -n "$ZHACKPID" ]; then + log_must kill -9 $ZHACKPID fi log_must wait_pool_imported $MMP_POOL "-d $MMP_DIR" -if [ -f $MMP_ZTEST_LOG ]; then - log_must rm $MMP_ZTEST_LOG +if [ -f $MMP_ZHACK_LOG ]; then + log_must rm $MMP_ZHACK_LOG fi # 5. Verify 'zpool import' fails with the expected error message, when @@ -99,6 +103,7 @@ fi # - hostid=matches - safe to import the pool # - hostid=different - previously imported on a different system # +log_note "Verify 'zpool import' fails with the expected error message" log_must mmp_clear_hostid MMP_IMPORTED_MSG="Set a unique system hostid" log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "action" "$MMP_IMPORTED_MSG" @@ -113,13 +118,16 @@ MMP_IMPORTED_MSG="The pool was last accessed by another system." log_must check_pool_import $MMP_POOL "-d $MMP_DIR" "status" "$MMP_IMPORTED_MSG" # 6. Verify 'zpool import $MMP_POOL' fails with the expected message. +log_note "Verify 'zpool import $MMP_POOL' fails with the expected message" MMP_IMPORTED_MSG="pool was previously in use from another system." log_must try_pool_import $MMP_POOL "-d $MMP_DIR" "$MMP_IMPORTED_MSG" # 7. Verify 'zpool import -f $MMP_POOL' can now import the pool. +log_note "Verify 'zpool import -f $MMP_POOL' can now import the pool" log_must import_activity_check $MMP_POOL "-f -d $MMP_DIR" # 8 Verify pool may be exported/imported without -f argument. +log_note "Verify pool may be exported/imported without -f argument" log_must zpool export $MMP_POOL log_must import_no_activity_check $MMP_POOL "-d $MMP_DIR" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_concurrent_import.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_concurrent_import.ksh new file mode 100755 index 000000000..7fab2634d --- /dev/null +++ b/tests/zfs-tests/tests/functional/mmp/mmp_concurrent_import.ksh @@ -0,0 +1,133 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by Lawrence Livermore National Security, LLC. +# + +# DESCRIPTION: +# Verify that even when importing a shared pool simultaneously +# on systems with different host ids at most one will succeed. +# +# STRATEGY: +# 1. Create an multihost enabled pool +# 2. zhack imports: $HOSTID1 (matching) and $HOSTID1 (matching) +# 3. zhack imports: $HOSTID1 (matching) and $HOSTID2 (different) +# 4. zhack imports: $HOSTID3 (different) and $HOSTID4 (different) +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/mmp/mmp.cfg +. $STF_SUITE/tests/functional/mmp/mmp.kshlib + +verify_runnable "both" + +function cleanup +{ + ZHACKPIDS=$(pgrep zhack) + if [ -n "$ZHACKPIDS" ]; then + for pid in $ZHACKPIDS; do + log_must kill -9 $pid + done + fi + + log_must rm -f $MMP_ZHACK_LOG.1 $MMP_ZHACK_LOG.2 + + mmp_pool_destroy $MMP_POOL $MMP_DIR + log_must mmp_clear_hostid +} + +# Verify that pool was imported by at most one of the zhack processes. +# Check both the return code and expected import message. +function verify_zhack +{ + IMPORT_COUNT=0 + IMPORT_MSGS=0 + + ZHACKPIDS=$(pgrep zhack) + for pid in $ZHACKPIDS; do + wait $pid + STATUS=$? + if [[ $STATUS -eq 0 ]]; then + (( IMPORT_COUNT++ )) + fi + log_note "PID $pid exited with status $STATUS" + done + + grep -H "Imported pool $MMP_POOL" $MMP_ZHACK_LOG.1 && (( IMPORT_MSGS++ )) + grep -H "Imported pool $MMP_POOL" $MMP_ZHACK_LOG.2 && (( IMPORT_MSGS++ )) + + if [[ $IMPORT_MSGS -gt 1 ]]; then + cat $MMP_ZHACK_LOG.* + log_fail "Multiple import success messages" + fi + + if [[ $IMPORT_COUNT -gt 1 ]]; then + cat $MMP_ZHACK_LOG.* + log_fail "Multiple import success return codes" + fi + + if [[ $IMPORT_MSGS -ne $IMPORT_COUNT ]]; then + cat $MMP_ZHACK_LOG.* + log_fail "Messages ($IMPORT_MSGS) differs from count ($IMPORT_COUNT)" + fi +} + +OPTS="-d $MMP_DIR action idle -t5 $MMP_POOL" + +log_assert "multihost=on concurrent imports" +log_onexit cleanup + +# 1. Create a multihost enabled pool with HOSTID1 +mmp_pool_create_simple $MMP_POOL $MMP_DIR +log_must zpool export -F $MMP_POOL + +# 2. zhack imports: $HOSTID1 (matching) and $HOSTID1 (matching) +# Activity check required because the pool was exported with -F above, the +# claim phase will detect the double import despite matching hostids. +log_note "zhack import with $HOSTID1 (matching) and $HOSTID1 (matching)" +log_must eval "ZFS_HOSTID=$HOSTID1 zhack $OPTS >$MMP_ZHACK_LOG.1 2>&1 &" +log_must eval "ZFS_HOSTID=$HOSTID1 zhack $OPTS >$MMP_ZHACK_LOG.2 2>&1 &" +log_must verify_zhack + +mmp_clear_hostid +mmp_set_hostid $HOSTID1 +log_must import_activity_check $MMP_POOL "-d $MMP_DIR" +log_must zpool export $MMP_POOL + +# 3. zhack imports: $HOSTID1 (matching) and $HOSTID2 (different) +# Activity check skipped for HOSTID1 it is expected to import successfully. +# zhack with HOSTID2 will run the activity check and detect the active pool. +log_note "zhack import with $HOSTID1 (matching) and $HOSTID2 (different)" +log_must eval "ZFS_HOSTID=$HOSTID1 zhack $OPTS >$MMP_ZHACK_LOG.1 2>&1 &" +log_must eval "ZFS_HOSTID=$HOSTID2 zhack $OPTS >$MMP_ZHACK_LOG.2 2>&1 &" +log_must verify_zhack + +mmp_clear_hostid +mmp_set_hostid $HOSTID3 +log_must import_activity_check $MMP_POOL "-d $MMP_DIR" +log_must zpool export $MMP_POOL + +# 4. zhack imports: $HOSTID1 (different) and $HOSTID2 (different) +# Both zhacks will run the activity checks, depending on the exact timing +# one may succeed and the other fail, or both may fail. +log_note "zhack import with $HOSTID1 (different) and $HOSTID2 (different)" +log_must eval "ZFS_HOSTID=$HOSTID1 zhack $OPTS >$MMP_ZHACK_LOG.1 2>&1 &" +log_must eval "ZFS_HOSTID=$HOSTID2 zhack $OPTS >$MMP_ZHACK_LOG.2 2>&1 &" +log_must verify_zhack + +log_pass "multihost=on concurrent imports" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh index 1cc55213f..d7846ea90 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_exported_import.ksh @@ -28,7 +28,7 @@ # 3. Verify multihost=off and hostids differ (no activity check) # 4. Verify multihost=off and hostid zero allowed (no activity check) # 5. Verify multihost=on and hostids match (no activity check) -# 6. Verify multihost=on and hostids differ (no activity check) +# 6. Verify multihost=on and hostids differ (activity check) # 7. Verify multihost=on and hostid zero fails (no activity check) # @@ -40,7 +40,7 @@ verify_runnable "both" function cleanup { - default_cleanup_noexit + datasetexists $TESTPOOL && destroy_pool $TESTPOOL log_must mmp_clear_hostid } @@ -49,9 +49,10 @@ log_onexit cleanup # 1. Create a zpool log_must mmp_set_hostid $HOSTID1 -default_setup_noexit $DISK +log_must zpool create -f $TESTPOOL $DISK # 2. Verify multihost=off and hostids match (no activity check) +log_note "Verify multihost=off and hostids match (no activity check)" log_must zpool set multihost=off $TESTPOOL for opt in "" "-f"; do @@ -60,6 +61,7 @@ for opt in "" "-f"; do done # 3. Verify multihost=off and hostids differ (no activity check) +log_note "Verify multihost=off and hostids differ (no activity check)" for opt in "" "-f"; do log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 log_must zpool export $TESTPOOL @@ -69,6 +71,7 @@ for opt in "" "-f"; do done # 4. Verify multihost=off and hostid zero allowed (no activity check) +log_note "Verify multihost=off and hostid zero allowed (no activity check)" log_must mmp_clear_hostid for opt in "" "-f"; do @@ -77,6 +80,7 @@ for opt in "" "-f"; do done # 5. Verify multihost=on and hostids match (no activity check) +log_note "Verify multihost=on and hostids match (no activity check)" log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 log_must zpool set multihost=on $TESTPOOL @@ -85,16 +89,18 @@ for opt in "" "-f"; do log_must import_no_activity_check $TESTPOOL $opt done -# 6. Verify multihost=on and hostids differ (no activity check) +# 6. Verify multihost=on and hostids differ (activity check) +log_note "Verify multihost=on and hostids differ (activity check)" for opt in "" "-f"; do log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 log_must zpool export $TESTPOOL log_must mmp_clear_hostid log_must mmp_set_hostid $HOSTID2 - log_must import_no_activity_check $TESTPOOL $opt + log_must import_activity_check $TESTPOOL $opt done # 7. Verify multihost=on and hostid zero fails (no activity check) +log_note "Verify multihost=on and hostid zero fails (no activity check)" log_must zpool export $TESTPOOL log_must mmp_clear_hostid diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_hostid.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_hostid.ksh index 6144db85c..e10954a4b 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_hostid.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_hostid.ksh @@ -40,7 +40,7 @@ verify_runnable "both" function cleanup { - default_cleanup_noexit + datasetexists $MMP_POOL && destroy_pool $MMP_POOL log_must rm $MMP_DIR/file.{0,1,2,3,4,5} log_must rmdir $MMP_DIR log_must mmp_clear_hostid @@ -56,7 +56,7 @@ log_must mkdir -p $MMP_DIR log_must truncate -s $MINVDEVSIZE $MMP_DIR/file.{0,1,2,3,4,5} # 1. Create a non-redundant pool -log_must zpool create $MMP_POOL $MMP_DIR/file.0 +log_must zpool create -f $MMP_POOL $MMP_DIR/file.0 # 2. Create an 'etc' dataset containing a valid hostid file; caching is # disabled on the dataset to force the hostid to be read from disk. @@ -71,16 +71,19 @@ mntpnt_fs=$(get_prop mountpoint $MMP_POOL/fs) log_must mkfile 1M $mntpnt_fs/file # 4. Verify multihost cannot be enabled until the /etc/hostid is linked +log_note "Verify multihost cannot be enabled until the /etc/hostid is linked" log_mustnot zpool set multihost=on $MMP_POOL log_mustnot ls -l $HOSTID_FILE log_must ln -s $mntpnt_etc/hostid $HOSTID_FILE log_must zpool set multihost=on $MMP_POOL # 5. Verify vdevs may be attached and detached +log_note "Verify vdevs may be attached and detached" log_must zpool attach $MMP_POOL $MMP_DIR/file.0 $MMP_DIR/file.1 log_must zpool detach $MMP_POOL $MMP_DIR/file.1 # 6. Verify normal, cache, log and special vdevs can be added +log_note "Verify normal, cache, log and special vdevs can be added" log_must zpool add $MMP_POOL $MMP_DIR/file.1 log_must zpool add $MMP_POOL $MMP_DIR/file.2 log_must zpool add $MMP_POOL cache $MMP_DIR/file.3 @@ -88,6 +91,7 @@ log_must zpool add $MMP_POOL log $MMP_DIR/file.4 log_must zpool add $MMP_POOL special $MMP_DIR/file.5 # 7. Verify normal, cache, and log vdevs can be removed +log_note "Verify normal, cache, and log vdevs can be removed" log_must zpool remove $MMP_POOL $MMP_DIR/file.2 log_must zpool remove $MMP_POOL $MMP_DIR/file.3 log_must zpool remove $MMP_POOL $MMP_DIR/file.4 diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh index 7aeb0c6f4..4c6b9c2e0 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_inactive_import.ksh @@ -27,7 +27,7 @@ # 2. Verify multihost=off and hostids match (no activity check) # 3. Verify multihost=off and hostids differ (no activity check) # 4. Verify multihost=off and hostid allowed (no activity check) -# 5. Verify multihost=on and hostids match (no activity check) +# 5. Verify multihost=on and hostids match (activity check) # 6. Verify multihost=on and hostids differ (activity check) # 7. Verify mmp_write and mmp_fail are set correctly # 8. Verify multihost=on and hostid zero fails (no activity check) @@ -42,7 +42,7 @@ verify_runnable "both" function cleanup { - default_cleanup_noexit + datasetexists $TESTPOOL && destroy_pool $TESTPOOL log_must mmp_clear_hostid log_must set_tunable64 MULTIHOST_INTERVAL $MMP_INTERVAL_DEFAULT } @@ -52,9 +52,10 @@ log_onexit cleanup # 1. Create a zpool log_must mmp_set_hostid $HOSTID1 -default_setup_noexit $DISK +log_must zpool create -f $TESTPOOL $DISK # 2. Verify multihost=off and hostids match (no activity check) +log_note "Verify multihost=off and hostids match (no activity check)" log_must zpool set multihost=off $TESTPOOL for opt in "" "-f"; do @@ -63,6 +64,7 @@ for opt in "" "-f"; do done # 3. Verify multihost=off and hostids differ (no activity check) +log_note "Verify multihost=off and hostids differ (no activity check)" log_must zpool export -F $TESTPOOL log_must mmp_clear_hostid log_must mmp_set_hostid $HOSTID2 @@ -70,21 +72,24 @@ log_mustnot import_no_activity_check $TESTPOOL "" log_must import_no_activity_check $TESTPOOL "-f" # 4. Verify multihost=off and hostid zero allowed (no activity check) +log_note "Verify multihost=off and hostid zero allowed (no activity check)" log_must zpool export -F $TESTPOOL log_must mmp_clear_hostid log_mustnot import_no_activity_check $TESTPOOL "" log_must import_no_activity_check $TESTPOOL "-f" -# 5. Verify multihost=on and hostids match (no activity check) +# 5. Verify multihost=on and hostids match (activity check) +log_note "Verify multihost=on and hostids match (activity check)" log_must mmp_pool_set_hostid $TESTPOOL $HOSTID1 log_must zpool set multihost=on $TESTPOOL for opt in "" "-f"; do log_must zpool export -F $TESTPOOL - log_must import_no_activity_check $TESTPOOL $opt + log_must import_activity_check $TESTPOOL $opt done # 6. Verify multihost=on and hostids differ (activity check) +log_note "Verify multihost=on and hostids differ (activity check)" log_must zpool export -F $TESTPOOL log_must mmp_clear_hostid log_must mmp_set_hostid $HOSTID2 @@ -92,10 +97,12 @@ log_mustnot import_activity_check $TESTPOOL "" log_must import_activity_check $TESTPOOL "-f" # 7. Verify mmp_write and mmp_fail are set correctly +log_note "Verify mmp_write and mmp_fail are set correctly" log_must zpool export -F $TESTPOOL log_must verify_mmp_write_fail_present ${DISK[0]} # 8. Verify multihost=on and hostid zero fails (no activity check) +log_note "Verify multihost=on and hostid zero fails (no activity check)" log_must mmp_clear_hostid MMP_IMPORTED_MSG="Set a unique system hostid" log_must check_pool_import $TESTPOOL "-f" "action" "$MMP_IMPORTED_MSG" @@ -104,9 +111,10 @@ log_mustnot import_no_activity_check $TESTPOOL "-f" # 9. Verify activity check duration based on mmp_write and mmp_fail # Specify a short test via tunables but import pool imported while # tunables set to default duration. +log_note "Verify activity check duration based on mmp_write and mmp_fail" log_must set_tunable64 MULTIHOST_INTERVAL $MMP_INTERVAL_MIN log_must mmp_clear_hostid log_must mmp_set_hostid $HOSTID1 -log_must import_activity_check $TESTPOOL "-f" $MMP_TEST_DURATION_DEFAULT +log_must import_activity_check $TESTPOOL "-f" log_pass "multihost=on|off inactive pool activity checks passed" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_on_off.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_on_off.ksh index 2dcebd432..b3f8f361a 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_on_off.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_on_off.ksh @@ -44,7 +44,7 @@ verify_runnable "both" function cleanup { - default_cleanup_noexit + datasetexists $TESTPOOL && destroy_pool $TESTPOOL log_must set_tunable64 TXG_TIMEOUT $TXG_TIMEOUT_DEFAULT log_must set_tunable64 MULTIHOST_INTERVAL $MMP_INTERVAL_DEFAULT log_must rm -f $PREV_UBER $CURR_UBER @@ -58,7 +58,7 @@ log_must set_tunable64 MULTIHOST_INTERVAL $MMP_INTERVAL_MIN log_must set_tunable64 TXG_TIMEOUT $TXG_TIMEOUT_LONG log_must mmp_set_hostid $HOSTID1 -default_setup_noexit $DISK +log_must zpool create -f $TESTPOOL $DISK log_must zpool set multihost=off $TESTPOOL log_must eval "zdb -u $TESTPOOL > $PREV_UBER" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh index 8b8d6dc44..932160d30 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_on_thread.ksh @@ -39,7 +39,7 @@ verify_runnable "both" function cleanup { - default_cleanup_noexit + datasetexists $TESTPOOL && destroy_pool $TESTPOOL log_must set_tunable64 TXG_TIMEOUT $TXG_TIMEOUT_DEFAULT log_must rm -f $PREV_UBER $CURR_UBER log_must mmp_clear_hostid @@ -51,7 +51,7 @@ log_onexit cleanup log_must set_tunable64 TXG_TIMEOUT $TXG_TIMEOUT_LONG log_must mmp_set_hostid $HOSTID1 -default_setup_noexit $DISK +log_must zpool create -f $TESTPOOL $DISK log_must zpool set multihost=on $TESTPOOL log_must eval "zdb -u $TESTPOOL > $PREV_UBER" diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh index 284ca9283..53c0b0729 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_on_uberblocks.ksh @@ -39,15 +39,16 @@ verify_runnable "both" UBER_CHANGES=0 -EXPECTED=$(($(echo $DISKS | wc -w) * 10)) +DURATION=10 +EXPECTED=$((($(echo $DISKS | wc -w) * $DURATION * 1000) / $MMP_INTERVAL_DEFAULT)) FUDGE=$((EXPECTED * 20 / 100)) MIN_UB_WRITES=$((EXPECTED - FUDGE)) MAX_UB_WRITES=$((EXPECTED + FUDGE)) -MIN_SEQ_VALUES=7 +MIN_SEQ_VALUES=10 function cleanup { - default_cleanup_noexit + datasetexists $TESTPOOL && destroy_pool $TESTPOOL log_must set_tunable64 MULTIHOST_INTERVAL $MMP_INTERVAL_DEFAULT set_tunable64 TXG_TIMEOUT $TXG_TIMEOUT_DEFAULT log_must mmp_clear_hostid @@ -56,13 +57,14 @@ function cleanup log_assert "Ensure MMP uberblocks update at the correct interval" log_onexit cleanup +log_must set_tunable64 MULTIHOST_INTERVAL $MMP_INTERVAL_DEFAULT log_must set_tunable64 TXG_TIMEOUT $TXG_TIMEOUT_LONG log_must mmp_set_hostid $HOSTID1 -default_setup_noexit "$DISKS" +log_must zpool create -f $TESTPOOL $DISKS log_must zpool set multihost=on $TESTPOOL clear_mmp_history -UBER_CHANGES=$(count_mmp_writes $TESTPOOL 10) +UBER_CHANGES=$(count_mmp_writes $TESTPOOL $DURATION) log_note "Uberblock changed $UBER_CHANGES times" @@ -76,7 +78,7 @@ fi log_must set_tunable64 MULTIHOST_INTERVAL $MMP_INTERVAL_MIN SEQ_BEFORE=$(zdb -luuuu ${DISK[0]} | awk '/mmp_seq/ {if ($NF>max) max=$NF}; END {print max}') -sleep 1 +sleep 5 SEQ_AFTER=$(zdb -luuuu ${DISK[0]} | awk '/mmp_seq/ {if ($NF>max) max=$NF}; END {print max}') if [ $((SEQ_AFTER - SEQ_BEFORE)) -lt $MIN_SEQ_VALUES ]; then zdb -luuuu ${DISK[0]} diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_on_zdb.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_on_zdb.ksh index 10e65c79f..ac058d342 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_on_zdb.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_on_zdb.ksh @@ -51,7 +51,8 @@ log_onexit cleanup verify_runnable "global" verify_disk_count "$DISKS" 2 -default_mirror_setup_noexit $DISKS +log_must zpool create -f $TESTPOOL mirror $DISKS +log_must zfs create $TESTPOOL/$TESTFS log_must mmp_set_hostid $HOSTID1 log_must zpool set multihost=on $TESTPOOL log_must zfs snap $TESTPOOL/$TESTFS@snap diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh index e5273aa91..6347cf6c5 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh @@ -43,7 +43,7 @@ verify_runnable "both" function cleanup { - default_cleanup_noexit + datasetexists $TESTPOOL && destroy_pool $TESTPOOL log_must set_tunable64 MULTIHOST_INTERVAL $MMP_INTERVAL_DEFAULT log_must set_tunable64 MULTIHOST_FAIL_INTERVALS \ $MMP_FAIL_INTERVALS_DEFAULT @@ -56,7 +56,7 @@ log_onexit cleanup log_must set_tunable64 MULTIHOST_INTERVAL $MMP_INTERVAL_HOUR log_must mmp_set_hostid $HOSTID1 -default_setup_noexit $DISK +log_must zpool create -f $TESTPOOL $DISK log_must zpool set multihost=on $TESTPOOL clear_mmp_history @@ -68,6 +68,7 @@ if [ $uber_count -eq 0 ]; then fi # 7. Verify mmp_write and mmp_fail are written +log_note "Verify mmp_write and mmp_fail are written" for fails in $(seq $MMP_FAIL_INTERVALS_MIN $((MMP_FAIL_INTERVALS_MIN*2))); do for interval in $(seq $MMP_INTERVAL_MIN 200 $MMP_INTERVAL_DEFAULT); do log_must set_tunable64 MULTIHOST_FAIL_INTERVALS $fails @@ -88,7 +89,8 @@ done # 8. Repeatedly change MULTIHOST_INTERVAL and fail_intervals -for x in $(seq 10); do +log_note "Repeatedly change MULTIHOST_INTERVAL and fail_intervals" +for x in $(seq 3); do typeset new_interval=$(( (RANDOM % 20 + 1) * $MMP_INTERVAL_MIN )) log_must set_tunable64 MULTIHOST_INTERVAL $new_interval typeset action=$((RANDOM %10)) diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_write_distribution.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_write_distribution.ksh index f7e5ceb92..6482434f9 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_write_distribution.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_write_distribution.ksh @@ -37,7 +37,7 @@ verify_runnable "both" function cleanup { - log_must zpool destroy $MMP_POOL + datasetexists $MMP_POOL && destroy_pool $MMP_POOL log_must rm $MMP_DIR/file.{0..7} log_must rm $MMP_HISTORY_TMP log_must rmdir $MMP_DIR diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_write_uberblocks.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_write_uberblocks.ksh index 51839b423..a13198fc1 100755 --- a/tests/zfs-tests/tests/functional/mmp/mmp_write_uberblocks.ksh +++ b/tests/zfs-tests/tests/functional/mmp/mmp_write_uberblocks.ksh @@ -38,7 +38,7 @@ verify_runnable "both" function cleanup { zinject -c all - default_cleanup_noexit + datasetexists $TESTPOOL && destroy_pool $TESTPOOL log_must mmp_clear_hostid } @@ -46,7 +46,7 @@ log_assert "mmp behaves correctly when failing to write uberblocks." log_onexit cleanup log_must mmp_set_hostid $HOSTID1 -default_mirror_setup_noexit $DISKS +log_must zpool create -f $TESTPOOL mirror $DISKS log_must zpool set multihost=on $TESTPOOL log_must zinject -d ${DISK[0]} -e io -T write -f 50 $TESTPOOL -L uber clear_mmp_history diff --git a/tests/zfs-tests/tests/functional/mmp/multihost_history.ksh b/tests/zfs-tests/tests/functional/mmp/multihost_history.ksh index cbffd28f6..689502a7f 100755 --- a/tests/zfs-tests/tests/functional/mmp/multihost_history.ksh +++ b/tests/zfs-tests/tests/functional/mmp/multihost_history.ksh @@ -50,6 +50,8 @@ function cleanup log_assert "zfs_multihost_history records writes and skipped writes" log_onexit cleanup +MMP_INTERVAL_DEFAULT=1000 + mmp_pool_create_simple $MMP_POOL $MMP_DIR log_must zinject -d $MMP_DIR/vdev1 -D$((2*MMP_INTERVAL_DEFAULT)):10 $MMP_POOL log_must zinject -d $MMP_DIR/vdev2 -D$((2*MMP_INTERVAL_DEFAULT)):10 $MMP_POOL