ZTS: test response of various sync methods under different failmodes

These are all the same shape: set up the pool to suspend on first write,
then perform some write+sync operation. The pool should suspend, and the
sync operation should respond according to the failmode= property.

We test fsync(), msync() and two forms of write() (open with O_SYNC, and
async with sync=always), which all take slightly different paths to
zil_commit() and back.

A helper function is included to do the write+sync sequence with mmap()
and msync(), since I didn't find a convenient tool to do that.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Alexander Motin <alexander.motin@TrueNAS.com>
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>
Closes #17398

tests/runfiles/common.run

@@ -725,7 +725,11 @@ tests = ['fadvise_willneed']
 tags = ['functional', 'fadvise']
 
 [tests/functional/failmode]
-tests = ['failmode_dmu_tx_wait', 'failmode_dmu_tx_continue']
+tests = ['failmode_dmu_tx_wait', 'failmode_dmu_tx_continue',
+    'failmode_fsync_wait', 'failmode_fsync_continue',
+    'failmode_msync_wait', 'failmode_msync_continue',
+    'failmode_osync_wait', 'failmode_osync_continue',
+    'failmode_syncalways_wait', 'failmode_syncalways_continue']
 tags = ['functional', 'failmode']
 
 [tests/functional/fallocate]

tests/zfs-tests/cmd/.gitignore

@@ -28,6 +28,7 @@
 /mmap_seek
 /mmap_sync
 /mmapwrite
+/mmap_write_sync
 /nvlist_to_lua
 /randfree_file
 /randwritecomp

tests/zfs-tests/cmd/Makefile.am

@@ -74,7 +74,7 @@ scripts_zfs_tests_bin_PROGRAMS += %D%/mkbusy %D%/mkfile %D%/mkfiles %D%/mktree
 
 scripts_zfs_tests_bin_PROGRAMS += \
 	%D%/mmap_exec %D%/mmap_ftruncate %D%/mmap_seek \
-	%D%/mmap_sync %D%/mmapwrite %D%/readmmap
+	%D%/mmap_sync %D%/mmapwrite %D%/readmmap %D%/mmap_write_sync
 %C%_mmapwrite_LDADD = -lpthread
 
 if WANT_MMAP_LIBAIO

tests/zfs-tests/cmd/mmap_write_sync.c

@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: CDDL-1.0
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2025, Klara, Inc.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+#define	PAGES	(8)
+
+int
+main(int argc, char **argv)
+{
+	if (argc != 2) {
+		fprintf(stderr, "usage: %s <filename>\n", argv[0]);
+		exit(1);
+	}
+
+	long page_size = sysconf(_SC_PAGESIZE);
+	if (page_size < 0) {
+		perror("sysconf");
+		exit(2);
+	}
+	size_t map_size = page_size * PAGES;
+
+	int fd = open(argv[1], O_CREAT|O_RDWR, S_IRWXU|S_IRWXG|S_IRWXO);
+	if (fd < 0) {
+		perror("open");
+		exit(2);
+	}
+
+	if (ftruncate(fd, map_size) < 0) {
+		perror("ftruncate");
+		close(fd);
+		exit(2);
+	}
+
+	uint64_t *p =
+	    mmap(NULL, map_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+	if (p == MAP_FAILED) {
+		perror("mmap");
+		close(fd);
+		exit(2);
+	}
+
+	for (int i = 0; i < (map_size / sizeof (uint64_t)); i++)
+		p[i] = 0x0123456789abcdef;
+
+	if (msync(p, map_size, MS_SYNC) < 0) {
+		perror("msync");
+		munmap(p, map_size);
+		close(fd);
+		exit(3);
+	}
+
+	munmap(p, map_size);
+	close(fd);
+	exit(0);
+}

tests/zfs-tests/include/commands.cfg

@@ -210,6 +210,7 @@ export ZFSTEST_FILES='badsend
     mmap_seek
     mmap_sync
     mmapwrite
+    mmap_write_sync
     nvlist_to_lua
     randfree_file
     randwritecomp

tests/zfs-tests/tests/Makefile.am

@@ -276,6 +276,7 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \
 	functional/direct/dio.kshlib \
 	functional/events/events.cfg \
 	functional/events/events_common.kshlib \
+	functional/failmode/failmode.kshlib \
 	functional/fault/fault.cfg \
 	functional/gang_blocks/gang_blocks.kshlib \
 	functional/grow/grow.cfg \
@@ -1541,6 +1542,14 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/failmode/cleanup.ksh \
 	functional/failmode/failmode_dmu_tx_wait.ksh \
 	functional/failmode/failmode_dmu_tx_continue.ksh \
+	functional/failmode/failmode_fsync_wait.ksh \
+	functional/failmode/failmode_fsync_continue.ksh \
+	functional/failmode/failmode_msync_wait.ksh \
+	functional/failmode/failmode_msync_continue.ksh \
+	functional/failmode/failmode_osync_wait.ksh \
+	functional/failmode/failmode_osync_continue.ksh \
+	functional/failmode/failmode_syncalways_wait.ksh \
+	functional/failmode/failmode_syncalways_continue.ksh \
 	functional/failmode/setup.ksh \
 	functional/fallocate/cleanup.ksh \
 	functional/fallocate/fallocate_prealloc.ksh \

tests/zfs-tests/tests/functional/failmode/failmode.kshlib

@@ -0,0 +1,149 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+typeset -A failmode_sync_helper_cmd=(
+    ["fsync"]='dd if=/dev/urandom of=DATAFILE bs=128k count=1 conv=fsync'
+    ["msync"]='mmap_write_sync DATAFILE'
+    ["osync"]='dd if=/dev/urandom of=DATAFILE bs=128k count=1 oflag=sync'
+    ["syncalways"]='dd if=/dev/urandom of=DATAFILE bs=128k count=1'
+)
+
+typeset -A failmode_sync_helper_dsopts=(
+    ["syncalways"]="-o sync=always"
+)
+
+function failmode_sync_cleanup
+{
+	zinject -c all || true
+	zpool clear $TESTPOOL || true
+	destroy_pool $TESTPOOL
+}
+
+#
+# failmode_sync_test <failmode> <helper>
+#
+# run a failmode sync test:
+# - failmode: wait|continue
+# - helper: fsync|msync|osync|syncalways
+#
+function failmode_sync_test
+{
+	typeset failmode=$1
+	typeset helper=$2
+
+	# we'll need two disks, one for the main pool, one for the log
+	read -r DISK1 DISK2 _ <<<"$DISKS"
+
+	# file to write to the pool
+	typeset datafile="/$TESTPOOL/$TESTFS/datafile"
+
+	# create a single-disk pool with a separate log and the wanted failmode
+	log_must zpool create \
+	    -f -o failmode=$failmode $TESTPOOL $DISK1 log $DISK2
+
+	# create the test dataset. we bias the ZIL towards the log device to
+	# try to ensure that the sync write never involves the main device
+	log_must zfs create \
+	    -o recordsize=128k -o logbias=latency \
+	    ${failmode_sync_helper_dsopts[$helper]} \
+	    $TESTPOOL/$TESTFS
+
+	# create the target file. the ZIL head structure is created on first
+	# use, and does a full txg wait to finish, which we want to avoid
+	log_must dd if=/dev/zero of=$datafile bs=128k count=1 conv=fsync
+	log_must zpool sync
+
+	# inject errors. writes will fail, as will the followup probes
+	zinject -d $DISK1 -e io -T write $TESTPOOL
+	zinject -d $DISK1 -e nxio -T probe $TESTPOOL
+	zinject -d $DISK2 -e io -T write $TESTPOOL
+	zinject -d $DISK2 -e nxio -T probe $TESTPOOL
+
+	# run the helper program in the background. the pool should immediately
+	# suspend, and the sync op block or fail based on the failmode
+	typeset helper_cmd=${failmode_sync_helper_cmd[$helper]/DATAFILE/$datafile}
+	log_note "running failmode sync helper: $helper_cmd"
+	$helper_cmd &
+	typeset -i pid=$!
+
+	# should only take a moment, but give it a chance
+	log_note "waiting for pool to suspend"
+	typeset -i tries=10
+	until [[ $(kstat_pool $TESTPOOL state) == "SUSPENDED" ]] ; do
+		if ((tries-- == 0)); then
+			log_fail "pool didn't suspend"
+		fi
+		sleep 1
+	done
+
+	# zil_commit() should have noticed the suspend by now
+	typeset -i zilerr=$(kstat zil.zil_commit_error_count)
+
+	# see if the helper program blocked
+	typeset -i blocked
+	if kill -0 $pid ; then
+		blocked=1
+		log_note "$helper: blocked in the kernel"
+	else
+		blocked=0
+		log_note "$helper: exited while pool suspended"
+	fi
+
+	# bring the pool back online
+	zinject -c all
+	zpool clear $TESTPOOL
+
+	# program definitely exited now, get its return code
+	wait $pid
+	typeset -i rc=$?
+
+	failmode_sync_cleanup
+
+	log_note "$helper: zilerr=$zilerr blocked=$blocked rc=$rc"
+
+	# confirm expected results for the failmode
+	if [[ $failmode = "wait" ]] ; then
+		# - the ZIL saw an error, and fell back to a txg sync
+		# - sync op blocked when the pool suspended
+		# - after resume, sync op succeeded, helper returned success
+		log_must test $zilerr -ne 0
+		log_must test $blocked -eq 1
+		log_must test $rc -eq 0
+	elif [[ $failmode = "continue" ]] ; then
+		# confirm expected results:
+		# - the ZIL saw an error, and fell back to a txg sync
+		# - helper exited when the pool suspended
+		# - sync op returned an error, so helper returned failure
+		log_must test $zilerr -ne 0
+		log_must test $blocked -eq 0
+		log_must test $rc -ne 0
+	else
+		log_fail "impossible failmode: $failmode"
+	fi
+}

tests/zfs-tests/tests/functional/failmode/failmode_fsync_continue.ksh

@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="fsync() returns when pool suspends with failmode=continue"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test continue fsync
+log_pass $desc

tests/zfs-tests/tests/functional/failmode/failmode_fsync_wait.ksh

@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="fsync() blocks when pool suspends with failmode=wait"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test wait fsync
+log_pass $desc

tests/zfs-tests/tests/functional/failmode/failmode_msync_continue.ksh

@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="msync() returns when pool suspends with failmode=continue"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test continue msync
+log_pass $desc

tests/zfs-tests/tests/functional/failmode/failmode_msync_wait.ksh

@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="msync() blocks when pool suspends with failmode=wait"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test wait msync
+log_pass $desc

tests/zfs-tests/tests/functional/failmode/failmode_osync_continue.ksh

@@ -0,0 +1,36 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="O_SYNC returns when pool suspends with failmode=continue"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test continue osync
+log_pass $desc

tests/zfs-tests/tests/functional/failmode/failmode_osync_wait.ksh

@@ -0,0 +1,37 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="O_SYNC blocks when pool suspends with failmode=wait"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test wait osync
+log_pass $desc
+

tests/zfs-tests/tests/functional/failmode/failmode_syncalways_continue.ksh

@@ -0,0 +1,37 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="write()+sync=always returns when pool suspends with failmode=continue"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test continue syncalways
+log_pass $desc
+

tests/zfs-tests/tests/functional/failmode/failmode_syncalways_wait.ksh

@@ -0,0 +1,37 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/failmode/failmode.kshlib
+
+typeset desc="write()+sync=always blocks when pool suspends with failmode=wait"
+
+log_assert $desc
+log_onexit failmode_sync_cleanup
+log_must failmode_sync_test wait syncalways
+log_pass $desc
+