zinject: add "probe" device injection type

Injecting a device probe failure is not possible by matching IO types,
because probe IO goes to the label regions, which is explicitly excluded
from injection. Even if it were possible, it would be awkward to do,
because a probe is sequence of reads and writes.

This commit adds a new IO "type" to match for injection, which looks for
the ZIO_FLAG_PROBE flag instead. Any probe IO will be match the
injection record and recieve the wanted error.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>
Closes #16947

tests/runfiles/common.run

@@ -159,7 +159,7 @@ tests = ['json_sanity']
 tags = ['functional', 'cli_root', 'json']
 
 [tests/functional/cli_root/zinject]
-tests = ['zinject_args', 'zinject_counts']
+tests = ['zinject_args', 'zinject_counts', 'zinject_probe']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zinject']

tests/zfs-tests/tests/Makefile.am

@@ -616,6 +616,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/json/json_sanity.ksh \
 	functional/cli_root/zinject/zinject_args.ksh \
 	functional/cli_root/zinject/zinject_counts.ksh \
+	functional/cli_root/zinject/zinject_probe.ksh \
 	functional/cli_root/zdb/zdb_002_pos.ksh \
 	functional/cli_root/zdb/zdb_003_pos.ksh \
 	functional/cli_root/zdb/zdb_004_pos.ksh \

tests/zfs-tests/tests/functional/cli_root/zinject/zinject_probe.ksh

@@ -0,0 +1,75 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025, Klara, Inc.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+verify_runnable "global"
+
+log_assert "Check zinject can correctly inject a probe failure."
+
+DISK1=${DISKS%% *}
+
+function cleanup
+{
+	log_pos zinject -c all
+	log_pos zpool clear $TESTPOOL
+	log_pos zpool destroy -f $TESTPOOL
+	log_pos restore_tunable TXG_TIMEOUT
+}
+
+log_onexit cleanup
+
+log_must zpool create $TESTPOOL $DISK1
+
+# set the txg timeout a long way out, to try and avoid the pool syncing
+# between error injection and writing
+save_tunable TXG_TIMEOUT
+log_must set_tunable32 TXG_TIMEOUT 600
+
+# force a sync now
+log_must zpool sync -f
+
+# write stuff. this should go into memory, not written yet
+log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=1
+
+# inject faults
+log_must zinject -d $DISK1 -e io -T probe $TESTPOOL
+log_must zinject -d $DISK1 -e io -T write $TESTPOOL
+
+# force the sync now. backgrounded, because the pool will suspend and we don't
+# want to block.
+log_pos zpool sync &
+
+log_note "waiting for pool to suspend"
+typeset -i tries=30
+until [[ $(kstat_pool $TESTPOOL state) == "SUSPENDED" ]] ; do
+	if ((tries-- == 0)); then
+		log_fail "pool didn't suspend"
+	fi
+	sleep 1
+done
+
+log_pass "zinject can correctly inject a probe failure."