From f43839e7fdeb2911e031179cca18a9444bdd59df Mon Sep 17 00:00:00 2001
From: Rob Norris <rob.norris@klarasystems.com>
Date: Wed, 22 Oct 2025 08:34:39 +1100
Subject: [PATCH] ZTS: fail test run if test runner crashes unexpectedly

zfs-tests.sh executes test-runner.py to do the actual test work. Any
exit code < 4 is interpreted as success, with the actual value
describing the outcome of the tests inside.

If a Python program crashes in some way (eg an uncaught exception), the
process exit code is 1.

Taken together, this means that test-runner.py can crash during setup,
but return a "success" error code to zfs-tests.sh, which will report and
exit 0. This in turn causes the CI runner to believe the test run
completed successfully.

This commit addresses this by making zfs-tests.sh interpret an exit code
of 255 as a failure in the runner itself. Then, in test-runner.py, the
"fail()" function defaults to a 255 return, and the main function gets
wrapped in a generic exception handler, which prints it and calls
fail().

All together, this should mean that any unexpected failure in the test
runner itself will be propagated out of zfs-tests.sh for CI or any other
calling program to deal with.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Signed-off-by: Rob Norris <rob.norris@klarasystems.com>
Closes #17858
---
 scripts/zfs-tests.sh                    |  4 +++
 tests/test-runner/bin/test-runner.py.in | 33 ++++++++++++++-----------
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh
index 5a0a1a609..09a15bafc 100755
--- a/scripts/zfs-tests.sh
+++ b/scripts/zfs-tests.sh
@@ -797,6 +797,10 @@ msg "${TEST_RUNNER}" \
     2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 read -r RUNRESULT <"$REPORT_FILE"
 
+if [[ "$RUNRESULT" -eq "255" ]] ; then
+    fail "$TEST_RUNNER failed, test aborted."
+fi 
+
 #
 # Analyze the results.
 #
diff --git a/tests/test-runner/bin/test-runner.py.in b/tests/test-runner/bin/test-runner.py.in
index d2c1185e4..6688b6c4b 100755
--- a/tests/test-runner/bin/test-runner.py.in
+++ b/tests/test-runner/bin/test-runner.py.in
@@ -25,6 +25,7 @@ import sys
 import ctypes
 import re
 import configparser
+import traceback
 
 from datetime import datetime
 from optparse import OptionParser
@@ -1138,7 +1139,7 @@ def filter_tests(testrun, options):
     testrun.filter(failed)
 
 
-def fail(retstr, ret=1):
+def fail(retstr, ret=255):
     print('%s: %s' % (sys.argv[0], retstr))
     exit(ret)
 
@@ -1247,23 +1248,27 @@ def parse_args():
 def main():
     options = parse_args()
 
-    testrun = TestRun(options)
+    try:
+        testrun = TestRun(options)
 
-    if options.runfiles:
-        testrun.read(options)
-    else:
-        find_tests(testrun, options)
+        if options.runfiles:
+            testrun.read(options)
+        else:
+            find_tests(testrun, options)
 
-    if options.logfile:
-        filter_tests(testrun, options)
+        if options.logfile:
+            filter_tests(testrun, options)
 
-    if options.template:
-        testrun.write(options)
-        exit(0)
+        if options.template:
+            testrun.write(options)
+            exit(0)
 
-    testrun.complete_outputdirs()
-    testrun.run(options)
-    exit(testrun.summary())
+        testrun.complete_outputdirs()
+        testrun.run(options)
+        exit(testrun.summary())
+
+    except Exception:
+        fail("Uncaught exception in test runner:\n" + traceback.format_exc())
 
 
 if __name__ == '__main__':