CI: run only sanity check on limited OSes for nonbehavioral changes

The commit uses heuristics to determine whether a PR is behavioral:

It runs "quick" CI (i.e., only use sanity.run on fewer OSes)
if (explicitly requested by user):
- the *last* commit message contains a line 'ZFS-CI-Type: quick',
or if (by heuristics):
- the files changed are not in the list of specified directory, and
- all commit messages does not contain 'ZFS-CI-Type: full'.

It runs "full" CI otherwise.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Tino Reichardt <milky-zfs@mcmilk.de>
Signed-off-by: Shengqi Chen <harry-chen@outlook.com>
Closes #16564

.github/CONTRIBUTING.md

@@ -147,6 +147,9 @@ Any required reviews can then be finalized and the pull request merged.
 #### Tests and Benchmarks
 * Every pull request is tested using a GitHub Actions workflow on multiple platforms by running the [zfs-tests.sh and zloop.sh](
 https://openzfs.github.io/openzfs-docs/Developer%20Resources/Building%20ZFS.html#running-zloop-sh-and-zfs-tests-sh) test suites.
+`.github/workflows/scripts/generate-ci-type.py` is used to determine whether the pull request is nonbehavior, i.e., not introducing behavior changes of any code, configuration or tests. If so, the CI will run on fewer platforms and only essential sanity tests will run. You can always override this by adding `ZFS-CI-Type` line to your commit message:
+  * If your last commit (or `HEAD` in git terms) contains a line `ZFS-CI-Type: quick`, quick mode is forced regardless of what files are changed.
+  * Otherwise, if any commit in a PR contains a line `ZFS-CI-Type: full`, full mode is forced.
 * To verify your changes conform to the [style guidelines](
 https://github.com/openzfs/zfs/blob/master/.github/CONTRIBUTING.md#style-guides
 ), please run `make checkstyle` and resolve any warnings.

.github/workflows/scripts/generate-ci-type.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+"""
+Determine the CI type based on the change list and commit message.
+
+Prints "quick" if (explicity required by user):
+- the *last* commit message contains 'ZFS-CI-Type: quick'
+or if (heuristics):
+- the files changed are not in the list of specified directories, and
+- all commit messages do not contain 'ZFS-CI-Type: full'
+
+Otherwise prints "full".
+"""
+
+import sys
+import subprocess
+import re
+
+"""
+Patterns of files that are not considered to trigger full CI.
+Note: not using pathlib.Path.match() because it does not support '**'
+"""
+FULL_RUN_IGNORE_REGEX = list(map(re.compile, [
+    r'.*\.md',
+    r'.*\.gitignore'
+]))
+
+"""
+Patterns of files that are considered to trigger full CI.
+"""
+FULL_RUN_REGEX = list(map(re.compile, [
+    r'cmd.*',
+    r'configs/.*',
+    r'META',
+    r'.*\.am',
+    r'.*\.m4',
+    r'autogen\.sh',
+    r'configure\.ac',
+    r'copy-builtin',
+    r'contrib',
+    r'etc',
+    r'include',
+    r'lib/.*',
+    r'module/.*',
+    r'scripts/.*',
+    r'tests/.*',
+    r'udev/.*'
+]))
+
+if __name__ == '__main__':
+
+    prog = sys.argv[0]
+
+    if len(sys.argv) != 3:
+        print(f'Usage: {prog} <head_ref> <base_ref>')
+        sys.exit(1)
+
+    head, base = sys.argv[1:3]
+
+    def output_type(type, reason):
+        print(f'{prog}: will run {type} CI: {reason}', file=sys.stderr)
+        print(type)
+        sys.exit(0)
+
+    # check last (HEAD) commit message
+    last_commit_message_raw = subprocess.run([
+        'git', 'show', '-s', '--format=%B', 'HEAD'
+    ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    for line in last_commit_message_raw.stdout.decode().splitlines():
+        if line.strip().lower() == 'zfs-ci-type: quick':
+            output_type('quick', f'explicitly requested by HEAD commit {head}')
+
+    # check all commit messages
+    all_commit_message_raw = subprocess.run([
+        'git', 'show', '-s',
+        '--format=ZFS-CI-Commit: %H%n%B', f'{head}...{base}'
+    ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    all_commit_message = all_commit_message_raw.stdout.decode().splitlines()
+
+    commit_ref = head
+    for line in all_commit_message:
+        if line.startswith('ZFS-CI-Commit:'):
+            commit_ref = line.lstrip('ZFS-CI-Commit:').rstrip()
+        if line.strip().lower() == 'zfs-ci-type: full':
+            output_type('full', f'explicitly requested by commit {commit_ref}')
+
+    # check changed files
+    changed_files_raw = subprocess.run([
+        'git', 'diff', '--name-only', head, base
+    ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    changed_files = changed_files_raw.stdout.decode().splitlines()
+
+    for f in changed_files:
+        for r in FULL_RUN_IGNORE_REGEX:
+            if r.match(f):
+                break
+        else:
+            for r in FULL_RUN_REGEX:
+                if r.match(f):
+                    output_type(
+                        'full',
+                        f'changed file "{f}" matches pattern "{r.pattern}"'
+                        )
+
+    # catch-all
+    output_type('quick', 'no changed file matches full CI patterns')

.github/workflows/scripts/qemu-6-tests.sh

@@ -48,7 +48,7 @@ if [ -z ${1:-} ]; then
   for i in $(seq 1 $VMs); do
     IP="192.168.122.1$i"
     daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \
-      $SSH zfs@$IP $TESTS $OS $i $VMs
+      $SSH zfs@$IP $TESTS $OS $i $VMs $CI_TYPE
     # handly line by line and add info prefix
     stdbuf -oL tail -fq vm${i}log.txt \
       | while read -r line; do prefix "$i" "$line"; done &
@@ -91,6 +91,9 @@ esac
 # run functional testings and save exitcode
 cd /var/tmp
 TAGS=$2/$3
+if [ "$4" == "quick" ]; then
+  export RUNFILES="sanity.run"
+fi
 sudo dmesg -c > dmesg-prerun.txt
 mount > mount.txt
 df -h > df-prerun.txt

.github/workflows/zfs-qemu.yml

@@ -9,15 +9,48 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  test-config:
+    name: Setup
+    runs-on: ubuntu-24.04
+    outputs:
+      test_os: ${{ steps.os.outputs.os }}
+      ci_type: ${{ steps.os.outputs.ci_type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generate OS config and CI type
+        id: os
+        run: |
+          FULL_OS='["almalinux8", "almalinux9", "centos-stream9", "debian11", "debian12", "fedora39", "fedora40", "freebsd13", "freebsd13r", "freebsd14", "freebsd14r", "ubuntu20", "ubuntu22", "ubuntu24"]'
+          QUICK_OS='["almalinux8", "almalinux9", "debian12", "fedora40", "freebsd13", "freebsd14", "ubuntu24"]'
+          # determine CI type when running on PR
+          ci_type="full"
+          if ${{ github.event_name == 'pull_request' }}; then
+            head=${{ github.event.pull_request.head.sha }}
+            base=${{ github.event.pull_request.base.sha }}
+            ci_type=$(python3 .github/workflows/scripts/generate-ci-type.py $head $base)
+          fi
+          if [ "$ci_type" == "quick" ]; then
+            os_selection="$QUICK_OS"
+          else
+            os_selection="$FULL_OS"
+          fi
+          os_json=$(echo ${os_selection} | jq -c)
+          echo "os=$os_json" >> $GITHUB_OUTPUT
+          echo "ci_type=$ci_type" >> $GITHUB_OUTPUT
+
   qemu-vm:
     name: qemu-x86
+    needs: [ test-config ]
     strategy:
       fail-fast: false
       matrix:
         # all:
         # os:  [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd13r, freebsd14, freebsd14r, freebsd15, ubuntu20, ubuntu22, ubuntu24]
         # openzfs:
-        os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, ubuntu20, ubuntu22, ubuntu24]
+        # os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, ubuntu20, ubuntu22, ubuntu24]
+        os: ${{ fromJson(needs.test-config.outputs.test_os) }}
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
@@ -67,6 +100,8 @@ jobs:
     - name: Run tests
       timeout-minutes: 270
       run: .github/workflows/scripts/qemu-6-tests.sh
+      env:
+        CI_TYPE: ${{ needs.test-config.outputs.ci_type }}
 
     - name: Prepare artifacts
       if: always()