CI: run only sanity check on limited OSes for nonbehavioral changes

The commit uses heuristics to determine whether a PR is behavioral:

It runs "quick" CI (i.e., only use sanity.run on fewer OSes)
if (explicitly requested by user):
- the *last* commit message contains a line 'ZFS-CI-Type: quick',
or if (by heuristics):
- the files changed are not in the list of specified directory, and
- all commit messages does not contain 'ZFS-CI-Type: full'.

It runs "full" CI otherwise.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Tino Reichardt <milky-zfs@mcmilk.de>
Signed-off-by: Shengqi Chen <harry-chen@outlook.com>
Closes #16564

.github/workflows/scripts/generate-ci-type.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+"""
+Determine the CI type based on the change list and commit message.
+
+Prints "quick" if (explicity required by user):
+- the *last* commit message contains 'ZFS-CI-Type: quick'
+or if (heuristics):
+- the files changed are not in the list of specified directories, and
+- all commit messages do not contain 'ZFS-CI-Type: full'
+
+Otherwise prints "full".
+"""
+
+import sys
+import subprocess
+import re
+
+"""
+Patterns of files that are not considered to trigger full CI.
+Note: not using pathlib.Path.match() because it does not support '**'
+"""
+FULL_RUN_IGNORE_REGEX = list(map(re.compile, [
+    r'.*\.md',
+    r'.*\.gitignore'
+]))
+
+"""
+Patterns of files that are considered to trigger full CI.
+"""
+FULL_RUN_REGEX = list(map(re.compile, [
+    r'cmd.*',
+    r'configs/.*',
+    r'META',
+    r'.*\.am',
+    r'.*\.m4',
+    r'autogen\.sh',
+    r'configure\.ac',
+    r'copy-builtin',
+    r'contrib',
+    r'etc',
+    r'include',
+    r'lib/.*',
+    r'module/.*',
+    r'scripts/.*',
+    r'tests/.*',
+    r'udev/.*'
+]))
+
+if __name__ == '__main__':
+
+    prog = sys.argv[0]
+
+    if len(sys.argv) != 3:
+        print(f'Usage: {prog} <head_ref> <base_ref>')
+        sys.exit(1)
+
+    head, base = sys.argv[1:3]
+
+    def output_type(type, reason):
+        print(f'{prog}: will run {type} CI: {reason}', file=sys.stderr)
+        print(type)
+        sys.exit(0)
+
+    # check last (HEAD) commit message
+    last_commit_message_raw = subprocess.run([
+        'git', 'show', '-s', '--format=%B', 'HEAD'
+    ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    for line in last_commit_message_raw.stdout.decode().splitlines():
+        if line.strip().lower() == 'zfs-ci-type: quick':
+            output_type('quick', f'explicitly requested by HEAD commit {head}')
+
+    # check all commit messages
+    all_commit_message_raw = subprocess.run([
+        'git', 'show', '-s',
+        '--format=ZFS-CI-Commit: %H%n%B', f'{head}...{base}'
+    ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    all_commit_message = all_commit_message_raw.stdout.decode().splitlines()
+
+    commit_ref = head
+    for line in all_commit_message:
+        if line.startswith('ZFS-CI-Commit:'):
+            commit_ref = line.lstrip('ZFS-CI-Commit:').rstrip()
+        if line.strip().lower() == 'zfs-ci-type: full':
+            output_type('full', f'explicitly requested by commit {commit_ref}')
+
+    # check changed files
+    changed_files_raw = subprocess.run([
+        'git', 'diff', '--name-only', head, base
+    ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    changed_files = changed_files_raw.stdout.decode().splitlines()
+
+    for f in changed_files:
+        for r in FULL_RUN_IGNORE_REGEX:
+            if r.match(f):
+                break
+        else:
+            for r in FULL_RUN_REGEX:
+                if r.match(f):
+                    output_type(
+                        'full',
+                        f'changed file "{f}" matches pattern "{r.pattern}"'
+                        )
+
+    # catch-all
+    output_type('quick', 'no changed file matches full CI patterns')

.github/workflows/scripts/qemu-6-tests.sh

@@ -48,7 +48,7 @@ if [ -z ${1:-} ]; then
   for i in $(seq 1 $VMs); do
     IP="192.168.122.1$i"
     daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \
-      $SSH zfs@$IP $TESTS $OS $i $VMs
+      $SSH zfs@$IP $TESTS $OS $i $VMs $CI_TYPE
     # handly line by line and add info prefix
     stdbuf -oL tail -fq vm${i}log.txt \
       | while read -r line; do prefix "$i" "$line"; done &
@@ -91,6 +91,9 @@ esac
 # run functional testings and save exitcode
 cd /var/tmp
 TAGS=$2/$3
+if [ "$4" == "quick" ]; then
+  export RUNFILES="sanity.run"
+fi
 sudo dmesg -c > dmesg-prerun.txt
 mount > mount.txt
 df -h > df-prerun.txt

.github/workflows/zfs-qemu.yml

@@ -9,15 +9,48 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  test-config:
+    name: Setup
+    runs-on: ubuntu-24.04
+    outputs:
+      test_os: ${{ steps.os.outputs.os }}
+      ci_type: ${{ steps.os.outputs.ci_type }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Generate OS config and CI type
+        id: os
+        run: |
+          FULL_OS='["almalinux8", "almalinux9", "centos-stream9", "debian11", "debian12", "fedora39", "fedora40", "freebsd13", "freebsd13r", "freebsd14", "freebsd14r", "ubuntu20", "ubuntu22", "ubuntu24"]'
+          QUICK_OS='["almalinux8", "almalinux9", "debian12", "fedora40", "freebsd13", "freebsd14", "ubuntu24"]'
+          # determine CI type when running on PR
+          ci_type="full"
+          if ${{ github.event_name == 'pull_request' }}; then
+            head=${{ github.event.pull_request.head.sha }}
+            base=${{ github.event.pull_request.base.sha }}
+            ci_type=$(python3 .github/workflows/scripts/generate-ci-type.py $head $base)
+          fi
+          if [ "$ci_type" == "quick" ]; then
+            os_selection="$QUICK_OS"
+          else
+            os_selection="$FULL_OS"
+          fi
+          os_json=$(echo ${os_selection} | jq -c)
+          echo "os=$os_json" >> $GITHUB_OUTPUT
+          echo "ci_type=$ci_type" >> $GITHUB_OUTPUT
+
   qemu-vm:
     name: qemu-x86
+    needs: [ test-config ]
     strategy:
       fail-fast: false
       matrix:
         # all:
         # os:  [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd13r, freebsd14, freebsd14r, freebsd15, ubuntu20, ubuntu22, ubuntu24]
         # openzfs:
-        os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, ubuntu20, ubuntu22, ubuntu24]
+        # os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, ubuntu20, ubuntu22, ubuntu24]
+        os: ${{ fromJson(needs.test-config.outputs.test_os) }}
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
@@ -67,6 +100,8 @@ jobs:
     - name: Run tests
       timeout-minutes: 270
       run: .github/workflows/scripts/qemu-6-tests.sh
+      env:
+        CI_TYPE: ${{ needs.test-config.outputs.ci_type }}
 
     - name: Prepare artifacts
       if: always()