From 05a7a9594e075c3393db270903b038fe1e7f0ddd Mon Sep 17 00:00:00 2001 From: Shengqi Chen Date: Wed, 25 Sep 2024 22:46:52 +0800 Subject: [PATCH] CI: run only sanity check on limited OSes for nonbehavioral changes The commit uses heuristics to determine whether a PR is behavioral: It runs "quick" CI (i.e., only use sanity.run on fewer OSes) if (explicitly requested by user): - the *last* commit message contains a line 'ZFS-CI-Type: quick', or if (by heuristics): - the files changed are not in the list of specified directory, and - all commit messages does not contain 'ZFS-CI-Type: full'. It runs "full" CI otherwise. Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Signed-off-by: Shengqi Chen Closes #16564 --- .github/CONTRIBUTING.md | 3 + .github/workflows/scripts/generate-ci-type.py | 107 ++++++++++++++++++ .github/workflows/scripts/qemu-6-tests.sh | 5 +- .github/workflows/zfs-qemu.yml | 37 +++++- 4 files changed, 150 insertions(+), 2 deletions(-) create mode 100755 .github/workflows/scripts/generate-ci-type.py diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 217c9b22c..7e8d1bafb 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -147,6 +147,9 @@ Any required reviews can then be finalized and the pull request merged. #### Tests and Benchmarks * Every pull request is tested using a GitHub Actions workflow on multiple platforms by running the [zfs-tests.sh and zloop.sh]( https://openzfs.github.io/openzfs-docs/Developer%20Resources/Building%20ZFS.html#running-zloop-sh-and-zfs-tests-sh) test suites. +`.github/workflows/scripts/generate-ci-type.py` is used to determine whether the pull request is nonbehavior, i.e., not introducing behavior changes of any code, configuration or tests. If so, the CI will run on fewer platforms and only essential sanity tests will run. You can always override this by adding `ZFS-CI-Type` line to your commit message: + * If your last commit (or `HEAD` in git terms) contains a line `ZFS-CI-Type: quick`, quick mode is forced regardless of what files are changed. + * Otherwise, if any commit in a PR contains a line `ZFS-CI-Type: full`, full mode is forced. * To verify your changes conform to the [style guidelines]( https://github.com/openzfs/zfs/blob/master/.github/CONTRIBUTING.md#style-guides ), please run `make checkstyle` and resolve any warnings. diff --git a/.github/workflows/scripts/generate-ci-type.py b/.github/workflows/scripts/generate-ci-type.py new file mode 100755 index 000000000..943aae254 --- /dev/null +++ b/.github/workflows/scripts/generate-ci-type.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +""" +Determine the CI type based on the change list and commit message. + +Prints "quick" if (explicity required by user): +- the *last* commit message contains 'ZFS-CI-Type: quick' +or if (heuristics): +- the files changed are not in the list of specified directories, and +- all commit messages do not contain 'ZFS-CI-Type: full' + +Otherwise prints "full". +""" + +import sys +import subprocess +import re + +""" +Patterns of files that are not considered to trigger full CI. +Note: not using pathlib.Path.match() because it does not support '**' +""" +FULL_RUN_IGNORE_REGEX = list(map(re.compile, [ + r'.*\.md', + r'.*\.gitignore' +])) + +""" +Patterns of files that are considered to trigger full CI. +""" +FULL_RUN_REGEX = list(map(re.compile, [ + r'cmd.*', + r'configs/.*', + r'META', + r'.*\.am', + r'.*\.m4', + r'autogen\.sh', + r'configure\.ac', + r'copy-builtin', + r'contrib', + r'etc', + r'include', + r'lib/.*', + r'module/.*', + r'scripts/.*', + r'tests/.*', + r'udev/.*' +])) + +if __name__ == '__main__': + + prog = sys.argv[0] + + if len(sys.argv) != 3: + print(f'Usage: {prog} ') + sys.exit(1) + + head, base = sys.argv[1:3] + + def output_type(type, reason): + print(f'{prog}: will run {type} CI: {reason}', file=sys.stderr) + print(type) + sys.exit(0) + + # check last (HEAD) commit message + last_commit_message_raw = subprocess.run([ + 'git', 'show', '-s', '--format=%B', 'HEAD' + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + for line in last_commit_message_raw.stdout.decode().splitlines(): + if line.strip().lower() == 'zfs-ci-type: quick': + output_type('quick', f'explicitly requested by HEAD commit {head}') + + # check all commit messages + all_commit_message_raw = subprocess.run([ + 'git', 'show', '-s', + '--format=ZFS-CI-Commit: %H%n%B', f'{head}...{base}' + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + all_commit_message = all_commit_message_raw.stdout.decode().splitlines() + + commit_ref = head + for line in all_commit_message: + if line.startswith('ZFS-CI-Commit:'): + commit_ref = line.lstrip('ZFS-CI-Commit:').rstrip() + if line.strip().lower() == 'zfs-ci-type: full': + output_type('full', f'explicitly requested by commit {commit_ref}') + + # check changed files + changed_files_raw = subprocess.run([ + 'git', 'diff', '--name-only', head, base + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + changed_files = changed_files_raw.stdout.decode().splitlines() + + for f in changed_files: + for r in FULL_RUN_IGNORE_REGEX: + if r.match(f): + break + else: + for r in FULL_RUN_REGEX: + if r.match(f): + output_type( + 'full', + f'changed file "{f}" matches pattern "{r.pattern}"' + ) + + # catch-all + output_type('quick', 'no changed file matches full CI patterns') diff --git a/.github/workflows/scripts/qemu-6-tests.sh b/.github/workflows/scripts/qemu-6-tests.sh index fac953c8f..2f023198b 100755 --- a/.github/workflows/scripts/qemu-6-tests.sh +++ b/.github/workflows/scripts/qemu-6-tests.sh @@ -48,7 +48,7 @@ if [ -z ${1:-} ]; then for i in $(seq 1 $VMs); do IP="192.168.122.1$i" daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \ - $SSH zfs@$IP $TESTS $OS $i $VMs + $SSH zfs@$IP $TESTS $OS $i $VMs $CI_TYPE # handly line by line and add info prefix stdbuf -oL tail -fq vm${i}log.txt \ | while read -r line; do prefix "$i" "$line"; done & @@ -91,6 +91,9 @@ esac # run functional testings and save exitcode cd /var/tmp TAGS=$2/$3 +if [ "$4" == "quick" ]; then + export RUNFILES="sanity.run" +fi sudo dmesg -c > dmesg-prerun.txt mount > mount.txt df -h > df-prerun.txt diff --git a/.github/workflows/zfs-qemu.yml b/.github/workflows/zfs-qemu.yml index 67d915adf..8922701f9 100644 --- a/.github/workflows/zfs-qemu.yml +++ b/.github/workflows/zfs-qemu.yml @@ -9,15 +9,48 @@ concurrency: cancel-in-progress: true jobs: + test-config: + name: Setup + runs-on: ubuntu-24.04 + outputs: + test_os: ${{ steps.os.outputs.os }} + ci_type: ${{ steps.os.outputs.ci_type }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Generate OS config and CI type + id: os + run: | + FULL_OS='["almalinux8", "almalinux9", "centos-stream9", "debian11", "debian12", "fedora39", "fedora40", "freebsd13", "freebsd13r", "freebsd14", "freebsd14r", "ubuntu20", "ubuntu22", "ubuntu24"]' + QUICK_OS='["almalinux8", "almalinux9", "debian12", "fedora40", "freebsd13", "freebsd14", "ubuntu24"]' + # determine CI type when running on PR + ci_type="full" + if ${{ github.event_name == 'pull_request' }}; then + head=${{ github.event.pull_request.head.sha }} + base=${{ github.event.pull_request.base.sha }} + ci_type=$(python3 .github/workflows/scripts/generate-ci-type.py $head $base) + fi + if [ "$ci_type" == "quick" ]; then + os_selection="$QUICK_OS" + else + os_selection="$FULL_OS" + fi + os_json=$(echo ${os_selection} | jq -c) + echo "os=$os_json" >> $GITHUB_OUTPUT + echo "ci_type=$ci_type" >> $GITHUB_OUTPUT + qemu-vm: name: qemu-x86 + needs: [ test-config ] strategy: fail-fast: false matrix: # all: # os: [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd13r, freebsd14, freebsd14r, freebsd15, ubuntu20, ubuntu22, ubuntu24] # openzfs: - os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, ubuntu20, ubuntu22, ubuntu24] + # os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, ubuntu20, ubuntu22, ubuntu24] + os: ${{ fromJson(needs.test-config.outputs.test_os) }} runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -67,6 +100,8 @@ jobs: - name: Run tests timeout-minutes: 270 run: .github/workflows/scripts/qemu-6-tests.sh + env: + CI_TYPE: ${{ needs.test-config.outputs.ci_type }} - name: Prepare artifacts if: always()