ZTS: Remove functional tests via matrix

This commit changes the workflow of the github actions.

- Ubuntu 20.04, 22.04, 24.04 will be tested via QEMU now
- remove unused scripts of this commit: b7bc334d1
- re-add the zloop standalone testings via zloop.yml

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>
Closes #16549

.github/workflows/build-dependencies.txt

@@ -1,57 +0,0 @@
-acl
-alien
-attr
-autoconf
-bc
-build-essential
-curl
-dbench
-debhelper-compat
-dh-python
-dkms
-fakeroot
-fio
-gdb
-gdebi
-git
-ksh
-lcov
-libacl1-dev
-libaio-dev
-libattr1-dev
-libblkid-dev
-libcurl4-openssl-dev
-libdevmapper-dev
-libelf-dev
-libffi-dev
-libmount-dev
-libpam0g-dev
-libselinux1-dev
-libssl-dev
-libtool
-libudev-dev
-linux-headers-generic
-lsscsi
-mdadm
-nfs-kernel-server
-pamtester
-parted
-po-debconf
-python3
-python3-all-dev
-python3-cffi
-python3-dev
-python3-packaging
-python3-pip
-python3-setuptools
-python3-sphinx
-rng-tools-debian
-rsync
-samba
-sysstat
-uuid-dev
-watchdog
-wget
-xfslibs-dev
-xz-utils
-zlib1g-dev

.github/workflows/checkstyle-dependencies.txt

@@ -1,5 +0,0 @@
-cppcheck
-devscripts
-mandoc
-pax-utils
-shellcheck

.github/workflows/checkstyle.yaml

@@ -13,15 +13,11 @@ jobs:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Install dependencies
       run: |
-        # https://github.com/orgs/community/discussions/47863
-        sudo apt-mark hold grub-efi-amd64-signed
-        sudo apt-get update --fix-missing
-        sudo apt-get upgrade
-        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
-        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/checkstyle-dependencies.txt apt-get install -qq
-        sudo python3 -m pip install --quiet flake8
-        sudo apt-get clean
-
+        # for x in lxd core20 snapd; do sudo snap remove $x; done
+        sudo apt-get purge -y snapd google-chrome-stable firefox
+        ONLY_DEPS=1 .github/workflows/scripts/qemu-3-deps.sh ubuntu22
+        sudo apt-get install -y cppcheck devscripts mandoc pax-utils shellcheck
+        sudo python -m pipx install --quiet flake8
         # confirm that the tools are installed
         # the build system doesn't fail when they are not
         checkbashisms --version
@@ -31,8 +27,13 @@ jobs:
         shellcheck --version
     - name: Prepare
       run: |
+        sed -i '/DEBUG_CFLAGS="-Werror"/s/^/#/' config/zfs-build.m4
         ./autogen.sh
+    - name: Configure
+      run: |
         ./configure
+    - name: Make
+      run: |
         make -j$(nproc) --no-print-directory --silent
     - name: Checkstyle
       run: |

.github/workflows/scripts/generate-summary.sh

@@ -1,119 +0,0 @@
-#!/usr/bin/env bash
-
-# for runtime reasons we split functional testings into N parts
-# - use a define to check for missing tarfiles
-FUNCTIONAL_PARTS="4"
-
-ZTS_REPORT="tests/test-runner/bin/zts-report.py"
-chmod +x $ZTS_REPORT
-
-function output() {
-  echo -e $* >> Summary.md
-}
-
-function error() {
-  output ":bangbang: $* :bangbang:\n"
-}
-
-# this function generates the real summary
-# - expects a logfile "log" in current directory
-function generate() {
-  # we issued some error already
-  test ! -s log && return
-
-  # for overview and zts-report
-  cat log | grep '^Test' > list
-
-  # error details
-  awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }
-    /\[SKIP\]|\[PASS\]/{ show=0; } show' log > err
-
-  # summary of errors
-  if [ -s err ]; then
-    output "<pre>"
-    $ZTS_REPORT --no-maybes ./list >> Summary.md
-    output "</pre>"
-
-    # generate seperate error logfile
-    ERRLOGS=$((ERRLOGS+1))
-    errfile="err-$ERRLOGS.md"
-    echo -e "\n## $headline (debugging)\n" >> $errfile
-    echo "<details><summary>Error Listing - with dmesg and dbgmsg</summary><pre>" >> $errfile
-    dd if=err bs=999k count=1 >> $errfile
-    echo "</pre></details>" >> $errfile
-  else
-    output "All tests passed :thumbsup:"
-  fi
-
-  output "<details><summary>Full Listing</summary><pre>"
-  cat list >> Summary.md
-  output "</pre></details>"
-
-  # remove tmp files
-  rm -f err list log
-}
-
-# check tarfiles and untar
-function check_tarfile() {
-  if [ -f "$1" ]; then
-    tar xf "$1" || error "Tarfile $1 returns some error"
-  else
-    error "Tarfile $1 not found"
-  fi
-}
-
-# check logfile and concatenate test results
-function check_logfile() {
-  if [ -f "$1" ]; then
-    cat "$1" >> log
-  else
-    error "Logfile $1 not found"
-  fi
-}
-
-# sanity
-function summarize_s() {
-  headline="$1"
-  output "\n## $headline\n"
-  rm -rf testfiles
-  check_tarfile "$2/sanity.tar"
-  check_logfile "testfiles/log"
-  generate
-}
-
-# functional
-function summarize_f() {
-  headline="$1"
-  output "\n## $headline\n"
-  rm -rf testfiles
-  for i in $(seq 1 $FUNCTIONAL_PARTS); do
-    tarfile="$2-part$i/part$i.tar"
-    check_tarfile "$tarfile"
-    check_logfile "testfiles/log"
-  done
-  generate
-}
-
-# https://docs.github.com/en/enterprise-server@3.6/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits
-# Job summaries are isolated between steps and each step is restricted to a maximum size of 1MiB.
-# [ ] can not show all error findings here
-# [x] split files into smaller ones and create additional steps
-
-ERRLOGS=0
-if [ ! -f Summary/Summary.md ]; then
-  # first call, we do the default summary (~500k)
-  echo -n > Summary.md
-  summarize_s "Sanity Tests Ubuntu 20.04" Logs-20.04-sanity
-  summarize_s "Sanity Tests Ubuntu 22.04" Logs-22.04-sanity
-  summarize_f "Functional Tests Ubuntu 20.04" Logs-20.04-functional
-  summarize_f "Functional Tests Ubuntu 22.04" Logs-22.04-functional
-
-  cat Summary.md >> $GITHUB_STEP_SUMMARY
-  mkdir -p Summary
-  mv *.md Summary
-else
-  # here we get, when errors where returned in first call
-  test -f Summary/err-$1.md && cat Summary/err-$1.md >> $GITHUB_STEP_SUMMARY
-fi
-
-exit 0

.github/workflows/scripts/qemu-3-deps.sh

@@ -141,6 +141,10 @@ case "$1" in
     ;;
 esac
 
+# This script is used for checkstyle + zloop deps also.
+# Install only the needed packages and exit - when used this way.
+test -z "${ONLY_DEPS:-}" || exit 0
+
 # Start services
 echo "##[group]Enable services"
 case "$1" in

.github/workflows/scripts/setup-dependencies.sh

@@ -1,88 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-function prerun() {
-  echo "::group::Install build dependencies"
-  # remove snap things, update+upgrade will be faster then
-  for x in lxd core20 snapd; do sudo snap remove $x; done
-  sudo apt-get purge snapd google-chrome-stable firefox
-  # https://github.com/orgs/community/discussions/47863
-  sudo apt-get remove grub-efi-amd64-bin grub-efi-amd64-signed shim-signed --allow-remove-essential
-  sudo apt-get update
-  sudo apt upgrade
-  sudo xargs --arg-file=.github/workflows/build-dependencies.txt apt-get install -qq
-  sudo apt-get clean
-  sudo dmesg -c > /var/tmp/dmesg-prerun
-  echo "::endgroup::"
-}
-
-function mod_build() {
-  echo "::group::Generate debian packages"
-  ./autogen.sh
-  ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
-  make --no-print-directory --silent native-deb-utils native-deb-kmod
-  mv ../*.deb .
-  rm ./openzfs-zfs-dracut*.deb ./openzfs-zfs-dkms*.deb
-  echo "$ImageOS-$ImageVersion" > tests/ImageOS.txt
-  echo "::endgroup::"
-}
-
-function mod_install() {
-  # install the pre-built module only on the same runner image
-  MOD=`cat tests/ImageOS.txt`
-  if [ "$MOD" != "$ImageOS-$ImageVersion" ]; then
-    rm -f *.deb
-    mod_build
-  fi
-
-  echo "::group::Install and load modules"
-  # don't use kernel-shipped zfs modules
-  sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
-  sudo apt-get install --fix-missing ./*.deb
-
-  # Native Debian packages enable and start the services
-  # Stop zfs-zed daemon, as it may interfere with some ZTS test cases
-  sudo systemctl stop zfs-zed
-  sudo depmod -a
-  sudo modprobe zfs
-  sudo dmesg
-  sudo dmesg -c > /var/tmp/dmesg-module-load
-  echo "::endgroup::"
-
-  echo "::group::Report CPU information"
-  lscpu
-  cat /proc/spl/kstat/zfs/chksum_bench
-  echo "::endgroup::"
-
-  echo "::group::Optimize storage for ZFS testings"
-  # remove swap and umount fast storage
-  # 89GiB -> rootfs + bootfs with ~80MB/s -> don't care
-  # 64GiB -> /mnt with 420MB/s -> new testing ssd
-  sudo swapoff -a
-
-  # this one is fast and mounted @ /mnt
-  # -> we reformat with ext4 + move it to /var/tmp
-  DEV="/dev/disk/azure/resource-part1"
-  sudo umount /mnt
-  sudo mkfs.ext4 -O ^has_journal -F $DEV
-  sudo mount -o noatime,barrier=0 $DEV /var/tmp
-  sudo chmod 1777 /var/tmp
-
-  # disk usage afterwards
-  sudo df -h /
-  sudo df -h /var/tmp
-  sudo fstrim -a
-  echo "::endgroup::"
-}
-
-case "$1" in
-  build)
-    prerun
-    mod_build
-    ;;
-  tests)
-    prerun
-    mod_install
-    ;;
-esac

.github/workflows/scripts/setup-functional.sh

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-TDIR="/usr/share/zfs/zfs-tests/tests/functional"
-echo -n "TODO="
-case "$1" in
-  part1)
-    # ~1h 20m
-    echo "cli_root"
-    ;;
-  part2)
-    # ~1h
-    ls $TDIR|grep '^[a-m]'|grep -v "cli_root"|xargs|tr -s ' ' ','
-    ;;
-  part3)
-    # ~1h
-    ls $TDIR|grep '^[n-qs-z]'|xargs|tr -s ' ' ','
-    ;;
-  part4)
-    # ~1h
-    ls $TDIR|grep '^r'|xargs|tr -s ' ' ','
-    ;;
-esac

.github/workflows/zfs-linux-tests.yml

@@ -1,124 +0,0 @@
-name: zfs-linux-tests
-
-on:
-  workflow_call:
-    inputs:
-      os:
-        description: 'The ubuntu version: 20.02 or 22.04'
-        required: true
-        type: string
-
-jobs:
-
-  zloop:
-    runs-on: ubuntu-${{ inputs.os }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v4
-      with:
-        name: modules-${{ inputs.os }}
-    - name: Install modules
-      run: |
-        tar xzf modules-${{ inputs.os }}.tgz
-        .github/workflows/scripts/setup-dependencies.sh tests
-    - name: Tests
-      timeout-minutes: 30
-      run: |
-        sudo mkdir -p /var/tmp/zloop
-        # run for 10 minutes or at most 2 iterations for a maximum runner
-        #   time of 20 minutes.
-        sudo /usr/share/zfs/zloop.sh -t 600 -I 2 -l -m1 -- -T 120 -P 60
-    - name: Prepare artifacts
-      if: failure()
-      run: |
-        sudo chmod +r -R /var/tmp/zloop/
-    - uses: actions/upload-artifact@v4
-      if: failure()
-      with:
-        name: Zloop-logs-${{ inputs.os }}
-        path: |
-          /var/tmp/zloop/*/
-          !/var/tmp/zloop/*/vdev/
-        retention-days: 14
-        if-no-files-found: ignore
-    - uses: actions/upload-artifact@v4
-      if: failure()
-      with:
-        name: Zloop-files-${{ inputs.os }}
-        path: |
-          /var/tmp/zloop/*/vdev/
-        retention-days: 14
-        if-no-files-found: ignore
-
-  sanity:
-    runs-on: ubuntu-${{ inputs.os }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v4
-      with:
-        name: modules-${{ inputs.os }}
-    - name: Install modules
-      run: |
-        tar xzf modules-${{ inputs.os }}.tgz
-        .github/workflows/scripts/setup-dependencies.sh tests
-    - name: Tests
-      timeout-minutes: 60
-      shell: bash
-      run: |
-        set -o pipefail
-        /usr/share/zfs/zfs-tests.sh -vKR -s 3G -r sanity | scripts/zfs-tests-color.sh
-    - name: Prepare artifacts
-      if: success() || failure()
-      run: |
-        RESPATH="/var/tmp/test_results"
-        mv -f $RESPATH/current $RESPATH/testfiles
-        tar cf $RESPATH/sanity.tar -h -C $RESPATH testfiles
-    - uses: actions/upload-artifact@v4
-      if: success() || failure()
-      with:
-        name: Logs-${{ inputs.os }}-sanity
-        path: /var/tmp/test_results/sanity.tar
-        if-no-files-found: ignore
-
-  functional:
-    runs-on: ubuntu-${{ inputs.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        tests: [ part1, part2, part3, part4 ]
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v4
-      with:
-        name: modules-${{ inputs.os }}
-    - name: Install modules
-      run: |
-        tar xzf modules-${{ inputs.os }}.tgz
-        .github/workflows/scripts/setup-dependencies.sh tests
-    - name: Setup tests
-      run: |
-        .github/workflows/scripts/setup-functional.sh ${{ matrix.tests }} >> $GITHUB_ENV
-    - name: Tests
-      timeout-minutes: 120
-      shell: bash
-      run: |
-        set -o pipefail
-        /usr/share/zfs/zfs-tests.sh -vKR -s 3G -T ${{ env.TODO }} | scripts/zfs-tests-color.sh
-    - name: Prepare artifacts
-      if: success() || failure()
-      run: |
-        RESPATH="/var/tmp/test_results"
-        mv -f $RESPATH/current $RESPATH/testfiles
-        tar cf $RESPATH/${{ matrix.tests }}.tar -h -C $RESPATH testfiles
-    - uses: actions/upload-artifact@v4
-      if: success() || failure()
-      with:
-        name: Logs-${{ inputs.os }}-functional-${{ matrix.tests }}
-        path: /var/tmp/test_results/${{ matrix.tests }}.tar
-        if-no-files-found: ignore

.github/workflows/zfs-linux.yml

@@ -1,64 +0,0 @@
-name: zfs-linux
-
-on:
-  push:
-  pull_request:
-
-jobs:
-
-  build:
-    name: Build
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [20.04, 22.04]
-    runs-on: ubuntu-${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-    - name: Build modules
-      run: .github/workflows/scripts/setup-dependencies.sh build
-    - name: Prepare modules upload
-      run: tar czf modules-${{ matrix.os }}.tgz *.deb .github tests/test-runner tests/ImageOS.txt
-    - uses: actions/upload-artifact@v4
-      with:
-        name: modules-${{ matrix.os }}
-        path: modules-${{ matrix.os }}.tgz
-        retention-days: 14
-
-  testings:
-    name: Testing
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [20.04, 22.04]
-    needs: build
-    uses: ./.github/workflows/zfs-linux-tests.yml
-    with:
-      os: ${{ matrix.os }}
-
-  cleanup:
-    if: always()
-    name: Cleanup
-    runs-on: ubuntu-22.04
-    needs: testings
-    steps:
-    - uses: actions/download-artifact@v4
-    - name: Generating summary
-      run: |
-        tar xzf modules-22.04/modules-22.04.tgz .github tests
-        .github/workflows/scripts/generate-summary.sh
-    # up to 4 steps, each can have 1 MiB output (for debugging log files)
-    - name: Summary for errors #1
-      run: .github/workflows/scripts/generate-summary.sh 1
-    - name: Summary for errors #2
-      run: .github/workflows/scripts/generate-summary.sh 2
-    - name: Summary for errors #3
-      run: .github/workflows/scripts/generate-summary.sh 3
-    - name: Summary for errors #4
-      run: .github/workflows/scripts/generate-summary.sh 4
-    - uses: actions/upload-artifact@v4
-      with:
-        name: Summary Files
-        path: Summary/

.github/workflows/zfs-qemu.yml

@@ -14,11 +14,7 @@ jobs:
         # all:
         # os:  [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd13r, freebsd14, freebsd14r, freebsd15, ubuntu20, ubuntu22, ubuntu24]
         # openzfs:
-        os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r]
-        # freebsd:
-        # os: [freebsd13, freebsd14]
-        # small test:
-        # os: [archlinux]
+        os: [almalinux8, almalinux9, centos-stream9, debian11, debian12, fedora39, fedora40, freebsd13, freebsd13r, freebsd14, freebsd14r, ubuntu20, ubuntu22, ubuntu24]
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4

.github/workflows/zloop.yml

@@ -0,0 +1,61 @@
+name: zloop
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  zloop:
+    runs-on: ubuntu-24.04
+    env:
+      TEST_DIR: /var/tmp/zloop
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Install dependencies
+      run: |
+        sudo apt-get purge -y snapd google-chrome-stable firefox
+        ONLY_DEPS=1 .github/workflows/scripts/qemu-3-deps.sh ubuntu24
+    - name: Autogen.sh
+      run: |
+        sed -i '/DEBUG_CFLAGS="-Werror"/s/^/#/' config/zfs-build.m4
+        ./autogen.sh
+    - name: Configure
+      run: |
+        ./configure --prefix=/usr --enable-debug --enable-debuginfo \
+           --enable-asan --enable-ubsan \
+           --enable-debug-kmem --enable-debug-kmem-tracking
+    - name: Make
+      run: |
+        make -j$(nproc)
+    - name: Install
+      run: |
+        sudo make install
+        sudo depmod
+        sudo modprobe zfs
+    - name: Tests
+      run: |
+        sudo mkdir -p $TEST_DIR
+        # run for 10 minutes or at most 6 iterations for a maximum runner
+        # time of 60 minutes.
+        sudo /usr/share/zfs/zloop.sh -t 600 -I 6 -l -m1 -- -T 120 -P 60
+    - name: Prepare artifacts
+      if: failure()
+      run: |
+        sudo chmod +r -R $TEST_DIR/
+    - uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: Logs
+        path: |
+          /var/tmp/zloop/*/
+          !/var/tmp/zloop/*/vdev/
+        if-no-files-found: ignore
+    - uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: Pool files
+        path: |
+          /var/tmp/zloop/*/vdev/
+        if-no-files-found: ignore