diff --git a/.github/workflows/build-dependencies.txt b/.github/workflows/build-dependencies.txt new file mode 100644 index 000000000..73921865c --- /dev/null +++ b/.github/workflows/build-dependencies.txt @@ -0,0 +1,57 @@ +acl +alien +attr +autoconf +bc +build-essential +curl +dbench +debhelper-compat +dh-python +dkms +fakeroot +fio +gdb +gdebi +git +ksh +lcov +libacl1-dev +libaio-dev +libattr1-dev +libblkid-dev +libcurl4-openssl-dev +libdevmapper-dev +libelf-dev +libffi-dev +libmount-dev +libpam0g-dev +libselinux1-dev +libssl-dev +libtool +libudev-dev +linux-headers-generic +lsscsi +mdadm +nfs-kernel-server +pamtester +parted +po-debconf +python3 +python3-all-dev +python3-cffi +python3-dev +python3-packaging +python3-pip +python3-setuptools +python3-sphinx +rng-tools-debian +rsync +samba +sysstat +uuid-dev +watchdog +wget +xfslibs-dev +xz-utils +zlib1g-dev diff --git a/.github/workflows/checkstyle-dependencies.txt b/.github/workflows/checkstyle-dependencies.txt new file mode 100644 index 000000000..cc68905d8 --- /dev/null +++ b/.github/workflows/checkstyle-dependencies.txt @@ -0,0 +1,2 @@ +pax-utils +shellcheck diff --git a/.github/workflows/checkstyle.yaml b/.github/workflows/checkstyle.yaml index 7b48d80c7..1a31f061e 100644 --- a/.github/workflows/checkstyle.yaml +++ b/.github/workflows/checkstyle.yaml @@ -17,12 +17,16 @@ jobs: sudo apt-mark hold grub-efi-amd64-signed sudo apt-get update --fix-missing sudo apt-get upgrade - sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot linux-headers-$(uname -r) - sudo apt-get install --yes -qq zlib1g-dev uuid-dev libattr1-dev libblkid-dev libselinux-dev libudev-dev libssl-dev python-dev python-setuptools python-cffi python3 python3-dev python3-setuptools python3-cffi - # packages for tests - sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio - sudo apt-get install --yes -qq mandoc cppcheck pax-utils devscripts - sudo -E pip --quiet install flake8 + sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq + sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/checkstyle-dependencies.txt apt-get install -qq + sudo python3 -m pip install --quiet flake8 + sudo apt-get clean + + # confirm that the tools are installed + # the build system doesn't fail when they are not + flake8 --version + scanelf --version + shellcheck --version - name: Prepare run: | sh ./autogen.sh diff --git a/.github/workflows/scripts/reclaim_disk_space.sh b/.github/workflows/scripts/reclaim_disk_space.sh new file mode 100755 index 000000000..ed23ce31d --- /dev/null +++ b/.github/workflows/scripts/reclaim_disk_space.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -eu + +# remove 4GiB of images +sudo systemd-run docker system prune --force --all --volumes + +# remove unused software +sudo systemd-run --wait rm -rf \ + "$AGENT_TOOLSDIRECTORY" \ + /opt/* \ + /usr/local/* \ + /usr/share/az* \ + /usr/share/dotnet \ + /usr/share/gradle* \ + /usr/share/miniconda \ + /usr/share/swift \ + /var/lib/gems \ + /var/lib/mysql \ + /var/lib/snapd + +# trim the cleaned space +sudo fstrim / diff --git a/.github/workflows/zfs-tests-functional.yml b/.github/workflows/zfs-tests-functional.yml index 403b20a40..1d30ddc64 100644 --- a/.github/workflows/zfs-tests-functional.yml +++ b/.github/workflows/zfs-tests-functional.yml @@ -21,16 +21,8 @@ jobs: sudo apt-mark hold grub-efi-amd64-signed sudo apt-get update --fix-missing sudo apt-get upgrade - sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \ - git alien fakeroot wget curl bc fio acl \ - sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \ - nfs-kernel-server samba rng-tools xz-utils \ - zlib1g-dev uuid-dev libblkid-dev libselinux-dev \ - xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \ - libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \ - libpam0g-dev pamtester python-dev python-setuptools python-cffi \ - python3 python3-dev python3-setuptools python3-cffi python3-packaging \ - libcurl4-openssl-dev + sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq + sudo apt-get clean - name: Autogen.sh run: | sh autogen.sh @@ -59,12 +51,13 @@ jobs: sudo udevadm control --reload-rules fi fi - # Workaround to provide additional free space for testing. - # https://github.com/actions/virtual-environments/issues/2840 - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: Clear the kernel ring buffer + run: | + sudo dmesg -c >/var/tmp/dmesg-prerun + - name: Reclaim and report disk space + run: | + ${{ github.workspace }}/.github/workflows/scripts/reclaim_disk_space.sh + df -h / - name: Tests run: | /usr/share/zfs/zfs-tests.sh -vR -s 3G @@ -74,7 +67,7 @@ jobs: run: | RESULTS_PATH=$(readlink -f /var/tmp/test_results/current) sudo dmesg > $RESULTS_PATH/dmesg - sudo cp /var/log/syslog $RESULTS_PATH/ + sudo cp /var/log/syslog /var/tmp/dmesg-prerun $RESULTS_PATH/ sudo chmod +r $RESULTS_PATH/* # Replace ':' in dir names, actions/upload-artifact doesn't support it for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done @@ -82,5 +75,7 @@ jobs: if: failure() with: name: Test logs Ubuntu-${{ matrix.os }} - path: /var/tmp/test_results/20*/ + path: | + /var/tmp/test_results/* + !/var/tmp/test_results/current if-no-files-found: ignore diff --git a/.github/workflows/zfs-tests-sanity.yml b/.github/workflows/zfs-tests-sanity.yml index 4b7765046..6a1432c29 100644 --- a/.github/workflows/zfs-tests-sanity.yml +++ b/.github/workflows/zfs-tests-sanity.yml @@ -17,16 +17,8 @@ jobs: sudo apt-mark hold grub-efi-amd64-signed sudo apt-get update --fix-missing sudo apt-get upgrade - sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \ - git alien fakeroot wget curl bc fio acl \ - sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \ - nfs-kernel-server samba rng-tools xz-utils \ - zlib1g-dev uuid-dev libblkid-dev libselinux-dev \ - xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \ - libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \ - libpam0g-dev pamtester python-dev python-setuptools python-cffi \ - python3 python3-dev python3-setuptools python3-cffi python3-packaging \ - libcurl4-openssl-dev + sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq + sudo apt-get clean - name: Autogen.sh run: | sh autogen.sh @@ -55,12 +47,13 @@ jobs: sudo udevadm control --reload-rules fi fi - # Workaround to provide additional free space for testing. - # https://github.com/actions/virtual-environments/issues/2840 - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: Clear the kernel ring buffer + run: | + sudo dmesg -c >/var/tmp/dmesg-prerun + - name: Reclaim and report disk space + run: | + ${{ github.workspace }}/.github/workflows/scripts/reclaim_disk_space.sh + df -h / - name: Tests run: | /usr/share/zfs/zfs-tests.sh -vR -s 3G -r sanity @@ -70,13 +63,15 @@ jobs: run: | RESULTS_PATH=$(readlink -f /var/tmp/test_results/current) sudo dmesg > $RESULTS_PATH/dmesg - sudo cp /var/log/syslog $RESULTS_PATH/ + sudo cp /var/log/syslog /var/tmp/dmesg-prerun $RESULTS_PATH/ sudo chmod +r $RESULTS_PATH/* # Replace ':' in dir names, actions/upload-artifact doesn't support it for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done - uses: actions/upload-artifact@v3 if: failure() with: - name: Test logs - path: /var/tmp/test_results/20*/ + name: Test logs Ubuntu-${{ matrix.os }} + path: | + /var/tmp/test_results/* + !/var/tmp/test_results/current if-no-files-found: ignore diff --git a/.github/workflows/zloop.yml b/.github/workflows/zloop.yml index f6b54d5f0..440ec01fa 100644 --- a/.github/workflows/zloop.yml +++ b/.github/workflows/zloop.yml @@ -19,14 +19,8 @@ jobs: sudo apt-mark hold grub-efi-amd64-signed sudo apt-get update --fix-missing sudo apt-get upgrade - sudo apt-get install --yes -qq build-essential autoconf libtool gdb \ - git alien fakeroot \ - zlib1g-dev uuid-dev libblkid-dev libselinux-dev \ - xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \ - libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \ - libpam0g-dev \ - python-dev python-setuptools python-cffi python-packaging \ - python3 python3-dev python3-setuptools python3-cffi python3-packaging + sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq + sudo apt-get clean - name: Autogen.sh run: | sh autogen.sh @@ -47,8 +41,9 @@ jobs: - name: Tests run: | sudo mkdir -p $TEST_DIR - # run for 20 minutes to have a total runner time of 30 minutes - sudo /usr/share/zfs/zloop.sh -t 1200 -l -m1 -- -T 120 -P 60 + # run for 10 minutes or at most 2 iterations for a maximum runner + # time of 20 minutes. + sudo /usr/share/zfs/zloop.sh -t 600 -I 2 -l -m1 -- -T 120 -P 60 - name: Prepare artifacts if: failure() run: | diff --git a/contrib/initramfs/scripts/Makefile.am b/contrib/initramfs/scripts/Makefile.am index 444a5f374..5bcbfb96b 100644 --- a/contrib/initramfs/scripts/Makefile.am +++ b/contrib/initramfs/scripts/Makefile.am @@ -7,5 +7,6 @@ dist_scripts_SCRIPTS = \ SUBDIRS = local-top +SHELLCHECK_IGNORE = ,SC2295 SHELLCHECKDIRS = $(SUBDIRS) SHELLCHECK_SHELL = sh diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 6c59fd7d4..047ae7eac 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -27,7 +27,7 @@ EXTRA_DIST = \ zol2zfs-patch.sed \ $(EXTRA_SCRIPTS) -SHELLCHECK_IGNORE = ,SC1117 +SHELLCHECK_IGNORE = ,SC1117,SC2086,SC2295 SHELLCHECKSCRIPTS = $(EXTRA_SCRIPTS) define EXTRA_ENVIRONMENT diff --git a/tests/Makefile.am b/tests/Makefile.am index 1dfc2cc5f..d8277ef2d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -4,5 +4,6 @@ SUBDIRS = runfiles test-runner zfs-tests EXTRA_DIST = README.md +SHELLCHECK_IGNORE = ,SC2155 SHELLCHECKSCRIPTS = $$(find . -name '*.sh') .PHONY: $(SHELLCHECKSCRIPTS)