Update workflows

Update the GitHub actions workflows using a subset of the changes from the master branch, commit 620a977. Cherry-picking each relevant commit would have resulted in a large number of conflicts so this change only applies a minimal set of useful updates. - Added build-dependencies.txt and checkstyle-dependencies.txt - Added reclaim_disk_space.sh script - Minor changes to build steps - Reduced ztest run time - checkbashisms, mandoc, and cppcheck were not included to avoid additional backports - Add exceptions for shellcheck Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
behlendorf · Mar 10, 2023 · 4f6e0ca · 4f6e0ca
1 parent 1d2aea7
commit 4f6e0ca
Show file tree

Hide file tree

Showing 10 changed files with 127 additions and 54 deletions.
diff --git a/.github/workflows/build-dependencies.txt b/.github/workflows/build-dependencies.txt
@@ -0,0 +1,57 @@
+acl
+alien
+attr
+autoconf
+bc
+build-essential
+curl
+dbench
+debhelper-compat
+dh-python
+dkms
+fakeroot
+fio
+gdb
+gdebi
+git
+ksh
+lcov
+libacl1-dev
+libaio-dev
+libattr1-dev
+libblkid-dev
+libcurl4-openssl-dev
+libdevmapper-dev
+libelf-dev
+libffi-dev
+libmount-dev
+libpam0g-dev
+libselinux1-dev
+libssl-dev
+libtool
+libudev-dev
+linux-headers-generic
+lsscsi
+mdadm
+nfs-kernel-server
+pamtester
+parted
+po-debconf
+python3
+python3-all-dev
+python3-cffi
+python3-dev
+python3-packaging
+python3-pip
+python3-setuptools
+python3-sphinx
+rng-tools-debian
+rsync
+samba
+sysstat
+uuid-dev
+watchdog
+wget
+xfslibs-dev
+xz-utils
+zlib1g-dev
diff --git a/.github/workflows/checkstyle-dependencies.txt b/.github/workflows/checkstyle-dependencies.txt
@@ -0,0 +1,2 @@
+pax-utils
+shellcheck
diff --git a/.github/workflows/checkstyle.yaml b/.github/workflows/checkstyle.yaml
@@ -17,12 +17,16 @@ jobs:
         sudo apt-mark hold grub-efi-amd64-signed
         sudo apt-get update --fix-missing
         sudo apt-get upgrade
-        sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot linux-headers-$(uname -r)
-        sudo apt-get install --yes -qq zlib1g-dev uuid-dev libattr1-dev libblkid-dev libselinux-dev libudev-dev libssl-dev python-dev python-setuptools python-cffi python3 python3-dev python3-setuptools python3-cffi
-        # packages for tests
-        sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio
-        sudo apt-get install --yes -qq mandoc cppcheck pax-utils devscripts
-        sudo -E pip --quiet install flake8
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/checkstyle-dependencies.txt apt-get install -qq
+        sudo python3 -m pip install --quiet flake8
+        sudo apt-get clean
+
+        # confirm that the tools are installed
+        # the build system doesn't fail when they are not
+        flake8 --version
+        scanelf --version
+        shellcheck --version
     - name: Prepare
       run: |
         sh ./autogen.sh

diff --git a/.github/workflows/scripts/reclaim_disk_space.sh b/.github/workflows/scripts/reclaim_disk_space.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -eu
+
+# remove 4GiB of images
+sudo systemd-run docker system prune --force --all --volumes
+
+# remove unused software
+sudo systemd-run --wait rm -rf \
+  "$AGENT_TOOLSDIRECTORY" \
+  /opt/* \
+  /usr/local/* \
+  /usr/share/az* \
+  /usr/share/dotnet \
+  /usr/share/gradle* \
+  /usr/share/miniconda \
+  /usr/share/swift \
+  /var/lib/gems \
+  /var/lib/mysql \
+  /var/lib/snapd
+
+# trim the cleaned space
+sudo fstrim /
diff --git a/.github/workflows/zfs-tests-functional.yml b/.github/workflows/zfs-tests-functional.yml
@@ -21,16 +21,8 @@ jobs:
         sudo apt-mark hold grub-efi-amd64-signed
         sudo apt-get update --fix-missing
         sudo apt-get upgrade
-        sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \
-          git alien fakeroot wget curl bc fio acl \
-          sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \
-          nfs-kernel-server samba rng-tools xz-utils \
-          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
-          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
-          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
-          libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi python3-packaging \
-          libcurl4-openssl-dev
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
+        sudo apt-get clean
     - name: Autogen.sh
       run: |
         sh autogen.sh
@@ -59,12 +51,13 @@ jobs:
             sudo udevadm control --reload-rules
           fi
         fi
-        # Workaround to provide additional free space for testing.
-        #   https://github.com/actions/virtual-environments/issues/2840
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf /opt/ghc
-        sudo rm -rf "/usr/local/share/boost"
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+    - name: Clear the kernel ring buffer
+      run: |
+        sudo dmesg -c >/var/tmp/dmesg-prerun
+    - name: Reclaim and report disk space
+      run: |
+        ${{ github.workspace }}/.github/workflows/scripts/reclaim_disk_space.sh
+        df -h /
     - name: Tests
       run: |
         /usr/share/zfs/zfs-tests.sh -vR -s 3G
@@ -74,13 +67,15 @@ jobs:
       run: |
         RESULTS_PATH=$(readlink -f /var/tmp/test_results/current)
         sudo dmesg > $RESULTS_PATH/dmesg
-        sudo cp /var/log/syslog $RESULTS_PATH/
+        sudo cp /var/log/syslog /var/tmp/dmesg-prerun $RESULTS_PATH/
         sudo chmod +r $RESULTS_PATH/*
         # Replace ':' in dir names, actions/upload-artifact doesn't support it
         for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done
     - uses: actions/upload-artifact@v3
       if: failure()
       with:
         name: Test logs Ubuntu-${{ matrix.os }}
-        path: /var/tmp/test_results/20*/
+        path: |
+          /var/tmp/test_results/*
+          !/var/tmp/test_results/current
         if-no-files-found: ignore
diff --git a/.github/workflows/zfs-tests-sanity.yml b/.github/workflows/zfs-tests-sanity.yml
@@ -17,16 +17,8 @@ jobs:
         sudo apt-mark hold grub-efi-amd64-signed
         sudo apt-get update --fix-missing
         sudo apt-get upgrade
-        sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \
-          git alien fakeroot wget curl bc fio acl \
-          sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \
-          nfs-kernel-server samba rng-tools xz-utils \
-          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
-          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
-          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
-          libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python3 python3-dev python3-setuptools python3-cffi python3-packaging \
-          libcurl4-openssl-dev
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
+        sudo apt-get clean
     - name: Autogen.sh
       run: |
         sh autogen.sh
@@ -55,12 +47,13 @@ jobs:
             sudo udevadm control --reload-rules
           fi
         fi
-        # Workaround to provide additional free space for testing.
-        #   https://github.com/actions/virtual-environments/issues/2840
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf /opt/ghc
-        sudo rm -rf "/usr/local/share/boost"
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+    - name: Clear the kernel ring buffer
+      run: |
+        sudo dmesg -c >/var/tmp/dmesg-prerun
+    - name: Reclaim and report disk space
+      run: |
+        ${{ github.workspace }}/.github/workflows/scripts/reclaim_disk_space.sh
+        df -h /
     - name: Tests
       run: |
         /usr/share/zfs/zfs-tests.sh -vR -s 3G -r sanity
@@ -70,13 +63,15 @@ jobs:
       run: |
         RESULTS_PATH=$(readlink -f /var/tmp/test_results/current)
         sudo dmesg > $RESULTS_PATH/dmesg
-        sudo cp /var/log/syslog $RESULTS_PATH/
+        sudo cp /var/log/syslog /var/tmp/dmesg-prerun $RESULTS_PATH/
         sudo chmod +r $RESULTS_PATH/*
         # Replace ':' in dir names, actions/upload-artifact doesn't support it
         for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done
     - uses: actions/upload-artifact@v3
       if: failure()
       with:
-        name: Test logs
-        path: /var/tmp/test_results/20*/
+        name: Test logs Ubuntu-${{ matrix.os }}
+        path: |
+          /var/tmp/test_results/*
+          !/var/tmp/test_results/current
         if-no-files-found: ignore
diff --git a/.github/workflows/zloop.yml b/.github/workflows/zloop.yml
@@ -19,14 +19,8 @@ jobs:
         sudo apt-mark hold grub-efi-amd64-signed
         sudo apt-get update --fix-missing
         sudo apt-get upgrade
-        sudo apt-get install --yes -qq build-essential autoconf libtool gdb \
-          git alien fakeroot \
-          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
-          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
-          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
-          libpam0g-dev \
-          python-dev python-setuptools python-cffi python-packaging \
-          python3 python3-dev python3-setuptools python3-cffi python3-packaging
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
+        sudo apt-get clean
     - name: Autogen.sh
       run: |
         sh autogen.sh
@@ -47,8 +41,9 @@ jobs:
     - name: Tests
       run: |
         sudo mkdir -p $TEST_DIR
-        # run for 20 minutes to have a total runner time of 30 minutes
-        sudo /usr/share/zfs/zloop.sh -t 1200 -l -m1 -- -T 120 -P 60
+        # run for 10 minutes or at most 2 iterations for a maximum runner
+        #   time of 20 minutes.
+        sudo /usr/share/zfs/zloop.sh -t 600 -I 2 -l -m1 -- -T 120 -P 60
     - name: Prepare artifacts
       if: failure()
       run: |

diff --git a/contrib/initramfs/scripts/Makefile.am b/contrib/initramfs/scripts/Makefile.am
@@ -7,5 +7,6 @@ dist_scripts_SCRIPTS = \
 
 SUBDIRS = local-top
 
+SHELLCHECK_IGNORE = ,SC2295
 SHELLCHECKDIRS = $(SUBDIRS)
 SHELLCHECK_SHELL = sh
diff --git a/scripts/Makefile.am b/scripts/Makefile.am
@@ -27,7 +27,7 @@ EXTRA_DIST = \
 	zol2zfs-patch.sed \
 	$(EXTRA_SCRIPTS)
 
-SHELLCHECK_IGNORE = ,SC1117
+SHELLCHECK_IGNORE = ,SC1117,SC2086,SC2295
 SHELLCHECKSCRIPTS = $(EXTRA_SCRIPTS)
 
 define EXTRA_ENVIRONMENT

diff --git a/tests/Makefile.am b/tests/Makefile.am
@@ -4,5 +4,6 @@ SUBDIRS = runfiles test-runner zfs-tests
 
 EXTRA_DIST = README.md
 
+SHELLCHECK_IGNORE = ,SC2155
 SHELLCHECKSCRIPTS = $$(find . -name '*.sh')
 .PHONY: $(SHELLCHECKSCRIPTS)