ci: add github action for CI testing

.github/workflows/ci.yml

@@ -0,0 +1,391 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build-fast-kernel:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout testing repository in root folder
+        uses: actions/checkout@v4
+        with:
+          repository: LabNConsulting/iptfs-dev
+
+      - name: Checkout source repository under linux
+        uses: actions/checkout@v4
+        with:
+          path: linux
+
+      - name: Install build depencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y ccache device-tree-compiler libelf-dev xz-utils
+
+      - name: Kernel build cache
+        id: linux-cache
+        uses: actions/cache@v4
+        with:
+          path: output-linux.tar.xz
+          # We need a unique key name in order for new versions to save
+          key: ${{ runner.os }}-kernel-${{ hashFiles('linux-fast.config', 'output-linux.tar.xz') }}
+          restore-keys: |
+            key: ${{ runner.os }}-kernel-
+
+      - name: Build kernel
+        run: |
+          set -x
+          rm -rf output-linux
+          if [ -f output-linux.tar.xz ]; then
+            echo "Cache found, extracting"
+            tar -xpJf output-linux.tar.xz
+            # If the kernel config is changed just wipe the cache
+            if ! diff -q -u1 linux-fast.config output-linux/.config -I '^[ \t]*#' -I '^[ \t]*$' 2>/dev/null; then
+              echo "Kernel configs differ erasing cache"
+              rm -rf output-linux output-linux.tar.xz
+            fi
+          fi
+
+          export CCACHE_DIR=$PWD/output-linux/.ccache
+          if [ ! -d output-linux ]; then
+            echo "Creating new output directory"
+            mkdir output-linux
+            cp linux-fast.config output-linux/.config
+          fi
+          cd output-linux
+          ccache -z
+          KBUILD_BUILD_TIMESTAMP='' make CC="ccache gcc" -j2 O=$PWD -C ../linux
+          ccache -sz
+          [ -e arch/x86/boot/bzImage ] && cp arch/x86/boot/bzImage ..
+          # Ccache keeps it's own objects
+          # make clean
+
+          # tar up the remaining files including .ccache
+          cd ..
+          echo "Collecting new cache contents"
+          tar -cf - output-linux | xz -T0 -c > output-linux.tar.xz
+          ls -hl output-linux.tar.xz
+
+      - name: Archive kernel bzImage
+        uses: actions/upload-artifact@v4
+        with:
+          name: kernel-bzImage
+          path: bzImage
+          retention-days: 1
+
+  build-cov-kernel:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout testing repository in root folder
+        uses: actions/checkout@v4
+        with:
+          repository: LabNConsulting/iptfs-dev
+
+      - name: Checkout source repository under linux
+        uses: actions/checkout@v4
+        with:
+          path: linux
+
+      - name: Install build depencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y ccache device-tree-compiler libelf-dev xz-utils
+
+      - name: Coverage kernel build cache
+        id: linux-cov-cache
+        uses: actions/cache@v4
+        with:
+          path: output-cov-linux.tar.xz
+          # We need a unique key name in order for new versions to save
+          key: ${{ runner.os }}-cov-kernel-${{ hashFiles('linux-cov.config', 'output-cov-linux.tar.xz') }}
+          restore-keys: |
+            key: ${{ runner.os }}-cov-kernel-
+
+      - name: Build coverage kernel
+        run: |
+          set -x
+          rm -rf output-linux
+          if [ -f output-cov-linux.tar.xz ]; then
+            echo "Cache found, extracting"
+            tar -xpJf output-cov-linux.tar.xz
+            # If the kernel config is changed just wipe the cache
+            if ! diff -q -u1 linux-cov.config output-linux/.config -I '^[ \t]*#' -I '^[ \t]*$' 2>/dev/null; then
+              echo "Kernel configs differ erasing cache"
+              rm -rf output-linux output-cov-linux.tar.xz
+            fi
+          fi
+
+          export CCACHE_DIR=$PWD/output-linux/.ccache
+          if [ ! -d output-linux ]; then
+            echo "Creating new output directory"
+            mkdir output-linux
+            cp linux-cov.config output-linux/.config
+          fi
+          cd output-linux
+          ccache -z
+          KBUILD_BUILD_TIMESTAMP='' make CC="ccache gcc" -j2 O=$PWD -C ../linux
+          ccache -sz
+          [ -e arch/x86/boot/bzImage ] && cp arch/x86/boot/bzImage ..
+          # Ccache keeps it's own objects
+          # We want the objects and images too for now, restroy once we know
+          # coverage works
+          # make clean
+
+          # tar up the remaining files including .ccache
+          cd ..
+          echo "Collecting new cache contents"
+          tar -cf - output-linux | xz -T0 -c > output-cov-linux.tar.xz
+          ls -hl output-cov-linux.tar.xz
+
+      - name: Archive coverage kernel bzImage
+        uses: actions/upload-artifact@v4
+        with:
+          name: kernel-cov-bzImage
+          path: bzImage
+          retention-days: 1
+
+  build-rootfs:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 600
+    steps:
+      - name: Checkout testing repository in root folder
+        uses: actions/checkout@v4
+        with:
+          repository: LabNConsulting/iptfs-dev
+
+      - name: Checkout kernel repository
+        uses: actions/checkout@v4
+        with:
+          path: linux
+
+      - name: Checkout other repos
+        run: make setup SHALLOW_CLONE=1
+
+      - name: Install build depencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y device-tree-compiler libelf-dev
+
+      - name: Buildroot Build Cache
+        id: buildroot-cache
+        uses: actions/cache@v4
+        with:
+          path: output-buildroot.tar.gz
+          key: ${{ runner.os }}-br-${{ hashFiles('buildroot.config', 'output-buildroot.tar.gz', 'root-key', 'root-key.pub') }}
+          restore-keys: |
+            key: ${{ runner.os }}-br-
+
+      - name: Build Rootfs
+        timeout-minutes: 540
+        run: |
+          set -x
+          if [ -f output-buildroot.tar.xz ] || [ -f output-buildroot.tar.gz ]; then
+            echo "Cache found, extracting"
+            if [ -f output-buildroot.tar.xz ]; then
+              tar -xpJf output-buildroot.tar.xz
+            else
+              tar -xpzf output-buildroot.tar.gz
+            fi
+            # If the buildroot config is changed just wipe the cache
+            if ! diff -q -u1 buildroot.config output-buildroot/.config -I '^[ \t]*#' -I '^[ \t]*$' 2>/dev/null; then
+              echo "Buildroot configs differ erasing cache"
+              rm -rf output-buildroot output-buildroot.tar.gz
+            fi
+          fi
+
+          if [ ! -d output-buildroot ]; then
+            echo "Creating new output directory"
+            mkdir -p output-buildroot
+            cp -p buildroot.config output-buildroot/.config
+          else
+            make -C buildroot -j6 V=0 O=$PWD/output-buildroot iproute2-dirclean
+          fi
+          # make rootfs
+          echo "Starting build: $(date)"
+          make -C buildroot -j6 V=1 O=$PWD/output-buildroot LOCALVERSION='' > build-buildroot.txt 2>&1
+          echo "Build complete: $(date)"
+
+      - name: Archive buildroot build log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: buildroot-build-log
+          path: build-buildroot.txt
+          retention-days: 30
+          if-no-files-found: error
+
+      - name: Collect cache contents
+        timeout-minutes: 120
+        run: |
+          # tar up the remaining files including .ccache
+          echo "Collecting new cache contents: $(date)"
+          tar -czvf output-buildroot.tar.gz output-buildroot root-key root-key.pub || echo "error with tarball creation"
+          ls -hl output-buildroot.tar.gz || true
+          echo "Done collecting new cache contents: $(date)"
+
+      - name: Archive buildroot rootfs.cpio.gz
+        uses: actions/upload-artifact@v4
+        with:
+          name: rootfs-compressed-cpio
+          path: output-buildroot/images/rootfs.cpio.gz
+          retention-days: 1
+          if-no-files-found: error
+
+      - name: Archive buildroot root-key
+        uses: actions/upload-artifact@v4
+        with:
+          name: root-key
+          path: |
+            root-key
+            root-key.pub
+          retention-days: 1
+          if-no-files-found: error
+
+  test:
+    runs-on: ubuntu-22.04
+    needs: ["build-cov-kernel", "build-rootfs"]
+    steps:
+      - name: Checkout testing repository in root folder
+        uses: actions/checkout@v4
+        with:
+          repository: LabNConsulting/iptfs-dev
+
+      - name: Checkout kernel repository
+        uses: actions/checkout@v4
+        with:
+          path: linux
+
+      - name: Install test depencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y lcov qemu-system-x86 socat
+          sudo python3 -m pip install -U munet pytest pytest-asyncio scapy
+
+      #
+      # For coverage data, extract the source and full build to get the gcno
+      # files.
+      #
+
+      - name: Restore coverage kernel build
+        id: linux-cov-cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: output-cov-linux.tar.xz
+          # We need a unique key name in order for new versions to save
+          key: ${{ runner.os }}-cov-kernel-${{ hashFiles('linux-cov.config', 'output-cov-linux.tar.xz') }}
+          restore-keys: |
+            key: ${{ runner.os }}-cov-kernel-
+
+      - name: Extract coverage kernel build
+        run: |
+          [ -d output-linux ] && echo "Error: found existinb build" && exit 1
+          tar -xvpJf output-cov-linux.tar.xz
+
+      #
+      # We only need to artifacts for buildroot
+      #
+      - name: Fetch buildroot rootfs.cpio.gz archive
+        uses: actions/download-artifact@v4
+        with:
+          name: rootfs-compressed-cpio
+
+      - name: Fetch root-key
+        uses: actions/download-artifact@v4
+        with:
+          name: root-key
+
+      # Would be nice to loop this w/ and w/o coverage
+
+      - name: Prep tests
+        run: |
+          test -f output-linux/arch/x86/boot/bzImage
+          mkdir -p output-buildroot/images/
+          mv rootfs.cpio.gz output-buildroot/images/
+          mkdir -p test-logs
+
+      - name: Print the environment under sudo
+        run: sudo -E env
+
+      - name: Config test
+        run: |
+          set -e
+          tmpf=test-logs/results-config.txt
+          sudo -E env CI=$CI python3 -m pytest -s --coverage tests/config | tee $tmpf
+          grep -v SKIPPED $tmpf
+
+      - name: Errors test
+        run: |
+          set -e
+          tmpf=test-logs/results-errors.txt
+          tmpf=/tmp/test-results-$$.txt
+          sudo -E env CI=$CI python3 -m pytest -s --coverage tests/errors | tee $tmpf
+          grep -v SKIPPED $tmpf
+
+      - name: Simple test
+        run: |
+          set -e
+          tmpf=test-logs/results-simplenet.txt
+          tmpf=/tmp/test-results-$$.txt
+          sudo -E env CI=$CI python3 -m pytest -s --coverage tests/simplenet | tee $tmpf
+          grep -v SKIPPED $tmpf
+
+      # Restore longer running test once working
+      - name: UT packet test
+        run: |
+          set -e
+          tmpf=test-logs/results-utpkt.txt
+          sudo python3 -m pytest -s --coverage tests/utpkt | tee $tmpf
+          grep -v SKIPPED $tmpf
+
+      - name: Extract coverage data
+        if: ${{ always() }}
+        run: |
+          make ci-extract-cov
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.info
+          flags: unittests
+          root_dir: ./linux
+          verbose: true
+
+      - name: Collect test logs
+        if: ${{ always() }}
+        run: |
+          sudo find /tmp/unet-test -name 'gcov-data.tgz' -exec rm {} +
+          sudo find /tmp/unet-test -type s -exec rm {} +
+          sudo tar -C /tmp/unet-test -cf - . | tar -C test-logs -xf -
+          tar -cjf test-logs.tar.bz2 test-logs
+
+      - name: Archive test logs tarball
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-logs-tar
+          path: test-logs.tar.bz2
+
+  deploy:
+    runs-on: ubuntu-22.04
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: ["build-fast-kernel", "build-rootfs", "test"]
+    steps:
+
+      - name: Download kernel bzImage
+        uses: actions/download-artifact@v4
+        with:
+          name: kernel-bzImage
+
+      - name: Download buildroot rootfs.cpio.gz
+        uses: actions/download-artifact@v4
+        with:
+          name: rootfs-compressed-cpio
+
+      - name: Deploy
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            bzImage
+            root-key
+            root-key.pub
+            rootfs.cpio.gz

.gitignore

@@ -169,3 +169,5 @@ sphinx_*/
 
 # Rust analyzer configuration
 /rust-project.json
+
+!.github