diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000000..2e0e25378e073e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,391 @@ +name: CI + +on: [push, pull_request] + +jobs: + build-fast-kernel: + runs-on: ubuntu-22.04 + steps: + - name: Checkout testing repository in root folder + uses: actions/checkout@v4 + with: + repository: LabNConsulting/iptfs-dev + + - name: Checkout source repository under linux + uses: actions/checkout@v4 + with: + path: linux + + - name: Install build depencies + run: | + sudo apt-get update -y + sudo apt-get install -y ccache device-tree-compiler libelf-dev xz-utils + + - name: Kernel build cache + id: linux-cache + uses: actions/cache@v4 + with: + path: output-linux.tar.xz + # We need a unique key name in order for new versions to save + key: ${{ runner.os }}-kernel-${{ hashFiles('linux-fast.config', 'output-linux.tar.xz') }} + restore-keys: | + key: ${{ runner.os }}-kernel- + + - name: Build kernel + run: | + set -x + rm -rf output-linux + if [ -f output-linux.tar.xz ]; then + echo "Cache found, extracting" + tar -xpJf output-linux.tar.xz + # If the kernel config is changed just wipe the cache + if ! diff -q -u1 linux-fast.config output-linux/.config -I '^[ \t]*#' -I '^[ \t]*$' 2>/dev/null; then + echo "Kernel configs differ erasing cache" + rm -rf output-linux output-linux.tar.xz + fi + fi + + export CCACHE_DIR=$PWD/output-linux/.ccache + if [ ! -d output-linux ]; then + echo "Creating new output directory" + mkdir output-linux + cp linux-fast.config output-linux/.config + fi + cd output-linux + ccache -z + KBUILD_BUILD_TIMESTAMP='' make CC="ccache gcc" -j2 O=$PWD -C ../linux + ccache -sz + [ -e arch/x86/boot/bzImage ] && cp arch/x86/boot/bzImage .. + # Ccache keeps it's own objects + # make clean + + # tar up the remaining files including .ccache + cd .. + echo "Collecting new cache contents" + tar -cf - output-linux | xz -T0 -c > output-linux.tar.xz + ls -hl output-linux.tar.xz + + - name: Archive kernel bzImage + uses: actions/upload-artifact@v4 + with: + name: kernel-bzImage + path: bzImage + retention-days: 1 + + build-cov-kernel: + runs-on: ubuntu-22.04 + steps: + - name: Checkout testing repository in root folder + uses: actions/checkout@v4 + with: + repository: LabNConsulting/iptfs-dev + + - name: Checkout source repository under linux + uses: actions/checkout@v4 + with: + path: linux + + - name: Install build depencies + run: | + sudo apt-get update -y + sudo apt-get install -y ccache device-tree-compiler libelf-dev xz-utils + + - name: Coverage kernel build cache + id: linux-cov-cache + uses: actions/cache@v4 + with: + path: output-cov-linux.tar.xz + # We need a unique key name in order for new versions to save + key: ${{ runner.os }}-cov-kernel-${{ hashFiles('linux-cov.config', 'output-cov-linux.tar.xz') }} + restore-keys: | + key: ${{ runner.os }}-cov-kernel- + + - name: Build coverage kernel + run: | + set -x + rm -rf output-linux + if [ -f output-cov-linux.tar.xz ]; then + echo "Cache found, extracting" + tar -xpJf output-cov-linux.tar.xz + # If the kernel config is changed just wipe the cache + if ! diff -q -u1 linux-cov.config output-linux/.config -I '^[ \t]*#' -I '^[ \t]*$' 2>/dev/null; then + echo "Kernel configs differ erasing cache" + rm -rf output-linux output-cov-linux.tar.xz + fi + fi + + export CCACHE_DIR=$PWD/output-linux/.ccache + if [ ! -d output-linux ]; then + echo "Creating new output directory" + mkdir output-linux + cp linux-cov.config output-linux/.config + fi + cd output-linux + ccache -z + KBUILD_BUILD_TIMESTAMP='' make CC="ccache gcc" -j2 O=$PWD -C ../linux + ccache -sz + [ -e arch/x86/boot/bzImage ] && cp arch/x86/boot/bzImage .. + # Ccache keeps it's own objects + # We want the objects and images too for now, restroy once we know + # coverage works + # make clean + + # tar up the remaining files including .ccache + cd .. + echo "Collecting new cache contents" + tar -cf - output-linux | xz -T0 -c > output-cov-linux.tar.xz + ls -hl output-cov-linux.tar.xz + + - name: Archive coverage kernel bzImage + uses: actions/upload-artifact@v4 + with: + name: kernel-cov-bzImage + path: bzImage + retention-days: 1 + + build-rootfs: + runs-on: ubuntu-22.04 + timeout-minutes: 600 + steps: + - name: Checkout testing repository in root folder + uses: actions/checkout@v4 + with: + repository: LabNConsulting/iptfs-dev + + - name: Checkout kernel repository + uses: actions/checkout@v4 + with: + path: linux + + - name: Checkout other repos + run: make setup SHALLOW_CLONE=1 + + - name: Install build depencies + run: | + sudo apt-get update -y + sudo apt-get install -y device-tree-compiler libelf-dev + + - name: Buildroot Build Cache + id: buildroot-cache + uses: actions/cache@v4 + with: + path: output-buildroot.tar.gz + key: ${{ runner.os }}-br-${{ hashFiles('buildroot.config', 'output-buildroot.tar.gz', 'root-key', 'root-key.pub') }} + restore-keys: | + key: ${{ runner.os }}-br- + + - name: Build Rootfs + timeout-minutes: 540 + run: | + set -x + if [ -f output-buildroot.tar.xz ] || [ -f output-buildroot.tar.gz ]; then + echo "Cache found, extracting" + if [ -f output-buildroot.tar.xz ]; then + tar -xpJf output-buildroot.tar.xz + else + tar -xpzf output-buildroot.tar.gz + fi + # If the buildroot config is changed just wipe the cache + if ! diff -q -u1 buildroot.config output-buildroot/.config -I '^[ \t]*#' -I '^[ \t]*$' 2>/dev/null; then + echo "Buildroot configs differ erasing cache" + rm -rf output-buildroot output-buildroot.tar.gz + fi + fi + + if [ ! -d output-buildroot ]; then + echo "Creating new output directory" + mkdir -p output-buildroot + cp -p buildroot.config output-buildroot/.config + else + make -C buildroot -j6 V=0 O=$PWD/output-buildroot iproute2-dirclean + fi + # make rootfs + echo "Starting build: $(date)" + make -C buildroot -j6 V=1 O=$PWD/output-buildroot LOCALVERSION='' > build-buildroot.txt 2>&1 + echo "Build complete: $(date)" + + - name: Archive buildroot build log + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: buildroot-build-log + path: build-buildroot.txt + retention-days: 30 + if-no-files-found: error + + - name: Collect cache contents + timeout-minutes: 120 + run: | + # tar up the remaining files including .ccache + echo "Collecting new cache contents: $(date)" + tar -czvf output-buildroot.tar.gz output-buildroot root-key root-key.pub || echo "error with tarball creation" + ls -hl output-buildroot.tar.gz || true + echo "Done collecting new cache contents: $(date)" + + - name: Archive buildroot rootfs.cpio.gz + uses: actions/upload-artifact@v4 + with: + name: rootfs-compressed-cpio + path: output-buildroot/images/rootfs.cpio.gz + retention-days: 1 + if-no-files-found: error + + - name: Archive buildroot root-key + uses: actions/upload-artifact@v4 + with: + name: root-key + path: | + root-key + root-key.pub + retention-days: 1 + if-no-files-found: error + + test: + runs-on: ubuntu-22.04 + needs: ["build-cov-kernel", "build-rootfs"] + steps: + - name: Checkout testing repository in root folder + uses: actions/checkout@v4 + with: + repository: LabNConsulting/iptfs-dev + + - name: Checkout kernel repository + uses: actions/checkout@v4 + with: + path: linux + + - name: Install test depencies + run: | + sudo apt-get update -y + sudo apt-get install -y lcov qemu-system-x86 socat + sudo python3 -m pip install -U munet pytest pytest-asyncio scapy + + # + # For coverage data, extract the source and full build to get the gcno + # files. + # + + - name: Restore coverage kernel build + id: linux-cov-cache-restore + uses: actions/cache/restore@v3 + with: + path: output-cov-linux.tar.xz + # We need a unique key name in order for new versions to save + key: ${{ runner.os }}-cov-kernel-${{ hashFiles('linux-cov.config', 'output-cov-linux.tar.xz') }} + restore-keys: | + key: ${{ runner.os }}-cov-kernel- + + - name: Extract coverage kernel build + run: | + [ -d output-linux ] && echo "Error: found existinb build" && exit 1 + tar -xvpJf output-cov-linux.tar.xz + + # + # We only need to artifacts for buildroot + # + - name: Fetch buildroot rootfs.cpio.gz archive + uses: actions/download-artifact@v4 + with: + name: rootfs-compressed-cpio + + - name: Fetch root-key + uses: actions/download-artifact@v4 + with: + name: root-key + + # Would be nice to loop this w/ and w/o coverage + + - name: Prep tests + run: | + test -f output-linux/arch/x86/boot/bzImage + mkdir -p output-buildroot/images/ + mv rootfs.cpio.gz output-buildroot/images/ + mkdir -p test-logs + + - name: Print the environment under sudo + run: sudo -E env + + - name: Config test + run: | + set -e + tmpf=test-logs/results-config.txt + sudo -E env CI=$CI python3 -m pytest -s --coverage tests/config | tee $tmpf + grep -v SKIPPED $tmpf + + - name: Errors test + run: | + set -e + tmpf=test-logs/results-errors.txt + tmpf=/tmp/test-results-$$.txt + sudo -E env CI=$CI python3 -m pytest -s --coverage tests/errors | tee $tmpf + grep -v SKIPPED $tmpf + + - name: Simple test + run: | + set -e + tmpf=test-logs/results-simplenet.txt + tmpf=/tmp/test-results-$$.txt + sudo -E env CI=$CI python3 -m pytest -s --coverage tests/simplenet | tee $tmpf + grep -v SKIPPED $tmpf + + # Restore longer running test once working + - name: UT packet test + run: | + set -e + tmpf=test-logs/results-utpkt.txt + sudo python3 -m pytest -s --coverage tests/utpkt | tee $tmpf + grep -v SKIPPED $tmpf + + - name: Extract coverage data + if: ${{ always() }} + run: | + make ci-extract-cov + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.info + flags: unittests + root_dir: ./linux + verbose: true + + - name: Collect test logs + if: ${{ always() }} + run: | + sudo find /tmp/unet-test -name 'gcov-data.tgz' -exec rm {} + + sudo find /tmp/unet-test -type s -exec rm {} + + sudo tar -C /tmp/unet-test -cf - . | tar -C test-logs -xf - + tar -cjf test-logs.tar.bz2 test-logs + + - name: Archive test logs tarball + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: test-logs-tar + path: test-logs.tar.bz2 + + deploy: + runs-on: ubuntu-22.04 + if: startsWith(github.ref, 'refs/tags/') + needs: ["build-fast-kernel", "build-rootfs", "test"] + steps: + + - name: Download kernel bzImage + uses: actions/download-artifact@v4 + with: + name: kernel-bzImage + + - name: Download buildroot rootfs.cpio.gz + uses: actions/download-artifact@v4 + with: + name: rootfs-compressed-cpio + + - name: Deploy + uses: softprops/action-gh-release@v1 + with: + files: | + bzImage + root-key + root-key.pub + rootfs.cpio.gz diff --git a/.gitignore b/.gitignore index 98274e1160d7b1..f1ecb9c2d08a3e 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,5 @@ sphinx_*/ # Rust analyzer configuration /rust-project.json + +!.github diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000000000..9957d9966f5681 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + target: auto + threshold: 10% +fixes: + - "/home/runner/work/iptfs-linux/iptfs-linux/linux::"