Merge pull request #374 from gabriel-samfira/make-workflow-pluggable

Make the kola test workflow reusable

.github/workflows/ci.yaml

@@ -42,9 +42,9 @@ jobs:
   packages:
     name: "Build Flatcar packages"
     runs-on:
+      - self-hosted
       - debian
       - build
-      - self-hosted
       - x64
     strategy:
       fail-fast: false
@@ -61,18 +61,14 @@ jobs:
         run: |
           sudo rm /bin/sh
           sudo ln -s /bin/bash /bin/sh
-          sudo apt-get install \
-            ca-certificates \
-            curl \
-            gnupg \
-            lsb-release
+          sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git
           sudo mkdir -p /etc/apt/keyrings
           curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
           echo \
             "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
             $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
           sudo apt-get update
-          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin qemu-user-static git
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
 
       - uses: actions/checkout@v3
         with:
@@ -287,3 +283,8 @@ jobs:
             scripts/images/**/flatcar_production_*.sh
             scripts/images/**/flatcar_test_update.gz
             scripts/ebuild_logs.tar.xz
+
+  test:
+    needs: packages
+    name: "Run kola tests"
+    uses: ./.github/workflows/run-kola-tests.yaml

.github/workflows/dispatch-kola-tests.yaml

@@ -0,0 +1,26 @@
+name: Dispatch kola tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      workflow_name_or_id:
+        type: string
+        required: true
+        default: ci.yaml
+        description: |
+          The workflow ID from where we'll download the artifacts to be tested.
+      workflow_run_id:
+        type: string
+        required: true
+        description: |
+          The run ID of the workflow specified in workflow_name_or_id
+
+permissions: {}
+
+jobs:
+  test:
+    name: "Trigger kola test - test"
+    uses: ./.github/workflows/run-kola-tests.yaml
+    with:
+      workflow_name_or_id: ${{ github.event.inputs.workflow_name_or_id }}
+      workflow_run_id: ${{ github.event.inputs.workflow_run_id }}

.github/workflows/run-kola-tests.yaml

@@ -0,0 +1,141 @@
+name: "Run kola tests"
+on:
+  workflow_call:
+    inputs:
+      workflow_name_or_id:
+        type: string
+        required: false
+        default: ci.yaml
+        description: |
+          The workflow ID from where we'll download the artifacts to be tested.
+      workflow_run_id:
+        type: string
+        required: false
+        description: |
+          The run ID of the workflow specified in workflow_name_or_id
+
+jobs:
+  tests:
+    name: "Run Kola tests"
+    runs-on:
+      - self-hosted
+      - debian
+      - kola
+      - ${{ matrix.arch }}
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: ["amd64", "arm64"]
+
+    steps:
+      - name: Prepare machine
+        shell: bash
+        working-directory: ${{ github.workspace }}
+        run: |
+          sudo rm /bin/sh
+          sudo ln -s /bin/bash /bin/sh
+          sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-system git bzip2 jq dnsmasq
+          sudo systemctl stop dnsmasq
+          sudo systemctl mask dnsmasq
+
+          # Install Docker-CE
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+
+          # Set up MASQUERADE. Don't care much to secure it.
+          # This is needed for the VMs kola spins up to have internet access.
+          DEFAULT_ROUTE_DEVICE=$(ip -j route sh default |jq -r .[0].dev)
+          sudo iptables -t nat -I POSTROUTING -o $DEFAULT_ROUTE_DEVICE -j MASQUERADE
+          sudo iptables -I FORWARD -o $DEFAULT_ROUTE_DEVICE -j ACCEPT
+          sudo iptables -I FORWARD -i $DEFAULT_ROUTE_DEVICE -j ACCEPT
+
+          # ARM64 tests run inside an LXD container instead of an LXD Virtual machine
+          # There are some limitations in terms of what we can customize.
+          if [ "${{ matrix.arch }}" == "amd64" ];then
+            # Enable IP forward
+            echo 'net.ipv4.ip_forward=1' | sudo tee /etc/sysctl.d/ip_forward.conf
+
+            # Enable unprivileged bpf
+            echo 'kernel.unprivileged_bpf_disabled=0' | sudo tee /etc/sysctl.d/enable_unprivileged_bpf.conf
+
+            sudo sysctl --system
+            sysctl -p
+
+            # Enable extra SWAP
+            sudo fallocate /swap.img -l 8GiB
+            sudo chmod 600 /swap.img
+            sudo mkswap /swap.img
+            sudo swapon /swap.img
+          fi
+
+      - uses: actions/checkout@v3
+        with:
+          path: scripts
+          fetch-depth: 0
+          submodules: true
+
+      - name: Download artifact
+        if: ${{ !inputs.workflow_run_id }}
+        uses: actions/download-artifact@v3
+        with:
+          name: images-${{ matrix.arch }}
+
+      - name: Download artifacts from other workflow
+        uses: gabriel-samfira/action-download-artifact@v5
+        if: ${{ inputs.workflow_run_id }}
+        with:
+          workflow: ${{ inputs.workflow_name_or_id }}
+          workflow_conclusion: success
+          run_id: ${{ inputs.workflow_run_id }}
+          name: images-${{ matrix.arch }}
+
+      - name: Run tests
+        shell: bash
+        run: |
+          exec 2>&1
+          set +x
+          set -euo pipefail
+
+          # extract the image.
+          IMG_ARCHIVE=$(readlink -f images/**/flatcar_production_image.bin.bz2)
+          QEMU_UEFI_BIOS_FILE=$(readlink -f images/**/flatcar_production_qemu_uefi_efi_code.fd)
+          bzip2 --decompress ${IMG_ARCHIVE}
+
+          cp ${IMG_ARCHIVE%%.bz2} ./scripts/
+          cp ${QEMU_UEFI_BIOS_FILE} ./scripts/
+
+          pushd scripts
+          source ci-automation/test.sh
+
+          PARALLEL_ARCH=2
+          if [ "${{ matrix.arch }}" == "arm64" ];then
+            # ARM64 servers have more memory and CPUs
+            PARALLEL_ARCH=10
+          fi
+
+          cat > sdk_container/.env <<EOF
+          # export the QEMU_IMAGE_NAME to avoid to download it.
+          export QEMU_IMAGE_NAME="/work/flatcar_production_image.bin"
+          export QEMU_UEFI_BIOS="/work/flatcar_production_qemu_uefi_efi_code.fd"
+          export PARALLEL_TESTS=${PARALLEL_ARCH}
+          EOF
+
+          export MAX_RETRIES=5
+          export SKIP_COPY_TO_BINCACHE=1
+
+          # run the test.
+          test_run ${{ matrix.arch }} qemu_uefi
+
+      - name: Upload artifacts
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-${{ matrix.arch }}
+          path: |
+            scripts/__TESTS__
+            scripts/results-.*.tap

ci-automation/ci-config.env

@@ -45,15 +45,15 @@ CONTAINER_IMAGE_ROOT="/home/sdk/build/images"
 
 # -- QEMU --
 
-QEMU_IMAGE_NAME="flatcar_production_image.bin"
+QEMU_IMAGE_NAME=${QEMU_IMAGE_NAME:-flatcar_production_image.bin}
 QEMU_PARALLEL="${PARALLEL_TESTS:-20}"
 
 # BIOS path within the SDK
 QEMU_BIOS="/usr/share/qemu/bios-256k.bin"
 
 # UEFI bios filename on build cache.
 # Published by vms.sh as part of the qemu vendor build.
-QEMU_UEFI_BIOS="flatcar_production_qemu_uefi_efi_code.fd"
+QEMU_UEFI_BIOS="${QEMU_UEFI_BIOS:-flatcar_production_qemu_uefi_efi_code.fd}"
 
 
 # -- Equinix Metal --

ci-automation/test.sh

@@ -120,6 +120,7 @@ function _test_run_impl() {
     fi
 
     local retries="${MAX_RETRIES:-20}"
+    local skip_copy_to_bincache=${SKIP_COPY_TO_BINCACHE:-0}
 
     source ci-automation/tapfile_helper_lib.sh
     source ci-automation/ci_automation_common.sh
@@ -216,14 +217,19 @@ function _test_run_impl() {
         echo "########### All re-runs exhausted ($retries). Giving up. ###########"
     fi
 
-    # publish kola output, TAP files to build cache
-    copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
-        "${tests_dir}/_kola_temp"
-    copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
-        "${tests_dir}/"*.tap
-    copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
-        "${tap_merged_summary}"
-    copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
-        "${tap_merged_detailed}"
+    if [ ${skip_copy_to_bincache} -eq 0 ];then
+        # publish kola output, TAP files to build cache
+        copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+            "${tests_dir}/_kola_temp"
+        copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+            "${tests_dir}/"*.tap
+        copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+            "${tap_merged_summary}"
+        copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+            "${tap_merged_detailed}"
+    fi
+    if ! $success; then
+        return 1
+    fi
 }
 # --

ci-automation/vendor-testing/qemu.sh

@@ -22,7 +22,7 @@ fi
 
 # Fetch image and BIOS if not present
 if [ -f "${QEMU_IMAGE_NAME}" ] ; then
-    echo "++++ ${CIA_TESTSCRIPT}: Using existing ./${QEMU_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+    echo "++++ ${CIA_TESTSCRIPT}: Using existing ${QEMU_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
 else
     echo "++++ ${CIA_TESTSCRIPT}: downloading ${QEMU_IMAGE_NAME} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
     rm -f "${QEMU_IMAGE_NAME}.bz2"
@@ -34,7 +34,7 @@ bios="${QEMU_BIOS}"
 if [ "${CIA_TESTSCRIPT}" = "qemu_uefi.sh" ] ; then
     bios="${QEMU_UEFI_BIOS}"
     if [ -f "${bios}" ] ; then
-        echo "++++ ${CIA_TESTSCRIPT}: Using existing ./${bios} ++++"
+        echo "++++ ${CIA_TESTSCRIPT}: Using existing ${bios} ++++"
     else
         echo "++++ ${CIA_TESTSCRIPT}: downloading ${bios} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
         copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${bios}" .
@@ -51,6 +51,7 @@ kola run \
     --qemu-image="${QEMU_IMAGE_NAME}" \
     --tapfile="${CIA_TAPFILE}" \
     --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+    --qemu-skip-mangle \
     "${@}"
 
 set +x

ci-automation/vendor-testing/qemu_update.sh

@@ -55,4 +55,5 @@ kola run \
     --tapfile="${CIA_TAPFILE}" \
     --torcx-manifest="${CIA_TORCX_MANIFEST}" \
     --update-payload=tmp/flatcar_test_update.gz \
+    --qemu-skip-mangle \
     cl.update.payload