pr.yml

name: PR
on:
  workflow_dispatch:
  pull_request:
    paths:
      - "go.mod"
      - "**.go"
      - "**.yml"
      - "**.yaml"
      - "Makefile"
      - "!deploy/kicbase/**"
      - "!deploy/iso/**"
env:
  GOPROXY: https://proxy.golang.org
  GO_VERSION: '1.21.1'
permissions:
  contents: read

jobs:
  # Runs before all other jobs
  # builds the minikube binaries
  build_minikube:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
        with:
          go-version: ${{env.GO_VERSION}}
          cache-dependency-path: ./go.sum
      - name: Download Dependencies
        run: go mod download
      - name: Build Binaries
        run: |
          make cross
          make e2e-cross
          cp -r test/integration/testdata ./out
          whoami
          echo github ref $GITHUB_REF
          echo workflow $GITHUB_WORKFLOW
          echo home $HOME
          echo event name $GITHUB_EVENT_NAME
          echo workspace $GITHUB_WORKSPACE
          echo "end of debug stuff"
          echo $(which jq)
      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
        with:
          name: minikube_binaries
          path: out
  lint:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
        with:
          go-version: ${{env.GO_VERSION}}
          cache-dependency-path: ./go.sum
      - name: Install libvirt
        run: |
          sudo apt-get update
          sudo apt-get install -y libvirt-dev
      - name: Download Dependencies
        run: go mod download
      - name: Lint
        env:
          TESTSUITE: lintall
        run: make test
        continue-on-error: false
  unit_test:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
        with:
          go-version: ${{env.GO_VERSION}}
          cache-dependency-path: ./go.sum
      - name: Install libvirt
        run: |
          sudo apt-get update
          sudo apt-get install -y libvirt-dev
      - name: Download Dependencies
        run: go mod download
      - name: Unit Test
        env:
          TESTSUITE: unittest
        run: make test
        continue-on-error: false
  # Run the following integration tests after the build_minikube
  # They will run in parallel and use the binaries in previous step
  functional_docker_ubuntu:
    permissions:
      contents: none
    needs: [build_minikube]
    env:
      TIME_ELAPSED: time
      JOB_NAME: "functional_docker_ubuntu"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    runs-on: ubuntu-20.04
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -sSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      - name: Docker Info
        shell: bash
        run: |
          echo "--------------------------"
          docker version || true
          echo "--------------------------"
          docker info || true
          echo "--------------------------"
          docker system df || true
          echo "--------------------------"
          docker system info --format='{{json .}}'|| true
          echo "--------------------------"
          docker ps || true
          echo "--------------------------"
      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
        with:
          go-version: ${{env.GO_VERSION}}
      - name: Install gopogh
        shell: bash
        run: |
          go install github.com/medyagh/gopogh/cmd/gopogh@v0.23.0
      - name: Download Binaries
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
        with:
          name: minikube_binaries
          path: minikube_binaries
      - name: Run Integration Test
        continue-on-error: false
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          MINIKUBE_HOME=$(pwd)/testhome ./minikube-linux-amd64 delete --all --purge
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker  -test.run TestFunctional -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
          echo 'STAT<<EOF' >> $GITHUB_ENV
          echo "${STAT}" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
        with:
          name: functional_docker_ubuntu
          path: minikube_binaries/report
      - name: The End Result functional_docker_ubuntu
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*******************${numPass} Passes :) *******************"
          echo $STAT | jq '.PassedTests' || true
          echo "*******************************************************"
          echo "---------------- ${numFail} Failures :( ----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
  functional_docker_containerd_ubuntu:
    permissions:
      contents: none
    needs: [build_minikube]
    env:
      TIME_ELAPSED: time
      JOB_NAME: "functional_docker_containerd_ubuntu"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    runs-on: ubuntu-20.04
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -sSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      - name: Docker Info
        shell: bash
        run: |
          echo "--------------------------"
          docker version || true
          echo "--------------------------"
          docker info || true
          echo "--------------------------"
          docker system df || true
          echo "--------------------------"
          docker system info --format='{{json .}}'|| true
          echo "--------------------------"
          docker ps || true
          echo "--------------------------"
      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
        with:
          go-version: ${{env.GO_VERSION}}
      - name: Install gopogh
        shell: bash
        run: |
          go install github.com/medyagh/gopogh/cmd/gopogh@v0.23.0
      - name: Download Binaries
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
        with:
          name: minikube_binaries
          path: minikube_binaries
      - name: Run Integration Test
        continue-on-error: false
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          MINIKUBE_HOME=$(pwd)/testhome ./minikube-linux-amd64 delete --all --purge
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args="--vm-driver=docker --container-runtime=containerd" -test.run TestFunctional -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
          echo 'STAT<<EOF' >> $GITHUB_ENV
          echo "${STAT}" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
        with:
          name: functional_docker_containerd_ubuntu
          path: minikube_binaries/report
      - name: The End Result functional_docker_containerd_ubuntu
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*******************${numPass} Passes :) *******************"
          echo $STAT | jq '.PassedTests' || true
          echo "*******************************************************"
          echo "---------------- ${numFail} Failures :( ----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi

  functional_docker_rootless_containerd_ubuntu:
    permissions:
      contents: none
    needs: [build_minikube]
    env:
      TIME_ELAPSED: time
      JOB_NAME: "functional_docker_rootless_containerd_ubuntu"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
      DEBIAN_FRONTEND: noninteractive
    # ubuntu-22.04 is needed for cgroup v2
    runs-on: ubuntu-22.04
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -sSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      # https://rootlesscontaine.rs/getting-started/common/cgroup2/
      - name: Set up cgroup v2 delegation
        run: |
          sudo mkdir -p /etc/systemd/system/user@.service.d
          cat <<EOF | sudo tee /etc/systemd/system/user@.service.d/delegate.conf
          [Service]
          Delegate=cpu cpuset io memory pids
          EOF
          sudo systemctl daemon-reload
      - name: Set up Rootless Docker
        run: |
          sudo apt-get remove moby-engine-*
          curl https://get.docker.com | sudo sh
          dockerd-rootless-setuptool.sh install -f
          docker context use rootless
      - name: Docker Info
        shell: bash
        run: |
          echo "--------------------------"
          docker version || true
          echo "--------------------------"
          docker info || true
          echo "--------------------------"
          docker system df || true
          echo "--------------------------"
          docker system info --format='{{json .}}'|| true
          echo "--------------------------"
          docker ps || true
          echo "--------------------------"
      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
        with:
          go-version: ${{env.GO_VERSION}}
      - name: Install gopogh
        shell: bash
        run: |
          go install github.com/medyagh/gopogh/cmd/gopogh@v0.23.0
      - name: Download Binaries
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
        with:
          name: minikube_binaries
          path: minikube_binaries
      - name: Run Integration Test
        continue-on-error: false
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          MINIKUBE_HOME=$(pwd)/testhome ./minikube-linux-amd64 delete --all --purge
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args="--vm-driver=docker --rootless --container-runtime=containerd" -test.run TestFunctional -test.timeout=30m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
          echo 'STAT<<EOF' >> $GITHUB_ENV
          echo "${STAT}" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
        with:
          name: functional_docker_rootless_containerd_ubuntu
          path: minikube_binaries/report
      - name: The End Result functional_docker_rootless_containerd_ubuntu
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*******************${numPass} Passes :) *******************"
          echo $STAT | jq '.PassedTests' || true
          echo "*******************************************************"
          echo "---------------- ${numFail} Failures :( ----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi

  functional_podman_ubuntu:
    permissions:
      contents: none
    needs: [ build_minikube ]
    env:
      TIME_ELAPSED: time
      JOB_NAME: functional_podman_ubuntu
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    runs-on: ubuntu-20.04
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -sSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true

      - name: Install Podman
        shell: bash
        run: |
          sudo apt update
          sudo apt install -y podman
          echo "--------------------------"
          podman version || true
          echo "--------------------------"
          podman info || true
          echo "--------------------------"
          podman system df || true
          echo "--------------------------"
          podman system info --format='{{json .}}'|| true
          echo "--------------------------"
          podman ps || true
          echo "--------------------------"
      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
        with:
          go-version: ${{env.GO_VERSION}}
      - name: Install gopogh
        shell: bash
        run: |
          go install github.com/medyagh/gopogh/cmd/gopogh@v0.23.0
      - name: Download Binaries
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
        with:
          name: minikube_binaries
          path: minikube_binaries
      - name: Run Integration Test
        continue-on-error: false
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          MINIKUBE_HOME=$(pwd)/testhome ./minikube-linux-amd64 delete --all --purge
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=podman -v=6 --alsologtostderr -test.run TestFunctional -test.timeout=15m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
          echo 'STAT<<EOF' >> $GITHUB_ENV
          echo "${STAT}" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
        with:
          name: functional_podman_ubuntu
          path: minikube_binaries/report
      - name: The End Result functional_podman_ubuntu
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*******************${numPass} Passes :) *******************"
          echo $STAT | jq '.PassedTests' || true
          echo "*******************************************************"
          echo "---------------- ${numFail} Failures :( ----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi

  functional_virtualbox_macos:
    permissions:
      contents: none
    needs: [build_minikube]
    env:
      TIME_ELAPSED: time
      JOB_NAME: "functional_virtualbox_macos"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    runs-on: macos-12
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -sSL https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl"
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
        with:
          go-version: ${{env.GO_VERSION}}
      - name: Install gopogh
        shell: bash
        run: |
          go install github.com/medyagh/gopogh/cmd/gopogh@v0.23.0
      - name: Install docker
        shell: bash
        run: |
          brew install docker-machine docker
          sudo docker --version
      - name: Info
        shell: bash
        run: |
          hostname
          VBoxManage --version
          sysctl hw.physicalcpu hw.logicalcpu
      - name: Disable firewall
        run: |
          sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
          sudo /usr/libexec/ApplicationFirewall/socketfilterfw -k
      - name: Download Binaries
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
        with:
          name: minikube_binaries
          path: minikube_binaries
      - name: Run Integration Test
        continue-on-error: false
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          MINIKUBE_HOME=$(pwd)/testhome ./minikube-darwin-amd64 delete --all --purge
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-darwin-amd64 -minikube-start-args=--vm-driver=virtualbox  -test.run "TestFunctional" -test.timeout=35m -test.v -timeout-multiplier=1.5 -binary=./minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
          echo 'STAT<<EOF' >> $GITHUB_ENV
          echo "${STAT}" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
        with:
          name: functional_virtualbox_macos
          path: minikube_binaries/report
      - name: The End Result functional_virtualbox_macos
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*******************${numPass} Passes :) *******************"
          echo $STAT | jq '.PassedTests' || true
          echo "*******************************************************"
          echo "----------------${numFail} Failures :(----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 33 ];then echo "*** Failed to pass at least 33 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
  functional_baremetal_ubuntu20_04:
    permissions:
      contents: none
    needs: [build_minikube]
    env:
      TIME_ELAPSED: time
      JOB_NAME: "functional_baremetal_ubuntu20_04"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    runs-on: ubuntu-20.04
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -sSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe
        with:
          go-version: ${{env.GO_VERSION}}
      # conntrack is required for kubernetes 1.18 and higher
      # socat is required for kubectl port forward which is used in some tests such as validateHelmTillerAddon
      # cri-dockerd is required for Kubernetes 1.24 and higher for none driver
      - name: Install tools for none
        shell: bash
        run: |
          sudo apt-get update -qq
          sudo apt-get -qq -y install conntrack
          sudo apt-get -qq -y install socat
          CRI_DOCKERD_VERSION="v0.3.3"
          CRI_DOCKERD_COMMIT="b58acf8f78f9d7bce1241d1cddb0932e7101f278"
          CRI_DOCKERD_BASE_URL="https://storage.googleapis.com/kicbase-artifacts/cri-dockerd/${CRI_DOCKERD_COMMIT}"
          sudo curl -L "${CRI_DOCKERD_BASE_URL}/amd64/cri-dockerd" -o /usr/bin/cri-dockerd
          sudo curl -L "${CRI_DOCKERD_BASE_URL}/cri-docker.socket" -o /usr/lib/systemd/system/cri-docker.socket
          sudo curl -L "${CRI_DOCKERD_BASE_URL}/cri-docker.service" -o /usr/lib/systemd/system/cri-docker.service
          sudo chmod +x /usr/bin/cri-dockerd
          CRICTL_VERSION="v1.17.0"
          curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz --output crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
          sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin
      - name: Install gopogh
        shell: bash
        run: |
          go install github.com/medyagh/gopogh/cmd/gopogh@v0.23.0
      - name: Set fs.protected_regular
        shell: bash
        run: |
          sudo sysctl fs.protected_regular=0
      - name: Download Binaries
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
        with:
          name: minikube_binaries
          path: minikube_binaries
      - name: Run Integration Test
        continue-on-error: true
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          MINIKUBE_HOME=$(pwd)/testhome ./minikube-linux-amd64 delete --all --purge
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=10m -test.v -timeout-multiplier=1.5 -test.run TestFunctional -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo "TIME_ELAPSED=${TIME_ELAPSED}" >> $GITHUB_ENV
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out_html ./report/testout.html -out_summary ./report/testout_summary.json -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo "GOPOGH_RESULT=${GOPOGH_RESULT}" >> $GITHUB_ENV
          echo 'STAT<<EOF' >> $GITHUB_ENV
          echo "${STAT}" >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV
      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
        with:
          name: functional_baremetal_ubuntu20_04
          path: minikube_binaries/report
      - name: The End Result functional_baremetal_ubuntu20_04
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*******************${numPass} Passes :) *******************"
          echo $STAT | jq '.PassedTests' || true
          echo "*******************************************************"
          echo "---------------- ${numFail} Failures :( ----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 26 ];then echo "*** Failed to pass at least 26 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
  # After all integration tests finished
  # collect all the reports and upload them
  upload_all_reports:
    permissions:
      contents: none
    if: always()
    needs:
      [
        functional_docker_ubuntu,
        functional_docker_containerd_ubuntu,
        functional_docker_rootless_containerd_ubuntu,
        functional_podman_ubuntu,
        functional_virtualbox_macos,
        functional_baremetal_ubuntu20_04,
      ]
    runs-on: ubuntu-20.04
    steps:
      - name: download all reports
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
      - name: upload all reports
        shell: bash {0}
        continue-on-error: true
        run: |
          mkdir -p all_reports
          ls -lah
          cp -r ./functional_docker_ubuntu ./all_reports/
          cp -r ./functional_docker_containerd_ubuntu ./all_reports/
          cp -r ./functional_docker_rootless_containerd_ubuntu ./all_reports/
          cp -r ./functional_podman_ubuntu ./all_reports/
          cp -r ./functional_virtualbox_macos ./all_reports/
          cp -r ./functional_baremetal_ubuntu20_04 ./all_reports/

      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
        with:
          name: all_reports
          path: all_reports