conformance-k8s-kind.yaml

name: Conformance K8s Kind

# Any change in triggers needs to be reflected in the concurrency group.
on:
  pull_request:
    paths-ignore:
      - 'Documentation/**'
      - 'test/**'
  push:
    branches:
      - main
      - ft/main/**
    paths-ignore:
      - 'Documentation/**'
      - 'test/**'

permissions: read-all

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }}
  cancel-in-progress: true

env:
  cluster_name: cilium-testing
  cilium_cli_ci_version:

jobs:
  kubernetes-e2e:
    name: Installation and Conformance Test
    runs-on: ubuntu-latest
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        # TODO add "dual" and "ipv6", "ipv6" fails to install cilium
        ipFamily: ["ipv4"]
    env:
      IP_FAMILY: ${{ matrix.ipFamily }}

    steps:
      - name: Collect Workflow Telemetry
        uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0
        with:
          comment_on_pr: false

      - name: Checkout target branch to access local actions
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ github.base_ref || github.ref }}
          persist-credentials: false

      - name: Set Environment Variables
        uses: ./.github/actions/set-env-variables

      - name: Enable ipv4 and ipv6 forwarding
        run: |
          sudo sysctl -w net.ipv6.conf.all.forwarding=1
          sudo sysctl -w net.ipv4.ip_forward=1

      - name: Set up environment (download Kubernetes dependencies)
        run: |
          TMP_DIR=$(mktemp -d)
          # Test binaries
          curl -L https://dl.k8s.io/${{ env.KIND_K8S_VERSION }}/kubernetes-test-linux-amd64.tar.gz -o ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz
          tar xvzf ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz \
            --directory ${TMP_DIR} \
            --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
          # kubectl
          curl -L https://dl.k8s.io/${{ env.KIND_K8S_VERSION }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
          # kind
          curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/${{ env.KIND_VERSION }}/kind-linux-amd64
          # Install
          sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo
          sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test
          sudo cp ${TMP_DIR}/kubectl /usr/local/bin/kubectl
          sudo cp ${TMP_DIR}/kind /usr/local/bin/kind
          sudo chmod +x /usr/local/bin/*
          sudo rm -rf ${TMP_DIR}

      - name: Create multi node cluster
        run: |
          cat <<EOF | /usr/local/bin/kind create cluster \
            --name ${{ env.cluster_name}}           \
            --image ${{ env.KIND_K8S_IMAGE }}  \
            -v7 --retain --config=-
          kind: Cluster
          apiVersion: kind.x-k8s.io/v1alpha4
          networking:
            ipFamily: ${IP_FAMILY}
            kubeProxyMode: "none"
            disableDefaultCNI: true
          nodes:
          - role: control-plane
          - role: worker
          - role: worker
          EOF

      - name: Workaround CoreDNS for IPv6 airgapped
        if: ${{ matrix.ipFamily == 'ipv6' }}
        run: |
          # Patch CoreDNS to work in Github CI
          # 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
          # to work in an offline environment:
          # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
          # 2. Github CI adds following domains to resolv.conf search field:
          # .net.
          # CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
          # otherwise pods stops trying to resolve the domain.
          # Get the current config
          original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns)
          echo "Original CoreDNS config:"
          echo "${original_coredns}"
          # Patch it
          fixed_coredns=$(
            printf '%s' "${original_coredns}" | sed \
              -e 's/^.*kubernetes cluster\.local/& net/' \
              -e '/^.*upstream$/d' \
              -e '/^.*fallthrough.*$/d' \
              -e '/^.*forward . \/etc\/resolv.conf$/d' \
              -e '/^.*loop$/d' \
          )
          echo "Patched CoreDNS config:"
          echo "${fixed_coredns}"
          printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f -

      - name: Get Cilium's default values
        id: default_vars
        uses: ./.github/actions/helm-default
        with:
          image-tag: ${{ github.event.pull_request.head.sha }}

      - name: Set up job variables
        id: vars
        run: |
          # Note: On Kind, we install Cilium with HostPort (portmap CNI chaining) enabled,
          # to ensure coverage of that feature in cilium connectivity test
          CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \
            --helm-set=cni.chainingMode=portmap \
            --helm-set-string=kubeProxyReplacement=true \
            --helm-set=sessionAffinity=true"
          echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT
          echo sha=${{ steps.default_vars.outputs.sha }} >> $GITHUB_OUTPUT

      - name: Install Cilium CLI
        uses: cilium/cilium-cli@a9579d1afc528b085930c5237a69ee77b2c2be76 # v0.16.4
        with:
          repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
          release-version: ${{ env.CILIUM_CLI_VERSION }}
          ci-version: ${{ env.cilium_cli_ci_version }}

      - name: Checkout code
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          ref: ${{ steps.vars.outputs.sha }}
          persist-credentials: false

      - name: Wait for images to be available
        timeout-minutes: 30
        shell: bash
        run: |
          for image in cilium-ci operator-generic-ci hubble-relay-ci ; do
            until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
          done

      - name: Install Cilium
        id: install-cilium
        run: |
          cilium install --wait ${{ steps.vars.outputs.cilium_install_defaults }}

      - name: Run Kubernetes sig-network conformance test
        run: |
          # output_dir
          mkdir -p _artifacts

          # get kubeconfig to pass to the e2e binary
          kind get kubeconfig --name ${{ env.cluster_name }} > _artifacts/kubeconfig.conf

          # Kubernetes e2e tests use ginkgo and tags to select the tests that should run based on two regex, focus and skip:
          # Focus tests:
          # \[Conformance\]|\[sig-network\]: Conformance tests are defined by the project to guarantee a consistent behaviour and some mandatory features on all clusters
          #                                  sig-network tests are defined by sig-networkto guarantee a consistent behaviour on all the the k8s network implementations
          # Skipped tests:
          # Disruptive|Serial : require to run in serial and perform disruptive operations on clusters (reboots, ...)
          # Federation|PerformanceDNS : unrelated sig-network tests
          # Feature : skip features that are not GA, however, some of them should be enabled, per example [Feature:ProxyTerminatingEndpoints]
          # DualStack : only with dualstack clusters
          # KubeProxy|kube-proxy : kube-proxy specifics
          # LoadBalancer|GCE|ExternalIP : require a cloud provider, some of them are GCE specifics
          # Netpol|NetworkPolicy : network policies, demand significant resources and use to be slow, better to run in a different job
          # Aggregator : Flaky, https://github.com/cilium/cilium/issues/24622.
          # same.port.number.but.different.protocols|HostPort|should.serve.endpoints.on.same.port.and.different.protocols : #9207
          # rejected : Kubernetes expect Services without endpoints associated to REJECT the connection to notify the client, Cilium silently drops the packet
          # externalTrafficPolicy : needs investigation

          # Run tests
          export KUBERNETES_CONFORMANCE_TEST='y'
          export E2E_REPORT_DIR=${PWD}/_artifacts
          /usr/local/bin/ginkgo --nodes=25                \
            --focus="\[Conformance\]|\[sig-network\]"     \
            --skip="Feature|Federation|PerformanceDNS|DualStack|Disruptive|Serial|KubeProxy|kube-proxy|ExternalIP|LoadBalancer|GCE|Netpol|NetworkPolicy|Aggregator|rejected|externalTrafficPolicy|HostPort|same.port.number.but.different.protocols|should.serve.endpoints.on.same.port.and.different.protocols"   \
            /usr/local/bin/e2e.test                       \
            --                                            \
            --kubeconfig=${PWD}/_artifacts/kubeconfig.conf     \
            --provider=local                              \
            --dump-logs-on-failure=true                   \
            --report-dir=${E2E_REPORT_DIR}                \
            --disable-log-dump=true

      - name: Post-test information gathering
        if: ${{ !success() && steps.install-cilium.outcome != 'skipped' }}
        run: |
          kubectl get pods --all-namespaces -o wide
          cilium status
          cilium sysdump --output-filename cilium-sysdump-final
          /usr/local/bin/kind export logs --name  ${{ env.cluster_name }} --loglevel=debug ./_artifacts/logs
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently

      - name: Upload artifacts
        if: ${{ !success() }}
        uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4.3.2
        with:
          name: cilium-sysdumps
          path: cilium-sysdump-*.zip
          retention-days: 5

      - name: Upload cluster logs
        if: ${{ !success() }}
        uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4.3.2
        with:
          name: kind-logs
          path: ./_artifacts/logs
          retention-days: 5

      - name: Upload Kubernetes e2e Junit Reports [junit]
        if: ${{ success() }}
        uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4
        with:
          name: kubernetes-e2e-junit
          path: './_artifacts/*.xml'

      - name: Publish Test Results As GitHub Summary
        if: ${{ always() }}
        uses: aanm/junit2md@332ebf0fddd34e91b03a832cfafaa826306558f9 # v0.0.3
        with:
          junit-directory: "_artifacts"