conformance-aks.yaml

name: ConformanceAKS (ci-aks)

# Any change in triggers needs to be reflected in the concurrency group.
on:
  issue_comment:
    types:
      - created
  # Run every 6 hours
  schedule:
    - cron:  '0 0/6 * * *'
  ### FOR TESTING PURPOSES
  # This workflow runs in the context of `master`, and ignores changes to
  # workflow files in PRs. For testing changes to this workflow from a PR:
  # - Make sure the PR uses a branch from the base repository (requires write
  #   privileges). It will not work with a branch from a fork (missing secrets).
  # - Uncomment the `pull_request` event below, commit separately with a `DO
  #   NOT MERGE` message, and push to the PR. As long as the commit is present,
  #   any push to the PR will trigger this workflow.
  # - Don't forget to remove the `DO NOT MERGE` commit once satisfied. The run
  #   will disappear from the PR checks: please provide a direct link to the
  #   successful workflow run (can be found from Actions tab) in a comment.
  # 
  # pull_request: {}
  ###

# By specifying the access of one of the scopes, all of those that are not
# specified are set to 'none'.
permissions:
  # To be able to access the repository with actions/checkout
  contents: read
  # To allow retrieving information from the PR API
  pull-requests: read
  # So that Sibz/github-status-action can write into the status API
  statuses: write

concurrency:
  # Structure:
  # - Workflow name
  # - Event type
  # - A unique identifier depending on event type:
  #   - schedule: SHA
  #   - issue_comment: PR number
  #   - pull_request: PR number
  #
  # This structure ensures a unique concurrency group name is generated for each
  # type of testing:
  # - schedule: {name} schedule {SHA}
  # - issue_comment: {name} issue_comment {PR number}
  # - pull_request: {name} pull_request {PR number}
  #
  # Note: for `issue_comment` triggers, we additionally need to filter out based
  # on comment content, otherwise any comment will interrupt workflow runs.
  group: |
    ${{ github.workflow }}
    ${{ github.event_name }}
    ${{
      (github.event_name == 'schedule' && github.sha) ||
      (github.event_name == 'issue_comment' &&
        (startsWith(github.event.comment.body, 'ci-aks') ||
        startsWith(github.event.comment.body, 'test-me-please')) &&
        github.event.issue.number) ||
      (github.event_name == 'pull_request' && github.event.pull_request.number)
    }}
  cancel-in-progress: true

env:
  name: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
  location: westeurope
  cilium_cli_version: v0.8.6
  check_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

jobs:
  check_changes:
    name: Deduce required tests from code changes
    if: |
      (github.event_name == 'issue_comment' && (
        startsWith(github.event.comment.body, 'ci-aks') ||
        (startsWith(github.event.comment.body, 'test-me-please'))
      )) ||
      (github.event_name == 'schedule' && github.repository == 'cilium/cilium') ||
      github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    outputs:
      tested: ${{ steps.tested-tree.outputs.src }}
    steps:
      # Because we run on issue comments, we need to checkout the code for
      # paths-filter to work.
      - name: Checkout code
        if: ${{ github.event.issue.pull_request }}
        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
        with:
          persist-credentials: false
      - name: Retrieve pull request's base and head
        if: ${{ github.event.issue.pull_request }}
        id: pr
        run: |
          curl ${{ github.event.issue.pull_request.url }} > pr.json
          echo "::set-output name=base::$(jq -r '.base.sha' pr.json)"
          echo "::set-output name=head::$(jq -r '.head.sha' pr.json)"
      - name: Check code changes
        if: ${{ github.event.issue.pull_request }}
        uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
        id: tested-tree
        with:
          base: ${{ steps.pr.outputs.base }}
          ref: ${{ steps.pr.outputs.head }}
          filters: |
            src:
              - '!(test|Documentation)/**'

  # When the test-me-please trigger is used, this job is skipped if the only
  # modified files were under test/ or Documentation/.
  installation-and-connectivity:
    needs: check_changes
    if: |
      (github.event_name == 'issue_comment' && (
        startsWith(github.event.comment.body, 'ci-aks') ||
        (startsWith(github.event.comment.body, 'test-me-please') && (needs.check_changes.outputs.tested == 'true'))
      )) ||
      (github.event_name == 'schedule' && github.repository == 'cilium/cilium') ||
      github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    timeout-minutes: 35
    steps:
      - name: Set up job variables
        id: vars
        run: |
          if [ ${{ github.event.issue.pull_request || github.event.pull_request }} ]; then
            PR_API_JSON=$(curl \
              -H "Accept: application/vnd.github.v3+json" \
              -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
              ${{ github.event.issue.pull_request.url || github.event.pull_request.url }})
            SHA=$(echo "$PR_API_JSON" | jq -r ".head.sha")
            OWNER=$(echo "$PR_API_JSON" | jq -r ".number")
          else
            SHA=${{ github.sha }}
            OWNER=${{ github.sha }}
          fi

          CILIUM_INSTALL_DEFAULTS="--cluster-name=${{ env.name }} \
            --agent-image=quay.io/${{ github.repository_owner }}/cilium-ci \
            --operator-image=quay.io/${{ github.repository_owner }}/operator-azure-ci \
            --version=${SHA} \
            --azure-resource-group ${{ env.name }} \
            --wait=false \
            --config monitor-aggregation=none"
          HUBBLE_ENABLE_DEFAULTS="--relay-image=quay.io/${{ github.repository_owner }}/hubble-relay-ci \
            --relay-version=${SHA}"
          echo ::set-output name=cilium_install_defaults::${CILIUM_INSTALL_DEFAULTS}
          echo ::set-output name=hubble_enable_defaults::${HUBBLE_ENABLE_DEFAULTS}
          echo ::set-output name=sha::${SHA}
          echo ::set-output name=owner::${OWNER}

      - name: Set commit status to pending
        uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d
        with:
          authToken: ${{ secrets.GITHUB_TOKEN }}
          sha: ${{ steps.vars.outputs.sha }}
          context: ${{ github.workflow }}
          description: Connectivity test in progress...
          state: pending
          target_url: ${{ env.check_url }}

      - name: Install Cilium CLI
        run: |
          curl -sSL --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${{ env.cilium_cli_version }}/cilium-linux-amd64.tar.gz{,.sha256sum}
          sha256sum --check cilium-linux-amd64.tar.gz.sha256sum
          sudo tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin
          rm cilium-linux-amd64.tar.gz{,.sha256sum}
          cilium version

      - name: Login to Azure
        uses: azure/login@77f1b2e3fb80c0e8645114159d17008b8a2e475a
        with:
          creds: ${{ secrets.AZURE_PR_SP_CREDS }}

      - name: Display Azure CLI info
        uses: azure/CLI@4b58c946a0f48d82cc2b6e31c0d15a6604859554
        with:
          azcliversion: 2.0.72
          inlineScript: |
            az account show

      - name: Create AKS cluster
        id: cluster-creation
        run: |
          # Create group
          az group create \
            --name ${{ env.name }} \
            --location ${{ env.location }} \
            --tags usage=${{ github.repository_owner }}-${{ github.event.repository.name }} owner=${{ steps.vars.outputs.owner }}

          # Create cluster with a 1 node-count (we will remove this node pool
          # afterwards)
          # Details: Basic load balancers are not supported with multiple node
          # pools. Create a cluster with standard load balancer selected to use
          # multiple node pools, learn more at https://aka.ms/aks/nodepools.
          az aks create \
            --resource-group ${{ env.name }} \
            --name ${{ env.name }} \
            --location ${{ env.location }} \
            --network-plugin azure \
            --node-count 1 \
            --load-balancer-sku standard \
            --node-vm-size Standard_B2s \
            --node-osdisk-size 30 \
            --generate-ssh-keys

          # Get the name of the node pool that we will delete afterwards
          echo ::set-output name=nodepool_to_delete::$(az aks nodepool list --cluster-name ${{ env.name }} -g ${{ env.name }} -o json | jq -r '.[0].name')

          # Create a node pool with the taint 'node.cilium.io/agent-not-ready=true:NoSchedule'
          # and with 'mode=system' as it it the same mode used for the nodepool
          # created with the cluster.
          az aks nodepool add \
            --name nodepool2 \
            --cluster-name ${{ env.name }} \
            --resource-group ${{ env.name }} \
            --node-count 2 \
            --node-vm-size Standard_B2s \
            --node-osdisk-size 30 \
            --mode system \
            --node-taints node.cilium.io/agent-not-ready=true:NoSchedule

      - name: Get cluster credentials
        run: |
          az aks get-credentials \
            --resource-group ${{ env.name }} \
            --name ${{ env.name }}

      - name: Wait for images to be available
        timeout-minutes: 10
        shell: bash
        run: |
          for image in cilium-ci operator-azure-ci hubble-relay-ci ; do
            until curl --silent -f -lSL "https://quay.io/api/v1/repository/${{ github.repository_owner }}/$image/tag/${{ steps.vars.outputs.sha }}/images" &> /dev/null; do sleep 45s; done
          done

      - name: Install Cilium
        run: |
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }}

      - name: Delete the first node pool
        run: |
          # We can only delete the first node pool after Cilium is installed
          # because some pods have Pod Disruption Budgets set. If we try to
          # delete the first node pool without the second node pool being ready,
          # AKS will not succeed with the pool deletion because some Deployments
          # can't cease to exist in the cluster.
          az aks nodepool delete --name ${{ steps.cluster-creation.outputs.nodepool_to_delete }} \
            --cluster-name ${{ env.name }} \
            --resource-group ${{ env.name }}

      - name: Enable Relay
        run: |
          cilium hubble enable ${{ steps.vars.outputs.hubble_enable_defaults }}

      - name: Wait for Cilium status to be ready
        run: |
          cilium status --wait

      - name: Port forward Relay
        run: |
          cilium hubble port-forward&
          sleep 10s
          [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]

      - name: Run connectivity test
        run: |
          cilium connectivity test --flow-validation=disabled

      - name: Clean up Cilium
        run: |
          cilium uninstall --wait
          pkill -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay"

      - name: Create custom IPsec secret
        run: |
          kubectl create -n kube-system secret generic cilium-ipsec-keys --from-literal=keys="15 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"

      - name: Install Cilium with encryption
        run: |
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }} \
            --encryption=ipsec

      - name: Enable Relay
        run: |
          cilium hubble enable ${{ steps.vars.outputs.hubble_enable_defaults }}

      - name: Wait for Cilium status to be ready
        run: |
          cilium status --wait

      - name: Port forward Relay
        run: |
          cilium hubble port-forward&
          sleep 10s
          [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]

      - name: Run connectivity test
        run: |
          cilium connectivity test --force-deploy --flow-validation=disabled

      - name: Post-test information gathering
        if: ${{ !success() }}
        run: |
          cilium status
          kubectl get pods --all-namespaces -o wide
          curl -sLO https://github.com/cilium/cilium-sysdump/releases/latest/download/cilium-sysdump.zip
          python cilium-sysdump.zip --output cilium-sysdump-out
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently

      - name: Clean up AKS
        if: ${{ always() }}
        run: |
          az group delete --name ${{ env.name }} --yes --no-wait
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently

      - name: Upload artifacts
        if: ${{ !success() }}
        uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
        with:
          name: cilium-sysdump-out.zip
          path: cilium-sysdump-out.zip
          retention-days: 5

      - name: Set commit status to success
        if: ${{ success() }}
        uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d
        with:
          authToken: ${{ secrets.GITHUB_TOKEN }}
          sha: ${{ steps.vars.outputs.sha }}
          context: ${{ github.workflow }}
          description: Connectivity test successful
          state: success
          target_url: ${{ env.check_url }}

      - name: Set commit status to failure
        if: ${{ failure() }}
        uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d
        with:
          authToken: ${{ secrets.GITHUB_TOKEN }}
          sha: ${{ steps.vars.outputs.sha }}
          context: ${{ github.workflow }}
          description: Connectivity test failed
          state: failure
          target_url: ${{ env.check_url }}

      - name: Set commit status to cancelled
        if: ${{ cancelled() }}
        uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d
        with:
          authToken: ${{ secrets.GITHUB_TOKEN }}
          sha: ${{ steps.vars.outputs.sha }}
          context: ${{ github.workflow }}
          description: Connectivity test cancelled
          state: error
          target_url: ${{ env.check_url }}

      - name: Send slack notification
        if: ${{ !success() && (github.event_name == 'schedule' || github.event_name == 'push') }}
        uses: 8398a7/action-slack@e74cd4e48f4452e8158dc4f8bcfc780ae6203364
        with:
          status: ${{ job.status }}
          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took # selectable (default: repo,message)
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}