conformance-multicluster.yaml

name: Multicluster / Cluster mesh (ci-multicluster)

# Any change in triggers needs to be reflected in the concurrency group.
on:
  issue_comment:
    types:
      - created
  # Run every 6 hours
  schedule:
    - cron:  '0 3/6 * * *'
  ### FOR TESTING PURPOSES
  # pull_request:
  #  types:
  #    - "labeled"
  ###

concurrency:
  # In case of PR comment, we can't simply append the comment to the group name
  # as that is too long and results in an error. Instead we check if it's a
  # trigger phrase and append a simple 'trigger-phrase'.
  group: |
    "${{ github.workflow }} ${{ github.event.issue.pull_request.url || 'scheduled' }}
     ${{ (startsWith(github.event.comment.body, 'ci-multicluster') ||
          startsWith(github.event.comment.body, 'test-me-please')) && 'trigger-phrase' }}"
  cancel-in-progress: true

env:
  clusterName1: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-mesh-1
  clusterName2: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-mesh-2
  zone: us-west2-a
  check_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

jobs:
  check_changes:
    name: Deduce required tests from code changes
    if: |
      (github.event.issue.pull_request && (
        startsWith(github.event.comment.body, 'ci-multicluster') ||
        (startsWith(github.event.comment.body, 'test-me-please'))
      )) ||
      (github.event_name == 'schedule' && github.repository == 'cilium/cilium') ||
      github.event.label.name == 'ci-run/multicluster'
    runs-on: ubuntu-latest
    outputs:
      tested: ${{ steps.tested-tree.outputs.src }}
    steps:
      - name: Retrieve pull request's base and head
        if: ${{ github.event.issue.pull_request }}
        id: pr
        run: |
          curl ${{ github.event.issue.pull_request.url }} > pr.json
          echo "::set-output name=base::$(jq -r '.base.sha' pr.json)"
          echo "::set-output name=head::$(jq -r '.head.sha' pr.json)"
      # Because we run on issue comments, we need to checkout the code for
      # paths-filter to work.
      - name: Checkout code
        if: ${{ github.event.issue.pull_request }}
        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
        with:
          persist-credentials: false
      - name: Check code changes
        if: ${{ github.event.issue.pull_request }}
        uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
        id: tested-tree
        with:
          base: ${{ steps.pr.outputs.base }}
          ref: ${{ steps.pr.outputs.head }}
          filters: |
            src:
              - '!(test|Documentation)/**'

  # When the test-me-please trigger is used, this job is skipped if the only
  # modified files were under test/ or Documentation/.
  installation-and-connectivity:
    needs: check_changes
    if: |
      (github.event.issue.pull_request && (
        startsWith(github.event.comment.body, 'ci-multicluster') ||
        (startsWith(github.event.comment.body, 'test-me-please') && (needs.check_changes.outputs.tested == 'true'))
      )) ||
      (github.event_name == 'schedule' && github.repository == 'cilium/cilium') ||
      github.event.label.name == 'ci-run/multicluster'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Set up job variables
        id: vars
        run: |
          if [ ${{ github.event.issue.pull_request || github.event.pull_request }} ]; then
            PR_API_JSON=$(curl \
              -H "Accept: application/vnd.github.v3+json" \
              -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
              ${{ github.event.issue.pull_request.url || github.event.pull_request.url }})
            SHA=$(echo "$PR_API_JSON" | jq -r ".head.sha")
            OWNER=$(echo "$PR_API_JSON" | jq -r ".number")
          else
            SHA=${{ github.sha }}
            OWNER=${{ github.sha }}
          fi

          CILIUM_INSTALL_DEFAULTS="--agent-image=quay.io/${{ github.repository_owner }}/cilium-ci \
            --operator-image=quay.io/${{ github.repository_owner }}/operator-generic-ci \
            --version=${SHA} \
            --wait=false \
            --config monitor-aggregation=none"
          HUBBLE_ENABLE_DEFAULTS="--relay-image=quay.io/${{ github.repository_owner }}/hubble-relay-ci \
            --relay-version=${SHA}"
          echo ::set-output name=cilium_install_defaults::${CILIUM_INSTALL_DEFAULTS}
          echo ::set-output name=hubble_enable_defaults::${HUBBLE_ENABLE_DEFAULTS}
          echo ::set-output name=sha::${SHA}
          echo ::set-output name=owner::${OWNER}

      - name: Set commit status to pending
        uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d
        with:
          authToken: ${{ secrets.GITHUB_TOKEN }}
          sha: ${{ steps.vars.outputs.sha }}
          context: ${{ github.workflow }}
          description: Connectivity test in progress...
          state: pending
          target_url: ${{ env.check_url }}

      - name: Install Cilium CLI
        run: |
          curl -LO https://github.com/cilium/cilium-cli/releases/download/v0.6/cilium-linux-amd64.tar.gz
          sudo tar xzvfC cilium-linux-amd64.tar.gz /usr/bin
          rm cilium-linux-amd64.tar.gz

      - name: Set up gcloud CLI
        uses: google-github-actions/setup-gcloud@daadedc81d5f9d3c06d2c92f49202a3cc2b919ba
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
          service_account_key: ${{ secrets.GCP_PR_SA_KEY }}
          export_default_credentials: true

      - name: Display gcloud CLI info
        run: |
          gcloud info

      - name: Create GKE cluster 1
        run: |
          gcloud container clusters create ${{ env.clusterName1 }} \
            --labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \
            --zone ${{ env.zone }} \
            --image-type COS_CONTAINERD \
            --num-nodes 2 \
            --machine-type e2-custom-2-4096 \
            --disk-type pd-standard \
            --disk-size 10GB \
            --preemptible

      - name: Create GKE cluster 2
        run: |
          gcloud container clusters create ${{ env.clusterName2 }} \
            --labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \
            --zone ${{ env.zone }} \
            --image-type COS_CONTAINERD \
            --num-nodes 2 \
            --machine-type e2-custom-2-4096 \
            --disk-type pd-standard \
            --disk-size 10GB \
            --preemptible

      - name: Get cluster credentials and setup contexts
        id: contexts
        run: |
          gcloud container clusters get-credentials ${{ env.clusterName1 }} --zone ${{ env.zone }}
          CONTEXT_1="$(kubectl config view | grep ${{ env.clusterName1 }} | head -1 | awk '{print $2}')"
          echo ::set-output name=context1::${CONTEXT_1}
          gcloud container clusters get-credentials ${{ env.clusterName2 }} --zone ${{ env.zone }}
          CONTEXT_2="$(kubectl config view | grep ${{ env.clusterName2 }} | head -1 | awk '{print $2}')"
          echo ::set-output name=context2::${CONTEXT_2}

      - name: Wait for images to be available
        timeout-minutes: 10
        shell: bash
        run: |
          until curl --silent -f -lSL "https://quay.io/api/v1/repository/${{ github.repository_owner }}/cilium-ci/tag/${{ steps.vars.outputs.sha }}/images" &> /dev/null; do sleep 45s; done
          until curl --silent -f -lSL "https://quay.io/api/v1/repository/${{ github.repository_owner }}/operator-generic-ci/tag/${{ steps.vars.outputs.sha }}/images" &> /dev/null; do sleep 45s; done

      - name: Install Cilium in cluster1
        run: |
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }} \
            --context ${{ steps.contexts.outputs.context1 }} \
            --cluster-name=${{ env.clusterName1 }} \
            --cluster-id 1

      - name: Install Cilium in cluster2
        run: |
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }} \
            --context ${{ steps.contexts.outputs.context2 }} \
            --cluster-name=${{ env.clusterName2 }} \
            --cluster-id 2

      - name: Enable Relay
        run: |
          cilium hubble enable --context ${{ steps.contexts.outputs.context1 }} ${{ steps.vars.outputs.hubble_enable_defaults }}
          cilium hubble enable --context ${{ steps.contexts.outputs.context2 }} ${{ steps.vars.outputs.hubble_enable_defaults }}

      - name: Wait for Cilium status to be ready
        run: |
          cilium status --wait --context ${{ steps.contexts.outputs.context1 }}
          cilium status --wait --context ${{ steps.contexts.outputs.context2 }}

      - name: Enable cluster mesh
        run: |
          cilium clustermesh enable --context ${{ steps.contexts.outputs.context1 }}
          cilium clustermesh enable --context ${{ steps.contexts.outputs.context2 }}

      - name: Wait for cluster mesh status to be ready
        run: |
          cilium clustermesh status --wait --context ${{ steps.contexts.outputs.context1 }}
          cilium clustermesh status --wait --context ${{ steps.contexts.outputs.context2 }}

      - name: Connect clusters
        run: |
          cilium clustermesh connect \
            --context ${{ steps.contexts.outputs.context1 }} \
            --destination-context ${{ steps.contexts.outputs.context2 }}

      - name: Wait for cluster mesh status to be ready
        run: |
          cilium clustermesh status --wait --context ${{ steps.contexts.outputs.context1 }}
          cilium clustermesh status --wait --context ${{ steps.contexts.outputs.context2 }}

      - name: Port forward Relay
        run: |
          kubectl port-forward \
            --context ${{ steps.contexts.outputs.context1 }} \
            -n kube-system deployment/hubble-relay 4245:4245&
          sleep 10s

      - name: Run connectivity test
        run: |
          cilium connectivity test \
            --context ${{ steps.contexts.outputs.context1 }} \
            --multi-cluster ${{ steps.contexts.outputs.context2 }} \
            --test '!pod-to-nodeport' \
            --test '!pod-to-local-nodeport' \
             --flow-validation=disabled

      - name: Post-test information gathering
        if: ${{ failure() }}
        run: |
          cilium status --context ${{ steps.contexts.outputs.context1 }}
          cilium clustermesh status --context ${{ steps.contexts.outputs.context1 }}
          cilium status --context ${{ steps.contexts.outputs.context2 }}
          cilium clustermesh status --context ${{ steps.contexts.outputs.context2 }}
          kubectl get pods --all-namespaces -o wide
          curl -sLO https://github.com/cilium/cilium-sysdump/releases/latest/download/cilium-sysdump.zip
          python cilium-sysdump.zip --output cilium-sysdump-out
        shell: bash {0}

      - name: Clean up GKE
        if: ${{ always() }}
        run: |
          gcloud container clusters delete ${{ env.clusterName1 }} --zone ${{ env.zone }} --quiet --async
          gcloud container clusters delete ${{ env.clusterName2 }} --zone ${{ env.zone }} --quiet --async
        shell: bash {0}

      - name: Upload artifacts
        if: ${{ failure() }}
        uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
        with:
          name: cilium-sysdump-out.zip
          path: cilium-sysdump-out.zip
          retention-days: 5

      - name: Set commit status to success
        if: ${{ success() }}
        uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d
        with:
          authToken: ${{ secrets.GITHUB_TOKEN }}
          sha: ${{ steps.vars.outputs.sha }}
          context: ${{ github.workflow }}
          description: Connectivity test successful
          state: success
          target_url: ${{ env.check_url }}

      - name: Set commit status to failure
        if: ${{ failure() }}
        uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d
        with:
          authToken: ${{ secrets.GITHUB_TOKEN }}
          sha: ${{ steps.vars.outputs.sha }}
          context: ${{ github.workflow }}
          description: Connectivity test failed
          state: failure
          target_url: ${{ env.check_url }}

      - name: Set commit status to cancelled
        if: ${{ cancelled() }}
        uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d
        with:
          authToken: ${{ secrets.GITHUB_TOKEN }}
          sha: ${{ steps.vars.outputs.sha }}
          context: ${{ github.workflow }}
          description: Connectivity test cancelled
          state: pending
          target_url: ${{ env.check_url }}

      - name: Send slack notification
        if: ${{ (cancelled() || failure()) && (github.event_name == 'schedule' || github.event_name == 'push') }}
        uses: 8398a7/action-slack@dcc8c8e9dd8802e21a712dc0c003db97b42efe43
        with:
          status: ${{ job.status }}
          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took # selectable (default: repo,message)
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}