Backport of ci: add test-integrations into release/1.13.x (#17039)

* backport of commit bc630a7 * backport of commit 1c8731c * backport of commit 76b7e2d * backport of commit 21f3d86 * backport of commit 12e053f * backport of commit 0fc2445 * backport of commit 025240c * backport of commit cbece51 * backport of commit 3d42297 * backport of commit 2f1af3b * backport of commit a5c24fb * backport of commit eb3e524 * ci: add test-integrations (#16915) * add test-integrations workflow * add test-integrations success job * update vault integration testing versions (#16949) * change parallelism to 4 forgotestsum. use env.CONSUL_VERSION so we can see the version. * use env for repeated values * match test to circleci * fix envvar * fix envvar 2 * fix envvar 3 * fix envvar 4 * fix envvar 5 * make upgrade and compatibility tests match circleci * run go env to check environment * debug docker Signed-off-by: Dan Bond <danbond@protonmail.com> * debug docker Signed-off-by: Dan Bond <danbond@protonmail.com> * revert debug docker Signed-off-by: Dan Bond <danbond@protonmail.com> * going back to command that worked 5 days ago for compatibility tests * Update Envoy versions to reflect changes in #16889 * cd to test dir * try running ubuntu latest * update PR with latest changes that work in enterprise * yaml still sucks * test GH fix (localhost resolution) * change for testing * test splitting and ipv6 lookup for compatibility and upgrade tests * fix indention * consul as image name * remove the on push * add gotestsum back in * removing the use of the gotestsum download action * yaml sucks today just like yesterday * fixing nomad tests * worked out the kinks on enterprise --------- Signed-off-by: Dan Bond <danbond@protonmail.com> Co-authored-by: John Eikenberry <jae@zhar.net> Co-authored-by: Dan Bond <danbond@protonmail.com> Co-authored-by: Nathan Coleman <nathan.coleman@hashicorp.com> Co-authored-by: Sarah <sthompson@hashicorp.com> --------- Signed-off-by: Dan Bond <danbond@protonmail.com> Co-authored-by: John Murret <john.murret@hashicorp.com> Co-authored-by: John Eikenberry <jae@zhar.net> Co-authored-by: Dan Bond <danbond@protonmail.com> Co-authored-by: Nathan Coleman <nathan.coleman@hashicorp.com> Co-authored-by: Sarah <sthompson@hashicorp.com>
hashicorp · Apr 19, 2023 · 5f996fa · 5f996fa
1 parent 177839e
commit 5f996fa
Show file tree

Hide file tree

Showing 2 changed files with 371 additions and 0 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -321,6 +321,7 @@ jobs:
             sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz
           environment:
             <<: *ENVIRONMENT
+      - run: go env
       - run: *install-gotestsum
       - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
       - run:

diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml
@@ -0,0 +1,370 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+name: test-integrations
+
+on:
+  pull_request:
+    branches-ignore:
+      - stable-website
+      - 'docs/**'
+      - 'ui/**'
+      - 'mktg-**' # Digital Team Terraform-generated branch prefix
+      - 'backport/docs/**'
+      - 'backport/ui/**'
+      - 'backport/mktg-**'
+
+env:
+  TEST_RESULTS_DIR: /tmp/test-results
+  TEST_RESULTS_ARTIFACT_NAME: test-results
+  CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }}
+  GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }}
+  GOTESTSUM_VERSION: "1.9.0"
+  CONSUL_BINARY_UPLOAD_NAME: consul-bin
+  # strip the hashicorp/ off the front of github.repository for consul
+  CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'consul' }}
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    name: Setup
+    outputs:
+      compute-small: ${{ steps.runners.outputs.compute-small }}
+      compute-medium: ${{ steps.runners.outputs.compute-medium }}
+      compute-large: ${{ steps.runners.outputs.compute-large }}
+      compute-xl: ${{ steps.runners.outputs.compute-xl }}
+      enterprise: ${{ steps.runners.outputs.enterprise }}
+    steps:
+      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - id: runners
+        run: .github/scripts/get_runner_classes.sh
+
+  dev-build:
+    needs: [setup]
+    uses: ./.github/workflows/reusable-dev-build.yml
+    with:
+      runs-on: ${{ needs.setup.outputs.compute-xl }}
+      repository-name: ${{ github.repository }}
+      uploaded-binary-name: 'consul-bin'
+    secrets:
+      elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+
+  nomad-integration-test:
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
+    needs:
+      - setup
+      - dev-build
+    strategy:
+      matrix:
+        nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16']
+    steps:
+      - name: Checkout Nomad
+        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+        with:
+          repository: hashicorp/nomad
+          ref: ${{ matrix.nomad-version }}
+
+      - name: Install Go
+        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Fetch Consul binary
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
+          path: ./bin
+      - name: Restore Consul permissions
+        run: |
+          chmod +x ./bin/consul
+          echo "$(pwd)/bin" >> $GITHUB_PATH
+
+      - name: Make Nomad dev build
+        run: make pkg/linux_amd64/nomad
+
+      - name: Run integration tests
+        run: |
+          go install gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} && \
+          gotestsum \
+            --format=short-verbose \
+            --rerun-fails \
+            --rerun-fails-report=/tmp/gotestsum-rerun-fails \
+            --packages="./command/agent/consul" \
+            --junitfile $TEST_RESULTS_DIR/results.xml -- \
+            -run TestConsul
+
+  vault-integration-test:
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
+    needs:
+      - setup
+      - dev-build
+    strategy:
+      matrix:
+        vault-version: ["1.12.2","1.11.6","1.10.9","1.9.10"]
+    env:
+      VAULT_BINARY_VERSION: ${{ matrix.vault-version }}
+    steps:
+      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+
+      # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.
+      - name: Setup Git
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com"
+
+      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Install Vault
+        run: |
+          wget -q -O /tmp/vault.zip "https://releases.hashicorp.com/vault/${{ env.VAULT_BINARY_VERSION }}/vault_${{ env.VAULT_BINARY_VERSION }}_linux_amd64.zip"
+          unzip -d /tmp /tmp/vault.zip
+          echo "/tmp" >> $GITHUB_PATH
+
+      - name: Run Connect CA Provider Tests
+        run: |
+          mkdir -p "${{ env.TEST_RESULTS_DIR }}"
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+            --format=short-verbose \
+            --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" \
+            -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage.txt ./agent/connect/ca
+          # Run leader tests that require Vault
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+            --format=short-verbose \
+            --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" \
+            -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul
+          # Run agent tests that require Vault
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+            --format=short-verbose \
+            --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \
+            -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent
+
+  generate-envoy-job-matrices:
+    needs: [setup]
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
+    name: Generate Envoy Job Matrices
+    outputs:
+      envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }}
+    steps:
+      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - name: Generate Envoy Job Matrix
+        id: set-matrix
+        env:
+          # this is further going to multiplied in envoy-integration tests by the 
+          # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
+          # multiplied by 8 based on these values:
+          # envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"]
+          # xds-target: ["server", "client"]
+          TOTAL_RUNNERS: 3 
+          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
+        run: |
+          {
+            echo -n "envoy-matrix="
+            find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \
+              | xargs -0 -n 1 basename \
+              | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \
+              | jq --compact-output 'map(join("|"))'
+          } >> "$GITHUB_OUTPUT"
+          cat "$GITHUB_OUTPUT"
+  
+  envoy-integration-test:
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read
+    needs:
+      - setup
+      - generate-envoy-job-matrices
+      - dev-build
+    strategy:
+      fail-fast: false
+      matrix:
+        envoy-version: ["1.20.7", "1.21.6","1.22.11","1.23.8"]
+        xds-target: ["server", "client"]
+        test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
+    env:
+      ENVOY_VERSION: ${{ matrix.envoy-version }}
+      XDS_TARGET: ${{ matrix.xds-target }}
+      AWS_LAMBDA_REGION: us-west-2
+    steps:
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Authenticate to Vault
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: vault-auth
+        run: vault-auth
+
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Fetch Secrets
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: secrets
+        uses: hashicorp/vault-action@v2.5.0
+        with:
+          url: ${{ steps.vault-auth.outputs.addr }}
+          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
+          token: ${{ steps.vault-auth.outputs.token }}
+          secrets: |
+            kv/data/github/${{ github.repository }}/aws arn | AWS_ROLE_ARN ;
+
+      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        with:
+          go-version-file: 'go.mod'
+
+      - name: fetch binary
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
+          path: ./bin
+      - name: restore mode+x
+        run: chmod +x ./bin/consul
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2.4.1
+
+      - name: Docker build
+        run: docker build -t consul-dev -f ./build-support/docker/Consul-Dev.dockerfile ./bin
+
+      - name: Envoy Integration Tests
+        env:
+          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
+          GOTESTSUM_FORMAT: standard-verbose
+          COMPOSE_INTERACTIVE_NO_CLI: 1
+          LAMBDA_TESTS_ENABLED: "true"
+          # tput complains if this isn't set to something.
+          TERM: ansi
+        run: |
+          # shellcheck disable=SC2001
+          echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
+          # shellcheck disable=SC2001
+          sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+              --debug \
+              --rerun-fails \
+              --rerun-fails-report=/tmp/gotestsum-rerun-fails \
+              --jsonfile /tmp/jsonfile/go-test.log \
+              --packages=./test/integration/connect/envoy \
+              -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})"
+          
+      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: container-logs
+          path: ./test/integration/connect/envoy/workdir/logs
+
+      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
+          path: ${{ env.TEST_RESULTS_DIR }}
+
+  generate-compatibility-job-matrices:
+    needs: [setup]
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
+    name: Generate Compatibility Job Matrices
+    outputs:
+      compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }}
+    steps:
+      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - name: Generate Compatibility Job Matrix
+        id: set-matrix
+        env:
+          TOTAL_RUNNERS: 5
+          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
+        run: |
+          cd ./test/integration/consul-container
+          {
+            echo -n "compatibility-matrix="
+            find . -maxdepth 2 -type d -print0 | xargs -0 -n 1 \
+              | grep -v lib  \
+              | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \
+              | jq --compact-output 'map(join(" "))'
+          } >> "$GITHUB_OUTPUT"
+          cat "$GITHUB_OUTPUT"
+          
+  compatibility-integration-test:
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    needs:
+      - setup
+      - dev-build
+      - generate-compatibility-job-matrices
+    strategy:
+      fail-fast: false
+      matrix:
+        test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }}
+    steps:
+      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        with:
+          go-version-file: 'go.mod'
+      - run: go env
+
+      # Build the consul:local image from the already built binary
+      - name: fetch binary
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}'
+          path: .
+      - name: restore mode+x
+        run: chmod +x consul
+
+      - name: Build consul:local image
+        run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
+      - name: Configure GH workaround for ipv6 loopback
+        if: ${{ !endsWith(github.repository, '-enterprise') }}
+        run: |
+          cat /etc/hosts && echo "-----------"
+          sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts
+          cat /etc/hosts
+      - name: Compatibility Integration Tests
+        run: |
+            mkdir -p "/tmp/test-results"
+            cd ./test/integration/consul-container
+            docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
+            echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
+            # shellcheck disable=SC2001
+            sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
+            go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+              --raw-command \
+              --format=short-verbose \
+              --debug \
+              --rerun-fails=3 \
+              -- \
+              go test \
+              -p=4 \
+              -tags "${{ env.GOTAGS }}" \
+              -timeout=30m \
+              -json \
+              "${{ matrix.test-cases }}" \
+              --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+              --target-version local \
+              --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+              --latest-version latest
+            ls -lrt
+        env:
+          # this is needed because of incompatibility between RYUK container and circleci
+          GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
+          GOTESTSUM_FORMAT: standard-verbose
+          COMPOSE_INTERACTIVE_NO_CLI: 1
+          # tput complains if this isn't set to something.
+          TERM: ansi
+
+      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
+          path: ${{ env.TEST_RESULTS_DIR }}
+
+  test-integrations-success:
+    needs: 
+    - setup
+    - dev-build
+    - nomad-integration-test
+    - vault-integration-test
+    - generate-envoy-job-matrices
+    - envoy-integration-test
+    - generate-compatibility-job-matrices
+    - compatibility-integration-test
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
+    if: |
+      (always() && ! cancelled()) &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled')
+    steps:
+      - run: echo "test-integrations succeeded"