diff --git a/.circleci/config.yml b/.circleci/config.yml index 52ace09f9c59..6ab18b4568d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -345,6 +345,7 @@ jobs: sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz environment: <<: *ENVIRONMENT + - run: go env - run: *install-gotestsum - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - run: @@ -408,6 +409,7 @@ jobs: sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz environment: <<: *ENVIRONMENT + - run: go env - run: *install-gotestsum - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . - run: diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml new file mode 100644 index 000000000000..6ba474643382 --- /dev/null +++ b/.github/workflows/test-integrations.yml @@ -0,0 +1,501 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: test-integrations + +on: + pull_request: + branches-ignore: + - stable-website + - 'docs/**' + - 'ui/**' + - 'mktg-**' # Digital Team Terraform-generated branch prefix + - 'backport/docs/**' + - 'backport/ui/**' + - 'backport/mktg-**' + +env: + TEST_RESULTS_DIR: /tmp/test-results + TEST_RESULTS_ARTIFACT_NAME: test-results + CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} + GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} + GOTESTSUM_VERSION: "1.9.0" + CONSUL_BINARY_UPLOAD_NAME: consul-bin + # strip the hashicorp/ off the front of github.repository for consul + CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'consul' }} + +jobs: + setup: + runs-on: ubuntu-latest + name: Setup + outputs: + compute-small: ${{ steps.runners.outputs.compute-small }} + compute-medium: ${{ steps.runners.outputs.compute-medium }} + compute-large: ${{ steps.runners.outputs.compute-large }} + compute-xl: ${{ steps.runners.outputs.compute-xl }} + enterprise: ${{ steps.runners.outputs.enterprise }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - id: runners + run: .github/scripts/get_runner_classes.sh + + dev-build: + needs: [setup] + uses: ./.github/workflows/reusable-dev-build.yml + with: + runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + uploaded-binary-name: 'consul-bin' + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + nomad-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + strategy: + matrix: + nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16'] + steps: + - name: Checkout Nomad + uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + with: + repository: hashicorp/nomad + ref: ${{ matrix.nomad-version }} + + - name: Install Go + uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: Fetch Consul binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: ./bin + - name: Restore Consul permissions + run: | + chmod +x ./bin/consul + echo "$(pwd)/bin" >> $GITHUB_PATH + + - name: Make Nomad dev build + run: make pkg/linux_amd64/nomad + + - name: Run integration tests + run: | + go install gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} && \ + gotestsum \ + --format=short-verbose \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="./command/agent/consul" \ + --junitfile $TEST_RESULTS_DIR/results.xml -- \ + -run TestConsul + + vault-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + strategy: + matrix: + vault-version: ["1.13.1", "1.12.5", "1.11.9", "1.10.11"] + env: + VAULT_BINARY_VERSION: ${{ matrix.vault-version }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: Install Vault + run: | + wget -q -O /tmp/vault.zip "https://releases.hashicorp.com/vault/${{ env.VAULT_BINARY_VERSION }}/vault_${{ env.VAULT_BINARY_VERSION }}_linux_amd64.zip" + unzip -d /tmp /tmp/vault.zip + echo "/tmp" >> $GITHUB_PATH + + - name: Run Connect CA Provider Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" \ + -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage.txt ./agent/connect/ca + # Run leader tests that require Vault + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" \ + -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul + # Run agent tests that require Vault + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --format=short-verbose \ + --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \ + -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent + + generate-envoy-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Envoy Job Matrices + outputs: + envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - name: Generate Envoy Job Matrix + id: set-matrix + env: + # this is further going to multiplied in envoy-integration tests by the + # other dimensions in the matrix. Currently TOTAL_RUNNERS would be + # multiplied by 8 based on these values: + # envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"] + # xds-target: ["server", "client"] + TOTAL_RUNNERS: 3 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + { + echo -n "envoy-matrix=" + find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ + | xargs -0 -n 1 basename \ + | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join("|"))' + } >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + envoy-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + needs: + - setup + - generate-envoy-job-matrices + - dev-build + strategy: + fail-fast: false + matrix: + envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"] + xds-target: ["server", "client"] + test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} + env: + ENVOY_VERSION: ${{ matrix.envoy-version }} + XDS_TARGET: ${{ matrix.xds-target }} + AWS_LAMBDA_REGION: us-west-2 + steps: + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/aws arn | AWS_ROLE_ARN ; + + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: ./bin + - name: restore mode+x + run: chmod +x ./bin/consul + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2.4.1 + + - name: Docker build + run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin + + - name: Envoy Integration Tests + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --debug \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --jsonfile /tmp/jsonfile/go-test.log \ + --packages=./test/integration/connect/envoy \ + -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: container-logs + path: ./test/integration/connect/envoy/workdir/logs + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} + path: ${{ env.TEST_RESULTS_DIR }} + + generate-compatibility-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Compatibility Job Matrices + outputs: + compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - name: Generate Compatibility Job Matrix + id: set-matrix + env: + TOTAL_RUNNERS: 5 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + cd ./test/integration/consul-container + { + echo -n "compatibility-matrix=" + find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \ + | grep -v util | grep -v upgrade \ + | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join(" "))' + } >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + compatibility-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + needs: + - setup + - dev-build + - generate-compatibility-job-matrices + strategy: + fail-fast: false + matrix: + test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - run: go env + + # Build the consul:local image from the already built binary + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: . + - name: restore mode+x + run: chmod +x consul + + - name: Build consul:local image + run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . + - name: Configure GH workaround for ipv6 loopback + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + cat /etc/hosts && echo "-----------" + sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts + cat /etc/hosts + - name: Compatibility Integration Tests + run: | + mkdir -p "/tmp/test-results" + cd ./test/integration/consul-container + docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --raw-command \ + --format=short-verbose \ + --debug \ + --rerun-fails=3 \ + -- \ + go test \ + -p=4 \ + -tags "${{ env.GOTAGS }}" \ + -timeout=30m \ + -json \ + "${{ matrix.test-cases }}" \ + --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --target-version local \ + --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --latest-version latest + ls -lrt + env: + # this is needed because of incompatibility between RYUK container and circleci + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} + path: ${{ env.TEST_RESULTS_DIR }} + + generate-upgrade-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Upgrade Job Matrices + outputs: + upgrade-matrix: ${{ steps.set-matrix.outputs.upgrade-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - name: Generate Updgrade Job Matrix + id: set-matrix + env: + TOTAL_RUNNERS: 4 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + cd ./test/integration/consul-container/test/upgrade + { + echo -n "upgrade-matrix=" + go test ./... -list=. -json | jq -r '.Output | select (. !=null) | select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' \ + | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join("|"))' + } >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + upgrade-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + needs: + - setup + - dev-build + - generate-upgrade-job-matrices + strategy: + fail-fast: false + matrix: + consul-version: [ "1.14", "1.15"] + test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }} + env: + CONSUL_VERSION: ${{ matrix.consul-version }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - run: go env + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/dockerhub username | DOCKERHUB_USERNAME; + kv/data/github/${{ github.repository }}/dockerhub token | DOCKERHUB_TOKEN; + + # NOTE: conditional specific logic as we store secrets in Vault in ENT and use GHA secrets in OSS. + - name: Login to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2.1.0 + with: + username: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }} + password: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_TOKEN || secrets.DOCKERHUB_TOKEN }} + + + # Get go binary from workspace + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: '${{ env.CONSUL_BINARY_UPLOAD_NAME }}' + path: . + - name: restore mode+x + run: chmod +x consul + - name: Build consul:local image + run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile . + - name: Configure GH workaround for ipv6 loopback + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + cat /etc/hosts && echo "-----------" + sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts + cat /etc/hosts + - name: Upgrade Integration Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + cd ./test/integration/consul-container/test/upgrade + docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ + --raw-command \ + --format=short-verbose \ + --debug \ + --rerun-fails=3 \ + --packages="./..." \ + -- \ + go test \ + -p=4 \ + -tags "${{ env.GOTAGS }}" \ + -timeout=30m \ + -json ./... \ + -run "${{ matrix.test-cases }}" \ + --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --target-version local \ + --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ + --latest-version "${{ env.CONSUL_VERSION }}" + ls -lrt + env: + # this is needed because of incompatibility between RYUK container and circleci + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} + path: ${{ env.TEST_RESULTS_DIR }} + + test-integrations-success: + needs: + - setup + - dev-build + - nomad-integration-test + - vault-integration-test + - generate-envoy-job-matrices + - envoy-integration-test + - generate-compatibility-job-matrices + - compatibility-integration-test + - generate-upgrade-job-matrices + - upgrade-integration-test + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: | + (always() && ! cancelled()) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + steps: + - run: echo "test-integrations succeeded"