diff --git a/.github/workflows/regression.yaml b/.github/workflows/regression.yaml new file mode 100644 index 0000000000..68533cf38b --- /dev/null +++ b/.github/workflows/regression.yaml @@ -0,0 +1,203 @@ +name: regression + +on: + workflow_call: + inputs: + version_tag_old: + description: 'The initial install version to use for upgrade scenarios' + required: true + type: string + version_tag_new: + description: 'The new version to use as the upgrade in upgrade scenarios and the initial version for fresh installs.' + required: true + type: string + secrets: + TESTIM_ACCESS_TOKEN: + required: true + E2E_TESTIM_AWS_ACCESS_KEY_ID: + required: true + E2E_TESTIM_AWS_SECRET_ACCESS_KEY: + required: true + E2E_GH_PAT: + required: true + KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL: + required: true + + workflow_dispatch: + inputs: + version_tag_old: + description: 'The initial install version to use for upgrade scenarios' + required: true + version_tag_new: + description: 'The new version to use as the upgrade in upgrade scenarios and the initial version for fresh installs.' + required: true + +env: + AWS_DEFAULT_REGION: us-east-1 + AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }} + TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }} + TF_WORKSPACE: automation + +jobs: + jumpbox: + runs-on: ubuntu-20.04 + container: + image: hashicorp/terraform:1.0.7 + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + repository: replicatedhq/kots-regression-automation + token: ${{ secrets.E2E_GH_PAT }} + path: automation + ref: main + - name: Create Jumpbox Environment + if: always() + working-directory: automation/jumpbox + run: | + terraform init + terraform apply --auto-approve + + - name: Notify Slack + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: custom + fields: repo,ref,workflow + custom_payload: | + { + attachments: [{ + color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning', + text: `Create jumpbox: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`, + }] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }} + + tests: + needs: jumpbox + runs-on: ubuntu-20.04 + container: + image: hashicorp/terraform:1.0.7 + strategy: + fail-fast: false + matrix: + test: [ + { + name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin", + backend_config: "embedded-airgapped-upgrade-backend-config.tfvars", + terraform_script: "embedded-airgapped-upgrade.sh", + is_upgrade: "1" + }, + { + name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin", + backend_config: "embedded-airgapped-install-backend-config.tfvars", + terraform_script: "embedded-airgapped-install.sh" + }, + { + name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin", + backend_config: "embedded-online-install-backend-config.tfvars", + terraform_script: "embedded-online-install.sh" + }, + { + name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin", + backend_config: "embedded-online-upgrade-backend-config.tfvars", + terraform_script: "embedded-online-upgrade.sh", + is_upgrade: "1" + }, + { + name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin", + backend_config: "existing-airgapped-install-admin-backend-config.tfvars", + terraform_script: "existing-airgapped-install-admin.sh" + }, + { + name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac", + backend_config: "existing-airgapped-install-minimum-backend-config.tfvars", + terraform_script: "existing-airgapped-install-minimum.sh" + }, + { + name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin", + backend_config: "existing-online-upgrade-admin-backend-config.tfvars", + terraform_script: "existing-online-upgrade-admin.sh", + is_upgrade: "1" + }, + { + name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac", + backend_config: "existing-online-upgrade-minimum-backend-config.tfvars", + terraform_script: "existing-online-upgrade-minimum.sh", + is_upgrade: "1" + }, + { + name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin", + backend_config: "existing-online-install-admin-backend-config.tfvars", + terraform_script: "existing-online-install-admin.sh" + }, + { + name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac", + backend_config: "existing-online-install-minimum-backend-config.tfvars", + terraform_script: "existing-online-install-minimum.sh" + }, + { + name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin", + backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars", + terraform_script: "existing-airgapped-upgrade-admin.sh", + is_upgrade: "1" + }, + { + name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac", + backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars", + terraform_script: "existing-airgapped-upgrade-minimum.sh", + is_upgrade: "1" + } + ] + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + repository: replicatedhq/kots-regression-automation + token: ${{ secrets.E2E_GH_PAT }} + path: automation + ref: adding-outputs + - name: Create test environment + if: always() + working-directory: automation/cluster + run: | + export CURRENT_TAG=${{ github.event.inputs.version_tag_old || inputs.version_tag_old }} + export NEXT_TAG=${{ github.event.inputs.version_tag_new || inputs.version_tag_new }} + + if [ "${{ matrix.test.is_upgrade }}" ]; then + export TF_VAR_kots_version_initial="${CURRENT_TAG:1}" + export TF_VAR_kots_version_upgrade="${NEXT_TAG:1}" + else + export TF_VAR_kots_version_initial="${NEXT_TAG:1}" + fi + terraform init -backend-config ${{ matrix.test.backend_config }} + ./${{ matrix.test.terraform_script }} apply + terraform taint -allow-missing aws_instance.control_plane + terraform taint -allow-missing 'aws_instance.worker[0]' + + - name: Run the test + working-directory: automation/cluster + run: | + terraform output -raw jumpbox_private_key > ssh.pem + chmod 600 ssh.pem + ssh -i ssh.pem ubuntu@$(terraform output -raw jumpbox_public_ip) -oStrictHostKeyChecking=no -oServerAliveInterval=60 -oServerAliveCountMax=10 "ssh -tt ubuntu@$(terraform output -raw control_plane_private_ip) -oServerAliveInterval=60 -oServerAliveCountMax=10 \"sudo /tmp/start.sh\"" + + - name: Notify Slack + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: custom + fields: repo,ref,workflow + custom_payload: | + { + attachments: [{ + color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning', + text: `Setting up: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`, + }] + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }} + MATRIX_CONTEXT: ${{ toJson(matrix) }} \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1392a09aff..5f03cb0b0d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,7 +6,6 @@ on: - "v*.*.*" jobs: - generate-schema: runs-on: ubuntu-18.04 container: @@ -28,7 +27,6 @@ jobs: name: schema path: ./migrations/fixtures/schema/fixtures.sql - generate-fixtures: runs-on: ubuntu-18.04 needs: [generate-schema] @@ -55,7 +53,6 @@ jobs: name: fixtures path: ./migrations/fixtures/fixtures.sql - publish-fixtures: runs-on: ubuntu-18.04 needs: [generate-fixtures] @@ -74,7 +71,6 @@ jobs: name: fixtures path: ./migrations/fixtures - build-schema-migrations: runs-on: ubuntu-20.04 needs: [publish-fixtures] @@ -106,7 +102,6 @@ jobs: name: migrations-image path: ./migrations/bin/docker-archive - test_web: runs-on: ubuntu-18.04 needs: [build-schema-migrations] @@ -126,21 +121,6 @@ jobs: run: npm i -g cross-env shell: bash - # - name: Test web - # env: - # PACT_BROKER_USERNAME: ${{ secrets.PACT_BROKER_USERNAME }} - # PACT_BROKER_PASSWORD: ${{ secrets.PACT_BROKER_PASSWORD }} - # run: | - # make -C kotsadm/web deps test publish-pact - - # TODO: migrate retry logic for pacts - # retry: - # automatic: - # # this command exiting with status 2 typically means that the yarn package failed to download - # - exit_status: 2 - # limit: 5 - - build_web: runs-on: ubuntu-18.04 needs: [build-schema-migrations] @@ -166,12 +146,6 @@ jobs: GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }} run: export $(cat .image.env | sed 's/#.*//g' | xargs) && make -C web deps build-kotsadm shell: bash - ## TODO: retry logic - # retry: - # automatic: - # # this command exiting with status 2 typically means that the yarn package failed to download - # - exit_status: 2 - # limit: 5 - name: Upload web artifact uses: actions/upload-artifact@v2 @@ -179,7 +153,6 @@ jobs: name: web path: ./web/dist - build_kurl_proxy: runs-on: ubuntu-18.04 needs: [build-schema-migrations] @@ -213,7 +186,6 @@ jobs: name: kurl_proxy path: ./kurl_proxy/bin - build_go_api: runs-on: ubuntu-18.04 needs: [test_web, build_web, build_kurl_proxy] @@ -268,7 +240,6 @@ jobs: name: go_api path: ./bin - release_go_api_tagged: runs-on: ubuntu-20.04 needs: [build_web, build_go_api] @@ -327,7 +298,6 @@ jobs: name: kotsadm-image path: ./bin/docker-archive - build_kurl_proxy_tagged: runs-on: ubuntu-18.04 needs: [build_kurl_proxy] @@ -377,7 +347,6 @@ jobs: run: | export $(cat .image.env | sed 's/#.*//g' | xargs) && make -C kurl_proxy build-release - goreleaser: runs-on: ubuntu-18.04 steps: @@ -437,7 +406,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - build_airgap: runs-on: ubuntu-18.04 needs: [release_go_api_tagged, goreleaser, build-schema-migrations] @@ -480,516 +448,45 @@ jobs: asset_name: kotsadm.tar.gz asset_content_type: application/gzip - - copy_nightly: + regression_test_setup: + name: Run regression testing if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly') - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest needs: [build_airgap] + outputs: + last_release_tag: ${{ steps.get_latest_release_tag.outputs.release }} steps: - name: Checkout uses: actions/checkout@v2 - - name: Get tags - id: get_tag - uses: ./actions/version-tag - - name: Copy assets uses: ./actions/copy-assets env: GITHUB_TOKEN: ${{ secrets.NIGHTLY_GH_PAT }} with: - srcReleaseTag: ${{ steps.get_tag.outputs.GIT_TAG }} + srcReleaseTag: ${{ github.ref_name }} dstReleaseTag: "v0.0.0-nightly" + - name: Get latest release tag + id: get_latest_release_tag + uses: pozetroninc/github-action-get-latest-release@master + with: + repository: ${{ github.repository }} + excludes: prerelease, draft - #### ---- START OF AUTOMATED REGRESSION TESTS ---- #### - - terraform-cleanup-test-instances: - if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly') - needs: copy_nightly - runs-on: ubuntu-20.04 - container: - image: hashicorp/terraform:1.0.7 - strategy: - fail-fast: false - matrix: - test: [ - { - name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin", - backend_config: "embedded-airgapped-upgrade-backend-config.tfvars", - terraform_script: "embedded-airgapped-upgrade.sh" - }, - { - name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin", - backend_config: "embedded-airgapped-install-backend-config.tfvars", - terraform_script: "embedded-airgapped-install.sh" - }, - { - name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin", - backend_config: "embedded-online-install-backend-config.tfvars", - terraform_script: "embedded-online-install.sh" - }, - { - name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin", - backend_config: "embedded-online-upgrade-backend-config.tfvars", - terraform_script: "embedded-online-upgrade.sh" - }, - { - name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin", - backend_config: "existing-airgapped-install-admin-backend-config.tfvars", - terraform_script: "existing-airgapped-install-admin.sh" - }, - { - name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac", - backend_config: "existing-airgapped-install-minimum-backend-config.tfvars", - terraform_script: "existing-airgapped-install-minimum.sh" - }, - { - name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin", - backend_config: "existing-online-upgrade-admin-backend-config.tfvars", - terraform_script: "existing-online-upgrade-admin.sh" - }, - { - name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac", - backend_config: "existing-online-upgrade-minimum-backend-config.tfvars", - terraform_script: "existing-online-upgrade-minimum.sh" - }, - { - name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin", - backend_config: "existing-online-install-admin-backend-config.tfvars", - terraform_script: "existing-online-install-admin.sh" - }, - { - name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac", - backend_config: "existing-online-install-minimum-backend-config.tfvars", - terraform_script: "existing-online-install-minimum.sh" - }, - { - name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin", - backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars", - terraform_script: "existing-airgapped-upgrade-admin.sh" - }, - { - name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac", - backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars", - terraform_script: "existing-airgapped-upgrade-minimum.sh" - } - ] - env: - AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }} - TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }} - TF_WORKSPACE: automation - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - repository: replicatedhq/kots-regression-automation - token: ${{ secrets.E2E_GH_PAT }} - path: automation - ref: main - - - name: Cleanup test environment - working-directory: automation/cluster - continue-on-error: true - run: | - terraform init -backend-config ${{ matrix.test.backend_config }} - ./${{ matrix.test.terraform_script }} destroy - - - name: Notify Slack - if: failure() - uses: 8398a7/action-slack@v3 - with: - status: custom - fields: repo,ref,workflow - custom_payload: | - { - attachments: [{ - color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning', - text: `Clean up: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`, - }] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }} - MATRIX_CONTEXT: ${{ toJson(matrix) }} - - - terraform-destroy-jumpbox: - if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly') - needs: terraform-cleanup-test-instances - runs-on: ubuntu-20.04 - container: - image: hashicorp/terraform:1.0.7 - env: - AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }} - TF_WORKSPACE: automation - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - repository: replicatedhq/kots-regression-automation - token: ${{ secrets.E2E_GH_PAT }} - path: automation - ref: main - - - name: Delete Jumpbox Environment - working-directory: automation/jumpbox - run: | - terraform init - terraform destroy --auto-approve - - - name: Notify Slack - if: failure() - uses: 8398a7/action-slack@v3 - with: - status: custom - fields: repo,ref,workflow - custom_payload: | - { - attachments: [{ - color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning', - text: `Destroy jumpbox: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`, - }] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }} - - - terraform-setup-jumpbox: - if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly') - needs: terraform-destroy-jumpbox - runs-on: ubuntu-20.04 - container: - image: hashicorp/terraform:1.0.7 - env: - AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }} - TF_WORKSPACE: automation - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - repository: replicatedhq/kots-regression-automation - token: ${{ secrets.E2E_GH_PAT }} - path: automation - ref: main - - - name: Create Jumpbox Environment - working-directory: automation/jumpbox - run: | - terraform init - terraform apply --auto-approve - - - name: Install openssl - working-directory: automation/jumpbox - run: | - apk add --update openssl - - - name: Encrypt ssh.pem - working-directory: automation/jumpbox - run: | - openssl enc -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 -pass pass:${{ secrets.ENCRYPTION_KEY }} -salt -in ssh.pem -out ssh.pem.enc - - - name: Upload ssh.pem.enc artifact - uses: actions/upload-artifact@v2 - with: - name: ssh.pem.enc - path: automation/jumpbox/ssh.pem.enc - - - name: Notify Slack - if: failure() - uses: 8398a7/action-slack@v3 - with: - status: custom - fields: repo,ref,workflow - custom_payload: | - { - attachments: [{ - color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning', - text: `Create jumpbox: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`, - }] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }} - - - terraform-setup-test-instances: + regression_test: if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly') - needs: terraform-setup-jumpbox - runs-on: ubuntu-20.04 - container: - image: hashicorp/terraform:1.0.7 - strategy: - fail-fast: false - matrix: - test: [ - { - name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin", - backend_config: "embedded-airgapped-upgrade-backend-config.tfvars", - terraform_script: "embedded-airgapped-upgrade.sh", - is_upgrade: "1" - }, - { - name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin", - backend_config: "embedded-airgapped-install-backend-config.tfvars", - terraform_script: "embedded-airgapped-install.sh" - }, - { - name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin", - backend_config: "embedded-online-install-backend-config.tfvars", - terraform_script: "embedded-online-install.sh" - }, - { - name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin", - backend_config: "embedded-online-upgrade-backend-config.tfvars", - terraform_script: "embedded-online-upgrade.sh", - is_upgrade: "1" - }, - { - name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin", - backend_config: "existing-airgapped-install-admin-backend-config.tfvars", - terraform_script: "existing-airgapped-install-admin.sh" - }, - { - name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac", - backend_config: "existing-airgapped-install-minimum-backend-config.tfvars", - terraform_script: "existing-airgapped-install-minimum.sh" - }, - { - name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin", - backend_config: "existing-online-upgrade-admin-backend-config.tfvars", - terraform_script: "existing-online-upgrade-admin.sh", - is_upgrade: "1" - }, - { - name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac", - backend_config: "existing-online-upgrade-minimum-backend-config.tfvars", - terraform_script: "existing-online-upgrade-minimum.sh", - is_upgrade: "1" - }, - { - name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin", - backend_config: "existing-online-install-admin-backend-config.tfvars", - terraform_script: "existing-online-install-admin.sh" - }, - { - name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac", - backend_config: "existing-online-install-minimum-backend-config.tfvars", - terraform_script: "existing-online-install-minimum.sh" - }, - { - name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin", - backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars", - terraform_script: "existing-airgapped-upgrade-admin.sh", - is_upgrade: "1" - }, - { - name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac", - backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars", - terraform_script: "existing-airgapped-upgrade-minimum.sh", - is_upgrade: "1" - } - ] - env: - AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }} - TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }} - TF_WORKSPACE: automation - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Get tags - id: get_tag - uses: ./actions/version-tag - - - name: Get latest tag - id: get_latest_tag - uses: pozetroninc/github-action-get-latest-release@master - with: - repository: ${{ github.repository }} - excludes: prerelease, draft - - - name: Checkout - uses: actions/checkout@v2 - with: - repository: replicatedhq/kots-regression-automation - token: ${{ secrets.E2E_GH_PAT }} - path: automation - ref: main - - - name: Create test environment - working-directory: automation/cluster - run: | - export CURRENT_TAG=${{ steps.get_latest_tag.outputs.release }} - export NEXT_TAG=${{ steps.get_tag.outputs.GIT_TAG }} - if [ "${{ matrix.test.is_upgrade }}" ]; then - export TF_VAR_kots_version_initial="${CURRENT_TAG:1}" - export TF_VAR_kots_version_upgrade="${NEXT_TAG:1}" - else - export TF_VAR_kots_version_initial="${NEXT_TAG:1}" - fi - terraform init -backend-config ${{ matrix.test.backend_config }} - ./${{ matrix.test.terraform_script }} apply - - - name: Notify Slack - if: failure() - uses: 8398a7/action-slack@v3 - with: - status: custom - fields: repo,ref,workflow - custom_payload: | - { - attachments: [{ - color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning', - text: `Setting up: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`, - }] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }} - MATRIX_CONTEXT: ${{ toJson(matrix) }} - - - testim-run-regression-tests: - if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly') - needs: terraform-setup-test-instances - runs-on: ubuntu-20.04 - container: - image: hashicorp/terraform:1.0.7 - strategy: - fail-fast: false - matrix: - test: [ - { - name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin", - backend_config: "embedded-airgapped-upgrade-backend-config.tfvars" - }, - { - name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin", - backend_config: "embedded-airgapped-install-backend-config.tfvars" - }, - { - name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin", - backend_config: "embedded-online-install-backend-config.tfvars" - }, - { - name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin", - backend_config: "embedded-online-upgrade-backend-config.tfvars" - }, - { - name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin", - backend_config: "existing-airgapped-install-admin-backend-config.tfvars" - }, - { - name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac", - backend_config: "existing-airgapped-install-minimum-backend-config.tfvars" - }, - { - name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin", - backend_config: "existing-online-upgrade-admin-backend-config.tfvars" - }, - { - name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac", - backend_config: "existing-online-upgrade-minimum-backend-config.tfvars" - }, - { - name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin", - backend_config: "existing-online-install-admin-backend-config.tfvars" - }, - { - name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac", - backend_config: "existing-online-install-minimum-backend-config.tfvars" - }, - { - name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin", - backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars" - }, - { - name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac", - backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars" - } - ] - env: - AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }} - TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }} - TF_WORKSPACE: automation - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - repository: replicatedhq/kots-regression-automation - token: ${{ secrets.E2E_GH_PAT }} - path: automation - ref: main - - - name: Set Jumpbox Public Ip - id: set_jumpbox_ip - working-directory: automation/jumpbox - run: | - terraform init - echo "::set-output name=jumpbox_ip::$(terraform output -raw jumpbox_ip_public)" - - - name: Download ssh.pem.enc artifact - uses: actions/download-artifact@v2 - with: - name: ssh.pem.enc - path: automation/jumpbox/ - - - name: Install openssl - working-directory: automation/jumpbox - run: | - apk add --update openssl - - - name: Decrypt ssh.pem - working-directory: automation/jumpbox - run: | - openssl enc -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 -pass pass:${{ secrets.ENCRYPTION_KEY }} -salt -d -in ssh.pem.enc -out ssh.pem - chmod 600 ssh.pem - - - name: Init test environment - id: init_test_env - working-directory: automation/cluster - run: | - terraform init -backend-config ${{ matrix.test.backend_config }} - echo "::set-output name=instance_ip::$(terraform output -raw control_plane_private_ip)" - - - name: Run the test - working-directory: automation/jumpbox - run: | - ssh -i ssh.pem ubuntu@${{ steps.set_jumpbox_ip.outputs.jumpbox_ip }} -oStrictHostKeyChecking=no -oServerAliveInterval=60 -oServerAliveCountMax=10 "ssh -tt ubuntu@${{ steps.init_test_env.outputs.instance_ip }} -oServerAliveInterval=60 -oServerAliveCountMax=10 \"sudo /tmp/start.sh\"" - - - name: Notify Slack - if: always() - uses: 8398a7/action-slack@v3 - with: - status: custom - fields: repo,ref,workflow - custom_payload: | - { - attachments: [{ - color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning', - text: `Test name: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`, - }] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }} - MATRIX_CONTEXT: ${{ toJson(matrix) }} - - - #### ---- END OF AUTOMATED REGRESSION TESTS ---- #### - + needs: regression_test_setup + uses: replicatedhq/kots/.github/workflows/regression.yaml@master + with: + version_tag_old: ${{ needs.regression_test_setup.outputs.last_release_tag }} + version_tag_new: ${{ github.ref_name }} + secrets: + E2E_TESTIM_AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }} + E2E_TESTIM_AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }} + TESTIM_ACCESS_TOKEN: ${{ secrets.TESTIM_ACCESS_TOKEN }} + E2E_GH_PAT: ${{ secrets.E2E_GH_PAT }} + KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }} start_kgrid_test: runs-on: ubuntu-18.04 @@ -1040,12 +537,10 @@ jobs: printf "\n\nSupport bundles are available in the Replicated production AWS account under the 'kgrid-support-bundles' S3 bucket. To download a support bundle, you can do so using the AWS Management Console, or by configuring the AWS cli tool with the appropriate credentials and running the following command: \n\naws s3 cp .tar.gz\n\n" ./hack/wait-kgrid.sh - generate-kurl-addon-pr: runs-on: ubuntu-18.04 needs: [release_go_api_tagged, build_kurl_proxy_tagged] steps: - - name: Checkout uses: actions/checkout@v2 @@ -1056,7 +551,7 @@ jobs: - name: Generate Kurl Addon PR env: GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }} - GH_PAT: ${{ secrets.GH_PAT }} + GH_PAT: ${{ secrets.GH_PAT }} run: | # Strips off the 'v' in version curl -H "Authorization: token $GH_PAT" \ @@ -1068,7 +563,6 @@ jobs: runs-on: ubuntu-18.04 needs: [release_go_api_tagged, build_kurl_proxy_tagged] steps: - - name: Checkout uses: actions/checkout@v2 @@ -1079,7 +573,7 @@ jobs: - name: Generate Kots Release Notes PR env: GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }} - GH_PAT: ${{ secrets.GH_PAT }} + GH_PAT: ${{ secrets.GH_PAT }} run: | curl -H "Authorization: token $GH_PAT" \ -H 'Accept: application/json' \