automated nightly regression tests (#2164)

* automated nightly regression tests
replicatedhq · Sep 17, 2021 · 85886b4 · 85886b4
1 parent 4fe89bc
commit 85886b4
Showing 1 changed file with 370 additions and 0 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -679,6 +679,376 @@ jobs:
         dstReleaseTag: "v0.0.0-nightly"
 
 
+  #### ---- START OF AUTOMATED REGRESSION TESTS ---- ####
+
+  terraform-setup-jumpbox:
+    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
+    needs: copy_nightly
+    runs-on: ubuntu-20.04
+    container:
+      image: hashicorp/terraform:1.0.7
+    env:
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
+      TF_WORKSPACE: automation
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          repository: replicatedhq/kots-regression-automation
+          token: ${{ secrets.E2E_GH_PAT }}
+          path: automation
+          ref: main
+
+      - name: Create Jumpbox Environment
+        working-directory: automation/jumpbox
+        run: |
+          terraform init
+          terraform apply --auto-approve
+
+      - name: Install openssl
+        working-directory: automation/jumpbox
+        run: |
+          apk add --update openssl
+
+      - name: Encrypt ssh.pem
+        working-directory: automation/jumpbox
+        run: |
+          openssl enc -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 -pass pass:${{ secrets.ENCRYPTION_KEY }} -salt -in ssh.pem -out ssh.pem.enc
+
+      - name: Upload ssh.pem.enc artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: ssh.pem.enc
+          path: automation/jumpbox/ssh.pem.enc
+
+
+  terraform-setup-test-instances:
+    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
+    needs: terraform-setup-jumpbox
+    runs-on: ubuntu-20.04
+    container:
+      image: hashicorp/terraform:1.0.7
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [
+          {
+            backend_config: "embedded-airgapped-upgrade-backend-config.tfvars",
+            terraform_script: "embedded-airgapped-upgrade.sh",
+            is_upgrade: "1"
+          },
+          {
+            backend_config: "embedded-airgapped-install-backend-config.tfvars",
+            terraform_script: "embedded-airgapped-install.sh"
+          },
+          {
+            backend_config: "embedded-online-install-backend-config.tfvars",
+            terraform_script: "embedded-online-install.sh"
+          },
+          {
+            backend_config: "embedded-online-upgrade-backend-config.tfvars",
+            terraform_script: "embedded-online-upgrade.sh",
+            is_upgrade: "1"
+          },
+          {
+            backend_config: "existing-airgapped-install-admin-backend-config.tfvars",
+            terraform_script: "existing-airgapped-install-admin.sh"
+          },
+          {
+            backend_config: "existing-airgapped-install-minimum-backend-config.tfvars",
+            terraform_script: "existing-airgapped-install-minimum.sh"
+          },
+          {
+            backend_config: "existing-online-upgrade-admin-backend-config.tfvars",
+            terraform_script: "existing-online-upgrade-admin.sh",
+            is_upgrade: "1"
+          },
+          {
+            backend_config: "existing-online-upgrade-minimum-backend-config.tfvars",
+            terraform_script: "existing-online-upgrade-minimum.sh",
+            is_upgrade: "1"
+          },
+          {
+            backend_config: "existing-online-install-admin-backend-config.tfvars",
+            terraform_script: "existing-online-install-admin.sh"
+          },
+          {
+            backend_config: "existing-online-install-minimum-backend-config.tfvars",
+            terraform_script: "existing-online-install-minimum.sh"
+          },
+          {
+            backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars",
+            terraform_script: "existing-airgapped-upgrade-admin.sh",
+            is_upgrade: "1"
+          },
+          {
+            backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars",
+            terraform_script: "existing-airgapped-upgrade-minimum.sh",
+            is_upgrade: "1"
+          }
+        ]
+    env:
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
+      TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
+      TF_WORKSPACE: automation
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Get tags
+        id: get_tag
+        uses: ./actions/version-tag
+
+      - name: Get latest tag
+        id: get_latest_tag
+        uses: pozetroninc/github-action-get-latest-release@master
+        with:
+          repository: ${{ github.repository }}
+          excludes: prerelease, draft
+
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          repository: replicatedhq/kots-regression-automation
+          token: ${{ secrets.E2E_GH_PAT }}
+          path: automation
+          ref: main
+
+      - name: Create test environment
+        working-directory: automation/cluster
+        run: |
+          export CURRENT_TAG=${{ steps.get_latest_tag.outputs.release }}
+          export NEXT_TAG=${{ steps.get_tag.outputs.GIT_TAG }}
+          if [ "${{ matrix.test.is_upgrade }}" ]; then
+            export TF_VAR_kots_version_initial="${CURRENT_TAG:1}"
+            export TF_VAR_kots_version_upgrade="${NEXT_TAG:1}"
+          else
+            export TF_VAR_kots_version_initial="${NEXT_TAG:1}"
+          fi
+          terraform init -backend-config ${{ matrix.test.backend_config }}
+          ./${{ matrix.test.terraform_script }} apply
+
+
+  testim-run-regression-tests:
+    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
+    needs: terraform-setup-test-instances
+    runs-on: ubuntu-20.04
+    container:
+      image: hashicorp/terraform:1.0.7
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [
+          {
+            backend_config: "embedded-airgapped-upgrade-backend-config.tfvars"
+          },
+          {
+            backend_config: "embedded-airgapped-install-backend-config.tfvars"
+          },
+          {
+            backend_config: "embedded-online-install-backend-config.tfvars"
+          },
+          {
+            backend_config: "embedded-online-upgrade-backend-config.tfvars"
+          },
+          {
+            backend_config: "existing-airgapped-install-admin-backend-config.tfvars"
+          },
+          {
+            backend_config: "existing-airgapped-install-minimum-backend-config.tfvars"
+          },
+          {
+            backend_config: "existing-online-upgrade-admin-backend-config.tfvars"
+          },
+          {
+            backend_config: "existing-online-upgrade-minimum-backend-config.tfvars"
+          },
+          {
+            backend_config: "existing-online-install-admin-backend-config.tfvars"
+          },
+          {
+            backend_config: "existing-online-install-minimum-backend-config.tfvars"
+          },
+          {
+            backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars"
+          },
+          {
+            backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars"
+          }
+        ]
+    env:
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
+      TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
+      TF_WORKSPACE: automation
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          repository: replicatedhq/kots-regression-automation
+          token: ${{ secrets.E2E_GH_PAT }}
+          path: automation
+          ref: main
+
+      - name: Set Jumpbox Public Ip
+        id: set_jumpbox_ip
+        working-directory: automation/jumpbox
+        run: |
+          terraform init
+          echo "::set-output name=jumpbox_ip::$(terraform output -raw jumpbox_ip_public)"
+
+      - name: Download ssh.pem.enc artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: ssh.pem.enc
+          path: automation/jumpbox/
+
+      - name: Install openssl
+        working-directory: automation/jumpbox
+        run: |
+          apk add --update openssl
+
+      - name: Decrypt ssh.pem
+        working-directory: automation/jumpbox
+        run: |
+          openssl enc -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 -pass pass:${{ secrets.ENCRYPTION_KEY }} -salt -d -in ssh.pem.enc -out ssh.pem
+          chmod 600 ssh.pem
+
+      - name: Init test environment
+        id: init_test_env
+        working-directory: automation/cluster
+        run: |
+          terraform init -backend-config ${{ matrix.test.backend_config }}
+          echo "::set-output name=instance_ip::$(terraform output -raw control_plane_public_ip)"
+
+      - name: Run the test
+        working-directory: automation/jumpbox
+        run: |
+          ssh -i ssh.pem ubuntu@${{ steps.set_jumpbox_ip.outputs.jumpbox_ip }} -oStrictHostKeyChecking=no <<EOT
+          ssh ubuntu@${{ steps.init_test_env.outputs.instance_ip }} "sudo bash /tmp/start.sh"
+          EOT
+
+
+  terraform-cleanup-test-instances:
+    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
+    needs: testim-run-regression-tests
+    runs-on: ubuntu-20.04
+    container:
+      image: hashicorp/terraform:1.0.7
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [
+          {
+            backend_config: "embedded-airgapped-upgrade-backend-config.tfvars",
+            terraform_script: "embedded-airgapped-upgrade.sh"
+          },
+          {
+            backend_config: "embedded-airgapped-install-backend-config.tfvars",
+            terraform_script: "embedded-airgapped-install.sh"
+          },
+          {
+            backend_config: "embedded-online-install-backend-config.tfvars",
+            terraform_script: "embedded-online-install.sh"
+          },
+          {
+            backend_config: "embedded-online-upgrade-backend-config.tfvars",
+            terraform_script: "embedded-online-upgrade.sh"
+          },
+          {
+            backend_config: "existing-airgapped-install-admin-backend-config.tfvars",
+            terraform_script: "existing-airgapped-install-admin.sh"
+          },
+          {
+            backend_config: "existing-airgapped-install-minimum-backend-config.tfvars",
+            terraform_script: "existing-airgapped-install-minimum.sh"
+          },
+          {
+            backend_config: "existing-online-upgrade-admin-backend-config.tfvars",
+            terraform_script: "existing-online-upgrade-admin.sh"
+          },
+          {
+            backend_config: "existing-online-upgrade-minimum-backend-config.tfvars",
+            terraform_script: "existing-online-upgrade-minimum.sh"
+          },
+          {
+            backend_config: "existing-online-install-admin-backend-config.tfvars",
+            terraform_script: "existing-online-install-admin.sh"
+          },
+          {
+            backend_config: "existing-online-install-minimum-backend-config.tfvars",
+            terraform_script: "existing-online-install-minimum.sh"
+          },
+          {
+            backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars",
+            terraform_script: "existing-airgapped-upgrade-admin.sh"
+          },
+          {
+            backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars",
+            terraform_script: "existing-airgapped-upgrade-minimum.sh"
+          }
+        ]
+    env:
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
+      TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
+      TF_WORKSPACE: automation
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          repository: replicatedhq/kots-regression-automation
+          token: ${{ secrets.E2E_GH_PAT }}
+          path: automation
+          ref: main
+
+      - name: Cleanup Test Environment for Embedded AirGapped Upgrade
+        working-directory: automation/cluster
+        run: |
+          terraform init -backend-config ${{ matrix.test.backend_config }}
+          ./${{ matrix.test.terraform_script }} destroy
+
+
+  terraform-destroy-jumpbox:
+    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
+    needs: terraform-cleanup-test-instances
+    runs-on: ubuntu-20.04
+    container:
+      image: hashicorp/terraform:1.0.7
+    env:
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
+      TF_WORKSPACE: automation
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          repository: replicatedhq/kots-regression-automation
+          token: ${{ secrets.E2E_GH_PAT }}
+          path: automation
+          ref: main
+
+      - name: Delete Jumpbox Environment
+        working-directory: automation/jumpbox
+        run: |
+          terraform init
+          terraform destroy --auto-approve
+
+  #### ---- END OF AUTOMATED REGRESSION TESTS ---- ####
+
+
   start_kgrid_test:
     runs-on: ubuntu-18.04
     if: false