Adding standalone trigger for regression testing

.github/workflows/regression.yaml

@@ -0,0 +1,203 @@
+name: regression
+
+on:
+  workflow_call:
+    inputs:
+      version_tag_old:
+        description: 'The initial install version to use for upgrade scenarios'
+        required: true
+        type: string
+      version_tag_new:
+        description: 'The new version to use as the upgrade in upgrade scenarios and the initial version for fresh installs.'
+        required: true
+        type: string
+    secrets:
+      TESTIM_ACCESS_TOKEN:
+        required: true
+      E2E_TESTIM_AWS_ACCESS_KEY_ID:
+        required: true
+      E2E_TESTIM_AWS_SECRET_ACCESS_KEY:
+        required: true
+      E2E_GH_PAT:
+        required: true
+      KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL:
+        required: true
+
+  workflow_dispatch:
+    inputs:
+      version_tag_old:
+        description: 'The initial install version to use for upgrade scenarios'
+        required: true
+      version_tag_new:
+        description: 'The new version to use as the upgrade in upgrade scenarios and the initial version for fresh installs.'
+        required: true
+
+env:
+  AWS_DEFAULT_REGION: us-east-1
+  AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
+  TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
+  TF_WORKSPACE: automation
+
+jobs:
+  jumpbox:
+    runs-on: ubuntu-20.04
+    container:
+      image: hashicorp/terraform:1.0.7
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          repository: replicatedhq/kots-regression-automation
+          token: ${{ secrets.E2E_GH_PAT }}
+          path: automation
+          ref: main
+      - name: Create Jumpbox Environment
+        if: always()
+        working-directory: automation/jumpbox
+        run: |
+          terraform init
+          terraform apply --auto-approve
+
+      - name: Notify Slack
+        if: failure()
+        uses: 8398a7/action-slack@v3
+        with:
+          status: custom
+          fields: repo,ref,workflow
+          custom_payload: |
+            {
+              attachments: [{
+                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
+                text: `Create jumpbox: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`,
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}
+
+  tests:
+    needs: jumpbox
+    runs-on: ubuntu-20.04
+    container:
+      image: hashicorp/terraform:1.0.7
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [
+          {
+            name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
+            backend_config: "embedded-airgapped-upgrade-backend-config.tfvars",
+            terraform_script: "embedded-airgapped-upgrade.sh",
+            is_upgrade: "1"
+          },
+          {
+            name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin",
+            backend_config: "embedded-airgapped-install-backend-config.tfvars",
+            terraform_script: "embedded-airgapped-install.sh"
+          },
+          {
+            name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin",
+            backend_config: "embedded-online-install-backend-config.tfvars",
+            terraform_script: "embedded-online-install.sh"
+          },
+          {
+            name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin",
+            backend_config: "embedded-online-upgrade-backend-config.tfvars",
+            terraform_script: "embedded-online-upgrade.sh",
+            is_upgrade: "1"
+          },
+          {
+            name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin",
+            backend_config: "existing-airgapped-install-admin-backend-config.tfvars",
+            terraform_script: "existing-airgapped-install-admin.sh"
+          },
+          {
+            name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac",
+            backend_config: "existing-airgapped-install-minimum-backend-config.tfvars",
+            terraform_script: "existing-airgapped-install-minimum.sh"
+          },
+          {
+            name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin",
+            backend_config: "existing-online-upgrade-admin-backend-config.tfvars",
+            terraform_script: "existing-online-upgrade-admin.sh",
+            is_upgrade: "1"
+          },
+          {
+            name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac",
+            backend_config: "existing-online-upgrade-minimum-backend-config.tfvars",
+            terraform_script: "existing-online-upgrade-minimum.sh",
+            is_upgrade: "1"
+          },
+          {
+            name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin",
+            backend_config: "existing-online-install-admin-backend-config.tfvars",
+            terraform_script: "existing-online-install-admin.sh"
+          },
+          {
+            name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac",
+            backend_config: "existing-online-install-minimum-backend-config.tfvars",
+            terraform_script: "existing-online-install-minimum.sh"
+          },
+          {
+            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
+            backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars",
+            terraform_script: "existing-airgapped-upgrade-admin.sh",
+            is_upgrade: "1"
+          },
+          {
+            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac",
+            backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars",
+            terraform_script: "existing-airgapped-upgrade-minimum.sh",
+            is_upgrade: "1"
+          }
+        ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          repository: replicatedhq/kots-regression-automation
+          token: ${{ secrets.E2E_GH_PAT }}
+          path: automation
+          ref: adding-outputs
+      - name: Create test environment
+        if: always()
+        working-directory: automation/cluster
+        run: |
+          export CURRENT_TAG=${{ github.event.inputs.version_tag_old || inputs.version_tag_old }}
+          export NEXT_TAG=${{ github.event.inputs.version_tag_new || inputs.version_tag_new }}
+
+          if [ "${{ matrix.test.is_upgrade }}" ]; then
+            export TF_VAR_kots_version_initial="${CURRENT_TAG:1}"
+            export TF_VAR_kots_version_upgrade="${NEXT_TAG:1}"
+          else
+            export TF_VAR_kots_version_initial="${NEXT_TAG:1}"
+          fi
+          terraform init -backend-config ${{ matrix.test.backend_config }}
+          ./${{ matrix.test.terraform_script }} apply
+          terraform taint -allow-missing aws_instance.control_plane
+          terraform taint -allow-missing 'aws_instance.worker[0]'
+
+      - name: Run the test
+        working-directory: automation/cluster
+        run: |
+          terraform output -raw jumpbox_private_key > ssh.pem
+          chmod 600 ssh.pem
+          ssh -i ssh.pem ubuntu@$(terraform output -raw jumpbox_public_ip) -oStrictHostKeyChecking=no -oServerAliveInterval=60 -oServerAliveCountMax=10 "ssh -tt ubuntu@$(terraform output -raw control_plane_private_ip) -oServerAliveInterval=60 -oServerAliveCountMax=10 \"sudo /tmp/start.sh\""
+
+      - name: Notify Slack
+        if: failure()
+        uses: 8398a7/action-slack@v3
+        with:
+          status: custom
+          fields: repo,ref,workflow
+          custom_payload: |
+            {
+              attachments: [{
+                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
+                text: `Setting up: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`,
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}
+          MATRIX_CONTEXT: ${{ toJson(matrix) }}