release.yaml

name: release

on:
  push:
    tags:
      - "v*.*.*"

jobs:

  generate-schema:
    runs-on: ubuntu-18.04
    container:
      image: schemahero/schemahero:0.12.2
    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Generate fixtures
        run: |
          /schemahero fixtures \
            --input-dir ./migrations/tables \
            --output-dir ./migrations/fixtures/schema \
            --dbname ship-cloud --driver postgres

      - name: Upload schema artifact
        uses: actions/upload-artifact@v2
        with:
          name: schema
          path: ./migrations/fixtures/schema/fixtures.sql


  generate-fixtures:
    runs-on: ubuntu-18.04
    needs: [generate-schema]
    container:
      image: replicated/gitops-builder:buildkite
      options: --user root
    steps:
    - name: Checkout
      uses: actions/checkout@v2

    - name: Download schema artifact
      uses: actions/download-artifact@v2
      with:
        name: schema
        path: ./migrations/fixtures/schema

    - name: Build fixtures
      run: export $(cat .image.env | sed 's/#.*//g' | xargs) && make -C migrations/fixtures deps build run
      shell: bash

    - name: Upload fixtures artifact
      uses: actions/upload-artifact@v2
      with:
        name: fixtures
        path: ./migrations/fixtures/fixtures.sql


  publish-fixtures:
    runs-on: ubuntu-18.04
    needs: [generate-fixtures]
    steps:
    - name: Checkout
      uses: actions/checkout@v2

    - uses: azure/docker-login@v1
      with:
        username: ${{ secrets.DOCKERHUB_USER }}
        password: ${{ secrets.DOCKERHUB_PASSWORD }}

    - name: Download fixtures artifact
      uses: actions/download-artifact@v2
      with:
        name: fixtures
        path: ./migrations/fixtures


  build-schema-migrations:
    runs-on: ubuntu-20.04
    needs: [publish-fixtures]
    steps:
    - name: Checkout
      uses: actions/checkout@v2

    - uses: azure/docker-login@v1
      env:
        DOCKER_CONFIG: ./migrations/.docker
      with:
        username: ${{ secrets.DOCKERHUB_USER }}
        password: ${{ secrets.DOCKERHUB_PASSWORD }}

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: "Release schema migrations on tag"
      env:
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
        DOCKER_CONFIG: ./.docker
      run: |
        export $(cat .image.env | sed 's/#.*//g' | xargs) && make -C migrations schema-release

    - name: Upload airgap image
      uses: actions/upload-artifact@v2
      with:
        name: migrations-image
        path: ./migrations/bin/docker-archive


  test_web:
    runs-on: ubuntu-18.04
    needs: [build-schema-migrations]
    container:
      image: replicated/gitops-builder:buildkite
      options: --user root
    steps:
    - name: Checkout
      uses: actions/checkout@v2

    - name: Setup Node.js environment
      uses: actions/setup-node@v2
      with:
        node-version: '10'

    - name: Install cross-env
      run: npm i -g cross-env
      shell: bash

    # - name: Test web
    #   env:
    #     PACT_BROKER_USERNAME: ${{ secrets.PACT_BROKER_USERNAME }}
    #     PACT_BROKER_PASSWORD: ${{ secrets.PACT_BROKER_PASSWORD }}
    #   run: |
    #     make -C kotsadm/web deps test publish-pact

      # TODO: migrate retry logic for pacts
      # retry:
      #   automatic:
      #     # this command exiting with status 2 typically means that the yarn package failed to download
      #     - exit_status: 2
      #       limit: 5


  build_web:
    runs-on: ubuntu-18.04
    needs: [build-schema-migrations]
    container:
      image: replicated/gitops-builder:buildkite
      options: --user root
    steps:
    - name: Checkout
      uses: actions/checkout@v2

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: Setup Node.js environment
      uses: actions/setup-node@v2
      with:
        node-version: '10'

    - name: Build web
      env:
        GIT_COMMIT: ${{ github.sha }}
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
      run: export $(cat .image.env | sed 's/#.*//g' | xargs) && make -C web deps build-kotsadm
      shell: bash
      ## TODO: retry logic
      # retry:
      #   automatic:
      #     # this command exiting with status 2 typically means that the yarn package failed to download
      #     - exit_status: 2
      #       limit: 5

    - name: Upload web artifact
      uses: actions/upload-artifact@v2
      with:
        name: web
        path: ./web/dist


  build_kurl_proxy:
    runs-on: ubuntu-18.04
    needs: [build-schema-migrations]
    container:
      image: replicated/gitops-builder:buildkite-go14-node10
      options: --user root
    steps:
    - name: Checkout
      uses: actions/checkout@v2

    - name: Setup Go
      uses: actions/setup-go@v2
      with:
        go-version: '^1.16.3'

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: Build kurl_proxy
      env:
        GIT_COMMIT: ${{ github.sha }}
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
        SCOPE_DSN_PUBLIC: ""
      run: export $(cat .image.env | sed 's/#.*//g' | xargs) && make -C kurl_proxy test build
      shell: bash

    - name: Upload kurl_proxy artifact
      uses: actions/upload-artifact@v2
      with:
        name: kurl_proxy
        path: ./kurl_proxy/bin


  build_go_api:
    runs-on: ubuntu-18.04
    needs: [test_web, build_web, build_kurl_proxy]
    container:
      image: replicated/gitops-builder:buildkite-go14-node10
      options: --user root
    steps:
    - uses: actions/setup-go@v2
      with:
        go-version: '^1.16.3'

    - id: go-cache-paths
      run: |
        echo "::set-output name=go-build::$(go env GOCACHE)"
        echo "::set-output name=go-mod::$(go env GOMODCACHE)"
    - name: Checkout
      uses: actions/checkout@v1 # not @v2 because of: https://github.com/actions/checkout/issues/126

    - uses: actions/cache@v2
      with:
        path: ${{ steps.go-cache-paths.outputs.go-build }}
        key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
    - uses: actions/cache@v2
      with:
        path: ${{ steps.go-cache-paths.outputs.go-mod }}
        key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: Download web artifact
      uses: actions/download-artifact@v2
      with:
        name: web
        path: ./web/dist

    - name: Build Go API
      env:
        GIT_COMMIT: ${{ github.sha }}
        # GITHUB_SHA: ${{ github.sha }}
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
        # GITHUB_REPOSITORY:
        # GITHUB_WORKSPACE:
        SCOPE_DSN_PUBLIC: ""
      run: export $(cat .image.env | sed 's/#.*//g' | xargs) && make test kotsadm
      shell: bash

    - name: Upload Go API artifact
      uses: actions/upload-artifact@v2
      with:
        name: go_api
        path: ./bin


  release_go_api_tagged:
    runs-on: ubuntu-20.04
    needs: [build_web, build_go_api]
    steps:
    - name: Setup Go
      uses: actions/setup-go@v2
      with:
        go-version: '^1.16.3'

    - id: go-cache-paths
      run: |
        echo "::set-output name=go-build::$(go env GOCACHE)"
        echo "::set-output name=go-mod::$(go env GOMODCACHE)"
    - name: Checkout
      uses: actions/checkout@v2

    - uses: actions/cache@v2
      with:
        path: ${{ steps.go-cache-paths.outputs.go-build }}
        key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
    - uses: actions/cache@v2
      with:
        path: ${{ steps.go-cache-paths.outputs.go-mod }}
        key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: Download go_api artifact
      uses: actions/download-artifact@v2
      with:
        name: go_api
        path: ./bin

    - name: Add executable permissions
      run: chmod a+x ./bin/kotsadm

    - uses: azure/docker-login@v1
      env:
        DOCKER_CONFIG: ./.docker
      with:
        username: ${{ secrets.DOCKERHUB_USER }}
        password: ${{ secrets.DOCKERHUB_PASSWORD }}

    - name: Build tagged release
      env:
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
        DOCKER_CONFIG: ./.docker
      run: |
        export $(cat .image.env | sed 's/#.*//g' | xargs) && make build-release

    - name: Upload airgap image
      uses: actions/upload-artifact@v2
      with:
        name: kotsadm-image
        path: ./bin/docker-archive


  build_kurl_proxy_tagged:
    runs-on: ubuntu-18.04
    needs: [build_kurl_proxy]
    steps:
    - uses: azure/docker-login@v1
      with:
        username: ${{ secrets.DOCKERHUB_USER }}
        password: ${{ secrets.DOCKERHUB_PASSWORD }}

    - name: Setup Go
      uses: actions/setup-go@v2
      with:
        go-version: '^1.16.3'

    - id: go-cache-paths
      run: |
        echo "::set-output name=go-build::$(go env GOCACHE)"
        echo "::set-output name=go-mod::$(go env GOMODCACHE)"
    - name: Checkout
      uses: actions/checkout@v2

    - uses: actions/cache@v2
      with:
        path: ${{ steps.go-cache-paths.outputs.go-build }}
        key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
    - uses: actions/cache@v2
      with:
        path: ${{ steps.go-cache-paths.outputs.go-mod }}
        key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: Download kurl_proxy artifact
      uses: actions/download-artifact@v2
      with:
        name: kurl_proxy
        path: ./kurl_proxy/bin

    - name: Add executable permissions
      run: chmod a+x ./kurl_proxy/bin/kurl_proxy

    - name: Build tagged release
      env:
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
      run: |
        export $(cat .image.env | sed 's/#.*//g' | xargs) && make -C kurl_proxy build-release


  goreleaser:
    runs-on: ubuntu-18.04
    steps:
    - id: go-cache-paths
      run: |
        echo "::set-output name=go-build::$(go env GOCACHE)"
        echo "::set-output name=go-mod::$(go env GOMODCACHE)"

    - name: Checkout
      uses: actions/checkout@v2

    - uses: actions/cache@v2
      with:
        path: ${{ steps.go-cache-paths.outputs.go-build }}
        key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
    - uses: actions/cache@v2
      with:
        path: ${{ steps.go-cache-paths.outputs.go-mod }}
        key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}

    - name: Unshallow
      run: git fetch --prune --unshallow

    - uses: actions/setup-go@v2
      with:
        go-version: '^1.16.3'

    - run: sudo apt-get update -y
    - run: sudo apt-get -qq -y install gnupg2 libdevmapper-dev libgpgme-dev libc6-dev-i386 btrfs-progs btrfs-tools pkg-config
    - name: set previous release tag for goreleaser
      shell: bash
      run: |
        export TAG=$(curl --silent "https://api.github.com/repos/replicatedhq/kots/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')
        echo "GORELEASER_PREVIOUS_TAG=${TAG}" >> $GITHUB_ENV

    - uses: sigstore/cosign-installer@main
      with:
        cosign-release: 'v1.2.1'

    - name: Get Cosign Key
      run: |
        echo $COSIGN_KEY | base64 -d > ./cosign.key
      env:
        COSIGN_KEY: ${{secrets.COSIGN_KEY}}

    - name: Generate SBOM
      run: |
        make sbom
      env:
        COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}

    - name: Run GoReleaser
      uses: goreleaser/goreleaser-action@v2
      with:
        version: "v0.166.1"
        args: release --rm-dist --config deploy/.goreleaser.yaml
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


  build_airgap:
    runs-on: ubuntu-18.04
    needs: [release_go_api_tagged, goreleaser, build-schema-migrations]
    steps:
    - name: Checkout
      uses: actions/checkout@v2
    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: Download migrations
      uses: actions/download-artifact@v2
      with:
        name: migrations-image
        path: ./docker-archive

    - name: Download kotsadm image
      uses: actions/download-artifact@v2
      with:
        name: kotsadm-image
        path: ./docker-archive

    - name: Get Release
      id: get_release
      uses: bruceadams/get-release@v1.2.3   # no v1 tag
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

    - name: Make kotsadm airgap archive
      run: |
        tar czf ./kotsadm.tar.gz -C ./ ./docker-archive

    - name: Upload airgap bundle
      uses: actions/upload-release-asset@v1
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        upload_url: https://uploads.github.com/repos/replicatedhq/kots/releases/${{ steps.get_release.outputs.id }}/assets?name=kotsadm.tar.gz
        asset_path: ./kotsadm.tar.gz
        asset_name: kotsadm.tar.gz
        asset_content_type: application/gzip


  copy_nightly:
    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
    runs-on: ubuntu-18.04
    needs: [build_airgap]
    steps:
    - name: Checkout
      uses: actions/checkout@v2

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: Copy assets
      uses: ./actions/copy-assets
      env:
        GITHUB_TOKEN: ${{ secrets.NIGHTLY_GH_PAT }}
      with:
        srcReleaseTag: ${{ steps.get_tag.outputs.GIT_TAG }}
        dstReleaseTag: "v0.0.0-nightly"


  #### ---- START OF AUTOMATED REGRESSION TESTS ---- ####

  terraform-cleanup-test-instances:
    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
    needs: copy_nightly
    runs-on: ubuntu-20.04
    container:
      image: hashicorp/terraform:1.0.7
    strategy:
      fail-fast: false
      matrix:
        test: [
          {
            name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
            backend_config: "embedded-airgapped-upgrade-backend-config.tfvars",
            terraform_script: "embedded-airgapped-upgrade.sh"
          },
          {
            name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin",
            backend_config: "embedded-airgapped-install-backend-config.tfvars",
            terraform_script: "embedded-airgapped-install.sh"
          },
          {
            name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin",
            backend_config: "embedded-online-install-backend-config.tfvars",
            terraform_script: "embedded-online-install.sh"
          },
          {
            name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin",
            backend_config: "embedded-online-upgrade-backend-config.tfvars",
            terraform_script: "embedded-online-upgrade.sh"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin",
            backend_config: "existing-airgapped-install-admin-backend-config.tfvars",
            terraform_script: "existing-airgapped-install-admin.sh"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac",
            backend_config: "existing-airgapped-install-minimum-backend-config.tfvars",
            terraform_script: "existing-airgapped-install-minimum.sh"
          },
          {
            name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin",
            backend_config: "existing-online-upgrade-admin-backend-config.tfvars",
            terraform_script: "existing-online-upgrade-admin.sh"
          },
          {
            name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac",
            backend_config: "existing-online-upgrade-minimum-backend-config.tfvars",
            terraform_script: "existing-online-upgrade-minimum.sh"
          },
          {
            name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin",
            backend_config: "existing-online-install-admin-backend-config.tfvars",
            terraform_script: "existing-online-install-admin.sh"
          },
          {
            name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac",
            backend_config: "existing-online-install-minimum-backend-config.tfvars",
            terraform_script: "existing-online-install-minimum.sh"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
            backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars",
            terraform_script: "existing-airgapped-upgrade-admin.sh"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac",
            backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars",
            terraform_script: "existing-airgapped-upgrade-minimum.sh"
          }
        ]
    env:
      AWS_DEFAULT_REGION: us-east-1
      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
      TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
      TF_WORKSPACE: automation

    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          repository: replicatedhq/kots-regression-automation
          token: ${{ secrets.E2E_GH_PAT }}
          path: automation
          ref: main

      - name: Cleanup test environment
        working-directory: automation/cluster
        run: |
          terraform init -backend-config ${{ matrix.test.backend_config }}
          ./${{ matrix.test.terraform_script }} destroy

      - name: Notify Slack
        if: always()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          fields: repo,ref,workflow
          custom_payload: |
            {
              attachments: [{
                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
                text: `Cleaning up: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}. Ref: ${process.env.AS_REF}`,
              }]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}
          MATRIX_CONTEXT: ${{ toJson(matrix) }}


  terraform-destroy-jumpbox:
    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
    needs: terraform-cleanup-test-instances
    runs-on: ubuntu-20.04
    container:
      image: hashicorp/terraform:1.0.7
    env:
      AWS_DEFAULT_REGION: us-east-1
      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
      TF_WORKSPACE: automation

    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          repository: replicatedhq/kots-regression-automation
          token: ${{ secrets.E2E_GH_PAT }}
          path: automation
          ref: main

      - name: Delete Jumpbox Environment
        working-directory: automation/jumpbox
        run: |
          terraform init
          terraform destroy --auto-approve

      - name: Notify Slack
        if: always()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          fields: repo,ref,workflow
          custom_payload: |
            {
              attachments: [{
                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
                text: `Destroying jumpbox: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}. Ref: ${process.env.AS_REF}`,
              }]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}


  terraform-setup-jumpbox:
    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
    needs: terraform-destroy-jumpbox
    runs-on: ubuntu-20.04
    container:
      image: hashicorp/terraform:1.0.7
    env:
      AWS_DEFAULT_REGION: us-east-1
      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
      TF_WORKSPACE: automation

    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          repository: replicatedhq/kots-regression-automation
          token: ${{ secrets.E2E_GH_PAT }}
          path: automation
          ref: main

      - name: Create Jumpbox Environment
        working-directory: automation/jumpbox
        run: |
          terraform init
          terraform apply --auto-approve

      - name: Install openssl
        working-directory: automation/jumpbox
        run: |
          apk add --update openssl

      - name: Encrypt ssh.pem
        working-directory: automation/jumpbox
        run: |
          openssl enc -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 -pass pass:${{ secrets.ENCRYPTION_KEY }} -salt -in ssh.pem -out ssh.pem.enc

      - name: Upload ssh.pem.enc artifact
        uses: actions/upload-artifact@v2
        with:
          name: ssh.pem.enc
          path: automation/jumpbox/ssh.pem.enc

      - name: Notify Slack
        if: always()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          fields: repo,ref,workflow
          custom_payload: |
            {
              attachments: [{
                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
                text: `Creating jumpbox: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}. Ref: ${process.env.AS_REF}`,
              }]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}


  terraform-setup-test-instances:
    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
    needs: terraform-setup-jumpbox
    runs-on: ubuntu-20.04
    container:
      image: hashicorp/terraform:1.0.7
    strategy:
      fail-fast: false
      matrix:
        test: [
          {
            name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
            backend_config: "embedded-airgapped-upgrade-backend-config.tfvars",
            terraform_script: "embedded-airgapped-upgrade.sh",
            is_upgrade: "1"
          },
          {
            name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin",
            backend_config: "embedded-airgapped-install-backend-config.tfvars",
            terraform_script: "embedded-airgapped-install.sh"
          },
          {
            name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin",
            backend_config: "embedded-online-install-backend-config.tfvars",
            terraform_script: "embedded-online-install.sh"
          },
          {
            name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin",
            backend_config: "embedded-online-upgrade-backend-config.tfvars",
            terraform_script: "embedded-online-upgrade.sh",
            is_upgrade: "1"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin",
            backend_config: "existing-airgapped-install-admin-backend-config.tfvars",
            terraform_script: "existing-airgapped-install-admin.sh"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac",
            backend_config: "existing-airgapped-install-minimum-backend-config.tfvars",
            terraform_script: "existing-airgapped-install-minimum.sh"
          },
          {
            name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin",
            backend_config: "existing-online-upgrade-admin-backend-config.tfvars",
            terraform_script: "existing-online-upgrade-admin.sh",
            is_upgrade: "1"
          },
          {
            name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac",
            backend_config: "existing-online-upgrade-minimum-backend-config.tfvars",
            terraform_script: "existing-online-upgrade-minimum.sh",
            is_upgrade: "1"
          },
          {
            name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin",
            backend_config: "existing-online-install-admin-backend-config.tfvars",
            terraform_script: "existing-online-install-admin.sh"
          },
          {
            name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac",
            backend_config: "existing-online-install-minimum-backend-config.tfvars",
            terraform_script: "existing-online-install-minimum.sh"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
            backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars",
            terraform_script: "existing-airgapped-upgrade-admin.sh",
            is_upgrade: "1"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac",
            backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars",
            terraform_script: "existing-airgapped-upgrade-minimum.sh",
            is_upgrade: "1"
          }
        ]
    env:
      AWS_DEFAULT_REGION: us-east-1
      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
      TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
      TF_WORKSPACE: automation

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Get tags
        id: get_tag
        uses: ./actions/version-tag

      - name: Get latest tag
        id: get_latest_tag
        uses: pozetroninc/github-action-get-latest-release@master
        with:
          repository: ${{ github.repository }}
          excludes: prerelease, draft

      - name: Checkout
        uses: actions/checkout@v2
        with:
          repository: replicatedhq/kots-regression-automation
          token: ${{ secrets.E2E_GH_PAT }}
          path: automation
          ref: main

      - name: Create test environment
        working-directory: automation/cluster
        run: |
          export CURRENT_TAG=${{ steps.get_latest_tag.outputs.release }}
          export NEXT_TAG=${{ steps.get_tag.outputs.GIT_TAG }}
          if [ "${{ matrix.test.is_upgrade }}" ]; then
            export TF_VAR_kots_version_initial="${CURRENT_TAG:1}"
            export TF_VAR_kots_version_upgrade="${NEXT_TAG:1}"
          else
            export TF_VAR_kots_version_initial="${NEXT_TAG:1}"
          fi
          terraform init -backend-config ${{ matrix.test.backend_config }}
          ./${{ matrix.test.terraform_script }} apply

      - name: Notify Slack
        if: always()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          fields: repo,ref,workflow
          custom_payload: |
            {
              attachments: [{
                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
                text: `Setting up: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}. Ref: ${process.env.AS_REF}`,
              }]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}
          MATRIX_CONTEXT: ${{ toJson(matrix) }}


  testim-run-regression-tests:
    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
    needs: terraform-setup-test-instances
    runs-on: ubuntu-20.04
    container:
      image: hashicorp/terraform:1.0.7
    strategy:
      fail-fast: false
      matrix:
        test: [
          {
            name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
            backend_config: "embedded-airgapped-upgrade-backend-config.tfvars"
          },
          {
            name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin",
            backend_config: "embedded-airgapped-install-backend-config.tfvars"
          },
          {
            name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin",
            backend_config: "embedded-online-install-backend-config.tfvars"
          },
          {
            name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin",
            backend_config: "embedded-online-upgrade-backend-config.tfvars"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin",
            backend_config: "existing-airgapped-install-admin-backend-config.tfvars"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac",
            backend_config: "existing-airgapped-install-minimum-backend-config.tfvars"
          },
          {
            name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin",
            backend_config: "existing-online-upgrade-admin-backend-config.tfvars"
          },
          {
            name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac",
            backend_config: "existing-online-upgrade-minimum-backend-config.tfvars"
          },
          {
            name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin",
            backend_config: "existing-online-install-admin-backend-config.tfvars"
          },
          {
            name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac",
            backend_config: "existing-online-install-minimum-backend-config.tfvars"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
            backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac",
            backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars"
          }
        ]
    env:
      AWS_DEFAULT_REGION: us-east-1
      AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
      TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
      TF_WORKSPACE: automation

    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          repository: replicatedhq/kots-regression-automation
          token: ${{ secrets.E2E_GH_PAT }}
          path: automation
          ref: main

      - name: Set Jumpbox Public Ip
        id: set_jumpbox_ip
        working-directory: automation/jumpbox
        run: |
          terraform init
          echo "::set-output name=jumpbox_ip::$(terraform output -raw jumpbox_ip_public)"

      - name: Download ssh.pem.enc artifact
        uses: actions/download-artifact@v2
        with:
          name: ssh.pem.enc
          path: automation/jumpbox/

      - name: Install openssl
        working-directory: automation/jumpbox
        run: |
          apk add --update openssl

      - name: Decrypt ssh.pem
        working-directory: automation/jumpbox
        run: |
          openssl enc -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 -pass pass:${{ secrets.ENCRYPTION_KEY }} -salt -d -in ssh.pem.enc -out ssh.pem
          chmod 600 ssh.pem

      - name: Init test environment
        id: init_test_env
        working-directory: automation/cluster
        run: |
          terraform init -backend-config ${{ matrix.test.backend_config }}
          echo "::set-output name=instance_ip::$(terraform output -raw control_plane_private_ip)"

      - name: Run the test
        working-directory: automation/jumpbox
        run: |
          ssh -i ssh.pem ubuntu@${{ steps.set_jumpbox_ip.outputs.jumpbox_ip }} -oStrictHostKeyChecking=no -oServerAliveInterval=60 -oServerAliveCountMax=10 "ssh -tt ubuntu@${{ steps.init_test_env.outputs.instance_ip }} -oServerAliveInterval=60 -oServerAliveCountMax=10 \"sudo /tmp/start.sh\""

      - name: Notify Slack
        if: always()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          fields: repo,ref,workflow
          custom_payload: |
            {
              attachments: [{
                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
                text: `Test name: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}. Ref: ${process.env.AS_REF}`,
              }]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}
          MATRIX_CONTEXT: ${{ toJson(matrix) }}


  #### ---- END OF AUTOMATED REGRESSION TESTS ---- ####


  start_kgrid_test:
    runs-on: ubuntu-18.04
    needs: [release_go_api_tagged]
    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-nightly')
    steps:
    - name: Checkout
      uses: actions/checkout@v2

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - uses: kceb/pull-request-url-action@v2
      id: pr_url

    - name: Expose git commit data
      uses: rlespinasse/git-commit-data-action@v1.x   # no v1 tag

    - name: Create staging gitops commit
      env:
        REPLICATEDCOM_GITHUB_PRIVATE_KEY: ${{ secrets.REPLICATEDCOM_GITHUB_PRIVATE_KEY }}
        PR_URL: ${{ steps.pr_url.outputs.url }}
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
        GITOPS_OWNER: replicatedcom
        GITOPS_REPO: gitops-deploy
        GITOPS_BRANCH: master
        RUN_ID: ${{ github.run_id }}
      run: ./hack/deploy-kgrid.sh

    - name: Create production gitops commit
      id: deploy_kgrid
      env:
        REPLICATEDCOM_GITHUB_PRIVATE_KEY: ${{ secrets.REPLICATEDCOM_GITHUB_PRIVATE_KEY }}
        PR_URL: ${{ steps.pr_url.outputs.url }}
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
        OVERLAY: production
        GITOPS_OWNER: replicatedcom
        GITOPS_REPO: gitops-deploy
        GITOPS_BRANCH: release
      run: ./hack/deploy-kgrid.sh

    - name: Wait for production kgrid tests
      env:
        KGRID_API_TOKEN: ${{ secrets.KGRID_API_TOKEN }}
        KGRID_RUN_ID: ${{ steps.deploy_kgrid.outputs.kgrid-run-id }}
      run: |
        printf "\n\nSupport bundles are available in the Replicated production AWS account under the 'kgrid-support-bundles' S3 bucket. To download a support bundle, you can do so using the AWS Management Console, or by configuring the AWS cli tool with the appropriate credentials and running the following command: \n\naws s3 cp <test-supportbundle-s3-url> <local-filename>.tar.gz\n\n"
        ./hack/wait-kgrid.sh


  generate-kurl-addon-pr:
    runs-on: ubuntu-18.04
    needs: [release_go_api_tagged, build_kurl_proxy_tagged]
    steps:

    - name: Checkout
      uses: actions/checkout@v2

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: Generate Kurl Addon PR
      env:
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
        GH_PAT: ${{ secrets.GH_PAT }}  
      run: |
        # Strips off the 'v' in version
        curl -H "Authorization: token $GH_PAT" \
          -H 'Accept: application/json' \
          -d "{\"event_type\": \"auto-kotsadm-update\", \"client_payload\": {\"version\": \"${GIT_TAG:1}\" }}" \
          "https://api.github.com/repos/replicatedhq/kurl/dispatches"

  generate-kots-release-notes-pr:
    runs-on: ubuntu-18.04
    needs: [release_go_api_tagged, build_kurl_proxy_tagged]
    steps:

    - name: Checkout
      uses: actions/checkout@v2

    - name: Get tags
      id: get_tag
      uses: ./actions/version-tag

    - name: Generate Kots Release Notes PR
      env:
        GIT_TAG: ${{ steps.get_tag.outputs.GIT_TAG }}
        GH_PAT: ${{ secrets.GH_PAT }}  
      run: |
        curl -H "Authorization: token $GH_PAT" \
          -H 'Accept: application/json' \
          -d "{\"event_type\": \"auto-release-notes\", \"client_payload\": {\"version\": \"${GIT_TAG}\" }}" \
          "https://api.github.com/repos/replicatedhq/kots.io/dispatches"