Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding standalone trigger for regression testing #2362

Merged
merged 1 commit into from
Nov 23, 2021
Merged

Adding standalone trigger for regression testing #2362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
203 changes: 203 additions & 0 deletions .github/workflows/regression.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
name: regression

on:
workflow_call:
inputs:
version_tag_old:
description: 'The initial install version to use for upgrade scenarios'
required: true
type: string
version_tag_new:
description: 'The new version to use as the upgrade in upgrade scenarios and the initial version for fresh installs.'
required: true
type: string
secrets:
TESTIM_ACCESS_TOKEN:
required: true
E2E_TESTIM_AWS_ACCESS_KEY_ID:
required: true
E2E_TESTIM_AWS_SECRET_ACCESS_KEY:
required: true
E2E_GH_PAT:
required: true
KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL:
required: true

workflow_dispatch:
inputs:
version_tag_old:
description: 'The initial install version to use for upgrade scenarios'
required: true
version_tag_new:
description: 'The new version to use as the upgrade in upgrade scenarios and the initial version for fresh installs.'
required: true

env:
AWS_DEFAULT_REGION: us-east-1
AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
TF_WORKSPACE: automation

jobs:
jumpbox:
runs-on: ubuntu-20.04
container:
image: hashicorp/terraform:1.0.7
steps:
- name: Checkout
uses: actions/checkout@v2
with:
repository: replicatedhq/kots-regression-automation
token: ${{ secrets.E2E_GH_PAT }}
path: automation
ref: main
- name: Create Jumpbox Environment
if: always()
working-directory: automation/jumpbox
run: |
terraform init
terraform apply --auto-approve

- name: Notify Slack
if: failure()
uses: 8398a7/action-slack@v3
with:
status: custom
fields: repo,ref,workflow
custom_payload: |
{
attachments: [{
color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
text: `Create jumpbox: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`,
}]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}

tests:
needs: jumpbox
runs-on: ubuntu-20.04
container:
image: hashicorp/terraform:1.0.7
strategy:
fail-fast: false
matrix:
test: [
{
name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
backend_config: "embedded-airgapped-upgrade-backend-config.tfvars",
terraform_script: "embedded-airgapped-upgrade.sh",
is_upgrade: "1"
},
{
name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin",
backend_config: "embedded-airgapped-install-backend-config.tfvars",
terraform_script: "embedded-airgapped-install.sh"
},
{
name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin",
backend_config: "embedded-online-install-backend-config.tfvars",
terraform_script: "embedded-online-install.sh"
},
{
name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin",
backend_config: "embedded-online-upgrade-backend-config.tfvars",
terraform_script: "embedded-online-upgrade.sh",
is_upgrade: "1"
},
{
name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin",
backend_config: "existing-airgapped-install-admin-backend-config.tfvars",
terraform_script: "existing-airgapped-install-admin.sh"
},
{
name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac",
backend_config: "existing-airgapped-install-minimum-backend-config.tfvars",
terraform_script: "existing-airgapped-install-minimum.sh"
},
{
name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin",
backend_config: "existing-online-upgrade-admin-backend-config.tfvars",
terraform_script: "existing-online-upgrade-admin.sh",
is_upgrade: "1"
},
{
name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac",
backend_config: "existing-online-upgrade-minimum-backend-config.tfvars",
terraform_script: "existing-online-upgrade-minimum.sh",
is_upgrade: "1"
},
{
name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin",
backend_config: "existing-online-install-admin-backend-config.tfvars",
terraform_script: "existing-online-install-admin.sh"
},
{
name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac",
backend_config: "existing-online-install-minimum-backend-config.tfvars",
terraform_script: "existing-online-install-minimum.sh"
},
{
name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars",
terraform_script: "existing-airgapped-upgrade-admin.sh",
is_upgrade: "1"
},
{
name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac",
backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars",
terraform_script: "existing-airgapped-upgrade-minimum.sh",
is_upgrade: "1"
}
]

steps:
- name: Checkout
uses: actions/checkout@v2
with:
repository: replicatedhq/kots-regression-automation
token: ${{ secrets.E2E_GH_PAT }}
path: automation
ref: adding-outputs
- name: Create test environment
if: always()
working-directory: automation/cluster
run: |
export CURRENT_TAG=${{ github.event.inputs.version_tag_old || inputs.version_tag_old }}
export NEXT_TAG=${{ github.event.inputs.version_tag_new || inputs.version_tag_new }}

if [ "${{ matrix.test.is_upgrade }}" ]; then
export TF_VAR_kots_version_initial="${CURRENT_TAG:1}"
export TF_VAR_kots_version_upgrade="${NEXT_TAG:1}"
else
export TF_VAR_kots_version_initial="${NEXT_TAG:1}"
fi
terraform init -backend-config ${{ matrix.test.backend_config }}
./${{ matrix.test.terraform_script }} apply
terraform taint -allow-missing aws_instance.control_plane
terraform taint -allow-missing 'aws_instance.worker[0]'

- name: Run the test
working-directory: automation/cluster
run: |
terraform output -raw jumpbox_private_key > ssh.pem
chmod 600 ssh.pem
ssh -i ssh.pem ubuntu@$(terraform output -raw jumpbox_public_ip) -oStrictHostKeyChecking=no -oServerAliveInterval=60 -oServerAliveCountMax=10 "ssh -tt ubuntu@$(terraform output -raw control_plane_private_ip) -oServerAliveInterval=60 -oServerAliveCountMax=10 \"sudo /tmp/start.sh\""

- name: Notify Slack
if: failure()
uses: 8398a7/action-slack@v3
with:
status: custom
fields: repo,ref,workflow
custom_payload: |
{
attachments: [{
color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
text: `Setting up: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`,
}]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}
MATRIX_CONTEXT: ${{ toJson(matrix) }}