Prep the GitHub Actions setup for handling more than one E2E test

.github/workflows/test-command.yml

@@ -8,7 +8,7 @@ on:
 
 defaults:
   run:
-    # We need -e -o pipefail for consistency with GitHub Actions's default behavior
+    # We need -e -o pipefail for consistency with GitHub Actions' default behavior
     shell: bash -e -o pipefail {0}
 
 jobs:
@@ -18,6 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       run-ping: ${{ steps.parse.outputs.ping }}
+      run-build: ${{ steps.parse.outputs.build }}
       run-e2e: ${{ steps.parse.outputs.e2e }}
     steps:
       - name: Parse Args
@@ -31,7 +32,7 @@ jobs:
           ARGS="${ARGS_V1}${ARGS_V2}"
           printf "Args are %s\n" "$ARGS"
           printf "\n\nslash_command is %s\n\n" "$DEBUG"
-          COMMANDS=(PING E2E)
+          COMMANDS=(PING BUILD E2E)
           if printf "%s" "${ARGS^^}" | grep -qE '\bALL\b'; then
             # "all" explicitly does not include "ping"
             for cmd in "${COMMANDS[@]}"; do
@@ -70,10 +71,107 @@ jobs:
           GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }}
           GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }}
 
-  # Run E2E tests
-  e2e:
+  # Build and upload the artifacts so they can be used later in the pipeline
+  build:
     runs-on: ubuntu-latest
     needs: parse
+    # Run if they explicitly want it, or run if they want a different stage that depends on this
+    if: needs.parse.outputs.run-build == 'true' || needs.parse.outputs.run-e2e == 'true'
+    container: cloudposse/test-harness:latest
+    steps:
+      # Update GitHub status for pending pipeline run
+      - name: "Update GitHub Status for pending"
+        uses: docker://cloudposse/github-status-updater
+        with:
+          args: "-action update_state -ref ${{ github.event.client_payload.pull_request.head.sha }} -repo ${{ github.event.client_payload.github.payload.repository.name }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+          GITHUB_STATE: pending
+          GITHUB_CONTEXT: "/test build"
+          GITHUB_DESCRIPTION: "started by @${{ github.event.client_payload.github.actor }}"
+          GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }}
+          GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }}
+
+      # Checkout the code from GitHub Pull Request
+      - name: "Checkout the code"
+        uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.PAT }}
+          repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.client_payload.pull_request.head.ref }}
+
+      - name: "Build the artifacts"
+        shell: bash -x -e -o pipefail {0}
+        run: |
+          # cloudposse/test-harness has golang 1.15, we need 1.16. This is the easiest way I know to do it. This should definitely be revisited and cleaned up.
+          git clone --branch v0.8.0 --depth 1 https://github.com/asdf-vm/asdf.git $HOME/.asdf
+          source ~/.asdf/asdf.sh
+          export PATH="$HOME/.asdf/bin:$PATH"
+          asdf plugin-add golang https://github.com/kennyp/asdf-golang.git
+          asdf install golang 1.16.7
+          asdf global golang 1.16.7
+          export GOPATH="$HOME/go"
+          export PATH="$PATH:$GOPATH/bin"
+          make build-cli-linux
+          ./build/zarf tools registry login registry1.dso.mil --username "${{ secrets.REGISTRY1_USERNAME_ROTHANDREW2 }}" --password "${{ secrets.REGISTRY1_PASSWORD_ROTHANDREW2 }}"
+          make init-package
+
+      - name: "Upload the artifacts"
+        uses: actions/upload-artifact@v2
+        with:
+          name: build
+          path: build
+          if-no-files-found: error
+
+      # Update GitHub status for failing pipeline run
+      - name: "Update GitHub Status for failure"
+        if: ${{ failure() }}
+        uses: docker://cloudposse/github-status-updater
+        with:
+          args: "-action update_state -ref ${{ github.event.client_payload.pull_request.head.sha }} -repo ${{ github.event.client_payload.github.payload.repository.name }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+          GITHUB_STATE: failure
+          GITHUB_CONTEXT: "/test build"
+          GITHUB_DESCRIPTION: "run failed"
+          GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }}
+          GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }}
+
+      # Update GitHub status for successful pipeline run
+      - name: "Update GitHub Status for success"
+        uses: docker://cloudposse/github-status-updater
+        with:
+          args: "-action update_state -ref ${{ github.event.client_payload.pull_request.head.sha }} -repo ${{ github.event.client_payload.github.payload.repository.name }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+          GITHUB_STATE: success
+          GITHUB_CONTEXT: "/test build"
+          GITHUB_DESCRIPTION: "run passed"
+          GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }}
+          GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }}
+
+      # Update GitHub status for cancelled pipeline run
+      - name: "Update GitHub Status for cancelled"
+        if: ${{ cancelled() }}
+        uses: docker://cloudposse/github-status-updater
+        with:
+          args: "-action update_state -ref ${{ github.event.client_payload.pull_request.head.sha }} -repo ${{ github.event.client_payload.github.payload.repository.name }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+          GITHUB_STATE: error
+          GITHUB_CONTEXT: "/test build"
+          GITHUB_DESCRIPTION: "run cancelled"
+          GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }}
+          GITHUB_OWNER: ${{ github.event.client_payload.github.payload.repository.owner.login }}
+
+  # Run the Game E2E test
+  e2e-game:
+    runs-on: ubuntu-latest
+    needs: [parse, build]
     if: needs.parse.outputs.run-e2e == 'true'
     container: cloudposse/test-harness:latest
     steps:
@@ -85,7 +183,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
           GITHUB_STATE: pending
-          GITHUB_CONTEXT: "/test e2e"
+          GITHUB_CONTEXT: "/test e2e - Game Example"
           GITHUB_DESCRIPTION: "started by @${{ github.event.client_payload.github.actor }}"
           GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
           GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }}
@@ -99,13 +197,9 @@ jobs:
           repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
           ref: ${{ github.event.client_payload.pull_request.head.ref }}
 
-#      # Checkout the code from GitHub Pull Request
-#      - name: "Checkout the code"
-#        uses: actions/checkout@v2
-#        with:
-#          token: ${{ secrets.PAT }}
-#          repository: defenseunicorns/zarf
-#          ref: feature/add-terratest-e2e-to-pipeline
+      # Download the built artifacts
+      - name: "Download the built artifacts"
+        uses: actions/download-artifact@v2
 
       - name: "Run E2E tests"
         shell: bash -x -e -o pipefail {0}
@@ -123,9 +217,8 @@ jobs:
           asdf global golang 1.16.7
           export GOPATH="$HOME/go"
           export PATH="$PATH:$GOPATH/bin"
-          make build-cli-linux
-          ./build/zarf tools registry login registry1.dso.mil --username "${{ secrets.REGISTRY1_USERNAME_ROTHANDREW2 }}" --password "${{ secrets.REGISTRY1_PASSWORD_ROTHANDREW2 }}"
-          make init-package test-e2e
+          chmod +x build/zarf
+          make package-example-game test-cloud-e2e-example-game
 
       # Update GitHub status for failing pipeline run
       - name: "Update GitHub Status for failure"
@@ -136,7 +229,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
           GITHUB_STATE: failure
-          GITHUB_CONTEXT: "/test e2e"
+          GITHUB_CONTEXT: "/test e2e - Game Example"
           GITHUB_DESCRIPTION: "run failed"
           GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
           GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }}
@@ -150,7 +243,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
           GITHUB_STATE: success
-          GITHUB_CONTEXT: "/test e2e"
+          GITHUB_CONTEXT: "/test e2e - Game Example"
           GITHUB_DESCRIPTION: "run passed"
           GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
           GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }}
@@ -165,7 +258,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.PAT }}
           GITHUB_STATE: error
-          GITHUB_CONTEXT: "/test e2e"
+          GITHUB_CONTEXT: "/test e2e - Game Example"
           GITHUB_DESCRIPTION: "run cancelled"
           GITHUB_TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
           GITHUB_REF: ${{ github.event.client_payload.pull_request.head.ref }}

Makefile

@@ -33,9 +33,6 @@ vm-init: ## usage -> make vm-init OS=ubuntu
 vm-destroy: ## Destroy the VM
 	vagrant destroy -f
 
-test-e2e: ## Run E2E tests. Requires access to an AWS account. Costs money. Make sure you ran the `build-cli` and `init-package` targets first
-	cd test/e2e && go test ./... -v -timeout 1200s
-
 e2e-ssh: ## Run this if you set SKIP_teardown=1 and want to SSH into the still-running test server. Don't forget to unset SKIP_teardown when you're done
 	cd test/tf/public-ec2-instance/.test-data && cat Ec2KeyPair.json | jq -r .PrivateKey > privatekey.pem && chmod 600 privatekey.pem
 	cd test/tf/public-ec2-instance && ssh -i .test-data/privatekey.pem ubuntu@$$(terraform output public_instance_ip)
@@ -61,5 +58,13 @@ build-test: build-cli init-package ## Build the CLI and create the init package
 
 ci-release: init-package ## Create the init package
 
-package-examples: ## automatically package all example directories and add the tarballs to the examples/sync directory
-	cd examples && $(MAKE) package-examples
+.PHONY: package-example-game
+package-example-game: ## Create the Doom example
+	cd examples/game && ../../$(ZARF_BIN) package create --confirm && mv zarf-package-* ../../build/
+
+.PHONY: test-cloud-e2e-example-game
+test-cloud-e2e-example-game: ## Runs the Doom game as an E2E test in the cloud. Requires access to an AWS account. Costs money. Make sure you ran the `build-cli`, `init-package`, and `package-example-game` targets first
+	cd test/e2e && go test ./... -run TestE2eExampleGame -v -timeout 1200s
+
+.PHONY: test-e2e
+test-e2e: package-example-game test-cloud-e2e-example-game ## DEPRECATED - to be replaced by individual e2e test targets

test/e2e/common.go

@@ -0,0 +1,139 @@
+package test
+
+import (
+  "bufio"
+  "encoding/base64"
+  "fmt"
+  "github.com/gruntwork-io/terratest/modules/random"
+  "github.com/gruntwork-io/terratest/modules/retry"
+  teststructure "github.com/gruntwork-io/terratest/modules/test-structure"
+  "github.com/stretchr/testify/require"
+  "io/ioutil"
+  "os"
+  "testing"
+  "time"
+
+  "github.com/gruntwork-io/terratest/modules/aws"
+  "github.com/gruntwork-io/terratest/modules/ssh"
+  "github.com/gruntwork-io/terratest/modules/terraform"
+)
+
+func teardown(t *testing.T, tmpFolder string) {
+  keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder)
+  aws.DeleteEC2KeyPair(t, keyPair)
+
+  terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder)
+  terraform.Destroy(t, terraformOptions)
+}
+
+func setup(t *testing.T, tmpFolder string) {
+  terraformOptions, keyPair, err := configureTerraformOptions(t, tmpFolder)
+  require.NoError(t, err)
+
+  // Save the options and key pair so later test stages can use them
+  teststructure.SaveTerraformOptions(t, tmpFolder, terraformOptions)
+  teststructure.SaveEc2KeyPair(t, tmpFolder, keyPair)
+
+  // This will run `terraform init` and `terraform apply` and fail the test if there are any errors
+  terraform.InitAndApply(t, terraformOptions)
+}
+
+func configureTerraformOptions(t *testing.T, tmpFolder string) (*terraform.Options, *aws.Ec2Keypair, error) {
+  // A unique ID we can use to namespace resources so we don't clash with anything already in the AWS account or
+  // tests running in parallel
+  uniqueID := random.UniqueId()
+  namespace := "zarf"
+  stage := "terratest"
+  name := fmt.Sprintf("e2e-%s", uniqueID)
+
+  // Get the region to use from the system's environment
+  awsRegion, err := getAwsRegion()
+  if err != nil {
+    return nil, nil, err
+  }
+
+  instanceType := "t3a.large"
+
+  // Create an EC2 KeyPair that we can use for SSH access
+  keyPairName := fmt.Sprintf("%s-%s-%s", namespace, stage, name)
+  keyPair := aws.CreateAndImportEC2KeyPair(t, awsRegion, keyPairName)
+
+  // Construct the terraform options with default retryable errors to handle the most common retryable errors in
+  // terraform testing.
+  terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
+    // The path to where our Terraform code is located
+    TerraformDir: tmpFolder,
+
+    // Variables to pass to our Terraform code using -var options
+    Vars: map[string]interface{}{
+      "aws_region":    awsRegion,
+      "namespace":     namespace,
+      "stage":         stage,
+      "name":          name,
+      "instance_type": instanceType,
+      "key_pair_name": keyPairName,
+    },
+  })
+
+  return terraformOptions, keyPair, nil
+}
+
+// syncFileToRemoteServer uses SCP to sync a file from source to destination. `destPath` can be absolute or relative to
+// the SSH user's home directory. It has to be in a directory that the SSH user is allowed to write to.
+func syncFileToRemoteServer(t *testing.T, terraformOptions *terraform.Options, keyPair *aws.Ec2Keypair, sshUsername string, srcPath string, destPath string, chmod string) {
+  // Run `terraform output` to get the value of an output variable
+  publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip")
+
+  // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu",
+  // as we know the Instance is running an Ubuntu AMI that has such a user
+  host := ssh.Host{
+    Hostname:    publicInstanceIP,
+    SshKeyPair:  keyPair.KeyPair,
+    SshUserName: sshUsername,
+  }
+
+  // It can take a minute or so for the Instance to boot up, so retry a few times
+  maxRetries := 15
+  timeBetweenRetries, err := time.ParseDuration("5s")
+  require.NoError(t, err)
+
+  // Wait for the instance to be ready
+  _, err = retry.DoWithRetryE(t, "Wait for the instance to be ready", maxRetries, timeBetweenRetries, func() (string, error){
+    _, err := ssh.CheckSshCommandE(t, host, "whoami")
+    if err != nil {
+      return "", err
+    }
+    return "", nil
+  })
+  require.NoError(t, err)
+
+  // Create the folder structure
+  output, err := ssh.CheckSshCommandE(t, host,fmt.Sprintf("bash -c 'install -m 644 -D /dev/null \"%s\"'", destPath))
+  require.NoError(t, err, output)
+
+  // The ssh lib only supports sending strings so we'll base64encode it first
+  f, err := os.Open(srcPath)
+  require.NoError(t, err)
+  reader := bufio.NewReader(f)
+  content, err := ioutil.ReadAll(reader)
+  require.NoError(t, err)
+  encodedContent := base64.StdEncoding.EncodeToString(content)
+  err = ssh.ScpFileToE(t, host, 0600, fmt.Sprintf("%s.b64", destPath), encodedContent)
+  require.NoError(t, err)
+  output, err = ssh.CheckSshCommandE(t, host, fmt.Sprintf("base64 -d \"%s.b64\" > \"%s\" && chmod \"%s\" \"%s\"", destPath, destPath, chmod, destPath))
+  require.NoError(t, err, output)
+}
+
+// getAwsRegion returns the desired AWS region to use by first checking the env var AWS_REGION, then checking
+// AWS_DEFAULT_REGION if AWS_REGION isn't set. If neither is set it returns an error
+func getAwsRegion() (string, error) {
+  val, present := os.LookupEnv("AWS_REGION")
+  if !present {
+    val, present = os.LookupEnv("AWS_DEFAULT_REGION")
+  }
+  if !present {
+    return "", fmt.Errorf("expected either AWS_REGION or AWS_DEFAULT_REGION env var to be set, but they were not")
+  } else {
+    return val, nil
+  }
+}