Merge pull request #1355 from hashicorp/sebasslash/enable-onprem-ci

Temporarily enable CI to test against enterprise instance
hashicorp · May 22, 2024 · 036e812 · 036e812
2 parents 5219dbf + 1a0b1eb
commit 036e812
Show file tree

Hide file tree

Showing 11 changed files with 140 additions and 105 deletions.
diff --git a/.github/actions/lint-provider-tfe/action.yml b/.github/actions/lint-provider-tfe/action.yml
@@ -7,7 +7,7 @@ runs:
   using: composite
   steps:
     - name: Setup Go Environment
-      uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+      uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
       with:
         go-version-file: "go.mod"
         cache: true

diff --git a/.github/actions/test-provider-tfe/action.yml b/.github/actions/test-provider-tfe/action.yml
@@ -1,7 +1,7 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: MPL-2.0
 
-name: TESTS ARE TEMPOARILY DISABLED
+name: Test
 description: Tests terraform-provider-tfe within a matrix
 inputs:
   admin_configuration_token:
@@ -51,76 +51,72 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: NO-OP
+    - name: Set up Go
+      uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+      with:
+        go-version-file: go.mod
+        cache: true
+
+    - name: Sync dependencies
       shell: bash
       run: |
-        echo "Tests are skipped. Please test manually."
-    # - name: Set up Go
-    #   uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
-    #   with:
-    #     go-version-file: go.mod
-    #     cache: true
-
-    # - name: Sync dependencies
-    #   shell: bash
-    #   run: |
-    #     go mod download
-    #     go mod tidy
+        go mod download
+        go mod tidy
 
-    # - name: Install gotestsum
-    #   shell: bash
-    #   run: go install gotest.tools/gotestsum@latest
+    - name: Install gotestsum
+      shell: bash
+      run: go install gotest.tools/gotestsum@latest
 
-    # - name: Download artifact
-    #   id: download-artifact
-    #   uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2.27.0
-    #   with:
-    #     workflow_conclusion: success
-    #     name: junit-test-summary
-    #     if_no_artifact_found: warn
-    #     branch: main
+    - name: Download artifact
+      id: download-artifact
+      uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2.27.0
+      with:
+        workflow_conclusion: success
+        name: junit-test-summary
+        if_no_artifact_found: warn
+        branch: main
 
-    # - name: Split acceptance tests
-    #   id: test_split
-    #   uses: hashicorp-forge/go-test-split-action@796beedbdb3d1bea14cad2d3057bab5c5cf15fe5 # v1.0.2
-    #   with:
-    #     index: ${{ inputs.matrix_index }}
-    #     total: ${{ inputs.matrix_total }}
-    #     junit-summary: ./ci-summary-provider.xml
-    #     # When tests are split and run concurrently, lists_tests arg in ci.yml will skip the TestAccTFESAMLSettings_omnibus test suite
-    #     list: ${{ inputs.list_tests }}
+    - name: Split acceptance tests
+      id: test_split
+      uses: hashicorp-forge/go-test-split-action@796beedbdb3d1bea14cad2d3057bab5c5cf15fe5 # v1.0.2
+      with:
+        index: ${{ inputs.matrix_index }}
+        total: ${{ inputs.matrix_total }}
+        junit-summary: ./ci-summary-provider.xml
+        # When tests are split and run concurrently, lists_tests arg in ci.yml will skip the TestAccTFESAMLSettings_omnibus test suite
+        list: ${{ inputs.list_tests }}
 
-    # - name: Run Tests
-    #   shell: bash
-    #   env:
-    #     TFE_HOSTNAME: "${{ inputs.hostname }}"
-    #     TFE_TOKEN: "${{ inputs.token }}"
-    #     TFE_ADMIN_CONFIGURATION_TOKEN: ${{ inputs.admin_configuration_token }}
-    #     TFE_ADMIN_PROVISION_LICENSES_TOKEN: ${{ inputs.admin_provision_licenses_token }}
-    #     TFE_ADMIN_SECURITY_MAINTENANCE_TOKEN: ${{ inputs.admin_security_maintenance_token }}
-    #     TFE_ADMIN_SITE_ADMIN_TOKEN: ${{ inputs.admin_site_admin_token }}
-    #     TFE_ADMIN_SUBSCRIPTION_TOKEN: ${{ inputs.admin_subscription_token }}
-    #     TFE_ADMIN_SUPPORT_TOKEN: ${{ inputs.admin_support_token }}
-    #     TFE_ADMIN_VERSION_MAINTENANCE_TOKEN: ${{ inputs.admin_version_maintenance_token }}
-    #     TFE_USER1: tfe-provider-user1
-    #     TFE_USER2: tfe-provider-user2
-    #     TF_ACC: "1"
-    #     ENABLE_TFE: "${{ inputs.enterprise }}"
-    #     RUN_TASKS_URL: "http://testing-mocks.tfe:22180/runtasks/pass"
-    #     GITHUB_POLICY_SET_IDENTIFIER: "hashicorp/test-policy-set"
-    #     GITHUB_REGISTRY_MODULE_IDENTIFIER: "hashicorp/terraform-random-module"
-    #     GITHUB_WORKSPACE_IDENTIFIER: "hashicorp/terraform-random-module"
-    #     GITHUB_WORKSPACE_BRANCH: "main"
-    #     GITHUB_TOKEN: "${{ inputs.testing-github-token }}"
-    #     MOD_PROVIDER: github.com/hashicorp/terraform-provider-tfe
-    #     MOD_TFE: github.com/hashicorp/terraform-provider-tfe/internal/provider
-    #     MOD_VERSION: github.com/hashicorp/terraform-provider-tfe/version
-    #   run: |
-    #     gotestsum --junitfile summary.xml --format short-verbose -- $MOD_PROVIDER $MOD_TFE $MOD_VERSION -v -timeout=30m -run "${{ steps.test_split.outputs.run }}"
+    - name: Run Tests
+      shell: bash
+      env:
+        TFE_HOSTNAME: "${{ inputs.hostname }}"
+        TFE_TOKEN: "${{ inputs.token }}"
+        TFE_ADMIN_CONFIGURATION_TOKEN: ${{ inputs.admin_configuration_token }}
+        TFE_ADMIN_PROVISION_LICENSES_TOKEN: ${{ inputs.admin_provision_licenses_token }}
+        TFE_ADMIN_SECURITY_MAINTENANCE_TOKEN: ${{ inputs.admin_security_maintenance_token }}
+        TFE_ADMIN_SITE_ADMIN_TOKEN: ${{ inputs.admin_site_admin_token }}
+        TFE_ADMIN_SUBSCRIPTION_TOKEN: ${{ inputs.admin_subscription_token }}
+        TFE_ADMIN_SUPPORT_TOKEN: ${{ inputs.admin_support_token }}
+        TFE_ADMIN_VERSION_MAINTENANCE_TOKEN: ${{ inputs.admin_version_maintenance_token }}
+        TFE_USER1: tfe-provider-user1
+        TFE_USER2: tfe-provider-user2
+        TF_ACC: "1"
+        ENABLE_TFE: "${{ inputs.enterprise }}"
+        RUN_TASKS_URL: "http://testing-mocks.tfe:22180/runtasks/pass"
+        GITHUB_POLICY_SET_IDENTIFIER: "hashicorp/test-policy-set"
+        GITHUB_REGISTRY_MODULE_IDENTIFIER: "hashicorp/terraform-random-module"
+        GITHUB_WORKSPACE_IDENTIFIER: "hashicorp/terraform-random-module"
+        GITHUB_WORKSPACE_BRANCH: "main"
+        GITHUB_TOKEN: "${{ inputs.testing-github-token }}"
+        MOD_PROVIDER: github.com/hashicorp/terraform-provider-tfe
+        MOD_TFE: github.com/hashicorp/terraform-provider-tfe/internal/provider
+        MOD_VERSION: github.com/hashicorp/terraform-provider-tfe/version
+      run: |
+        gotestsum --junitfile summary.xml --format short-verbose -- $MOD_PROVIDER $MOD_TFE $MOD_VERSION -v -timeout=30m -run "${{ steps.test_split.outputs.run }}"
 
-    # - name: Upload test artifacts
-    #   uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-    #   with:
-    #     name: junit-test-summary-${{ matrix.index }}
-    #     path: summary.xml
-    #     retention-days: 1
+    - name: Upload test artifacts
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+      with:
+        name: junit-test-summary-${{ matrix.index }}
+        path: summary.xml
+        retention-days: 1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,13 +9,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  notice:
-    name: TESTS ARE TEMPORARILY DISABLED- RUN CHANGED RESOURCE TESTS LOCALLY
-    runs-on: ubuntu-latest
-    steps:
-      - name: NO-OP
-        run: |
-          echo "Tests are skipped. Please test manually."
   lint:
     name: lint
     runs-on: ubuntu-latest
@@ -31,8 +24,8 @@ jobs:
       fail-fast: false
       matrix:
         # If you adjust these parameters, also adjust the jrm input files on the "Merge reports" step below
-        total: [ 1 ]
-        index: [ 0 ]
+        total: [ 5 ]
+        index: [ 0, 1, 2, 3, 4 ]
     steps:
       - name: Fetch Outputs
         id: tflocal
@@ -80,7 +73,7 @@ jobs:
         run: npm install -g junit-report-merger
 
       - name: Merge reports
-        run: jrm ./ci-summary-provider.xml "junit-test-summary-0/*.xml"
+        run: jrm ./ci-summary-provider.xml "junit-test-summary-0/*.xml" "junit-test-summary-1/*.xml" "junit-test-summary-2/*.xml" "junit-test-summary-3/*.xml" "junit-test-summary-4/*.xml"
 
       - name: Upload test artifacts
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3

diff --git a/.github/workflows/create-release-pr.yml b/.github/workflows/create-release-pr.yml
@@ -35,7 +35,7 @@ jobs:
     steps:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       - name: Set up Go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
           go-version-file: go.mod
           cache: true

diff --git a/.github/workflows/nightly-tfe-test.yml b/.github/workflows/nightly-tfe-test.yml
@@ -25,8 +25,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        total: [ 1 ]
-        index: [ 0 ]
+        total: [ 5 ]
+        index: [ 0, 1, 2, 3, 4 ]
     steps:
       - name: Fetch Outputs
         id: tflocal
@@ -99,7 +99,7 @@ jobs:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
 
       - name: Set up Go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
           go-version-file: go.mod
           check-latest: true

diff --git a/internal/provider/helper_test.go b/internal/provider/helper_test.go
@@ -200,6 +200,18 @@ func skipUnlessBeta(t *testing.T) {
 	}
 }
 
+// Temporarily skip a test that may be experiencing API errors. This method
+// purposefully errors after the set date to remind contributors to remove this check
+// and verify that the API errors are no longer occurring.
+func skipUnlessAfterDate(t *testing.T, d time.Time) {
+	today := time.Now()
+	if today.After(d) {
+		t.Fatalf("This test was temporarily skipped and has now expired. Remove this check to run this test.")
+	} else {
+		t.Skipf("Temporarily skipping test due to external issues: %s", t.Name())
+	}
+}
+
 func enterpriseEnabled() bool {
 	return os.Getenv("ENABLE_TFE") == "1"
 }
@@ -255,9 +267,9 @@ func randomString(t *testing.T) string {
 	return v
 }
 
-type retryableFn func() (interface{}, error)
+type retryableFn func() (any, error)
 
-func retryFn(maxRetries, secondsBetween int, f retryableFn) (interface{}, error) {
+func retryFn(maxRetries, secondsBetween int, f retryableFn) (any, error) {
 	tick := time.NewTicker(time.Duration(secondsBetween) * time.Second)
 	retries := 0
 

diff --git a/internal/provider/provider_test.go b/internal/provider/provider_test.go
@@ -213,6 +213,8 @@ func TestConfigureEnvOrganization(t *testing.T) {
 // The TFE Provider tests use these environment variables, which are set in the
 // GitHub Action workflow file .github/workflows/ci.yml.
 func testAccGithubPreCheck(t *testing.T) {
+	skipUnlessAfterDate(t, time.Date(2024, 5, 24, 0, 0, 0, 0, time.UTC))
+
 	if envGithubToken == "" {
 		t.Skip("Please set GITHUB_TOKEN to run this test")
 	}

diff --git a/internal/provider/resource_tfe_policy_set_test.go b/internal/provider/resource_tfe_policy_set_test.go
@@ -4,6 +4,7 @@
 package provider
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"regexp"
@@ -60,6 +61,19 @@ func TestAccTFEPolicySet_pinnedPolicyRuntimeVersion(t *testing.T) {
 	sha := genSentinelSha(t, "secret", "data")
 	version := genSafeRandomSentinelVersion()
 
+	adminClient := testAdminClient(t, versionMaintenanceAdmin)
+
+	opts := tfe.AdminSentinelVersionCreateOptions{
+		Version: version,
+		SHA:     sha,
+		URL:     "https://hashicorp.com",
+	}
+
+	tool, err := adminClient.Admin.SentinelVersions.Create(context.Background(), opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	org, orgCleanup := createBusinessOrganization(t, tfeClient)
 	t.Cleanup(orgCleanup)
 
@@ -71,7 +85,7 @@ func TestAccTFEPolicySet_pinnedPolicyRuntimeVersion(t *testing.T) {
 		CheckDestroy: testAccCheckTFEPolicySetDestroy,
 		Steps: []resource.TestStep{
 			{
-				Config: testAccTFEPolicySet_pinnedPolicyRuntimeVersion(org.Name, version, sha),
+				Config: testAccTFEPolicySet_pinnedPolicyRuntimeVersion(org.Name, tool.Version),
 				Check: resource.ComposeTestCheckFunc(
 					testAccCheckTFEPolicySetExists("tfe_policy_set.foobar", policySet),
 					testAccCheckTFEPolicySetAttributes(policySet),
@@ -1034,14 +1048,8 @@ resource "tfe_policy_set" "foobar" {
 }`, organization, organization)
 }
 
-func testAccTFEPolicySet_pinnedPolicyRuntimeVersion(organization string, version string, sha string) string {
+func testAccTFEPolicySet_pinnedPolicyRuntimeVersion(organization string, version string) string {
 	return fmt.Sprintf(`
-resource "tfe_sentinel_version" "foobar" {
-  version = "%s"
-  url = "https://www.hashicorp.com"
-  sha = "%s"
-}
-
 resource "tfe_sentinel_policy" "foo" {
   name         = "policy-foo"
   policy       = "main = rule { true }"
@@ -1055,7 +1063,7 @@ resource "tfe_policy_set" "foobar" {
   agent_enabled = true
   policy_tool_version = "%s"
   policy_ids   = [tfe_sentinel_policy.foo.id]
-}`, version, sha, organization, organization, version)
+}`, organization, organization, version)
 }
 
 func testAccTFEPolicySetOPA_basic(organization string, version string, sha string) string {

diff --git a/internal/provider/resource_tfe_registry_module_test.go b/internal/provider/resource_tfe_registry_module_test.go
@@ -942,6 +942,8 @@ func testAccCheckTFERegistryModuleDestroy(s *terraform.State) error {
 }
 
 func testAccPreCheckTFERegistryModule(t *testing.T) {
+	skipUnlessAfterDate(t, time.Date(2024, 5, 24, 0, 0, 0, 0, time.UTC))
+
 	if envGithubToken == "" {
 		t.Skip("Please set GITHUB_TOKEN to run this test")
 	}

diff --git a/internal/provider/resource_tfe_test_variable_test.go b/internal/provider/resource_tfe_test_variable_test.go
@@ -19,7 +19,10 @@ func TestAccTFETestVariable_basic(t *testing.T) {
 	rInt := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
 
 	resource.Test(t, resource.TestCase{
-		PreCheck:                 func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			testAccGithubPreCheck(t)
+		},
 		ProtoV5ProviderFactories: testAccMuxedProviders,
 		CheckDestroy:             testAccCheckTFETestVariableDestroy,
 		Steps: []resource.TestStep{
@@ -52,7 +55,10 @@ func TestAccTFETestVariable_update(t *testing.T) {
 	rInt := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
 
 	resource.Test(t, resource.TestCase{
-		PreCheck:                 func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			testAccGithubPreCheck(t)
+		},
 		ProtoV5ProviderFactories: testAccMuxedProviders,
 		CheckDestroy:             testAccCheckTFETestVariableDestroy,
 		Steps: []resource.TestStep{
@@ -199,7 +205,7 @@ resource "tfe_organization" "foobar" {
   name  = "tst-terraform-%d"
   email = "admin@company.com"
 }
-  
+
 resource "tfe_oauth_client" "foobar" {
   organization     = tfe_organization.foobar.name
   api_url          = "https://api.github.com"