Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Chore: run acceptance tests in parallel #314

Merged
merged 4 commits into from
Apr 3, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions env0/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,18 @@ import (

func Provider(version string) plugin.ProviderFunc {
return func() *schema.Provider {
apiKeyEnv := "ENV0_API_KEY"
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is required when running the provider unit test (which modified the env).
Unfortunately, there's is no good way to configure env variables when running tests in parallel.
E.g. the new t.SetEnv(...) method is not supported when running tests in parallel (maybe future golang versions will have better support for it).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It'd be helpful if you can add some comment around that so any other person that reaches this piece of logic will understand what's going on

apiSecretEnv := "ENV0_API_SECRET"

// version "TEST" is used for acceptance testing.
// Due to race conditions related to env variables:
// Using different env variables during testing prevetns the race conditions.
if version == "TEST" {
version = ""
apiKeyEnv = "ENV0_API_KEY_TEST"
apiSecretEnv = "ENV0_API_SECRET_TEST"
}

provider := &schema.Provider{
Schema: map[string]*schema.Schema{
"api_endpoint": {
Expand All @@ -24,14 +36,14 @@ func Provider(version string) plugin.ProviderFunc {
"api_key": {
Type: schema.TypeString,
Description: "env0 api key (https://developer.env0.com/docs/api/YXBpOjY4Njc2-env0-api#creating-an-api-key)",
DefaultFunc: schema.EnvDefaultFunc("ENV0_API_KEY", nil),
DefaultFunc: schema.EnvDefaultFunc(apiKeyEnv, nil),
Required: true,
Sensitive: true,
},
"api_secret": {
Type: schema.TypeString,
Description: "env0 api key secret",
DefaultFunc: schema.EnvDefaultFunc("ENV0_API_SECRET", nil),
DefaultFunc: schema.EnvDefaultFunc(apiSecretEnv, nil),
Required: true,
Sensitive: true,
},
Expand Down
55 changes: 24 additions & 31 deletions env0/provider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,55 +3,48 @@ package env0
import (
"context"
"fmt"
"os"
"strings"
"testing"

"github.com/env0/terraform-provider-env0/client"
"github.com/env0/terraform-provider-env0/utils"
"github.com/golang/mock/gomock"
"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
"os"
"strings"
"testing"
)

var (
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These two vars should not be global but per test. (Otherwise mocking doesn't work properly).

apiClientMock *client.MockApiClientInterface
ctrl *gomock.Controller
)

var testUnitProviders = map[string]func() (*schema.Provider, error){
"env0": func() (*schema.Provider, error) {
provider := Provider("")()
provider.ConfigureContextFunc = func(ctx context.Context, d *schema.ResourceData) (interface{}, diag.Diagnostics) {
return apiClientMock, nil
}
return provider, nil
},
}

func runUnitTest(t *testing.T, testCase resource.TestCase, mockFunc func(mockFunc *client.MockApiClientInterface)) {
os.Setenv("TF_ACC", "1")
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TF_ACC is required when running terraform acceptance tests.
(Will error if not set).
In addition, this actually uncovered several bugs. When this env variable is set, some errors turn into panics.

os.Setenv("ENV0_API_KEY", "value")
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the secret and key env variables must not be unset. (Specifically when running tests in parallel).

os.Setenv("ENV0_API_SECRET", "value")

testPattern := os.Getenv("TEST_PATTERN")
if testPattern != "" && !strings.Contains(t.Name(), testPattern) {
t.SkipNow()
return
}

testReporter := utils.TestReporter{T: t}
ctrl = gomock.NewController(&testReporter)
ctrl := gomock.NewController(&testReporter)
defer ctrl.Finish()

os.Setenv("ENV0_API_KEY", "value")
os.Setenv("ENV0_API_SECRET", "value")
defer os.Setenv("ENV0_API_KEY", "")
defer os.Setenv("ENV0_API_SECRET", "")

apiClientMock = client.NewMockApiClientInterface(ctrl)
apiClientMock := client.NewMockApiClientInterface(ctrl)
mockFunc(apiClientMock)

testCase.ProviderFactories = testUnitProviders
testCase.ProviderFactories = map[string]func() (*schema.Provider, error){
"env0": func() (*schema.Provider, error) {
provider := Provider("")()
provider.ConfigureContextFunc = func(ctx context.Context, d *schema.ResourceData) (interface{}, diag.Diagnostics) {
return apiClientMock, nil
}
return provider, nil
},
}
testCase.PreventPostDestroyRefresh = true
resource.UnitTest(&testReporter, testCase)
resource.ParallelTest(&testReporter, testCase)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This makes it all run in Parallel.

}

func TestProvider(t *testing.T) {
Expand Down Expand Up @@ -80,12 +73,12 @@ func testMissingEnvVar(t *testing.T, envVars map[string]string, expectedKey stri
defer os.Setenv(key, "")
}

diags := Provider("")().Validate(&terraform.ResourceConfig{})
diags := Provider("TEST")().Validate(&terraform.ResourceConfig{})
testExpectedProviderError(t, diags, expectedKey)
}

func testMissingConfig(t *testing.T, config map[string]interface{}, expectedKey string) {
diags := Provider("")().Validate(terraform.NewResourceConfigRaw(config))
diags := Provider("TEST")().Validate(terraform.NewResourceConfigRaw(config))
testExpectedProviderError(t, diags, expectedKey)
}

Expand All @@ -108,10 +101,10 @@ func TestMissingConfigurations(t *testing.T) {

envVarsTestCases := map[string]map[string]string{
expectedApiKeyConfig: {
"ENV0_API_SECRET": "value",
"ENV0_API_SECRET_TEST": "value",
},
expectedApiSecretConfig: {
"ENV0_API_KEY": "value",
"ENV0_API_KEY_TEST": "value",
},
}

Expand Down
19 changes: 11 additions & 8 deletions env0/resource_azure_credentials_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,15 +135,18 @@ func TestUnitAzureCredentialsResource(t *testing.T) {
})

t.Run("validate missing arguments", func(t *testing.T) {
missingArgumentsTestCases := []resource.TestCase{
missingArgumentTestCase(resourceType, resourceName, map[string]interface{}{}, "client_id"),
missingArgumentTestCase(resourceType, resourceName, map[string]interface{}{}, "client_secret"),
missingArgumentTestCase(resourceType, resourceName, map[string]interface{}{}, "subscription_id"),
missingArgumentTestCase(resourceType, resourceName, map[string]interface{}{}, "tenant_id"),
missingArgumentTestCase(resourceType, resourceName, map[string]interface{}{}, "name"),
arguments := []string{
"client_id",
"client_secret",
"subscription_id",
"tenant_id",
"name",
}
for _, testCase := range missingArgumentsTestCases {
runUnitTest(t, testCase, func(mock *client.MockApiClientInterface) {

for _, argument := range arguments {
tc := missingArgumentTestCase(resourceType, resourceName, map[string]interface{}{}, argument)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are several places with similar fixes. Doing it in the old manner causes a parallel test inside a parallel, which is not allowed in Golang.

t.Run("validate missing arrguments "+argument, func(t *testing.T) {
runUnitTest(t, tc, func(mock *client.MockApiClientInterface) {})
})
}
})
Expand Down
24 changes: 17 additions & 7 deletions env0/resource_environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,10 +185,10 @@ func setEnvironmentSchema(d *schema.ResourceData, environment client.Environment
d.Set("id", environment.Id)
d.Set("name", environment.Name)
d.Set("project_id", environment.ProjectId)
d.Set("workspace", environment.WorkspaceName)
d.Set("auto_deploy_by_custom_glob", environment.AutoDeployByCustomGlob)
d.Set("ttl", environment.LifespanEndAt)
d.Set("terragrunt_working_directory", environment.TerragruntWorkingDirectory)
safeSet(d, "workspace", environment.WorkspaceName)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor bug. These calls return errors (fields are not in the schema).
With TF_ACC it panics.

safeSet first checks if the field is in the schema.

safeSet(d, "auto_deploy_by_custom_glob", environment.AutoDeployByCustomGlob)
safeSet(d, "ttl", environment.LifespanEndAt)
safeSet(d, "terragrunt_working_directory", environment.TerragruntWorkingDirectory)
if environment.LatestDeploymentLog != (client.DeploymentLog{}) {
d.Set("template_id", environment.LatestDeploymentLog.BlueprintId)
d.Set("revision", environment.LatestDeploymentLog.BlueprintRevision)
Expand All @@ -209,11 +209,17 @@ func setEnvironmentSchema(d *schema.ResourceData, environment client.Environment
}

func setEnvironmentConfigurationSchema(d *schema.ResourceData, configurationVariables []client.ConfigurationVariable) {
for index, configurationVariable := range configurationVariables {
var variables []interface{}
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have no idea how this worked and how it had passed the acceptance tests.
Took my a while to figure out what was wrong and how to fix it.

(With TF_ACC it panics).


for _, configurationVariable := range configurationVariables {
variable := make(map[string]interface{})
variable["name"] = configurationVariable.Name
variable["value"] = configurationVariable.Value
variable["type"] = configurationVariable.Type
if configurationVariable.Type == nil || *configurationVariable.Type == 0 {
variable["type"] = "environment"
} else {
variable["type"] = "terraform"
}
if configurationVariable.Description != "" {
variable["description"] = configurationVariable.Description
}
Expand All @@ -225,7 +231,11 @@ func setEnvironmentConfigurationSchema(d *schema.ResourceData, configurationVari
variable["schema_enum"] = configurationVariable.Schema.Enum
variable["schema_format"] = configurationVariable.Schema.Format
}
d.Set(fmt.Sprintf(`configuration.%d`, index), variable)
variables = append(variables, variable)
}

if variables != nil {
d.Set("configuration", variables)
}
}

Expand Down
2 changes: 2 additions & 0 deletions env0/resource_environment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ func TestUnitEnvironmentResource(t *testing.T) {
}
autoDeployOnPathChangesOnlyDefault := true
autoDeployByCustomGlobDefault := ""

t.Run("Success in create", func(t *testing.T) {
testCase := resource.TestCase{
Steps: []resource.TestStep{
Expand Down Expand Up @@ -106,6 +107,7 @@ func TestUnitEnvironmentResource(t *testing.T) {

mock.EXPECT().EnvironmentDestroy(environment.Id).Times(1)
})

})

t.Run("Success in create and deploy with variables update", func(t *testing.T) {
Expand Down
7 changes: 6 additions & 1 deletion env0/resource_gcp_credentials_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ func TestUnitGcpCredentialsResource(t *testing.T) {
})

t.Run("validate missing arguments", func(t *testing.T) {

missingArgumentsTestCases := []resource.TestCase{
missingArgumentTestCase(resourceType, resourceName, map[string]interface{}{
"name": "update",
Expand All @@ -130,8 +131,12 @@ func TestUnitGcpCredentialsResource(t *testing.T) {
}, "name"),
}
for _, testCase := range missingArgumentsTestCases {
runUnitTest(t, testCase, func(mock *client.MockApiClientInterface) {
tc := testCase

t.Run("validate missing arguments", func(t *testing.T) {
runUnitTest(t, tc, func(mock *client.MockApiClientInterface) {})
})

}
})

Expand Down
10 changes: 8 additions & 2 deletions env0/resource_template_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -632,7 +632,10 @@ func TestUnitTemplateResource(t *testing.T) {
}

for _, testCase := range testCases {
runUnitTest(t, testCase, func(mockFunc *client.MockApiClientInterface) {})
tc := testCase
t.Run("Invalid retry times field", func(t *testing.T) {
runUnitTest(t, tc, func(mockFunc *client.MockApiClientInterface) {})
})
}
})

Expand All @@ -655,7 +658,10 @@ func TestUnitTemplateResource(t *testing.T) {
}

for _, testCase := range testCases {
runUnitTest(t, testCase, func(mockFunc *client.MockApiClientInterface) {})
tc := testCase
t.Run("Invalid retry regex field", func(t *testing.T) {
runUnitTest(t, tc, func(mockFunc *client.MockApiClientInterface) {})
})
}
})

Expand Down
7 changes: 7 additions & 0 deletions env0/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,3 +167,10 @@ func writeResourceData(i interface{}, d *schema.ResourceData) error {

return nil
}

func safeSet(d *schema.ResourceData, k string, v interface{}) {
// Checks that the key exist in the schema before setting the value.
if test := d.Get(k); test != nil {
d.Set(k, v)
}
}