Merge branch 'master' of github.com:databrickslabs/databricks-terrafo…

…rm into fix-job-new-cluster

.gitignore

@@ -329,3 +329,4 @@ website/public/**
 
 .vscode/private.env
 tf.log
+*.env

Dockerfile

@@ -3,7 +3,7 @@ WORKDIR /go/src/github.com/databrickslabs/databricks-terraform/
 RUN curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v0.4.2/gotestsum_0.4.2_linux_amd64.tar.gz" | tar -xz -C /usr/local/bin gotestsum
 RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.25.0
 COPY . .
-RUN make vendor build
+RUN CGO_ENABLED=0 make vendor build
 
 FROM hashicorp/terraform:latest
 COPY --from=0 /go/src/github.com/databrickslabs/databricks-terraform/terraform-provider-databricks /root/.terraform.d/plugins/

Makefile

@@ -39,25 +39,6 @@ fmt: lint
 	@echo "==> Formatting source code with gofmt..."
 	@go fmt ./...
 
-
-python-setup:
-	@echo "==> Setting up virtual env and installing python libraries..."
-	@python -m pip install virtualenv
-	@cd docs && python -m virtualenv venv
-	@cd docs && source venv/bin/activate && python -m pip install -r requirements.txt
-
-docs: python-setup
-	@echo "==> Building Docs ..."
-	@cd docs && source venv/bin/activate && make clean && make html
-
-opendocs: python-setup docs
-	@echo "==> Opening Docs ..."
-	@cd docs && open build/html/index.html
-
-singlehtmldocs: python-setup
-	@echo "==> Building Docs ..."
-	@cd docs && source venv/bin/activate && make clean && make singlehtml
-
 vendor:
 	@echo "==> Filling vendor folder with library code..."
 	@go mod vendor
@@ -70,7 +51,12 @@ terraform-acc-azure: fmt
 # INTEGRATION TESTING WITH AWS
 terraform-acc-aws: fmt
 	@echo "==> Running Terraform Acceptance Tests for AWS..."
-	@CLOUD_ENV="aws" TF_ACC=1 gotestsum --format short-verbose --raw-command go test -v -json -tags=aws  -short -coverprofile=coverage.out ./...
+	@CLOUD_ENV="aws" TF_ACC=1 gotestsum --format short-verbose --raw-command go test -v -json -short -coverprofile=coverage.out -run 'TestAccAws' ./...
+
+# INTEGRATION TESTING WITH AWS
+terraform-acc-mws: fmt
+	@echo "==> Running Terraform Acceptance Tests for Multiple Workspace APIs on AWS..."
+	@/bin/bash integration-environment-mws/run.sh
 
 terraform-setup: build
 	@echo "==> Initializing Terraform..."

client/model/mws.go

@@ -0,0 +1,110 @@
+package model
+
+// StsRole is the object that contains cross account role arn and external app id
+type StsRole struct {
+	RoleArn    string `json:"role_arn,omitempty"`
+	ExternalID string `json:"external_id,omitempty"`
+}
+
+// AwsCredentials is the object that points to the cross account role
+type AwsCredentials struct {
+	StsRole *StsRole `json:"sts_role,omitempty"`
+}
+
+// MWSCredentials is the object that contains all the information for the credentials to create a workspace
+type MWSCredentials struct {
+	CredentialsID   string          `json:"credentials_id,omitempty"`
+	CredentialsName string          `json:"credentials_name,omitempty"`
+	AwsCredentials  *AwsCredentials `json:"aws_credentials,omitempty"`
+	AccountID       string          `json:"account_id,omitempty"`
+	CreationTime    int64           `json:"creation_time,omitempty"`
+}
+
+// RootBucketInfo points to a bucket name
+type RootBucketInfo struct {
+	BucketName string `json:"bucket_name,omitempty"`
+}
+
+// MWSStorageConfigurations is the object that contains all the information for the root storage bucket
+type MWSStorageConfigurations struct {
+	StorageConfigurationID   string          `json:"storage_configuration_id,omitempty"`
+	StorageConfigurationName string          `json:"storage_configuration_name,omitempty"`
+	RootBucketInfo           *RootBucketInfo `json:"root_bucket_info,omitempty"`
+	AccountID                string          `json:"account_id,omitempty"`
+	CreationTime             int64           `json:"creation_time,omitempty"`
+}
+
+// NetworkHealth is the object that contains all the error message when attaching a network to workspace
+type NetworkHealth struct {
+	ErrorType    string `json:"error_type,omitempty"`
+	ErrorMessage string `json:"error_message,omitempty"`
+}
+
+// MWSNetwork is the object that contains all the information for BYOVPC
+type MWSNetwork struct {
+	NetworkID        string          `json:"network_id,omitempty"`
+	NetworkName      string          `json:"network_name,omitempty"`
+	VPCID            string          `json:"vpc_id,omitempty"`
+	SubnetIds        []string        `json:"subnet_ids,omitempty"`
+	SecurityGroupIds []string        `json:"security_group_ids,omitempty"`
+	VPCStatus        string          `json:"vpc_status,omitempty"`
+	ErrorMessages    []NetworkHealth `json:"error_messages,omitempty"`
+	WorkspaceID      int64           `json:"workspace_id,omitempty"`
+	AccountID        string          `json:"account_id,omitempty"`
+	CreationTime     int64           `json:"creation_time,omitempty"`
+}
+
+// AwsKeyInfo has information about the KMS key for BYOK
+type AwsKeyInfo struct {
+	KeyArn    string `json:"key_arn,omitempty"`
+	KeyAlias  string `json:"key_alias,omitempty"`
+	KeyRegion string `json:"key_region,omitempty"`
+}
+
+// MWSCustomerManagedKey contains key information and metadata for BYOK for E2
+type MWSCustomerManagedKey struct {
+	CustomerManagedKeyID string      `json:"customer_managed_key_id,omitempty"`
+	AwsKeyInfo           *AwsKeyInfo `json:"aws_key_info,omitempty"`
+	AccountID            string      `json:"account_id,omitempty"`
+	CreationTime         int64       `json:"creation_time,omitempty"`
+}
+
+// List of workspace statuses for provisioning the workspace
+const (
+	WorkspaceStatusNotProvisioned = "NOT_PROVISIONED"
+	WorkspaceStatusProvisioning   = "PROVISIONING"
+	WorkspaceStatusRunning        = "RUNNING"
+	WorkspaceStatusFailed         = "FAILED"
+	WorkspaceStatusCanceled       = "CANCELLED"
+)
+
+// WorkspaceStatusesNonRunnable is a list of statuses in which the workspace is not runnable
+var WorkspaceStatusesNonRunnable = []string{WorkspaceStatusCanceled, WorkspaceStatusFailed}
+
+// ContainsWorkspaceState given a list of workspaceStates and the search state
+// it will return true if it found the search state
+func ContainsWorkspaceState(workspaceStates []string, searchState string) bool {
+	for _, state := range workspaceStates {
+		if state == searchState {
+			return true
+		}
+	}
+	return false
+}
+
+// MWSWorkspace is the object that contains all the information for deploying a E2 workspace
+type MWSWorkspace struct {
+	WorkspaceID            int64  `json:"workspace_id,omitempty"`
+	WorkspaceName          string `json:"workspace_name,omitempty"`
+	DeploymentName         string `json:"deployment_name,omitempty"`
+	AwsRegion              string `json:"aws_region,omitempty"`
+	CredentialsID          string `json:"credentials_id,omitempty"`
+	StorageConfigurationID string `json:"storage_configuration_id,omitempty"`
+	NetworkID              string `json:"network_id,omitempty"`
+	CustomerManagedKeyID   string `json:"customer_managed_key_id,omitempty"`
+	IsNoPublicIpEnabled    bool   `json:"is_no_public_ip_enabled,omitempty"`
+	AccountID              string `json:"account_id,omitempty"`
+	WorkspaceStatus        string `json:"workspace_status,omitempty"`
+	WorkspaceStatusMessage string `json:"workspace_status_message,omitempty"`
+	CreationTime           int64  `json:"creation_time,omitempty"`
+}

client/service/apis.go

@@ -17,22 +17,22 @@ func (c *DBApiClient) SetConfig(clientConfig *DBApiClientConfig) DBApiClient {
 }
 
 // Clusters returns an instance of ClustersAPI
-func (c DBApiClient) Clusters() ClustersAPI {
+func (c *DBApiClient) Clusters() ClustersAPI {
 	return ClustersAPI{Client: c}
 }
 
 // Secrets returns an instance of SecretsAPI
-func (c DBApiClient) Secrets() SecretsAPI {
+func (c *DBApiClient) Secrets() SecretsAPI {
 	return SecretsAPI{Client: c}
 }
 
 // SecretScopes returns an instance of SecretScopesAPI
-func (c DBApiClient) SecretScopes() SecretScopesAPI {
+func (c *DBApiClient) SecretScopes() SecretScopesAPI {
 	return SecretScopesAPI{Client: c}
 }
 
 // SecretAcls returns an instance of SecretAclsAPI
-func (c DBApiClient) SecretAcls() SecretAclsAPI {
+func (c *DBApiClient) SecretAcls() SecretAclsAPI {
 	return SecretAclsAPI{Client: c}
 }
 
@@ -42,50 +42,79 @@ func (c *DBApiClient) Tokens() TokensAPI {
 }
 
 // Users returns an instance of UsersAPI
-func (c DBApiClient) Users() UsersAPI {
+func (c *DBApiClient) Users() UsersAPI {
 	return UsersAPI{Client: c}
 }
 
 // Groups returns an instance of GroupsAPI
-func (c DBApiClient) Groups() GroupsAPI {
+func (c *DBApiClient) Groups() GroupsAPI {
 	return GroupsAPI{Client: c}
 }
 
 // Notebooks returns an instance of NotebooksAPI
-func (c DBApiClient) Notebooks() NotebooksAPI {
+func (c *DBApiClient) Notebooks() NotebooksAPI {
 	return NotebooksAPI{Client: c}
 }
 
 // Jobs returns an instance of JobsAPI
-func (c DBApiClient) Jobs() JobsAPI {
+func (c *DBApiClient) Jobs() JobsAPI {
 	return JobsAPI{Client: c}
 }
 
 // DBFS returns an instance of DBFSAPI
-func (c DBApiClient) DBFS() DBFSAPI {
+func (c *DBApiClient) DBFS() DBFSAPI {
 	return DBFSAPI{Client: c}
 }
 
 // Libraries returns an instance of LibrariesAPI
-func (c DBApiClient) Libraries() LibrariesAPI {
+func (c *DBApiClient) Libraries() LibrariesAPI {
 	return LibrariesAPI{Client: c}
 }
 
 // InstancePools returns an instance of InstancePoolsAPI
-func (c DBApiClient) InstancePools() InstancePoolsAPI {
+func (c *DBApiClient) InstancePools() InstancePoolsAPI {
 	return InstancePoolsAPI{Client: c}
 }
 
 // InstanceProfiles returns an instance of InstanceProfilesAPI
-func (c DBApiClient) InstanceProfiles() InstanceProfilesAPI {
+func (c *DBApiClient) InstanceProfiles() InstanceProfilesAPI {
 	return InstanceProfilesAPI{Client: c}
 }
 
 // Commands returns an instance of CommandsAPI
-func (c DBApiClient) Commands() CommandsAPI {
+func (c *DBApiClient) Commands() CommandsAPI {
 	return CommandsAPI{Client: c}
 }
 
-func (c DBApiClient) performQuery(method, path string, apiVersion string, headers map[string]string, data interface{}, secretsMask *SecretsMask) ([]byte, error) {
+// MWSCredentials returns an instance of MWSCredentialsAPI
+func (c *DBApiClient) MWSCredentials() MWSCredentialsAPI {
+	return MWSCredentialsAPI{Client: c}
+}
+
+// MWSStorageConfigurations returns an instance of MWSStorageConfigurationsAPI
+func (c *DBApiClient) MWSStorageConfigurations() MWSStorageConfigurationsAPI {
+	return MWSStorageConfigurationsAPI{Client: c}
+}
+
+// MWSWorkspaces returns an instance of MWSWorkspacesAPI
+func (c *DBApiClient) MWSWorkspaces() MWSWorkspacesAPI {
+	return MWSWorkspacesAPI{Client: c}
+}
+
+// MWSNetworks returns an instance of MWSNetworksAPI
+func (c *DBApiClient) MWSNetworks() MWSNetworksAPI {
+	return MWSNetworksAPI{Client: c}
+}
+
+// MWSCustomerManagedKeys returns an instance of MWSCustomerManagedKeysAPI
+func (c *DBApiClient) MWSCustomerManagedKeys() MWSCustomerManagedKeysAPI {
+	return MWSCustomerManagedKeysAPI{Client: c}
+}
+
+func (c *DBApiClient) performQuery(method, path string, apiVersion string, headers map[string]string, data interface{}, secretsMask *SecretsMask) ([]byte, error) {
+	err := c.Config.getOrCreateToken()
+	if err != nil {
+		return []byte(""), err
+	}
 	return PerformQuery(c.Config, method, path, apiVersion, headers, true, false, data, secretsMask)
 }

client/service/client.go

@@ -12,6 +12,7 @@ import (
 	"net/url"
 	"reflect"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/google/go-querystring/query"
@@ -56,6 +57,8 @@ const (
 	BasicAuth AuthType = "BASIC"
 )
 
+var clientAuthorizerMutex sync.Mutex
+
 // DBApiClientConfig is used to configure the DataBricks Client
 type DBApiClientConfig struct {
 	Host               string
@@ -65,12 +68,14 @@ type DBApiClientConfig struct {
 	DefaultHeaders     map[string]string
 	InsecureSkipVerify bool
 	TimeoutSeconds     int
+	CustomAuthorizer   func(*DBApiClientConfig) error
 	client             *retryablehttp.Client
 }
 
 var transientErrorStringMatches []string = []string{ // TODO: Should we make these regexes to match more of the message or is this sufficient?
 	"com.databricks.backend.manager.util.UnknownWorkerEnvironmentException",
 	"does not have any associated worker environments",
+	"There is no worker environment with id",
 }
 
 // Setup initializes the client
@@ -123,7 +128,6 @@ func checkHTTPRetry(ctx context.Context, resp *http.Response, err error) (bool,
 		if err != nil {
 			return false, err
 		}
-
 		var errorBody DBApiErrorBody
 		err = json.Unmarshal(body, &errorBody)
 		if err != nil {
@@ -145,7 +149,22 @@ func checkHTTPRetry(ctx context.Context, resp *http.Response, err error) (bool,
 	return false, nil
 }
 
-func (c *DBApiClientConfig) getAuthHeader() map[string]string {
+func (c *DBApiClientConfig) getOrCreateToken() error {
+	if c.CustomAuthorizer != nil {
+		// Lock incase terraform tries to getOrCreateToken from multiple go routines on the same client ptr.
+		clientAuthorizerMutex.Lock()
+		defer clientAuthorizerMutex.Unlock()
+		if reflect.ValueOf(c.Token).IsZero() {
+			log.Println("NOT AUTHORIZED SO ATTEMPTING TO AUTHORIZE")
+			return c.CustomAuthorizer(c)
+		}
+		log.Println("ALREADY AUTHORIZED")
+		return nil
+	}
+	return nil
+}
+
+func (c DBApiClientConfig) getAuthHeader() map[string]string {
 	auth := make(map[string]string)
 	if c.AuthType == BasicAuth {
 		auth["Authorization"] = "Basic " + c.Token

client/service/clusters.go

@@ -11,7 +11,7 @@ import (
 
 // ClustersAPI is a struct that contains the Databricks api client to perform queries
 type ClustersAPI struct {
-	Client DBApiClient
+	Client *DBApiClient
 }
 
 // Create creates a new Spark cluster

client/service/commands.go

@@ -12,7 +12,7 @@ import (
 
 // CommandsAPI exposes the Context & Commands API
 type CommandsAPI struct {
-	Client DBApiClient
+	Client *DBApiClient
 }
 
 // Execute creates a spark context and executes a command and then closes context

client/service/dbfs.go

@@ -10,7 +10,7 @@ import (
 
 // DBFSAPI exposes the DBFS API
 type DBFSAPI struct {
-	Client DBApiClient
+	Client *DBApiClient
 }
 
 // Create creates a file in DBFS given data string in base64

client/service/groups.go

@@ -12,7 +12,7 @@ import (
 
 // GroupsAPI exposes the scim groups API
 type GroupsAPI struct {
-	Client DBApiClient
+	Client *DBApiClient
 }
 
 // Create creates a scim group in the Databricks workspace

client/service/instance_pools.go

@@ -8,7 +8,7 @@ import (
 
 // InstancePoolsAPI exposes the instance pools api
 type InstancePoolsAPI struct {
-	Client DBApiClient
+	Client *DBApiClient
 }
 
 // Create creates the instance pool to given the instance pool configuration