Merge branches 'cluster-policies' and 'master' of github.com:databric…

…kslabs/databricks-terraform into cluster-policies

� Conflicts:
�	databricks/provider.go

.gitignore

@@ -329,3 +329,4 @@ website/public/**
 
 .vscode/private.env
 tf.log
+*.env

Makefile

@@ -39,25 +39,6 @@ fmt: lint
 	@echo "==> Formatting source code with gofmt..."
 	@go fmt ./...
 
-
-python-setup:
-	@echo "==> Setting up virtual env and installing python libraries..."
-	@python -m pip install virtualenv
-	@cd docs && python -m virtualenv venv
-	@cd docs && source venv/bin/activate && python -m pip install -r requirements.txt
-
-docs: python-setup
-	@echo "==> Building Docs ..."
-	@cd docs && source venv/bin/activate && make clean && make html
-
-opendocs: python-setup docs
-	@echo "==> Opening Docs ..."
-	@cd docs && open build/html/index.html
-
-singlehtmldocs: python-setup
-	@echo "==> Building Docs ..."
-	@cd docs && source venv/bin/activate && make clean && make singlehtml
-
 vendor:
 	@echo "==> Filling vendor folder with library code..."
 	@go mod vendor
@@ -70,7 +51,12 @@ terraform-acc-azure: fmt
 # INTEGRATION TESTING WITH AWS
 terraform-acc-aws: fmt
 	@echo "==> Running Terraform Acceptance Tests for AWS..."
-	@CLOUD_ENV="aws" TF_ACC=1 gotestsum --format short-verbose --raw-command go test -v -json -tags=aws  -short -coverprofile=coverage.out ./...
+	@CLOUD_ENV="aws" TF_ACC=1 gotestsum --format short-verbose --raw-command go test -v -json -short -coverprofile=coverage.out -run 'TestAccAws' ./...
+
+# INTEGRATION TESTING WITH AWS
+terraform-acc-mws: fmt
+	@echo "==> Running Terraform Acceptance Tests for Multiple Workspace APIs on AWS..."
+	@/bin/bash integration-environment-mws/run.sh
 
 terraform-setup: build
 	@echo "==> Initializing Terraform..."

client/model/mws.go

@@ -0,0 +1,110 @@
+package model
+
+// StsRole is the object that contains cross account role arn and external app id
+type StsRole struct {
+	RoleArn    string `json:"role_arn,omitempty"`
+	ExternalID string `json:"external_id,omitempty"`
+}
+
+// AwsCredentials is the object that points to the cross account role
+type AwsCredentials struct {
+	StsRole *StsRole `json:"sts_role,omitempty"`
+}
+
+// MWSCredentials is the object that contains all the information for the credentials to create a workspace
+type MWSCredentials struct {
+	CredentialsID   string          `json:"credentials_id,omitempty"`
+	CredentialsName string          `json:"credentials_name,omitempty"`
+	AwsCredentials  *AwsCredentials `json:"aws_credentials,omitempty"`
+	AccountID       string          `json:"account_id,omitempty"`
+	CreationTime    int64           `json:"creation_time,omitempty"`
+}
+
+// RootBucketInfo points to a bucket name
+type RootBucketInfo struct {
+	BucketName string `json:"bucket_name,omitempty"`
+}
+
+// MWSStorageConfigurations is the object that contains all the information for the root storage bucket
+type MWSStorageConfigurations struct {
+	StorageConfigurationID   string          `json:"storage_configuration_id,omitempty"`
+	StorageConfigurationName string          `json:"storage_configuration_name,omitempty"`
+	RootBucketInfo           *RootBucketInfo `json:"root_bucket_info,omitempty"`
+	AccountID                string          `json:"account_id,omitempty"`
+	CreationTime             int64           `json:"creation_time,omitempty"`
+}
+
+// NetworkHealth is the object that contains all the error message when attaching a network to workspace
+type NetworkHealth struct {
+	ErrorType    string `json:"error_type,omitempty"`
+	ErrorMessage string `json:"error_message,omitempty"`
+}
+
+// MWSNetwork is the object that contains all the information for BYOVPC
+type MWSNetwork struct {
+	NetworkID        string          `json:"network_id,omitempty"`
+	NetworkName      string          `json:"network_name,omitempty"`
+	VPCID            string          `json:"vpc_id,omitempty"`
+	SubnetIds        []string        `json:"subnet_ids,omitempty"`
+	SecurityGroupIds []string        `json:"security_group_ids,omitempty"`
+	VPCStatus        string          `json:"vpc_status,omitempty"`
+	ErrorMessages    []NetworkHealth `json:"error_messages,omitempty"`
+	WorkspaceID      int64           `json:"workspace_id,omitempty"`
+	AccountID        string          `json:"account_id,omitempty"`
+	CreationTime     int64           `json:"creation_time,omitempty"`
+}
+
+// AwsKeyInfo has information about the KMS key for BYOK
+type AwsKeyInfo struct {
+	KeyArn    string `json:"key_arn,omitempty"`
+	KeyAlias  string `json:"key_alias,omitempty"`
+	KeyRegion string `json:"key_region,omitempty"`
+}
+
+// MWSCustomerManagedKey contains key information and metadata for BYOK for E2
+type MWSCustomerManagedKey struct {
+	CustomerManagedKeyID string      `json:"customer_managed_key_id,omitempty"`
+	AwsKeyInfo           *AwsKeyInfo `json:"aws_key_info,omitempty"`
+	AccountID            string      `json:"account_id,omitempty"`
+	CreationTime         int64       `json:"creation_time,omitempty"`
+}
+
+// List of workspace statuses for provisioning the workspace
+const (
+	WorkspaceStatusNotProvisioned = "NOT_PROVISIONED"
+	WorkspaceStatusProvisioning   = "PROVISIONING"
+	WorkspaceStatusRunning        = "RUNNING"
+	WorkspaceStatusFailed         = "FAILED"
+	WorkspaceStatusCanceled       = "CANCELLED"
+)
+
+// WorkspaceStatusesNonRunnable is a list of statuses in which the workspace is not runnable
+var WorkspaceStatusesNonRunnable = []string{WorkspaceStatusCanceled, WorkspaceStatusFailed}
+
+// ContainsWorkspaceState given a list of workspaceStates and the search state
+// it will return true if it found the search state
+func ContainsWorkspaceState(workspaceStates []string, searchState string) bool {
+	for _, state := range workspaceStates {
+		if state == searchState {
+			return true
+		}
+	}
+	return false
+}
+
+// MWSWorkspace is the object that contains all the information for deploying a E2 workspace
+type MWSWorkspace struct {
+	WorkspaceID            int64  `json:"workspace_id,omitempty"`
+	WorkspaceName          string `json:"workspace_name,omitempty"`
+	DeploymentName         string `json:"deployment_name,omitempty"`
+	AwsRegion              string `json:"aws_region,omitempty"`
+	CredentialsID          string `json:"credentials_id,omitempty"`
+	StorageConfigurationID string `json:"storage_configuration_id,omitempty"`
+	NetworkID              string `json:"network_id,omitempty"`
+	CustomerManagedKeyID   string `json:"customer_managed_key_id,omitempty"`
+	IsNoPublicIpEnabled    bool   `json:"is_no_public_ip_enabled,omitempty"`
+	AccountID              string `json:"account_id,omitempty"`
+	WorkspaceStatus        string `json:"workspace_status,omitempty"`
+	WorkspaceStatusMessage string `json:"workspace_status_message,omitempty"`
+	CreationTime           int64  `json:"creation_time,omitempty"`
+}

client/service/apis.go

@@ -86,6 +86,31 @@ func (c *DBApiClient) Commands() CommandsAPI {
 	return CommandsAPI{Client: c}
 }
 
+// MWSCredentials returns an instance of MWSCredentialsAPI
+func (c *DBApiClient) MWSCredentials() MWSCredentialsAPI {
+	return MWSCredentialsAPI{Client: c}
+}
+
+// MWSStorageConfigurations returns an instance of MWSStorageConfigurationsAPI
+func (c *DBApiClient) MWSStorageConfigurations() MWSStorageConfigurationsAPI {
+	return MWSStorageConfigurationsAPI{Client: c}
+}
+
+// MWSWorkspaces returns an instance of MWSWorkspacesAPI
+func (c *DBApiClient) MWSWorkspaces() MWSWorkspacesAPI {
+	return MWSWorkspacesAPI{Client: c}
+}
+
+// MWSNetworks returns an instance of MWSNetworksAPI
+func (c *DBApiClient) MWSNetworks() MWSNetworksAPI {
+	return MWSNetworksAPI{Client: c}
+}
+
+// MWSCustomerManagedKeys returns an instance of MWSCustomerManagedKeysAPI
+func (c *DBApiClient) MWSCustomerManagedKeys() MWSCustomerManagedKeysAPI {
+	return MWSCustomerManagedKeysAPI{Client: c}
+}
+
 func (c *DBApiClient) performQuery(method, path string, apiVersion string, headers map[string]string, data interface{}, secretsMask *SecretsMask) ([]byte, error) {
 	err := c.Config.getOrCreateToken()
 	if err != nil {

client/service/main_test.go

@@ -1,6 +1,7 @@
 package service
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"github.com/joho/godotenv"
@@ -51,6 +52,18 @@ func GetIntegrationDBAPIClient() *DBApiClient {
 	return &c
 }
 
+func GetIntegrationMWSAPIClient() *DBApiClient {
+	var config DBApiClientConfig
+	tokenUnB64 := fmt.Sprintf("%s:%s", os.Getenv("DATABRICKS_USERNAME"), os.Getenv("DATABRICKS_PASSWORD"))
+	config.AuthType = BasicAuth
+	config.Token = base64.StdEncoding.EncodeToString([]byte(tokenUnB64))
+	config.Host = os.Getenv("DATABRICKS_MWS_HOST")
+
+	var c DBApiClient
+	c.SetConfig(&config)
+	return &c
+}
+
 func GetCloudInstanceType(c *DBApiClient) string {
 	if strings.Contains(c.Config.Host, "azure") {
 		return "Standard_DS3_v2"

client/service/mws_credentials.go

@@ -0,0 +1,77 @@
+package service
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/databrickslabs/databricks-terraform/client/model"
+	"net/http"
+)
+
+// MWSCredentialsAPI exposes the mws credentials API
+type MWSCredentialsAPI struct {
+	Client *DBApiClient
+}
+
+// Create creates a set of MWS Credentials for the cross account role
+func (a MWSCredentialsAPI) Create(mwsAcctId, credentialsName string, roleArn string) (model.MWSCredentials, error) {
+	var mwsCreds model.MWSCredentials
+
+	credentialsAPIPath := fmt.Sprintf("/accounts/%s/credentials", mwsAcctId)
+
+	mwsCredentialsRequest := model.MWSCredentials{
+		CredentialsName: credentialsName,
+		AwsCredentials: &model.AwsCredentials{
+			StsRole: &model.StsRole{
+				RoleArn: roleArn,
+			},
+		},
+	}
+
+	resp, err := a.Client.performQuery(http.MethodPost, credentialsAPIPath, "2.0", nil, mwsCredentialsRequest, nil)
+	if err != nil {
+		return mwsCreds, err
+	}
+
+	err = json.Unmarshal(resp, &mwsCreds)
+	return mwsCreds, err
+}
+
+// Read returns the credentials object along with metadata
+func (a MWSCredentialsAPI) Read(mwsAcctId, credentialsID string) (model.MWSCredentials, error) {
+	var mwsCreds model.MWSCredentials
+
+	credentialsAPIPath := fmt.Sprintf("/accounts/%s/credentials/%s", mwsAcctId, credentialsID)
+
+	resp, err := a.Client.performQuery(http.MethodGet, credentialsAPIPath, "2.0", nil, nil, nil)
+	if err != nil {
+		return mwsCreds, err
+	}
+
+	err = json.Unmarshal(resp, &mwsCreds)
+	return mwsCreds, err
+}
+
+// Delete deletes the credentials object given a credentials id
+func (a MWSCredentialsAPI) Delete(mwsAcctId, credentialsID string) error {
+
+	credentialsAPIPath := fmt.Sprintf("/accounts/%s/credentials/%s", mwsAcctId, credentialsID)
+
+	_, err := a.Client.performQuery(http.MethodDelete, credentialsAPIPath, "2.0", nil, nil, nil)
+
+	return err
+}
+
+// List lists all the available credentials object in the mws account
+func (a MWSCredentialsAPI) List(mwsAcctId string) ([]model.MWSCredentials, error) {
+	var mwsCredsList []model.MWSCredentials
+
+	credentialsAPIPath := fmt.Sprintf("/accounts/%s/credentials", mwsAcctId)
+
+	resp, err := a.Client.performQuery(http.MethodGet, credentialsAPIPath, "2.0", nil, nil, nil)
+	if err != nil {
+		return mwsCredsList, err
+	}
+
+	err = json.Unmarshal(resp, &mwsCredsList)
+	return mwsCredsList, err
+}

client/service/mws_credentials_integration_test.go

@@ -0,0 +1,30 @@
+package service
+
+import (
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestMWSCreds(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode.")
+	}
+	acctId := os.Getenv("DATABRICKS_MWS_ACCT_ID")
+	client := GetIntegrationMWSAPIClient()
+	credsList, err := client.MWSCredentials().List(acctId)
+	assert.NoError(t, err, err)
+	t.Log(credsList)
+
+	myCreds, err := client.MWSCredentials().Create(acctId, "sri-mws-terraform-automation-role", "arn:aws:iam::997819999999:role/sri-e2-terraform-automation-role")
+	assert.NoError(t, err, err)
+
+	myCredsFull, err := client.MWSCredentials().Read(acctId, myCreds.CredentialsID)
+	assert.NoError(t, err, err)
+	t.Log(myCredsFull.AwsCredentials.StsRole.ExternalID)
+
+	defer func() {
+		err = client.MWSCredentials().Delete(acctId, myCreds.CredentialsID)
+		assert.NoError(t, err, err)
+	}()
+}

client/service/mws_customer_managed_keys.go

@@ -0,0 +1,69 @@
+package service
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/databrickslabs/databricks-terraform/client/model"
+	"net/http"
+)
+
+// MWSCustomerManagedKeysAPI exposes the mws customerManagedKeys API
+type MWSCustomerManagedKeysAPI struct {
+	Client *DBApiClient
+}
+
+// Create creates a set of MWS CustomerManagedKeys for the BYOVPC
+func (a MWSCustomerManagedKeysAPI) Create(mwsAcctId, keyArn, keyAlias, keyRegion string) (model.MWSCustomerManagedKey, error) {
+	var mwsCustomerManagedKey model.MWSCustomerManagedKey
+
+	customerManagedKeysAPIPath := fmt.Sprintf("/accounts/%s/customer-managed-keys", mwsAcctId)
+	mwsCustomerManagedKeysRequest := model.MWSCustomerManagedKey{
+		AwsKeyInfo: &model.AwsKeyInfo{
+			KeyArn:    keyArn,
+			KeyAlias:  keyAlias,
+			KeyRegion: keyRegion,
+		},
+	}
+	resp, err := a.Client.performQuery(http.MethodPost, customerManagedKeysAPIPath, "2.0", nil, mwsCustomerManagedKeysRequest, nil)
+	if err != nil {
+		return mwsCustomerManagedKey, err
+	}
+	err = json.Unmarshal(resp, &mwsCustomerManagedKey)
+	return mwsCustomerManagedKey, err
+}
+
+// Read returns the customer managed key object along with metadata
+func (a MWSCustomerManagedKeysAPI) Read(mwsAcctId, customerManagedKeysID string) (model.MWSCustomerManagedKey, error) {
+	var mwsCustomerManagedKey model.MWSCustomerManagedKey
+	customerManagedKeysAPIPath := fmt.Sprintf("/accounts/%s/customer-managed-keys/%s", mwsAcctId, customerManagedKeysID)
+	resp, err := a.Client.performQuery(http.MethodGet, customerManagedKeysAPIPath, "2.0", nil, nil, nil)
+	if err != nil {
+		return mwsCustomerManagedKey, err
+	}
+	err = json.Unmarshal(resp, &mwsCustomerManagedKey)
+	return mwsCustomerManagedKey, err
+}
+
+// Delete deletes the customer managed key object given a network id
+func (a MWSCustomerManagedKeysAPI) Delete(customerManagedKeysID string) error {
+	//customerManagedKeysAPIPath := fmt.Sprintf("/accounts/%s/customer-managed-keys/%s", a.Client.Config.E2AcctID, customerManagedKeysID)
+	//_, err := a.Client.performQuery(http.MethodDelete, customerManagedKeysAPIPath, "2.0", nil, nil, nil)
+	//return err
+	return errors.New("delete is not yet supported")
+}
+
+// List lists all the available customer managed key objects in the mws account
+func (a MWSCustomerManagedKeysAPI) List(mwsAcctId string) ([]model.MWSCustomerManagedKey, error) {
+	var mwsCustomerManagedKeyList []model.MWSCustomerManagedKey
+
+	customerManagedKeysAPIPath := fmt.Sprintf("/accounts/%s/customer-managed-keys", mwsAcctId)
+
+	resp, err := a.Client.performQuery(http.MethodGet, customerManagedKeysAPIPath, "2.0", nil, nil, nil)
+	if err != nil {
+		return mwsCustomerManagedKeyList, err
+	}
+
+	err = json.Unmarshal(resp, &mwsCustomerManagedKeyList)
+	return mwsCustomerManagedKeyList, err
+}