Merge pull request #3 from databrickslabs/develop

Develop

.dockerignore

@@ -0,0 +1,3 @@
+.terraform
+docs
+examples

.gitignore

@@ -320,4 +320,5 @@ pip-selfcheck.json
 # Remove tfvars to prevent secrets from leaking and ignore any terraform files in root folder
 **/*.tfvars
 /*.tf
-.DS_Store
+.DS_Store
+/*.dbc

Dockerfile

@@ -0,0 +1,9 @@
+FROM golang:1.13-alpine
+WORKDIR /go/src/github.com/databrickslabs/databricks-terraform/
+COPY . .
+RUN apk add --update make
+RUN make build
+
+FROM hashicorp/terraform:latest
+COPY --from=0 /go/src/github.com/databrickslabs/databricks-terraform/terraform-provider-db /root/.terraform.d/plugins/
+RUN ls ~/.terraform.d/plugins

Makefile

@@ -29,11 +29,11 @@ python-setup:
 	@cd docs && python -m virtualenv venv
 	@cd docs && source venv/bin/activate && python -m pip install -r requirements.txt
 
-docs:
+docs: python-setup
 	@echo "==> Building Docs ..."
-	@cd docs && source venv/bin/activate && make html
+	@cd docs && source venv/bin/activate && make clean && make html
 
-opendocs: docs
+opendocs: python-setup docs
 	@echo "==> Opening Docs ..."
 	@cd docs && open build/html/index.html
 
@@ -42,6 +42,10 @@ vendor:
 	@go mod vendor
 
 # INTEGRATION TESTING WITH TERRAFORM EXAMPLES
+terraform-acc: fmt build
+	@echo "==> Running Terraform Acceptance Tests..."
+	@TF_ACC=1 go test -v -short ./db/...
+
 terraform-setup: fmt build
 	@echo "==> Initializing Terraform..."
 	@terraform init

README.md

@@ -80,7 +80,13 @@ This program built for i386-apple-darwin11.3.0
 * After installing `golang`, `terraform`, and `make` you will now build the artifact.
 
 ```bash
-$ go get -u github.com/databrickslabs/databricks-terraform && cd $GOPATH/src/github.com/databrickslabs/databricks-terraform 
+$ go get -v -u github.com/databrickslabs/databricks-terraform && cd $GOPATH/src/github.com/databrickslabs/databricks-terraform 
+```
+
+:warning: If you are fetching from a private repository please use the following command:
+
+```bash
+$ GOSUMDB=off GOPROXY=direct go get -v -u github.com/databrickslabs/databricks-terraform && cd $GOPATH/src/github.com/databrickslabs/databricks-terraform
 ```
 
 * When you are in the root directory of the repository please run:
@@ -104,7 +110,7 @@ Now your plugin for the Databricks Terraform provider is installed correctly. Yo
 
 Sample terraform code
 
-```hcl-terraform
+```terraform
 provider "db" {
   host = "http://databrickshost.com"
   token = "dapitokenhere"
@@ -123,6 +129,30 @@ Please refer to the detailed documentation provided in the html documentation fo
 
 Also refer to these [examples](examples/) for more scenarios. 
 
+
+### Docker commands
+
+To install and build the code if you dont want to install golang, terraform, etc. All you need is docker and git.
+
+First make sure you clone the repository and you are in the directory.
+
+Then build the docker image with this command:
+
+```bash
+$ docker build -t databricks-terraform . 
+```
+
+Then run the execute the terraform binary via the following command and volume mount. Make sure that you are in the directory
+ with the terraform code. The following command you can execute the following commands and additional ones as part of 
+ the terraform binary.
+
+```bash
+$ docker run -it -v $(pwd):/workpace -w /workpace databricks-terraform init
+$ docker run -it -v $(pwd):/workpace -w /workpace databricks-terraform plan
+$ docker run -it -v $(pwd):/workpace -w /workpace databricks-terraform apply
+
+```
+
 ## Project Components
 
 ### High Level Databricks Client CR[U]D
@@ -245,21 +275,39 @@ Note: For updating a user use Patch for entitlements, Put for everything else. W
 
 ### Databricks terraform provider resources
 
+* [x] Instance pools
+* [x] Scim groups
+* [x] Scim users
+* [x] Secret scopes
+* [x] Secrets
+* [x] Secret acls
+* [x] Jobs
+* [x] Clusters + Libraries
+* [x] IAM Instance Profiles
+* [x] Notebooks
+* [x] DBFS File
+* [x] DBFS File Sync (stateful sync from src to target)
+* [ ] Cluster Policies (private preview)
+* [ ] Entitlements (private preview)
+
+### Databricks terraform provider docs
+
 * [x] Instance pools
 * [x] Scim groups
 * [x] Scim users
 * [x] Secret scopes
 * [x] Secrets
 * [x] Secret acls
 * [ ] Jobs
-* [ ] Clusters (EC2 based example)
+* [ ] Clusters + Libraries
 * [ ] IAM Instance Profiles
 * [ ] Notebooks
-* [ ] DBFS
-* [ ] Libraries 
+* [ ] DBFS File
+* [ ] DBFS File Sync (stateful sync from src to target)
 * [ ] Cluster Policies (private preview)
 * [ ] Entitlements (private preview)
 
+
 ## Testing
 
 * [ ] Integration tests should be run at a client level against both azure and aws to maintain sdk parity against both apis **(currently only on one cloud)**

client/client.go

@@ -96,6 +96,14 @@ func (o *DBClientOption) getRequestURI(path string, apiVersion string) (string,
 	return requestURI, nil
 }
 
+func onlyNBytes(buf []byte, numBytes int64) []byte {
+	if len(buf) > int(numBytes) {
+		return buf[:numBytes]
+	} else {
+		return buf
+	}
+}
+
 // PerformQuery can be used in a client or directly
 func PerformQuery(option DBClientOption, method, path string, apiVersion string, headers map[string]string, marshalJson bool, useRawPath bool, data interface{}) ([]byte, error) {
 
@@ -133,7 +141,7 @@ func PerformQuery(option DBClientOption, method, path string, apiVersion string,
 			}
 
 			requestBody = bodyBytes
-			log.Println(string(requestBody))
+			log.Println(string(onlyNBytes(requestBody, 1e3)))
 		} else {
 			requestBody = []byte(data.(string))
 		}

client/model/cluster.go

@@ -86,6 +86,11 @@ func ContainsClusterState(clusterStates []ClusterState, searchState ClusterState
 	return false
 }
 
+type ZonesInfo struct {
+	Zones       []string `json:"zones,omitempty"`
+	DefaultZone string   `json:"default_zone,omitempty"`
+}
+
 type AwsAttributes struct {
 	FirstOnDemand       int32           `json:"first_on_demand,omitempty"`
 	Availability        AwsAvailability `json:"availability,omitempty"`
@@ -198,21 +203,31 @@ type NodeType struct {
 	NodeInfo       *ClusterCloudProviderNodeInfo `json:"node_info,omitempty"`
 }
 
+type DockerBasicAuth struct {
+	Username string `json:"username,omitempty"`
+	Password string `json:"password,omitempty"`
+}
+
+type DockerImage struct {
+	Url       string           `json:"url,omitempty"`
+	BasicAuth *DockerBasicAuth `json:"basic_auth,omitempty"`
+}
+
 type Cluster struct {
-	ClusterId        string            `json:"cluster_id,omitempty"`
-	NumWorkers       int32             `json:"num_workers,omitempty"`
-	Autoscale        *AutoScale        `json:"autoscale,omitempty"`
-	ClusterName      string            `json:"cluster_name,omitempty"`
-	SparkVersion     string            `json:"spark_version,omitempty"`
-	SparkConf        map[string]string `json:"spark_conf,omitempty"`
-	AwsAttributes    *AwsAttributes    `json:"aws_attributes,omitempty"`
-	NodeTypeID       string            `json:"node_type_id,omitempty"`
-	DriverNodeTypeID string            `json:"driver_node_type_id,omitempty"`
-	SSHPublicKeys    []string          `json:"ssh_public_keys,omitempty"`
-	CustomTags       map[string]string `json:"custom_tags,omitempty"`
-	ClusterLogConf   *StorageInfo      `json:"cluster_log_conf,omitempty"`
-	InitScripts      []StorageInfo     `json:"init_scripts,omitempty"`
-	//TODO: Update docker image portion here
+	ClusterId              string            `json:"cluster_id,omitempty"`
+	NumWorkers             int32             `json:"num_workers,omitempty"`
+	Autoscale              *AutoScale        `json:"autoscale,omitempty"`
+	ClusterName            string            `json:"cluster_name,omitempty"`
+	SparkVersion           string            `json:"spark_version,omitempty"`
+	SparkConf              map[string]string `json:"spark_conf,omitempty"`
+	AwsAttributes          *AwsAttributes    `json:"aws_attributes,omitempty"`
+	NodeTypeID             string            `json:"node_type_id,omitempty"`
+	DriverNodeTypeID       string            `json:"driver_node_type_id,omitempty"`
+	SSHPublicKeys          []string          `json:"ssh_public_keys,omitempty"`
+	CustomTags             map[string]string `json:"custom_tags,omitempty"`
+	ClusterLogConf         *StorageInfo      `json:"cluster_log_conf,omitempty"`
+	InitScripts            []StorageInfo     `json:"init_scripts,omitempty"`
+	DockerImage            *DockerImage      `json:"docker_image,omitempty"`
 	SparkEnvVars           map[string]string `json:"spark_env_vars,omitempty"`
 	AutoterminationMinutes int32             `json:"autotermination_minutes,omitempty"`
 	EnableElasticDisk      bool              `json:"enable_elastic_disk,omitempty"`
@@ -242,7 +257,9 @@ type ClusterInfo struct {
 	SparkEnvVars           map[string]string  `json:"spark_env_vars,omitempty"`
 	AutoterminationMinutes int32              `json:"autotermination_minutes,omitempty"`
 	EnableElasticDisk      bool               `json:"enable_elastic_disk,omitempty"`
+	InstancePoolId         string             `json:"instance_pool_id,omitempty"`
 	ClusterSource          AwsAvailability    `json:"cluster_source,omitempty"`
+	DockerImage            *DockerImage       `json:"docker_image,omitempty"`
 	State                  ClusterState       `json:"state,omitempty"`
 	StateMessage           string             `json:"state_message,omitempty"`
 	StartTime              int64              `json:"start_time,omitempty"`
@@ -251,7 +268,7 @@ type ClusterInfo struct {
 	LastActivityTime       int64              `json:"last_activity_time,omitempty"`
 	ClusterMemoryMb        int64              `json:"cluster_memory_mb,omitempty"`
 	ClusterCores           float32            `json:"cluster_cores,omitempty"`
-	DefaultTags            map[string]string  `json:"DefaultTags,omitempty"`
+	DefaultTags            map[string]string  `json:"default_tags,omitempty"`
 	ClusterLogStatus       *LogSyncStatus     `json:"cluster_log_status,omitempty"`
 	TerminationReason      *TerminationReason `json:"termination_reason,omitempty"`
 }

client/model/command.go

@@ -2,6 +2,8 @@ package model
 
 type CommandResults struct {
 	ResultType   string      `json:"resultType,omitempty"`
+	Summary      string      `json:"summary,omitempty"`
+	Cause        string      `json:"cause,omitempty"`
 	Data         interface{} `json:"data,omitempty"`
 	Schema       interface{} `json:"schema,omitempty"`
 	Truncated    bool        `json:"truncated,omitempty"`
@@ -13,3 +15,9 @@ type Command struct {
 	Status  string          `json:"status,omitempty"`
 	Results *CommandResults `json:"results,omitempty"`
 }
+
+type ExecutionContext struct {
+	ContextId string   `json:"contextId,omitempty"`
+	ClusterId string   `json:"clusterId,omitempty"`
+	Language  Language `json:"language,omitempty"`
+}

client/model/dbfs.go

@@ -0,0 +1,7 @@
+package model
+
+type FileInfo struct {
+	Path     string `json:"path,omitempty"`
+	IsDir    bool   `json:"is_dir,omitempty"`
+	FileSize int64  `json:"file_size,omitempty"`
+}

client/model/instance_pool.go

@@ -2,7 +2,7 @@ package model
 
 type InstancePoolAwsAttributes struct {
 	Availability        AwsAvailability `json:"availability,omitempty"`
-	ZoneId              string          `json:"zone_id,omitempty"`
+	ZoneID              string          `json:"zone_id,omitempty"`
 	SpotBidPricePercent int32           `json:"spot_bid_price_percent,omitempty"`
 }
 

client/model/instance_profile.go

@@ -0,0 +1,5 @@
+package model
+
+type InstanceProfileInfo struct {
+	InstanceProfileArn string `json:"instance_profile_arn,omitempty"`
+}

client/model/job.go

@@ -0,0 +1,60 @@
+package model
+
+//go:generate easytags $GOFILE
+
+type NotebookTask struct {
+	NotebookPath   string            `json:"notebook_path,omitempty"`
+	BaseParameters map[string]string `json:"base_parameters,omitempty"`
+}
+
+type SparkPythonTask struct {
+	PythonFile string   `json:"python_file,omitempty"`
+	Parameters []string `json:"parameters,omitempty"`
+}
+
+type SparkJarTask struct {
+	JarUri        string   `json:"jar_uri,omitempty"`
+	MainClassName string   `json:"main_class_name,omitempty"`
+	Parameters    []string `json:"parameters,omitempty"`
+}
+
+type SparkSubmitTask struct {
+	Parameters []string `json:"parameters,omitempty"`
+}
+
+type JobEmailNotifications struct {
+	OnStart               []string `json:"on_start,omitempty"`
+	OnSuccess             []string `json:"on_success,omitempty"`
+	OnFailure             []string `json:"on_failure,omitempty"`
+	NoAlertForSkippedRuns bool     `json:"no_alert_for_skipped_runs,omitempty"`
+}
+
+type CronSchedule struct {
+	QuartzCronExpression string `json:"quartz_cron_expression,omitempty"`
+	TimezoneId           string `json:"timezone_id,omitempty"`
+}
+
+type JobSettings struct {
+	ExistingClusterId      string                 `json:"existing_cluster_id,omitempty"`
+	NewCluster             *Cluster               `json:"new_cluster,omitempty"`
+	NotebookTask           *NotebookTask          `json:"notebook_task,omitempty"`
+	SparkJarTask           *SparkJarTask          `json:"spark_jar_task,omitempty"`
+	SparkPythonTask        *SparkPythonTask       `json:"spark_python_task,omitempty"`
+	SparkSubmitTask        *SparkSubmitTask       `json:"spark_submit_task,omitempty"`
+	Name                   string                 `json:"name,omitempty"`
+	Libraries              []Library              `json:"libraries,omitempty"`
+	EmailNotifications     *JobEmailNotifications `json:"email_notifications,omitempty"`
+	TimeoutSeconds         int32                  `json:"timeout_seconds,omitempty"`
+	MaxRetries             int32                  `json:"max_retries,omitempty"`
+	MinRetryIntervalMillis int32                  `json:"max_retry_interval_millis,omitempty"`
+	RetryOnTimeout         bool                   `json:"retry_on_timeout,omitempty"`
+	Schedule               *CronSchedule          `json:"schedule,omitempty"`
+	MaxConcurrentRuns      int32                  `json:"max_concurrent_runs,omitempty"`
+}
+
+type Job struct {
+	JobId           int64        `json:"job_id,omitempty"`
+	CreatorUserName string       `json:"creator_user_name,omitempty"`
+	Settings        *JobSettings `json:"settings,omitempty"`
+	CreatedTime     int64        `json:"created_time,omitempty"`
+}

client/model/library.go

@@ -0,0 +1,33 @@
+package model
+
+type PyPi struct {
+	Package string `json:"package,omitempty"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type Maven struct {
+	Coordinates string   `json:"coordinates,omitempty"`
+	Repo        string   `json:"repo,omitempty"`
+	Exclusions  []string `json:"exclusions,omitempty"`
+}
+
+type Cran struct {
+	Package string `json:"package,omitempty"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type Library struct {
+	Jar   string `json:"jar,omitempty"`
+	Egg   string `json:"egg,omitempty"`
+	Whl   string `json:"whl,omitempty"`
+	Pypi  *PyPi  `json:"pypi,omitempty"`
+	Maven *Maven `json:"maven,omitempty"`
+	Cran  *Cran  `json:"cran,omitempty"`
+}
+
+type LibraryStatus struct {
+	Library                         *Library `json:"library,omitempty"`
+	Status                          string   `json:"status,omitempty"`
+	IsLibraryInstalledOnAllClusters bool     `json:"is_library_for_all_clusters,omitempty"`
+	Messages                        []string `json:"messages,omitempty"`
+}