Allow init scripts on ADLS in databricks_cluster and related resour…

…ces (#1845) This allows to specify path to init script as `abfss://container@storage.dfs.core.windows.net/path` this fixes #1786
databricks · Dec 14, 2022 · 2f6dc75 · 2f6dc75
1 parent c822784
commit 2f6dc75
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 5 deletions.
diff --git a/clusters/clusters_api.go b/clusters/clusters_api.go
@@ -191,6 +191,11 @@ type GcsStorageInfo struct {
 	Destination string `json:"destination,omitempty"`
 }
 
+// AbfssStorageInfo contains the struct for when storing files in ADLS
+type AbfssStorageInfo struct {
+	Destination string `json:"destination,omitempty"`
+}
+
 // LocalFileInfo represents a local file on disk, e.g. in a customer's container.
 type LocalFileInfo struct {
 	Destination string `json:"destination,omitempty"`
@@ -204,10 +209,11 @@ type StorageInfo struct {
 
 // InitScriptStorageInfo captures the allowed sources of init scripts.
 type InitScriptStorageInfo struct {
-	Dbfs *DbfsStorageInfo `json:"dbfs,omitempty" tf:"group:storage"`
-	Gcs  *GcsStorageInfo  `json:"gcs,omitempty" tf:"group:storage"`
-	S3   *S3StorageInfo   `json:"s3,omitempty" tf:"group:storage"`
-	File *LocalFileInfo   `json:"file,omitempty"`
+	Dbfs  *DbfsStorageInfo  `json:"dbfs,omitempty" tf:"group:storage"`
+	Gcs   *GcsStorageInfo   `json:"gcs,omitempty" tf:"group:storage"`
+	S3    *S3StorageInfo    `json:"s3,omitempty" tf:"group:storage"`
+	Abfss *AbfssStorageInfo `json:"abfss,omitempty" tf:"group:storage"`
+	File  *LocalFileInfo    `json:"file,omitempty"`
 }
 
 // SparkNodeAwsAttributes is the struct that determines if the node is a spot instance or not

diff --git a/docs/resources/cluster.md b/docs/resources/cluster.md
@@ -288,7 +288,19 @@ init_scripts {
 }
 ```
 
- Clusters with [custom Docker containers](https://docs.databricks.com/clusters/custom-containers.html) also allow a local file location for init scripts as follows:
+Similarly, for an init script stored in ADLS:
+
+```hcl
+init_scripts {
+  abfss {
+    destination = "abfss://container@storage.dfs.core.windows.net/install-elk.sh"
+  }
+}
+```
+
+Please note that you need to provide Spark Hadoop configuration (`spark.hadoop.fs.azure...`) to authenticate to ADLS to get access to the init script.
+
+Clusters with [custom Docker containers](https://docs.databricks.com/clusters/custom-containers.html) also allow a local file location for init scripts as follows:
 
 ```hcl
 init_scripts {