Added support for Unity Catalog databricks_metastores data source (#…

…2017) * add documentation for databricks_metastores data source * add API endpoint for listing metastores * add metastores data resource * add test for metastores data source * add metastores datasource to resource mapping * fix reference to wrong resource docs * add a Metastores struct for the response of the API, use this in the datSource * update terraform specific object attributes * add new data test * remove slice_set property from MetastoreData * use databricks-go-sdk for data_metastore.go * removed listMetastores endpoint since it's unused * make sure tests also use the unitycatalog.MetastoreInfo from the sdk * remove redundant resource * test -dev * fix * fmt * cleanup * Added AccountClient to DatabricksClient and AccountData * upd * cleanup * accountLevel * upd * add example * list * cleanup * docs * remove dead code * wip * use maps * upd * cleanup * comments * - * remove redundant test --------- Co-authored-by: Tanmay Rustagi <tanmay.rustagi@databricks.com> Co-authored-by: vuong-nguyen <44292934+nkvuong@users.noreply.github.com>
databricks · Jul 19, 2023 · 0a23d5e · 0a23d5e
1 parent b52eed9
commit 0a23d5e
Show file tree

Hide file tree

Showing 6 changed files with 148 additions and 5 deletions.
diff --git a/catalog/data_metastores.go b/catalog/data_metastores.go
@@ -0,0 +1,32 @@
+package catalog
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/terraform-provider-databricks/common"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+func DataSourceMetastores() *schema.Resource {
+	type metastoresData struct {
+		Ids map[string]string `json:"ids,omitempty" tf:"computed"`
+	}
+	return common.AccountData(func(ctx context.Context, data *metastoresData, acc *databricks.AccountClient) error {
+		metastores, err := acc.Metastores.List(ctx)
+		if err != nil {
+			return err
+		}
+		data.Ids = map[string]string{}
+		for _, v := range metastores.Metastores {
+			name := v.Name
+			_, duplicateName := data.Ids[name]
+			if duplicateName {
+				return fmt.Errorf("duplicate metastore name detected: %s", name)
+			}
+			data.Ids[name] = v.MetastoreId
+		}
+		return nil
+	})
+}
diff --git a/catalog/data_metastores_test.go b/catalog/data_metastores_test.go
@@ -0,0 +1,55 @@
+package catalog
+
+import (
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/service/catalog"
+	"github.com/databricks/terraform-provider-databricks/qa"
+)
+
+func TestMetastoresDataContainsName(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/accounts/testaccount/metastores",
+				Response: catalog.ListMetastoresResponse{
+					Metastores: []catalog.MetastoreInfo{
+						{
+							Name:                      "a",
+							StorageRoot:               "abc",
+							DefaultDataAccessConfigId: "sth",
+							Owner:                     "John.Doe@example.com",
+							MetastoreId:               "abc",
+						},
+						{
+							Name:                      "b",
+							StorageRoot:               "dcw",
+							DefaultDataAccessConfigId: "sth",
+							Owner:                     "John.Doe@example.com",
+							MetastoreId:               "ded",
+						},
+					},
+				},
+			},
+		},
+		Resource:    DataSourceMetastores(),
+		Read:        true,
+		NonWritable: true,
+		ID:          "_",
+		AccountID:   "testaccount",
+	}.ApplyAndExpectData(t, map[string]any{
+		"ids": map[string]interface{}{"a": "abc", "b": "ded"},
+	})
+}
+
+func TestMetastoresData_Error(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures:    qa.HTTPFailures,
+		Resource:    DataSourceMetastores(),
+		Read:        true,
+		NonWritable: true,
+		ID:          "_",
+		AccountID:   "_",
+	}.ExpectError(t, "I'm a teapot")
+}
diff --git a/catalog/resource_metastore.go b/catalog/resource_metastore.go
@@ -43,11 +43,6 @@ type CreateMetastore struct {
 	StorageRoot string `json:"storage_root"`
 }
 
-// func (a MetastoresAPI) listMetastores() (mis []MetastoreInfo, err error) {
-// 	err = a.client.Get(a.context, "/unity-catalog/metastores", nil, &mis)
-// 	return
-// }
-
 func (a MetastoresAPI) createMetastore(cm CreateMetastore) (mi MetastoreInfo, err error) {
 	err = a.client.Post(a.context, "/unity-catalog/metastores", cm, &mi)
 	return

diff --git a/docs/data-sources/metastores.md b/docs/data-sources/metastores.md
@@ -0,0 +1,33 @@
+---
+subcategory: "Unity Catalog"
+---
+# databricks_metastores Data Source
+
+Retrieves a mapping of name to id of [databricks_metastore](../resources/metastore.md) objects, that were created by Terraform or manually, so that special handling could be applied. 
+
+-> **Note** [`account_id`](../index.md#account_id) provider configuration property is required for this resource to work. Data resource will error in case of metastores with duplicate names. This data source is only available for users & service principals with account admin status
+
+## Example Usage
+
+Mapping of name to id of all metastores:
+
+```hcl
+data "databricks_metastores" "all" {}
+
+output "all_metastores" {
+  value = data.databricks_metastores.all.ids
+}
+```
+
+## Attribute Reference
+
+This data source exports the following attributes:
+
+* `ids` - Mapping of name to id of [databricks_metastore](../resources/metastore.md)
+
+## Related Resources
+
+The following resources are used in the same context:
+
+* [databricks_metastore](../resources/metastore.md) to manage Metastores within Unity Catalog.
+* [databricks_catalog](../resources/catalog.md) to manage catalogs within Unity Catalog.
diff --git a/internal/acceptance/data_metastores_test.go b/internal/acceptance/data_metastores_test.go
@@ -0,0 +1,27 @@
+package acceptance
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
+)
+
+func TestUcAccDataSourceMetastores(t *testing.T) {
+	accountLevel(t, step{
+		Template: `
+		data "databricks_metastores" "this" {
+		}`,
+		Check: func(s *terraform.State) error {
+			r, ok := s.RootModule().Resources["data.databricks_metastores.this"]
+			if !ok {
+				return fmt.Errorf("data not found in state")
+			}
+			ids := r.Primary.Attributes["ids.%"]
+			if ids == "" {
+				return fmt.Errorf("ids is empty: %v", r.Primary.Attributes)
+			}
+			return nil
+		},
+	})
+}
diff --git a/provider/provider.go b/provider/provider.go
@@ -62,6 +62,7 @@ func DatabricksProvider() *schema.Provider {
 			"databricks_instance_pool":           pools.DataSourceInstancePool(),
 			"databricks_jobs":                    jobs.DataSourceJobs(),
 			"databricks_job":                     jobs.DataSourceJob(),
+			"databricks_metastores":              catalog.DataSourceMetastores(),
 			"databricks_mws_credentials":         mws.DataSourceMwsCredentials(),
 			"databricks_mws_workspaces":          mws.DataSourceMwsWorkspaces(),
 			"databricks_node_type":               clusters.DataSourceNodeType(),