From 0a23d5e301df4c79875c8e91a8a710f17859aab5 Mon Sep 17 00:00:00 2001 From: guillesd <74136033+guillesd@users.noreply.github.com> Date: Wed, 19 Jul 2023 16:11:55 +0200 Subject: [PATCH] Added support for Unity Catalog `databricks_metastores` data source (#2017) * add documentation for databricks_metastores data source * add API endpoint for listing metastores * add metastores data resource * add test for metastores data source * add metastores datasource to resource mapping * fix reference to wrong resource docs * add a Metastores struct for the response of the API, use this in the datSource * update terraform specific object attributes * add new data test * remove slice_set property from MetastoreData * use databricks-go-sdk for data_metastore.go * removed listMetastores endpoint since it's unused * make sure tests also use the unitycatalog.MetastoreInfo from the sdk * remove redundant resource * test -dev * fix * fmt * cleanup * Added AccountClient to DatabricksClient and AccountData * upd * cleanup * accountLevel * upd * add example * list * cleanup * docs * remove dead code * wip * use maps * upd * cleanup * comments * - * remove redundant test --------- Co-authored-by: Tanmay Rustagi Co-authored-by: vuong-nguyen <44292934+nkvuong@users.noreply.github.com> --- catalog/data_metastores.go | 32 ++++++++++++ catalog/data_metastores_test.go | 55 +++++++++++++++++++++ catalog/resource_metastore.go | 5 -- docs/data-sources/metastores.md | 33 +++++++++++++ internal/acceptance/data_metastores_test.go | 27 ++++++++++ provider/provider.go | 1 + 6 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 catalog/data_metastores.go create mode 100644 catalog/data_metastores_test.go create mode 100644 docs/data-sources/metastores.md create mode 100644 internal/acceptance/data_metastores_test.go diff --git a/catalog/data_metastores.go b/catalog/data_metastores.go new file mode 100644 index 0000000000..98cc05bdb7 --- /dev/null +++ b/catalog/data_metastores.go @@ -0,0 +1,32 @@ +package catalog + +import ( + "context" + "fmt" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/terraform-provider-databricks/common" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +func DataSourceMetastores() *schema.Resource { + type metastoresData struct { + Ids map[string]string `json:"ids,omitempty" tf:"computed"` + } + return common.AccountData(func(ctx context.Context, data *metastoresData, acc *databricks.AccountClient) error { + metastores, err := acc.Metastores.List(ctx) + if err != nil { + return err + } + data.Ids = map[string]string{} + for _, v := range metastores.Metastores { + name := v.Name + _, duplicateName := data.Ids[name] + if duplicateName { + return fmt.Errorf("duplicate metastore name detected: %s", name) + } + data.Ids[name] = v.MetastoreId + } + return nil + }) +} diff --git a/catalog/data_metastores_test.go b/catalog/data_metastores_test.go new file mode 100644 index 0000000000..d10dbc9e6e --- /dev/null +++ b/catalog/data_metastores_test.go @@ -0,0 +1,55 @@ +package catalog + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/terraform-provider-databricks/qa" +) + +func TestMetastoresDataContainsName(t *testing.T) { + qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.0/accounts/testaccount/metastores", + Response: catalog.ListMetastoresResponse{ + Metastores: []catalog.MetastoreInfo{ + { + Name: "a", + StorageRoot: "abc", + DefaultDataAccessConfigId: "sth", + Owner: "John.Doe@example.com", + MetastoreId: "abc", + }, + { + Name: "b", + StorageRoot: "dcw", + DefaultDataAccessConfigId: "sth", + Owner: "John.Doe@example.com", + MetastoreId: "ded", + }, + }, + }, + }, + }, + Resource: DataSourceMetastores(), + Read: true, + NonWritable: true, + ID: "_", + AccountID: "testaccount", + }.ApplyAndExpectData(t, map[string]any{ + "ids": map[string]interface{}{"a": "abc", "b": "ded"}, + }) +} + +func TestMetastoresData_Error(t *testing.T) { + qa.ResourceFixture{ + Fixtures: qa.HTTPFailures, + Resource: DataSourceMetastores(), + Read: true, + NonWritable: true, + ID: "_", + AccountID: "_", + }.ExpectError(t, "I'm a teapot") +} diff --git a/catalog/resource_metastore.go b/catalog/resource_metastore.go index d68cc56455..86b40ce213 100644 --- a/catalog/resource_metastore.go +++ b/catalog/resource_metastore.go @@ -43,11 +43,6 @@ type CreateMetastore struct { StorageRoot string `json:"storage_root"` } -// func (a MetastoresAPI) listMetastores() (mis []MetastoreInfo, err error) { -// err = a.client.Get(a.context, "/unity-catalog/metastores", nil, &mis) -// return -// } - func (a MetastoresAPI) createMetastore(cm CreateMetastore) (mi MetastoreInfo, err error) { err = a.client.Post(a.context, "/unity-catalog/metastores", cm, &mi) return diff --git a/docs/data-sources/metastores.md b/docs/data-sources/metastores.md new file mode 100644 index 0000000000..b595636e31 --- /dev/null +++ b/docs/data-sources/metastores.md @@ -0,0 +1,33 @@ +--- +subcategory: "Unity Catalog" +--- +# databricks_metastores Data Source + +Retrieves a mapping of name to id of [databricks_metastore](../resources/metastore.md) objects, that were created by Terraform or manually, so that special handling could be applied. + +-> **Note** [`account_id`](../index.md#account_id) provider configuration property is required for this resource to work. Data resource will error in case of metastores with duplicate names. This data source is only available for users & service principals with account admin status + +## Example Usage + +Mapping of name to id of all metastores: + +```hcl +data "databricks_metastores" "all" {} + +output "all_metastores" { + value = data.databricks_metastores.all.ids +} +``` + +## Attribute Reference + +This data source exports the following attributes: + +* `ids` - Mapping of name to id of [databricks_metastore](../resources/metastore.md) + +## Related Resources + +The following resources are used in the same context: + +* [databricks_metastore](../resources/metastore.md) to manage Metastores within Unity Catalog. +* [databricks_catalog](../resources/catalog.md) to manage catalogs within Unity Catalog. diff --git a/internal/acceptance/data_metastores_test.go b/internal/acceptance/data_metastores_test.go new file mode 100644 index 0000000000..af6c30ff17 --- /dev/null +++ b/internal/acceptance/data_metastores_test.go @@ -0,0 +1,27 @@ +package acceptance + +import ( + "fmt" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" +) + +func TestUcAccDataSourceMetastores(t *testing.T) { + accountLevel(t, step{ + Template: ` + data "databricks_metastores" "this" { + }`, + Check: func(s *terraform.State) error { + r, ok := s.RootModule().Resources["data.databricks_metastores.this"] + if !ok { + return fmt.Errorf("data not found in state") + } + ids := r.Primary.Attributes["ids.%"] + if ids == "" { + return fmt.Errorf("ids is empty: %v", r.Primary.Attributes) + } + return nil + }, + }) +} diff --git a/provider/provider.go b/provider/provider.go index 04406cb61f..d71425fe90 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -62,6 +62,7 @@ func DatabricksProvider() *schema.Provider { "databricks_instance_pool": pools.DataSourceInstancePool(), "databricks_jobs": jobs.DataSourceJobs(), "databricks_job": jobs.DataSourceJob(), + "databricks_metastores": catalog.DataSourceMetastores(), "databricks_mws_credentials": mws.DataSourceMwsCredentials(), "databricks_mws_workspaces": mws.DataSourceMwsWorkspaces(), "databricks_node_type": clusters.DataSourceNodeType(),