From 17c22dc2853d3ce36ac0d404b708716ee04f832c Mon Sep 17 00:00:00 2001 From: Stuart Auld Date: Tue, 10 Mar 2020 11:20:42 +1100 Subject: [PATCH] [feature] storage integration (#140) --- README.md | 46 ++- pkg/provider/provider.go | 37 +- pkg/resources/database.go | 18 +- pkg/resources/database_test.go | 2 +- pkg/resources/grant_helpers.go | 4 +- pkg/resources/grant_helpers_internal_test.go | 4 +- pkg/resources/helpers_test.go | 8 + pkg/resources/list_expansion_internal_test.go | 28 ++ pkg/resources/stage.go | 26 ++ pkg/resources/storage_integration.go | 341 ++++++++++++++++++ pkg/resources/storage_integration_test.go | 88 +++++ pkg/resources/view_grant.go | 2 +- pkg/snowflake/generic.go | 119 ++++-- pkg/snowflake/generic_test.go | 25 ++ pkg/snowflake/grant.go | 1 - pkg/snowflake/stage.go | 34 +- pkg/snowflake/stage_test.go | 9 + pkg/snowflake/storage_integration.go | 18 + pkg/snowflake/storage_integration_test.go | 26 ++ 19 files changed, 749 insertions(+), 87 deletions(-) create mode 100644 pkg/resources/list_expansion_internal_test.go create mode 100644 pkg/resources/storage_integration.go create mode 100644 pkg/resources/storage_integration_test.go create mode 100644 pkg/snowflake/generic_test.go create mode 100644 pkg/snowflake/storage_integration.go create mode 100644 pkg/snowflake/storage_integration_test.go diff --git a/README.md b/README.md index 2b72c896ff..9ac70b95bb 100644 --- a/README.md +++ b/README.md @@ -208,19 +208,20 @@ These resources do not enforce exclusive attachment of a grant, it is the user's #### properties -| NAME | TYPE | DESCRIPTION | OPTIONAL | REQUIRED | COMPUTED | DEFAULT | -|--------------------|--------|-------------------------------------------------------------------------------------------------------------------|----------|-----------|----------|---------| -| aws_external_id | string | | true | false | true | | -| comment | string | Specifies a comment for the stage. | true | false | false | | -| copy_options | string | Specifies the copy options for the stage. | true | false | false | | -| credentials | string | Specifies the credentials for the stage. | true | false | false | | -| database | string | The database in which to create the stage. | false | true | false | | -| encryption | string | Specifies the encryption settings for the stage. | true | false | false | | -| file_format | string | Specifies the file format for the stage. | true | false | false | | -| name | string | Specifies the identifier for the stage; must be unique for the database and schema in which the stage is created. | false | true | false | | -| schema | string | The schema in which to create the stage. | false | true | false | | -| snowflake_iam_user | string | | true | false | true | | -| url | string | Specifies the URL for the stage. | true | false | false | | +| NAME | TYPE | DESCRIPTION | OPTIONAL | REQUIRED | COMPUTED | DEFAULT | +|---------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-----------|----------|---------| +| aws_external_id | string | | true | false | true | | +| comment | string | Specifies a comment for the stage. | true | false | false | | +| copy_options | string | Specifies the copy options for the stage. | true | false | false | | +| credentials | string | Specifies the credentials for the stage. | true | false | false | | +| database | string | The database in which to create the stage. | false | true | false | | +| encryption | string | Specifies the encryption settings for the stage. | true | false | false | | +| file_format | string | Specifies the file format for the stage. | true | false | false | | +| name | string | Specifies the identifier for the stage; must be unique for the database and schema in which the stage is created. | false | true | false | | +| schema | string | The schema in which to create the stage. | false | true | false | | +| snowflake_iam_user | string | | true | false | true | | +| storage_integration | string | Specifies the name of the storage integration used to delegate authentication responsibility for external cloud storage to a Snowflake identity and access management (IAM) entity. | true | false | false | | +| url | string | Specifies the URL for the stage. | true | false | false | | ### snowflake_stage_grant @@ -241,6 +242,25 @@ These resources do not enforce exclusive attachment of a grant, it is the user's | shares | set | Grants privilege to these shares. | true | false | false | | | stage_name | string | The name of the stage on which to grant privileges. | false | true | false | | +### snowflake_storage_integration + +#### properties + +| NAME | TYPE | DESCRIPTION | OPTIONAL | REQUIRED | COMPUTED | DEFAULT | +|---------------------------|--------|---------------------------------------------------------------------------------------------------------------|----------|-----------|----------|------------------| +| azure_tenant_id | string | | true | false | false | "" | +| comment | string | | true | false | false | "" | +| created_on | string | Date and time when the storage integration was created. | false | false | true | | +| enabled | bool | | true | false | false | true | +| name | string | | false | true | false | | +| storage_allowed_locations | list | Explicitly limits external stages that use the integration to reference one or more storage locations. | false | true | false | | +| storage_aws_external_id | string | The external ID that Snowflake will use when assuming the AWS role. | false | false | true | | +| storage_aws_iam_user_arn | string | The Snowflake user that will attempt to assume the AWS role. | false | false | true | | +| storage_aws_role_arn | string | | true | false | false | "" | +| storage_blocked_locations | list | Explicitly prohibits external stages that use the integration from referencing one or more storage locations. | true | false | false | | +| storage_provider | string | | false | true | false | | +| type | string | | true | false | false | "EXTERNAL_STAGE" | + ### snowflake_table_grant **Note**: The snowflake_table_grant resource creates exclusive attachments of grants. diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index e982497648..8e27d20119 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -60,24 +60,25 @@ func Provider() *schema.Provider { }, }, ResourcesMap: map[string]*schema.Resource{ - "snowflake_database": resources.Database(), - "snowflake_database_grant": resources.DatabaseGrant(), - "snowflake_managed_account": resources.ManagedAccount(), - "snowflake_pipe": resources.Pipe(), - "snowflake_resource_monitor": resources.ResourceMonitor(), - "snowflake_role": resources.Role(), - "snowflake_role_grants": resources.RoleGrants(), - "snowflake_schema": resources.Schema(), - "snowflake_schema_grant": resources.SchemaGrant(), - "snowflake_share": resources.Share(), - "snowflake_stage": resources.Stage(), - "snowflake_stage_grant": resources.StageGrant(), - "snowflake_user": resources.User(), - "snowflake_view": resources.View(), - "snowflake_view_grant": resources.ViewGrant(), - "snowflake_table_grant": resources.TableGrant(), - "snowflake_warehouse": resources.Warehouse(), - "snowflake_warehouse_grant": resources.WarehouseGrant(), + "snowflake_database": resources.Database(), + "snowflake_database_grant": resources.DatabaseGrant(), + "snowflake_managed_account": resources.ManagedAccount(), + "snowflake_pipe": resources.Pipe(), + "snowflake_resource_monitor": resources.ResourceMonitor(), + "snowflake_role": resources.Role(), + "snowflake_role_grants": resources.RoleGrants(), + "snowflake_schema": resources.Schema(), + "snowflake_schema_grant": resources.SchemaGrant(), + "snowflake_share": resources.Share(), + "snowflake_stage": resources.Stage(), + "snowflake_stage_grant": resources.StageGrant(), + "snowflake_storage_integration": resources.StorageIntegration(), + "snowflake_user": resources.User(), + "snowflake_view": resources.View(), + "snowflake_view_grant": resources.ViewGrant(), + "snowflake_table_grant": resources.TableGrant(), + "snowflake_warehouse": resources.Warehouse(), + "snowflake_warehouse_grant": resources.WarehouseGrant(), }, DataSourcesMap: map[string]*schema.Resource{}, ConfigureFunc: ConfigureProvider, diff --git a/pkg/resources/database.go b/pkg/resources/database.go index 595cfd07ad..36207a4ddb 100644 --- a/pkg/resources/database.go +++ b/pkg/resources/database.go @@ -29,17 +29,17 @@ var databaseSchema = map[string]*schema.Schema{ Computed: true, }, "from_share": &schema.Schema{ - Type: schema.TypeMap, - Description: "Specify a provider and a share in this map to create a database from a share.", - Optional: true, - ForceNew: true, + Type: schema.TypeMap, + Description: "Specify a provider and a share in this map to create a database from a share.", + Optional: true, + ForceNew: true, ConflictsWith: []string{"from_database"}, }, "from_database": &schema.Schema{ - Type: schema.TypeString, - Description: "Specify a database to create a clone from.", - Optional: true, - ForceNew: true, + Type: schema.TypeString, + Description: "Specify a database to create a clone from.", + Optional: true, + ForceNew: true, ConflictsWith: []string{"from_share"}, }, } @@ -66,7 +66,7 @@ func CreateDatabase(data *schema.ResourceData, meta interface{}) error { if _, ok := data.GetOk("from_share"); ok { return createDatabaseFromShare(data, meta) } - + if _, ok := data.GetOk("from_database"); ok { return createDatabaseFromDatabase(data, meta) } diff --git a/pkg/resources/database_test.go b/pkg/resources/database_test.go index 3eeb14260f..7ff83a1336 100644 --- a/pkg/resources/database_test.go +++ b/pkg/resources/database_test.go @@ -94,7 +94,7 @@ func TestDatabaseCreateFromDatabase(t *testing.T) { a := assert.New(t) in := map[string]interface{}{ - "name": "good_name", + "name": "good_name", "from_database": "abc123", } d := schema.TestResourceDataRaw(t, resources.Database().Schema, in) diff --git a/pkg/resources/grant_helpers.go b/pkg/resources/grant_helpers.go index 74ddf7aa4e..30dbb4b105 100644 --- a/pkg/resources/grant_helpers.go +++ b/pkg/resources/grant_helpers.go @@ -59,7 +59,7 @@ type grant struct { type grantID struct { ResourceName string SchemaName string - ObjectName string + ObjectName string Privilege string } @@ -153,7 +153,7 @@ func grantIDFromString(stringID string) (*grantID, error) { grantResult := &grantID{ ResourceName: lines[0][0], SchemaName: lines[0][1], - ObjectName: lines[0][2], + ObjectName: lines[0][2], Privilege: lines[0][3], } return grantResult, nil diff --git a/pkg/resources/grant_helpers_internal_test.go b/pkg/resources/grant_helpers_internal_test.go index 71ab07abee..e48684373d 100644 --- a/pkg/resources/grant_helpers_internal_test.go +++ b/pkg/resources/grant_helpers_internal_test.go @@ -62,7 +62,7 @@ func TestGrantStruct(t *testing.T) { grant := &grantID{ ResourceName: "database_name", SchemaName: "schema", - ObjectName: "view_name", + ObjectName: "view_name", Privilege: "priv", } gID, err := grant.String() @@ -79,7 +79,7 @@ func TestGrantStruct(t *testing.T) { grant = &grantID{ ResourceName: "database|name", SchemaName: "schema|name", - ObjectName: "view|name", + ObjectName: "view|name", Privilege: "priv", } gID, err = grant.String() diff --git a/pkg/resources/helpers_test.go b/pkg/resources/helpers_test.go index adfbb2cf7a..9eeb111e97 100644 --- a/pkg/resources/helpers_test.go +++ b/pkg/resources/helpers_test.go @@ -56,6 +56,14 @@ func roleGrants(t *testing.T, id string, params map[string]interface{}) *schema. return d } +func storageIntegration(t *testing.T, id string, params map[string]interface{}) *schema.ResourceData { + a := assert.New(t) + d := schema.TestResourceDataRaw(t, resources.StorageIntegration().Schema, params) + a.NotNil(d) + d.SetId(id) + return d +} + func user(t *testing.T, id string, params map[string]interface{}) *schema.ResourceData { a := assert.New(t) d := schema.TestResourceDataRaw(t, resources.User().Schema, params) diff --git a/pkg/resources/list_expansion_internal_test.go b/pkg/resources/list_expansion_internal_test.go new file mode 100644 index 0000000000..65822fa175 --- /dev/null +++ b/pkg/resources/list_expansion_internal_test.go @@ -0,0 +1,28 @@ +package resources + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExpandStringList(t *testing.T) { + r := require.New(t) + + in := []interface{}{"this", "is", "just", "a", "test"} + out := expandStringList(in) + + r.Equal("this", out[0]) + r.Equal("is", out[1]) + r.Equal("just", out[2]) + r.Equal("a", out[3]) + r.Equal("test", out[4]) +} + +func TestExpandBlankStringList(t *testing.T) { + r := require.New(t) + in := []interface{}{} + out := expandStringList(in) + + r.Equal(len(out), 0) +} diff --git a/pkg/resources/stage.go b/pkg/resources/stage.go index 8460c47713..9a53382537 100644 --- a/pkg/resources/stage.go +++ b/pkg/resources/stage.go @@ -45,6 +45,11 @@ var stageSchema = map[string]*schema.Schema{ Optional: true, Description: "Specifies the credentials for the stage.", }, + "storage_integration": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: "Specifies the name of the storage integration used to delegate authentication responsibility for external cloud storage to a Snowflake identity and access management (IAM) entity.", + }, "file_format": &schema.Schema{ Type: schema.TypeString, Optional: true, @@ -157,6 +162,10 @@ func CreateStage(data *schema.ResourceData, meta interface{}) error { builder.WithCredentials(v.(string)) } + if v, ok := data.GetOk("storage_integration"); ok { + builder.WithStorageIntegration(v.(string)) + } + if v, ok := data.GetOk("file_format"); ok { builder.WithFileFormat(v.(string)) } @@ -249,6 +258,11 @@ func ReadStage(data *schema.ResourceData, meta interface{}) error { return err } + err = data.Set("storage_integration", stageShow.storageIntegration) + if err != nil { + return err + } + err = data.Set("comment", stageShow.comment) if err != nil { return err @@ -305,6 +319,18 @@ func UpdateStage(data *schema.ResourceData, meta interface{}) error { data.SetPartial("credentials") } + + if data.HasChange("storage_integration") { + _, si := data.GetChange("storage_integration") + q := builder.ChangeStorageIntegration(si.(string)) + err := DBExec(db, q) + if err != nil { + return errors.Wrapf(err, "error updating stage storage integration on %v", data.Id()) + } + + data.SetPartial("storage_integration") + } + if data.HasChange("encryption") { _, encryption := data.GetChange("encryption") q := builder.ChangeEncryption(encryption.(string)) diff --git a/pkg/resources/storage_integration.go b/pkg/resources/storage_integration.go new file mode 100644 index 0000000000..809b6b4b72 --- /dev/null +++ b/pkg/resources/storage_integration.go @@ -0,0 +1,341 @@ +package resources + +import ( + "database/sql" + "fmt" + "log" + "strings" + + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" + "github.com/hashicorp/terraform-plugin-sdk/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/helper/validation" +) + +var storageIntegrationSchema = map[string]*schema.Schema{ + // The first part of the schema is shared between all integration vendors + "name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "comment": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Default: "", + }, + "type": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Default: "EXTERNAL_STAGE", + ValidateFunc: validation.StringInSlice([]string{"EXTERNAL_STAGE"}, true), + }, + "enabled": &schema.Schema{ + Type: schema.TypeBool, + Optional: true, + Default: true, + }, + "storage_allowed_locations": &schema.Schema{ + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Required: true, + Description: "Explicitly limits external stages that use the integration to reference one or more storage locations.", + }, + "storage_blocked_locations": &schema.Schema{ + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + Description: "Explicitly prohibits external stages that use the integration from referencing one or more storage locations.", + }, + // This part of the schema is the cloudProviderParams in the Snowflake documentation and differs between vendors + "storage_provider": &schema.Schema{ + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringInSlice([]string{"S3", "GCS", "AZURE"}, false), + }, + "storage_aws_external_id": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + Description: "The external ID that Snowflake will use when assuming the AWS role.", + }, + "storage_aws_iam_user_arn": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + Description: "The Snowflake user that will attempt to assume the AWS role.", + }, + "storage_aws_role_arn": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Default: "", + }, + "azure_tenant_id": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Default: "", + }, + "created_on": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + Description: "Date and time when the storage integration was created.", + }, +} + +// StorageIntegration returns a pointer to the resource representing a storage integration +func StorageIntegration() *schema.Resource { + return &schema.Resource{ + Create: CreateStorageIntegration, + Read: ReadStorageIntegration, + Update: UpdateStorageIntegration, + Delete: DeleteStorageIntegration, + Exists: StorageIntegrationExists, + + Schema: storageIntegrationSchema, + Importer: &schema.ResourceImporter{ + State: schema.ImportStatePassthrough, + }, + } +} + +// CreateStorageIntegration implements schema.CreateFunc +func CreateStorageIntegration(data *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + name := data.Get("name").(string) + + stmt := snowflake.StorageIntegration(name).Create() + + // Set required fields + stmt.SetString(`TYPE`, data.Get("type").(string)) + stmt.SetBool(`ENABLED`, data.Get("enabled").(bool)) + + stmt.SetStringList("STORAGE_ALLOWED_LOCATIONS", expandStringList(data.Get("storage_allowed_locations").([]interface{}))) + + // Set optional fields + if v, ok := data.GetOk("comment"); ok { + stmt.SetString(`COMMENT`, v.(string)) + } + + if _, ok := data.GetOk("storage_blocked_locations"); ok { + stmt.SetStringList("STORAGE_BLOCKED_LOCATIONS", expandStringList(data.Get("storage_blocked_locations").([]interface{}))) + } + + // Now, set the storage provider + err := setStorageProviderSettings(data, stmt) + if err != nil { + return err + } + + err = DBExec(db, stmt.Statement()) + if err != nil { + return fmt.Errorf("error creating storage integration: %w", err) + } + + data.SetId(name) + + return ReadStorageIntegration(data, meta) +} + +// ReadStorageIntegration implements schema.ReadFunc +func ReadStorageIntegration(data *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + id := data.Id() + + stmt := snowflake.StorageIntegration(data.Id()).Show() + row := db.QueryRow(stmt) + + // Some properties can come from the SHOW INTEGRATION call + var name, integrationType, category, createdOn sql.NullString + var enabled sql.NullBool + if err := row.Scan(&name, &integrationType, &category, &enabled, &createdOn); err != nil { + return fmt.Errorf("Could not show storage integration: %w", err) + } + + // Note: category must be STORAGE or something is broken + if c := category.String; c != "STORAGE" { + return fmt.Errorf("Expected %v to be a STORAGE integration, got %v", id, c) + } + + if err := data.Set("name", name.String); err != nil { + return err + } + + if err := data.Set("type", integrationType.String); err != nil { + return err + } + + if err := data.Set("created_on", createdOn.String); err != nil { + return err + } + + if err := data.Set("enabled", enabled.Bool); err != nil { + return err + } + + // Some properties come from the DESCRIBE INTEGRATION call + // We need to grab them in a loop + var k, pType string + var v, d interface{} + stmt = snowflake.StorageIntegration(data.Id()).Describe() + rows, err := db.Query(stmt) + if err != nil { + return fmt.Errorf("Could not describe storage integration: %w", err) + } + defer rows.Close() + for rows.Next() { + if err := rows.Scan(&k, &pType, &v, &d); err != nil { + return err + } + switch k { + case "ENABLED": + // We set this using the SHOW INTEGRATION call so let's ignore it here + case "STORAGE_PROVIDER": + if err = data.Set("storage_provider", v.(string)); err != nil { + return err + } + case "STORAGE_ALLOWED_LOCATIONS": + if err = data.Set("storage_allowed_locations", strings.Split(v.(string), ",")); err != nil { + return err + } + case "STORAGE_BLOCKED_LOCATIONS": + if val := v.(string); val != "" { + if err = data.Set("storage_blocked_locations", strings.Split(val, ",")); err != nil { + return err + } + } + case "STORAGE_AWS_IAM_USER_ARN": + if err = data.Set("storage_aws_iam_user_arn", v.(string)); err != nil { + return err + } + case "STORAGE_AWS_ROLE_ARN": + if err = data.Set("storage_aws_role_arn", v.(string)); err != nil { + return err + } + case "STORAGE_AWS_EXTERNAL_ID": + if err = data.Set("storage_aws_external_id", v.(string)); err != nil { + return err + } + default: + log.Printf("[WARN] unexpected property %v returned from Snowflake", k) + } + } + + return err +} + +// UpdateStorageIntegration implements schema.UpdateFunc +func UpdateStorageIntegration(data *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + id := data.Id() + + stmt := snowflake.StorageIntegration(id).Alter() + + // This is required in case the only change is to UNSET STORAGE_ALLOWED_LOCATIONS. + // Not sure if there is a more elegant way of determining this + var runSetStatement bool + + if data.HasChange("comment") { + runSetStatement = true + stmt.SetString("COMMENT", data.Get("comment").(string)) + } + + if data.HasChange("type") { + runSetStatement = true + stmt.SetString("TYPE", data.Get("type").(string)) + } + + if data.HasChange("enabled") { + runSetStatement = true + stmt.SetBool(`ENABLED`, data.Get("enabled").(bool)) + } + + if data.HasChange("storage_allowed_locations") { + runSetStatement = true + stmt.SetStringList("STORAGE_ALLOWED_LOCATIONS", expandStringList(data.Get("storage_allowed_locations").([]interface{}))) + } + + // We need to UNSET this if we remove all storage blocked locations. I don't think + // this is documented by Snowflake, but this is how it works. + // + // @TODO move the SQL back to the snowflake package + if data.HasChange("storage_blocked_locations") { + v := data.Get("storage_blocked_locations").([]interface{}) + if len(v) == 0 { + err := DBExec(db, fmt.Sprintf(`ALTER STORAGE INTEGRATION %v UNSET STORAGE_BLOCKED_LOCATIONS`, data.Id())) + if err != nil { + return fmt.Errorf("error unsetting storage_blocked_locations: %w", err) + } + } else { + runSetStatement = true + stmt.SetStringList("STORAGE_BLOCKED_LOCATIONS", expandStringList(v)) + } + } + + if data.HasChange("storage_provider") { + runSetStatement = true + setStorageProviderSettings(data, stmt) + } else { + if data.HasChange("storage_aws_role_arn") { + runSetStatement = true + stmt.SetString("STORAGE_AWS_ROLE_ARN", data.Get("storage_aws_role_arn").(string)) + } + if data.HasChange("azure_tenant_id") { + runSetStatement = true + stmt.SetString("AZURE_TENANT_ID", data.Get("azure_tenant_id").(string)) + } + } + + if runSetStatement { + if err := DBExec(db, stmt.Statement()); err != nil { + return fmt.Errorf("error updating storage integration: %w", err) + } + } + + return ReadStorageIntegration(data, meta) +} + +// DeleteStorageIntegration implements schema.DeleteFunc +func DeleteStorageIntegration(data *schema.ResourceData, meta interface{}) error { + return DeleteResource("", snowflake.StorageIntegration)(data, meta) +} + +// StorageIntegrationExists implements schema.ExistsFunc +func StorageIntegrationExists(data *schema.ResourceData, meta interface{}) (bool, error) { + db := meta.(*sql.DB) + id := data.Id() + + stmt := snowflake.StorageIntegration(id).Show() + rows, err := db.Query(stmt) + if err != nil { + return false, err + } + defer rows.Close() + + if rows.Next() { + return true, nil + } + return false, nil +} + +func setStorageProviderSettings(data *schema.ResourceData, stmt snowflake.SettingBuilder) error { + storageProvider := data.Get("storage_provider").(string) + stmt.SetString("STORAGE_PROVIDER", storageProvider) + + switch storageProvider { + case "S3": + v, ok := data.GetOk("storage_aws_role_arn") + if !ok { + return fmt.Errorf("If you use the S3 storage provider you must specify a storage_aws_role_arn") + } + stmt.SetString(`STORAGE_AWS_ROLE_ARN`, v.(string)) + case "AZURE": + v, ok := data.GetOk("azure_tenant_id") + if !ok { + return fmt.Errorf("If you use the Azure storage provider you must specify an azure_tenant_id") + } + stmt.SetString(`AZURE_TENANT_ID`, v.(string)) + case "GCS": + // nothing to set here + default: + return fmt.Errorf("Unexpected provider %v", storageProvider) + } + + return nil +} diff --git a/pkg/resources/storage_integration_test.go b/pkg/resources/storage_integration_test.go new file mode 100644 index 0000000000..c9641393d0 --- /dev/null +++ b/pkg/resources/storage_integration_test.go @@ -0,0 +1,88 @@ +package resources_test + +import ( + "database/sql" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/provider" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/resources" + . "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/testhelpers" + "github.com/hashicorp/terraform-plugin-sdk/helper/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStorageIntegration(t *testing.T) { + r := require.New(t) + err := resources.StorageIntegration().InternalValidate(provider.Provider().Schema, true) + r.NoError(err) +} + +func TestStorageIntegrationCreate(t *testing.T) { + a := assert.New(t) + + in := map[string]interface{}{ + "name": "test_storage_integration", + "comment": "great comment", + "storage_allowed_locations": []interface{}{"s3://great-bucket/great-path/"}, + "storage_provider": "S3", + "storage_aws_role_arn": "we-should-probably-validate-this-string", + } + d := schema.TestResourceDataRaw(t, resources.StorageIntegration().Schema, in) + a.NotNil(d) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `^CREATE STORAGE INTEGRATION "test_storage_integration" COMMENT='great comment' STORAGE_AWS_ROLE_ARN='we-should-probably-validate-this-string' STORAGE_PROVIDER='S3' TYPE='EXTERNAL_STAGE' STORAGE_ALLOWED_LOCATIONS=\('s3://great-bucket/great-path/'\) ENABLED=true$`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + expectReadStorageIntegration(mock) + + err := resources.CreateStorageIntegration(d, db) + a.NoError(err) + }) +} + +func TestStorageIntegrationRead(t *testing.T) { + a := assert.New(t) + + d := storageIntegration(t, "test_storage_integration", map[string]interface{}{"name": "test_storage_integration"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + expectReadStorageIntegration(mock) + + err := resources.ReadStorageIntegration(d, db) + a.NoError(err) + }) +} + +func TestStorageIntegrationDelete(t *testing.T) { + a := assert.New(t) + + d := storageIntegration(t, "drop_it", map[string]interface{}{"name": "drop_it"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP STORAGE INTEGRATION "drop_it"`).WillReturnResult(sqlmock.NewResult(1, 1)) + err := resources.DeleteStorageIntegration(d, db) + a.NoError(err) + }) +} + +func expectReadStorageIntegration(mock sqlmock.Sqlmock) { + showRows := sqlmock.NewRows([]string{ + "name", "type", "category", "enabled", "created_on"}, + ).AddRow("test_storage_integration", "EXTERNAL_STAGE", "STORAGE", true, "now") + mock.ExpectQuery(`^SHOW STORAGE INTEGRATIONS LIKE 'test_storage_integration'$`).WillReturnRows(showRows) + + descRows := sqlmock.NewRows([]string{ + "property", "property_type", "property_value", "property_default", + }).AddRow("ENABLED", "Boolean", true, false). + AddRow("STORAGE_PROVIDER", "String", "S3", nil). + AddRow("STORAGE_ALLOWED_LOCATIONS", "List", "s3://bucket-a/path-a/,s3://bucket-b/", nil). + AddRow("STORAGE_BLOCKED_LOCATIONS", "List", "s3://bucket-c/path-c/,s3://bucket-d/", nil). + AddRow("STORAGE_AWS_IAM_USER_ARN", "String", "arn:aws:iam::000000000000:/user/test", nil). + AddRow("STORAGE_AWS_ROLE_ARN", "String", "arn:aws:iam::000000000001:/role/test", nil). + AddRow("STORAGE_AWS_EXTERNAL_ID", "String", "AGreatExternalID", nil) + + mock.ExpectQuery(`DESCRIBE STORAGE INTEGRATION "test_storage_integration"$`).WillReturnRows(descRows) +} diff --git a/pkg/resources/view_grant.go b/pkg/resources/view_grant.go index 7c00bea1d4..74de53f9dd 100644 --- a/pkg/resources/view_grant.go +++ b/pkg/resources/view_grant.go @@ -113,7 +113,7 @@ func CreateViewGrant(data *schema.ResourceData, meta interface{}) error { grant := &grantID{ ResourceName: dbName, SchemaName: schemaName, - ObjectName: viewName, + ObjectName: viewName, Privilege: priv, } dataIDInput, err := grant.String() diff --git a/pkg/snowflake/generic.go b/pkg/snowflake/generic.go index 1e4d92bddb..6b0bb3a533 100644 --- a/pkg/snowflake/generic.go +++ b/pkg/snowflake/generic.go @@ -1,21 +1,24 @@ package snowflake import ( + "bytes" "fmt" "sort" "strings" + "text/template" ) type EntityType string const ( - DatabaseType EntityType = "DATABASE" - ManagedAccountType EntityType = "MANAGED ACCOUNT" - ResourceMonitorType EntityType = "RESOURCE MONITOR" - RoleType EntityType = "ROLE" - ShareType EntityType = "SHARE" - UserType EntityType = "USER" - WarehouseType EntityType = "WAREHOUSE" + DatabaseType EntityType = "DATABASE" + ManagedAccountType EntityType = "MANAGED ACCOUNT" + ResourceMonitorType EntityType = "RESOURCE MONITOR" + RoleType EntityType = "ROLE" + ShareType EntityType = "SHARE" + StorageIntegrationType EntityType = "STORAGE INTEGRATION" + UserType EntityType = "USER" + WarehouseType EntityType = "WAREHOUSE" ) type Builder struct { @@ -27,6 +30,10 @@ func (b *Builder) Show() string { return fmt.Sprintf(`SHOW %sS LIKE '%s'`, b.entityType, b.name) } +func (b *Builder) Describe() string { + return fmt.Sprintf(`DESCRIBE %s "%s"`, b.entityType, b.name) +} + func (b *Builder) Drop() string { return fmt.Sprintf(`DROP %s "%s"`, b.entityType, b.name) } @@ -35,23 +42,34 @@ func (b *Builder) Rename(newName string) string { return fmt.Sprintf(`ALTER %s "%s" RENAME TO "%s"`, b.entityType, b.name, newName) } +// SettingBuilder is an interface for a builder that allows you to set key value pairs +type SettingBuilder interface { + SetString(string, string) + SetStringList(string, []string) + SetBool(string, bool) + SetInt(string, int) + SetFloat(string, float64) +} + type AlterPropertiesBuilder struct { - name string - entityType EntityType - stringProperties map[string]string - boolProperties map[string]bool - intProperties map[string]int - floatProperties map[string]float64 + name string + entityType EntityType + stringProperties map[string]string + stringListProperties map[string][]string + boolProperties map[string]bool + intProperties map[string]int + floatProperties map[string]float64 } func (b *Builder) Alter() *AlterPropertiesBuilder { return &AlterPropertiesBuilder{ - name: b.name, - entityType: b.entityType, - stringProperties: make(map[string]string), - boolProperties: make(map[string]bool), - intProperties: make(map[string]int), - floatProperties: make(map[string]float64), + name: b.name, + entityType: b.entityType, + stringProperties: make(map[string]string), + stringListProperties: make(map[string][]string), + boolProperties: make(map[string]bool), + intProperties: make(map[string]int), + floatProperties: make(map[string]float64), } } @@ -59,6 +77,10 @@ func (ab *AlterPropertiesBuilder) SetString(key, value string) { ab.stringProperties[key] = value } +func (ab *AlterPropertiesBuilder) SetStringList(key string, value []string) { + ab.stringListProperties[key] = value +} + func (ab *AlterPropertiesBuilder) SetBool(key string, value bool) { ab.boolProperties[key] = value } @@ -79,6 +101,10 @@ func (ab *AlterPropertiesBuilder) Statement() string { sb.WriteString(fmt.Sprintf(" %s='%s'", strings.ToUpper(k), EscapeString(v))) } + for k, v := range ab.stringListProperties { + sb.WriteString(fmt.Sprintf(" %s=%s", strings.ToUpper(k), formatStringList(v))) + } + for k, v := range ab.boolProperties { sb.WriteString(fmt.Sprintf(" %s=%t", strings.ToUpper(k), v)) } @@ -95,22 +121,24 @@ func (ab *AlterPropertiesBuilder) Statement() string { } type CreateBuilder struct { - name string - entityType EntityType - stringProperties map[string]string - boolProperties map[string]bool - intProperties map[string]int - floatProperties map[string]float64 + name string + entityType EntityType + stringProperties map[string]string + stringListProperties map[string][]string + boolProperties map[string]bool + intProperties map[string]int + floatProperties map[string]float64 } func (b *Builder) Create() *CreateBuilder { return &CreateBuilder{ - name: b.name, - entityType: b.entityType, - stringProperties: make(map[string]string), - boolProperties: make(map[string]bool), - intProperties: make(map[string]int), - floatProperties: make(map[string]float64), + name: b.name, + entityType: b.entityType, + stringProperties: make(map[string]string), + stringListProperties: make(map[string][]string), + boolProperties: make(map[string]bool), + intProperties: make(map[string]int), + floatProperties: make(map[string]float64), } } @@ -118,6 +146,10 @@ func (b *CreateBuilder) SetString(key, value string) { b.stringProperties[key] = value } +func (b *CreateBuilder) SetStringList(key string, value []string) { + b.stringListProperties[key] = value +} + func (b *CreateBuilder) SetBool(key string, value bool) { b.boolProperties[key] = value } @@ -144,6 +176,15 @@ func (b *CreateBuilder) Statement() string { sb.WriteString(fmt.Sprintf(" %s='%s'", strings.ToUpper(k), EscapeString(b.stringProperties[k]))) } + sortedStringListProperties := make([]string, 0) + for k := range b.stringListProperties { + sortedStringListProperties = append(sortedStringListProperties, k) + } + + for _, k := range sortedStringListProperties { + sb.WriteString(fmt.Sprintf(" %s=%s", strings.ToUpper(k), formatStringList(b.stringListProperties[k]))) + } + sortedBoolProperties := make([]string, 0) for k := range b.boolProperties { sortedBoolProperties = append(sortedBoolProperties, k) @@ -176,3 +217,19 @@ func (b *CreateBuilder) Statement() string { return sb.String() } + +func formatStringList(list []string) string { + t, err := template.New("StringList").Funcs(template.FuncMap{ + "escapeString": EscapeString, + }).Parse(`({{ range $i, $v := .}}{{ if $i }}, {{ end }}'{{ escapeString $v }}'{{ end }})`) + if err != nil { + return "" + } + var buf bytes.Buffer + + if err := t.Execute(&buf, list); err != nil { + return "" + } + + return buf.String() +} diff --git a/pkg/snowflake/generic_test.go b/pkg/snowflake/generic_test.go new file mode 100644 index 0000000000..7ccbb55a14 --- /dev/null +++ b/pkg/snowflake/generic_test.go @@ -0,0 +1,25 @@ +package snowflake + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFormatStringList(t *testing.T) { + r := require.New(t) + + in := []string{"this", "is", "just", "a", "test"} + out := formatStringList(in) + + r.Equal("('this', 'is', 'just', 'a', 'test')", out) +} + +func TestFormatStringListWithEscape(t *testing.T) { + r := require.New(t) + + in := []string{"th'is", "is", "just", "a", "test"} + out := formatStringList(in) + + r.Equal("('th\\'is', 'is', 'just', 'a', 'test')", out) +} diff --git a/pkg/snowflake/grant.go b/pkg/snowflake/grant.go index cf43adbe38..f15b43dbad 100644 --- a/pkg/snowflake/grant.go +++ b/pkg/snowflake/grant.go @@ -67,7 +67,6 @@ func StageGrant(db, schema, stage string) GrantBuilder { } } - // ViewGrant returns a pointer to a CurrentGrantBuilder for a view func ViewGrant(db, schema, view string) GrantBuilder { return &CurrentGrantBuilder{ diff --git a/pkg/snowflake/stage.go b/pkg/snowflake/stage.go index 7a21641968..ffb08e1d34 100644 --- a/pkg/snowflake/stage.go +++ b/pkg/snowflake/stage.go @@ -7,15 +7,16 @@ import ( // StageBuilder abstracts the creation of SQL queries for a Snowflake stage type StageBuilder struct { - name string - db string - schema string - url string - credentials string - encryption string - fileFormat string - copyOptions string - comment string + name string + db string + schema string + url string + credentials string + storageIntegration string + encryption string + fileFormat string + copyOptions string + comment string } // QualifiedName prepends the db and schema and escapes everything nicely @@ -39,6 +40,12 @@ func (sb *StageBuilder) WithCredentials(c string) *StageBuilder { return sb } +// WithStorageIntegration adds a storage integration to the StageBuilder +func (sb *StageBuilder) WithStorageIntegration(s string) *StageBuilder { + sb.storageIntegration = s + return sb +} + // WithEncryption adds encryption to the StageBuilder func (sb *StageBuilder) WithEncryption(e string) *StageBuilder { sb.encryption = e @@ -96,6 +103,10 @@ func (sb *StageBuilder) Create() string { q.WriteString(fmt.Sprintf(` CREDENTIALS = (%v)`, sb.credentials)) } + if sb.storageIntegration != "" { + q.WriteString(fmt.Sprintf(` STORAGE_INTEGRATION = %v`, sb.storageIntegration)) + } + if sb.encryption != "" { q.WriteString(fmt.Sprintf(` ENCRYPTION = (%v)`, sb.encryption)) } @@ -140,6 +151,11 @@ func (sb *StageBuilder) ChangeCredentials(c string) string { return fmt.Sprintf(`ALTER STAGE %v SET CREDENTIALS = (%v)`, sb.QualifiedName(), c) } +// ChangeStorageIntegration returns the SQL query that will update the storage integration on the stage. +func (sb *StageBuilder) ChangeStorageIntegration(s string) string { + return fmt.Sprintf(`ALTER STAGE %v SET STORAGE_INTEGRATION = %v`, sb.QualifiedName(), s) +} + // ChangeEncryption returns the SQL query that will update the encryption on the stage. func (sb *StageBuilder) ChangeEncryption(e string) string { return fmt.Sprintf(`ALTER STAGE %v SET ENCRYPTION = (%v)`, sb.QualifiedName(), e) diff --git a/pkg/snowflake/stage_test.go b/pkg/snowflake/stage_test.go index 641f047cd0..8dfb56d8ee 100644 --- a/pkg/snowflake/stage_test.go +++ b/pkg/snowflake/stage_test.go @@ -30,6 +30,9 @@ func TestStageCreate(t *testing.T) { s.WithComment("Yeehaw") a.Equal(s.Create(), `CREATE STAGE "test_db"."test_schema"."test_stage" URL = 's3://load/encrypted_files/' CREDENTIALS = (aws_role='arn:aws:iam::001234567890:role/mysnowflakerole') ENCRYPTION = (type='AWS_SSE_KMS' kms_key_id = 'aws/key') FILE_FORMAT = (format_name=my_csv_format) COPY_OPTIONS = (on_error='skip_file') COMMENT = 'Yeehaw'`) + + s.WithStorageIntegration("MY_INTEGRATION") + a.Equal(s.Create(), `CREATE STAGE "test_db"."test_schema"."test_stage" URL = 's3://load/encrypted_files/' CREDENTIALS = (aws_role='arn:aws:iam::001234567890:role/mysnowflakerole') STORAGE_INTEGRATION = MY_INTEGRATION ENCRYPTION = (type='AWS_SSE_KMS' kms_key_id = 'aws/key') FILE_FORMAT = (format_name=my_csv_format) COPY_OPTIONS = (on_error='skip_file') COMMENT = 'Yeehaw'`) } func TestStageRename(t *testing.T) { @@ -68,6 +71,12 @@ func TestStageChangeCredentials(t *testing.T) { a.Equal(s.ChangeCredentials("aws_role='arn:aws:iam::001234567890:role/mysnowflakerole'"), `ALTER STAGE "test_db"."test_schema"."test_stage" SET CREDENTIALS = (aws_role='arn:aws:iam::001234567890:role/mysnowflakerole')`) } +func TestStageChangeStorageIntegration(t *testing.T) { + a := assert.New(t) + s := Stage("test_stage", "test_db", "test_schema") + a.Equal(s.ChangeStorageIntegration("MY_INTEGRATION"), `ALTER STAGE "test_db"."test_schema"."test_stage" SET STORAGE_INTEGRATION = MY_INTEGRATION`) +} + func TestStageChangeCopyOptions(t *testing.T) { a := assert.New(t) s := Stage("test_stage", "test_db", "test_schema") diff --git a/pkg/snowflake/storage_integration.go b/pkg/snowflake/storage_integration.go new file mode 100644 index 0000000000..d13a505bb7 --- /dev/null +++ b/pkg/snowflake/storage_integration.go @@ -0,0 +1,18 @@ +package snowflake + +// StorageIntegration returns a pointer to a Builder that abstracts the DDL operations for a storage integration. +// +// Supported DDL operations are: +// - CREATE STORAGE INTEGRATION +// - ALTER STORAGE INTEGRATION +// - DROP INTEGRATION +// - SHOW INTEGRATIONS +// - DESCRIBE INTEGRATION +// +// [Snowflake Reference](https://docs.snowflake.net/manuals/sql-reference/ddl-user-security.html#storage-integrations) +func StorageIntegration(name string) *Builder { + return &Builder{ + entityType: StorageIntegrationType, + name: name, + } +} diff --git a/pkg/snowflake/storage_integration_test.go b/pkg/snowflake/storage_integration_test.go new file mode 100644 index 0000000000..e4bcfce340 --- /dev/null +++ b/pkg/snowflake/storage_integration_test.go @@ -0,0 +1,26 @@ +package snowflake_test + +import ( + "testing" + + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" + "github.com/stretchr/testify/assert" +) + +func TestStorageIntegration(t *testing.T) { + a := assert.New(t) + builder := snowflake.StorageIntegration("aws") + a.NotNil(builder) + + q := builder.Show() + a.Equal("SHOW STORAGE INTEGRATIONS LIKE 'aws'", q) + + c := builder.Create() + + c.SetString(`type`, `EXTERNAL_STAGE`) + c.SetStringList(`storage_allowed_locations`, []string{"s3://my-bucket/my-path/", "s3://another-bucket/"}) + c.SetBool(`enabled`, true) + q = c.Statement() + + a.Equal(`CREATE STORAGE INTEGRATION "aws" TYPE='EXTERNAL_STAGE' STORAGE_ALLOWED_LOCATIONS=('s3://my-bucket/my-path/', 's3://another-bucket/') ENABLED=true`, q) +}