From 9f3b761e2a62c1e45f368879f22b9451c734d869 Mon Sep 17 00:00:00 2001 From: Tom Bamford Date: Tue, 9 Apr 2024 19:38:27 +0100 Subject: [PATCH 1/5] azurerm_storage_container: support for the `default_encryption_scope` and `encryption_scope_override_enabled` properties --- internal/services/storage/shim/containers.go | 10 +++--- .../storage/shim/containers_data_plane.go | 10 +++--- .../storage/storage_container_data_source.go | 13 +++++++ .../storage_container_data_source_test.go | 24 +++++++++---- .../storage/storage_container_resource.go | 28 +++++++++++++++ .../storage_container_resource_test.go | 36 +++++++++++++++++++ .../docs/d/storage_container.html.markdown | 4 +++ .../docs/r/storage_container.html.markdown | 4 +++ 8 files changed, 114 insertions(+), 15 deletions(-) diff --git a/internal/services/storage/shim/containers.go b/internal/services/storage/shim/containers.go index c29325ce4aa2..73677c93e3af 100644 --- a/internal/services/storage/shim/containers.go +++ b/internal/services/storage/shim/containers.go @@ -19,8 +19,10 @@ type StorageContainerWrapper interface { } type StorageContainerProperties struct { - AccessLevel containers.AccessLevel - MetaData map[string]string - HasImmutabilityPolicy bool - HasLegalHold bool + AccessLevel containers.AccessLevel + DefaultEncryptionScope string + EncryptionScopeOverrideDisabled bool + MetaData map[string]string + HasImmutabilityPolicy bool + HasLegalHold bool } diff --git a/internal/services/storage/shim/containers_data_plane.go b/internal/services/storage/shim/containers_data_plane.go index e365ef9819fb..3a0aa10b27d9 100644 --- a/internal/services/storage/shim/containers_data_plane.go +++ b/internal/services/storage/shim/containers_data_plane.go @@ -60,10 +60,12 @@ func (w DataPlaneStorageContainerWrapper) Get(ctx context.Context, containerName } return &StorageContainerProperties{ - AccessLevel: props.AccessLevel, - MetaData: props.MetaData, - HasImmutabilityPolicy: props.HasImmutabilityPolicy, - HasLegalHold: props.HasLegalHold, + AccessLevel: props.AccessLevel, + DefaultEncryptionScope: props.DefaultEncryptionScope, + EncryptionScopeOverrideDisabled: props.EncryptionScopeOverrideDisabled, + MetaData: props.MetaData, + HasImmutabilityPolicy: props.HasImmutabilityPolicy, + HasLegalHold: props.HasLegalHold, }, nil } diff --git a/internal/services/storage/storage_container_data_source.go b/internal/services/storage/storage_container_data_source.go index 2cb74d84d77e..b82840f72d9c 100644 --- a/internal/services/storage/storage_container_data_source.go +++ b/internal/services/storage/storage_container_data_source.go @@ -40,6 +40,16 @@ func dataSourceStorageContainer() *pluginsdk.Resource { Computed: true, }, + "default_encryption_scope": { + Type: pluginsdk.TypeString, + Computed: true, + }, + + "encryption_scope_override_enabled": { + Type: pluginsdk.TypeBool, + Computed: true, + }, + "metadata": MetaDataComputedSchema(), // TODO: support for ACL's, Legal Holds and Immutability Policies @@ -111,6 +121,9 @@ func dataSourceStorageContainerRead(d *pluginsdk.ResourceData, meta interface{}) d.Set("storage_account_name", accountName) d.Set("container_access_type", flattenStorageContainerAccessLevel(props.AccessLevel)) + d.Set("default_encryption_scope", props.DefaultEncryptionScope) + d.Set("encryption_scope_override_enabled", !props.EncryptionScopeOverrideDisabled) + if err = d.Set("metadata", FlattenMetaData(props.MetaData)); err != nil { return fmt.Errorf("setting `metadata`: %v", err) } diff --git a/internal/services/storage/storage_container_data_source_test.go b/internal/services/storage/storage_container_data_source_test.go index b3bff3787014..b878446c32c6 100644 --- a/internal/services/storage/storage_container_data_source_test.go +++ b/internal/services/storage/storage_container_data_source_test.go @@ -22,6 +22,8 @@ func TestAccDataSourceStorageContainer_basic(t *testing.T) { Check: acceptance.ComposeTestCheckFunc( check.That(data.ResourceName).Key("container_access_type").HasValue("private"), check.That(data.ResourceName).Key("has_immutability_policy").HasValue("false"), + check.That(data.ResourceName).Key("default_encryption_scope").HasValue(fmt.Sprintf("acctestEScontainer%d", data.RandomInteger)), + check.That(data.ResourceName).Key("encryption_scope_override_enabled").HasValue("true"), check.That(data.ResourceName).Key("metadata.%").HasValue("2"), check.That(data.ResourceName).Key("metadata.k1").HasValue("v1"), check.That(data.ResourceName).Key("metadata.k2").HasValue("v2"), @@ -37,12 +39,12 @@ provider "azurerm" { } resource "azurerm_resource_group" "test" { - name = "containerdstest-%s" - location = "%s" + name = "containerdstest-%[1]s" + location = "%[2]s" } resource "azurerm_storage_account" "test" { - name = "acctestsadsc%s" + name = "acctestsadsc%[1]s" resource_group_name = "${azurerm_resource_group.test.name}" location = "${azurerm_resource_group.test.location}" @@ -50,10 +52,18 @@ resource "azurerm_storage_account" "test" { account_replication_type = "LRS" } +resource "azurerm_storage_encryption_scope" "test" { + name = "acctestEScontainer%[3]d" + storage_account_id = azurerm_storage_account.test.id + source = "Microsoft.Storage" +} + resource "azurerm_storage_container" "test" { - name = "containerdstest-%s" - storage_account_name = "${azurerm_storage_account.test.name}" - container_access_type = "private" + name = "containerdstest-%[1]s" + storage_account_name = "${azurerm_storage_account.test.name}" + container_access_type = "private" + default_encryption_scope = azurerm_storage_encryption_scope.test.name + encryption_scope_override_enabled = true metadata = { k1 = "v1" k2 = "v2" @@ -64,5 +74,5 @@ data "azurerm_storage_container" "test" { name = azurerm_storage_container.test.name storage_account_name = azurerm_storage_container.test.storage_account_name } -`, data.RandomString, data.Locations.Primary, data.RandomString, data.RandomString) +`, data.RandomString, data.Locations.Primary, data.RandomInteger) } diff --git a/internal/services/storage/storage_container_resource.go b/internal/services/storage/storage_container_resource.go index 4196064ca621..4515f646b3ac 100644 --- a/internal/services/storage/storage_container_resource.go +++ b/internal/services/storage/storage_container_resource.go @@ -72,6 +72,22 @@ func resourceStorageContainer() *pluginsdk.Resource { }, false), }, + "default_encryption_scope": { + Type: pluginsdk.TypeString, + Optional: true, + Computed: true, // needed because a dummy value is returned when unspecified + ForceNew: true, + ValidateFunc: validate.StorageEncryptionScopeName, + }, + + "encryption_scope_override_enabled": { + Type: pluginsdk.TypeBool, + Optional: true, + Default: true, // defaulting to false would be preferable here, but the API defaults this to true when unspecified + ForceNew: true, + RequiredWith: []string{"default_encryption_scope"}, + }, + "metadata": MetaDataComputedSchema(), // TODO: support for ACL's, Legal Holds and Immutability Policies @@ -148,6 +164,15 @@ func resourceStorageContainerCreate(d *pluginsdk.ResourceData, meta interface{}) MetaData: metaData, } + if encryptionScope := d.Get("default_encryption_scope"); encryptionScope.(string) != "" { + input.DefaultEncryptionScope = encryptionScope.(string) + input.EncryptionScopeOverrideDisabled = false + + if encryptionScopeOverrideEnabled := d.Get("encryption_scope_override_enabled"); !encryptionScopeOverrideEnabled.(bool) { + input.EncryptionScopeOverrideDisabled = true + } + } + if err = containersDataPlaneClient.Create(ctx, containerName, input); err != nil { return fmt.Errorf("creating %s: %v", id, err) } @@ -257,6 +282,9 @@ func resourceStorageContainerRead(d *pluginsdk.ResourceData, meta interface{}) e d.Set("container_access_type", flattenStorageContainerAccessLevel(props.AccessLevel)) + d.Set("default_encryption_scope", props.DefaultEncryptionScope) + d.Set("encryption_scope_override_enabled", !props.EncryptionScopeOverrideDisabled) + if err = d.Set("metadata", FlattenMetaData(props.MetaData)); err != nil { return fmt.Errorf("setting `metadata`: %v", err) } diff --git a/internal/services/storage/storage_container_resource_test.go b/internal/services/storage/storage_container_resource_test.go index 4590fe642c0e..072c351c40d4 100644 --- a/internal/services/storage/storage_container_resource_test.go +++ b/internal/services/storage/storage_container_resource_test.go @@ -113,6 +113,21 @@ func TestAccStorageContainer_update(t *testing.T) { }) } +func TestAccStorageContainer_encryptionScope(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_storage_container", "test") + r := StorageContainerResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.encryptionScope(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + func TestAccStorageContainer_metaData(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_storage_container", "test") r := StorageContainerResource{} @@ -314,6 +329,27 @@ resource "azurerm_storage_container" "test" { `, template, accessType, metadataVal) } +func (r StorageContainerResource) encryptionScope(data acceptance.TestData) string { + template := r.template(data) + return fmt.Sprintf(` +%[1]s + +resource "azurerm_storage_encryption_scope" "test" { + name = "acctestEScontainer%[2]d" + storage_account_id = azurerm_storage_account.test.id + source = "Microsoft.Storage" +} + +resource "azurerm_storage_container" "test" { + name = "vhds" + storage_account_name = azurerm_storage_account.test.name + container_access_type = "private" + + default_encryption_scope = azurerm_storage_encryption_scope.test.name +} +`, template, data.RandomInteger) +} + func (r StorageContainerResource) metaData(data acceptance.TestData) string { template := r.template(data) return fmt.Sprintf(` diff --git a/website/docs/d/storage_container.html.markdown b/website/docs/d/storage_container.html.markdown index e570dd1e9047..62be44b89610 100644 --- a/website/docs/d/storage_container.html.markdown +++ b/website/docs/d/storage_container.html.markdown @@ -31,6 +31,10 @@ The following arguments are supported: * `container_access_type` - The Access Level configured for this Container. +* `default_encryption_scope` - The default encryption scope in use for blobs uploaded to this container. + +* `encryption_scope_override_enabled` - Whether blobs are allowed to override the default encryption scope for this container. + * `has_immutability_policy` - Is there an Immutability Policy configured on this Storage Container? * `has_legal_hold` - Is there a Legal Hold configured on this Storage Container? diff --git a/website/docs/r/storage_container.html.markdown b/website/docs/r/storage_container.html.markdown index 2136d1f4fdca..bad59ba65a3b 100644 --- a/website/docs/r/storage_container.html.markdown +++ b/website/docs/r/storage_container.html.markdown @@ -49,6 +49,10 @@ The following arguments are supported: ~> **Note** When updating `container_access_type` for an existing storage container resource, Shared Key authentication will always be used, as AzureAD authentication is not supported. +* `default_encryption_scope` - (Optional) The default encryption scope to use for blobs uploaded to this container. Changing this forces a new resource to be created. + +* `encryption_scope_override_enabled` - (Optional) Whether to allow blobs to override the default encryption scope for this container. Can only be set when specifying `default_encryption_scope`. Defaults to `true`. Changing this forces a new resource to be created. + * `metadata` - (Optional) A mapping of MetaData for this Container. All metadata keys should be lowercase. ## Attributes Reference From 5efa7599e3c0ca3d713277c4a2207d960a697bf8 Mon Sep 17 00:00:00 2001 From: Tom Bamford Date: Tue, 9 Apr 2024 19:39:30 +0100 Subject: [PATCH 2/5] azurerm_storage_blob: support for the `encryption_scope` property --- internal/services/storage/blobs.go | 53 +++++++++++++------ .../storage/storage_blob_data_source.go | 7 +++ .../storage/storage_blob_data_source_test.go | 20 ++++--- .../services/storage/storage_blob_resource.go | 14 +++++ .../storage/storage_blob_resource_test.go | 44 +++++++++++++++ website/docs/d/storage_blob.html.markdown | 2 + website/docs/r/storage_blob.html.markdown | 2 + 7 files changed, 119 insertions(+), 23 deletions(-) diff --git a/internal/services/storage/blobs.go b/internal/services/storage/blobs.go index 70e59082642b..95abec03616f 100644 --- a/internal/services/storage/blobs.go +++ b/internal/services/storage/blobs.go @@ -15,7 +15,7 @@ import ( "strings" "sync" - "github.com/hashicorp/terraform-provider-azurerm/utils" + "github.com/hashicorp/go-azure-helpers/lang/pointer" "github.com/tombuildsstuff/giovanni/storage/2023-11-03/blob/blobs" ) @@ -26,16 +26,17 @@ type BlobUpload struct { BlobName string ContainerName string - BlobType string - CacheControl string - ContentType string - ContentMD5 string - MetaData map[string]string - Parallelism int - Size int - Source string - SourceContent string - SourceUri string + BlobType string + CacheControl string + ContentType string + ContentMD5 string + EncryptionScope string + MetaData map[string]string + Parallelism int + Size int + Source string + SourceContent string + SourceUri string } func (sbu BlobUpload) Create(ctx context.Context) error { @@ -93,6 +94,9 @@ func (sbu BlobUpload) copy(ctx context.Context) error { CopySource: sbu.SourceUri, MetaData: sbu.MetaData, } + if sbu.EncryptionScope != "" { + input.EncryptionScope = pointer.To(sbu.EncryptionScope) + } if err := sbu.Client.CopyAndWait(ctx, sbu.ContainerName, sbu.BlobName, input); err != nil { return fmt.Errorf("copy/waiting: %s", err) } @@ -102,9 +106,12 @@ func (sbu BlobUpload) copy(ctx context.Context) error { func (sbu BlobUpload) createEmptyAppendBlob(ctx context.Context) error { input := blobs.PutAppendBlobInput{ - ContentType: utils.String(sbu.ContentType), + ContentType: pointer.To(sbu.ContentType), MetaData: sbu.MetaData, } + if sbu.EncryptionScope != "" { + input.EncryptionScope = pointer.To(sbu.EncryptionScope) + } if _, err := sbu.Client.PutAppendBlob(ctx, sbu.ContainerName, sbu.BlobName, input); err != nil { return fmt.Errorf("PutAppendBlob: %s", err) } @@ -118,9 +125,12 @@ func (sbu BlobUpload) createEmptyBlockBlob(ctx context.Context) error { } input := blobs.PutBlockBlobInput{ - ContentType: utils.String(sbu.ContentType), + ContentType: pointer.To(sbu.ContentType), MetaData: sbu.MetaData, } + if sbu.EncryptionScope != "" { + input.EncryptionScope = pointer.To(sbu.EncryptionScope) + } if _, err := sbu.Client.PutBlockBlob(ctx, sbu.ContainerName, sbu.BlobName, input); err != nil { return fmt.Errorf("PutBlockBlob: %s", err) } @@ -152,11 +162,14 @@ func (sbu BlobUpload) uploadBlockBlob(ctx context.Context) error { defer file.Close() input := blobs.PutBlockBlobInput{ - ContentType: utils.String(sbu.ContentType), + ContentType: pointer.To(sbu.ContentType), MetaData: sbu.MetaData, } if sbu.ContentMD5 != "" { - input.ContentMD5 = utils.String(sbu.ContentMD5) + input.ContentMD5 = pointer.To(sbu.ContentMD5) + } + if sbu.EncryptionScope != "" { + input.EncryptionScope = pointer.To(sbu.EncryptionScope) } if err := sbu.Client.PutBlockBlobFromFile(ctx, sbu.ContainerName, sbu.BlobName, file, input); err != nil { return fmt.Errorf("PutBlockBlobFromFile: %s", err) @@ -172,9 +185,12 @@ func (sbu BlobUpload) createEmptyPageBlob(ctx context.Context) error { input := blobs.PutPageBlobInput{ BlobContentLengthBytes: int64(sbu.Size), - ContentType: utils.String(sbu.ContentType), + ContentType: pointer.To(sbu.ContentType), MetaData: sbu.MetaData, } + if sbu.EncryptionScope != "" { + input.EncryptionScope = pointer.To(sbu.EncryptionScope) + } if _, err := sbu.Client.PutPageBlob(ctx, sbu.ContainerName, sbu.BlobName, input); err != nil { return fmt.Errorf("PutPageBlob: %s", err) } @@ -222,9 +238,12 @@ func (sbu BlobUpload) uploadPageBlob(ctx context.Context) error { // first let's create a file of the specified file size input := blobs.PutPageBlobInput{ BlobContentLengthBytes: fileSize, - ContentType: utils.String(sbu.ContentType), + ContentType: pointer.To(sbu.ContentType), MetaData: sbu.MetaData, } + if sbu.EncryptionScope != "" { + input.EncryptionScope = pointer.To(sbu.EncryptionScope) + } if _, err := sbu.Client.PutPageBlob(ctx, sbu.ContainerName, sbu.BlobName, input); err != nil { return fmt.Errorf("PutPageBlob: %s", err) } diff --git a/internal/services/storage/storage_blob_data_source.go b/internal/services/storage/storage_blob_data_source.go index 0cd3e05a2481..a1fc8866e86e 100644 --- a/internal/services/storage/storage_blob_data_source.go +++ b/internal/services/storage/storage_blob_data_source.go @@ -63,6 +63,11 @@ func dataSourceStorageBlob() *pluginsdk.Resource { Computed: true, }, + "encryption_scope": { + Type: pluginsdk.TypeString, + Computed: true, + }, + "url": { Type: pluginsdk.TypeString, Computed: true, @@ -136,6 +141,8 @@ func dataSourceStorageBlobRead(d *pluginsdk.ResourceData, meta interface{}) erro } d.Set("content_md5", contentMD5) + d.Set("encryption_scope", props.EncryptionScope) + d.Set("type", strings.TrimSuffix(string(props.BlobType), "Blob")) d.SetId(id.ID()) diff --git a/internal/services/storage/storage_blob_data_source_test.go b/internal/services/storage/storage_blob_data_source_test.go index 57cd3e0666ef..308bf41d5c44 100644 --- a/internal/services/storage/storage_blob_data_source_test.go +++ b/internal/services/storage/storage_blob_data_source_test.go @@ -30,6 +30,7 @@ func TestAccDataSourceStorageBlob_basic(t *testing.T) { { Config: StorageBlobDataSource{}.basicWithDataSource(data, sourceBlob.Name()), Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).Key("encryption_scope").HasValue(fmt.Sprintf("acctestEScontainer%d", data.RandomInteger)), check.That(data.ResourceName).Key("type").HasValue("Block"), check.That(data.ResourceName).Key("metadata.%").HasValue("2"), check.That(data.ResourceName).Key("metadata.k1").HasValue("v1"), @@ -46,12 +47,12 @@ provider "azurerm" { } resource "azurerm_resource_group" "test" { - name = "blobdstest-%s" - location = "%s" + name = "blobdstest-%[1]s" + location = "%[2]s" } resource "azurerm_storage_account" "test" { - name = "acctestsadsc%s" + name = "acctestsadsc%[1]s" resource_group_name = "${azurerm_resource_group.test.name}" location = "${azurerm_resource_group.test.location}" @@ -59,8 +60,14 @@ resource "azurerm_storage_account" "test" { account_replication_type = "LRS" } +resource "azurerm_storage_encryption_scope" "test" { + name = "acctestEScontainer%[3]d" + storage_account_id = azurerm_storage_account.test.id + source = "Microsoft.Storage" +} + resource "azurerm_storage_container" "test" { - name = "containerdstest-%s" + name = "containerdstest-%[1]s" storage_account_name = "${azurerm_storage_account.test.name}" container_access_type = "private" } @@ -69,15 +76,16 @@ resource "azurerm_storage_blob" "test" { name = "example.vhd" storage_account_name = azurerm_storage_account.test.name storage_container_name = azurerm_storage_container.test.name + encryption_scope = azurerm_storage_encryption_scope.test.name type = "Block" - source = "%s" + source = "%[4]s" metadata = { k1 = "v1" k2 = "v2" } } -`, data.RandomString, data.Locations.Primary, data.RandomString, data.RandomString, fileName) +`, data.RandomString, data.Locations.Primary, data.RandomInteger, fileName) } func (d StorageBlobDataSource) basicWithDataSource(data acceptance.TestData, fileName string) string { diff --git a/internal/services/storage/storage_blob_resource.go b/internal/services/storage/storage_blob_resource.go index 181ffbb3c8d1..d306f3a57c0d 100644 --- a/internal/services/storage/storage_blob_resource.go +++ b/internal/services/storage/storage_blob_resource.go @@ -111,6 +111,13 @@ func resourceStorageBlob() *pluginsdk.Resource { Optional: true, }, + "encryption_scope": { + Type: pluginsdk.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validate.StorageEncryptionScopeName, + }, + "source": { Type: pluginsdk.TypeString, Optional: true, @@ -239,6 +246,11 @@ func resourceStorageBlobCreate(d *pluginsdk.ResourceData, meta interface{}) erro SourceContent: d.Get("source_content").(string), SourceUri: d.Get("source_uri").(string), } + + if encryptionScope := d.Get("encryption_scope"); encryptionScope.(string) != "" { + blobInput.EncryptionScope = encryptionScope.(string) + } + if err = blobInput.Create(ctx); err != nil { return fmt.Errorf("creating %s: %v", id, err) } @@ -380,6 +392,8 @@ func resourceStorageBlobRead(d *pluginsdk.ResourceData, meta interface{}) error } d.Set("content_md5", contentMD5) + d.Set("encryption_scope", props.EncryptionScope) + d.Set("type", strings.TrimSuffix(string(props.BlobType), "Blob")) d.Set("url", d.Id()) diff --git a/internal/services/storage/storage_blob_resource_test.go b/internal/services/storage/storage_blob_resource_test.go index 970dec107b74..ebe8a8a8379c 100644 --- a/internal/services/storage/storage_blob_resource_test.go +++ b/internal/services/storage/storage_blob_resource_test.go @@ -328,6 +328,22 @@ func TestAccStorageBlob_contentTypePremium(t *testing.T) { }) } +func TestAccStorageBlob_encryptionScope(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_storage_blob", "test") + r := StorageBlobResource{} + content := "Wubba Lubba Dub Dub" + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.encryptionScope(data, content), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("parallelism", "size", "source_content", "type"), + }) +} + func TestAccStorageBlob_pageEmpty(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_storage_blob", "test") r := StorageBlobResource{} @@ -945,6 +961,34 @@ resource "azurerm_storage_blob" "test" { `, template) } +func (r StorageBlobResource) encryptionScope(data acceptance.TestData, content string) string { + template := r.template(data, "blob") + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +%[1]s + +resource "azurerm_storage_encryption_scope" "test" { + name = "acctestEScontainer%[2]d" + storage_account_id = azurerm_storage_account.test.id + source = "Microsoft.Storage" +} + +resource "azurerm_storage_blob" "test" { + name = "rick.morty" + storage_account_name = azurerm_storage_account.test.name + storage_container_name = azurerm_storage_container.test.name + type = "Block" + encryption_scope = azurerm_storage_encryption_scope.test.name + source_content = < **NOTE:** This property is intended to be used with the Terraform internal [filemd5](https://www.terraform.io/docs/configuration/functions/filemd5.html) and [md5](https://www.terraform.io/docs/configuration/functions/md5.html) functions when `source` or `source_content`, respectively, are defined. +* `encryption_scope` - (Optional) The encryption scope to use for this blob. + * `source` - (Optional) An absolute path to a file on the local system. This field cannot be specified for Append blobs and cannot be specified if `source_content` or `source_uri` is specified. Changing this forces a new resource to be created. * `source_content` - (Optional) The content for this blob which should be defined inline. This field can only be specified for Block blobs and cannot be specified if `source` or `source_uri` is specified. Changing this forces a new resource to be created. From b8ff4a8a057b1b550758b9a72ef84fd8a07e9b46 Mon Sep 17 00:00:00 2001 From: Tom Bamford Date: Tue, 9 Apr 2024 21:06:10 +0100 Subject: [PATCH 3/5] dependencies: updating to `v0.26.1` of `github.com/tombuildsstuff/giovanni` --- go.mod | 2 +- go.sum | 4 ++-- .../storage/2023-11-03/blob/blobs/metadata_set.go | 6 ++++++ .../2023-11-03/datalakestore/filesystems/create.go | 14 +++++++++++--- .../datalakestore/filesystems/properties_get.go | 5 +++++ vendor/modules.txt | 2 +- 6 files changed, 26 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 43d3c56b8900..c44c9a34fba4 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/rickb777/date v1.12.5-0.20200422084442-6300e543c4d9 github.com/sergi/go-diff v1.2.0 - github.com/tombuildsstuff/giovanni v0.25.5 + github.com/tombuildsstuff/giovanni v0.26.1 github.com/tombuildsstuff/kermit v0.20240122.1123108 golang.org/x/crypto v0.18.0 golang.org/x/tools v0.13.0 diff --git a/go.sum b/go.sum index 3d6ff64ceea2..9db169f9cf35 100644 --- a/go.sum +++ b/go.sum @@ -225,8 +225,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tombuildsstuff/giovanni v0.25.5 h1:YjVuiurzRAZDwTsWtelYOJV3373fMHXEazirh6h7nhU= -github.com/tombuildsstuff/giovanni v0.25.5/go.mod h1:s7xbU2lN5Iz9MBglmDDv9p2QPbn6x3UkJBtpCfUerLs= +github.com/tombuildsstuff/giovanni v0.26.1 h1:RZgnpyIHtgw0GXYpw3xttNk35obJNoI1hztCZsh/Djo= +github.com/tombuildsstuff/giovanni v0.26.1/go.mod h1:s7xbU2lN5Iz9MBglmDDv9p2QPbn6x3UkJBtpCfUerLs= github.com/tombuildsstuff/kermit v0.20240122.1123108 h1:icQaxsv/ANv/KC4Sr0V1trrWA/XIL+3QAVBDpiSTgj8= github.com/tombuildsstuff/kermit v0.20240122.1123108/go.mod h1:T3YBVFhRV4qA7SbnRaNE6eapIMpKDA9rG/V7Ocsjlno= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= diff --git a/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/blob/blobs/metadata_set.go b/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/blob/blobs/metadata_set.go index a9f9c00eee58..94f8918c156f 100644 --- a/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/blob/blobs/metadata_set.go +++ b/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/blob/blobs/metadata_set.go @@ -18,6 +18,9 @@ type SetMetaDataInput struct { // Any metadata which should be added to this blob MetaData map[string]string + + // The encryption scope for the blob. + EncryptionScope *string } type SetMetaDataResponse struct { @@ -85,6 +88,9 @@ func (s setMetadataOptions) ToHeaders() *client.Headers { if s.input.LeaseID != nil { headers.Append("x-ms-lease-id", *s.input.LeaseID) } + if s.input.EncryptionScope != nil { + headers.Append("x-ms-encryption-scope", *s.input.EncryptionScope) + } headers.Merge(metadata.SetMetaDataHeaders(s.input.MetaData)) return headers } diff --git a/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/datalakestore/filesystems/create.go b/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/datalakestore/filesystems/create.go index 9472e1e42880..a250c07c4fd8 100644 --- a/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/datalakestore/filesystems/create.go +++ b/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/datalakestore/filesystems/create.go @@ -10,6 +10,9 @@ import ( ) type CreateInput struct { + // The encryption scope to set as the default on the filesystem. + DefaultEncryptionScope string + // A map of base64-encoded strings to store as user-defined properties with the File System // Note that items may only contain ASCII characters in the ISO-8859-1 character set. // This automatically gets converted to a comma-separated list of name and @@ -35,7 +38,7 @@ func (c Client) Create(ctx context.Context, fileSystemName string, input CreateI }, HttpMethod: http.MethodPut, OptionsObject: createOptions{ - properties: input.Properties, + input: input, }, Path: fmt.Sprintf("/%s", fileSystemName), @@ -61,12 +64,17 @@ func (c Client) Create(ctx context.Context, fileSystemName string, input CreateI } type createOptions struct { - properties map[string]string + input CreateInput } func (o createOptions) ToHeaders() *client.Headers { headers := &client.Headers{} - props := buildProperties(o.properties) + + if o.input.DefaultEncryptionScope != "" { + headers.Append("x-ms-default-encryption-scope", o.input.DefaultEncryptionScope) + } + + props := buildProperties(o.input.Properties) if props != "" { headers.Append("x-ms-properties", props) } diff --git a/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/datalakestore/filesystems/properties_get.go b/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/datalakestore/filesystems/properties_get.go index d5d424e912c3..eae1c95d34e0 100644 --- a/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/datalakestore/filesystems/properties_get.go +++ b/vendor/github.com/tombuildsstuff/giovanni/storage/2023-11-03/datalakestore/filesystems/properties_get.go @@ -12,6 +12,9 @@ import ( type GetPropertiesResponse struct { HttpResponse *http.Response + // The default encryption scope for the filesystem. + DefaultEncryptionScope string + // A map of base64-encoded strings to store as user-defined properties with the File System // Note that items may only contain ASCII characters in the ISO-8859-1 character set. // This automatically gets converted to a comma-separated list of name and @@ -51,6 +54,8 @@ func (c Client) GetProperties(ctx context.Context, fileSystemName string) (resul result.HttpResponse = resp.Response if resp.Header != nil { + result.DefaultEncryptionScope = resp.Header.Get("x-ms-default-encryption-scope") + propertiesRaw := resp.Header.Get("x-ms-properties") var properties *map[string]string properties, err = parseProperties(propertiesRaw) diff --git a/vendor/modules.txt b/vendor/modules.txt index 973bb90c91f8..377df1c3664e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1277,7 +1277,7 @@ github.com/rickb777/plural # github.com/sergi/go-diff v1.2.0 ## explicit; go 1.12 github.com/sergi/go-diff/diffmatchpatch -# github.com/tombuildsstuff/giovanni v0.25.5 +# github.com/tombuildsstuff/giovanni v0.26.1 ## explicit; go 1.21 github.com/tombuildsstuff/giovanni/storage/2023-11-03/blob/accounts github.com/tombuildsstuff/giovanni/storage/2023-11-03/blob/blobs From 7f9a098a17404b467658f335656b9b8d4a0ad224 Mon Sep 17 00:00:00 2001 From: Tom Bamford Date: Tue, 9 Apr 2024 20:50:35 +0100 Subject: [PATCH 4/5] azurerm_storage_data_lake_gen2_filesystem: support for the `default_encryption_scope` property --- ...rage_data_lake_gen2_filesystem_resource.go | 13 +++++++ ...data_lake_gen2_filesystem_resource_test.go | 35 +++++++++++++++++++ ...ge_data_lake_gen2_filesystem.html.markdown | 2 ++ 3 files changed, 50 insertions(+) diff --git a/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go b/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go index a722c241fb29..8285294c6d41 100644 --- a/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go +++ b/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go @@ -83,6 +83,14 @@ func resourceStorageDataLakeGen2FileSystem() *pluginsdk.Resource { "properties": MetaDataSchema(), + "default_encryption_scope": { + Type: pluginsdk.TypeString, + Optional: true, + Computed: true, // needed because a dummy value is returned when unspecified + ForceNew: true, + ValidateFunc: validate.StorageEncryptionScopeName, + }, + "owner": { Type: pluginsdk.TypeString, Optional: true, @@ -207,6 +215,10 @@ func resourceStorageDataLakeGen2FileSystemCreate(d *pluginsdk.ResourceData, meta input := filesystems.CreateInput{ Properties: properties, } + if encryptionScope := d.Get("default_encryption_scope"); encryptionScope.(string) != "" { + input.DefaultEncryptionScope = encryptionScope.(string) + } + if _, err = dataPlaneFilesystemsClient.Create(ctx, id.FileSystemName, input); err != nil { return fmt.Errorf("creating %s: %v", id, err) } @@ -368,6 +380,7 @@ func resourceStorageDataLakeGen2FileSystemRead(d *pluginsdk.ResourceData, meta i } d.Set("name", id.FileSystemName) + d.Set("default_encryption_scope", resp.DefaultEncryptionScope) if err = d.Set("properties", resp.Properties); err != nil { return fmt.Errorf("setting `properties`: %v", err) diff --git a/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go b/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go index c93968fe5135..6b62ea3a8f5a 100644 --- a/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go +++ b/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go @@ -86,6 +86,21 @@ func TestAccStorageDataLakeGen2FileSystem_UpdateDefaultACL(t *testing.T) { }) } +func TestAccStorageDataLakeGen2FileSystem_encryptionScope(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_storage_data_lake_gen2_filesystem", "test") + r := StorageDataLakeGen2FileSystemResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.encryptionScope(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + func TestAccStorageDataLakeGen2FileSystem_properties(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_storage_data_lake_gen2_filesystem", "test") r := StorageDataLakeGen2FileSystemResource{} @@ -227,6 +242,26 @@ resource "azurerm_storage_data_lake_gen2_filesystem" "import" { `, template) } +func (r StorageDataLakeGen2FileSystemResource) encryptionScope(data acceptance.TestData) string { + template := r.template(data) + return fmt.Sprintf(` +%[1]s + +resource "azurerm_storage_encryption_scope" "test" { + name = "acctestEScontainer%[2]d" + storage_account_id = azurerm_storage_account.test.id + source = "Microsoft.Storage" +} + +resource "azurerm_storage_data_lake_gen2_filesystem" "test" { + name = "acctest-%[2]d" + storage_account_id = azurerm_storage_account.test.id + + default_encryption_scope = azurerm_storage_encryption_scope.test.name +} +`, template, data.RandomInteger) +} + func (r StorageDataLakeGen2FileSystemResource) properties(data acceptance.TestData, value string) string { template := r.template(data) return fmt.Sprintf(` diff --git a/website/docs/r/storage_data_lake_gen2_filesystem.html.markdown b/website/docs/r/storage_data_lake_gen2_filesystem.html.markdown index 44040e7ab36b..3810ee53420a 100644 --- a/website/docs/r/storage_data_lake_gen2_filesystem.html.markdown +++ b/website/docs/r/storage_data_lake_gen2_filesystem.html.markdown @@ -48,6 +48,8 @@ The following arguments are supported: * `storage_account_id` - (Required) Specifies the ID of the Storage Account in which the Data Lake Gen2 File System should exist. Changing this forces a new resource to be created. +* `default_encryption_scope` - (Optional) The default encryption scope to use for this filesystem. Changing this forces a new resource to be created. + * `properties` - (Optional) A mapping of Key to Base64-Encoded Values which should be assigned to this Data Lake Gen2 File System. * `ace` - (Optional) One or more `ace` blocks as defined below to specify the entries for the ACL for the path. From 4d1d6222dac9fbedc20cb7caa9ddf56931f34745 Mon Sep 17 00:00:00 2001 From: Tom Bamford Date: Tue, 9 Apr 2024 23:50:49 +0100 Subject: [PATCH 5/5] bugfix: encryption scope must be specified when updating storage blog metadata --- .../services/storage/storage_blob_resource.go | 27 +++- .../storage/storage_blob_resource_test.go | 138 ++++++++++++++++++ 2 files changed, 159 insertions(+), 6 deletions(-) diff --git a/internal/services/storage/storage_blob_resource.go b/internal/services/storage/storage_blob_resource.go index d306f3a57c0d..56db509abe3c 100644 --- a/internal/services/storage/storage_blob_resource.go +++ b/internal/services/storage/storage_blob_resource.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/hashicorp/go-azure-helpers/lang/pointer" "github.com/hashicorp/go-azure-helpers/lang/response" "github.com/hashicorp/terraform-provider-azurerm/helpers/tf" "github.com/hashicorp/terraform-provider-azurerm/internal/clients" @@ -19,7 +20,6 @@ import ( "github.com/hashicorp/terraform-provider-azurerm/internal/tf/pluginsdk" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/validation" "github.com/hashicorp/terraform-provider-azurerm/internal/timeouts" - "github.com/hashicorp/terraform-provider-azurerm/utils" "github.com/tombuildsstuff/giovanni/storage/2023-11-03/blob/accounts" "github.com/tombuildsstuff/giovanni/storage/2023-11-03/blob/blobs" ) @@ -285,11 +285,24 @@ func resourceStorageBlobUpdate(d *pluginsdk.ResourceData, meta interface{}) erro return fmt.Errorf("building Blobs Client: %v", err) } + log.Printf("[INFO] Retrieving %s", id) + input := blobs.GetPropertiesInput{} + props, err := blobsClient.GetProperties(ctx, id.ContainerName, id.BlobName, input) + if err != nil { + if response.WasNotFound(props.HttpResponse) { + log.Printf("[INFO] Blob %q was not found in Container %q / Account %q - assuming removed & removing from state...", id.BlobName, id.ContainerName, id.AccountId.AccountName) + d.SetId("") + return nil + } + + return fmt.Errorf("retrieving properties for %s: %v", id, err) + } + if d.HasChange("content_type") || d.HasChange("cache_control") { log.Printf("[DEBUG] Updating Properties for %s...", id) input := blobs.SetPropertiesInput{ - ContentType: utils.String(d.Get("content_type").(string)), - CacheControl: utils.String(d.Get("cache_control").(string)), + ContentType: pointer.To(d.Get("content_type").(string)), + CacheControl: pointer.To(d.Get("cache_control").(string)), } // `content_md5` is `ForceNew` but must be included in the `SetPropertiesInput` update payload, or it will be zeroed on the blob. @@ -298,10 +311,8 @@ func resourceStorageBlobUpdate(d *pluginsdk.ResourceData, meta interface{}) erro if err != nil { return fmt.Errorf("converting hex to base64 encoding for content_md5: %v", err) } - - input.ContentMD5 = utils.String(data) + input.ContentMD5 = pointer.To(data) } - if _, err = blobsClient.SetProperties(ctx, id.ContainerName, id.BlobName, input); err != nil { return fmt.Errorf("updating Properties for %s: %v", id, err) } @@ -314,6 +325,10 @@ func resourceStorageBlobUpdate(d *pluginsdk.ResourceData, meta interface{}) erro input := blobs.SetMetaDataInput{ MetaData: ExpandMetaData(metaDataRaw), } + // Encryption Scope must be specified when updating metadata + if props.EncryptionScope != "" { + input.EncryptionScope = pointer.To(props.EncryptionScope) + } if _, err = blobsClient.SetMetaData(ctx, id.ContainerName, id.BlobName, input); err != nil { return fmt.Errorf("updating MetaData for %s: %v", id, err) } diff --git a/internal/services/storage/storage_blob_resource_test.go b/internal/services/storage/storage_blob_resource_test.go index ebe8a8a8379c..d90cf6e65adc 100644 --- a/internal/services/storage/storage_blob_resource_test.go +++ b/internal/services/storage/storage_blob_resource_test.go @@ -344,6 +344,43 @@ func TestAccStorageBlob_encryptionScope(t *testing.T) { }) } +func TestAccStorageBlob_encryptionScopeUpdate(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_storage_blob", "test") + r := StorageBlobResource{} + content := "Wubba Lubba Dub Dub" + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.encryptionScope(data, content), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("parallelism", "size", "source_content", "type"), + { + Config: r.encryptionScopeUpdateMetadata(data, content), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("parallelism", "size", "source_content", "type"), + { + Config: r.encryptionScopeUpdateProperties(data, content), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("parallelism", "size", "source_content", "type"), + { + Config: r.encryptionScopeUpdateAccessTier(data, content), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("parallelism", "size", "source_content", "type"), + }) +} + func TestAccStorageBlob_pageEmpty(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_storage_blob", "test") r := StorageBlobResource{} @@ -989,6 +1026,107 @@ EOT `, template, data.RandomInteger, content) } +func (r StorageBlobResource) encryptionScopeUpdateMetadata(data acceptance.TestData, content string) string { + template := r.template(data, "blob") + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +%[1]s + +resource "azurerm_storage_encryption_scope" "test" { + name = "acctestEScontainer%[2]d" + storage_account_id = azurerm_storage_account.test.id + source = "Microsoft.Storage" +} + +resource "azurerm_storage_blob" "test" { + name = "rick.morty" + storage_account_name = azurerm_storage_account.test.name + storage_container_name = azurerm_storage_container.test.name + type = "Block" + encryption_scope = azurerm_storage_encryption_scope.test.name + source_content = <