From 51e0aa992c54367fbd4ffb51976855f739834a2c Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Thu, 17 Nov 2022 02:06:51 -0800 Subject: [PATCH 01/11] Azure Blobstorage Binding: Migrate to Track2 Azure SDK Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- bindings/azure/blobstorage/blobstorage.go | 284 ++++++++++-------- .../azure/blobstorage/blobstorage_test.go | 10 +- go.mod | 3 +- go.sum | 6 +- .../azure/blobstorage/blobstorage_test.go | 46 +-- .../bindings/azure/blobstorage/go.mod | 4 +- .../bindings/azure/blobstorage/go.sum | 8 +- .../bindings/azure/cosmosdb/go.mod | 2 +- .../bindings/azure/cosmosdb/go.sum | 4 +- .../bindings/azure/eventhubs/go.mod | 2 +- .../bindings/azure/eventhubs/go.sum | 4 +- .../bindings/azure/servicebusqueues/go.mod | 2 +- .../bindings/azure/servicebusqueues/go.sum | 4 +- .../bindings/azure/storagequeues/go.mod | 2 +- .../bindings/azure/storagequeues/go.sum | 4 +- .../pubsub/azure/eventhubs/go.mod | 2 +- .../pubsub/azure/eventhubs/go.sum | 4 +- .../pubsub/azure/servicebus/topics/go.mod | 2 +- .../pubsub/azure/servicebus/topics/go.sum | 4 +- .../secretstores/azure/keyvault/go.mod | 2 +- .../secretstores/azure/keyvault/go.sum | 4 +- .../state/azure/blobstorage/go.mod | 2 +- .../state/azure/blobstorage/go.sum | 4 +- .../certification/state/azure/cosmosdb/go.mod | 2 +- .../certification/state/azure/cosmosdb/go.sum | 4 +- .../state/azure/tablestorage/go.mod | 2 +- .../state/azure/tablestorage/go.sum | 4 +- 27 files changed, 236 insertions(+), 185 deletions(-) diff --git a/bindings/azure/blobstorage/blobstorage.go b/bindings/azure/blobstorage/blobstorage.go index ddcd2f8e03..34d10f2ae9 100644 --- a/bindings/azure/blobstorage/blobstorage.go +++ b/bindings/azure/blobstorage/blobstorage.go @@ -14,24 +14,29 @@ limitations under the License. package blobstorage import ( + "bytes" "context" b64 "encoding/base64" "encoding/json" "errors" "fmt" - "io" "net/url" "strconv" - "strings" "time" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/google/uuid" "github.com/dapr/components-contrib/bindings" azauth "github.com/dapr/components-contrib/internal/authentication/azure" mdutils "github.com/dapr/components-contrib/metadata" "github.com/dapr/kit/logger" + "github.com/dapr/kit/ptr" ) const ( @@ -63,25 +68,28 @@ const ( // Specifies the maximum number of blobs to return, including all BlobPrefix elements. If the request does not // specify maxresults the server will return up to 5,000 items. // See: https://docs.microsoft.com/en-us/rest/api/storageservices/list-blobs#uri-parameters - maxResults = 5000 + maxResults = 5000 + endpointKey = "endpoint" ) var ErrMissingBlobName = errors.New("blobName is a required attribute") // AzureBlobStorage allows saving blobs to an Azure Blob Storage account. type AzureBlobStorage struct { - metadata *blobStorageMetadata - containerURL azblob.ContainerURL + metadata *blobStorageMetadata + // containerURL azblob.ContainerURL + containerClient *container.Client logger logger.Logger } type blobStorageMetadata struct { - AccountName string - Container string - GetBlobRetryCount int - DecodeBase64 bool - PublicAccessLevel azblob.PublicAccessType + StorageAccount string `json:"storageAccount"` + StorageAccessKey string `json:"storageAccessKey"` + Container string `json:"container"` + GetBlobRetryCount int `json:"getBlobRetryCount,string"` + DecodeBase64 bool `json:"decodeBase64,string"` + PublicAccessLevel azblob.PublicAccessType `json:"publicAccessLevel"` } type createResponse struct { @@ -117,79 +125,100 @@ func (a *AzureBlobStorage) Init(metadata bindings.Metadata) error { } a.metadata = m - credential, env, err := azauth.GetAzureStorageBlobCredentials(a.logger, m.AccountName, metadata.Properties) - if err != nil { - return fmt.Errorf("invalid credentials with error: %s", err.Error()) - } - userAgent := "dapr-" + logger.DaprVersion - options := azblob.PipelineOptions{ - Telemetry: azblob.TelemetryOptions{Value: userAgent}, + options := container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Retry: policy.RetryOptions{ + MaxRetries: int32(a.metadata.GetBlobRetryCount), + }, + Telemetry: policy.TelemetryOptions{ + ApplicationID: userAgent, + }, + }, + } + + settings, err := azauth.NewEnvironmentSettings("storage", metadata.Properties) + if err != nil { + return err } - p := azblob.NewPipeline(credential, options) - - var containerURL azblob.ContainerURL - customEndpoint, ok := mdutils.GetMetadataProperty(metadata.Properties, azauth.StorageEndpointKeys...) + customEndpoint, ok := metadata.Properties[endpointKey] + var URL *url.URL if ok && customEndpoint != "" { - URL, parseErr := url.Parse(fmt.Sprintf("%s/%s/%s", customEndpoint, m.AccountName, m.Container)) + var parseErr error + URL, parseErr = url.Parse(fmt.Sprintf("%s/%s/%s", customEndpoint, m.StorageAccount, m.Container)) if parseErr != nil { return parseErr } - containerURL = azblob.NewContainerURL(*URL, p) } else { - URL, _ := url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", m.AccountName, env.StorageEndpointSuffix, m.Container)) - containerURL = azblob.NewContainerURL(*URL, p) + env := settings.AzureEnvironment + URL, _ = url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", m.StorageAccount, env.StorageEndpointSuffix, m.Container)) } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - _, err = containerURL.Create(ctx, azblob.Metadata{}, m.PublicAccessLevel) + var clientErr error + var client *container.Client + // Try using shared key credentials first + if m.StorageAccessKey != "" { + credential, newSharedKeyErr := azblob.NewSharedKeyCredential(m.StorageAccount, m.StorageAccessKey) + if err != nil { + return fmt.Errorf("invalid credentials with error: %w", newSharedKeyErr) + } + client, clientErr = container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) + if clientErr != nil { + return fmt.Errorf("cannot init Blobstorage container client: %w", err) + } + container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) + a.containerClient = client + } else { + // fallback to AAD + credential, tokenErr := settings.GetTokenCredential() + if err != nil { + return fmt.Errorf("invalid credentials with error: %w", tokenErr) + } + client, clientErr = container.NewClient(URL.String(), credential, &options) + } + if clientErr != nil { + return fmt.Errorf("cannot init Blobstorage client: %w", clientErr) + } + + createContainerOptions := container.CreateOptions{ + Access: &m.PublicAccessLevel, + Metadata: map[string]string{}, + } + timeoutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + _, err = client.Create(timeoutCtx, &createContainerOptions) cancel() // Don't return error, container might already exist a.logger.Debugf("error creating container: %w", err) - a.containerURL = containerURL + a.containerClient = client return nil } -func (a *AzureBlobStorage) parseMetadata(metadata bindings.Metadata) (*blobStorageMetadata, error) { - var m blobStorageMetadata - if val, ok := mdutils.GetMetadataProperty(metadata.Properties, azauth.StorageAccountNameKeys...); ok && val != "" { - m.AccountName = val +func (a *AzureBlobStorage) parseMetadata(meta bindings.Metadata) (*blobStorageMetadata, error) { + m := blobStorageMetadata{ + GetBlobRetryCount: defaultGetBlobRetryCount, + } + mdutils.DecodeMetadata(meta.Properties, &m) + + if val, ok := mdutils.GetMetadataProperty(meta.Properties, azauth.StorageAccountNameKeys...); ok && val != "" { + m.StorageAccount = val } else { return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageAccountNameKeys[0]) } - if val, ok := mdutils.GetMetadataProperty(metadata.Properties, azauth.StorageContainerNameKeys...); ok && val != "" { + if val, ok := mdutils.GetMetadataProperty(meta.Properties, azauth.StorageContainerNameKeys...); ok && val != "" { m.Container = val } else { return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageContainerNameKeys[0]) } - m.GetBlobRetryCount = defaultGetBlobRetryCount - if val, ok := metadata.Properties["getBlobRetryCount"]; ok { - n, err := strconv.Atoi(val) - if err != nil || n == 0 { - return nil, fmt.Errorf("invalid getBlobRetryCount field from metadata") - } - m.GetBlobRetryCount = n - } - - m.DecodeBase64 = false - if val, ok := metadata.Properties["decodeBase64"]; ok { - n, err := strconv.ParseBool(val) - if err != nil { - return nil, fmt.Errorf("invalid decodeBase64 field from metadata") - } - m.DecodeBase64 = n - } - - m.PublicAccessLevel = azblob.PublicAccessType(strings.ToLower(metadata.Properties["publicAccessLevel"])) // per the Dapr documentation "none" is a valid value if m.PublicAccessLevel == "none" { m.PublicAccessLevel = "" } - if !a.isValidPublicAccessType(m.PublicAccessLevel) { - return nil, fmt.Errorf("invalid public access level: %s; allowed: %s", m.PublicAccessLevel, azblob.PossiblePublicAccessTypeValues()) + if m.PublicAccessLevel != "" && !a.isValidPublicAccessType(m.PublicAccessLevel) { + return nil, fmt.Errorf("invalid public access level: %s; allowed: %s", + m.PublicAccessLevel, azblob.PossiblePublicAccessTypeValues()) } return &m, nil @@ -205,8 +234,6 @@ func (a *AzureBlobStorage) Operations() []bindings.OperationKind { } func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeRequest) (*bindings.InvokeResponse, error) { - var blobHTTPHeaders azblob.BlobHTTPHeaders - var blobURL azblob.BlockBlobURL var blobName string if val, ok := req.Metadata[metadataKeyBlobName]; ok && val != "" { blobName = val @@ -214,34 +241,38 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque } else { blobName = uuid.New().String() } - blobURL = a.getBlobURL(blobName) + + blobHTTPHeaders := blob.HTTPHeaders{} if val, ok := req.Metadata[metadataKeyContentType]; ok && val != "" { - blobHTTPHeaders.ContentType = val + blobHTTPHeaders.BlobContentType = ptr.Of(val) delete(req.Metadata, metadataKeyContentType) } + var contentMD5 *[]byte + if val, ok := req.Metadata[metadataKeyContentMD5]; ok && val != "" { sDec, err := b64.StdEncoding.DecodeString(val) if err != nil || len(sDec) != 16 { return nil, fmt.Errorf("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") } - blobHTTPHeaders.ContentMD5 = sDec + blobHTTPHeaders.BlobContentMD5 = sDec + contentMD5 = &sDec delete(req.Metadata, metadataKeyContentMD5) } if val, ok := req.Metadata[metadataKeyContentEncoding]; ok && val != "" { - blobHTTPHeaders.ContentEncoding = val + blobHTTPHeaders.BlobContentEncoding = ptr.Of(val) delete(req.Metadata, metadataKeyContentEncoding) } if val, ok := req.Metadata[metadataKeyContentLanguage]; ok && val != "" { - blobHTTPHeaders.ContentLanguage = val + blobHTTPHeaders.BlobContentLanguage = ptr.Of(val) delete(req.Metadata, metadataKeyContentLanguage) } if val, ok := req.Metadata[metadataKeyContentDisposition]; ok && val != "" { - blobHTTPHeaders.ContentDisposition = val + blobHTTPHeaders.BlobContentDisposition = ptr.Of(val) delete(req.Metadata, metadataKeyContentDisposition) } if val, ok := req.Metadata[metadataKeyCacheControl]; ok && val != "" { - blobHTTPHeaders.CacheControl = val + blobHTTPHeaders.BlobCacheControl = ptr.Of(val) delete(req.Metadata, metadataKeyCacheControl) } @@ -258,17 +289,22 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque req.Data = decoded } - _, err = azblob.UploadBufferToBlockBlob(ctx, req.Data, blobURL, azblob.UploadToBlockBlobOptions{ - Parallelism: 16, - Metadata: a.sanitizeMetadata(req.Metadata), - BlobHTTPHeaders: blobHTTPHeaders, - }) + uploadOptions := azblob.UploadBufferOptions{ + Metadata: a.sanitizeMetadata(req.Metadata), + HTTPHeaders: &blobHTTPHeaders, + Concurrency: 16, + TransactionalContentMD5: contentMD5, + } + + blockBlobClient := a.containerClient.NewBlockBlobClient(blobName) + _, err = blockBlobClient.UploadBuffer(ctx, req.Data, &uploadOptions) + if err != nil { return nil, fmt.Errorf("error uploading az blob: %w", err) } resp := createResponse{ - BlobURL: blobURL.String(), + BlobURL: blockBlobClient.URL(), } b, err := json.Marshal(resp) if err != nil { @@ -286,23 +322,30 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque } func (a *AzureBlobStorage) get(ctx context.Context, req *bindings.InvokeRequest) (*bindings.InvokeResponse, error) { - var blobURL azblob.BlockBlobURL + var blockBlobClient *blockblob.Client if val, ok := req.Metadata[metadataKeyBlobName]; ok && val != "" { - blobURL = a.getBlobURL(val) + blockBlobClient = a.containerClient.NewBlockBlobClient(val) } else { return nil, ErrMissingBlobName } - resp, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false) + downloadOptions := azblob.DownloadStreamOptions{ + AccessConditions: &blob.AccessConditions{}, + } + + blobDownloadResponse, err := blockBlobClient.DownloadStream(ctx, &downloadOptions) if err != nil { return nil, fmt.Errorf("error downloading az blob: %w", err) } - - bodyStream := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: a.metadata.GetBlobRetryCount}) - - data, err := io.ReadAll(bodyStream) + blobData := &bytes.Buffer{} + reader := blobDownloadResponse.Body + _, err = blobData.ReadFrom(reader) if err != nil { - return nil, fmt.Errorf("error reading az blob body: %w", err) + return nil, fmt.Errorf("error reading az blob: %w", err) + } + err = reader.Close() + if err != nil { + return nil, fmt.Errorf("error closing az blob reader: %w", err) } var metadata map[string]string @@ -311,45 +354,54 @@ func (a *AzureBlobStorage) get(ctx context.Context, req *bindings.InvokeRequest) return nil, fmt.Errorf("error parsing metadata: %w", err) } + getPropertiesOptions := blob.GetPropertiesOptions{ + AccessConditions: &blob.AccessConditions{}, + } + if fetchMetadata { - props, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{}) + props, err := blockBlobClient.GetProperties(ctx, &getPropertiesOptions) if err != nil { return nil, fmt.Errorf("error reading blob metadata: %w", err) } - metadata = props.NewMetadata() + metadata = props.Metadata } return &bindings.InvokeResponse{ - Data: data, + Data: blobData.Bytes(), Metadata: metadata, }, nil } func (a *AzureBlobStorage) delete(ctx context.Context, req *bindings.InvokeRequest) (*bindings.InvokeResponse, error) { - var blobURL azblob.BlockBlobURL - if val, ok := req.Metadata[metadataKeyBlobName]; ok && val != "" { - blobURL = a.getBlobURL(val) - } else { + var blockBlobClient *blockblob.Client + val, ok := req.Metadata[metadataKeyBlobName] + if !ok || val == "" { return nil, ErrMissingBlobName } - deleteSnapshotsOptions := azblob.DeleteSnapshotsOptionNone - if val, ok := req.Metadata[metadataKeyDeleteSnapshots]; ok && val != "" { - deleteSnapshotsOptions = azblob.DeleteSnapshotsOptionType(val) + var deleteSnapshotsOptions blob.DeleteSnapshotsOptionType + if deleteSnapShotOption, ok := req.Metadata[metadataKeyDeleteSnapshots]; ok && val != "" { + deleteSnapshotsOptions = azblob.DeleteSnapshotsOptionType(deleteSnapShotOption) if !a.isValidDeleteSnapshotsOptionType(deleteSnapshotsOptions) { return nil, fmt.Errorf("invalid delete snapshot option type: %s; allowed: %s", deleteSnapshotsOptions, azblob.PossibleDeleteSnapshotsOptionTypeValues()) } } - _, err := blobURL.Delete(ctx, deleteSnapshotsOptions, azblob.BlobAccessConditions{}) + deleteOptions := blob.DeleteOptions{ + DeleteSnapshots: &deleteSnapshotsOptions, + AccessConditions: &blob.AccessConditions{}, + } + + blockBlobClient = a.containerClient.NewBlockBlobClient(val) + _, err := blockBlobClient.Delete(context.Background(), &deleteOptions) return nil, err } func (a *AzureBlobStorage) list(ctx context.Context, req *bindings.InvokeRequest) (*bindings.InvokeResponse, error) { - options := azblob.ListBlobsSegmentOptions{} + options := container.ListBlobsFlatOptions{} hasPayload := false var payload listPayload @@ -360,50 +412,52 @@ func (a *AzureBlobStorage) list(ctx context.Context, req *bindings.InvokeRequest } hasPayload = true } - if hasPayload { - options.Details.Copy = payload.Include.Copy - options.Details.Metadata = payload.Include.Metadata - options.Details.Snapshots = payload.Include.Snapshots - options.Details.UncommittedBlobs = payload.Include.UncommittedBlobs - options.Details.Deleted = payload.Include.Deleted + options.Include.Copy = payload.Include.Copy + options.Include.Metadata = payload.Include.Metadata + options.Include.Snapshots = payload.Include.Snapshots + options.Include.UncommittedBlobs = payload.Include.UncommittedBlobs + options.Include.Deleted = payload.Include.Deleted } if hasPayload && payload.MaxResults != int32(0) { - options.MaxResults = payload.MaxResults + options.MaxResults = ptr.Of(payload.MaxResults) } else { - options.MaxResults = maxResults + options.MaxResults = ptr.Of(int32(maxResults)) } if hasPayload && payload.Prefix != "" { - options.Prefix = payload.Prefix + options.Prefix = ptr.Of(payload.Prefix) } - var initialMarker azblob.Marker + var initialMarker string if hasPayload && payload.Marker != "" { - initialMarker = azblob.Marker{Val: &payload.Marker} + initialMarker = payload.Marker } else { - initialMarker = azblob.Marker{} + initialMarker = "" } + options.Marker = ptr.Of(initialMarker) - var blobs []azblob.BlobItem metadata := map[string]string{} - for currentMaker := initialMarker; currentMaker.NotDone(); { - var listBlob *azblob.ListBlobsFlatSegmentResponse - listBlob, err := a.containerURL.ListBlobsFlatSegment(ctx, currentMaker, options) + blobs := []*container.BlobItem{} + pager := a.containerClient.NewListBlobsFlatPager(&options) + + for pager.More() { + resp, err := pager.NextPage(ctx) if err != nil { return nil, fmt.Errorf("error listing blobs: %w", err) } - blobs = append(blobs, listBlob.Segment.BlobItems...) - + blobs = append(blobs, resp.Segment.BlobItems...) numBlobs := len(blobs) - currentMaker = listBlob.NextMarker - metadata[metadataKeyMarker] = *currentMaker.Val metadata[metadataKeyNumber] = strconv.FormatInt(int64(numBlobs), 10) + metadata[metadataKeyMarker] = "" + if resp.Marker != nil { + metadata[metadataKeyMarker] = *resp.Marker + } - if options.MaxResults-maxResults > 0 { - options.MaxResults -= maxResults + if *options.MaxResults-maxResults > 0 { + *options.MaxResults -= maxResults } else { break } @@ -435,12 +489,6 @@ func (a *AzureBlobStorage) Invoke(ctx context.Context, req *bindings.InvokeReque } } -func (a *AzureBlobStorage) getBlobURL(name string) azblob.BlockBlobURL { - blobURL := a.containerURL.NewBlockBlobURL(name) - - return blobURL -} - func (a *AzureBlobStorage) isValidPublicAccessType(accessType azblob.PublicAccessType) bool { validTypes := azblob.PossiblePublicAccessTypeValues() for _, item := range validTypes { diff --git a/bindings/azure/blobstorage/blobstorage_test.go b/bindings/azure/blobstorage/blobstorage_test.go index d0a459141e..52810feab2 100644 --- a/bindings/azure/blobstorage/blobstorage_test.go +++ b/bindings/azure/blobstorage/blobstorage_test.go @@ -17,7 +17,7 @@ import ( "context" "testing" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/stretchr/testify/assert" "github.com/dapr/components-contrib/bindings" @@ -39,11 +39,11 @@ func TestParseMetadata(t *testing.T) { meta, err := blobStorage.parseMetadata(m) assert.Nil(t, err) assert.Equal(t, "test", meta.Container) - assert.Equal(t, "account", meta.AccountName) + assert.Equal(t, "account", meta.StorageAccount) // storageAccessKey is parsed in the azauth package assert.Equal(t, true, meta.DecodeBase64) assert.Equal(t, 5, meta.GetBlobRetryCount) - assert.Equal(t, azblob.PublicAccessNone, meta.PublicAccessLevel) + assert.Equal(t, "", string(meta.PublicAccessLevel)) }) t.Run("parse metadata with publicAccessLevel = blob", func(t *testing.T) { @@ -55,7 +55,7 @@ func TestParseMetadata(t *testing.T) { } meta, err := blobStorage.parseMetadata(m) assert.Nil(t, err) - assert.Equal(t, azblob.PublicAccessBlob, meta.PublicAccessLevel) + assert.Equal(t, azblob.PublicAccessTypeBlob, meta.PublicAccessLevel) }) t.Run("parse metadata with publicAccessLevel = container", func(t *testing.T) { @@ -67,7 +67,7 @@ func TestParseMetadata(t *testing.T) { } meta, err := blobStorage.parseMetadata(m) assert.Nil(t, err) - assert.Equal(t, azblob.PublicAccessContainer, meta.PublicAccessLevel) + assert.Equal(t, azblob.PublicAccessTypeContainer, meta.PublicAccessLevel) }) t.Run("parse metadata with invalid publicAccessLevel", func(t *testing.T) { diff --git a/go.mod b/go.mod index 5b94f30342..84ce825b36 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.10.1 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/Azure/azure-storage-blob-go v0.10.0 github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd github.com/Azure/go-amqp v0.17.5 @@ -133,7 +134,7 @@ require ( github.com/99designs/keyring v1.2.1 // indirect github.com/AthenZ/athenz v1.10.39 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect diff --git a/go.sum b/go.sum index 13862b899e..72cb6d3dcc 100644 --- a/go.sum +++ b/go.sum @@ -110,14 +110,16 @@ github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2/go.mod h1:Fy3bbChFm4c github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgDAUZ+I2bO8J7s8JxXmu9fhi2ss= github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1/go.mod h1:l3wvZkG9oW07GLBW5Cd0WwG5asOfJ8aqE8raUvNzLpk= github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.10.1 h1:AhZnZn4kUKz36bHJ8AK/FH2tH/q3CAkG+Gme+2ibuak= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.10.1/go.mod h1:S78i9yTr4o/nXlH76bKjGUye9Z2wSxO5Tz7GoDr4vfI= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 h1:Lg6BW0VPmCwcMlvOviL3ruHFO+H9tZNqscK0AeuFjGM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.1 h1:Zm7A6yKHT3evC/0lquPWJ9hrkRGVIeZOmIvHPv6xV9Q= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.1/go.mod h1:LH9XQnMr2ZYxQdVdCrzLO9mxeDyrDFa6wbSI3x5zCZk= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= diff --git a/tests/certification/bindings/azure/blobstorage/blobstorage_test.go b/tests/certification/bindings/azure/blobstorage/blobstorage_test.go index 964b2910de..ad715e91b6 100644 --- a/tests/certification/bindings/azure/blobstorage/blobstorage_test.go +++ b/tests/certification/bindings/azure/blobstorage/blobstorage_test.go @@ -39,6 +39,8 @@ import ( "github.com/dapr/components-contrib/tests/certification/flow/sidecar" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" ) const ( @@ -192,12 +194,12 @@ func TestBlobStorage(t *testing.T) { // confirm the deletion. _, invokeSecondGetErr := getBlobRequest(ctx, client, blobName, false) assert.Error(t, invokeSecondGetErr) - assert.Contains(t, invokeSecondGetErr.Error(), "ServiceCode=BlobNotFound") + assert.Contains(t, invokeSecondGetErr.Error(), "ERROR CODE: BlobNotFound") // deleting the key again should fail. _, invokeDeleteErr2 := deleteBlobRequest(ctx, client, blobName, "") assert.Error(t, invokeDeleteErr2) - assert.Contains(t, invokeDeleteErr2.Error(), "ServiceCode=BlobNotFound") + assert.Contains(t, invokeDeleteErr2.Error(), "ERROR CODE: BlobNotFound") return nil } @@ -230,7 +232,7 @@ func TestBlobStorage(t *testing.T) { _, invokeCreateErr := client.InvokeBinding(ctx, invokeCreateRequest) assert.Error(t, invokeCreateErr) - assert.Contains(t, invokeCreateErr.Error(), "ServiceCode=Md5Mismatch") + assert.Contains(t, invokeCreateErr.Error(), "ERROR CODE: Md5Mismatch") return nil } @@ -279,7 +281,7 @@ func TestBlobStorage(t *testing.T) { // confirm the deletion. _, invokeSecondGetErr := getBlobRequest(ctx, client, blobName, false) assert.Error(t, invokeSecondGetErr) - assert.Contains(t, invokeSecondGetErr.Error(), "ServiceCode=BlobNotFound") + assert.Contains(t, invokeSecondGetErr.Error(), "ERROR CODE: BlobNotFound") return nil } @@ -384,9 +386,9 @@ func TestBlobStorage(t *testing.T) { out, invokeGetErr := client.InvokeBinding(ctx, invokeGetRequest) assert.NoError(t, invokeGetErr) - assert.Equal(t, string(out.Data), input) - assert.Contains(t, out.Metadata, "custom") - assert.Equal(t, out.Metadata["custom"], "hello-world") + assert.Equal(t, input, string(out.Data)) + assert.Contains(t, out.Metadata, "Custom") + assert.Equal(t, "hello-world", out.Metadata["Custom"]) out, invokeErr := listBlobRequest(ctx, client, "", "", -1, true, false, false, false, false) assert.NoError(t, invokeErr) @@ -419,7 +421,7 @@ func TestBlobStorage(t *testing.T) { // confirm the deletion. _, invokeSecondGetErr := getBlobRequest(ctx, client, "filename.txt", false) assert.Error(t, invokeSecondGetErr) - assert.Contains(t, invokeSecondGetErr.Error(), "ServiceCode=BlobNotFound") + assert.Contains(t, invokeSecondGetErr.Error(), "ERROR CODE: BlobNotFound") return nil } @@ -505,8 +507,8 @@ func TestBlobStorage(t *testing.T) { unmarshalErr := json.Unmarshal(out.Data, &output) assert.NoError(t, unmarshalErr) - assert.Equal(t, len(output), 1) - assert.Equal(t, output[0]["Name"], "prefixA/filename.txt") + assert.Equal(t, 1, len(output)) + assert.Contains(t, output[0]["Name"], "prefixA") nextMarker := out.Metadata["marker"] @@ -518,8 +520,8 @@ func TestBlobStorage(t *testing.T) { err2 := json.Unmarshal(out2.Data, &output2) assert.NoError(t, err2) - assert.Equal(t, len(output2), 1) - assert.Equal(t, output2[0]["Name"], "prefixAfilename.txt") + assert.Equal(t, 1, len(output2)) + assert.Contains(t, output2[0]["Name"], "prefixA") // cleanup. _, invokeDeleteErr1 := deleteBlobRequest(ctx, client, "prefixA/filename.txt", "") @@ -530,15 +532,15 @@ func TestBlobStorage(t *testing.T) { assert.NoError(t, invokeDeleteErr3) // list deleted items with prefix. - out3, listErr3 := listBlobRequest(ctx, client, "prefixA", "", -1, false, false, false, false, true) + out3, listErr3 := listBlobRequest(ctx, client, "prefixA/", "", -1, false, false, false, false, true) assert.NoError(t, listErr3) // this will only return the deleted items if soft delete policy is enabled for the blob service. - assert.Equal(t, out3.Metadata["number"], "2") + assert.Equal(t, "1", out3.Metadata["number"]) var output3 []map[string]interface{} err3 := json.Unmarshal(out3.Data, &output3) assert.NoError(t, err3) - assert.Equal(t, len(output3), 2) + assert.Equal(t, len(output3), 1) return nil } @@ -553,17 +555,15 @@ func TestBlobStorage(t *testing.T) { defer client.Close() cred, _ := azblob.NewSharedKeyCredential(os.Getenv("AzureBlobStorageAccount"), os.Getenv("AzureBlobStorageAccessKey")) - service, _ := azblob.NewServiceClientWithSharedKey(fmt.Sprintf("https://%s.blob.core.windows.net/", os.Getenv("AzureBlobStorageAccount")), cred, nil) - containerClient, _ := service.NewContainerClient(os.Getenv("AzureBlobStorageContainer")) + containerClient, _ := container.NewClientWithSharedKeyCredential(fmt.Sprintf("https://%s.blob.core.windows.net/%s", os.Getenv("AzureBlobStorageAccount"), os.Getenv("AzureBlobStorageContainer")), cred, nil) - blobClient, _ := containerClient.NewBlockBlobClient("snapshotthis.txt") - uploadResp, uploadErr := blobClient.UploadBuffer( + blobClient := containerClient.NewBlockBlobClient("snapshotthis.txt") + _, uploadErr := blobClient.UploadBuffer( ctx, []byte("some example content"), - azblob.UploadOption{}) //nolint:exhaustivestruct + &azblob.UploadBufferOptions{}) //nolint:exhaustivestruct assert.NoError(t, uploadErr) - uploadResp.Body.Close() _, createSnapshotErr := blobClient.CreateSnapshot( - ctx, &azblob.BlobCreateSnapshotOptions{}) //nolint:exhaustivestruct + ctx, &blob.CreateSnapshotOptions{}) //nolint:exhaustivestruct assert.NoError(t, createSnapshotErr) // list the contents of the container including snapshots for the specific blob only. @@ -582,7 +582,7 @@ func TestBlobStorage(t *testing.T) { // create another snapshot. _, createSnapshotErr2 := blobClient.CreateSnapshot( - ctx, &azblob.BlobCreateSnapshotOptions{}) //nolint:exhaustivestruct + ctx, &blob.CreateSnapshotOptions{}) //nolint:exhaustivestruct assert.NoError(t, createSnapshotErr2) // delete base blob and snapshots all at once. diff --git a/tests/certification/bindings/azure/blobstorage/go.mod b/tests/certification/bindings/azure/blobstorage/go.mod index df4503ea58..39df468c9d 100644 --- a/tests/certification/bindings/azure/blobstorage/go.mod +++ b/tests/certification/bindings/azure/blobstorage/go.mod @@ -3,7 +3,7 @@ module github.com/dapr/components-contrib/tests/certification/bindings/azure/blo go 1.19 require ( - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/dapr/components-contrib v1.9.1-0.20221025205611-e38369028650 github.com/dapr/components-contrib/tests/certification v0.0.0-20211130185200-4918900c09e1 github.com/dapr/dapr v1.9.1-0.20221101183153-7e3635f1491e @@ -19,7 +19,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/bindings/azure/blobstorage/go.sum b/tests/certification/bindings/azure/blobstorage/go.sum index 3a90fb97c0..0200c2d339 100644 --- a/tests/certification/bindings/azure/blobstorage/go.sum +++ b/tests/certification/bindings/azure/blobstorage/go.sum @@ -45,10 +45,10 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 h1:QSdcrd/UFJv6Bp/CfoVf2SrENpFn9P6Yh8yb+xNhYMM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/bindings/azure/cosmosdb/go.mod b/tests/certification/bindings/azure/cosmosdb/go.mod index fd5bfac032..886fc97c83 100644 --- a/tests/certification/bindings/azure/cosmosdb/go.mod +++ b/tests/certification/bindings/azure/cosmosdb/go.mod @@ -22,7 +22,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/bindings/azure/cosmosdb/go.sum b/tests/certification/bindings/azure/cosmosdb/go.sum index b63e24fa22..82ff71cf61 100644 --- a/tests/certification/bindings/azure/cosmosdb/go.sum +++ b/tests/certification/bindings/azure/cosmosdb/go.sum @@ -49,8 +49,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4Sath github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2 h1:yJegJqjhrMJ3Oe5s43jOTGL2AsE7pJyx+7Yqls/65tw= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2/go.mod h1:Fy3bbChFm4cZn6oIxYYqKB2FG3rBDxk3NZDLDJCHl+Q= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/bindings/azure/eventhubs/go.mod b/tests/certification/bindings/azure/eventhubs/go.mod index 2319830400..e700c189cf 100644 --- a/tests/certification/bindings/azure/eventhubs/go.mod +++ b/tests/certification/bindings/azure/eventhubs/go.mod @@ -22,7 +22,7 @@ require ( github.com/Azure/azure-sdk-for-go v67.0.0+incompatible // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-amqp v0.17.5 // indirect diff --git a/tests/certification/bindings/azure/eventhubs/go.sum b/tests/certification/bindings/azure/eventhubs/go.sum index 6e62d9e587..2ffcf1e774 100644 --- a/tests/certification/bindings/azure/eventhubs/go.sum +++ b/tests/certification/bindings/azure/eventhubs/go.sum @@ -51,8 +51,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= diff --git a/tests/certification/bindings/azure/servicebusqueues/go.mod b/tests/certification/bindings/azure/servicebusqueues/go.mod index b03a1078a9..3f276e42d4 100644 --- a/tests/certification/bindings/azure/servicebusqueues/go.mod +++ b/tests/certification/bindings/azure/servicebusqueues/go.mod @@ -19,7 +19,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect diff --git a/tests/certification/bindings/azure/servicebusqueues/go.sum b/tests/certification/bindings/azure/servicebusqueues/go.sum index d6685e8d6a..aab8e83276 100644 --- a/tests/certification/bindings/azure/servicebusqueues/go.sum +++ b/tests/certification/bindings/azure/servicebusqueues/go.sum @@ -45,8 +45,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.1 h1:Zm7A6yKHT3evC/0lquPWJ9hrkRGVIeZOmIvHPv6xV9Q= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.1/go.mod h1:LH9XQnMr2ZYxQdVdCrzLO9mxeDyrDFa6wbSI3x5zCZk= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= diff --git a/tests/certification/bindings/azure/storagequeues/go.mod b/tests/certification/bindings/azure/storagequeues/go.mod index 64645190d9..6698ef29e0 100644 --- a/tests/certification/bindings/azure/storagequeues/go.mod +++ b/tests/certification/bindings/azure/storagequeues/go.mod @@ -19,7 +19,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/bindings/azure/storagequeues/go.sum b/tests/certification/bindings/azure/storagequeues/go.sum index b9d06c9b63..9776a90191 100644 --- a/tests/certification/bindings/azure/storagequeues/go.sum +++ b/tests/certification/bindings/azure/storagequeues/go.sum @@ -45,8 +45,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/pubsub/azure/eventhubs/go.mod b/tests/certification/pubsub/azure/eventhubs/go.mod index 7055be7b17..a49ec755bd 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.mod +++ b/tests/certification/pubsub/azure/eventhubs/go.mod @@ -22,7 +22,7 @@ require ( github.com/Azure/azure-sdk-for-go v67.0.0+incompatible // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-amqp v0.17.5 // indirect diff --git a/tests/certification/pubsub/azure/eventhubs/go.sum b/tests/certification/pubsub/azure/eventhubs/go.sum index 7848bdb158..bba9d03a7f 100644 --- a/tests/certification/pubsub/azure/eventhubs/go.sum +++ b/tests/certification/pubsub/azure/eventhubs/go.sum @@ -51,8 +51,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= diff --git a/tests/certification/pubsub/azure/servicebus/topics/go.mod b/tests/certification/pubsub/azure/servicebus/topics/go.mod index 62e0f47b8a..eaf00930ee 100644 --- a/tests/certification/pubsub/azure/servicebus/topics/go.mod +++ b/tests/certification/pubsub/azure/servicebus/topics/go.mod @@ -20,7 +20,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect diff --git a/tests/certification/pubsub/azure/servicebus/topics/go.sum b/tests/certification/pubsub/azure/servicebus/topics/go.sum index d6685e8d6a..aab8e83276 100644 --- a/tests/certification/pubsub/azure/servicebus/topics/go.sum +++ b/tests/certification/pubsub/azure/servicebus/topics/go.sum @@ -45,8 +45,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.1 h1:Zm7A6yKHT3evC/0lquPWJ9hrkRGVIeZOmIvHPv6xV9Q= github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.1/go.mod h1:LH9XQnMr2ZYxQdVdCrzLO9mxeDyrDFa6wbSI3x5zCZk= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= diff --git a/tests/certification/secretstores/azure/keyvault/go.mod b/tests/certification/secretstores/azure/keyvault/go.mod index 77c7dd5d7c..50595d2435 100644 --- a/tests/certification/secretstores/azure/keyvault/go.mod +++ b/tests/certification/secretstores/azure/keyvault/go.mod @@ -18,7 +18,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.10.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect diff --git a/tests/certification/secretstores/azure/keyvault/go.sum b/tests/certification/secretstores/azure/keyvault/go.sum index b6d23539f6..4fa12be754 100644 --- a/tests/certification/secretstores/azure/keyvault/go.sum +++ b/tests/certification/secretstores/azure/keyvault/go.sum @@ -45,8 +45,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.10.1 h1:AhZnZn4kUKz36bHJ8AK/FH2tH/q3CAkG+Gme+2ibuak= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.10.1/go.mod h1:S78i9yTr4o/nXlH76bKjGUye9Z2wSxO5Tz7GoDr4vfI= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.0 h1:Lg6BW0VPmCwcMlvOviL3ruHFO+H9tZNqscK0AeuFjGM= diff --git a/tests/certification/state/azure/blobstorage/go.mod b/tests/certification/state/azure/blobstorage/go.mod index 5022f12798..b7c33bfbe8 100644 --- a/tests/certification/state/azure/blobstorage/go.mod +++ b/tests/certification/state/azure/blobstorage/go.mod @@ -18,7 +18,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/state/azure/blobstorage/go.sum b/tests/certification/state/azure/blobstorage/go.sum index e1c955cbce..3b5acc6ec8 100644 --- a/tests/certification/state/azure/blobstorage/go.sum +++ b/tests/certification/state/azure/blobstorage/go.sum @@ -45,8 +45,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/state/azure/cosmosdb/go.mod b/tests/certification/state/azure/cosmosdb/go.mod index 8dbfc9a340..44f1f6f019 100644 --- a/tests/certification/state/azure/cosmosdb/go.mod +++ b/tests/certification/state/azure/cosmosdb/go.mod @@ -20,7 +20,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/state/azure/cosmosdb/go.sum b/tests/certification/state/azure/cosmosdb/go.sum index 5f83b83bcc..e1969c129c 100644 --- a/tests/certification/state/azure/cosmosdb/go.sum +++ b/tests/certification/state/azure/cosmosdb/go.sum @@ -49,8 +49,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4Sath github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2 h1:yJegJqjhrMJ3Oe5s43jOTGL2AsE7pJyx+7Yqls/65tw= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.2/go.mod h1:Fy3bbChFm4cZn6oIxYYqKB2FG3rBDxk3NZDLDJCHl+Q= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= diff --git a/tests/certification/state/azure/tablestorage/go.mod b/tests/certification/state/azure/tablestorage/go.mod index c7ad889bc7..b6a64984a7 100644 --- a/tests/certification/state/azure/tablestorage/go.mod +++ b/tests/certification/state/azure/tablestorage/go.mod @@ -19,7 +19,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/state/azure/tablestorage/go.sum b/tests/certification/state/azure/tablestorage/go.sum index a592f8d136..658835afc9 100644 --- a/tests/certification/state/azure/tablestorage/go.sum +++ b/tests/certification/state/azure/tablestorage/go.sum @@ -47,8 +47,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4Sath github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 h1:bFa9IcjvrCber6gGgDAUZ+I2bO8J7s8JxXmu9fhi2ss= github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1/go.mod h1:l3wvZkG9oW07GLBW5Cd0WwG5asOfJ8aqE8raUvNzLpk= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= From c3027b8fb1ae884366b56c083884e44e119e8f31 Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Thu, 17 Nov 2022 11:52:37 -0800 Subject: [PATCH 02/11] some small improvements Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- bindings/azure/blobstorage/blobstorage.go | 32 +++++++++---------- .../azure/blobstorage/blobstorage_test.go | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/bindings/azure/blobstorage/blobstorage.go b/bindings/azure/blobstorage/blobstorage.go index 34d10f2ae9..e97101b850 100644 --- a/bindings/azure/blobstorage/blobstorage.go +++ b/bindings/azure/blobstorage/blobstorage.go @@ -62,14 +62,14 @@ const ( metadataKeyContentLanguage = "contentLanguage" metadataKeyContentDisposition = "contentDisposition" metadataKeyCacheControl = "cacheControl" - // Specifies the maximum number of HTTP GET requests that will be made while reading from a RetryReader. A value - // of zero means that no additional HTTP GET requests will be made. + // Specifies the maximum number of HTTP requests that will be made to retry blob operations. A value + // of zero means that no additional HTTP requests will be made. defaultGetBlobRetryCount = 10 // Specifies the maximum number of blobs to return, including all BlobPrefix elements. If the request does not // specify maxresults the server will return up to 5,000 items. // See: https://docs.microsoft.com/en-us/rest/api/storageservices/list-blobs#uri-parameters - maxResults = 5000 - endpointKey = "endpoint" + maxResults int32 = 5000 + endpointKey = "endpoint" ) var ErrMissingBlobName = errors.New("blobName is a required attribute") @@ -87,7 +87,7 @@ type blobStorageMetadata struct { StorageAccount string `json:"storageAccount"` StorageAccessKey string `json:"storageAccessKey"` Container string `json:"container"` - GetBlobRetryCount int `json:"getBlobRetryCount,string"` + GetBlobRetryCount int32 `json:"getBlobRetryCount,string"` DecodeBase64 bool `json:"decodeBase64,string"` PublicAccessLevel azblob.PublicAccessType `json:"publicAccessLevel"` } @@ -129,7 +129,7 @@ func (a *AzureBlobStorage) Init(metadata bindings.Metadata) error { options := container.ClientOptions{ ClientOptions: azcore.ClientOptions{ Retry: policy.RetryOptions{ - MaxRetries: int32(a.metadata.GetBlobRetryCount), + MaxRetries: a.metadata.GetBlobRetryCount, }, Telemetry: policy.TelemetryOptions{ ApplicationID: userAgent, @@ -245,7 +245,7 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque blobHTTPHeaders := blob.HTTPHeaders{} if val, ok := req.Metadata[metadataKeyContentType]; ok && val != "" { - blobHTTPHeaders.BlobContentType = ptr.Of(val) + blobHTTPHeaders.BlobContentType = &val delete(req.Metadata, metadataKeyContentType) } var contentMD5 *[]byte @@ -253,26 +253,26 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque if val, ok := req.Metadata[metadataKeyContentMD5]; ok && val != "" { sDec, err := b64.StdEncoding.DecodeString(val) if err != nil || len(sDec) != 16 { - return nil, fmt.Errorf("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") + return nil, errors.New("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") } blobHTTPHeaders.BlobContentMD5 = sDec contentMD5 = &sDec delete(req.Metadata, metadataKeyContentMD5) } if val, ok := req.Metadata[metadataKeyContentEncoding]; ok && val != "" { - blobHTTPHeaders.BlobContentEncoding = ptr.Of(val) + blobHTTPHeaders.BlobContentEncoding = &val delete(req.Metadata, metadataKeyContentEncoding) } if val, ok := req.Metadata[metadataKeyContentLanguage]; ok && val != "" { - blobHTTPHeaders.BlobContentLanguage = ptr.Of(val) + blobHTTPHeaders.BlobContentLanguage = &val delete(req.Metadata, metadataKeyContentLanguage) } if val, ok := req.Metadata[metadataKeyContentDisposition]; ok && val != "" { - blobHTTPHeaders.BlobContentDisposition = ptr.Of(val) + blobHTTPHeaders.BlobContentDisposition = &val delete(req.Metadata, metadataKeyContentDisposition) } if val, ok := req.Metadata[metadataKeyCacheControl]; ok && val != "" { - blobHTTPHeaders.BlobCacheControl = ptr.Of(val) + blobHTTPHeaders.BlobCacheControl = &val delete(req.Metadata, metadataKeyCacheControl) } @@ -421,13 +421,13 @@ func (a *AzureBlobStorage) list(ctx context.Context, req *bindings.InvokeRequest } if hasPayload && payload.MaxResults != int32(0) { - options.MaxResults = ptr.Of(payload.MaxResults) + options.MaxResults = &payload.MaxResults } else { - options.MaxResults = ptr.Of(int32(maxResults)) + options.MaxResults = ptr.Of(maxResults) // cannot get address of constant directly } if hasPayload && payload.Prefix != "" { - options.Prefix = ptr.Of(payload.Prefix) + options.Prefix = &payload.Prefix } var initialMarker string @@ -436,7 +436,7 @@ func (a *AzureBlobStorage) list(ctx context.Context, req *bindings.InvokeRequest } else { initialMarker = "" } - options.Marker = ptr.Of(initialMarker) + options.Marker = &initialMarker metadata := map[string]string{} blobs := []*container.BlobItem{} diff --git a/bindings/azure/blobstorage/blobstorage_test.go b/bindings/azure/blobstorage/blobstorage_test.go index 52810feab2..8462e497d6 100644 --- a/bindings/azure/blobstorage/blobstorage_test.go +++ b/bindings/azure/blobstorage/blobstorage_test.go @@ -42,7 +42,7 @@ func TestParseMetadata(t *testing.T) { assert.Equal(t, "account", meta.StorageAccount) // storageAccessKey is parsed in the azauth package assert.Equal(t, true, meta.DecodeBase64) - assert.Equal(t, 5, meta.GetBlobRetryCount) + assert.Equal(t, int32(5), meta.GetBlobRetryCount) assert.Equal(t, "", string(meta.PublicAccessLevel)) }) From 17d38d7f655edc1ef05f23bcbf63bf5ef3612b0d Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Thu, 17 Nov 2022 12:02:22 -0800 Subject: [PATCH 03/11] fix blob storage cert test Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- .../azure/blobstorage/blobstorage_test.go | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/certification/bindings/azure/blobstorage/blobstorage_test.go b/tests/certification/bindings/azure/blobstorage/blobstorage_test.go index ad715e91b6..79c4855e18 100644 --- a/tests/certification/bindings/azure/blobstorage/blobstorage_test.go +++ b/tests/certification/bindings/azure/blobstorage/blobstorage_test.go @@ -33,6 +33,7 @@ import ( dapr_testing "github.com/dapr/dapr/pkg/testing" daprsdk "github.com/dapr/go-sdk/client" "github.com/dapr/kit/logger" + "github.com/dapr/kit/ptr" "github.com/dapr/components-contrib/tests/certification/embedded" "github.com/dapr/components-contrib/tests/certification/flow" @@ -106,10 +107,12 @@ func listBlobRequest(ctx flow.Context, client daprsdk.Client, prefix string, mar } // deleteBlobRequest is used to make a common binding request for the delete operation. -func deleteBlobRequest(ctx flow.Context, client daprsdk.Client, name string, deleteSnapshotsOption string) (out *daprsdk.BindingEvent, err error) { +func deleteBlobRequest(ctx flow.Context, client daprsdk.Client, name string, deleteSnapshotsOption *string) (out *daprsdk.BindingEvent, err error) { invokeDeleteMetadata := map[string]string{ - "blobName": name, - "deleteSnapshots": deleteSnapshotsOption, + "blobName": name, + } + if deleteSnapshotsOption != nil { + invokeDeleteMetadata["deleteSnapshots"] = *deleteSnapshotsOption } invokeGetRequest := &daprsdk.InvokeBindingRequest{ @@ -187,7 +190,7 @@ func TestBlobStorage(t *testing.T) { assert.Equal(t, newString, input2) // cleanup. - out, invokeDeleteErr := deleteBlobRequest(ctx, client, blobName, "") + out, invokeDeleteErr := deleteBlobRequest(ctx, client, blobName, nil) assert.NoError(t, invokeDeleteErr) assert.Empty(t, out.Data) @@ -197,7 +200,7 @@ func TestBlobStorage(t *testing.T) { assert.Contains(t, invokeSecondGetErr.Error(), "ERROR CODE: BlobNotFound") // deleting the key again should fail. - _, invokeDeleteErr2 := deleteBlobRequest(ctx, client, blobName, "") + _, invokeDeleteErr2 := deleteBlobRequest(ctx, client, blobName, nil) assert.Error(t, invokeDeleteErr2) assert.Contains(t, invokeDeleteErr2.Error(), "ERROR CODE: BlobNotFound") @@ -274,7 +277,7 @@ func TestBlobStorage(t *testing.T) { assert.Equal(t, responseData, dataBytes) assert.Empty(t, out.Metadata) - out, invokeDeleteErr := deleteBlobRequest(ctx, client, blobName, "") + out, invokeDeleteErr := deleteBlobRequest(ctx, client, blobName, nil) assert.NoError(t, invokeDeleteErr) assert.Empty(t, out.Data) @@ -329,7 +332,7 @@ func TestBlobStorage(t *testing.T) { } // cleanup. - _, invokeDeleteErr := deleteBlobRequest(ctx, client, blobName, "") + _, invokeDeleteErr := deleteBlobRequest(ctx, client, blobName, nil) assert.NoError(t, invokeDeleteErr) return nil @@ -414,7 +417,7 @@ func TestBlobStorage(t *testing.T) { } assert.True(t, found) - out, invokeDeleteErr := deleteBlobRequest(ctx, client, "filename.txt", "") + out, invokeDeleteErr := deleteBlobRequest(ctx, client, "filename.txt", nil) assert.NoError(t, invokeDeleteErr) assert.Empty(t, out.Data) @@ -524,11 +527,11 @@ func TestBlobStorage(t *testing.T) { assert.Contains(t, output2[0]["Name"], "prefixA") // cleanup. - _, invokeDeleteErr1 := deleteBlobRequest(ctx, client, "prefixA/filename.txt", "") + _, invokeDeleteErr1 := deleteBlobRequest(ctx, client, "prefixA/filename.txt", nil) assert.NoError(t, invokeDeleteErr1) - _, invokeDeleteErr2 := deleteBlobRequest(ctx, client, "prefixAfilename.txt", "") + _, invokeDeleteErr2 := deleteBlobRequest(ctx, client, "prefixAfilename.txt", nil) assert.NoError(t, invokeDeleteErr2) - _, invokeDeleteErr3 := deleteBlobRequest(ctx, client, "prefixB/filename.txt", "") + _, invokeDeleteErr3 := deleteBlobRequest(ctx, client, "prefixB/filename.txt", nil) assert.NoError(t, invokeDeleteErr3) // list deleted items with prefix. @@ -572,13 +575,13 @@ func TestBlobStorage(t *testing.T) { assert.Equal(t, out.Metadata["number"], "2") // delete snapshots. - _, invokeDeleteErr := deleteBlobRequest(ctx, client, "snapshotthis.txt", "only") + _, invokeDeleteErr := deleteBlobRequest(ctx, client, "snapshotthis.txt", ptr.Of(string(blob.DeleteSnapshotsOptionTypeOnly))) assert.NoError(t, invokeDeleteErr) // verify snapshot is deleted. out2, listErr2 := listBlobRequest(ctx, client, "snapshotthis.txt", "", -1, false, true, false, false, false) assert.NoError(t, listErr2) - assert.Equal(t, out2.Metadata["number"], "1") + assert.Equal(t, "1", out2.Metadata["number"]) // create another snapshot. _, createSnapshotErr2 := blobClient.CreateSnapshot( @@ -586,13 +589,13 @@ func TestBlobStorage(t *testing.T) { assert.NoError(t, createSnapshotErr2) // delete base blob and snapshots all at once. - _, invokeDeleteErr2 := deleteBlobRequest(ctx, client, "snapshotthis.txt", "include") + _, invokeDeleteErr2 := deleteBlobRequest(ctx, client, "snapshotthis.txt", ptr.Of(string(blob.DeleteSnapshotsOptionTypeInclude))) assert.NoError(t, invokeDeleteErr2) // verify base blob and snapshots are deleted. out3, listErr3 := listBlobRequest(ctx, client, "snapshotthis.txt", "", -1, false, true, false, false, false) assert.NoError(t, listErr3) - assert.Equal(t, out3.Metadata["number"], "0") + assert.Equal(t, "0", out3.Metadata["number"]) return nil } From 7ccca4def8a90b4055935a0008db87127b03adff Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Thu, 17 Nov 2022 19:55:02 -0800 Subject: [PATCH 04/11] AzBlob State: Migrate to Track2 SDK Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- state/azure/blobstorage/blobstorage.go | 208 +++++++++++------- state/azure/blobstorage/blobstorage_test.go | 12 +- .../state/azure/blobstorage/go.mod | 1 + .../state/azure/blobstorage/go.sum | 2 + 4 files changed, 139 insertions(+), 84 deletions(-) diff --git a/state/azure/blobstorage/blobstorage.go b/state/azure/blobstorage/blobstorage.go index b493ae162d..a03e7c6635 100644 --- a/state/azure/blobstorage/blobstorage.go +++ b/state/azure/blobstorage/blobstorage.go @@ -36,16 +36,21 @@ Concurrency is supported with ETags according to https://docs.microsoft.com/en-u package blobstorage import ( + "bytes" "context" b64 "encoding/base64" "fmt" - "io" - "net" "net/url" "reflect" "strings" - - "github.com/Azure/azure-storage-blob-go/azblob" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" jsoniter "github.com/json-iterator/go" azauth "github.com/dapr/components-contrib/internal/authentication/azure" @@ -63,13 +68,14 @@ const ( contentLanguage = "ContentLanguage" contentDisposition = "ContentDisposition" cacheControl = "CacheControl" + endpointKey = "endpoint" ) // StateStore Type. type StateStore struct { state.DefaultBulkStore - containerURL azblob.ContainerURL - json jsoniter.API + containerClient *container.Client + json jsoniter.API features []state.Feature logger logger.Logger @@ -78,49 +84,77 @@ type StateStore struct { type blobStorageMetadata struct { AccountName string ContainerName string + AccountKey string } // Init the connection to blob storage, optionally creates a blob container if it doesn't exist. func (r *StateStore) Init(metadata state.Metadata) error { - meta, err := getBlobStorageMetadata(metadata.Properties) + m, err := getBlobStorageMetadata(metadata.Properties) if err != nil { return err } - credential, env, err := azauth.GetAzureStorageBlobCredentials(r.logger, meta.AccountName, metadata.Properties) - if err != nil { - return fmt.Errorf("invalid credentials with error: %s", err.Error()) - } - userAgent := "dapr-" + logger.DaprVersion - options := azblob.PipelineOptions{ - Telemetry: azblob.TelemetryOptions{Value: userAgent}, + options := container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Telemetry: policy.TelemetryOptions{ + ApplicationID: userAgent, + }, + }, } - p := azblob.NewPipeline(credential, options) + settings, err := azauth.NewEnvironmentSettings("storage", metadata.Properties) + if err != nil { + return err + } + customEndpoint, ok := metadata.Properties[endpointKey] var URL *url.URL - customEndpoint, ok := mdutils.GetMetadataProperty(metadata.Properties, azauth.StorageEndpointKeys...) if ok && customEndpoint != "" { - URL, err = url.Parse(fmt.Sprintf("%s/%s/%s", customEndpoint, meta.AccountName, meta.ContainerName)) + var parseErr error + URL, parseErr = url.Parse(fmt.Sprintf("%s/%s/%s", customEndpoint, m.AccountName, m.ContainerName)) + if parseErr != nil { + return parseErr + } } else { - URL, err = url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", meta.AccountName, env.StorageEndpointSuffix, meta.ContainerName)) + env := settings.AzureEnvironment + URL, _ = url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", m.AccountName, env.StorageEndpointSuffix, m.ContainerName)) } - if err != nil { - return err - } - containerURL := azblob.NewContainerURL(*URL, p) - _, err = net.LookupHost(URL.Hostname()) - if err != nil { - return err + var clientErr error + var client *container.Client + // Try using shared key credentials first + if m.AccountKey != "" { + credential, newSharedKeyErr := azblob.NewSharedKeyCredential(m.AccountName, m.AccountKey) + if err != nil { + return fmt.Errorf("invalid credentials with error: %w", newSharedKeyErr) + } + client, clientErr = container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) + if clientErr != nil { + return fmt.Errorf("cannot init Blobstorage container client: %w", err) + } + container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) + r.containerClient = client + } else { + // fallback to AAD + credential, tokenErr := settings.GetTokenCredential() + if err != nil { + return fmt.Errorf("invalid credentials with error: %w", tokenErr) + } + client, clientErr = container.NewClient(URL.String(), credential, &options) + } + if clientErr != nil { + return fmt.Errorf("cannot init Blobstorage client: %w", clientErr) } - ctx := context.Background() - _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) - r.logger.Debugf("error creating container: %s", err) - - r.containerURL = containerURL - r.logger.Debugf("using container '%s'", meta.ContainerName) + createContainerOptions := container.CreateOptions{ + Access: nil, + } + timeoutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + _, err = client.Create(timeoutCtx, &createContainerOptions) + cancel() + // Don't return error, container might already exist + r.logger.Debugf("error creating container: %w", err) + r.containerClient = client return nil } @@ -149,10 +183,11 @@ func (r *StateStore) Set(req *state.SetRequest) error { } func (r *StateStore) Ping() error { - accessConditions := azblob.BlobAccessConditions{} - - if _, err := r.containerURL.GetProperties(context.Background(), accessConditions.LeaseAccessConditions); err != nil { - return fmt.Errorf("blob storage: error connecting to Blob storage at %s: %s", r.containerURL.URL().Host, err) + getPropertiesOptions := container.GetPropertiesOptions{ + LeaseAccessConditions: &container.LeaseAccessConditions{}, + } + if _, err := r.containerClient.GetProperties(context.Background(), &getPropertiesOptions); err != nil { + return fmt.Errorf("blob storage: error connecting to Blob storage at %s: %s", r.containerClient.URL(), err) } return nil @@ -197,9 +232,13 @@ func getBlobStorageMetadata(meta map[string]string) (*blobStorageMetadata, error } func (r *StateStore) readFile(ctx context.Context, req *state.GetRequest) (*state.GetResponse, error) { - blobURL := r.containerURL.NewBlockBlobURL(getFileName(req.Key)) + blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key)) + + downloadOptions := azblob.DownloadStreamOptions{ + AccessConditions: &blob.AccessConditions{}, + } - resp, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false) + blobDownloadResponse, err := blockBlobClient.DownloadStream(ctx, &downloadOptions) if err != nil { r.logger.Debugf("download file %s, err %s", req.Key, err) @@ -210,107 +249,124 @@ func (r *StateStore) readFile(ctx context.Context, req *state.GetRequest) (*stat return &state.GetResponse{}, err } - bodyStream := resp.Body(azblob.RetryReaderOptions{}) - data, err := io.ReadAll(bodyStream) + blobData := &bytes.Buffer{} + reader := blobDownloadResponse.Body + _, err = blobData.ReadFrom(reader) if err != nil { - r.logger.Debugf("read file %s, err %s", req.Key, err) - return &state.GetResponse{}, err + return &state.GetResponse{}, fmt.Errorf("error reading az blob: %w", err) + } + err = reader.Close() + if err != nil { + return &state.GetResponse{}, fmt.Errorf("error closing az blob reader: %w", err) } - contentType := resp.ContentType() + contentType := blobDownloadResponse.ContentType return &state.GetResponse{ - Data: data, - ETag: ptr.Of(string(resp.ETag())), - ContentType: &contentType, + Data: blobData.Bytes(), + ETag: ptr.Of(string(*blobDownloadResponse.ETag)), + ContentType: contentType, }, nil } func (r *StateStore) writeFile(ctx context.Context, req *state.SetRequest) error { - accessConditions := azblob.BlobAccessConditions{} + modifiedAccessConditions := blob.ModifiedAccessConditions{} if req.ETag != nil && *req.ETag != "" { - accessConditions.IfMatch = azblob.ETag(*req.ETag) + modifiedAccessConditions.IfMatch = ptr.Of(azcore.ETag(*req.ETag)) } if req.Options.Concurrency == state.FirstWrite && (req.ETag == nil || *req.ETag == "") { - accessConditions.IfNoneMatch = azblob.ETag("*") + modifiedAccessConditions.IfNoneMatch = ptr.Of(azcore.ETagAny) } - blobURL := r.containerURL.NewBlockBlobURL(getFileName(req.Key)) + accessConditions := blob.AccessConditions{ + ModifiedAccessConditions: &modifiedAccessConditions, + } blobHTTPHeaders, err := r.createBlobHTTPHeadersFromRequest(req) if err != nil { return err } - _, err = azblob.UploadBufferToBlockBlob(ctx, r.marshal(req), blobURL, azblob.UploadToBlockBlobOptions{ + + uploadOptions := azblob.UploadBufferOptions{ + AccessConditions: &accessConditions, Metadata: req.Metadata, - AccessConditions: accessConditions, - BlobHTTPHeaders: blobHTTPHeaders, - }) - if err != nil { - r.logger.Debugf("write file %s, err %s", req.Key, err) + HTTPHeaders: &blobHTTPHeaders, + Concurrency: 16, + } + + blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key)) + _, err = blockBlobClient.UploadBuffer(ctx, r.marshal(req), &uploadOptions) + if err != nil { // Check if the error is due to ETag conflict if req.ETag != nil && isETagConflictError(err) { return state.NewETagError(state.ETagMismatch, err) } - return err + return fmt.Errorf("error uploading az blob: %w", err) } return nil } -func (r *StateStore) createBlobHTTPHeadersFromRequest(req *state.SetRequest) (azblob.BlobHTTPHeaders, error) { - var blobHTTPHeaders azblob.BlobHTTPHeaders +func (r *StateStore) createBlobHTTPHeadersFromRequest(req *state.SetRequest) (blob.HTTPHeaders, error) { + blobHTTPHeaders := blob.HTTPHeaders{} if val, ok := req.Metadata[contentType]; ok && val != "" { - blobHTTPHeaders.ContentType = val + blobHTTPHeaders.BlobContentType = &val delete(req.Metadata, contentType) } if req.ContentType != nil { - if blobHTTPHeaders.ContentType != "" { - r.logger.Warnf("ContentType received from request Metadata %s, as well as ContentType property %s, choosing value from contentType property", blobHTTPHeaders.ContentType, *req.ContentType) + if blobHTTPHeaders.BlobContentType != nil { + r.logger.Warnf("ContentType received from request Metadata %s, as well as ContentType property %s, choosing value from contentType property", blobHTTPHeaders.BlobContentType, req.ContentType) } - blobHTTPHeaders.ContentType = *req.ContentType + blobHTTPHeaders.BlobContentType = req.ContentType } if val, ok := req.Metadata[contentMD5]; ok && val != "" { sDec, err := b64.StdEncoding.DecodeString(val) if err != nil || len(sDec) != 16 { - return azblob.BlobHTTPHeaders{}, fmt.Errorf("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") + return blob.HTTPHeaders{}, fmt.Errorf("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") } - blobHTTPHeaders.ContentMD5 = sDec + blobHTTPHeaders.BlobContentMD5 = sDec delete(req.Metadata, contentMD5) } if val, ok := req.Metadata[contentEncoding]; ok && val != "" { - blobHTTPHeaders.ContentEncoding = val + blobHTTPHeaders.BlobContentEncoding = &val delete(req.Metadata, contentEncoding) } if val, ok := req.Metadata[contentLanguage]; ok && val != "" { - blobHTTPHeaders.ContentLanguage = val + blobHTTPHeaders.BlobContentLanguage = &val delete(req.Metadata, contentLanguage) } if val, ok := req.Metadata[contentDisposition]; ok && val != "" { - blobHTTPHeaders.ContentDisposition = val + blobHTTPHeaders.BlobContentDisposition = &val delete(req.Metadata, contentDisposition) } if val, ok := req.Metadata[cacheControl]; ok && val != "" { - blobHTTPHeaders.CacheControl = val + blobHTTPHeaders.BlobCacheControl = &val delete(req.Metadata, cacheControl) } return blobHTTPHeaders, nil } func (r *StateStore) deleteFile(ctx context.Context, req *state.DeleteRequest) error { - blobURL := r.containerURL.NewBlockBlobURL(getFileName(req.Key)) - accessConditions := azblob.BlobAccessConditions{} + blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key)) + modifiedAccessConditions := blob.ModifiedAccessConditions{} if req.ETag != nil && *req.ETag != "" { - accessConditions.IfMatch = azblob.ETag(*req.ETag) + modifiedAccessConditions.IfMatch = ptr.Of(azcore.ETag(*req.ETag)) } - _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, accessConditions) + deleteOptions := blob.DeleteOptions{ + DeleteSnapshots: nil, + AccessConditions: &blob.AccessConditions{ + ModifiedAccessConditions: &modifiedAccessConditions, + }, + } + + _, err := blockBlobClient.Delete(ctx, &deleteOptions) if err != nil { r.logger.Debugf("delete file %s, err %s", req.Key, err) @@ -349,13 +405,9 @@ func (r *StateStore) marshal(req *state.SetRequest) []byte { } func isNotFoundError(err error) bool { - azureError, ok := err.(azblob.StorageError) - - return ok && azureError.ServiceCode() == azblob.ServiceCodeBlobNotFound + return bloberror.HasCode(err, bloberror.BlobNotFound) } func isETagConflictError(err error) bool { - azureError, ok := err.(azblob.StorageError) - - return ok && azureError.ServiceCode() == azblob.ServiceCodeConditionNotMet + return bloberror.HasCode(err, bloberror.ConditionNotMet) } diff --git a/state/azure/blobstorage/blobstorage_test.go b/state/azure/blobstorage/blobstorage_test.go index f511239b1d..e86505d516 100644 --- a/state/azure/blobstorage/blobstorage_test.go +++ b/state/azure/blobstorage/blobstorage_test.go @@ -34,8 +34,7 @@ func TestInit(t *testing.T) { } err := s.Init(m) assert.Nil(t, err) - assert.Equal(t, "acc.blob.core.windows.net", s.containerURL.URL().Host) - assert.Equal(t, "/dapr", s.containerURL.URL().Path) + assert.Equal(t, "https://acc.blob.core.windows.net/dapr", s.containerClient.URL()) }) t.Run("Init with missing metadata", func(t *testing.T) { @@ -53,7 +52,8 @@ func TestInit(t *testing.T) { "accountKey": "e+Dnvl8EOxYxV94nurVaRQ==", "containerName": "dapr", } - err := s.Init(m) + s.Init(m) + err := s.Ping() assert.NotNil(t, err) }) } @@ -100,7 +100,7 @@ func TestBlobHTTPHeaderGeneration(t *testing.T) { blobHeaders, err := s.createBlobHTTPHeadersFromRequest(req) assert.Nil(t, err) - assert.Equal(t, "application/json", blobHeaders.ContentType) + assert.Equal(t, "application/json", *blobHeaders.BlobContentType) }) t.Run("Content type and metadata provided (conflict), content type chosen", func(t *testing.T) { contentType := "application/json" @@ -113,7 +113,7 @@ func TestBlobHTTPHeaderGeneration(t *testing.T) { blobHeaders, err := s.createBlobHTTPHeadersFromRequest(req) assert.Nil(t, err) - assert.Equal(t, "application/json", blobHeaders.ContentType) + assert.Equal(t, "application/json", *blobHeaders.BlobContentType) }) t.Run("ContentType not provided, metadata provided set backward compatibility", func(t *testing.T) { req := &state.SetRequest{ @@ -124,6 +124,6 @@ func TestBlobHTTPHeaderGeneration(t *testing.T) { blobHeaders, err := s.createBlobHTTPHeadersFromRequest(req) assert.Nil(t, err) - assert.Equal(t, "text/plain", blobHeaders.ContentType) + assert.Equal(t, "text/plain", *blobHeaders.BlobContentType) }) } diff --git a/tests/certification/state/azure/blobstorage/go.mod b/tests/certification/state/azure/blobstorage/go.mod index b7c33bfbe8..1caa631ff2 100644 --- a/tests/certification/state/azure/blobstorage/go.mod +++ b/tests/certification/state/azure/blobstorage/go.mod @@ -19,6 +19,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 // indirect github.com/Azure/azure-storage-blob-go v0.10.0 // indirect github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/tests/certification/state/azure/blobstorage/go.sum b/tests/certification/state/azure/blobstorage/go.sum index 3b5acc6ec8..0200c2d339 100644 --- a/tests/certification/state/azure/blobstorage/go.sum +++ b/tests/certification/state/azure/blobstorage/go.sum @@ -47,6 +47,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4Sath github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= From 70eb9f3a9c43c7829e36a8ceb2f77a50efc6e276 Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Thu, 17 Nov 2022 20:35:05 -0800 Subject: [PATCH 05/11] Address code review comments Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- bindings/azure/blobstorage/blobstorage.go | 23 ++++++++----------- state/azure/blobstorage/blobstorage.go | 9 ++++---- .../azure/blobstorage/blobstorage_test.go | 11 +++++---- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/bindings/azure/blobstorage/blobstorage.go b/bindings/azure/blobstorage/blobstorage.go index e97101b850..5a8dbbc06d 100644 --- a/bindings/azure/blobstorage/blobstorage.go +++ b/bindings/azure/blobstorage/blobstorage.go @@ -14,12 +14,12 @@ limitations under the License. package blobstorage import ( - "bytes" "context" b64 "encoding/base64" "encoding/json" "errors" "fmt" + "io" "net/url" "strconv" "time" @@ -64,7 +64,7 @@ const ( metadataKeyCacheControl = "cacheControl" // Specifies the maximum number of HTTP requests that will be made to retry blob operations. A value // of zero means that no additional HTTP requests will be made. - defaultGetBlobRetryCount = 10 + defaultBlobRetryCount = 3 // Specifies the maximum number of blobs to return, including all BlobPrefix elements. If the request does not // specify maxresults the server will return up to 5,000 items. // See: https://docs.microsoft.com/en-us/rest/api/storageservices/list-blobs#uri-parameters @@ -76,8 +76,7 @@ var ErrMissingBlobName = errors.New("blobName is a required attribute") // AzureBlobStorage allows saving blobs to an Azure Blob Storage account. type AzureBlobStorage struct { - metadata *blobStorageMetadata - // containerURL azblob.ContainerURL + metadata *blobStorageMetadata containerClient *container.Client logger logger.Logger @@ -166,7 +165,6 @@ func (a *AzureBlobStorage) Init(metadata bindings.Metadata) error { if clientErr != nil { return fmt.Errorf("cannot init Blobstorage container client: %w", err) } - container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) a.containerClient = client } else { // fallback to AAD @@ -196,7 +194,7 @@ func (a *AzureBlobStorage) Init(metadata bindings.Metadata) error { func (a *AzureBlobStorage) parseMetadata(meta bindings.Metadata) (*blobStorageMetadata, error) { m := blobStorageMetadata{ - GetBlobRetryCount: defaultGetBlobRetryCount, + GetBlobRetryCount: defaultBlobRetryCount, } mdutils.DecodeMetadata(meta.Properties, &m) @@ -248,8 +246,8 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque blobHTTPHeaders.BlobContentType = &val delete(req.Metadata, metadataKeyContentType) } - var contentMD5 *[]byte + var contentMD5 *[]byte if val, ok := req.Metadata[metadataKeyContentMD5]; ok && val != "" { sDec, err := b64.StdEncoding.DecodeString(val) if err != nil || len(sDec) != 16 { @@ -292,7 +290,6 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque uploadOptions := azblob.UploadBufferOptions{ Metadata: a.sanitizeMetadata(req.Metadata), HTTPHeaders: &blobHTTPHeaders, - Concurrency: 16, TransactionalContentMD5: contentMD5, } @@ -337,9 +334,9 @@ func (a *AzureBlobStorage) get(ctx context.Context, req *bindings.InvokeRequest) if err != nil { return nil, fmt.Errorf("error downloading az blob: %w", err) } - blobData := &bytes.Buffer{} reader := blobDownloadResponse.Body - _, err = blobData.ReadFrom(reader) + defer reader.Close() + blobData, err := io.ReadAll(reader) if err != nil { return nil, fmt.Errorf("error reading az blob: %w", err) } @@ -368,7 +365,7 @@ func (a *AzureBlobStorage) get(ctx context.Context, req *bindings.InvokeRequest) } return &bindings.InvokeResponse{ - Data: blobData.Bytes(), + Data: blobData, Metadata: metadata, }, nil } @@ -395,7 +392,7 @@ func (a *AzureBlobStorage) delete(ctx context.Context, req *bindings.InvokeReque } blockBlobClient = a.containerClient.NewBlockBlobClient(val) - _, err := blockBlobClient.Delete(context.Background(), &deleteOptions) + _, err := blockBlobClient.Delete(ctx, &deleteOptions) return nil, err } @@ -420,7 +417,7 @@ func (a *AzureBlobStorage) list(ctx context.Context, req *bindings.InvokeRequest options.Include.Deleted = payload.Include.Deleted } - if hasPayload && payload.MaxResults != int32(0) { + if hasPayload && payload.MaxResults > 0 { options.MaxResults = &payload.MaxResults } else { options.MaxResults = ptr.Of(maxResults) // cannot get address of constant directly diff --git a/state/azure/blobstorage/blobstorage.go b/state/azure/blobstorage/blobstorage.go index a03e7c6635..5c6f6cd387 100644 --- a/state/azure/blobstorage/blobstorage.go +++ b/state/azure/blobstorage/blobstorage.go @@ -36,10 +36,10 @@ Concurrency is supported with ETags according to https://docs.microsoft.com/en-u package blobstorage import ( - "bytes" "context" b64 "encoding/base64" "fmt" + "io" "net/url" "reflect" "strings" @@ -132,7 +132,6 @@ func (r *StateStore) Init(metadata state.Metadata) error { if clientErr != nil { return fmt.Errorf("cannot init Blobstorage container client: %w", err) } - container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) r.containerClient = client } else { // fallback to AAD @@ -249,9 +248,9 @@ func (r *StateStore) readFile(ctx context.Context, req *state.GetRequest) (*stat return &state.GetResponse{}, err } - blobData := &bytes.Buffer{} reader := blobDownloadResponse.Body - _, err = blobData.ReadFrom(reader) + defer reader.Close() + blobData, err := io.ReadAll(reader) if err != nil { return &state.GetResponse{}, fmt.Errorf("error reading az blob: %w", err) } @@ -263,7 +262,7 @@ func (r *StateStore) readFile(ctx context.Context, req *state.GetRequest) (*stat contentType := blobDownloadResponse.ContentType return &state.GetResponse{ - Data: blobData.Bytes(), + Data: blobData, ETag: ptr.Of(string(*blobDownloadResponse.ETag)), ContentType: contentType, }, nil diff --git a/tests/certification/bindings/azure/blobstorage/blobstorage_test.go b/tests/certification/bindings/azure/blobstorage/blobstorage_test.go index 79c4855e18..a775cc53cc 100644 --- a/tests/certification/bindings/azure/blobstorage/blobstorage_test.go +++ b/tests/certification/bindings/azure/blobstorage/blobstorage_test.go @@ -41,6 +41,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" ) @@ -197,12 +198,12 @@ func TestBlobStorage(t *testing.T) { // confirm the deletion. _, invokeSecondGetErr := getBlobRequest(ctx, client, blobName, false) assert.Error(t, invokeSecondGetErr) - assert.Contains(t, invokeSecondGetErr.Error(), "ERROR CODE: BlobNotFound") + assert.Contains(t, invokeSecondGetErr.Error(), bloberror.BlobNotFound) // deleting the key again should fail. _, invokeDeleteErr2 := deleteBlobRequest(ctx, client, blobName, nil) assert.Error(t, invokeDeleteErr2) - assert.Contains(t, invokeDeleteErr2.Error(), "ERROR CODE: BlobNotFound") + assert.Contains(t, invokeDeleteErr2.Error(), bloberror.BlobNotFound) return nil } @@ -235,7 +236,7 @@ func TestBlobStorage(t *testing.T) { _, invokeCreateErr := client.InvokeBinding(ctx, invokeCreateRequest) assert.Error(t, invokeCreateErr) - assert.Contains(t, invokeCreateErr.Error(), "ERROR CODE: Md5Mismatch") + assert.Contains(t, invokeCreateErr.Error(), bloberror.MD5Mismatch) return nil } @@ -284,7 +285,7 @@ func TestBlobStorage(t *testing.T) { // confirm the deletion. _, invokeSecondGetErr := getBlobRequest(ctx, client, blobName, false) assert.Error(t, invokeSecondGetErr) - assert.Contains(t, invokeSecondGetErr.Error(), "ERROR CODE: BlobNotFound") + assert.Contains(t, invokeSecondGetErr.Error(), bloberror.BlobNotFound) return nil } @@ -424,7 +425,7 @@ func TestBlobStorage(t *testing.T) { // confirm the deletion. _, invokeSecondGetErr := getBlobRequest(ctx, client, "filename.txt", false) assert.Error(t, invokeSecondGetErr) - assert.Contains(t, invokeSecondGetErr.Error(), "ERROR CODE: BlobNotFound") + assert.Contains(t, invokeSecondGetErr.Error(), bloberror.BlobNotFound) return nil } From 8811d5e64fa0feb4cc3b1c10ebdda3610aa0a236 Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Thu, 17 Nov 2022 21:37:17 -0800 Subject: [PATCH 06/11] AzBlob components: Extract shared code Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- bindings/azure/blobstorage/blobstorage.go | 172 +----------------- .../azure/blobstorage/blobstorage_test.go | 71 -------- .../component/azure/blobstorage/client.go | 108 +++++++++++ .../component/azure/blobstorage/metadata.go | 123 +++++++++++++ .../azure/blobstorage/metadata_test.go | 93 ++++++++++ state/azure/blobstorage/blobstorage.go | 99 +--------- state/azure/blobstorage/blobstorage_test.go | 20 -- 7 files changed, 334 insertions(+), 352 deletions(-) create mode 100644 internal/component/azure/blobstorage/client.go create mode 100644 internal/component/azure/blobstorage/metadata.go create mode 100644 internal/component/azure/blobstorage/metadata_test.go diff --git a/bindings/azure/blobstorage/blobstorage.go b/bindings/azure/blobstorage/blobstorage.go index 5a8dbbc06d..5150a9b0ed 100644 --- a/bindings/azure/blobstorage/blobstorage.go +++ b/bindings/azure/blobstorage/blobstorage.go @@ -20,12 +20,8 @@ import ( "errors" "fmt" "io" - "net/url" "strconv" - "time" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" @@ -33,8 +29,7 @@ import ( "github.com/google/uuid" "github.com/dapr/components-contrib/bindings" - azauth "github.com/dapr/components-contrib/internal/authentication/azure" - mdutils "github.com/dapr/components-contrib/metadata" + storageinternal "github.com/dapr/components-contrib/internal/component/azure/blobstorage" "github.com/dapr/kit/logger" "github.com/dapr/kit/ptr" ) @@ -62,9 +57,6 @@ const ( metadataKeyContentLanguage = "contentLanguage" metadataKeyContentDisposition = "contentDisposition" metadataKeyCacheControl = "cacheControl" - // Specifies the maximum number of HTTP requests that will be made to retry blob operations. A value - // of zero means that no additional HTTP requests will be made. - defaultBlobRetryCount = 3 // Specifies the maximum number of blobs to return, including all BlobPrefix elements. If the request does not // specify maxresults the server will return up to 5,000 items. // See: https://docs.microsoft.com/en-us/rest/api/storageservices/list-blobs#uri-parameters @@ -76,21 +68,12 @@ var ErrMissingBlobName = errors.New("blobName is a required attribute") // AzureBlobStorage allows saving blobs to an Azure Blob Storage account. type AzureBlobStorage struct { - metadata *blobStorageMetadata + metadata *storageinternal.BlobStorageMetadata containerClient *container.Client logger logger.Logger } -type blobStorageMetadata struct { - StorageAccount string `json:"storageAccount"` - StorageAccessKey string `json:"storageAccessKey"` - Container string `json:"container"` - GetBlobRetryCount int32 `json:"getBlobRetryCount,string"` - DecodeBase64 bool `json:"decodeBase64,string"` - PublicAccessLevel azblob.PublicAccessType `json:"publicAccessLevel"` -} - type createResponse struct { BlobURL string `json:"blobURL"` BlobName string `json:"blobName"` @@ -118,110 +101,14 @@ func NewAzureBlobStorage(logger logger.Logger) bindings.OutputBinding { // Init performs metadata parsing. func (a *AzureBlobStorage) Init(metadata bindings.Metadata) error { - m, err := a.parseMetadata(metadata) - if err != nil { - return err - } - a.metadata = m - - userAgent := "dapr-" + logger.DaprVersion - options := container.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Retry: policy.RetryOptions{ - MaxRetries: a.metadata.GetBlobRetryCount, - }, - Telemetry: policy.TelemetryOptions{ - ApplicationID: userAgent, - }, - }, - } - - settings, err := azauth.NewEnvironmentSettings("storage", metadata.Properties) + var err error + a.containerClient, a.metadata, err = storageinternal.CreateContainerStorageClient(a.logger, metadata.Properties) if err != nil { return err } - customEndpoint, ok := metadata.Properties[endpointKey] - var URL *url.URL - if ok && customEndpoint != "" { - var parseErr error - URL, parseErr = url.Parse(fmt.Sprintf("%s/%s/%s", customEndpoint, m.StorageAccount, m.Container)) - if parseErr != nil { - return parseErr - } - } else { - env := settings.AzureEnvironment - URL, _ = url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", m.StorageAccount, env.StorageEndpointSuffix, m.Container)) - } - - var clientErr error - var client *container.Client - // Try using shared key credentials first - if m.StorageAccessKey != "" { - credential, newSharedKeyErr := azblob.NewSharedKeyCredential(m.StorageAccount, m.StorageAccessKey) - if err != nil { - return fmt.Errorf("invalid credentials with error: %w", newSharedKeyErr) - } - client, clientErr = container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) - if clientErr != nil { - return fmt.Errorf("cannot init Blobstorage container client: %w", err) - } - a.containerClient = client - } else { - // fallback to AAD - credential, tokenErr := settings.GetTokenCredential() - if err != nil { - return fmt.Errorf("invalid credentials with error: %w", tokenErr) - } - client, clientErr = container.NewClient(URL.String(), credential, &options) - } - if clientErr != nil { - return fmt.Errorf("cannot init Blobstorage client: %w", clientErr) - } - - createContainerOptions := container.CreateOptions{ - Access: &m.PublicAccessLevel, - Metadata: map[string]string{}, - } - timeoutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - _, err = client.Create(timeoutCtx, &createContainerOptions) - cancel() - // Don't return error, container might already exist - a.logger.Debugf("error creating container: %w", err) - a.containerClient = client - return nil } -func (a *AzureBlobStorage) parseMetadata(meta bindings.Metadata) (*blobStorageMetadata, error) { - m := blobStorageMetadata{ - GetBlobRetryCount: defaultBlobRetryCount, - } - mdutils.DecodeMetadata(meta.Properties, &m) - - if val, ok := mdutils.GetMetadataProperty(meta.Properties, azauth.StorageAccountNameKeys...); ok && val != "" { - m.StorageAccount = val - } else { - return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageAccountNameKeys[0]) - } - - if val, ok := mdutils.GetMetadataProperty(meta.Properties, azauth.StorageContainerNameKeys...); ok && val != "" { - m.Container = val - } else { - return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageContainerNameKeys[0]) - } - - // per the Dapr documentation "none" is a valid value - if m.PublicAccessLevel == "none" { - m.PublicAccessLevel = "" - } - if m.PublicAccessLevel != "" && !a.isValidPublicAccessType(m.PublicAccessLevel) { - return nil, fmt.Errorf("invalid public access level: %s; allowed: %s", - m.PublicAccessLevel, azblob.PossiblePublicAccessTypeValues()) - } - - return &m, nil -} - func (a *AzureBlobStorage) Operations() []bindings.OperationKind { return []bindings.OperationKind{ bindings.CreateOperation, @@ -288,7 +175,7 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque } uploadOptions := azblob.UploadBufferOptions{ - Metadata: a.sanitizeMetadata(req.Metadata), + Metadata: storageinternal.SanitizeMetadata(a.logger, req.Metadata), HTTPHeaders: &blobHTTPHeaders, TransactionalContentMD5: contentMD5, } @@ -486,17 +373,6 @@ func (a *AzureBlobStorage) Invoke(ctx context.Context, req *bindings.InvokeReque } } -func (a *AzureBlobStorage) isValidPublicAccessType(accessType azblob.PublicAccessType) bool { - validTypes := azblob.PossiblePublicAccessTypeValues() - for _, item := range validTypes { - if item == accessType { - return true - } - } - - return false -} - func (a *AzureBlobStorage) isValidDeleteSnapshotsOptionType(accessType azblob.DeleteSnapshotsOptionType) bool { validTypes := azblob.PossibleDeleteSnapshotsOptionTypeValues() for _, item := range validTypes { @@ -507,41 +383,3 @@ func (a *AzureBlobStorage) isValidDeleteSnapshotsOptionType(accessType azblob.De return false } - -func (a *AzureBlobStorage) sanitizeMetadata(metadata map[string]string) map[string]string { - for key, val := range metadata { - // Keep only letters and digits - n := 0 - newKey := make([]byte, len(key)) - for i := 0; i < len(key); i++ { - if (key[i] >= 'A' && key[i] <= 'Z') || - (key[i] >= 'a' && key[i] <= 'z') || - (key[i] >= '0' && key[i] <= '9') { - newKey[n] = key[i] - n++ - } - } - - if n != len(key) { - nks := string(newKey[:n]) - a.logger.Warnf("metadata key %s contains disallowed characters, sanitized to %s", key, nks) - delete(metadata, key) - metadata[nks] = val - key = nks - } - - // Remove all non-ascii characters - n = 0 - newVal := make([]byte, len(val)) - for i := 0; i < len(val); i++ { - if val[i] > 127 { - continue - } - newVal[n] = val[i] - n++ - } - metadata[key] = string(newVal[:n]) - } - - return metadata -} diff --git a/bindings/azure/blobstorage/blobstorage_test.go b/bindings/azure/blobstorage/blobstorage_test.go index 8462e497d6..fbcb6e54e1 100644 --- a/bindings/azure/blobstorage/blobstorage_test.go +++ b/bindings/azure/blobstorage/blobstorage_test.go @@ -17,83 +17,12 @@ import ( "context" "testing" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/stretchr/testify/assert" "github.com/dapr/components-contrib/bindings" "github.com/dapr/kit/logger" ) -func TestParseMetadata(t *testing.T) { - m := bindings.Metadata{} - blobStorage := NewAzureBlobStorage(logger.NewLogger("test")).(*AzureBlobStorage) - - t.Run("parse all metadata", func(t *testing.T) { - m.Properties = map[string]string{ - "storageAccount": "account", - "storageAccessKey": "key", - "container": "test", - "getBlobRetryCount": "5", - "decodeBase64": "true", - } - meta, err := blobStorage.parseMetadata(m) - assert.Nil(t, err) - assert.Equal(t, "test", meta.Container) - assert.Equal(t, "account", meta.StorageAccount) - // storageAccessKey is parsed in the azauth package - assert.Equal(t, true, meta.DecodeBase64) - assert.Equal(t, int32(5), meta.GetBlobRetryCount) - assert.Equal(t, "", string(meta.PublicAccessLevel)) - }) - - t.Run("parse metadata with publicAccessLevel = blob", func(t *testing.T) { - m.Properties = map[string]string{ - "storageAccount": "account", - "storageAccessKey": "key", - "container": "test", - "publicAccessLevel": "blob", - } - meta, err := blobStorage.parseMetadata(m) - assert.Nil(t, err) - assert.Equal(t, azblob.PublicAccessTypeBlob, meta.PublicAccessLevel) - }) - - t.Run("parse metadata with publicAccessLevel = container", func(t *testing.T) { - m.Properties = map[string]string{ - "storageAccount": "account", - "storageAccessKey": "key", - "container": "test", - "publicAccessLevel": "container", - } - meta, err := blobStorage.parseMetadata(m) - assert.Nil(t, err) - assert.Equal(t, azblob.PublicAccessTypeContainer, meta.PublicAccessLevel) - }) - - t.Run("parse metadata with invalid publicAccessLevel", func(t *testing.T) { - m.Properties = map[string]string{ - "storageAccount": "account", - "storageAccessKey": "key", - "container": "test", - "publicAccessLevel": "invalid", - } - _, err := blobStorage.parseMetadata(m) - assert.Error(t, err) - }) - - t.Run("sanitize metadata if necessary", func(t *testing.T) { - m.Properties = map[string]string{ - "somecustomfield": "some-custom-value", - "specialfield": "special:valueÜ", - "not-allowed:": "not-allowed", - } - meta := blobStorage.sanitizeMetadata(m.Properties) - assert.Equal(t, meta["somecustomfield"], "some-custom-value") - assert.Equal(t, meta["specialfield"], "special:value") - assert.Equal(t, meta["notallowed"], "not-allowed") - }) -} - func TestGetOption(t *testing.T) { blobStorage := NewAzureBlobStorage(logger.NewLogger("test")).(*AzureBlobStorage) diff --git a/internal/component/azure/blobstorage/client.go b/internal/component/azure/blobstorage/client.go new file mode 100644 index 0000000000..64eb5a6d2f --- /dev/null +++ b/internal/component/azure/blobstorage/client.go @@ -0,0 +1,108 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package blobstorage + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + + azauth "github.com/dapr/components-contrib/internal/authentication/azure" + "github.com/dapr/kit/logger" +) + +const ( + endpointKey = "endpoint" + // Specifies the maximum number of HTTP requests that will be made to retry blob operations. A value + // of zero means that no additional HTTP requests will be made. + defaultBlobRetryCount = 3 +) + +func CreateContainerStorageClient(log logger.Logger, meta map[string]string) (*container.Client, *BlobStorageMetadata, error) { + m, err := parseMetadata(meta) + if err != nil { + return nil, nil, err + } + + userAgent := "dapr-" + logger.DaprVersion + options := container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Retry: policy.RetryOptions{ + MaxRetries: m.RetryCount, + }, + Telemetry: policy.TelemetryOptions{ + ApplicationID: userAgent, + }, + }, + } + + settings, err := azauth.NewEnvironmentSettings("storage", meta) + if err != nil { + return nil, nil, err + } + customEndpoint, ok := meta[endpointKey] + var URL *url.URL + if ok && customEndpoint != "" { + var parseErr error + URL, parseErr = url.Parse(fmt.Sprintf("%s/%s/%s", customEndpoint, m.AccountName, m.ContainerName)) + if parseErr != nil { + return nil, nil, parseErr + } + } else { + env := settings.AzureEnvironment + URL, _ = url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", m.AccountName, env.StorageEndpointSuffix, m.ContainerName)) + } + + var clientErr error + var client *container.Client + // Try using shared key credentials first + if m.AccountKey != "" { + credential, newSharedKeyErr := azblob.NewSharedKeyCredential(m.AccountName, m.AccountKey) + if err != nil { + return nil, nil, fmt.Errorf("invalid credentials with error: %w", newSharedKeyErr) + } + client, clientErr = container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) + if clientErr != nil { + return nil, nil, fmt.Errorf("cannot init Blobstorage container client: %w", err) + } + } else { + // fallback to AAD + credential, tokenErr := settings.GetTokenCredential() + if err != nil { + return nil, nil, fmt.Errorf("invalid credentials with error: %w", tokenErr) + } + client, clientErr = container.NewClient(URL.String(), credential, &options) + } + if clientErr != nil { + return nil, nil, fmt.Errorf("cannot init Blobstorage client: %w", clientErr) + } + + createContainerOptions := container.CreateOptions{ + Access: &m.PublicAccessLevel, + Metadata: map[string]string{}, + } + timeoutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + _, err = client.Create(timeoutCtx, &createContainerOptions) + cancel() + // Don't return error, container might already exist + log.Debugf("error creating container: %w", err) + + return client, m, nil +} diff --git a/internal/component/azure/blobstorage/metadata.go b/internal/component/azure/blobstorage/metadata.go new file mode 100644 index 0000000000..14223c9ef3 --- /dev/null +++ b/internal/component/azure/blobstorage/metadata.go @@ -0,0 +1,123 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package blobstorage + +import ( + "fmt" + "strconv" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + + azauth "github.com/dapr/components-contrib/internal/authentication/azure" + mdutils "github.com/dapr/components-contrib/metadata" + "github.com/dapr/kit/logger" +) + +type BlobStorageMetadata struct { + AccountName string + AccountKey string + ContainerName string + RetryCount int32 `json:"retryCount,string"` + DecodeBase64 bool `json:"decodeBase64,string"` + PublicAccessLevel azblob.PublicAccessType +} + +func parseMetadata(meta map[string]string) (*BlobStorageMetadata, error) { + m := BlobStorageMetadata{ + RetryCount: defaultBlobRetryCount, + } + mdutils.DecodeMetadata(meta, &m) + + if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageAccountNameKeys...); ok && val != "" { + m.AccountName = val + } else { + return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageAccountNameKeys[0]) + } + + if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageContainerNameKeys...); ok && val != "" { + m.ContainerName = val + } else { + return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageContainerNameKeys[0]) + } + + // per the Dapr documentation "none" is a valid value + if m.PublicAccessLevel == "none" { + m.PublicAccessLevel = "" + } + if m.PublicAccessLevel != "" && !isValidPublicAccessType(m.PublicAccessLevel) { + return nil, fmt.Errorf("invalid public access level: %s; allowed: %s", + m.PublicAccessLevel, azblob.PossiblePublicAccessTypeValues()) + } + + // we need this key for backwards compatibility + if val, ok := meta["getBlobRetryCount"]; ok && val != "" { + // convert val from string to int32 + parseInt, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return nil, err + } + m.RetryCount = int32(parseInt) + } + + return &m, nil +} + +func isValidPublicAccessType(accessType azblob.PublicAccessType) bool { + validTypes := azblob.PossiblePublicAccessTypeValues() + for _, item := range validTypes { + if item == accessType { + return true + } + } + + return false +} + +func SanitizeMetadata(log logger.Logger, metadata map[string]string) map[string]string { + for key, val := range metadata { + // Keep only letters and digits + n := 0 + newKey := make([]byte, len(key)) + for i := 0; i < len(key); i++ { + if (key[i] >= 'A' && key[i] <= 'Z') || + (key[i] >= 'a' && key[i] <= 'z') || + (key[i] >= '0' && key[i] <= '9') { + newKey[n] = key[i] + n++ + } + } + + if n != len(key) { + nks := string(newKey[:n]) + log.Warnf("metadata key %s contains disallowed characters, sanitized to %s", key, nks) + delete(metadata, key) + metadata[nks] = val + key = nks + } + + // Remove all non-ascii characters + n = 0 + newVal := make([]byte, len(val)) + for i := 0; i < len(val); i++ { + if val[i] > 127 { + continue + } + newVal[n] = val[i] + n++ + } + metadata[key] = string(newVal[:n]) + } + + return metadata +} diff --git a/internal/component/azure/blobstorage/metadata_test.go b/internal/component/azure/blobstorage/metadata_test.go new file mode 100644 index 0000000000..e60ef9c149 --- /dev/null +++ b/internal/component/azure/blobstorage/metadata_test.go @@ -0,0 +1,93 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package blobstorage + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/dapr/kit/logger" + "github.com/stretchr/testify/assert" +) + +func TestParseMetadata(t *testing.T) { + logger := logger.NewLogger("test") + var m map[string]string + + t.Run("parse all metadata", func(t *testing.T) { + m = map[string]string{ + "storageAccount": "account", + "storageAccessKey": "key", + "container": "test", + "getBlobRetryCount": "5", + "decodeBase64": "true", + } + meta, err := parseMetadata(m) + assert.Nil(t, err) + assert.Equal(t, "test", meta.ContainerName) + assert.Equal(t, "account", meta.AccountName) + // storageAccessKey is parsed in the azauth package + assert.Equal(t, true, meta.DecodeBase64) + assert.Equal(t, int32(5), meta.RetryCount) + assert.Equal(t, "", string(meta.PublicAccessLevel)) + }) + + t.Run("parse metadata with publicAccessLevel = blob", func(t *testing.T) { + m = map[string]string{ + "storageAccount": "account", + "storageAccessKey": "key", + "container": "test", + "publicAccessLevel": "blob", + } + meta, err := parseMetadata(m) + assert.Nil(t, err) + assert.Equal(t, azblob.PublicAccessTypeBlob, meta.PublicAccessLevel) + }) + + t.Run("parse metadata with publicAccessLevel = container", func(t *testing.T) { + m = map[string]string{ + "storageAccount": "account", + "storageAccessKey": "key", + "container": "test", + "publicAccessLevel": "container", + } + meta, err := parseMetadata(m) + assert.Nil(t, err) + assert.Equal(t, azblob.PublicAccessTypeContainer, meta.PublicAccessLevel) + }) + + t.Run("parse metadata with invalid publicAccessLevel", func(t *testing.T) { + m = map[string]string{ + "storageAccount": "account", + "storageAccessKey": "key", + "container": "test", + "publicAccessLevel": "invalid", + } + _, err := parseMetadata(m) + assert.Error(t, err) + }) + + t.Run("sanitize metadata if necessary", func(t *testing.T) { + m = map[string]string{ + "somecustomfield": "some-custom-value", + "specialfield": "special:valueÜ", + "not-allowed:": "not-allowed", + } + meta := SanitizeMetadata(logger, m) + assert.Equal(t, meta["somecustomfield"], "some-custom-value") + assert.Equal(t, meta["specialfield"], "special:value") + assert.Equal(t, meta["notallowed"], "not-allowed") + }) +} diff --git a/state/azure/blobstorage/blobstorage.go b/state/azure/blobstorage/blobstorage.go index 5c6f6cd387..fb92126dd0 100644 --- a/state/azure/blobstorage/blobstorage.go +++ b/state/azure/blobstorage/blobstorage.go @@ -40,20 +40,17 @@ import ( b64 "encoding/base64" "fmt" "io" - "net/url" "reflect" "strings" - "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" jsoniter "github.com/json-iterator/go" - azauth "github.com/dapr/components-contrib/internal/authentication/azure" + storageinternal "github.com/dapr/components-contrib/internal/component/azure/blobstorage" mdutils "github.com/dapr/components-contrib/metadata" "github.com/dapr/components-contrib/state" "github.com/dapr/kit/logger" @@ -81,80 +78,13 @@ type StateStore struct { logger logger.Logger } -type blobStorageMetadata struct { - AccountName string - ContainerName string - AccountKey string -} - // Init the connection to blob storage, optionally creates a blob container if it doesn't exist. func (r *StateStore) Init(metadata state.Metadata) error { - m, err := getBlobStorageMetadata(metadata.Properties) - if err != nil { - return err - } - - userAgent := "dapr-" + logger.DaprVersion - options := container.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Telemetry: policy.TelemetryOptions{ - ApplicationID: userAgent, - }, - }, - } - - settings, err := azauth.NewEnvironmentSettings("storage", metadata.Properties) + var err error + r.containerClient, _, err = storageinternal.CreateContainerStorageClient(r.logger, metadata.Properties) if err != nil { return err } - customEndpoint, ok := metadata.Properties[endpointKey] - var URL *url.URL - if ok && customEndpoint != "" { - var parseErr error - URL, parseErr = url.Parse(fmt.Sprintf("%s/%s/%s", customEndpoint, m.AccountName, m.ContainerName)) - if parseErr != nil { - return parseErr - } - } else { - env := settings.AzureEnvironment - URL, _ = url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", m.AccountName, env.StorageEndpointSuffix, m.ContainerName)) - } - - var clientErr error - var client *container.Client - // Try using shared key credentials first - if m.AccountKey != "" { - credential, newSharedKeyErr := azblob.NewSharedKeyCredential(m.AccountName, m.AccountKey) - if err != nil { - return fmt.Errorf("invalid credentials with error: %w", newSharedKeyErr) - } - client, clientErr = container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) - if clientErr != nil { - return fmt.Errorf("cannot init Blobstorage container client: %w", err) - } - r.containerClient = client - } else { - // fallback to AAD - credential, tokenErr := settings.GetTokenCredential() - if err != nil { - return fmt.Errorf("invalid credentials with error: %w", tokenErr) - } - client, clientErr = container.NewClient(URL.String(), credential, &options) - } - if clientErr != nil { - return fmt.Errorf("cannot init Blobstorage client: %w", clientErr) - } - - createContainerOptions := container.CreateOptions{ - Access: nil, - } - timeoutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - _, err = client.Create(timeoutCtx, &createContainerOptions) - cancel() - // Don't return error, container might already exist - r.logger.Debugf("error creating container: %w", err) - r.containerClient = client - return nil } @@ -193,7 +123,7 @@ func (r *StateStore) Ping() error { } func (r *StateStore) GetComponentMetadata() map[string]string { - metadataStruct := blobStorageMetadata{} + metadataStruct := storageinternal.BlobStorageMetadata{} metadataInfo := map[string]string{} mdutils.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo) return metadataInfo @@ -211,25 +141,6 @@ func NewAzureBlobStorageStore(logger logger.Logger) state.Store { return s } -func getBlobStorageMetadata(meta map[string]string) (*blobStorageMetadata, error) { - m := blobStorageMetadata{} - err := mdutils.DecodeMetadata(meta, &m) - - if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageAccountNameKeys...); ok && val != "" { - m.AccountName = val - } else { - return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageAccountNameKeys[0]) - } - - if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageContainerNameKeys...); ok && val != "" { - m.ContainerName = val - } else { - return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageContainerNameKeys[0]) - } - - return &m, err -} - func (r *StateStore) readFile(ctx context.Context, req *state.GetRequest) (*state.GetResponse, error) { blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key)) @@ -289,7 +200,7 @@ func (r *StateStore) writeFile(ctx context.Context, req *state.SetRequest) error uploadOptions := azblob.UploadBufferOptions{ AccessConditions: &accessConditions, - Metadata: req.Metadata, + Metadata: storageinternal.SanitizeMetadata(r.logger, req.Metadata), HTTPHeaders: &blobHTTPHeaders, Concurrency: 16, } diff --git a/state/azure/blobstorage/blobstorage_test.go b/state/azure/blobstorage/blobstorage_test.go index e86505d516..2e69af0708 100644 --- a/state/azure/blobstorage/blobstorage_test.go +++ b/state/azure/blobstorage/blobstorage_test.go @@ -58,26 +58,6 @@ func TestInit(t *testing.T) { }) } -func TestGetBlobStorageMetaData(t *testing.T) { - t.Run("Nothing at all passed", func(t *testing.T) { - m := make(map[string]string) - _, err := getBlobStorageMetadata(m) - - assert.NotNil(t, err) - }) - - t.Run("All parameters passed and parsed", func(t *testing.T) { - m := make(map[string]string) - m["accountName"] = "acc" - m["containerName"] = "dapr" - meta, err := getBlobStorageMetadata(m) - - assert.Nil(t, err) - assert.Equal(t, "acc", meta.AccountName) - assert.Equal(t, "dapr", meta.ContainerName) - }) -} - func TestFileName(t *testing.T) { t.Run("Valid composite key", func(t *testing.T) { key := getFileName("app_id||key") From f11fc2421ed2ed1aadf2db5f84787580c9b92bf7 Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Thu, 17 Nov 2022 21:43:20 -0800 Subject: [PATCH 07/11] fix linter issue Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- internal/component/azure/blobstorage/metadata_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/component/azure/blobstorage/metadata_test.go b/internal/component/azure/blobstorage/metadata_test.go index e60ef9c149..c466fb0934 100644 --- a/internal/component/azure/blobstorage/metadata_test.go +++ b/internal/component/azure/blobstorage/metadata_test.go @@ -18,8 +18,9 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/dapr/kit/logger" "github.com/stretchr/testify/assert" + + "github.com/dapr/kit/logger" ) func TestParseMetadata(t *testing.T) { From 23d36a08d78e47d64b60553def2262bd6e5bd326 Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Fri, 18 Nov 2022 17:56:02 -0800 Subject: [PATCH 08/11] more refactoring and address code review Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- bindings/azure/blobstorage/blobstorage.go | 51 +++--------- .../component/azure/blobstorage/client.go | 15 ++-- .../azure/blobstorage/client_test.go | 64 +++++++++++++++ .../component/azure/blobstorage/metadata.go | 4 + .../component/azure/blobstorage/request.go | 73 ++++++++++++++++++ .../azure/blobstorage/request_test.go | 53 +++++++++++++ state/azure/blobstorage/blobstorage.go | 77 +------------------ state/azure/blobstorage/blobstorage_test.go | 38 --------- 8 files changed, 216 insertions(+), 159 deletions(-) create mode 100644 internal/component/azure/blobstorage/client_test.go create mode 100644 internal/component/azure/blobstorage/request.go create mode 100644 internal/component/azure/blobstorage/request_test.go diff --git a/bindings/azure/blobstorage/blobstorage.go b/bindings/azure/blobstorage/blobstorage.go index 5150a9b0ed..582de98787 100644 --- a/bindings/azure/blobstorage/blobstorage.go +++ b/bindings/azure/blobstorage/blobstorage.go @@ -49,14 +49,6 @@ const ( // Defines the delete snapshots option for the delete operation. // See: https://docs.microsoft.com/en-us/rest/api/storageservices/delete-blob#request-headers metadataKeyDeleteSnapshots = "deleteSnapshots" - // HTTP headers to be associated with the blob. - // See: https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob#request-headers-all-blob-types - metadataKeyContentType = "contentType" - metadataKeyContentMD5 = "contentMD5" - metadataKeyContentEncoding = "contentEncoding" - metadataKeyContentLanguage = "contentLanguage" - metadataKeyContentDisposition = "contentDisposition" - metadataKeyCacheControl = "cacheControl" // Specifies the maximum number of blobs to return, including all BlobPrefix elements. If the request does not // specify maxresults the server will return up to 5,000 items. // See: https://docs.microsoft.com/en-us/rest/api/storageservices/list-blobs#uri-parameters @@ -124,41 +116,16 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque blobName = val delete(req.Metadata, metadataKeyBlobName) } else { - blobName = uuid.New().String() - } - - blobHTTPHeaders := blob.HTTPHeaders{} - - if val, ok := req.Metadata[metadataKeyContentType]; ok && val != "" { - blobHTTPHeaders.BlobContentType = &val - delete(req.Metadata, metadataKeyContentType) - } - - var contentMD5 *[]byte - if val, ok := req.Metadata[metadataKeyContentMD5]; ok && val != "" { - sDec, err := b64.StdEncoding.DecodeString(val) - if err != nil || len(sDec) != 16 { - return nil, errors.New("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") + id, err := uuid.NewRandom() + if err != nil { + return nil, err } - blobHTTPHeaders.BlobContentMD5 = sDec - contentMD5 = &sDec - delete(req.Metadata, metadataKeyContentMD5) - } - if val, ok := req.Metadata[metadataKeyContentEncoding]; ok && val != "" { - blobHTTPHeaders.BlobContentEncoding = &val - delete(req.Metadata, metadataKeyContentEncoding) - } - if val, ok := req.Metadata[metadataKeyContentLanguage]; ok && val != "" { - blobHTTPHeaders.BlobContentLanguage = &val - delete(req.Metadata, metadataKeyContentLanguage) + blobName = id.String() } - if val, ok := req.Metadata[metadataKeyContentDisposition]; ok && val != "" { - blobHTTPHeaders.BlobContentDisposition = &val - delete(req.Metadata, metadataKeyContentDisposition) - } - if val, ok := req.Metadata[metadataKeyCacheControl]; ok && val != "" { - blobHTTPHeaders.BlobCacheControl = &val - delete(req.Metadata, metadataKeyCacheControl) + + blobHTTPHeaders, err := storageinternal.CreateBlobHTTPHeadersFromRequest(req.Metadata, nil, a.logger) + if err != nil { + return nil, err } d, err := strconv.Unquote(string(req.Data)) @@ -177,7 +144,7 @@ func (a *AzureBlobStorage) create(ctx context.Context, req *bindings.InvokeReque uploadOptions := azblob.UploadBufferOptions{ Metadata: storageinternal.SanitizeMetadata(a.logger, req.Metadata), HTTPHeaders: &blobHTTPHeaders, - TransactionalContentMD5: contentMD5, + TransactionalContentMD5: &blobHTTPHeaders.BlobContentMD5, } blockBlobClient := a.containerClient.NewBlockBlobClient(blobName) diff --git a/internal/component/azure/blobstorage/client.go b/internal/component/azure/blobstorage/client.go index 64eb5a6d2f..7a9250421a 100644 --- a/internal/component/azure/blobstorage/client.go +++ b/internal/component/azure/blobstorage/client.go @@ -25,11 +25,11 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" azauth "github.com/dapr/components-contrib/internal/authentication/azure" + mdutils "github.com/dapr/components-contrib/metadata" "github.com/dapr/kit/logger" ) const ( - endpointKey = "endpoint" // Specifies the maximum number of HTTP requests that will be made to retry blob operations. A value // of zero means that no additional HTTP requests will be made. defaultBlobRetryCount = 3 @@ -57,9 +57,12 @@ func CreateContainerStorageClient(log logger.Logger, meta map[string]string) (*c if err != nil { return nil, nil, err } - customEndpoint, ok := meta[endpointKey] + var customEndpoint string + if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageEndpointKeys...); ok && val != "" { + customEndpoint = val + } var URL *url.URL - if ok && customEndpoint != "" { + if customEndpoint != "" { var parseErr error URL, parseErr = url.Parse(fmt.Sprintf("%s/%s/%s", customEndpoint, m.AccountName, m.ContainerName)) if parseErr != nil { @@ -76,7 +79,7 @@ func CreateContainerStorageClient(log logger.Logger, meta map[string]string) (*c if m.AccountKey != "" { credential, newSharedKeyErr := azblob.NewSharedKeyCredential(m.AccountName, m.AccountKey) if err != nil { - return nil, nil, fmt.Errorf("invalid credentials with error: %w", newSharedKeyErr) + return nil, nil, fmt.Errorf("invalid shared key credentials with error: %w", newSharedKeyErr) } client, clientErr = container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) if clientErr != nil { @@ -86,7 +89,7 @@ func CreateContainerStorageClient(log logger.Logger, meta map[string]string) (*c // fallback to AAD credential, tokenErr := settings.GetTokenCredential() if err != nil { - return nil, nil, fmt.Errorf("invalid credentials with error: %w", tokenErr) + return nil, nil, fmt.Errorf("invalid token credentials with error: %w", tokenErr) } client, clientErr = container.NewClient(URL.String(), credential, &options) } @@ -102,7 +105,7 @@ func CreateContainerStorageClient(log logger.Logger, meta map[string]string) (*c _, err = client.Create(timeoutCtx, &createContainerOptions) cancel() // Don't return error, container might already exist - log.Debugf("error creating container: %w", err) + log.Debugf("error creating container: %v", err) return client, m, nil } diff --git a/internal/component/azure/blobstorage/client_test.go b/internal/component/azure/blobstorage/client_test.go new file mode 100644 index 0000000000..61ebe43bac --- /dev/null +++ b/internal/component/azure/blobstorage/client_test.go @@ -0,0 +1,64 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package blobstorage + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + azauth "github.com/dapr/components-contrib/internal/authentication/azure" + "github.com/dapr/kit/logger" +) + +type scenario struct { + metadata map[string]string + expectedFailureSubString string +} + +func TestClientInitFailures(t *testing.T) { + log := logger.NewLogger("test") + + scenarios := map[string]scenario{ + "missing accountName": { + metadata: createTestMetadata(false, true, true), + expectedFailureSubString: "missing or empty accountName field from metadata", + }, + "missing container": { + metadata: createTestMetadata(true, true, false), + expectedFailureSubString: "missing or empty containerName field from metadata", + }, + } + + for name, s := range scenarios { + t.Run(name, func(t *testing.T) { + _, _, err := CreateContainerStorageClient(log, s.metadata) + assert.Contains(t, err.Error(), s.expectedFailureSubString) + }) + } +} + +func createTestMetadata(accountName bool, accountKey bool, container bool) map[string]string { + m := map[string]string{} + if accountName { + m[azauth.StorageAccountNameKeys[0]] = "account" + } + if accountKey { + m[azauth.StorageAccountKeyKeys[0]] = "key" + } + if container { + m[azauth.StorageContainerNameKeys[0]] = "test" + } + return m +} diff --git a/internal/component/azure/blobstorage/metadata.go b/internal/component/azure/blobstorage/metadata.go index 14223c9ef3..04a3499e58 100644 --- a/internal/component/azure/blobstorage/metadata.go +++ b/internal/component/azure/blobstorage/metadata.go @@ -51,6 +51,10 @@ func parseMetadata(meta map[string]string) (*BlobStorageMetadata, error) { return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageContainerNameKeys[0]) } + if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageAccountKeyKeys...); ok && val != "" { + m.AccountKey = val + } + // per the Dapr documentation "none" is a valid value if m.PublicAccessLevel == "none" { m.PublicAccessLevel = "" diff --git a/internal/component/azure/blobstorage/request.go b/internal/component/azure/blobstorage/request.go new file mode 100644 index 0000000000..38bc5e4e27 --- /dev/null +++ b/internal/component/azure/blobstorage/request.go @@ -0,0 +1,73 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package blobstorage + +import ( + b64 "encoding/base64" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + + "github.com/dapr/kit/logger" +) + +const ( + contentTypeKey = "ContentType" + contentMD5Key = "ContentMD5" + contentEncodingKey = "ContentEncoding" + contentLanguageKey = "ContentLanguage" + contentDispositionKey = "ContentDisposition" + cacheControlKey = "CacheControl" +) + +func CreateBlobHTTPHeadersFromRequest(meta map[string]string, contentType *string, log logger.Logger) (blob.HTTPHeaders, error) { + blobHTTPHeaders := blob.HTTPHeaders{} + if val, ok := meta[contentTypeKey]; ok && val != "" { + blobHTTPHeaders.BlobContentType = &val + delete(meta, contentTypeKey) + } + + if contentType != nil { + if blobHTTPHeaders.BlobContentType != nil { + log.Warnf("ContentType received from request Metadata %s, as well as ContentType property %s, choosing value from contentType property", blobHTTPHeaders.BlobContentType, *contentType) + } + blobHTTPHeaders.BlobContentType = contentType + } + + if val, ok := meta[contentMD5Key]; ok && val != "" { + sDec, err := b64.StdEncoding.DecodeString(val) + if err != nil || len(sDec) != 16 { + return blob.HTTPHeaders{}, fmt.Errorf("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") + } + blobHTTPHeaders.BlobContentMD5 = sDec + delete(meta, contentMD5Key) + } + if val, ok := meta[contentEncodingKey]; ok && val != "" { + blobHTTPHeaders.BlobContentEncoding = &val + delete(meta, contentEncodingKey) + } + if val, ok := meta[contentLanguageKey]; ok && val != "" { + blobHTTPHeaders.BlobContentLanguage = &val + delete(meta, contentLanguageKey) + } + if val, ok := meta[contentDispositionKey]; ok && val != "" { + blobHTTPHeaders.BlobContentDisposition = &val + delete(meta, contentDispositionKey) + } + if val, ok := meta[cacheControlKey]; ok && val != "" { + blobHTTPHeaders.BlobCacheControl = &val + delete(meta, cacheControlKey) + } + return blobHTTPHeaders, nil +} diff --git a/internal/component/azure/blobstorage/request_test.go b/internal/component/azure/blobstorage/request_test.go new file mode 100644 index 0000000000..051eb6f0f6 --- /dev/null +++ b/internal/component/azure/blobstorage/request_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2021 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package blobstorage + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/dapr/kit/logger" +) + +func TestBlobHTTPHeaderGeneration(t *testing.T) { + log := logger.NewLogger("test") + + t.Run("Content type is set from request, forward compatibility", func(t *testing.T) { + contentType := "application/json" + requestMetadata := map[string]string{} + + blobHeaders, err := CreateBlobHTTPHeadersFromRequest(requestMetadata, &contentType, log) + assert.Nil(t, err) + assert.Equal(t, "application/json", *blobHeaders.BlobContentType) + }) + t.Run("Content type and metadata provided (conflict), content type chosen", func(t *testing.T) { + contentType := "application/json" + requestMetadata := map[string]string{ + contentTypeKey: "text/plain", + } + + blobHeaders, err := CreateBlobHTTPHeadersFromRequest(requestMetadata, &contentType, log) + assert.Nil(t, err) + assert.Equal(t, "application/json", *blobHeaders.BlobContentType) + }) + t.Run("ContentType not provided, metadata provided set backward compatibility", func(t *testing.T) { + requestMetadata := map[string]string{ + contentTypeKey: "text/plain", + } + blobHeaders, err := CreateBlobHTTPHeadersFromRequest(requestMetadata, nil, log) + assert.Nil(t, err) + assert.Equal(t, "text/plain", *blobHeaders.BlobContentType) + }) +} diff --git a/state/azure/blobstorage/blobstorage.go b/state/azure/blobstorage/blobstorage.go index fb92126dd0..c3d2387066 100644 --- a/state/azure/blobstorage/blobstorage.go +++ b/state/azure/blobstorage/blobstorage.go @@ -37,7 +37,6 @@ package blobstorage import ( "context" - b64 "encoding/base64" "fmt" "io" "reflect" @@ -58,14 +57,7 @@ import ( ) const ( - keyDelimiter = "||" - contentType = "ContentType" - contentMD5 = "ContentMD5" - contentEncoding = "ContentEncoding" - contentLanguage = "ContentLanguage" - contentDisposition = "ContentDisposition" - cacheControl = "CacheControl" - endpointKey = "endpoint" + keyDelimiter = "||" ) // StateStore Type. @@ -95,27 +87,21 @@ func (r *StateStore) Features() []state.Feature { // Delete the state. func (r *StateStore) Delete(req *state.DeleteRequest) error { - r.logger.Debugf("delete %s", req.Key) return r.deleteFile(context.Background(), req) } // Get the state. func (r *StateStore) Get(req *state.GetRequest) (*state.GetResponse, error) { - r.logger.Debugf("get %s", req.Key) return r.readFile(context.Background(), req) } // Set the state. func (r *StateStore) Set(req *state.SetRequest) error { - r.logger.Debugf("saving %s", req.Key) return r.writeFile(context.Background(), req) } func (r *StateStore) Ping() error { - getPropertiesOptions := container.GetPropertiesOptions{ - LeaseAccessConditions: &container.LeaseAccessConditions{}, - } - if _, err := r.containerClient.GetProperties(context.Background(), &getPropertiesOptions); err != nil { + if _, err := r.containerClient.GetProperties(context.Background(), nil); err != nil { return fmt.Errorf("blob storage: error connecting to Blob storage at %s: %s", r.containerClient.URL(), err) } @@ -143,15 +129,8 @@ func NewAzureBlobStorageStore(logger logger.Logger) state.Store { func (r *StateStore) readFile(ctx context.Context, req *state.GetRequest) (*state.GetResponse, error) { blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key)) - - downloadOptions := azblob.DownloadStreamOptions{ - AccessConditions: &blob.AccessConditions{}, - } - - blobDownloadResponse, err := blockBlobClient.DownloadStream(ctx, &downloadOptions) + blobDownloadResponse, err := blockBlobClient.DownloadStream(ctx, nil) if err != nil { - r.logger.Debugf("download file %s, err %s", req.Key, err) - if isNotFoundError(err) { return &state.GetResponse{}, nil } @@ -165,10 +144,6 @@ func (r *StateStore) readFile(ctx context.Context, req *state.GetRequest) (*stat if err != nil { return &state.GetResponse{}, fmt.Errorf("error reading az blob: %w", err) } - err = reader.Close() - if err != nil { - return &state.GetResponse{}, fmt.Errorf("error closing az blob reader: %w", err) - } contentType := blobDownloadResponse.ContentType @@ -193,7 +168,7 @@ func (r *StateStore) writeFile(ctx context.Context, req *state.SetRequest) error ModifiedAccessConditions: &modifiedAccessConditions, } - blobHTTPHeaders, err := r.createBlobHTTPHeadersFromRequest(req) + blobHTTPHeaders, err := storageinternal.CreateBlobHTTPHeadersFromRequest(req.Metadata, req.ContentType, r.logger) if err != nil { return err } @@ -202,7 +177,6 @@ func (r *StateStore) writeFile(ctx context.Context, req *state.SetRequest) error AccessConditions: &accessConditions, Metadata: storageinternal.SanitizeMetadata(r.logger, req.Metadata), HTTPHeaders: &blobHTTPHeaders, - Concurrency: 16, } blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key)) @@ -220,47 +194,6 @@ func (r *StateStore) writeFile(ctx context.Context, req *state.SetRequest) error return nil } -func (r *StateStore) createBlobHTTPHeadersFromRequest(req *state.SetRequest) (blob.HTTPHeaders, error) { - blobHTTPHeaders := blob.HTTPHeaders{} - if val, ok := req.Metadata[contentType]; ok && val != "" { - blobHTTPHeaders.BlobContentType = &val - delete(req.Metadata, contentType) - } - - if req.ContentType != nil { - if blobHTTPHeaders.BlobContentType != nil { - r.logger.Warnf("ContentType received from request Metadata %s, as well as ContentType property %s, choosing value from contentType property", blobHTTPHeaders.BlobContentType, req.ContentType) - } - blobHTTPHeaders.BlobContentType = req.ContentType - } - - if val, ok := req.Metadata[contentMD5]; ok && val != "" { - sDec, err := b64.StdEncoding.DecodeString(val) - if err != nil || len(sDec) != 16 { - return blob.HTTPHeaders{}, fmt.Errorf("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") - } - blobHTTPHeaders.BlobContentMD5 = sDec - delete(req.Metadata, contentMD5) - } - if val, ok := req.Metadata[contentEncoding]; ok && val != "" { - blobHTTPHeaders.BlobContentEncoding = &val - delete(req.Metadata, contentEncoding) - } - if val, ok := req.Metadata[contentLanguage]; ok && val != "" { - blobHTTPHeaders.BlobContentLanguage = &val - delete(req.Metadata, contentLanguage) - } - if val, ok := req.Metadata[contentDisposition]; ok && val != "" { - blobHTTPHeaders.BlobContentDisposition = &val - delete(req.Metadata, contentDisposition) - } - if val, ok := req.Metadata[cacheControl]; ok && val != "" { - blobHTTPHeaders.BlobCacheControl = &val - delete(req.Metadata, cacheControl) - } - return blobHTTPHeaders, nil -} - func (r *StateStore) deleteFile(ctx context.Context, req *state.DeleteRequest) error { blockBlobClient := r.containerClient.NewBlockBlobClient(getFileName(req.Key)) @@ -278,8 +211,6 @@ func (r *StateStore) deleteFile(ctx context.Context, req *state.DeleteRequest) e _, err := blockBlobClient.Delete(ctx, &deleteOptions) if err != nil { - r.logger.Debugf("delete file %s, err %s", req.Key, err) - if req.ETag != nil && isETagConflictError(err) { return state.NewETagError(state.ETagMismatch, err) } else if isNotFoundError(err) { diff --git a/state/azure/blobstorage/blobstorage_test.go b/state/azure/blobstorage/blobstorage_test.go index 2e69af0708..e8ec8a97b2 100644 --- a/state/azure/blobstorage/blobstorage_test.go +++ b/state/azure/blobstorage/blobstorage_test.go @@ -69,41 +69,3 @@ func TestFileName(t *testing.T) { assert.Equal(t, "key", key) }) } - -func TestBlobHTTPHeaderGeneration(t *testing.T) { - s := NewAzureBlobStorageStore(logger.NewLogger("logger")).(*StateStore) - t.Run("Content type is set from request, forward compatibility", func(t *testing.T) { - contentType := "application/json" - req := &state.SetRequest{ - ContentType: &contentType, - } - - blobHeaders, err := s.createBlobHTTPHeadersFromRequest(req) - assert.Nil(t, err) - assert.Equal(t, "application/json", *blobHeaders.BlobContentType) - }) - t.Run("Content type and metadata provided (conflict), content type chosen", func(t *testing.T) { - contentType := "application/json" - req := &state.SetRequest{ - ContentType: &contentType, - Metadata: map[string]string{ - contentType: "text/plain", - }, - } - - blobHeaders, err := s.createBlobHTTPHeadersFromRequest(req) - assert.Nil(t, err) - assert.Equal(t, "application/json", *blobHeaders.BlobContentType) - }) - t.Run("ContentType not provided, metadata provided set backward compatibility", func(t *testing.T) { - req := &state.SetRequest{ - Metadata: map[string]string{ - contentType: "text/plain", - }, - } - - blobHeaders, err := s.createBlobHTTPHeadersFromRequest(req) - assert.Nil(t, err) - assert.Equal(t, "text/plain", *blobHeaders.BlobContentType) - }) -} From f1b9374f9a72da8c0a1c6e94f75184cdc31dfc10 Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Fri, 18 Nov 2022 18:02:09 -0800 Subject: [PATCH 09/11] Move tests around Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- .../component/azure/blobstorage/metadata.go | 39 ------------------- .../azure/blobstorage/metadata_test.go | 15 ------- .../component/azure/blobstorage/request.go | 38 ++++++++++++++++++ .../azure/blobstorage/request_test.go | 15 +++++++ 4 files changed, 53 insertions(+), 54 deletions(-) diff --git a/internal/component/azure/blobstorage/metadata.go b/internal/component/azure/blobstorage/metadata.go index 04a3499e58..af3db20611 100644 --- a/internal/component/azure/blobstorage/metadata.go +++ b/internal/component/azure/blobstorage/metadata.go @@ -21,7 +21,6 @@ import ( azauth "github.com/dapr/components-contrib/internal/authentication/azure" mdutils "github.com/dapr/components-contrib/metadata" - "github.com/dapr/kit/logger" ) type BlobStorageMetadata struct { @@ -87,41 +86,3 @@ func isValidPublicAccessType(accessType azblob.PublicAccessType) bool { return false } - -func SanitizeMetadata(log logger.Logger, metadata map[string]string) map[string]string { - for key, val := range metadata { - // Keep only letters and digits - n := 0 - newKey := make([]byte, len(key)) - for i := 0; i < len(key); i++ { - if (key[i] >= 'A' && key[i] <= 'Z') || - (key[i] >= 'a' && key[i] <= 'z') || - (key[i] >= '0' && key[i] <= '9') { - newKey[n] = key[i] - n++ - } - } - - if n != len(key) { - nks := string(newKey[:n]) - log.Warnf("metadata key %s contains disallowed characters, sanitized to %s", key, nks) - delete(metadata, key) - metadata[nks] = val - key = nks - } - - // Remove all non-ascii characters - n = 0 - newVal := make([]byte, len(val)) - for i := 0; i < len(val); i++ { - if val[i] > 127 { - continue - } - newVal[n] = val[i] - n++ - } - metadata[key] = string(newVal[:n]) - } - - return metadata -} diff --git a/internal/component/azure/blobstorage/metadata_test.go b/internal/component/azure/blobstorage/metadata_test.go index c466fb0934..e0d6481846 100644 --- a/internal/component/azure/blobstorage/metadata_test.go +++ b/internal/component/azure/blobstorage/metadata_test.go @@ -19,12 +19,9 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/stretchr/testify/assert" - - "github.com/dapr/kit/logger" ) func TestParseMetadata(t *testing.T) { - logger := logger.NewLogger("test") var m map[string]string t.Run("parse all metadata", func(t *testing.T) { @@ -79,16 +76,4 @@ func TestParseMetadata(t *testing.T) { _, err := parseMetadata(m) assert.Error(t, err) }) - - t.Run("sanitize metadata if necessary", func(t *testing.T) { - m = map[string]string{ - "somecustomfield": "some-custom-value", - "specialfield": "special:valueÜ", - "not-allowed:": "not-allowed", - } - meta := SanitizeMetadata(logger, m) - assert.Equal(t, meta["somecustomfield"], "some-custom-value") - assert.Equal(t, meta["specialfield"], "special:value") - assert.Equal(t, meta["notallowed"], "not-allowed") - }) } diff --git a/internal/component/azure/blobstorage/request.go b/internal/component/azure/blobstorage/request.go index 38bc5e4e27..38f017bbce 100644 --- a/internal/component/azure/blobstorage/request.go +++ b/internal/component/azure/blobstorage/request.go @@ -71,3 +71,41 @@ func CreateBlobHTTPHeadersFromRequest(meta map[string]string, contentType *strin } return blobHTTPHeaders, nil } + +func SanitizeMetadata(log logger.Logger, metadata map[string]string) map[string]string { + for key, val := range metadata { + // Keep only letters and digits + n := 0 + newKey := make([]byte, len(key)) + for i := 0; i < len(key); i++ { + if (key[i] >= 'A' && key[i] <= 'Z') || + (key[i] >= 'a' && key[i] <= 'z') || + (key[i] >= '0' && key[i] <= '9') { + newKey[n] = key[i] + n++ + } + } + + if n != len(key) { + nks := string(newKey[:n]) + log.Warnf("metadata key %s contains disallowed characters, sanitized to %s", key, nks) + delete(metadata, key) + metadata[nks] = val + key = nks + } + + // Remove all non-ascii characters + n = 0 + newVal := make([]byte, len(val)) + for i := 0; i < len(val); i++ { + if val[i] > 127 { + continue + } + newVal[n] = val[i] + n++ + } + metadata[key] = string(newVal[:n]) + } + + return metadata +} diff --git a/internal/component/azure/blobstorage/request_test.go b/internal/component/azure/blobstorage/request_test.go index 051eb6f0f6..74d92e7edd 100644 --- a/internal/component/azure/blobstorage/request_test.go +++ b/internal/component/azure/blobstorage/request_test.go @@ -51,3 +51,18 @@ func TestBlobHTTPHeaderGeneration(t *testing.T) { assert.Equal(t, "text/plain", *blobHeaders.BlobContentType) }) } + +func TestSanitizeRequestMetadata(t *testing.T) { + log := logger.NewLogger("test") + t.Run("sanitize metadata if necessary", func(t *testing.T) { + m := map[string]string{ + "somecustomfield": "some-custom-value", + "specialfield": "special:valueÜ", + "not-allowed:": "not-allowed", + } + meta := SanitizeMetadata(log, m) + assert.Equal(t, meta["somecustomfield"], "some-custom-value") + assert.Equal(t, meta["specialfield"], "special:value") + assert.Equal(t, meta["notallowed"], "not-allowed") + }) +} From 7d5c06bc8df7b69ecc9aea5761357805f3751469 Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Wed, 23 Nov 2022 01:43:24 +0000 Subject: [PATCH 10/11] Make blob metadata case insensitive Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- .../component/azure/blobstorage/request.go | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/internal/component/azure/blobstorage/request.go b/internal/component/azure/blobstorage/request.go index 38f017bbce..2d3dade3d2 100644 --- a/internal/component/azure/blobstorage/request.go +++ b/internal/component/azure/blobstorage/request.go @@ -16,6 +16,7 @@ package blobstorage import ( b64 "encoding/base64" "fmt" + "strings" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" @@ -23,19 +24,25 @@ import ( ) const ( - contentTypeKey = "ContentType" - contentMD5Key = "ContentMD5" - contentEncodingKey = "ContentEncoding" - contentLanguageKey = "ContentLanguage" - contentDispositionKey = "ContentDisposition" - cacheControlKey = "CacheControl" + contentTypeKey = "contenttype" + contentMD5Key = "contentmd5" + contentEncodingKey = "contentencoding" + contentLanguageKey = "contentlanguage" + contentDispositionKey = "contentdisposition" + cacheControlKey = "cachecontrol" ) func CreateBlobHTTPHeadersFromRequest(meta map[string]string, contentType *string, log logger.Logger) (blob.HTTPHeaders, error) { + // build map to support arbitrary case + caseMap := make(map[string]string) + for k := range meta { + caseMap[strings.ToLower(k)] = k + } + blobHTTPHeaders := blob.HTTPHeaders{} - if val, ok := meta[contentTypeKey]; ok && val != "" { + if val, ok := meta[caseMap[contentTypeKey]]; ok && val != "" { blobHTTPHeaders.BlobContentType = &val - delete(meta, contentTypeKey) + delete(meta, caseMap[contentTypeKey]) } if contentType != nil { @@ -45,29 +52,29 @@ func CreateBlobHTTPHeadersFromRequest(meta map[string]string, contentType *strin blobHTTPHeaders.BlobContentType = contentType } - if val, ok := meta[contentMD5Key]; ok && val != "" { + if val, ok := meta[caseMap[contentMD5Key]]; ok && val != "" { sDec, err := b64.StdEncoding.DecodeString(val) if err != nil || len(sDec) != 16 { return blob.HTTPHeaders{}, fmt.Errorf("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") } blobHTTPHeaders.BlobContentMD5 = sDec - delete(meta, contentMD5Key) + delete(meta, caseMap[contentMD5Key]) } - if val, ok := meta[contentEncodingKey]; ok && val != "" { + if val, ok := meta[caseMap[contentEncodingKey]]; ok && val != "" { blobHTTPHeaders.BlobContentEncoding = &val - delete(meta, contentEncodingKey) + delete(meta, caseMap[contentEncodingKey]) } - if val, ok := meta[contentLanguageKey]; ok && val != "" { + if val, ok := meta[caseMap[contentLanguageKey]]; ok && val != "" { blobHTTPHeaders.BlobContentLanguage = &val - delete(meta, contentLanguageKey) + delete(meta, caseMap[contentLanguageKey]) } - if val, ok := meta[contentDispositionKey]; ok && val != "" { + if val, ok := meta[caseMap[contentDispositionKey]]; ok && val != "" { blobHTTPHeaders.BlobContentDisposition = &val - delete(meta, contentDispositionKey) + delete(meta, caseMap[contentDispositionKey]) } - if val, ok := meta[cacheControlKey]; ok && val != "" { + if val, ok := meta[caseMap[cacheControlKey]]; ok && val != "" { blobHTTPHeaders.BlobCacheControl = &val - delete(meta, cacheControlKey) + delete(meta, caseMap[cacheControlKey]) } return blobHTTPHeaders, nil } From 385c6b79b67039ece5507ab31f796483049fd47e Mon Sep 17 00:00:00 2001 From: Bernd Verst <4535280+berndverst@users.noreply.github.com> Date: Mon, 28 Nov 2022 15:51:03 -0800 Subject: [PATCH 11/11] remove unnecessary reader close() Signed-off-by: Bernd Verst <4535280+berndverst@users.noreply.github.com> --- bindings/azure/blobstorage/blobstorage.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bindings/azure/blobstorage/blobstorage.go b/bindings/azure/blobstorage/blobstorage.go index 582de98787..3724a7e0ff 100644 --- a/bindings/azure/blobstorage/blobstorage.go +++ b/bindings/azure/blobstorage/blobstorage.go @@ -194,10 +194,6 @@ func (a *AzureBlobStorage) get(ctx context.Context, req *bindings.InvokeRequest) if err != nil { return nil, fmt.Errorf("error reading az blob: %w", err) } - err = reader.Close() - if err != nil { - return nil, fmt.Errorf("error closing az blob reader: %w", err) - } var metadata map[string]string fetchMetadata, err := req.GetMetadataAsBool(metadataKeyIncludeMetadata)