From 04aefb4d8379760f5202ebba1c675d519b23e285 Mon Sep 17 00:00:00 2001 From: Ruslan Dudchenko Date: Tue, 22 Mar 2022 21:37:37 +0200 Subject: [PATCH 1/3] changed upload method --- ManagedCode.Storage.Aws/AWSStorage.cs | 12 ++- ManagedCode.Storage.Azure/AzureStorage.cs | 78 ++++++++++++++----- .../FileSystemStorage.cs | 1 - ManagedCode.Storage.Gcp/GCPStorage.cs | 44 ++++------- 4 files changed, 86 insertions(+), 49 deletions(-) diff --git a/ManagedCode.Storage.Aws/AWSStorage.cs b/ManagedCode.Storage.Aws/AWSStorage.cs index e22d8632..3b086c6c 100644 --- a/ManagedCode.Storage.Aws/AWSStorage.cs +++ b/ManagedCode.Storage.Aws/AWSStorage.cs @@ -311,8 +311,15 @@ private async Task UploadStreamInternalAsync(string blobName, Stream dataStream, ServerSideEncryptionMethod = null }; - await _s3Client.EnsureBucketExistsAsync(_bucket); - await _s3Client.PutObjectAsync(putRequest, cancellationToken); + try + { + await _s3Client.PutObjectAsync(putRequest, cancellationToken); + } + catch + { + await CreateContainerAsync(); + await _s3Client.PutObjectAsync(putRequest, cancellationToken); + } } #endregion @@ -325,5 +332,4 @@ public async Task CreateContainerAsync() } #endregion - } \ No newline at end of file diff --git a/ManagedCode.Storage.Azure/AzureStorage.cs b/ManagedCode.Storage.Azure/AzureStorage.cs index 41871ca0..272b4aff 100644 --- a/ManagedCode.Storage.Azure/AzureStorage.cs +++ b/ManagedCode.Storage.Azure/AzureStorage.cs @@ -26,14 +26,6 @@ public AzureStorage(AzureStorageOptions options) options.Container, options.OriginalOptions ); - - - if (_options.ShouldCreateIfNotExists) - { - _blobContainerClient.CreateIfNotExists(PublicAccessType.BlobContainer); - } - - _blobContainerClient.SetAccessPolicy(_options.PublicAccessType); } public void Dispose() @@ -188,13 +180,31 @@ public async IAsyncEnumerable GetBlobListAsync(CancellationToken c public async Task UploadStreamAsync(string blobName, Stream dataStream, CancellationToken cancellationToken = default) { var blobClient = _blobContainerClient.GetBlobClient(blobName); - await blobClient.UploadAsync(dataStream, cancellationToken); + + try + { + await blobClient.UploadAsync(dataStream, cancellationToken); + } + catch + { + await CreateContainerAsync(); + await blobClient.UploadAsync(dataStream, cancellationToken); + } } public async Task UploadAsync(string blobName, string content, CancellationToken cancellationToken = default) { var blobClient = _blobContainerClient.GetBlobClient(blobName); - await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); + + try + { + await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); + } + catch + { + await CreateContainerAsync(); + await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); + } } public async Task UploadFileAsync(string blobName, string pathToFile, CancellationToken cancellationToken = default) @@ -203,7 +213,15 @@ public async Task UploadFileAsync(string blobName, string pathToFile, Cancellati using (var fs = new FileStream(pathToFile, FileMode.Open, FileAccess.Read)) { - await blobClient.UploadAsync(fs, cancellationToken); + try + { + await blobClient.UploadAsync(fs, cancellationToken); + } + catch + { + await CreateContainerAsync(); + await blobClient.UploadAsync(fs, cancellationToken); + } } } @@ -225,7 +243,16 @@ public async Task UploadAsync(BlobMetadata blobMetadata, string content, Cancell public async Task UploadAsync(BlobMetadata blobMetadata, byte[] data, CancellationToken cancellationToken = default) { var blobClient = _blobContainerClient.GetBlobClient(blobMetadata.Name); - await blobClient.UploadAsync(BinaryData.FromBytes(data), cancellationToken); + + try + { + await blobClient.UploadAsync(BinaryData.FromBytes(data), cancellationToken); + } + catch + { + await CreateContainerAsync(); + await blobClient.UploadAsync(BinaryData.FromBytes(data), cancellationToken); + } } public async Task UploadAsync(string content, CancellationToken cancellationToken = default) @@ -233,17 +260,34 @@ public async Task UploadAsync(string content, CancellationToken cancella string fileName = $"{Guid.NewGuid().ToString("N").ToLowerInvariant()}.txt"; var blobClient = _blobContainerClient.GetBlobClient(fileName); - await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); + try + { + await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); + } + catch + { + await CreateContainerAsync(); + await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); + } + return fileName; } public async Task UploadAsync(Stream dataStream, CancellationToken cancellationToken = default) { - string fileName = Guid.NewGuid().ToString("N").ToLowerInvariant(); - + var fileName = Guid.NewGuid().ToString("N").ToLowerInvariant(); var blobClient = _blobContainerClient.GetBlobClient(fileName); - await blobClient.UploadAsync(dataStream, cancellationToken); + + try + { + await blobClient.UploadAsync(dataStream, cancellationToken); + } + catch + { + await CreateContainerAsync(); + await blobClient.UploadAsync(dataStream, cancellationToken); + } return fileName; } @@ -254,7 +298,6 @@ public async Task UploadAsync(Stream dataStream, CancellationToken cance public async Task CreateContainerAsync() { - if (_options.ShouldCreateIfNotExists) { await _blobContainerClient.CreateIfNotExistsAsync(PublicAccessType.BlobContainer); @@ -264,5 +307,4 @@ public async Task CreateContainerAsync() } #endregion - } \ No newline at end of file diff --git a/ManagedCode.Storage.FileSystem/FileSystemStorage.cs b/ManagedCode.Storage.FileSystem/FileSystemStorage.cs index bf32f4f8..569e3a8d 100644 --- a/ManagedCode.Storage.FileSystem/FileSystemStorage.cs +++ b/ManagedCode.Storage.FileSystem/FileSystemStorage.cs @@ -270,5 +270,4 @@ public async Task CreateContainerAsync() } #endregion - } \ No newline at end of file diff --git a/ManagedCode.Storage.Gcp/GCPStorage.cs b/ManagedCode.Storage.Gcp/GCPStorage.cs index 0230d580..5dfddb64 100644 --- a/ManagedCode.Storage.Gcp/GCPStorage.cs +++ b/ManagedCode.Storage.Gcp/GCPStorage.cs @@ -34,21 +34,6 @@ public GCPStorage(GCPStorageOptions gcpStorageOptions) { _storageClient = StorageClient.Create(gcpStorageOptions.GoogleCredential); } - - try - { - if (_gcpStorageOptions.OriginalOptions != null) - { - _storageClient.CreateBucket(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, _gcpStorageOptions.OriginalOptions); - } - else - { - _storageClient.CreateBucket(_gcpStorageOptions.BucketOptions.ProjectId, _bucket); - } - } - catch - { - } } public void Dispose() @@ -262,28 +247,33 @@ private async Task UploadStreamInternalAsync(string blobName, Stream dataStream, CancellationToken cancellationToken = default) { contentType ??= Constants.ContentType; - await _storageClient.UploadObjectAsync(_bucket, blobName, contentType, dataStream, null, cancellationToken); + + try + { + await _storageClient.UploadObjectAsync(_bucket, blobName, contentType, dataStream, null, cancellationToken); + } + catch + { + await CreateContainerAsync(); + await _storageClient.UploadObjectAsync(_bucket, blobName, contentType, dataStream, null, cancellationToken); + } } #endregion #region CreateContainer - public async Task CreateContainerAsync() + + public async Task CreateContainerAsync() { - try + if (_gcpStorageOptions.OriginalOptions != null) { - if (_gcpStorageOptions.OriginalOptions != null) - { - await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, _gcpStorageOptions.OriginalOptions); - } - else - { - await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket); - } + await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, _gcpStorageOptions.OriginalOptions); } - catch + else { + await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket); } } + #endregion } \ No newline at end of file From 2be55eaaf41ec307b0f669b9a8d0bf760458f1b9 Mon Sep 17 00:00:00 2001 From: Vitaliy Basanets Date: Tue, 22 Mar 2022 22:02:10 +0200 Subject: [PATCH 2/3] added createcontainer test --- ManagedCode.Storage.Aws/AWSStorage.cs | 2 +- ManagedCode.Storage.Azure/AzureStorage.cs | 4 ++-- ManagedCode.Storage.Core/IStorage.cs | 2 +- .../FileSystemStorage.cs | 2 +- ManagedCode.Storage.Gcp/GCPStorage.cs | 6 +++--- ManagedCode.Storage.Tests/StorageBaseTests.cs | 15 ++++++++++++++- 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/ManagedCode.Storage.Aws/AWSStorage.cs b/ManagedCode.Storage.Aws/AWSStorage.cs index 3b086c6c..3602eaa5 100644 --- a/ManagedCode.Storage.Aws/AWSStorage.cs +++ b/ManagedCode.Storage.Aws/AWSStorage.cs @@ -326,7 +326,7 @@ private async Task UploadStreamInternalAsync(string blobName, Stream dataStream, #region CreateContainer - public async Task CreateContainerAsync() + public async Task CreateContainerAsync(CancellationToken cancellationToken = default) { await _s3Client.EnsureBucketExistsAsync(_bucket); } diff --git a/ManagedCode.Storage.Azure/AzureStorage.cs b/ManagedCode.Storage.Azure/AzureStorage.cs index 272b4aff..b84bc8e4 100644 --- a/ManagedCode.Storage.Azure/AzureStorage.cs +++ b/ManagedCode.Storage.Azure/AzureStorage.cs @@ -296,11 +296,11 @@ public async Task UploadAsync(Stream dataStream, CancellationToken cance #region CreateContainer - public async Task CreateContainerAsync() + public async Task CreateContainerAsync(CancellationToken cancellationToken = default) { if (_options.ShouldCreateIfNotExists) { - await _blobContainerClient.CreateIfNotExistsAsync(PublicAccessType.BlobContainer); + await _blobContainerClient.CreateIfNotExistsAsync(PublicAccessType.BlobContainer, cancellationToken: cancellationToken); } await _blobContainerClient.SetAccessPolicyAsync(_options.PublicAccessType); diff --git a/ManagedCode.Storage.Core/IStorage.cs b/ManagedCode.Storage.Core/IStorage.cs index bc295610..f1e80930 100644 --- a/ManagedCode.Storage.Core/IStorage.cs +++ b/ManagedCode.Storage.Core/IStorage.cs @@ -38,5 +38,5 @@ public interface IStorage : IDisposable IAsyncEnumerable ExistsAsync(IEnumerable blobNames, CancellationToken cancellationToken = default); IAsyncEnumerable ExistsAsync(IEnumerable blobs, CancellationToken cancellationToken = default); - Task CreateContainerAsync(); + Task CreateContainerAsync(CancellationToken cancellationToken = default); } \ No newline at end of file diff --git a/ManagedCode.Storage.FileSystem/FileSystemStorage.cs b/ManagedCode.Storage.FileSystem/FileSystemStorage.cs index 569e3a8d..89680389 100644 --- a/ManagedCode.Storage.FileSystem/FileSystemStorage.cs +++ b/ManagedCode.Storage.FileSystem/FileSystemStorage.cs @@ -259,7 +259,7 @@ public async Task UploadAsync(Stream dataStream, CancellationToken cance #region CreateContainer - public async Task CreateContainerAsync() + public async Task CreateContainerAsync(CancellationToken cancellationToken = default) { await Task.Yield(); diff --git a/ManagedCode.Storage.Gcp/GCPStorage.cs b/ManagedCode.Storage.Gcp/GCPStorage.cs index 5dfddb64..0d0b4ae6 100644 --- a/ManagedCode.Storage.Gcp/GCPStorage.cs +++ b/ManagedCode.Storage.Gcp/GCPStorage.cs @@ -263,15 +263,15 @@ private async Task UploadStreamInternalAsync(string blobName, Stream dataStream, #region CreateContainer - public async Task CreateContainerAsync() + public async Task CreateContainerAsync(CancellationToken cancellationToken = default) { if (_gcpStorageOptions.OriginalOptions != null) { - await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, _gcpStorageOptions.OriginalOptions); + await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, _gcpStorageOptions.OriginalOptions, cancellationToken); } else { - await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket); + await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, cancellationToken: cancellationToken); } } diff --git a/ManagedCode.Storage.Tests/StorageBaseTests.cs b/ManagedCode.Storage.Tests/StorageBaseTests.cs index a5f9fb1c..d3807154 100644 --- a/ManagedCode.Storage.Tests/StorageBaseTests.cs +++ b/ManagedCode.Storage.Tests/StorageBaseTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; @@ -602,6 +603,18 @@ public async Task ExistFileByListBlobMetadataAsync() #endregion + #region CreateContainer + + [Fact] + public async Task CreateContainerAsync() + { + await FluentActions.Awaiting(() => Storage.CreateContainerAsync()) + .Should().NotThrowAsync(); + } + + #endregion + + private async Task PrepareFileToTest(string content, string fileName) { if (await Storage.ExistsAsync(fileName)) From 391002fe644fe982169aefd29ae488a311137767 Mon Sep 17 00:00:00 2001 From: Vitaliy Basanets Date: Tue, 22 Mar 2022 22:32:32 +0200 Subject: [PATCH 3/3] fixed tests --- ManagedCode.Storage.Aws/AWSStorage.cs | 4 ++-- ManagedCode.Storage.Azure/AzureStorage.cs | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ManagedCode.Storage.Aws/AWSStorage.cs b/ManagedCode.Storage.Aws/AWSStorage.cs index 3602eaa5..2f963e37 100644 --- a/ManagedCode.Storage.Aws/AWSStorage.cs +++ b/ManagedCode.Storage.Aws/AWSStorage.cs @@ -306,7 +306,7 @@ private async Task UploadStreamInternalAsync(string blobName, Stream dataStream, BucketName = _bucket, Key = blobName, InputStream = dataStream, - AutoCloseStream = true, + AutoCloseStream = false, ContentType = contentType ?? Constants.ContentType, ServerSideEncryptionMethod = null }; @@ -315,7 +315,7 @@ private async Task UploadStreamInternalAsync(string blobName, Stream dataStream, { await _s3Client.PutObjectAsync(putRequest, cancellationToken); } - catch + catch (AmazonS3Exception) { await CreateContainerAsync(); await _s3Client.PutObjectAsync(putRequest, cancellationToken); diff --git a/ManagedCode.Storage.Azure/AzureStorage.cs b/ManagedCode.Storage.Azure/AzureStorage.cs index b84bc8e4..523e0a9b 100644 --- a/ManagedCode.Storage.Azure/AzureStorage.cs +++ b/ManagedCode.Storage.Azure/AzureStorage.cs @@ -5,6 +5,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using ManagedCode.Storage.Azure.Options; @@ -185,7 +186,7 @@ public async Task UploadStreamAsync(string blobName, Stream dataStream, Cancella { await blobClient.UploadAsync(dataStream, cancellationToken); } - catch + catch (RequestFailedException) { await CreateContainerAsync(); await blobClient.UploadAsync(dataStream, cancellationToken); @@ -200,7 +201,7 @@ public async Task UploadAsync(string blobName, string content, CancellationToken { await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); } - catch + catch (RequestFailedException) { await CreateContainerAsync(); await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); @@ -217,7 +218,7 @@ public async Task UploadFileAsync(string blobName, string pathToFile, Cancellati { await blobClient.UploadAsync(fs, cancellationToken); } - catch + catch (RequestFailedException) { await CreateContainerAsync(); await blobClient.UploadAsync(fs, cancellationToken); @@ -248,7 +249,7 @@ public async Task UploadAsync(BlobMetadata blobMetadata, byte[] data, Cancellati { await blobClient.UploadAsync(BinaryData.FromBytes(data), cancellationToken); } - catch + catch (RequestFailedException) { await CreateContainerAsync(); await blobClient.UploadAsync(BinaryData.FromBytes(data), cancellationToken); @@ -265,7 +266,7 @@ public async Task UploadAsync(string content, CancellationToken cancella { await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); } - catch + catch (RequestFailedException) { await CreateContainerAsync(); await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken); @@ -283,7 +284,7 @@ public async Task UploadAsync(Stream dataStream, CancellationToken cance { await blobClient.UploadAsync(dataStream, cancellationToken); } - catch + catch (RequestFailedException) { await CreateContainerAsync(); await blobClient.UploadAsync(dataStream, cancellationToken);