Skip to content

Change upload method #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vitalik23 merged 4 commits into from
Mar 22, 2022
Merged

Change upload method #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
04aefb4
changed upload method
Mar 22, 2022
2be55ea
added createcontainer test
Mar 22, 2022
391002f
fixed tests
Mar 22, 2022
490d9d5
Merge branch 'main' into change-upload-method
Mar 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions ManagedCode.Storage.Aws/AWSStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,24 +306,30 @@ private async Task UploadStreamInternalAsync(string blobName, Stream dataStream,
BucketName = _bucket,
Key = blobName,
InputStream = dataStream,
AutoCloseStream = true,
AutoCloseStream = false,
ContentType = contentType ?? Constants.ContentType,
ServerSideEncryptionMethod = null
};

await _s3Client.EnsureBucketExistsAsync(_bucket);
await _s3Client.PutObjectAsync(putRequest, cancellationToken);
try
{
await _s3Client.PutObjectAsync(putRequest, cancellationToken);
}
catch (AmazonS3Exception)
{
await CreateContainerAsync();
await _s3Client.PutObjectAsync(putRequest, cancellationToken);
}
}

#endregion

#region CreateContainer

public async Task CreateContainerAsync()
public async Task CreateContainerAsync(CancellationToken cancellationToken = default)
{
await _s3Client.EnsureBucketExistsAsync(_bucket);
}

#endregion

}
83 changes: 63 additions & 20 deletions ManagedCode.Storage.Azure/AzureStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using ManagedCode.Storage.Azure.Options;
Expand All @@ -26,14 +27,6 @@ public AzureStorage(AzureStorageOptions options)
options.Container,
options.OriginalOptions
);


if (_options.ShouldCreateIfNotExists)
{
_blobContainerClient.CreateIfNotExists(PublicAccessType.BlobContainer);
}

_blobContainerClient.SetAccessPolicy(_options.PublicAccessType);
}

public void Dispose()
Expand Down Expand Up @@ -188,13 +181,31 @@ public async IAsyncEnumerable<BlobMetadata> GetBlobListAsync(CancellationToken c
public async Task UploadStreamAsync(string blobName, Stream dataStream, CancellationToken cancellationToken = default)
{
var blobClient = _blobContainerClient.GetBlobClient(blobName);
await blobClient.UploadAsync(dataStream, cancellationToken);

try
{
await blobClient.UploadAsync(dataStream, cancellationToken);
}
catch (RequestFailedException)
{
await CreateContainerAsync();
await blobClient.UploadAsync(dataStream, cancellationToken);
}
}

public async Task UploadAsync(string blobName, string content, CancellationToken cancellationToken = default)
{
var blobClient = _blobContainerClient.GetBlobClient(blobName);
await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken);

try
{
await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken);
}
catch (RequestFailedException)
{
await CreateContainerAsync();
await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken);
}
}

public async Task UploadFileAsync(string blobName, string pathToFile, CancellationToken cancellationToken = default)
Expand All @@ -203,7 +214,15 @@ public async Task UploadFileAsync(string blobName, string pathToFile, Cancellati

using (var fs = new FileStream(pathToFile, FileMode.Open, FileAccess.Read))
{
await blobClient.UploadAsync(fs, cancellationToken);
try
{
await blobClient.UploadAsync(fs, cancellationToken);
}
catch (RequestFailedException)
{
await CreateContainerAsync();
await blobClient.UploadAsync(fs, cancellationToken);
}
}
}

Expand All @@ -225,25 +244,51 @@ public async Task UploadAsync(BlobMetadata blobMetadata, string content, Cancell
public async Task UploadAsync(BlobMetadata blobMetadata, byte[] data, CancellationToken cancellationToken = default)
{
var blobClient = _blobContainerClient.GetBlobClient(blobMetadata.Name);
await blobClient.UploadAsync(BinaryData.FromBytes(data), cancellationToken);

try
{
await blobClient.UploadAsync(BinaryData.FromBytes(data), cancellationToken);
}
catch (RequestFailedException)
{
await CreateContainerAsync();
await blobClient.UploadAsync(BinaryData.FromBytes(data), cancellationToken);
}
}

public async Task<string> UploadAsync(string content, CancellationToken cancellationToken = default)
{
string fileName = $"{Guid.NewGuid().ToString("N").ToLowerInvariant()}.txt";

var blobClient = _blobContainerClient.GetBlobClient(fileName);
await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken);

try
{
await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken);
}
catch (RequestFailedException)
{
await CreateContainerAsync();
await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken);
}

return fileName;
}

public async Task<string> UploadAsync(Stream dataStream, CancellationToken cancellationToken = default)
{
string fileName = Guid.NewGuid().ToString("N").ToLowerInvariant();

var fileName = Guid.NewGuid().ToString("N").ToLowerInvariant();
var blobClient = _blobContainerClient.GetBlobClient(fileName);
await blobClient.UploadAsync(dataStream, cancellationToken);

try
{
await blobClient.UploadAsync(dataStream, cancellationToken);
}
catch (RequestFailedException)
{
await CreateContainerAsync();
await blobClient.UploadAsync(dataStream, cancellationToken);
}

return fileName;
}
Expand All @@ -252,17 +297,15 @@ public async Task<string> UploadAsync(Stream dataStream, CancellationToken cance

#region CreateContainer

public async Task CreateContainerAsync()
public async Task CreateContainerAsync(CancellationToken cancellationToken = default)
{

if (_options.ShouldCreateIfNotExists)
{
await _blobContainerClient.CreateIfNotExistsAsync(PublicAccessType.BlobContainer);
await _blobContainerClient.CreateIfNotExistsAsync(PublicAccessType.BlobContainer, cancellationToken: cancellationToken);
}

await _blobContainerClient.SetAccessPolicyAsync(_options.PublicAccessType);
}

#endregion

}
2 changes: 1 addition & 1 deletion ManagedCode.Storage.Core/IStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,5 @@ public interface IStorage : IDisposable
IAsyncEnumerable<bool> ExistsAsync(IEnumerable<string> blobNames, CancellationToken cancellationToken = default);
IAsyncEnumerable<bool> ExistsAsync(IEnumerable<BlobMetadata> blobs, CancellationToken cancellationToken = default);

Task CreateContainerAsync();
Task CreateContainerAsync(CancellationToken cancellationToken = default);
}
3 changes: 1 addition & 2 deletions ManagedCode.Storage.FileSystem/FileSystemStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ public async Task<string> UploadAsync(Stream dataStream, CancellationToken cance

#region CreateContainer

public async Task CreateContainerAsync()
public async Task CreateContainerAsync(CancellationToken cancellationToken = default)
{
await Task.Yield();

Expand All @@ -268,5 +268,4 @@ public async Task CreateContainerAsync()
}

#endregion

}
44 changes: 17 additions & 27 deletions ManagedCode.Storage.Gcp/GCPStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,21 +34,6 @@ public GCPStorage(GCPStorageOptions gcpStorageOptions)
{
_storageClient = StorageClient.Create(gcpStorageOptions.GoogleCredential);
}

try
{
if (_gcpStorageOptions.OriginalOptions != null)
{
_storageClient.CreateBucket(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, _gcpStorageOptions.OriginalOptions);
}
else
{
_storageClient.CreateBucket(_gcpStorageOptions.BucketOptions.ProjectId, _bucket);
}
}
catch
{
}
}

public void Dispose()
Expand Down Expand Up @@ -262,28 +247,33 @@ private async Task UploadStreamInternalAsync(string blobName, Stream dataStream,
CancellationToken cancellationToken = default)
{
contentType ??= Constants.ContentType;
await _storageClient.UploadObjectAsync(_bucket, blobName, contentType, dataStream, null, cancellationToken);

try
{
await _storageClient.UploadObjectAsync(_bucket, blobName, contentType, dataStream, null, cancellationToken);
}
catch
{
await CreateContainerAsync();
await _storageClient.UploadObjectAsync(_bucket, blobName, contentType, dataStream, null, cancellationToken);
}
}

#endregion

#region CreateContainer
public async Task CreateContainerAsync()

public async Task CreateContainerAsync(CancellationToken cancellationToken = default)
{
try
if (_gcpStorageOptions.OriginalOptions != null)
{
if (_gcpStorageOptions.OriginalOptions != null)
{
await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, _gcpStorageOptions.OriginalOptions);
}
else
{
await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket);
}
await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, _gcpStorageOptions.OriginalOptions, cancellationToken);
}
catch
else
{
await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectId, _bucket, cancellationToken: cancellationToken);
}
}

#endregion
}
15 changes: 14 additions & 1 deletion ManagedCode.Storage.Tests/StorageBaseTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
Expand Down Expand Up @@ -602,6 +603,18 @@ public async Task ExistFileByListBlobMetadataAsync()

#endregion

#region CreateContainer

[Fact]
public async Task CreateContainerAsync()
{
await FluentActions.Awaiting(() => Storage.CreateContainerAsync())
.Should().NotThrowAsync<Exception>();
}

#endregion


private async Task PrepareFileToTest(string content, string fileName)
{
if (await Storage.ExistsAsync(fileName))
Expand Down