Skip to content

Commit

Permalink
Fix API
Browse files Browse the repository at this point in the history
  • Loading branch information
teoadal committed Apr 27, 2024
1 parent 3f367fb commit 402878f
Show file tree
Hide file tree
Showing 16 changed files with 664 additions and 401 deletions.
36 changes: 24 additions & 12 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

Привет! Это обертка над HttpClient для работы с S3 хранилищами. Мотивация создания была простейшей - я не понимал,
почему клиенты [AWS](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/welcome.html)
и [Minio](https://github.com/minio/minio-dotnet) едят так много памяти . Результат моих экспериментов: скорость
и [Minio](https://github.com/minio/minio-dotnet) едят так много памяти. Результат моих экспериментов: скорость
почти как у Minio, а памяти потребляю почти в 200 раз меньше, чем клиент для AWS.

```ini
Expand All @@ -18,13 +18,13 @@ Unknown processor
[Host] : .NET 8.0.4 (8.0.424.16909), Arm64 RyuJIT AdvSIMD
.NET 8.0 : .NET 8.0.4 (8.0.424.16909), Arm64 RyuJIT AdvSIMD

Job=.NET 8.0 Runtime=.NET 8.0
Job=.NET 8.0 Runtime=.NET 8.0
```
| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio |
|-------- |--------:|---------:|---------:|------:|--------:|-----------:|----------:|-------------:|------------:|
| Aws | 1.51 s | 0.030 s | 0.035 s | 1.42 | 0.03 | 80000.00 | 5000.00 | 201710 KB | 420.92 |
| Minio | 1.48 s | 0.027 s | 0.023 s | 1.39 | 0.02 | - | - | 279524 KB | 583.30 |
| Storage | 1.06 s | 0.014 s | 0.013 s | 1.00 | 0.00 | - | - | 479 KB | 1.00 |
| Method | Mean | Ratio | Gen0 | Gen1 | Allocated | Alloc Ratio |
|-------- |--------:|------:|------------:|-----------:|--------------:|------------:|
| Aws | 1.497 s | 1.45 | 80 000 | 6 000 | 201 728.07 KB | 333.24 |
| Minio | 1.468 s | 1.43 | - | - | 279 532.97 KB | 461.76 |
| Storage | 1.031 s | 1.00 | - | - | 605.36 KB | 1.00 |

## Создание клиента

Expand Down Expand Up @@ -57,7 +57,7 @@ Amazon S3 не тестировался.
bool bucketCreateResult = await storageClient.CreateBucket(cancellationToken);
Console.WriteLine(bucketCreateResult
? "Bucket создан"
: $"Bucket не был создан");
: "Bucket не был создан");
```

### Проверка существования bucket'a
Expand Down Expand Up @@ -100,8 +100,8 @@ if (fileUploadResult) Console.WriteLine("Файл загружен");

using S3Upload upload = await storageClient.UploadFile(fileName, fileType, cancellationToken);

await upload.Upload(stream, cancellationToken); // загружаем часть документа
if (!await upload.Upload(byteArray, cancellationToken)) { // загружаем другую часть документа
await upload.AddParts(stream, cancellationToken); // загружаем части документа
if (!await upload.AddParts(byteArray, cancellationToken)) { // загружаем другую часть документа
await upload.Abort(cancellationToken); // отменяем загрузку
}
else {
Expand All @@ -125,11 +125,21 @@ else {
}
```

### Получение файла как Stream

```csharp
var fileStream = await storageClient.GetFileStream(fileName, cancellationToken);
```

В случае, если файл не существует, возвратится `Stream.Null`.

### Проверка существования файла

```csharp
bool fileExistsResult = await storageClient.IsFileExists(fileName, cancellationToken);
if (fileExistsResult) Console.WriteLine("Файл существует");
if (fileExistsResult) {
Console.WriteLine("Файл существует");
}
```

### Создание подписанной ссылки на файл
Expand All @@ -139,7 +149,9 @@ if (fileExistsResult) Console.WriteLine("Файл существует");

```csharp
string? preSignedFileUrl = storageClient.GetFileUrl(fileName, expiration);
if (preSignedFileUrl != null) Console.WriteLine($"URL получен: {preSignedFileUrl}");
if (preSignedFileUrl != null) {
Console.WriteLine($"URL получен: {preSignedFileUrl}");
}
```

Существует не безопасный способ создать ссылку, без проверки наличия файла в S3.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ public void Config()
var config = BenchmarkHelper.ReadConfiguration();
var settings = BenchmarkHelper.ReadSettings(config);

_cancellation = new CancellationToken();
_cancellation = CancellationToken.None;
_fileId = "привет-как-делаdcd156a8-b6bd-4130-a2c7-8a38dbfebbc7";
_s3Client = BenchmarkHelper.CreateStoragesClient(settings);

Expand Down
2 changes: 1 addition & 1 deletion src/Storage.Benchmark/Storage.Benchmark.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="Minio" Version="6.0.2" />
<PackageReference Include="AWSSDK.S3" Version="3.7.307.23" />
<PackageReference Include="AWSSDK.S3" Version="3.7.307.24" />
</ItemGroup>


Expand Down
61 changes: 60 additions & 1 deletion src/Storage.Tests/ObjectShould.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,23 @@ public ObjectShould(StorageFixture fixture)
_notExistsBucketClient = TestHelper.CloneClient(_fixture);
}

[Fact]
public async Task AbortMultipart()
{
var fileName = _fixture.Create<string>();
var data = GetByteArray(50 * 1024 * 1024);

using var uploader = await _client.UploadFile(fileName, StreamContentType, _ct);

var addResult = await uploader.AddPart(data, _ct);
addResult
.Should().BeTrue();

var abortResult = await uploader.Abort(_ct);
abortResult
.Should().BeTrue();
}

[Fact]
public async Task AllowParallelUploadMultipleFiles()
{
Expand Down Expand Up @@ -121,12 +138,26 @@ public async Task DisposeStorageFile()
var fileName = await CreateTestFile();
using var fileGetResult = await _client.GetFile(fileName, _ct);

// ReSharper disable once MethodHasAsyncOverload
// ReSharper disable once DisposeOnUsingVariable
fileGetResult.Dispose();

await DeleteTestFile(fileName);
}

[Fact]
public async Task GetFileStream()
{
var fileName = await CreateTestFile();

var fileStream = await _client.GetFileStream(fileName, _ct);

using var bufferStream = GetEmptyByteStream();
await fileStream.CopyToAsync(bufferStream, _ct);

await EnsureFileSame(fileName, bufferStream.ToArray());
await DeleteTestFile(fileName);
}

[Fact]
public async Task GetFileUrl()
{
Expand Down Expand Up @@ -261,6 +292,20 @@ public async Task PutByteArray()
await DeleteTestFile(fileName);
}

[Fact]
public async Task PutBigByteArray()
{
var fileName = _fixture.Create<string>();
var data = GetByteArray(50 * 1024 * 1024);
var filePutResult = await _client.UploadFile(fileName, StreamContentType, data, _ct);

filePutResult
.Should().BeTrue();

await EnsureFileSame(fileName, data);
await DeleteTestFile(fileName);
}

[Fact]
public async Task PutStream()
{
Expand All @@ -275,6 +320,20 @@ public async Task PutStream()
await DeleteTestFile(fileName);
}

[Fact]
public async Task PutBigStream()
{
var fileName = _fixture.Create<string>();
var data = GetByteStream(50 * 1024 * 1024);
var filePutResult = await _client.UploadFile(fileName, StreamContentType, data, _ct);

filePutResult
.Should().BeTrue();

await EnsureFileSame(fileName, data);
await DeleteTestFile(fileName);
}

[Fact]
public async Task Upload()
{
Expand Down
2 changes: 1 addition & 1 deletion src/Storage.Tests/StorageFixture.cs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public static MemoryStream GetByteStream(int size = DefaultByteArraySize)
return new MemoryStream(GetByteArray(size));
}

public static MemoryStream GetEmptyByteStream(long? size)
public static MemoryStream GetEmptyByteStream(long? size = null)
{
return size.HasValue
? new MemoryStream(new byte[(int)size])
Expand Down
76 changes: 76 additions & 0 deletions src/Storage/S3Client.Buckets.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
using Storage.Utils;
using static Storage.Utils.HashHelper;

namespace Storage;

/// <summary>
/// Функции управления бакетом
/// </summary>
public sealed partial class S3Client
{
public async Task<bool> CreateBucket(CancellationToken ct)
{
HttpResponseMessage response;
using (var request = CreateRequest(HttpMethod.Put))
{
response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false);
}

switch (response.StatusCode)
{
case HttpStatusCode.OK:
response.Dispose();
return true;
case HttpStatusCode.Conflict: // already exists
response.Dispose();
return false;
default:
Errors.UnexpectedResult(response);
return false;
}
}

public async Task<bool> DeleteBucket(CancellationToken ct)
{
HttpResponseMessage response;
using (var request = CreateRequest(HttpMethod.Delete))
{
response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false);
}

switch (response.StatusCode)
{
case HttpStatusCode.NoContent:
response.Dispose();
return true;
case HttpStatusCode.NotFound:
response.Dispose();
return false;
default:
Errors.UnexpectedResult(response);
return false;
}
}

public async Task<bool> IsBucketExists(CancellationToken ct)
{
HttpResponseMessage response;
using (var request = CreateRequest(HttpMethod.Head))
{
response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false);
}

switch (response.StatusCode)
{
case HttpStatusCode.OK:
response.Dispose();
return true;
case HttpStatusCode.NotFound:
response.Dispose();
return false;
default:
Errors.UnexpectedResult(response);
return false;
}
}
}

0 comments on commit 402878f

Please sign in to comment.