develop (#67)

* aws * Added Managed Identity for Azure Storage * Release alpha version * base controller * Add methods from StorageFromFileExt and StorageExt (#46) Co-authored-by: Hurko Volodymyr <hurko@managed-code.com> * prototype * test containsers azure + aws * checks * actions * test containsers * test * rename * #54 * refactoring * tests * tests * tests * fix all tests * naming * fix test container for google * ndepend * Upload chunks (#69) * add integration tests project * add client * add test controller and controller tests * fix * add IStorageClient * fix storage client * accept file and content names in client method * add content name property * return status code when request is failed * add test for file upload from file info * move tests * add tests for upload methods with azure * fix tests * return downloaded file as stream * return localFile from download method * return result on download * add base test classes for controller tests * add methods upload in chunk --------- Co-authored-by: TRybina132 <rybina@managed-code.com> * add stream tests classes * Add integration tests (#70) * add integration tests project * add client * add test controller and controller tests * fix * add IStorageClient * fix storage client * accept file and content names in client method * add content name property * return status code when request is failed * add test for file upload from file info * move tests * add tests for upload methods with azure * fix tests * return downloaded file as stream * return localFile from download method * return result on download * add base test classes for controller tests * use another controller for base controller * use another method for crc calculation * remove duplicated controller * remove local folder path and replace it with path from method --------- Co-authored-by: TRybina132 <rybina@managed-code.com> Co-authored-by: Olexandr Ivanitskyi <6saaanchik6@gmail.com> * fix azure blob stream lenght * fix tests * add write test for azure blob * add seek test * Updated code coverage in pipeline (#74) * Temporarily removed some code from dotnet.yml pipeline * codecoverage * Modified test csproj * Rollbacked changes to csproj * Added stopIfQGFailed: false * Updated dotnet.yml * Updated dotnet.yml * Modified pipeline * Fixed type and temporarily removed some code * Added ndepend * Brought back missing code * Added --no-build flag --------- Co-authored-by: Kostiantyn Bondarenko <bondarenko@managed-code.com> Co-authored-by: ksemenenko <KSemenenko@users.noreply.github.com> * add properties variable --------- Co-authored-by: dtymakhov <dtymakhov@gmail.com> Co-authored-by: Alexandr Ivanitskyi <6saaanchik6@gmail.com> Co-authored-by: Hurko Volodymyr <92858780+Hurko-Volodymyr@users.noreply.github.com> Co-authored-by: Hurko Volodymyr <hurko@managed-code.com> Co-authored-by: Eduard <eduard@managed-code.com> Co-authored-by: Alexandr Ivanitskyi <91984393+Sanyyazzz@users.noreply.github.com> Co-authored-by: TRybina132 <rybina@managed-code.com> Co-authored-by: TRybina132 <98330987+TRybina132@users.noreply.github.com> Co-authored-by: Kostiantyn Bondarenko <92333402+techcat18@users.noreply.github.com> Co-authored-by: Kostiantyn Bondarenko <bondarenko@managed-code.com>
managedcode · Oct 20, 2023 · cf6b55e · cf6b55e
1 parent 9b91d02
commit cf6b55e
Show file tree

Hide file tree

Showing 32 changed files with 1,160 additions and 86 deletions.
diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml
@@ -5,59 +5,46 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
-
 
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 jobs:
 
   build-and-test:
     runs-on: ubuntu-latest
-          
+
     steps:
 
-    - name: checkout
-      uses: actions/checkout@v3
-
-    - name: Setup .NET
-      uses: actions/setup-dotnet@v3
-      with:
-        dotnet-version: 7.0.x
-
-    - name: Restore dependencies
-      run: dotnet restore
-
-    - name: Build
-      run: dotnet build --no-restore
-
-    - name: Test
-      run: dotnet test --no-build --logger 'trx;LogFileName=test-results.trx'
-
-    - name: Collect Code Coverage
-      run: dotnet test --no-build --verbosity normal /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=ManagedCode.Storage.Tests/lcov.info
-
-
-    - name: NDepend
-      uses: ndepend/ndepend-action@v1
-      with:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        license: ${{ secrets.NDEPENDLICENSE }}
-        coveragefolder: ManagedCode.Storage.Tests
-        baseline: recent
-        #baseline: main_recent
-
-    - name : coverlet
-      uses: b3b00/coverlet-action@1.1.9
-      with:
-        testProject: 'ManagedCode.Storage.Tests/ManagedCode.Storage.Tests.csproj'
-        output: 'lcov.info'
-        outputFormat: 'lcov'
-        excludes: '[program]*,[test]test.*'
-    - name: coveralls
-      uses: coverallsapp/github-action@master
-      with:
-        github-token: ${{secrets.GITHUB_TOKEN }}
-        path-to-lcov: ManagedCode.Storage.Tests/lcov.info
-
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 7.0.x
+
+      - name: Restore dependencies
+        run: dotnet restore
+
+      - name: Build
+        run: dotnet build --no-restore
+
+      - name: Test
+        run: dotnet test --no-build --logger 'trx;LogFileName=test-results.trx'
+
+      - name: Collect Code Coverage
+        run: dotnet test --no-build /p:CollectCoverage=true /p:CoverletOutput=TestResults/ /p:CoverletOutputFormat=lcov
 
+      - name: NDepend
+        uses: ndepend/ndepend-action@v1
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          license: ${{ secrets.NDEPENDLICENSE }}
+          coveragefolder: ManagedCode.Storage.Tests/TestResults
+          baseline: recent
+
+      - name: coveralls
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          path-to-lcov: ManagedCode.Storage.Tests/TestResults/coverage.info
diff --git a/ManagedCode.Storage.Azure/BlobStream.cs b/ManagedCode.Storage.Azure/BlobStream.cs
@@ -37,18 +37,19 @@ public override long Length
     {
         get
         {
-            var realLength = 0L;
-            var metadata = _pageBlob.GetProperties().Value.Metadata;
+            var properties = _pageBlob.GetProperties();
+            var metadata = properties.Value.Metadata;
             if (metadata.TryGetValue(MetadataLengthKey, out var length))
             {
-                if (long.TryParse(length, out realLength))
+                if (long.TryParse(length, out var realLength))
                 {
                     return realLength;
                 }
             }
-
-            SetLengthInternal(realLength);
-            return realLength;
+
+            var contentLenght = properties.Value.ContentLength;
+            SetLengthInternal(contentLenght);
+            return contentLenght;
         }
     }
 

diff --git a/ManagedCode.Storage.Client/Class1.cs b/ManagedCode.Storage.Client/Class1.cs
diff --git a/ManagedCode.Storage.Client/IStorageClient.cs b/ManagedCode.Storage.Client/IStorageClient.cs
@@ -0,0 +1,17 @@
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using ManagedCode.Communication;
+using ManagedCode.Storage.Core.Models;
+
+namespace ManagedCode.Storage.Client;
+
+public interface IStorageClient
+{
+    Task<Result<BlobMetadata>> UploadFile(Stream stream, string apiUrl, string contentName, CancellationToken cancellationToken = default);
+    Task<Result<BlobMetadata>> UploadFile(FileInfo fileInfo, string apiUrl, string contentName, CancellationToken cancellationToken = default);
+    Task<Result<BlobMetadata>> UploadFile(byte[] bytes, string apiUrl, string contentName, CancellationToken cancellationToken = default);
+    Task<Result<BlobMetadata>> UploadFile(string base64, string apiUrl, string contentName, CancellationToken cancellationToken = default);
+    Task<Result> UploadFileInChunks(Stream file, string apiUrl, int chunkSize, CancellationToken cancellationToken = default);
+    Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl, string? path = null, CancellationToken cancellationToken = default);
+}
diff --git a/ManagedCode.Storage.Client/StorageClient.cs b/ManagedCode.Storage.Client/StorageClient.cs
@@ -0,0 +1,162 @@
+using System;
+using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Json;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Net.Http.Json;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using ManagedCode.Communication;
+using ManagedCode.Storage.Core;
+using ManagedCode.Storage.Core.Models;
+
+namespace ManagedCode.Storage.Client;
+
+public class StorageClient : IStorageClient
+{
+    private readonly HttpClient _httpClient;
+
+    public StorageClient(HttpClient httpClient)
+    {
+        _httpClient = httpClient;
+    }
+
+    public async Task<Result<BlobMetadata>> UploadFile(Stream stream, string apiUrl, string contentName, CancellationToken cancellationToken = default)
+    {
+        var streamContent = new StreamContent(stream);
+
+        using (var formData = new MultipartFormDataContent())
+        {
+            formData.Add(streamContent, contentName, contentName);
+
+            var response = await _httpClient.PostAsync(apiUrl, formData, cancellationToken);
+
+            if (response.IsSuccessStatusCode)
+            {
+                var result = await response.Content.ReadFromJsonAsync<Result<BlobMetadata>>(cancellationToken: cancellationToken);
+                return result;
+            }
+
+            string content = await response.Content.ReadAsStringAsync(cancellationToken: cancellationToken);
+
+            return Result<BlobMetadata>.Fail(response.StatusCode, content);
+        }
+    }
+
+    public async Task<Result<BlobMetadata>> UploadFile(FileInfo fileInfo, string apiUrl, string contentName, CancellationToken cancellationToken = default)
+    {
+        using var streamContent = new StreamContent(fileInfo.OpenRead());
+
+        using (var formData = new MultipartFormDataContent())
+        {
+            formData.Add(streamContent, contentName, contentName);
+
+            var response = await _httpClient.PostAsync(apiUrl, formData, cancellationToken);
+
+            if (response.IsSuccessStatusCode)
+            {
+                var result = await response.Content.ReadFromJsonAsync<Result<BlobMetadata>>(cancellationToken: cancellationToken);
+                return result;
+            }
+
+            return Result<BlobMetadata>.Fail(response.StatusCode);
+        }
+    }
+
+    public async Task<Result<BlobMetadata>> UploadFile(byte[] bytes, string apiUrl, string contentName, CancellationToken cancellationToken = default)
+    {
+        using (var stream = new MemoryStream())
+        {
+            stream.Write(bytes, 0, bytes.Length);
+
+            using var streamContent = new StreamContent(stream);
+
+            using (var formData = new MultipartFormDataContent())
+            {
+                formData.Add(streamContent, contentName, contentName);
+
+                var response = await _httpClient.PostAsync(apiUrl, formData, cancellationToken);
+
+                if (response.IsSuccessStatusCode)
+                {
+                    var result = await response.Content.ReadFromJsonAsync<Result<BlobMetadata>>(cancellationToken: cancellationToken);
+                    return result;
+                }
+
+                return Result<BlobMetadata>.Fail(response.StatusCode);
+            }
+        }
+    }   
+
+    public async Task<Result<BlobMetadata>> UploadFile(string base64, string apiUrl, string contentName, CancellationToken cancellationToken = default)
+    {
+        byte[] fileAsBytes = Convert.FromBase64String(base64);
+        using var fileContent = new ByteArrayContent(fileAsBytes);
+
+        using var formData = new MultipartFormDataContent();
+
+        formData.Add(fileContent, contentName, contentName);
+
+        var response = await _httpClient.PostAsync(apiUrl, formData, cancellationToken);
+
+        if (response.IsSuccessStatusCode)
+        {
+            return await response.Content.ReadFromJsonAsync<Result<BlobMetadata>>(cancellationToken: cancellationToken);
+        }
+
+        return Result<BlobMetadata>.Fail(response.StatusCode);
+    }
+
+    public async Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl, string? path = null,  CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            var response = await _httpClient.GetStreamAsync($"{apiUrl}/{fileName}", cancellationToken);
+
+            var localFile = path is null ? await LocalFile.FromStreamAsync(response, fileName) : await LocalFile.FromStreamAsync(response, path, fileName);
+
+            return Result<LocalFile>.Succeed(localFile);
+        }
+        catch (HttpRequestException e)
+        {
+            return Result<LocalFile>.Fail(e.StatusCode ?? HttpStatusCode.InternalServerError);
+        }
+    }
+
+    public async Task<Result> UploadFileInChunks(Stream file, string apiUrl, int chunkSize, CancellationToken cancellationToken)
+    {
+        var buffer = new byte[chunkSize];
+        int bytesRead;
+        int chunkIndex = 0;
+
+        while ((bytesRead = await file.ReadAsync(buffer, 0, buffer.Length, cancellationToken)) > 0)
+        {
+            // Create a MemoryStream for the current chunk.
+            using (var memoryStream = new MemoryStream(buffer, 0, bytesRead))
+            {
+                var content = new StreamContent(memoryStream);
+
+                using (var chunk = new MultipartFormDataContent())
+                {
+                    chunk.Add(content, "file", "file");
+
+                    var byteArrayContent = new ByteArrayContent(await chunk.ReadAsByteArrayAsync(cancellationToken));
+                    // Send the current chunk to the API endpoint.
+                    var response = await _httpClient.PostAsync(apiUrl, byteArrayContent, cancellationToken);
+
+                    if (!response.IsSuccessStatusCode)
+                    {
+                        return Result.Fail();
+                    }
+                }
+            }
+        }
+
+        var mergeResult = await _httpClient.PostAsync(apiUrl + "/complete", JsonContent.Create("file"));
+
+        return await mergeResult.Content.ReadFromJsonAsync<Result>();
+    }
+}
diff --git a/ManagedCode.Storage.Core/Models/LocalFile.cs b/ManagedCode.Storage.Core/Models/LocalFile.cs
@@ -150,6 +150,17 @@ public static async Task<LocalFile> FromStreamAsync(Stream stream)
         await file.FileStream.DisposeAsync();
         return file;
     }
+
+    public static async Task<LocalFile> FromStreamAsync(Stream stream, string path, string fileName)
+    {
+        var pathWithName = Path.Combine(path, $"{fileName}.tmp");
+        var file = new LocalFile(pathWithName);
+
+        await stream.CopyToAsync(file.FileStream);
+        await file.FileStream.DisposeAsync();
+
+        return file;
+    }
 
     public static async Task<LocalFile> FromStreamAsync(Stream stream, string fileName)
     {

diff --git a/ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs b/ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs
@@ -0,0 +1,13 @@
+namespace ManagedCode.Storage.IntegrationTests.Constants;
+
+public static class ApiEndpoints
+{
+    public const string Azure = "azure";
+
+    public static class Base
+    {
+        public const string UploadFile = "{0}/upload";
+        public const string UploadFileChunks = "{0}/upload-chunks";
+        public const string DownloadFile = "{0}/download";
+    }
+}