From 081185fb1af5bdbd73dbbaaf3ff13f9df54293d6 Mon Sep 17 00:00:00 2001 From: Stuart Ferguson Date: Fri, 7 May 2021 18:37:18 +0100 Subject: [PATCH 1/2] Added duplicate file checking --- .../FileRequestHandlerTests.cs | 10 ++--- .../RequestTests.cs | 3 +- .../RequestHandlers/FileRequestHandler.cs | 42 ++++++++++++++++--- .../Requests/UploadFileRequest.cs | 13 +----- .../FileImportLogAggregateTests.cs | 12 ++++++ ...cessor.FileImportLogAggregate.Tests.csproj | 2 +- .../FileImportLogAggregate.cs | 11 ++++- FileProcessor/Controllers/FileController.cs | 5 +-- FileProcessor/Startup.cs | 2 +- 9 files changed, 69 insertions(+), 31 deletions(-) diff --git a/FileProcessor.BusinessLogic.Tests/FileRequestHandlerTests.cs b/FileProcessor.BusinessLogic.Tests/FileRequestHandlerTests.cs index 5a7716b..5e482b3 100644 --- a/FileProcessor.BusinessLogic.Tests/FileRequestHandlerTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileRequestHandlerTests.cs @@ -67,7 +67,7 @@ public async Task FileRequestHandler_UploadFileRequest_RequestIsHandled() fileSystem); UploadFileRequest uploadFileRequest = - new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); + new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); Should.NotThrow(async () => { @@ -109,7 +109,7 @@ public async Task FileRequestHandler_UploadFileRequest_ImportLogAlreadyCreated_R fileSystem); UploadFileRequest uploadFileRequest = - new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); + new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); Should.NotThrow(async () => { @@ -150,7 +150,7 @@ public async Task FileRequestHandler_UploadFileRequest_NoFileProfiles_RequestIsH fileSystem); UploadFileRequest uploadFileRequest = - new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); + new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); Should.Throw(async () => { @@ -191,7 +191,7 @@ public async Task FileRequestHandler_UploadFileRequest_FileNotFound_RequestIsHan fileSystem); UploadFileRequest uploadFileRequest = - new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); + new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); Should.Throw(async () => { @@ -233,7 +233,7 @@ public async Task FileRequestHandler_UploadFileRequest_DestinationDirectoryNotFo fileSystem); UploadFileRequest uploadFileRequest = - new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); + new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId); Should.Throw(async () => { diff --git a/FileProcessor.BusinessLogic.Tests/RequestTests.cs b/FileProcessor.BusinessLogic.Tests/RequestTests.cs index bffca78..c21fe15 100644 --- a/FileProcessor.BusinessLogic.Tests/RequestTests.cs +++ b/FileProcessor.BusinessLogic.Tests/RequestTests.cs @@ -13,11 +13,10 @@ public class RequestTests public void UploadFileRequest_CanBeCreated_IsCreated() { UploadFileRequest uploadFileRequest = - new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePath, TestData.FileProfileId); + new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePath, TestData.FileProfileId); uploadFileRequest.EstateId.ShouldBe(TestData.EstateId); uploadFileRequest.MerchantId.ShouldBe(TestData.MerchantId); - uploadFileRequest.FileId.ShouldBe(TestData.FileId); uploadFileRequest.UserId.ShouldBe(TestData.UserId); uploadFileRequest.FilePath.ShouldBe(TestData.FilePath); uploadFileRequest.FileProfileId.ShouldBe(TestData.FileProfileId); diff --git a/FileProcessor.BusinessLogic/RequestHandlers/FileRequestHandler.cs b/FileProcessor.BusinessLogic/RequestHandlers/FileRequestHandler.cs index 01a76d8..b590c7c 100644 --- a/FileProcessor.BusinessLogic/RequestHandlers/FileRequestHandler.cs +++ b/FileProcessor.BusinessLogic/RequestHandlers/FileRequestHandler.cs @@ -8,6 +8,7 @@ namespace FileProcessor.BusinessLogic.RequestHandlers { using System.IO; using System.IO.Abstractions; + using System.Security.Cryptography; using MediatR; using System.Threading; using EstateManagement.Client; @@ -36,7 +37,7 @@ namespace FileProcessor.BusinessLogic.RequestHandlers /// /// /// - public class FileRequestHandler : IRequestHandler, + public class FileRequestHandler : IRequestHandler, IRequestHandler, IRequestHandler, IRequestHandler @@ -118,7 +119,7 @@ public FileRequestHandler(IFileProcessorManager fileProcessorManager, /// No file profile found with Id {request.FileProfileId} /// File {file.FullName} not found /// Directory {fileProfile.ListeningDirectory} not found - public async Task Handle(UploadFileRequest request, + public async Task Handle(UploadFileRequest request, CancellationToken cancellationToken) { DateTime importLogDateTime = DateTime.Now; @@ -155,17 +156,46 @@ public async Task Handle(UploadFileRequest request, { throw new DirectoryNotFoundException($"Directory {fileProfile.ListeningDirectory} not found"); } + + // Read the file data + String fileContent = null; + //Open file for Read\Write + using (Stream fs = file.Open(FileMode.OpenOrCreate, FileAccess.Read, FileShare.Read)) + { + //Create object of StreamReader by passing FileStream object on which it needs to operates on + using (StreamReader sr = new StreamReader(fs)) + { + //Use ReadToEnd method to read all the content from file + fileContent = await sr.ReadToEndAsync(); + } + } - String fileDestination = $"{fileProfile.ListeningDirectory}\\{request.EstateId:N}-{request.FileId:N}"; - file.MoveTo(fileDestination, overwrite:true);; + Guid fileId = this.CreateGuidFromFileData(fileContent); + + String fileDestination = $"{fileProfile.ListeningDirectory}\\{request.EstateId:N}-{fileId:N}"; + file.MoveTo(fileDestination, overwrite: true); // Update Import log aggregate - fileImportLogAggregate.AddImportedFile(request.FileId, request.MerchantId, request.UserId, request.FileProfileId, originalName, fileDestination); + fileImportLogAggregate.AddImportedFile(fileId, request.MerchantId, request.UserId, request.FileProfileId, originalName, fileDestination); // Save changes await this.FileImportLogAggregateRepository.SaveChanges(fileImportLogAggregate, cancellationToken); - return new Unit(); + return fileId; + } + + private Guid CreateGuidFromFileData(String fileContents) + { + using (SHA256 sha256Hash = SHA256.Create()) + { + //Generate hash from the key + Byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(fileContents)); + + Byte[] j = bytes.Skip(Math.Max(0, bytes.Count() - 16)).ToArray(); //Take last 16 + + //Create our Guid. + return new Guid(j); + } } private Guid CreateGuidFromDateTime(DateTime dateTime) diff --git a/FileProcessor.BusinessLogic/Requests/UploadFileRequest.cs b/FileProcessor.BusinessLogic/Requests/UploadFileRequest.cs index 36cd6e5..40bbb26 100644 --- a/FileProcessor.BusinessLogic/Requests/UploadFileRequest.cs +++ b/FileProcessor.BusinessLogic/Requests/UploadFileRequest.cs @@ -10,7 +10,7 @@ /// /// /// - public record UploadFileRequest : IRequest + public record UploadFileRequest : IRequest { #region Constructors @@ -19,20 +19,17 @@ public record UploadFileRequest : IRequest /// /// The estate identifier. /// The merchant identifier. - /// The file identifier. /// The user identifier. /// The file path. /// The file profile identifier. public UploadFileRequest(Guid estateId, Guid merchantId, - Guid fileId, Guid userId, String filePath, Guid fileProfileId) { this.EstateId = estateId; this.MerchantId = merchantId; - this.FileId = fileId; this.UserId = userId; this.FilePath = filePath; this.FileProfileId = fileProfileId; @@ -50,14 +47,6 @@ public UploadFileRequest(Guid estateId, /// public Guid EstateId { get; init; } - /// - /// Gets or sets the file identifier. - /// - /// - /// The file identifier. - /// - public Guid FileId { get; init; } - /// /// Gets or sets the file path. /// diff --git a/FileProcessor.FileImportLogAggregate.Tests/FileImportLogAggregateTests.cs b/FileProcessor.FileImportLogAggregate.Tests/FileImportLogAggregateTests.cs index a391eb0..16f46c0 100644 --- a/FileProcessor.FileImportLogAggregate.Tests/FileImportLogAggregateTests.cs +++ b/FileProcessor.FileImportLogAggregate.Tests/FileImportLogAggregateTests.cs @@ -79,5 +79,17 @@ public void FileImportLogAggregate_AddImportedFile_ImportLogNotCreated_ErrorThro fileImportLogAggregate.AddImportedFile(TestData.FileId, TestData.MerchantId, TestData.UserId, TestData.FileProfileId, TestData.OriginalFileName, TestData.FilePath); }); } + + [Fact] + public void FileImportLogAggregate_AddImportedFile_DuplicateFileId_ErrorThrown() + { + FileImportLogAggregate fileImportLogAggregate = FileImportLogAggregate.Create(TestData.FileImportLogId); + fileImportLogAggregate.CreateImportLog(TestData.EstateId, TestData.ImportLogDateTime); + fileImportLogAggregate.AddImportedFile(TestData.FileId, TestData.MerchantId, TestData.UserId, TestData.FileProfileId, TestData.OriginalFileName, TestData.FilePath); + Should.Throw(() => + { + fileImportLogAggregate.AddImportedFile(TestData.FileId, TestData.MerchantId, TestData.UserId, TestData.FileProfileId, TestData.OriginalFileName, TestData.FilePath); + }); + } } } diff --git a/FileProcessor.FileImportLogAggregate.Tests/FileProcessor.FileImportLogAggregate.Tests.csproj b/FileProcessor.FileImportLogAggregate.Tests/FileProcessor.FileImportLogAggregate.Tests.csproj index 81608e4..adac6fd 100644 --- a/FileProcessor.FileImportLogAggregate.Tests/FileProcessor.FileImportLogAggregate.Tests.csproj +++ b/FileProcessor.FileImportLogAggregate.Tests/FileProcessor.FileImportLogAggregate.Tests.csproj @@ -2,7 +2,7 @@ net5.0 - + None false diff --git a/FileProcessor.FileImportLogAggregate/FileImportLogAggregate.cs b/FileProcessor.FileImportLogAggregate/FileImportLogAggregate.cs index 9532a8a..14d5170 100644 --- a/FileProcessor.FileImportLogAggregate/FileImportLogAggregate.cs +++ b/FileProcessor.FileImportLogAggregate/FileImportLogAggregate.cs @@ -4,6 +4,7 @@ namespace FileProcessor.FileImportLogAggregate { using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; + using System.Linq; using FileImportLog.DomainEvents; using FIleProcessor.Models; using Microsoft.AspNetCore.Mvc.Rendering; @@ -94,7 +95,15 @@ public void CreateImportLog(Guid estateId, DateTime importLogDateTime) /// The file path. public void AddImportedFile(Guid fileId, Guid merchantId,Guid userId, Guid fileProfileId, String originalFileName, String filePath) { - // TODO: Do we check here for a duplicate (somehow!!) + if (this.IsCreated == false) + { + throw new InvalidOperationException("Import log has not been created"); + } + + if (this.Files.Any(f => f.FileId == fileId)) + { + throw new InvalidOperationException($"Duplicate file {originalFileName} detected File Id [{fileId}]"); + } FileAddedToImportLogEvent fileAddedToImportLogEvent = new FileAddedToImportLogEvent(this.AggregateId, fileId, this.EstateId, merchantId, userId, fileProfileId, originalFileName, filePath); diff --git a/FileProcessor/Controllers/FileController.cs b/FileProcessor/Controllers/FileController.cs index 85f3c37..bba4af9 100644 --- a/FileProcessor/Controllers/FileController.cs +++ b/FileProcessor/Controllers/FileController.cs @@ -40,7 +40,6 @@ public async Task UploadFile([FromForm] UploadFileRequest request var file = formCollection.Files.First(); var temporaryFileLocation = ConfigurationReader.GetValue("AppSettings", "TemporaryFileLocation"); - Guid fileId = Guid.NewGuid(); var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"'); var fullPath = Path.Combine(temporaryFileLocation, fileName); @@ -52,9 +51,9 @@ public async Task UploadFile([FromForm] UploadFileRequest request // Create a command with the file in it BusinessLogic.Requests.UploadFileRequest uploadFileRequest = - new BusinessLogic.Requests.UploadFileRequest(request.EstateId, request.MerchantId, fileId, request.UserId, fullPath, request.FileProfileId); + new BusinessLogic.Requests.UploadFileRequest(request.EstateId, request.MerchantId, request.UserId, fullPath, request.FileProfileId); - await this.Mediator.Send(uploadFileRequest, cancellationToken); + Guid fileId = await this.Mediator.Send(uploadFileRequest, cancellationToken); return this.Accepted(fileId); } diff --git a/FileProcessor/Startup.cs b/FileProcessor/Startup.cs index 7cf09fd..140749d 100644 --- a/FileProcessor/Startup.cs +++ b/FileProcessor/Startup.cs @@ -203,7 +203,7 @@ public void ConfigureServices(IServiceCollection services) return t => context.GetService(t); }); - services.AddSingleton, FileRequestHandler>(); + services.AddSingleton, FileRequestHandler>(); services.AddSingleton, FileRequestHandler>(); services.AddSingleton, FileRequestHandler>(); services.AddSingleton, FileRequestHandler>(); From 4023efc24bc45023ca37c38661f7b562eeb02a20 Mon Sep 17 00:00:00 2001 From: Stuart Ferguson Date: Fri, 7 May 2021 18:39:02 +0100 Subject: [PATCH 2/2] Workflow tweaks --- .github/workflows/createrelease.yml | 1 + .github/workflows/pullrequest.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/createrelease.yml b/.github/workflows/createrelease.yml index 8580a3b..f7c6b75 100644 --- a/.github/workflows/createrelease.yml +++ b/.github/workflows/createrelease.yml @@ -30,6 +30,7 @@ jobs: echo "ASPNETCORE_ENVIRONMENT are > ${ASPNETCORE_ENVIRONMENT}" dotnet test "FileProcessor.BusinessLogic.Tests\FileProcessor.BusinessLogic.Tests.csproj" dotnet test "FileProcessor.FileAggregate.Tests\FileProcessor.FileAggregate.Tests.csproj" + dotnet test "FileProcessor.FileImportLogAggregate.Tests\FileProcessor.FileImportLogAggregate.Tests.csproj" dotnet test "FileProcessor.DomainEvents.Tests\FileProcessor.DomainEvents.Tests.csproj" - name: Build Docker Images diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index 87331a7..019f7ba 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -27,6 +27,7 @@ jobs: echo "ASPNETCORE_ENVIRONMENT are > ${ASPNETCORE_ENVIRONMENT}" dotnet test "FileProcessor.BusinessLogic.Tests\FileProcessor.BusinessLogic.Tests.csproj" dotnet test "FileProcessor.FileAggregate.Tests\FileProcessor.FileAggregate.Tests.csproj" + dotnet test "FileProcessor.FileImportLogAggregate.Tests\FileProcessor.FileImportLogAggregate.Tests.csproj" dotnet test "FileProcessor.DomainEvents.Tests\FileProcessor.DomainEvents.Tests.csproj" - name: Build Docker Image