Prevent Duplicate File Uploads

.github/workflows/createrelease.yml

@@ -30,6 +30,7 @@ jobs:
         echo "ASPNETCORE_ENVIRONMENT are > ${ASPNETCORE_ENVIRONMENT}"
         dotnet test "FileProcessor.BusinessLogic.Tests\FileProcessor.BusinessLogic.Tests.csproj"
         dotnet test "FileProcessor.FileAggregate.Tests\FileProcessor.FileAggregate.Tests.csproj"
+        dotnet test "FileProcessor.FileImportLogAggregate.Tests\FileProcessor.FileImportLogAggregate.Tests.csproj"
         dotnet test "FileProcessor.DomainEvents.Tests\FileProcessor.DomainEvents.Tests.csproj"      
 
     - name: Build Docker Images 

.github/workflows/pullrequest.yml

@@ -27,6 +27,7 @@ jobs:
         echo "ASPNETCORE_ENVIRONMENT are > ${ASPNETCORE_ENVIRONMENT}"
         dotnet test "FileProcessor.BusinessLogic.Tests\FileProcessor.BusinessLogic.Tests.csproj"
         dotnet test "FileProcessor.FileAggregate.Tests\FileProcessor.FileAggregate.Tests.csproj"
+        dotnet test "FileProcessor.FileImportLogAggregate.Tests\FileProcessor.FileImportLogAggregate.Tests.csproj"
         dotnet test "FileProcessor.DomainEvents.Tests\FileProcessor.DomainEvents.Tests.csproj"
 
     - name: Build Docker Image

FileProcessor.BusinessLogic.Tests/FileRequestHandlerTests.cs

@@ -67,7 +67,7 @@ public async Task FileRequestHandler_UploadFileRequest_RequestIsHandled()
                                                                            fileSystem);
 
             UploadFileRequest uploadFileRequest =
-                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
+                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
 
             Should.NotThrow(async () =>
                             {
@@ -109,7 +109,7 @@ public async Task FileRequestHandler_UploadFileRequest_ImportLogAlreadyCreated_R
                                                                            fileSystem);
 
             UploadFileRequest uploadFileRequest =
-                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
+                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
 
             Should.NotThrow(async () =>
             {
@@ -150,7 +150,7 @@ public async Task FileRequestHandler_UploadFileRequest_NoFileProfiles_RequestIsH
                                                                            fileSystem);
 
             UploadFileRequest uploadFileRequest =
-                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
+                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
 
             Should.Throw<NotFoundException>(async () =>
             {
@@ -191,7 +191,7 @@ public async Task FileRequestHandler_UploadFileRequest_FileNotFound_RequestIsHan
                                                                            fileSystem);
 
             UploadFileRequest uploadFileRequest =
-                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
+                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
 
             Should.Throw<FileNotFoundException>(async () =>
                                                 {
@@ -233,7 +233,7 @@ public async Task FileRequestHandler_UploadFileRequest_DestinationDirectoryNotFo
                                                                            fileSystem);
 
             UploadFileRequest uploadFileRequest =
-                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
+                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePathWithName, TestData.FileProfileId);
 
             Should.Throw<DirectoryNotFoundException>(async () =>
             {

FileProcessor.BusinessLogic.Tests/RequestTests.cs

@@ -13,11 +13,10 @@ public class RequestTests
         public void UploadFileRequest_CanBeCreated_IsCreated()
         {
             UploadFileRequest uploadFileRequest =
-                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.FileId, TestData.UserId, TestData.FilePath, TestData.FileProfileId);
+                new UploadFileRequest(TestData.EstateId, TestData.MerchantId, TestData.UserId, TestData.FilePath, TestData.FileProfileId);
 
             uploadFileRequest.EstateId.ShouldBe(TestData.EstateId);
             uploadFileRequest.MerchantId.ShouldBe(TestData.MerchantId);
-            uploadFileRequest.FileId.ShouldBe(TestData.FileId);
             uploadFileRequest.UserId.ShouldBe(TestData.UserId);
             uploadFileRequest.FilePath.ShouldBe(TestData.FilePath);
             uploadFileRequest.FileProfileId.ShouldBe(TestData.FileProfileId);

FileProcessor.BusinessLogic/RequestHandlers/FileRequestHandler.cs

@@ -8,6 +8,7 @@ namespace FileProcessor.BusinessLogic.RequestHandlers
 {
     using System.IO;
     using System.IO.Abstractions;
+    using System.Security.Cryptography;
     using MediatR;
     using System.Threading;
     using EstateManagement.Client;
@@ -36,7 +37,7 @@ namespace FileProcessor.BusinessLogic.RequestHandlers
     /// <seealso cref="UploadFileRequest" />
     /// <seealso cref="SafaricomTopupRequest" />
     /// <seealso cref="ProcessTransactionForFileLineRequest" />
-    public class FileRequestHandler : IRequestHandler<UploadFileRequest>,
+    public class FileRequestHandler : IRequestHandler<UploadFileRequest,Guid>,
                                       IRequestHandler<ProcessUploadedFileRequest>,
                                       IRequestHandler<SafaricomTopupRequest>,
                                       IRequestHandler<ProcessTransactionForFileLineRequest>
@@ -118,7 +119,7 @@ public FileRequestHandler(IFileProcessorManager fileProcessorManager,
         /// <exception cref="NotFoundException">No file profile found with Id {request.FileProfileId}</exception>
         /// <exception cref="System.IO.FileNotFoundException">File {file.FullName} not found</exception>
         /// <exception cref="System.IO.DirectoryNotFoundException">Directory {fileProfile.ListeningDirectory} not found</exception>
-        public async Task<Unit> Handle(UploadFileRequest request,
+        public async Task<Guid> Handle(UploadFileRequest request,
                                        CancellationToken cancellationToken)
         {
             DateTime importLogDateTime = DateTime.Now;
@@ -155,17 +156,46 @@ public async Task<Unit> Handle(UploadFileRequest request,
             {
                 throw new DirectoryNotFoundException($"Directory {fileProfile.ListeningDirectory} not found");
             }
+
+            // Read the file data
+            String fileContent = null;
+            //Open file for Read\Write
+            using (Stream fs = file.Open(FileMode.OpenOrCreate, FileAccess.Read, FileShare.Read))
+            {
+                //Create object of StreamReader by passing FileStream object on which it needs to operates on
+                using (StreamReader sr = new StreamReader(fs))
+                {
+                    //Use ReadToEnd method to read all the content from file
+                    fileContent = await sr.ReadToEndAsync();
+                }
+            }
 
-            String fileDestination = $"{fileProfile.ListeningDirectory}\\{request.EstateId:N}-{request.FileId:N}";
-            file.MoveTo(fileDestination, overwrite:true);;
+            Guid fileId = this.CreateGuidFromFileData(fileContent);
+
+            String fileDestination = $"{fileProfile.ListeningDirectory}\\{request.EstateId:N}-{fileId:N}";
+            file.MoveTo(fileDestination, overwrite: true);
 
             // Update Import log aggregate
-            fileImportLogAggregate.AddImportedFile(request.FileId, request.MerchantId, request.UserId, request.FileProfileId, originalName, fileDestination);
+            fileImportLogAggregate.AddImportedFile(fileId, request.MerchantId, request.UserId, request.FileProfileId, originalName, fileDestination);
 
             // Save changes
             await this.FileImportLogAggregateRepository.SaveChanges(fileImportLogAggregate, cancellationToken);
 
-            return new Unit();
+            return fileId;
+        }
+
+        private Guid CreateGuidFromFileData(String fileContents)
+        {
+            using (SHA256 sha256Hash = SHA256.Create())
+            {
+                //Generate hash from the key
+                Byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(fileContents));
+
+                Byte[] j = bytes.Skip(Math.Max(0, bytes.Count() - 16)).ToArray(); //Take last 16
+
+                //Create our Guid.
+                return new Guid(j);
+            }
         }
 
         private Guid CreateGuidFromDateTime(DateTime dateTime)

FileProcessor.BusinessLogic/Requests/UploadFileRequest.cs

@@ -10,7 +10,7 @@
     /// <seealso cref="MediatR.IRequest{MediatR.Unit}" />
     /// <seealso cref="MediatR.IBaseRequest" />
     /// <seealso cref="System.IEquatable{FileProcessor.BusinessLogic.Requests.UploadFileRequest}" />
-    public record UploadFileRequest : IRequest
+    public record UploadFileRequest : IRequest<Guid>
     {
         #region Constructors
 
@@ -19,20 +19,17 @@ public record UploadFileRequest : IRequest
         /// </summary>
         /// <param name="estateId">The estate identifier.</param>
         /// <param name="merchantId">The merchant identifier.</param>
-        /// <param name="fileId">The file identifier.</param>
         /// <param name="userId">The user identifier.</param>
         /// <param name="filePath">The file path.</param>
         /// <param name="fileProfileId">The file profile identifier.</param>
         public UploadFileRequest(Guid estateId,
                                  Guid merchantId,
-                                 Guid fileId,
                                  Guid userId,
                                  String filePath,
                                  Guid fileProfileId)
         {
             this.EstateId = estateId;
             this.MerchantId = merchantId;
-            this.FileId = fileId;
             this.UserId = userId;
             this.FilePath = filePath;
             this.FileProfileId = fileProfileId;
@@ -50,14 +47,6 @@ public UploadFileRequest(Guid estateId,
         /// </value>
         public Guid EstateId { get; init; }
 
-        /// <summary>
-        /// Gets or sets the file identifier.
-        /// </summary>
-        /// <value>
-        /// The file identifier.
-        /// </value>
-        public Guid FileId { get; init; }
-
         /// <summary>
         /// Gets or sets the file path.
         /// </summary>

FileProcessor.FileImportLogAggregate.Tests/FileImportLogAggregateTests.cs

@@ -79,5 +79,17 @@ public void FileImportLogAggregate_AddImportedFile_ImportLogNotCreated_ErrorThro
                                                         fileImportLogAggregate.AddImportedFile(TestData.FileId, TestData.MerchantId, TestData.UserId, TestData.FileProfileId, TestData.OriginalFileName, TestData.FilePath);
                                                     });
         }
+
+        [Fact]
+        public void FileImportLogAggregate_AddImportedFile_DuplicateFileId_ErrorThrown()
+        {
+            FileImportLogAggregate fileImportLogAggregate = FileImportLogAggregate.Create(TestData.FileImportLogId);
+            fileImportLogAggregate.CreateImportLog(TestData.EstateId, TestData.ImportLogDateTime);
+            fileImportLogAggregate.AddImportedFile(TestData.FileId, TestData.MerchantId, TestData.UserId, TestData.FileProfileId, TestData.OriginalFileName, TestData.FilePath);
+            Should.Throw<InvalidOperationException>(() =>
+                                                    {
+                                                        fileImportLogAggregate.AddImportedFile(TestData.FileId, TestData.MerchantId, TestData.UserId, TestData.FileProfileId, TestData.OriginalFileName, TestData.FilePath);
+                                                    });
+        }
     }
 }

FileProcessor.FileImportLogAggregate.Tests/FileProcessor.FileImportLogAggregate.Tests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net5.0</TargetFramework>
-
+	  <DebugType>None</DebugType>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 

FileProcessor.FileImportLogAggregate/FileImportLogAggregate.cs

@@ -4,6 +4,7 @@ namespace FileProcessor.FileImportLogAggregate
 {
     using System.Collections.Generic;
     using System.Diagnostics.CodeAnalysis;
+    using System.Linq;
     using FileImportLog.DomainEvents;
     using FIleProcessor.Models;
     using Microsoft.AspNetCore.Mvc.Rendering;
@@ -94,7 +95,15 @@ public void CreateImportLog(Guid estateId, DateTime importLogDateTime)
         /// <param name="filePath">The file path.</param>
         public void AddImportedFile(Guid fileId, Guid merchantId,Guid userId, Guid fileProfileId, String originalFileName, String filePath)
         {
-            // TODO: Do we check here for a duplicate (somehow!!)
+            if (this.IsCreated == false)
+            {
+                throw new InvalidOperationException("Import log has not been created");
+            }
+
+            if (this.Files.Any(f => f.FileId == fileId))
+            {
+                throw new InvalidOperationException($"Duplicate file {originalFileName} detected File Id [{fileId}]");
+            }
 
             FileAddedToImportLogEvent fileAddedToImportLogEvent =
                 new FileAddedToImportLogEvent(this.AggregateId, fileId, this.EstateId, merchantId, userId, fileProfileId, originalFileName, filePath);

FileProcessor/Controllers/FileController.cs

@@ -40,7 +40,6 @@ public async Task<IActionResult> UploadFile([FromForm] UploadFileRequest request
             var file = formCollection.Files.First();
 
             var temporaryFileLocation = ConfigurationReader.GetValue("AppSettings", "TemporaryFileLocation");
-            Guid fileId = Guid.NewGuid();
             var fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"');
 
             var fullPath = Path.Combine(temporaryFileLocation, fileName);
@@ -52,9 +51,9 @@ public async Task<IActionResult> UploadFile([FromForm] UploadFileRequest request
 
             // Create a command with the file in it
             BusinessLogic.Requests.UploadFileRequest uploadFileRequest =
-                new BusinessLogic.Requests.UploadFileRequest(request.EstateId, request.MerchantId, fileId, request.UserId, fullPath, request.FileProfileId);
+                new BusinessLogic.Requests.UploadFileRequest(request.EstateId, request.MerchantId, request.UserId, fullPath, request.FileProfileId);
 
-            await this.Mediator.Send(uploadFileRequest, cancellationToken);
+            Guid fileId = await this.Mediator.Send(uploadFileRequest, cancellationToken);
 
             return this.Accepted(fileId);
         }

FileProcessor/Startup.cs

@@ -203,7 +203,7 @@ public void ConfigureServices(IServiceCollection services)
                                                       return t => context.GetService(t);
                                                   });
 
-            services.AddSingleton<IRequestHandler<UploadFileRequest, Unit>, FileRequestHandler>();
+            services.AddSingleton<IRequestHandler<UploadFileRequest, Guid>, FileRequestHandler>();
             services.AddSingleton<IRequestHandler<ProcessUploadedFileRequest, Unit>, FileRequestHandler>();
             services.AddSingleton<IRequestHandler<SafaricomTopupRequest, Unit>, FileRequestHandler>();
             services.AddSingleton<IRequestHandler<ProcessTransactionForFileLineRequest, Unit>, FileRequestHandler>();