Mark invalid format lines as rejected

FIleProcessor.Models/FileLine.cs

@@ -42,6 +42,14 @@ public class FileLine
         /// </value>
         public Guid TransactionId { get; set; }
 
+        /// <summary>
+        /// Gets or sets the rejected reason.
+        /// </summary>
+        /// <value>
+        /// The rejected reason.
+        /// </value>
+        public String RejectedReason { get; set; }
+
         #endregion
     }
 }

FIleProcessor.Models/ProcessingResult.cs

@@ -23,6 +23,11 @@ public enum ProcessingResult
         /// <summary>
         /// The ignored
         /// </summary>
-        Ignored
+        Ignored,
+
+        /// <summary>
+        /// The rejected
+        /// </summary>
+        Rejected
     }
 }

FIleProcessor.Models/ProcessingSummary.cs

@@ -25,6 +25,14 @@ public class ProcessingSummary
         /// </value>
         public Int32 IgnoredLines { get; set; }
 
+        /// <summary>
+        /// Gets or sets the rejected lines.
+        /// </summary>
+        /// <value>
+        /// The rejectedt lines.
+        /// </value>
+        public Int32 RejectedLines { get; set; }
+
         /// <summary>
         /// Gets or sets the not processed lines.
         /// </summary>

FileProcessor.BusinessLogic/RequestHandlers/FileRequestHandler.cs

@@ -30,6 +30,7 @@ namespace FileProcessor.BusinessLogic.RequestHandlers
     using Shared.Logger;
     using TransactionProcessor.Client;
     using TransactionProcessor.DataTransferObjects;
+    using Exception = System.Exception;
 
     /// <summary>
     /// 
@@ -370,6 +371,14 @@ public async Task<Unit> Handle(ProcessTransactionForFileLineRequest request,
             // need to now parse the line (based on the file format), this builds the metadata
             Dictionary<String, String> transactionMetadata = this.ParseFileLine(fileLine.LineData, fileProfile.FileFormatHandler);
 
+            if (transactionMetadata == null)
+            {
+                // Line failed to parse so record this
+                fileAggregate.RecordFileLineAsRejected(fileLine.LineNumber, "Invalid Format");
+                await this.FileAggregateRepository.SaveChanges(fileAggregate, cancellationToken);
+                return new Unit();
+            }
+
             // Add the file data to the request metadata
             transactionMetadata.Add("FileId", request.FileId.ToString());
             transactionMetadata.Add("FileLineNumber", fileLine.LineNumber.ToString());
@@ -502,9 +511,17 @@ private Boolean FileLineCanBeIgnored(String domainEventFileLine,
         private Dictionary<String, String> ParseFileLine(String domainEventFileLine,
                                                         String fileProfileFileFormatHandler)
         {
-            IFileFormatHandler fileFormatHandler = this.FileFormatHandlerResolver(fileProfileFileFormatHandler);
+            try
+            {
+                IFileFormatHandler fileFormatHandler = this.FileFormatHandlerResolver(fileProfileFileFormatHandler);
 
-            return fileFormatHandler.ParseFileLine(domainEventFileLine);
+                return fileFormatHandler.ParseFileLine(domainEventFileLine);
+            }
+            catch(InvalidDataException iex)
+            {
+                Logger.LogWarning(iex.Message);
+                return null;
+            }
         }
 
         /// <summary>

FileProcessor.DataTransferObjects/Responses/FileLine.cs

@@ -43,6 +43,15 @@ public class FileLine
         [JsonProperty("transaction_id")]
         public Guid TransactionId { get; set; }
 
+        /// <summary>
+        /// Gets or sets the rejection reason.
+        /// </summary>
+        /// <value>
+        /// The rejection reason.
+        /// </value>
+        [JsonProperty("rejection_reason")]
+        public String RejectionReason { get; set; }
+
         #endregion
     }
 }

FileProcessor.DataTransferObjects/Responses/FileLineProcessingResult.cs

@@ -25,6 +25,11 @@ public enum FileLineProcessingResult
         /// <summary>
         /// The ignored
         /// </summary>
-        Ignored
+        Ignored,
+
+        /// <summary>
+        /// The rejected
+        /// </summary>
+        Rejected
     }
 }

FileProcessor.DataTransferObjects/Responses/FileProcessingSummary.cs

@@ -34,6 +34,15 @@ public class FileProcessingSummary
         [JsonProperty("not_processed_lines")]
         public Int32 NotProcessedLines { get; set; }
 
+        /// <summary>
+        /// Gets or sets the rejected lines.
+        /// </summary>
+        /// <value>
+        /// The rejected lines.
+        /// </value>
+        [JsonProperty("rejected_lines")]
+        public Int32 RejectedLines { get; set; }
+
         /// <summary>
         /// Gets or sets the successfully processed lines.
         /// </summary>

FileProcessor.DomainEvents.Tests/FileAggregateDomainEventTests.cs

@@ -80,6 +80,18 @@ public void FileLineProcessingIgnoredEvent_CanBeCreated_IsCreated()
             fileLineProcessingIgnoredEvent.EstateId.ShouldBe(TestData.EstateId);
         }
 
+        [Fact]
+        public void FileLineProcessingRejectedEvent_CanBeCreated_IsCreated()
+        {
+            FileLineProcessingRejectedEvent fileLineProcessingRejectedEvent =
+                new FileLineProcessingRejectedEvent(TestData.FileId, TestData.EstateId, TestData.LineNumber, TestData.RejectionReason);
+
+            fileLineProcessingRejectedEvent.FileId.ShouldBe(TestData.FileId);
+            fileLineProcessingRejectedEvent.LineNumber.ShouldBe(TestData.LineNumber);
+            fileLineProcessingRejectedEvent.EstateId.ShouldBe(TestData.EstateId);
+            fileLineProcessingRejectedEvent.Reason.ShouldBe(TestData.RejectionReason);
+        }
+
         [Fact]
         public void FileProcessingCompletedEvent_CanBeCreated_IsCreated()
         {

FileProcessor.File.DomainEvents/FileLineProcessingRejectedEvent.cs

@@ -0,0 +1,57 @@
+namespace FileProcessor.File.DomainEvents
+{
+    using System;
+    using Shared.DomainDrivenDesign.EventSourcing;
+
+    public record FileLineProcessingRejectedEvent : DomainEventRecord.DomainEvent
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FileLineProcessingFailedEvent" /> class.
+        /// </summary>
+        /// <param name="aggregateId">The aggregate identifier.</param>
+        /// <param name="estateId">The estate identifier.</param>
+        /// <param name="lineNumber">The line number.</param>
+        /// <param name="reason">The reason.</param>
+        public FileLineProcessingRejectedEvent(Guid aggregateId,
+                                               Guid estateId, Int32 lineNumber,
+                                               String reason) : base(aggregateId, Guid.NewGuid())
+        {
+            this.FileId = aggregateId;
+            this.EstateId = estateId;
+            this.LineNumber = lineNumber;
+            this.Reason = reason;
+        }
+
+        /// <summary>
+        /// Gets or sets the estate identifier.
+        /// </summary>
+        /// <value>
+        /// The estate identifier.
+        /// </value>
+        public Guid EstateId { get; init; }
+
+        /// <summary>
+        /// Gets or sets the file identifier.
+        /// </summary>
+        /// <value>
+        /// The file identifier.
+        /// </value>
+        public Guid FileId { get; init; }
+
+        /// <summary>
+        /// Gets or sets the line number.
+        /// </summary>
+        /// <value>
+        /// The line number.
+        /// </value>
+        public Int32 LineNumber { get; init; }
+
+        /// <summary>
+        /// Gets or sets the reason.
+        /// </summary>
+        /// <value>
+        /// The reason.
+        /// </value>
+        public String Reason { get; init; }
+    }
+}

FileProcessor.FileAggregate.Tests/FileAggregateTests.cs

@@ -287,5 +287,71 @@ public void FileAggregate_RecordFileLineAsIgnored_LineNotFound_ErrorThrown()
                 fileAggregate.RecordFileLineAsIgnored(TestData.NotFoundLineNumber);
             });
         }
+
+
+
+        [Fact]
+        public void FileAggregate_RecordFileLineAsRejected_FileLineUpdatedAsRejected()
+        {
+            FileAggregate fileAggregate = FileAggregate.Create(TestData.FileId);
+            fileAggregate.CreateFile(TestData.FileImportLogId, TestData.EstateId, TestData.MerchantId, TestData.UserId,
+                                     TestData.FileProfileId, TestData.FileLocation, TestData.FileUploadedDateTime);
+            fileAggregate.AddFileLine(TestData.FileLine);
+            fileAggregate.RecordFileLineAsRejected(TestData.LineNumber,TestData.RejectionReason);
+
+            FileDetails fileDetails = fileAggregate.GetFile();
+            fileDetails.FileLines.ShouldNotBeNull();
+            fileDetails.FileLines.ShouldNotBeEmpty();
+            fileDetails.FileLines.ShouldHaveSingleItem();
+            fileDetails.FileLines.Single().LineNumber.ShouldBe(1);
+            fileDetails.FileLines.Single().LineData.ShouldBe(TestData.FileLine);
+            fileDetails.FileLines.Single().ProcessingResult.ShouldBe(ProcessingResult.Rejected);
+            fileDetails.FileLines.Single().RejectedReason.ShouldBe(TestData.RejectionReason);
+            fileDetails.ProcessingCompleted.ShouldBeTrue();
+            fileDetails.ProcessingSummary.ShouldNotBeNull();
+            fileDetails.ProcessingSummary.TotalLines.ShouldBe(1);
+            fileDetails.ProcessingSummary.NotProcessedLines.ShouldBe(0);
+            fileDetails.ProcessingSummary.FailedLines.ShouldBe(0);
+            fileDetails.ProcessingSummary.SuccessfullyProcessedLines.ShouldBe(0);
+            fileDetails.ProcessingSummary.IgnoredLines.ShouldBe(0);
+            fileDetails.ProcessingSummary.RejectedLines.ShouldBe(1);
+        }
+
+        [Fact]
+        public void FileAggregate_RecordFileLineAsRejected_FileNotCreated_ErrorThrown()
+        {
+            FileAggregate fileAggregate = FileAggregate.Create(TestData.FileId);
+
+            Should.Throw<InvalidOperationException>(() =>
+            {
+                fileAggregate.RecordFileLineAsRejected(TestData.LineNumber,TestData.RejectionReason);
+            });
+        }
+
+        [Fact]
+        public void FileAggregate_RecordFileLineAsRejected_FileHasNoLine_ErrorThrown()
+        {
+            FileAggregate fileAggregate = FileAggregate.Create(TestData.FileId);
+            fileAggregate.CreateFile(TestData.FileImportLogId, TestData.EstateId, TestData.MerchantId, TestData.UserId,
+                                     TestData.FileProfileId, TestData.FileLocation, TestData.FileUploadedDateTime);
+
+            Should.Throw<InvalidOperationException>(() =>
+            {
+                fileAggregate.RecordFileLineAsRejected(TestData.LineNumber,TestData.RejectionReason);
+            });
+        }
+
+        [Fact]
+        public void FileAggregate_RecordFileLineAsRejected_LineNotFound_ErrorThrown()
+        {
+            FileAggregate fileAggregate = FileAggregate.Create(TestData.FileId);
+            fileAggregate.CreateFile(TestData.FileImportLogId, TestData.EstateId, TestData.MerchantId, TestData.UserId,
+                                     TestData.FileProfileId, TestData.FileLocation, TestData.FileUploadedDateTime);
+            fileAggregate.AddFileLine(TestData.FileLine);
+            Should.Throw<NotFoundException>(() =>
+            {
+                fileAggregate.RecordFileLineAsRejected(TestData.NotFoundLineNumber, TestData.RejectionReason);
+            });
+        }
     }
 }

FileProcessor.FileAggregate/FileAggregate.cs

@@ -162,7 +162,8 @@ public FileDetails GetFile()
                                                FailedLines = this.FileLines.Count(x => x.ProcessingResult == ProcessingResult.Failed),
                                                IgnoredLines = this.FileLines.Count(x => x.ProcessingResult == ProcessingResult.Ignored),
                                                NotProcessedLines = this.FileLines.Count(x => x.ProcessingResult == ProcessingResult.NotProcessed),
-                                               SuccessfullyProcessedLines = this.FileLines.Count(x => x.ProcessingResult == ProcessingResult.Successful)
+                                               SuccessfullyProcessedLines = this.FileLines.Count(x => x.ProcessingResult == ProcessingResult.Successful),
+                                               RejectedLines = this.FileLines.Count(x => x.ProcessingResult == ProcessingResult.Rejected)
                        }
                    };
         }
@@ -191,6 +192,14 @@ private void PlayEvent(FileLineProcessingIgnoredEvent domainEvent)
             fileLine.ProcessingResult = ProcessingResult.Ignored;
         }
 
+        private void PlayEvent(FileLineProcessingRejectedEvent domainEvent)
+        {
+            // find the line 
+            FileLine fileLine = this.FileLines.Single(f => f.LineNumber == domainEvent.LineNumber);
+            fileLine.ProcessingResult = ProcessingResult.Rejected;
+            fileLine.RejectedReason = domainEvent.Reason;
+        }
+
         /// <summary>
         /// Plays the event.
         /// </summary>
@@ -291,6 +300,33 @@ public void RecordFileLineAsIgnored(Int32 lineNumber)
             this.CompletedChecks();
         }
 
+        /// <summary>
+        /// Records the file line as rejected.
+        /// </summary>
+        /// <param name="lineNumber">The line number.</param>
+        /// <param name="reason">The reason.</param>
+        /// <exception cref="InvalidOperationException">File has no lines to mark as rejected</exception>
+        /// <exception cref="NotFoundException">File line with number {lineNumber} not found to mark as rejected</exception>
+        public void RecordFileLineAsRejected(Int32 lineNumber, String reason)
+        {
+            if (this.FileLines.Any() == false)
+            {
+                throw new InvalidOperationException("File has no lines to mark as rejected");
+            }
+
+            if (this.FileLines.SingleOrDefault(l => l.LineNumber == lineNumber) == null)
+            {
+                throw new NotFoundException($"File line with number {lineNumber} not found to mark as rejected");
+            }
+
+            FileLineProcessingRejectedEvent fileLineProcessingRejectedEvent =
+                new FileLineProcessingRejectedEvent(this.AggregateId, this.EstateId, lineNumber, reason);
+
+            this.ApplyAndAppend(fileLineProcessingRejectedEvent);
+
+            this.CompletedChecks();
+        }
+
         /// <summary>
         /// Records the file line as successful.
         /// </summary>

FileProcessor.Testing/TestData.cs

@@ -445,6 +445,8 @@ public static List<ContractResponse> GetMerchantContractsResponseNoNullValueProd
         public static DateTime ImportLogStartDate = new DateTime(2021,7,1);
         public static DateTime ImportLogEndDate = new DateTime(2021,7,2);
 
+        public static String RejectionReason = "Invalid Line Format";
+
         public static FileDetails FileDetailsModel => new FileDetails
                                                       {
                                                           EstateId = TestData.EstateId,
@@ -458,10 +460,11 @@ public static List<ContractResponse> GetMerchantContractsResponseNoNullValueProd
                                                           ProcessingSummary = new ProcessingSummary
                                                                               {
                                                                                   FailedLines = 1,
-                                                                                  TotalLines = 8,
+                                                                                  TotalLines = 10,
                                                                                   IgnoredLines = 2,
                                                                                   NotProcessedLines = 1,
-                                                                                  SuccessfullyProcessedLines = 4
+                                                                                  SuccessfullyProcessedLines = 4,
+                                                                                  RejectedLines = 1
                                                                               },
                                                           FileLines = new List<FIleProcessor.Models.FileLine>
                                                                       {
@@ -516,15 +519,22 @@ public static List<ContractResponse> GetMerchantContractsResponseNoNullValueProd
                                                                           },
                                                                           new FIleProcessor.Models.FileLine
                                                                           {
-                                                                              ProcessingResult = (ProcessingResult)99, // Invalid status
+                                                                              ProcessingResult = ProcessingResult.Rejected,
                                                                               LineNumber = 8,
+                                                                              LineData = TestData.GetSafaricomDetailLine("7","700"),
+                                                                              TransactionId = Guid.Empty
+                                                                          },
+                                                                          new FIleProcessor.Models.FileLine
+                                                                          {
+                                                                              ProcessingResult = (ProcessingResult)99, // Invalid status
+                                                                              LineNumber = 9,
                                                                               LineData = TestData.GetSafaricomDetailLine("8","800"),
                                                                               TransactionId = Guid.Empty
                                                                           },
                                                                           new FIleProcessor.Models.FileLine
                                                                           {
                                                                               ProcessingResult = ProcessingResult.Ignored,
-                                                                              LineNumber = 9,
+                                                                              LineNumber = 10,
                                                                               LineData = "T",
                                                                               TransactionId = Guid.Empty
                                                                           }