Add retry policy and tests for event handling

TransactionProcessor.BusinessLogic.Tests/DomainEventHandlers/MerchantSettlementDomainEventHandlerTests.cs

@@ -0,0 +1,59 @@
+using Microsoft.AspNetCore.Mvc.ModelBinding.Binders;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using EventStore.Client;
+using Shouldly;
+using TransactionProcessor.BusinessLogic.EventHandling;
+using TransactionProcessor.Testing;
+using Xunit;
+using static TransactionProcessor.DomainEvents.SettlementDomainEvents;
+using MediatR;
+using Moq;
+using Shared.Logger;
+using SimpleResults;
+using TransactionProcessor.BusinessLogic.Requests;
+using Grpc.Core;
+
+namespace TransactionProcessor.BusinessLogic.Tests.DomainEventHandlers
+{
+    public class MerchantSettlementDomainEventHandlerTests
+    {
+        private readonly MerchantSettlementDomainEventHandler EventHandler;
+        private readonly Mock<IMediator> Mediator;
+
+        public MerchantSettlementDomainEventHandlerTests()
+        {
+            this.Mediator = new Mock<IMediator>();
+            this.EventHandler = new MerchantSettlementDomainEventHandler(this.Mediator.Object);
+            Logger.Initialise(new NullLogger());
+        }
+
+        [Fact]
+        public async Task MerchantSettlementDomainEventHandler_Handle_MerchantFeeSettledEvent_EventIsHandled()
+        {
+            MerchantFeeSettledEvent domainEvent = TestData.DomainEvents.MerchantFeeSettledEvent;
+            this.Mediator.Setup(m => m.Send(It.IsAny<MerchantStatementCommands.AddSettledFeeToMerchantStatementCommand>(), It.IsAny<CancellationToken>()))
+                .ReturnsAsync(Result.Success());
+
+            Result result = await this.EventHandler.Handle(domainEvent, CancellationToken.None);
+            result.IsSuccess.ShouldBeTrue();
+        }
+
+        [Fact]
+        public async Task MerchantSettlementDomainEventHandler_Handle_MerchantFeeSettledEvent_Retry_EventIsHandled()
+        {
+            MerchantFeeSettledEvent domainEvent = TestData.DomainEvents.MerchantFeeSettledEvent;
+            this.Mediator.SetupSequence(m => m.Send(It.IsAny<MerchantStatementCommands.AddSettledFeeToMerchantStatementCommand>(), It.IsAny<CancellationToken>()))
+                .ThrowsAsync(new WrongExpectedVersionException("Stream1", StreamRevision.None, StreamRevision.None))
+                .ThrowsAsync(new RpcException(new Status(StatusCode.DeadlineExceeded, "Deadline Exceeded")))
+                .ReturnsAsync(Result.Success());                       
+
+            Result result = await this.EventHandler.Handle(domainEvent, CancellationToken.None);
+            result.IsSuccess.ShouldBeTrue();
+        }
+    }
+}

TransactionProcessor.BusinessLogic/Common/PolicyFactory.cs

@@ -0,0 +1,54 @@
+using System;
+using EventStore.Client;
+using Grpc.Core;
+using Polly;
+using Polly.Fallback;
+using Polly.Retry;
+using Polly.Wrap;
+using Shared.Logger;
+using SimpleResults;
+
+namespace TransactionProcessor.BusinessLogic.Common;
+
+public static class PolicyFactory{
+    public static IAsyncPolicy<Result> CreatePolicy(Int32 retryCount=5, TimeSpan? retryDelay = null, String policyTag="", Boolean withFallBack=false) {
+
+        TimeSpan retryDelayValue = retryDelay.GetValueOrDefault(TimeSpan.FromSeconds(1));
+
+        AsyncRetryPolicy<Result> retryPolicy = CreateRetryPolicy(retryCount, retryDelayValue, policyTag);
+
+        return withFallBack switch {
+            true => CreateFallbackPolicy(policyTag, retryPolicy),
+            _ => retryPolicy
+        };
+    }
+
+    private static AsyncRetryPolicy<Result> CreateRetryPolicy(int retryCount, TimeSpan retryDelay, String policyTag)
+    {
+        //
+        return Policy<Result>
+            .Handle<WrongExpectedVersionException>()
+            .Or<RpcException>(ex => ex.StatusCode == StatusCode.DeadlineExceeded)
+            .WaitAndRetryAsync(retryCount,
+                _ => retryDelay, // Fixed delay
+                (exception, timeSpan, retryCount, context) =>
+                {
+                    Logger.LogWarning($"{policyTag} - Retry {retryCount} due to {exception.GetType().Name}. Waiting {timeSpan} before retrying...");
+                });
+    }
+
+    private static AsyncPolicyWrap<Result> CreateFallbackPolicy(String policyTag, AsyncRetryPolicy<Result> retryPolicy)
+    {
+        AsyncFallbackPolicy<Result> fallbackPolicy = Policy<Result>
+            .Handle<WrongExpectedVersionException>()
+            .Or<TimeoutException>()
+            .FallbackAsync(async (cancellationToken) =>
+            {
+                Logger.LogWarning($"{policyTag}  -All retries failed. Executing fallback action...");
+                // Log failure, notify monitoring system, or enqueue for later processing
+                return Result.Failure("Fallback action executed");
+            });
+
+        return fallbackPolicy.WrapAsync(retryPolicy);
+    }
+}

TransactionProcessor.BusinessLogic/EventHandling/MerchantSettlementDomainEventHandler.cs

@@ -1,27 +1,26 @@
-using System.Threading;
-using System.Threading.Tasks;
-using MediatR;
+using MediatR;
+using Polly;
 using Shared.DomainDrivenDesign.EventSourcing;
 using Shared.EventStore.EventHandling;
 using SimpleResults;
+using System.Threading;
+using System.Threading.Tasks;
+using TransactionProcessor.BusinessLogic.Common;
 using TransactionProcessor.BusinessLogic.Requests;
 using TransactionProcessor.DomainEvents;
 
 namespace TransactionProcessor.BusinessLogic.EventHandling;
 
-public class MerchantSettlementDomainEventHandler : IDomainEventHandler
-{
+public class MerchantSettlementDomainEventHandler : IDomainEventHandler {
     private readonly IMediator Mediator;
 
     public MerchantSettlementDomainEventHandler(IMediator mediator) {
         this.Mediator = mediator;
     }
 
-    public async Task<Result>  Handle(IDomainEvent domainEvent,
-                             CancellationToken cancellationToken)
-    {
-        Task<Result> t = domainEvent switch
-        {
+    public async Task<Result> Handle(IDomainEvent domainEvent,
+                                     CancellationToken cancellationToken) {
+        Task<Result> t = domainEvent switch {
             SettlementDomainEvents.MerchantFeeSettledEvent mfse => this.HandleSpecificDomainEvent(mfse, cancellationToken),
             _ => null
         };
@@ -32,15 +31,13 @@ public async Task<Result>  Handle(IDomainEvent domainEvent,
     }
 
     private async Task<Result> HandleSpecificDomainEvent(SettlementDomainEvents.MerchantFeeSettledEvent domainEvent,
-                                                         CancellationToken cancellationToken)
-    {
-        MerchantStatementCommands.AddSettledFeeToMerchantStatementCommand  command = new(domainEvent.EstateId,
-            domainEvent.MerchantId,
-            domainEvent.FeeCalculatedDateTime,
-            domainEvent.CalculatedValue,
-            domainEvent.TransactionId,
-            domainEvent.FeeId);
-
-        return await this.Mediator.Send(command, cancellationToken);
+                                                         CancellationToken cancellationToken) {
+        IAsyncPolicy<Result> retryPolicy = PolicyFactory.CreatePolicy(2, policyTag: "MerchantSettlementDomainEventHandler - MerchantFeeSettledEvent");
+
+        return await retryPolicy.ExecuteAsync(async () => {
+            MerchantStatementCommands.AddSettledFeeToMerchantStatementCommand command = new(domainEvent.EstateId, domainEvent.MerchantId, domainEvent.FeeCalculatedDateTime, domainEvent.CalculatedValue, domainEvent.TransactionId, domainEvent.FeeId);
+
+            return await this.Mediator.Send(command, cancellationToken);
+        });
     }
 }