From 406301ab4aa05299601533e6f2fa87319ed1fae7 Mon Sep 17 00:00:00 2001 From: hippy Date: Thu, 4 Aug 2022 01:09:19 -0700 Subject: [PATCH] dh: Add ExponentialBackoffWithJitterRetryOnSignal handler --- ...alBackoffWithJitterRetryOnSignalHandler.cs | 71 +++++++++ .../misc/RequestProperties.cs | 1 + ...koffWithJitterRetryOnSignalHandlerTests.cs | 136 ++++++++++++++++++ .../misc/RetrySignalingOnConditionHandler.cs | 34 +++++ 4 files changed, 242 insertions(+) create mode 100644 src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryOnSignalHandler.cs create mode 100644 tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryOnSignalHandlerTests.cs create mode 100644 tests/rm.DelegatingHandlersTest/misc/RetrySignalingOnConditionHandler.cs diff --git a/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryOnSignalHandler.cs b/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryOnSignalHandler.cs new file mode 100644 index 0000000..63d52cd --- /dev/null +++ b/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryOnSignalHandler.cs @@ -0,0 +1,71 @@ +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Polly; +using Polly.Contrib.WaitAndRetry; + +namespace rm.DelegatingHandlers +{ + /// + /// Retries on certain conditions with exponential backoff jitter (DecorrelatedJitterBackoffV2). + /// + /// Retry conditions: + /// On signal. + /// + /// + /// source + /// + public class ExponentialBackoffWithJitterRetryOnSignalHandler : DelegatingHandler + { + private readonly IAsyncPolicy<(HttpRequestMessage request, HttpResponseMessage response)> retryPolicy; + + /// + public ExponentialBackoffWithJitterRetryOnSignalHandler( + IRetrySettings retrySettings) + { + _ = retrySettings + ?? throw new ArgumentNullException(nameof(retrySettings)); + + var sleepDurationsWithJitter = Backoff.DecorrelatedJitterBackoffV2( + medianFirstRetryDelay: TimeSpan.FromMilliseconds(retrySettings.RetryDelayInMilliseconds), + retryCount: retrySettings.RetryCount); + + // note: response can't be null + // ref: https://github.com/dotnet/runtime/issues/19925#issuecomment-272664671 + retryPolicy = Policy + .HandleResult<(HttpRequestMessage request, HttpResponseMessage response)>(tuple => + tuple.request.Properties.TryGetValue(RequestProperties.RetrySignal, out var retrySignaledObj) && (bool)retrySignaledObj) + .WaitAndRetryAsync( + sleepDurations: sleepDurationsWithJitter, + onRetry: (responseResult, delay, retryAttempt, context) => + { + // note: response can be null in case of handled exception + responseResult.Result.response?.Dispose(); + context[ContextKey.RetryAttempt] = retryAttempt; + }); + } + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var tuple = await retryPolicy.ExecuteAsync( + action: async (context, ct) => + { + if (context.TryGetValue(ContextKey.RetryAttempt, out var retryAttempt)) + { + request.Properties[RequestProperties.PollyRetryAttempt] = retryAttempt; + } + request.Properties.Remove(RequestProperties.RetrySignal); + var response = await base.SendAsync(request, ct) + .ConfigureAwait(false); + return (request, response); + }, + context: new Context(), + cancellationToken: cancellationToken) + .ConfigureAwait(false); + return tuple.response; + } + } +} diff --git a/src/rm.DelegatingHandlers/misc/RequestProperties.cs b/src/rm.DelegatingHandlers/misc/RequestProperties.cs index dc4e177..34aebac 100644 --- a/src/rm.DelegatingHandlers/misc/RequestProperties.cs +++ b/src/rm.DelegatingHandlers/misc/RequestProperties.cs @@ -3,5 +3,6 @@ public static class RequestProperties { public static readonly string PollyRetryAttempt = "PollyRetryAttempt"; + public static readonly string RetrySignal = "RetrySignal"; } } diff --git a/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryOnSignalHandlerTests.cs b/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryOnSignalHandlerTests.cs new file mode 100644 index 0000000..de3b74a --- /dev/null +++ b/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryOnSignalHandlerTests.cs @@ -0,0 +1,136 @@ +using System.Net; +using AutoFixture; +using AutoFixture.AutoMoq; +using NUnit.Framework; +using rm.DelegatingHandlers; + +namespace rm.DelegatingHandlersTest +{ + [TestFixture] + public class ExponentialBackoffWithJitterRetryOnSignalHandlerTests + { + [Test] + public async Task Retries_On_Signal() + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var shortCircuitingCannedResponsesHandler = new ShortCircuitingCannedResponsesHandler( + new HttpResponseMessage() { StatusCode = (HttpStatusCode)404 }, // retry + new HttpResponseMessage() { StatusCode = (HttpStatusCode)200, Content = new StringContent("yawn!") }, // retry + new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 }, // NO retry + new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 }, // not used + new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 } // not used + ); + var retrySignalingOnConditionHandler = new RetrySignalingOnConditionHandler(); + var retryAttempt = -1; + var delegateHandler = new DelegateHandler( + (request, ct) => + { + retryAttempt++; + return Task.CompletedTask; + }); + var retryHandler = new ExponentialBackoffWithJitterRetryOnSignalHandler( + new RetrySettings + { + RetryCount = 5, + RetryDelayInMilliseconds = 0, + }); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, retrySignalingOnConditionHandler, shortCircuitingCannedResponsesHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + + Assert.AreEqual(2, retryAttempt); + } + + [Test] + public async Task Does_Not_Retry_If_No_Signal() + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var shortCircuitingCannedResponsesHandler = new ShortCircuitingCannedResponsesHandler( + new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 } // NO retry + ); + var retrySignalingOnConditionHandler = new RetrySignalingOnConditionHandler(); + var retryAttempt = -1; + var delegateHandler = new DelegateHandler( + (request, ct) => + { + retryAttempt++; + return Task.CompletedTask; + }); + var retryHandler = new ExponentialBackoffWithJitterRetryOnSignalHandler( + new RetrySettings + { + RetryCount = 5, + RetryDelayInMilliseconds = 0, + }); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, retrySignalingOnConditionHandler, shortCircuitingCannedResponsesHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + + Assert.AreEqual(0, retryAttempt); + } + + [Test] + public async Task When_0_Retries_PollyRetryAttempt_Property_Is_Not_Present() + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var shortCircuitingCannedResponsesHandler = new ShortCircuitingCannedResponsesHandler( + new HttpResponseMessage() { StatusCode = (HttpStatusCode)404 }); + var retrySignalingOnConditionHandler = new RetrySignalingOnConditionHandler(); + var retryHandler = new ExponentialBackoffWithJitterRetryOnSignalHandler( + new RetrySettings + { + RetryCount = 0, + RetryDelayInMilliseconds = 0, + }); + + using var invoker = HttpMessageInvokerFactory.Create( + fixture.Create(), retryHandler, retrySignalingOnConditionHandler, shortCircuitingCannedResponsesHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + +#pragma warning disable CS0618 // Type or member is obsolete + Assert.IsFalse(requestMessage.Properties.ContainsKey(RequestProperties.PollyRetryAttempt)); +#pragma warning restore CS0618 // Type or member is obsolete + } + + [Test] + [TestCase(1)] + [TestCase(2)] + public async Task When_N_Retries_PollyRetryAttempt_Property_Is_Present(int retryCount) + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var shortCircuitingCannedResponsesHandler = new ShortCircuitingCannedResponsesHandler( + new HttpResponseMessage() { StatusCode = (HttpStatusCode)404 }, + new HttpResponseMessage() { StatusCode = (HttpStatusCode)200, Content = new StringContent("yawn!") }, + new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 }); + var retrySignalingOnConditionHandler = new RetrySignalingOnConditionHandler(); + var retryHandler = new ExponentialBackoffWithJitterRetryOnSignalHandler( + new RetrySettings + { + RetryCount = retryCount, + RetryDelayInMilliseconds = 0, + }); + + using var invoker = HttpMessageInvokerFactory.Create( + fixture.Create(), retryHandler, retrySignalingOnConditionHandler, shortCircuitingCannedResponsesHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + +#pragma warning disable CS0618 // Type or member is obsolete + Assert.AreEqual(retryCount, requestMessage.Properties[RequestProperties.PollyRetryAttempt]); +#pragma warning restore CS0618 // Type or member is obsolete + } + } +} diff --git a/tests/rm.DelegatingHandlersTest/misc/RetrySignalingOnConditionHandler.cs b/tests/rm.DelegatingHandlersTest/misc/RetrySignalingOnConditionHandler.cs new file mode 100644 index 0000000..e7cbaff --- /dev/null +++ b/tests/rm.DelegatingHandlersTest/misc/RetrySignalingOnConditionHandler.cs @@ -0,0 +1,34 @@ +using System.Net; +using rm.DelegatingHandlers; + +namespace rm.DelegatingHandlersTest +{ + public class RetrySignalingOnConditionHandler : DelegatingHandler + { + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + var response = await base.SendAsync(request, cancellationToken); + + // tweak conditions accordingly + if (response.StatusCode == (HttpStatusCode)404) + { +#pragma warning disable CS0618 // Type or member is obsolete + request.Properties[RequestProperties.RetrySignal] = true; +#pragma warning restore CS0618 // Type or member is obsolete + return response; + } + var content = await response.Content.ReadAsStringAsync(cancellationToken); + if (content.Contains("yawn!")) + { +#pragma warning disable CS0618 // Type or member is obsolete + request.Properties[RequestProperties.RetrySignal] = true; +#pragma warning restore CS0618 // Type or member is obsolete + return response; + } + + return response; + } + } +}