dh: Add ExponentialBackoffWithJitterRetryOnSignal handler

src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryOnSignalHandler.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Polly;
+using Polly.Contrib.WaitAndRetry;
+
+namespace rm.DelegatingHandlers
+{
+	/// <summary>
+	/// Retries on certain conditions with exponential backoff jitter (DecorrelatedJitterBackoffV2).
+	/// <para></para>
+	/// Retry conditions:
+	///   On signal.
+	/// </summary>
+	/// <remarks>
+	/// <see href="https://github.com/App-vNext/Polly/wiki/Retry-with-jitter">source</see>
+	/// </remarks>
+	public class ExponentialBackoffWithJitterRetryOnSignalHandler : DelegatingHandler
+	{
+		private readonly IAsyncPolicy<(HttpRequestMessage request, HttpResponseMessage response)> retryPolicy;
+
+		/// <inheritdoc cref="ExponentialBackoffWithJitterRetryOnSignalHandler" />
+		public ExponentialBackoffWithJitterRetryOnSignalHandler(
+			IRetrySettings retrySettings)
+		{
+			_ = retrySettings
+				?? throw new ArgumentNullException(nameof(retrySettings));
+
+			var sleepDurationsWithJitter = Backoff.DecorrelatedJitterBackoffV2(
+				medianFirstRetryDelay: TimeSpan.FromMilliseconds(retrySettings.RetryDelayInMilliseconds),
+				retryCount: retrySettings.RetryCount);
+
+			// note: response can't be null
+			// ref: https://github.com/dotnet/runtime/issues/19925#issuecomment-272664671
+			retryPolicy = Policy
+				.HandleResult<(HttpRequestMessage request, HttpResponseMessage response)>(tuple =>
+					tuple.request.Properties.TryGetValue(RequestProperties.RetrySignal, out var retrySignaledObj) && (bool)retrySignaledObj)
+				.WaitAndRetryAsync(
+					sleepDurations: sleepDurationsWithJitter,
+					onRetry: (responseResult, delay, retryAttempt, context) =>
+					{
+						// note: response can be null in case of handled exception
+						responseResult.Result.response?.Dispose();
+						context[ContextKey.RetryAttempt] = retryAttempt;
+					});
+		}
+
+		protected override async Task<HttpResponseMessage> SendAsync(
+			HttpRequestMessage request,
+			CancellationToken cancellationToken)
+		{
+			var tuple = await retryPolicy.ExecuteAsync(
+				action: async (context, ct) =>
+				{
+					if (context.TryGetValue(ContextKey.RetryAttempt, out var retryAttempt))
+					{
+						request.Properties[RequestProperties.PollyRetryAttempt] = retryAttempt;
+					}
+					request.Properties.Remove(RequestProperties.RetrySignal);
+					var response = await base.SendAsync(request, ct)
+						.ConfigureAwait(false);
+					return (request, response);
+				},
+				context: new Context(),
+				cancellationToken: cancellationToken)
+					.ConfigureAwait(false);
+			return tuple.response;
+		}
+	}
+}

src/rm.DelegatingHandlers/misc/RequestProperties.cs

@@ -3,5 +3,6 @@
 	public static class RequestProperties
 	{
 		public static readonly string PollyRetryAttempt = "PollyRetryAttempt";
+		public static readonly string RetrySignal = "RetrySignal";
 	}
 }

tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryOnSignalHandlerTests.cs

@@ -0,0 +1,136 @@
+using System.Net;
+using AutoFixture;
+using AutoFixture.AutoMoq;
+using NUnit.Framework;
+using rm.DelegatingHandlers;
+
+namespace rm.DelegatingHandlersTest
+{
+	[TestFixture]
+	public class ExponentialBackoffWithJitterRetryOnSignalHandlerTests
+	{
+		[Test]
+		public async Task Retries_On_Signal()
+		{
+			var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+			var shortCircuitingCannedResponsesHandler = new ShortCircuitingCannedResponsesHandler(
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)404 }, // retry
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)200, Content = new StringContent("yawn!") }, // retry
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 }, // NO retry
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 }, // not used
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 }  // not used
+				);
+			var retrySignalingOnConditionHandler = new RetrySignalingOnConditionHandler();
+			var retryAttempt = -1;
+			var delegateHandler = new DelegateHandler(
+				(request, ct) =>
+				{
+					retryAttempt++;
+					return Task.CompletedTask;
+				});
+			var retryHandler = new ExponentialBackoffWithJitterRetryOnSignalHandler(
+				new RetrySettings
+				{
+					RetryCount = 5,
+					RetryDelayInMilliseconds = 0,
+				});
+
+			using var invoker = HttpMessageInvokerFactory.Create(
+				retryHandler, delegateHandler, retrySignalingOnConditionHandler, shortCircuitingCannedResponsesHandler);
+
+			using var requestMessage = fixture.Create<HttpRequestMessage>();
+			using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+			Assert.AreEqual(2, retryAttempt);
+		}
+
+		[Test]
+		public async Task Does_Not_Retry_If_No_Signal()
+		{
+			var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+			var shortCircuitingCannedResponsesHandler = new ShortCircuitingCannedResponsesHandler(
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 } // NO retry
+				);
+			var retrySignalingOnConditionHandler = new RetrySignalingOnConditionHandler();
+			var retryAttempt = -1;
+			var delegateHandler = new DelegateHandler(
+				(request, ct) =>
+				{
+					retryAttempt++;
+					return Task.CompletedTask;
+				});
+			var retryHandler = new ExponentialBackoffWithJitterRetryOnSignalHandler(
+				new RetrySettings
+				{
+					RetryCount = 5,
+					RetryDelayInMilliseconds = 0,
+				});
+
+			using var invoker = HttpMessageInvokerFactory.Create(
+				retryHandler, delegateHandler, retrySignalingOnConditionHandler, shortCircuitingCannedResponsesHandler);
+
+			using var requestMessage = fixture.Create<HttpRequestMessage>();
+			using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+			Assert.AreEqual(0, retryAttempt);
+		}
+
+		[Test]
+		public async Task When_0_Retries_PollyRetryAttempt_Property_Is_Not_Present()
+		{
+			var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+			var shortCircuitingCannedResponsesHandler = new ShortCircuitingCannedResponsesHandler(
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)404 });
+			var retrySignalingOnConditionHandler = new RetrySignalingOnConditionHandler();
+			var retryHandler = new ExponentialBackoffWithJitterRetryOnSignalHandler(
+				new RetrySettings
+				{
+					RetryCount = 0,
+					RetryDelayInMilliseconds = 0,
+				});
+
+			using var invoker = HttpMessageInvokerFactory.Create(
+				fixture.Create<HttpMessageHandler>(), retryHandler, retrySignalingOnConditionHandler, shortCircuitingCannedResponsesHandler);
+
+			using var requestMessage = fixture.Create<HttpRequestMessage>();
+			using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+#pragma warning disable CS0618 // Type or member is obsolete
+			Assert.IsFalse(requestMessage.Properties.ContainsKey(RequestProperties.PollyRetryAttempt));
+#pragma warning restore CS0618 // Type or member is obsolete
+		}
+
+		[Test]
+		[TestCase(1)]
+		[TestCase(2)]
+		public async Task When_N_Retries_PollyRetryAttempt_Property_Is_Present(int retryCount)
+		{
+			var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+			var shortCircuitingCannedResponsesHandler = new ShortCircuitingCannedResponsesHandler(
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)404 },
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)200, Content = new StringContent("yawn!") },
+				new HttpResponseMessage() { StatusCode = (HttpStatusCode)200 });
+			var retrySignalingOnConditionHandler = new RetrySignalingOnConditionHandler();
+			var retryHandler = new ExponentialBackoffWithJitterRetryOnSignalHandler(
+				new RetrySettings
+				{
+					RetryCount = retryCount,
+					RetryDelayInMilliseconds = 0,
+				});
+
+			using var invoker = HttpMessageInvokerFactory.Create(
+				fixture.Create<HttpMessageHandler>(), retryHandler, retrySignalingOnConditionHandler, shortCircuitingCannedResponsesHandler);
+
+			using var requestMessage = fixture.Create<HttpRequestMessage>();
+			using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+#pragma warning disable CS0618 // Type or member is obsolete
+			Assert.AreEqual(retryCount, requestMessage.Properties[RequestProperties.PollyRetryAttempt]);
+#pragma warning restore CS0618 // Type or member is obsolete
+		}
+	}
+}

tests/rm.DelegatingHandlersTest/misc/RetrySignalingOnConditionHandler.cs

@@ -0,0 +1,34 @@
+using System.Net;
+using rm.DelegatingHandlers;
+
+namespace rm.DelegatingHandlersTest
+{
+	public class RetrySignalingOnConditionHandler : DelegatingHandler
+	{
+		protected override async Task<HttpResponseMessage> SendAsync(
+			HttpRequestMessage request,
+			CancellationToken cancellationToken)
+		{
+			var response = await base.SendAsync(request, cancellationToken);
+
+			// tweak conditions accordingly
+			if (response.StatusCode == (HttpStatusCode)404)
+			{
+#pragma warning disable CS0618 // Type or member is obsolete
+				request.Properties[RequestProperties.RetrySignal] = true;
+#pragma warning restore CS0618 // Type or member is obsolete
+				return response;
+			}
+			var content = await response.Content.ReadAsStringAsync(cancellationToken);
+			if (content.Contains("yawn!"))
+			{
+#pragma warning disable CS0618 // Type or member is obsolete
+				request.Properties[RequestProperties.RetrySignal] = true;
+#pragma warning restore CS0618 // Type or member is obsolete
+				return response;
+			}
+
+			return response;
+		}
+	}
+}