Add httpclient polly

ATI.Services.Common/ATI.Services.Common.csproj

@@ -19,12 +19,14 @@
     <PackageReference Include="JetBrains.Annotations" Version="2020.1.0" />
     <PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.0" />
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="3.1.2" />
-    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="3.1.3" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.3" />
     <PackageReference Include="NLog" Version="4.7.6" />
     <PackageReference Include="NLog.DiagnosticSource" Version="1.3.0" />
     <PackageReference Include="NLog.Web.AspNetCore" Version="4.9.3" />
     <PackageReference Include="Npgsql" Version="7.0.6" />
-    <PackageReference Include="Polly" Version="7.2.3" />
+    <PackageReference Include="Polly" Version="7.2.4" />
+    <PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
     <PackageReference Include="prometheus-net" Version="8.2.1" />
     <PackageReference Include="StackExchange.Redis" Version="2.2.50" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />

ATI.Services.Common/Caching/Redis/HitRatioCounter.cs

@@ -13,8 +13,8 @@ public class HitRatioCounter
 
         public HitRatioCounter(string hitRatioCounterService, string[] lables)
         {
-            _hitRatio = Prometheus.Metrics.CreateGauge($"{hitRatioCounterService}_hits_ratio", "", lables);
-            _hitRatioNetto = Prometheus.Metrics.CreateGauge($"{hitRatioCounterService}_hits_ratio_netto", "", lables);
+            _hitRatio = Prometheus.Metrics.CreateGauge($"{hitRatioCounterService}_hits_ratio", string.Empty, lables);
+            _hitRatioNetto = Prometheus.Metrics.CreateGauge($"{hitRatioCounterService}_hits_ratio_netto", string.Empty, lables);
         }
 
         public void Hit(int count = 1)

ATI.Services.Common/Caching/Redis/RedisProvider.cs

@@ -3,6 +3,7 @@
 using System.Linq;
 using System.Threading.Tasks;
 using ATI.Services.Common.Caching.Redis.Abstractions;
+using ATI.Services.Common.Metrics;
 using ATI.Services.Common.Serializers;
 using JetBrains.Annotations;
 using Microsoft.Extensions.Options;
@@ -15,7 +16,7 @@ public class RedisProvider
     private readonly Dictionary<string, RedisCache> _redisCaches = new();
     private readonly Logger _logger = LogManager.GetCurrentClassLogger();
 
-    public RedisProvider(IOptions<CacheManagerOptions> cacheManagerOptions, SerializerProvider serializerProvider)
+    public RedisProvider(IOptions<CacheManagerOptions> cacheManagerOptions, SerializerProvider serializerProvider, MetricsFactory metricsFactory)
     {
         var cacheOptions = cacheManagerOptions.Value.CacheOptions;
         var manager = new CacheHitRatioManager(cacheManagerOptions.Value.HitRatioManagerUpdatePeriod);
@@ -33,7 +34,7 @@ public RedisProvider(IOptions<CacheManagerOptions> cacheManagerOptions, Serializ
                 { } jsonSerializer => new RedisStringSerializer(jsonSerializer)
             };
 
-            var cache = new RedisCache(options, manager, serializer);
+            var cache = new RedisCache(options, manager, serializer, metricsFactory);
             _redisCaches.Add(redisName, cache);
         }
     }

ATI.Services.Common/Caching/Redis/RedisScriptCache.cs

@@ -16,7 +16,7 @@ namespace ATI.Services.Common.Caching.Redis;
 
 internal class RedisScriptCache : BaseRedisCache
 {
-    private readonly MetricsFactory _metricsFactory;
+    private readonly MetricsInstance _metrics;
     private readonly IDatabase _redisDb;
     private readonly AsyncCircuitBreakerPolicy _circuitBreakerPolicy;
     private readonly AsyncPolicyWrap _policy;
@@ -26,21 +26,21 @@ internal class RedisScriptCache : BaseRedisCache
     /// </summary>
     /// <param name="redisDb"></param>
     /// <param name="redisOptions"></param>
-    /// <param name="metricsFactory"></param>
+    /// <param name="metrics"></param>
     /// <param name="circuitBreakerPolicy"></param>
     /// <param name="policy"></param>
     public RedisScriptCache(
         IDatabase redisDb,
         RedisOptions redisOptions,
-        MetricsFactory metricsFactory,
+        MetricsInstance metrics,
         AsyncCircuitBreakerPolicy circuitBreakerPolicy,
         AsyncPolicyWrap policy,
         IRedisSerializer serializer
     )
     {
         Options = redisOptions;
         _redisDb = redisDb;
-        _metricsFactory = metricsFactory;
+        _metrics = metrics;
         _circuitBreakerPolicy = circuitBreakerPolicy;
         _policy = policy;
         Serializer = serializer;
@@ -52,7 +52,7 @@ IRedisSerializer serializer
         if (redisValue.Count < 0)
             return OperationResult.Ok;
 
-        using (_metricsFactory.CreateMetricsTimerWithLogging(metricEntity,
+        using (_metrics.CreateMetricsTimerWithLogging(metricEntity,
                    requestParams: new { RedisValues = redisValue }, longRequestTime: longRequestTime))
         {
             var result = await InsertManyByScriptAsync(redisValue.Select(v => v.GetKey()), redisValue);
@@ -66,7 +66,7 @@ IRedisSerializer serializer
         if (redisValues == null || redisValues.Count == 0)
             return OperationResult.Ok;
 
-        using (_metricsFactory.CreateMetricsTimerWithLogging(metricEntity,
+        using (_metrics.CreateMetricsTimerWithLogging(metricEntity,
                    requestParams: new { RedisValues = redisValues },
                    longRequestTime: longRequestTime))
         {

ATI.Services.Common/Extensions/ServiceCollectionExtensions.cs

@@ -19,7 +19,7 @@ public static void ConfigureByName<T>(this IServiceCollection serviceCollection,
             if(checkSectionExists && !section.Exists())
                 throw new Exception($"Секции {sectionName} нет в appsettings.json");
 
-            serviceCollection.Configure<T>(ConfigurationManager.GetSection(typeof(T).Name));
+            serviceCollection.Configure<T>(section);
         }
 
         [UsedImplicitly]

ATI.Services.Common/Http/HttpClientBuilderPolicyExtensions.cs

@@ -0,0 +1,132 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using ATI.Services.Common.Logging;
+using ATI.Services.Common.Options;
+using Microsoft.Extensions.DependencyInjection;
+using NLog;
+using Polly;
+using Polly.CircuitBreaker;
+using Polly.Contrib.WaitAndRetry;
+using Polly.Extensions.Http;
+using Polly.Registry;
+using Polly.Timeout;
+
+namespace ATI.Services.Common.Http;
+
+/// <summary>
+/// https://habr.com/ru/companies/dododev/articles/503376/
+/// </summary>
+public static class HttpClientBuilderPolicyExtensions
+{
+    /// <summary>
+    /// Func, because we need new iterator for every request (different time interval)
+    /// </summary>
+    private static readonly Func<TimeSpan, int, IEnumerable<TimeSpan>> RetryPolicyDelay =  (medianFirstRetryDelay, retryCount) => Backoff.DecorrelatedJitterBackoffV2(
+        medianFirstRetryDelay: medianFirstRetryDelay,
+        retryCount: retryCount);
+
+    public static IHttpClientBuilder AddRetryPolicy(
+        this IHttpClientBuilder clientBuilder,
+        BaseServiceOptions serviceOptions,
+        ILogger logger)
+    {
+        var methodsToRetry = serviceOptions.HttpMethodsToRetry ?? new List<string> { HttpMethod.Get.Method };
+
+        return clientBuilder
+            .AddPolicyHandler((_, message) =>
+            {
+                if (!methodsToRetry.Contains(message.Method.Method, StringComparer.OrdinalIgnoreCase))
+                {
+                    return Policy.NoOpAsync<HttpResponseMessage>();
+                }
+
+                return HttpPolicyExtensions
+                    .HandleTransientHttpError()
+                    .Or<TimeoutRejectedException>()
+                    .OrResult(r => r.StatusCode == (HttpStatusCode) 429) // Too Many Requests
+                    .WaitAndRetryAsync(RetryPolicyDelay(serviceOptions.MedianFirstRetryDelay, serviceOptions.RetryCount),
+                        (response, sleepDuration, retryCount, _) =>
+                        {
+                            logger.ErrorWithObject(response?.Exception, "Error while WaitAndRetry",  new
+                            {
+                                serviceOptions.ConsulName,
+                                message.RequestUri,
+                                message.Method,
+                                response?.Result?.StatusCode,
+                                sleepDuration,
+                                retryCount
+                            } );
+                        });
+            });
+    }
+
+    public static IHttpClientBuilder AddHostSpecificCircuitBreakerPolicy(
+        this IHttpClientBuilder clientBuilder,
+        BaseServiceOptions serviceOptions,
+        ILogger logger)
+    {
+        var registry = new PolicyRegistry();
+        return clientBuilder.AddPolicyHandler(message =>
+        {
+            var policyKey = message.RequestUri.Host;
+            var policy = registry.GetOrAdd(policyKey, BuildCircuitBreakerPolicy(message, serviceOptions, logger));
+            return policy;
+        });
+    }
+
+    public static IHttpClientBuilder AddTimeoutPolicy(
+        this IHttpClientBuilder httpClientBuilder,
+        TimeSpan timeout)
+    {
+        return httpClientBuilder.AddPolicyHandler(Policy.TimeoutAsync<HttpResponseMessage>(timeout));
+    }
+
+
+    private static AsyncCircuitBreakerPolicy<HttpResponseMessage> BuildCircuitBreakerPolicy(
+        HttpRequestMessage message,
+        BaseServiceOptions serviceOptions,
+        ILogger logger)
+    {
+        return HttpPolicyExtensions
+            .HandleTransientHttpError()
+            .Or<TimeoutRejectedException>()
+            .OrResult(r => r.StatusCode == (HttpStatusCode) 429) // Too Many Requests
+            .CircuitBreakerAsync(
+                serviceOptions.CircuitBreakerExceptionsCount,
+                serviceOptions.CircuitBreakerDuration,
+                (response, circuitState, timeSpan, _) =>
+                {
+                    logger.ErrorWithObject(null, "CB onBreak", new
+                    {
+                        serviceOptions.ConsulName,
+                        message.RequestUri,
+                        message.Method,
+                        response?.Result?.StatusCode,
+                        circuitState,
+                        timeSpan
+                    });
+                },
+                context =>
+                {
+                    logger.ErrorWithObject(null, "CB onReset", new
+                    {
+                        serviceOptions.ConsulName,
+                        message.RequestUri,
+                        message.Method,
+                        context
+                    });
+                },
+                () =>
+                {
+                    logger.ErrorWithObject(null, "CB onHalfOpen", new
+                    {
+                        serviceOptions.ConsulName,
+                        message.RequestUri,
+                        message.Method,
+                    });
+                });
+    }
+}

ATI.Services.Common/Http/HttpClientExtensions.cs

@@ -0,0 +1,32 @@
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Net.Http.Headers;
+
+namespace ATI.Services.Common.Http;
+
+internal static class HttpClientExtensions
+{
+    public static void SetBaseFields(
+        this HttpClient httpClient, 
+        string serviceAsClientName,
+        string serviceAsClientHeaderName,
+        Dictionary<string,string> additionalHeaders)
+    {
+        httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        if (!string.IsNullOrEmpty(serviceAsClientName) &&
+            !string.IsNullOrEmpty(serviceAsClientHeaderName))
+        {
+            httpClient.DefaultRequestHeaders.Add(
+                serviceAsClientHeaderName,
+                serviceAsClientName);
+        }
+
+        if (additionalHeaders is { Count: > 0 })
+        {
+            foreach (var header in additionalHeaders)
+            {
+                httpClient.DefaultRequestHeaders.Add(header.Key, header.Value);
+            }
+        }
+    }
+}

ATI.Services.Common/Variables/AppHttpContext.cs → ATI.Services.Common/Http/HttpContextHelper.cs

@@ -4,56 +4,24 @@
 using System.Linq;
 using ATI.Services.Common.Metrics;
 using Microsoft.AspNetCore.Http;
-using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Primitives;
 
 namespace ATI.Services.Common.Variables;
 
-internal static class AppHttpContext
+internal static class HttpContextHelper
 {
-    private static IServiceProvider? _services;
+    public static string[] MetricsHeadersValues(HttpContext? httpContext) => GetHeadersValues(httpContext, MetricsLabelsAndHeaders.UserHeaders);
+    public static Dictionary<string, string> HeadersAndValuesToProxy(HttpContext? httpContext,IReadOnlyCollection<string>? headersToProxy) => GetHeadersAndValues(httpContext, headersToProxy);
+
 
-    /// <summary>
-    /// Provides static access to the framework's services provider
-    /// </summary>
-    public static IServiceProvider? Services
-    {
-        get => _services;
-        set
-        {
-            // If _services already initialized -> return
-            if (_services is not null)
-                return;
-
-            _services = value;
-        }
-    }
-
-    public static string[] MetricsHeadersValues => GetHeadersValues(MetricsLabelsAndHeaders.UserHeaders);
-    public static Dictionary<string, string> HeadersAndValuesToProxy(IReadOnlyCollection<string>? headersToProxy) => GetHeadersAndValues(headersToProxy);
-
-    /// <summary>
-    /// Provides static access to the current HttpContext
-    /// </summary>
-    private static HttpContext? Ctx
-    {
-        get
-        {
-            ArgumentNullException.ThrowIfNull(Services, "IServiceProvider can't be null. Try do services.AddServiceVariables() in app configuration");
-
-            var httpContextAccessor = Services.GetService<IHttpContextAccessor>();
-            return httpContextAccessor?.HttpContext;
-        }
-    }
-
-    private static string[] GetHeadersValues(IReadOnlyCollection<string>? headersNames)
+    private static string[] GetHeadersValues(HttpContext? httpContext, IReadOnlyCollection<string>? headersNames)
     {
         try
         {
             if (headersNames is null || headersNames.Count == 0)
                 return Array.Empty<string>();
 
-            var headersValues = headersNames.Select(label => GetHeaderValue(Ctx, label)).ToArray();
+            var headersValues = headersNames.Select(label => GetHeaderValue(httpContext, label)).ToArray();
             return headersValues;
         }
         catch (ObjectDisposedException) // when thing happen outside http ctx e.g eventbus event handler
@@ -68,18 +36,18 @@
             return "This service";
 
         if (context.Request.Headers.TryGetValue(headerName, out var headerValues) && !StringValues.IsNullOrEmpty(headerValues))
             return headerValues[0];
 
         return "Empty";
     }
 
-    private static Dictionary<string, string> GetHeadersAndValues(IReadOnlyCollection<string>? headersNames)
+    private static Dictionary<string, string> GetHeadersAndValues(HttpContext? httpContext, IReadOnlyCollection<string>? headersNames)
     {
-        if (headersNames is null || headersNames.Count == 0 || Ctx is null)
+        if (headersNames is null || headersNames.Count == 0 || httpContext is null)
             return new Dictionary<string, string>();
 
         return headersNames
-            .Select(header => Ctx.Request.Headers.TryGetValue(header, out var headerValues)
+            .Select(header => httpContext.Request.Headers.TryGetValue(header, out var headerValues)
                               && !StringValues.IsNullOrEmpty(headerValues)
                 ? new
                 {