Bug Fixes

ReleaseNotes/version2.2.md

@@ -1,5 +1,16 @@
 # Release Notes #
 
+2.2.10.5-d1
+
+* Bug Fix: Start App Insights with app config setting
+* Bug Fix: environment defaults and BackendOptions were getting mixedup 
+* Bug Fix: Probes not getting logged
+* Bug Fix: Environment variables not being defaulted to
+* Bug Fix: Make sure program does not exit until shutdown completes
+* Make shared_iterator use either singlePass or maxAttempts, default to singlePass.
+* Don't invalidate shared iterators unless using latency load balancer and the order changed
+* In the case of a 404, use the same response path as a normal response
+
 2.2.10.4
 
 Proxy:

deployment/AppConfiguration/deploy.sh

@@ -1,19 +1,19 @@
 #!/bin/bash
 
-# Deploy/Update Azure App Configuration for BackendOptions
+# Deploy/Update Azure App Configuration for ProxyConfig
 #
 # Goals:
 #   1. Migration  – seed App Configuration from a live Container App's
 #                   env vars (with fallback to local shell variables).
 #   2. Catalog    – every publishable setting is always written so that
 #                   operators can see the full list in the portal.
 #                   When no env value exists, the C# default from
-#                   BackendOptions.cs is used. If even that is empty,
+#                   ProxyConfig.cs is used. If even that is empty,
 #                   a "-" placeholder is written, meaning "use the
 #                   built-in code default".
 #
 # Discovers publishable keys dynamically from [ConfigOption("...")]
-# decorations in BackendOptions.cs.
+# decorations in ProxyConfig.cs.
 #
 # Three modes (ConfigMode enum):
 #   Warm   – published under "Warm:" prefix, hot-reloaded (~30 s)
@@ -60,7 +60,7 @@ APPCONFIG_LABEL="${APPCONFIG_LABEL:-}"
 AZURE_APPCONFIG_REFRESH_SECONDS="${AZURE_APPCONFIG_REFRESH_SECONDS:-30}"
 UPDATE_CONTAINER_APP_ENV="${UPDATE_CONTAINER_APP_ENV:-true}"
 
-BACKEND_OPTIONS_FILE="${BACKEND_OPTIONS_FILE:-${REPO_ROOT}/src/SimpleL7Proxy/Config/BackendOptions.cs}"
+BACKEND_OPTIONS_FILE="${BACKEND_OPTIONS_FILE:-${REPO_ROOT}/src/SimpleL7Proxy/Config/ProxyConfig.cs}"
 
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
@@ -76,7 +76,7 @@ if ! command -v az >/dev/null 2>&1; then
 fi
 
 if [ ! -f "${BACKEND_OPTIONS_FILE}" ]; then
-    echo -e "${RED}Error: BackendOptions file not found: ${BACKEND_OPTIONS_FILE}${NC}"
+    echo -e "${RED}Error: ProxyConfig file not found: ${BACKEND_OPTIONS_FILE}${NC}"
     exit 1
 fi
 
@@ -310,7 +310,7 @@ for entry in "${CONFIG_ENTRIES[@]}"; do
         [ -n "${VALUE}" ] && SOURCE="local-env"
     fi
 
-    # 3) Fallback to C# default from BackendOptions.cs
+    # 3) Fallback to C# default from ProxyConfig.cs
     if [ -z "${VALUE}" ] && [ -n "${CS_DEFAULT}" ]; then
         VALUE="${CS_DEFAULT}"
         SOURCE="cs-default"

src/SimpleL7Proxy/Backend/Backends.cs

@@ -29,7 +29,7 @@ public class Backends : IBackendService
   /// </summary>
   private List<BaseHostHealth> _backendHosts => _backendHostCollection.Current.Hosts;
 
-  private readonly BackendOptions _options;
+  private readonly ProxyConfig _options;
   private static readonly bool _debug = false;
 
   private static double _successRate;
@@ -44,6 +44,7 @@ public class Backends : IBackendService
 
   private readonly IEventClient _eventClient;
   private readonly ISharedIteratorRegistry? _sharedIteratorRegistry;
+  private List<Guid> _lastLatencyOrder = new();
 
   // Reusable ProxyEvent instances for backend poller to reduce allocations
   private readonly ProxyEvent _statusEvent = new ProxyEvent(25);  // 4 fixed (Timestamp, LoadBalanceMode, ActiveHostsCount, SuccessRate) + 7*N per host (assumes ~3 hosts)
@@ -57,7 +58,7 @@ public class Backends : IBackendService
   private Task? PollerTask;
   //public Backends(List<BackendHost> hosts, HttpClient client, int interval, int successRate)
   public Backends(
-      IOptions<BackendOptions> options,
+      IOptions<ProxyConfig> options,
       ICircuitBreaker circuitBreaker,
       IHostHealthCollection backendHostCollection, //
       IHostApplicationLifetime appLifetime,               //
@@ -98,13 +99,13 @@ public Backends(
 
     // Hosts are staged and activated by ConfigBootstrapper.RegisterBackends
 
-    _logger.LogDebug("[INIT] Backends service starting");
+    _logger.LogDebug("[INIT] Backend health-polling service created");
 
   }
 
   public Task Stop()
   {
-    _logger.LogInformation("[SHUTDOWN] ⏹ Backend stopping");
+    _logger.LogInformation("[SHUTDOWN] ⏹ Backend health poller stopping");
     _cancellationTokenSource.Cancel();
 
     return PollerTask ?? Task.CompletedTask;
@@ -119,13 +120,9 @@ public void Start()
     {
       if (task.Exception != null)
       {
-        Console.WriteLine($"[BACKENDS-POLLER-ERROR] {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} {task.Exception.Flatten()}");
+        _logger.LogError(task.Exception, "[SERVICE] ✗ Backend health poller task faulted {Time} {Exception}", DateTime.UtcNow, task.Exception.Flatten());
       }
     }, TaskContinuationOptions.OnlyOnFaulted);
-
-    // If OAuth is enabled, start token refresh
-
-    _logger.LogInformation("[SERVICE] ✓ Backend service started");
   }
 
 
@@ -165,13 +162,10 @@ public async Task WaitForStartup(int timeout)
     var completedTask = await Task.WhenAny(allTasks, delayTask).ConfigureAwait(false);
     if (completedTask == delayTask)
     {
-      Console.WriteLine($"[BACKENDS-STARTUP-ERROR] {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} Backend Token Provider did not initialize tokens in the last {timeout} seconds.");
-      _logger.LogError($"Backend Token Provider did not initialize tokens in the last {timeout} seconds.");
+      // Console.WriteLine($"[BACKENDS-STARTUP-ERROR] {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} Backend Token Provider did not initialize tokens in the last {timeout} seconds.");
+      _logger.LogError("Backend Token Provider did not initialize tokens in the last {Timeout} seconds.", timeout);
       throw new Exception("Backend Token Provider did not initialize tokens in time.");
     }
-
-    _logger.LogInformation($"[SERVICE] ✓ Backend Poller started in {(DateTime.Now - start).TotalSeconds:F3}s");
-
   }
 
   private readonly Dictionary<string, bool> currentHostStatus = [];
@@ -183,7 +177,7 @@ private async Task Run()
       {
         var intervalTime = TimeSpan.FromMilliseconds(_options.PollInterval).ToString(@"hh\:mm\:ss");
         var timeoutTime = TimeSpan.FromMilliseconds(_options.PollTimeout).ToString(@"hh\:mm\:ss\.fff");
-        _logger.LogInformation($"[SERVICE] ✓ Backend Poller starting - Interval: {intervalTime} | Success Rate: {_successRate} | Timeout: {timeoutTime}");
+        _logger.LogInformation($"[SERVICE] ✓ Backend health poller started — polling every {intervalTime}, healthy threshold: {_successRate}, probe timeout: {timeoutTime}");
 
         _client.Timeout = TimeSpan.FromMilliseconds(_options.PollTimeout);
 
@@ -205,24 +199,23 @@ private async Task Run()
             }
             catch (OperationCanceledException)
             {
-              _logger.LogInformation("[SHUTDOWN] ⏹ Backend Poller stopping");
+              _logger.LogInformation("[SHUTDOWN] ⏹ Backend health poller cancelled — draining");
               break;
             }
             catch (Exception e)
             {
-              _logger.LogError(e, "[BACKENDS] Unexpected error in poller loop - continuing");
+              _logger.LogError(e, "[SERVICE] ⚠ Backend health poller hit an error — retrying next cycle");
             }
           }
         }
 
-        _logger.LogInformation("[SHUTDOWN] ✓ Backend Poller stopped");
+        _logger.LogInformation("[SHUTDOWN] ✓ Backend health poller stopped");
       }
     }
     catch (Exception ex)
     {
       // Catch any unhandled exceptions to prevent background service from crashing the host
-      Console.WriteLine($"[BACKENDS-RUN-ERROR] {DateTime.UtcNow:yyyy-MM-dd HH:mm:ss} {ex}");
-      _logger.LogError(ex, "[BACKENDS] CRITICAL: Unhandled exception in backend poller - service stopping");
+      _logger.LogError(ex, "[SERVICE] ✗ Backend health poller crashed — service stopping");
       throw; // Rethrow to let the host know the background service failed, but at least we logged it
     }
   }
@@ -325,6 +318,7 @@ private async Task<bool> GetHostStatus(BaseHostHealth host, HttpClient client)
         }
 
         // If the response is successful, add the host to the active hosts
+        _probeEvent["Success"] = response.IsSuccessStatusCode.ToString();
         return response.IsSuccessStatusCode;
       }
       finally
@@ -409,17 +403,22 @@ private void FilterActiveHosts()
 
     _activeHosts = newActiveHosts;
 
-    // Invalidate iterator cache only if hosts actually changed
     if (hostsChanged)
     {
       InvalidateIteratorCache();
+      _lastLatencyOrder = newActiveHosts.OrderBy(h => h.CalculatedAverageLatency).Select(h => h.guid).ToList();
     }
-    else
+    else if (string.Equals(_options.LoadBalanceMode, Constants.Latency, StringComparison.OrdinalIgnoreCase))
     {
-      // Even if host list didn't change, invalidate shared iterators
-      // so they get fresh latency ordering on next request
-      _sharedIteratorRegistry?.InvalidateAll();
+      // Only invalidate shared iterators when the latency-based ordering actually changed
+      var newOrder = newActiveHosts.OrderBy(h => h.CalculatedAverageLatency).Select(h => h.guid).ToList();
+      if (!newOrder.SequenceEqual(_lastLatencyOrder))
+      {
+        _sharedIteratorRegistry?.InvalidateAll();
+        _lastLatencyOrder = newOrder;
+      }
     }
+    // For roundrobin/random modes with unchanged host list, no invalidation needed
   }
 
   /// <summary>
@@ -459,7 +458,7 @@ private void DisplayHostStatus()
     if (_backendHosts != null)
       foreach (var host in _backendHosts.OrderBy(h => h.AverageLatency()))
       {
-        statusIndicator = host.SuccessRate() >= _successRate ? "Good  " : "Errors";
+        statusIndicator = host.SuccessRate() >= _successRate ? "✓ Active" : "✗ Below threshold";
         var roundedLatency = Math.Round(host.AverageLatency(), 3);
         var successRatePercentage = Math.Round(host.SuccessRate() * 100, 2);
         var hoststatus = host.GetStatus(out int calls, out int errors, out double average);

src/SimpleL7Proxy/Backend/CircuitBreaker.cs

@@ -11,10 +11,11 @@ namespace SimpleL7Proxy.Backend;
 
 public class CircuitBreaker : ICircuitBreaker
 {
+    static ProxyConfig _options = null!;
     private ConcurrentQueue<DateTime> hostFailureTimes2 = new();
-    private readonly int _failureThreshold;
-    private readonly int _failureTimeFrame;
-    private readonly HashSet<int> _allowableCodes;
+    private int _failureThreshold;
+    private  int _failureTimeFrame;
+    private HashSet<int> _allowableCodes = null!;
     private readonly ILogger<CircuitBreaker> _logger;
 
     // Global counters using Interlocked operations
@@ -40,22 +41,16 @@ public class CircuitBreaker : ICircuitBreaker
 
     public string ID { get; set; } = "";
 
-    public CircuitBreaker(IOptions<BackendOptions> options, ILogger<CircuitBreaker> logger)
+    public CircuitBreaker(IOptions<ProxyConfig> options, ILogger<CircuitBreaker> logger)
     {
         ArgumentNullException.ThrowIfNull(options?.Value, nameof(options));
         ArgumentNullException.ThrowIfNull(logger, nameof(logger));
 
         var backendOptions = options.Value;
-        _failureThreshold = backendOptions.CircuitBreakerErrorThreshold;
-        _failureTimeFrame = backendOptions.CircuitBreakerTimeslice;
-        _allowableCodes = new HashSet<int>(backendOptions.AcceptableStatusCodes ?? new[] { 200, 401, 403, 408, 410, 412, 417, 400 });
-
-        count_50percent = (int)(_failureThreshold * 0.5);
-        count_60percent = (int)(_failureThreshold * 0.6);
-        count_70percent = (int)(_failureThreshold * 0.7);
-        count_80percent = (int)(_failureThreshold * 0.8);
-        count_90percent = (int)(_failureThreshold * 0.9);
         _logger = logger;
+        _options = backendOptions;
+
+        InitVars();
 
         // Register this circuit breaker globally
         Interlocked.Increment(ref _totalCircuitBreakersCount);
@@ -69,6 +64,19 @@ public CircuitBreaker(IOptions<BackendOptions> options, ILogger<CircuitBreaker>
             ID, _failureThreshold, _failureTimeFrame, _totalCircuitBreakersCount);
     }
 
+    public void InitVars()
+    {
+        _failureThreshold = _options.CircuitBreakerErrorThreshold;
+        _failureTimeFrame = _options.CircuitBreakerTimeslice;
+        _allowableCodes = new HashSet<int>(_options.AcceptableStatusCodes ?? new[] { 200, 401, 403, 408, 410, 412, 417, 400 });
+
+        count_50percent = (int)(_failureThreshold * 0.5);
+        count_60percent = (int)(_failureThreshold * 0.6);
+        count_70percent = (int)(_failureThreshold * 0.7);
+        count_80percent = (int)(_failureThreshold * 0.8);
+        count_90percent = (int)(_failureThreshold * 0.9);
+    }
+
     public void TrackStatus(int code, bool wasFailure, string state)
     {
         if (_allowableCodes.Contains(code) && !wasFailure)

src/SimpleL7Proxy/Backend/HostCollectionManager.cs

@@ -91,7 +91,6 @@ public void Activate()
       }
 
       // Activate circuit breakers only for configs that survived dedup
-      _logger.LogInformation("[HOST-MANAGER] Activating {Count} host(s)...", uniqueConfigs.Count);
       foreach (var config in uniqueConfigs)
       {
         config.Activate();

src/SimpleL7Proxy/Backend/HostConfig.cs

@@ -307,7 +307,7 @@ private static ParsedConfig TryParseConfig(string hostname, string? probepath, s
         result.Hostname = result.Host;
       } else 
       {
-        Console.WriteLine($"Direct mode host detected: {result.Host}");
+        // Console.WriteLine($"Direct mode host detected: {result.Host}");
         result.Hostname = new Uri(result.Host).Host;
         if (string.IsNullOrEmpty(result.Processor))
         {

src/SimpleL7Proxy/Backend/Iterators/SharedIteratorRegistry.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Threading;
 using Microsoft.Extensions.Logging;
+using SimpleL7Proxy.Config;
 
 namespace SimpleL7Proxy.Backend.Iterators;
 
@@ -27,7 +28,7 @@ namespace SimpleL7Proxy.Backend.Iterators;
 /// │                                                                             │
 /// └─────────────────────────────────────────────────────────────────────────────┘
 /// </summary>
-public sealed class SharedIteratorRegistry : ISharedIteratorRegistry, IShutdownParticipant, IDisposable
+public sealed class SharedIteratorRegistry : ISharedIteratorRegistry, IShutdownParticipant, IDisposable, IConfigChangeSubscriber
 {
     public int ShutdownOrder => 200;
 
@@ -40,38 +41,53 @@ public Task ShutdownAsync(CancellationToken cancellationToken)
     private readonly Dictionary<string, SharedHostIterator> _iterators = new();
     private readonly object _lock = new();
     private readonly ILogger<SharedIteratorRegistry> _logger;
+    private readonly ProxyConfig _options;
     private readonly Timer _cleanupTimer;
-    private readonly TimeSpan _iteratorTTL;
-    private readonly TimeSpan _cleanupInterval;
+    private TimeSpan _iteratorTTL;
+    private TimeSpan _cleanupInterval;
     private volatile bool _disposed;
 
     /// <summary>
     /// Creates a new SharedIteratorRegistry with the specified TTL and cleanup interval.
     /// </summary>
     /// <param name="logger">Logger for diagnostic output</param>
-    /// <param name="iteratorTTLSeconds">How long an unused iterator lives before cleanup (default: 60 seconds)</param>
-    /// <param name="cleanupIntervalSeconds">How often to run cleanup (default: 30 seconds)</param>
+    /// <param name="backendOptions">Backend configuration options</param>
     public SharedIteratorRegistry(
         ILogger<SharedIteratorRegistry> logger,
-        int iteratorTTLSeconds = 60,
-        int cleanupIntervalSeconds = 30)
+        ProxyConfig backendOptions,
+        ConfigChangeNotifier configChangeNotifier)
     {
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
-        _iteratorTTL = TimeSpan.FromSeconds(Math.Max(10, iteratorTTLSeconds));
-        _cleanupInterval = TimeSpan.FromSeconds(Math.Max(5, cleanupIntervalSeconds));
+        _options = backendOptions ?? throw new ArgumentNullException(nameof(backendOptions));
 
+        InitVars();
         // Start cleanup timer
         _cleanupTimer = new Timer(
             CleanupStaleIterators,
             null,
             _cleanupInterval,
             _cleanupInterval);
 
-        _logger.LogInformation(
+        _logger.LogDebug(
             "[SharedIteratorRegistry] Initialized with TTL={TTL}s, CleanupInterval={Interval}s",
             _iteratorTTL.TotalSeconds, _cleanupInterval.TotalSeconds);
+
+        configChangeNotifier.Subscribe(this,
+            o => o.SharedIteratorTTLSeconds,
+            o => o.SharedIteratorCleanupIntervalSeconds);
+    }
+
+    public Task OnConfigChangedAsync(IReadOnlyList<ConfigChange> changes, ProxyConfig backendOptions, CancellationToken cancellationToken)
+    {
+        InitVars();
+        return Task.CompletedTask;
     }
 
+    public void InitVars()
+    {
+        _iteratorTTL = TimeSpan.FromSeconds(Math.Max(10, _options.SharedIteratorTTLSeconds));
+        _cleanupInterval = TimeSpan.FromSeconds(Math.Max(5, _options.SharedIteratorCleanupIntervalSeconds));
+    }
     /// <inheritdoc/>
     public int Count
     {
@@ -265,7 +281,6 @@ public void Dispose()
         {
             iterator.Dispose();
         }
-
-        _logger.LogInformation("[SharedIteratorRegistry] Disposed");
     }
+
 }