Improved shutdown and app configuration

ReleaseNotes/version2.2.md

@@ -1,5 +1,28 @@
 # Release Notes #
 
+2.2.10.2
+
+Proxy:
+* Performance improvements in server for Probe path and header checks
+* Bug fix when checking for change config parameters
+* Spindown old hosts that are no longer in the config
+* De-register old hosts from the global circuit breaker when removed
+* Bug Fix for gov cloud default credential
+* Add MaxEvents to control how many undrained events can be in memory
+* Slow down the incoming requests as we start to use up the event buffer 
+
+
+2.2.10.1
+
+Deployment:
+* Update script to assign the data reader role to the ACA serivce prinicpal for app configuration
+* Bug fix for truncated parameters
+
+Proxy:
+* bug fix for reading UserIDFieldName
+* bug fix when multiple host defined for the same hostname
+* Migrate eventhub config into AppConfiguration
+
 
 2.2.10
 

SimpleL7Proxy.sln

@@ -66,8 +66,7 @@ Global
 	GlobalSection(NestedProjects) = preSolution
 		{1BEAB9EB-9991-41ED-99B5-F7A8D374491A} = {806C4881-406B-4504-BCC9-781F89BFCCB9}
 		{F562EE95-23FC-48A2-B5CD-21438BFE8926} = {7FB4896E-B4A7-4030-A112-06B8E8A701B7}
-		{8911A389-8D4E-4268-90B7-9AC12C59861E} = {7FB4896E-B4A7-4030-A112-06B8E8A701B7}
-		{9F502DB0-81A5-4AAB-B915-AAABAA55B0A3} = {8911A389-8D4E-4268-90B7-9AC12C59861E}
+		{9F502DB0-81A5-4AAB-B915-AAABAA55B0A3} = {7FB4896E-B4A7-4030-A112-06B8E8A701B7}
 		{5315AE06-D9C2-456F-BE0C-4B6996540CC9} = {7FB4896E-B4A7-4030-A112-06B8E8A701B7}
 		{F045A36C-FFC3-4732-B898-1FF6ED71B4AE} = {5315AE06-D9C2-456F-BE0C-4B6996540CC9}
 		{B7220CAA-AAAA-4E8E-BECE-6420B7002D4A} = {7FB4896E-B4A7-4030-A112-06B8E8A701B7}

deployment/AppConfiguration/deploy.sh

@@ -160,6 +160,32 @@ else
     echo -e "${YELLOW}Warning: Could not determine signed-in user principal. Ensure you have 'App Configuration Data Owner' role.${NC}"
 fi
 
+# ----------------------------------------------------------------------------
+# Ensure the Container App's managed identity has data-plane read access
+# ----------------------------------------------------------------------------
+CA_PRINCIPAL_ID="$(echo "${CA_JSON}" | jq -r '.identity.principalId // empty')"
+if [ -n "${CA_PRINCIPAL_ID}" ]; then
+    EXISTING_CA_ROLE="$(az role assignment list \
+        --assignee "${CA_PRINCIPAL_ID}" \
+        --role "App Configuration Data Reader" \
+        --scope "${APPCONFIG_RESOURCE_ID}" \
+        --query "[0].id" -o tsv 2>/dev/null || true)"
+
+    if [ -z "${EXISTING_CA_ROLE}" ]; then
+        echo -e "${YELLOW}Assigning 'App Configuration Data Reader' role to Container App managed identity (${CA_PRINCIPAL_ID})...${NC}"
+        az role assignment create \
+            --assignee "${CA_PRINCIPAL_ID}" \
+            --role "App Configuration Data Reader" \
+            --scope "${APPCONFIG_RESOURCE_ID}" \
+            >/dev/null
+        echo -e "${GREEN}✓ Role assigned. RBAC propagation may take a few minutes.${NC}"
+    else
+        echo -e "${GREEN}Container App managed identity already has 'App Configuration Data Reader' role.${NC}"
+    fi
+else
+    echo -e "${YELLOW}Warning: Container App has no system-assigned managed identity. Skipping RBAC assignment.${NC}"
+fi
+
 # ----------------------------------------------------------------------------
 # Discover config options dynamically from [ConfigOption("...")] decorations.
 # Handles:
@@ -208,11 +234,19 @@ mapfile -t CONFIG_ENTRIES < <(
                         defVal = "";
                         if (match($0, /\}[[:space:]]*=[[:space:]]*(.+);/, dv)) {
                             defVal = dv[1];
-                            sub(/[[:space:]]*\/\/.*$/, "", defVal);
                             sub(/^[[:space:]]+/, "", defVal);
                             sub(/[[:space:]]+$/, "", defVal);
                             if (match(defVal, /^"(.*)"$/, q)) {
+                                # Quoted string — extract content directly.
+                                # Do NOT strip // comments here; the URL
+                                # inside (e.g. https://) is part of the value.
                                 defVal = q[1];
+                            } else {
+                                # Non-string (number, bool, enum) — safe to
+                                # strip trailing C# inline // comments.
+                                sub(/[[:space:]]*\/\/.*$/, "", defVal);
+                                sub(/^[[:space:]]+/, "", defVal);
+                                sub(/[[:space:]]+$/, "", defVal);
                             }
                         }
                         break;
@@ -226,7 +260,7 @@ mapfile -t CONFIG_ENTRIES < <(
                 print prop "|" key "|" configName "|" mode "|" defVal;
             }
         }
-    ' "${BACKEND_OPTIONS_FILE}"
+    ' "${BACKEND_OPTIONS_FILE}" | tr -d '\r'
 )
 
 if [ "${#CONFIG_ENTRIES[@]}" -eq 0 ]; then
@@ -256,7 +290,7 @@ for entry in "${CONFIG_ENTRIES[@]}"; do
     KEY_PATH="$(echo "${entry}" | cut -d'|' -f2)"
     CONFIG_NAME="$(echo "${entry}" | cut -d'|' -f3)"
     MODE="$(echo "${entry}" | cut -d'|' -f4)"
-    CS_DEFAULT="$(echo "${entry}" | cut -d'|' -f5)"
+    CS_DEFAULT="$(echo "${entry}" | cut -d'|' -f5-)"
     # Prefix matches the mode: Warm:Section:Key or Cold:Section:Key
     APP_CONFIG_KEY="${MODE}:${KEY_PATH}"
 
@@ -282,8 +316,9 @@ for entry in "${CONFIG_ENTRIES[@]}"; do
         SOURCE="cs-default"
         # Handle enum defaults like "TypeName.Value" → "Value"
         # Only match Identifier.Identifier (e.g. IterationModeEnum.SinglePass)
-        # Avoid mangling URLs, file paths, or floats that also contain dots
-        if [[ "${VALUE}" =~ ^[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*$ ]]; then
+        # Require PascalCase first segment (uppercase start) so filenames
+        # like "eventslog.json" are not mistaken for enum qualifiers.
+        if [[ "${VALUE}" =~ ^[A-Z][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*$ ]]; then
             VALUE="${VALUE##*.}"
         fi
     fi
@@ -294,6 +329,14 @@ for entry in "${CONFIG_ENTRIES[@]}"; do
         SOURCE="placeholder"
     fi
 
+    # Strip wrapping double quotes that may survive from C# string literal
+    # defaults (AWK quote-strip can fail on non-gawk) or from Container App
+    # env vars / local env vars that were set with accidental quoting.
+    if [[ "${VALUE}" == '"'*'"' ]]; then
+        VALUE="${VALUE#\"}"
+        VALUE="${VALUE%\"}"
+    fi
+
     # Escape for JSON (handle backslashes, quotes, newlines)
     JSON_VALUE="$(printf '%s' "${VALUE}" | sed 's/\\/\\\\/g; s/"/\\"/g')"
 

docs/AZURE_APP_CONFIGURATION.md

@@ -0,0 +1,258 @@
+# Azure App Configuration Integration
+
+This document describes how to use Azure App Configuration for hot-reloading **[Warm]** settings without restarting the proxy.
+
+## Overview
+
+The proxy supports three types of configuration settings:
+
+| Type | Behavior | Label in App Config | Example |
+|------|----------|--------------------|---------|
+| **Warm** | Hot-reloaded from Azure App Config | *(none)* or `APPCONFIG_LABEL` | `MaxAttempts`, `LogConsole`, `DefaultPriority` |
+| **Cold** | Read at startup, requires restart | `Cold` | `PollInterval`, `Timeout`, `Workers` |
+| **Hidden** | Not published | — | `AsyncBlobStorageConnectionString` (parsed at runtime) |
+
+Both Warm and Cold settings are stored under the `Warm:` key prefix (for a single `Select("Warm:*")` query), but are distinguished by their **label**:
+- Warm settings use the deployment label (default: no label)
+- Cold settings always use label `Cold`
+
+This makes it easy to identify which settings require a restart when browsing the Azure portal's Configuration Explorer — just look at the **Label** column.
+
+## Environment Variables
+
+Configure the connection to Azure App Configuration:
+
+```bash
+# Option 1: Managed Identity (recommended for production)
+AZURE_APPCONFIG_ENDPOINT=https://your-appconfig.azconfig.io
+
+# Option 2: Connection String (for development)
+AZURE_APPCONFIG_CONNECTION_STRING=Endpoint=https://...;Id=...;Secret=...
+
+# Optional: Label filter (default: no label)
+AZURE_APPCONFIG_LABEL=Production
+
+# Optional: Refresh interval in seconds (default: 30)
+AZURE_APPCONFIG_REFRESH_SECONDS=30
+```
+
+## Azure App Configuration Key Structure
+
+All settings are stored under the `Warm:` key prefix. **Labels** distinguish the reload mode:
+
+```
+# Warm settings (label = none or APPCONFIG_LABEL) — hot-reloaded
+Warm:MaxAttempts = 3
+Warm:LogConsole = true
+Warm:DefaultPriority = 5
+Warm:Sentinel = 1  # Change this to trigger refresh
+
+# Cold settings (label = Cold) — read at startup only
+Warm:Server:Port = 8000
+Warm:Server:Workers = 100
+Warm:Server:Timeout = 100000
+```
+
+### Sentinel Key Pattern
+
+The `Warm:Sentinel` key is used to trigger configuration refresh:
+
+1. The refresh service polls Azure App Configuration every N seconds
+2. It only checks if `Warm:Sentinel` has changed
+3. If changed, **all** Warm settings are reloaded
+4. This minimizes API calls while allowing instant updates
+
+**To trigger a refresh:** Update `Warm:Sentinel` to any new value (e.g., increment a counter or use a timestamp).
+
+## Setting Up Azure App Configuration
+
+### 1. Create the Resource
+
+```bash
+# Create resource group
+az group create --name rg-proxy --location eastus
+
+# Create App Configuration store
+az appconfig create \
+  --name appconfig-proxy \
+  --resource-group rg-proxy \
+  --location eastus \
+  --sku Standard
+```
+
+### 2. Assign Managed Identity Access
+
+```bash
+# Get the Container App's managed identity
+IDENTITY_ID=$(az containerapp show \
+  --name your-proxy-app \
+  --resource-group rg-proxy \
+  --query identity.principalId -o tsv)
+
+# Get App Configuration resource ID
+APPCONFIG_ID=$(az appconfig show \
+  --name appconfig-proxy \
+  --resource-group rg-proxy \
+  --query id -o tsv)
+
+# Assign App Configuration Data Reader role
+az role assignment create \
+  --role "App Configuration Data Reader" \
+  --assignee $IDENTITY_ID \
+  --scope $APPCONFIG_ID
+```
+
+### 3. Import Initial Settings
+
+Create a JSON file with your warm settings:
+
+```json
+{
+  "Warm:MaxAttempts": 3,
+  "Warm:DefaultPriority": 5,
+  "Warm:LogConsole": true,
+  "Warm:LogProbes": false,
+  "Warm:AcceptableStatusCodes": "[200, 201, 202]",
+  "Warm:Sentinel": "1"
+}
+```
+
+Import to App Configuration:
+
+```bash
+az appconfig kv import \
+  --name appconfig-proxy \
+  --source file \
+  --path warm-settings.json \
+  --format json \
+  --label Production
+```
+
+### 4. Update Container App Environment
+
+```bash
+az containerapp update \
+  --name your-proxy-app \
+  --resource-group rg-proxy \
+  --set-env-vars \
+    AZURE_APPCONFIG_ENDPOINT=https://appconfig-proxy.azconfig.io \
+    AZURE_APPCONFIG_LABEL=Production \
+    AZURE_APPCONFIG_REFRESH_SECONDS=30
+```
+
+## Available Warm Settings
+
+These settings can be hot-reloaded:
+
+### Logging
+- `LogConsole` - Enable console logging
+- `LogConsoleEvent` - Enable console event logging  
+- `LogPoller` - Log poller activity
+- `LogProbes` - Log health probe activity
+- `LogHeaders` - Headers to include in logs
+- `LogAllRequestHeaders` - Log all request headers
+- `LogAllRequestHeadersExcept` - Exclude specific request headers
+- `LogAllResponseHeaders` - Log all response headers
+- `LogAllResponseHeadersExcept` - Exclude specific response headers
+
+### Request Processing
+- `MaxAttempts` - Maximum retry attempts
+- `DefaultPriority` - Default request priority
+- `DefaultTTLSecs` - Default time-to-live
+- `TimeoutHeader` - Header containing timeout value
+- `TTLHeader` - Header containing TTL value
+
+### Validation
+- `RequiredHeaders` - Headers that must be present
+- `DisallowedHeaders` - Headers that are not allowed
+- `ValidateHeaders` - Header validation rules
+- `ValidateAuthAppID` - Enable app ID validation
+- `ValidateAuthAppIDUrl` - URL for app ID validation
+- `ValidateAuthAppFieldName` - Field name for app ID
+- `ValidateAuthAppIDHeader` - Header containing app ID
+
+### User Management
+- `UserConfigUrl` - URL for user configuration
+- `SuspendedUserConfigUrl` - URL for suspended users
+- `UserProfileHeader` - Header containing user profile
+- `UserIDFieldName` - Field name for user ID
+- `UniqueUserHeaders` - Headers that identify unique users
+- `UserPriorityThreshold` - Priority threshold for users
+
+### Priority
+- `PriorityKeyHeader` - Header containing priority key
+- `PriorityKeys` - List of priority keys
+- `PriorityValues` - Priority values for each key
+
+### Response Handling
+- `AcceptableStatusCodes` - Status codes to accept
+- `StripResponseHeaders` - Headers to remove from response
+- `StripRequestHeaders` - Headers to remove from request
+
+### Async Settings (timing only)
+- `AsyncTimeout` - Async operation timeout
+- `AsyncTTLSecs` - Async TTL in seconds
+- `AsyncTriggerTimeout` - Trigger timeout
+- `AsyncClientRequestHeader` - Client async header
+- `AsyncClientConfigFieldName` - Config field name
+
+## Monitoring Refresh
+
+The proxy logs configuration refresh activity:
+
+```
+[CONFIG] ✓ Azure App Configuration initialized with Warm settings refresh
+[CONFIG] Azure App Configuration refresh service started with 30s interval
+[CONFIG] Configuration refresh check completed - changes detected
+[CONFIG] Warm settings changed - applying to BackendOptions
+[CONFIG] ✓ Warm settings applied successfully
+```
+
+## Troubleshooting
+
+### Settings Not Refreshing
+
+1. Check the sentinel key was updated:
+   ```bash
+   az appconfig kv show --name appconfig-proxy --key "Warm:Sentinel"
+   ```
+
+2. Verify the label filter matches:
+   ```bash
+   az appconfig kv list --name appconfig-proxy --label Production
+   ```
+
+3. Check proxy logs for refresh errors
+
+### Authentication Failures
+
+1. Verify managed identity is enabled on the Container App
+2. Check role assignment is correct (App Configuration Data Reader)
+3. Ensure the endpoint URL is correct
+
+### Performance Considerations
+
+- Default refresh interval is 30 seconds
+- Only the sentinel key is checked on each poll
+- Full refresh only occurs when sentinel changes
+- Consider longer intervals for production (60-120 seconds)
+
+## Example: Changing MaxAttempts at Runtime
+
+```bash
+# Update the setting
+az appconfig kv set \
+  --name appconfig-proxy \
+  --key "Warm:MaxAttempts" \
+  --value "5" \
+  --label Production
+
+# Trigger refresh by updating sentinel
+az appconfig kv set \
+  --name appconfig-proxy \
+  --key "Warm:Sentinel" \
+  --value "$(date +%s)" \
+  --label Production
+```
+
+Within 30 seconds (or your configured interval), all proxy instances will pick up the new value without restart.