From 99d96dfca980aedf5cebd02a73f9abe46393c00c Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Wed, 8 Apr 2026 00:33:09 -0300 Subject: [PATCH 1/4] Add support for multiple ALBs --- CHANGELOG.md | 5 + k8s/scope/build_context | 16 +- k8s/scope/networking/resolve_balancer | 249 +++++++++ .../tests/networking/resolve_balancer.bats | 471 ++++++++++++++++++ 4 files changed, 726 insertions(+), 15 deletions(-) create mode 100755 k8s/scope/networking/resolve_balancer create mode 100644 k8s/scope/tests/networking/resolve_balancer.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 217b0a10..578eb022 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add unit tests for **k8s/backup** module (backup_templates and s3 operations) - Add ALB capacity validation on scope creation. Requires additional AWS permissions: `elasticloadbalancing:DescribeLoadBalancers`, `elasticloadbalancing:DescribeListeners`, `elasticloadbalancing:DescribeRules` - Add ALB target group capacity validation on deployment. Requires additional AWS permission: `elasticloadbalancing:DescribeTargetGroups` +- Extract ALB resolution into dedicated **k8s/scope/networking/resolve_balancer** script +- Add support for distributing scopes across multiple ALBs via `additional_public_balancers` and `additional_private_balancers` provider configuration (selects the ALB with fewest listener rules) +- Ensure deployment-time ALB consistency with DNS by looking up the existing ALB from K8s ingress or Route53 record instead of recalculating +- Add 25 BATS tests for the resolve_balancer module +- **New AWS permission required** (only when using additional balancers): `route53:ListResourceRecordSets` — used at deployment time to look up the ALB assigned during scope creation ## [1.10.1] - 2026-02-13 - Hotfix on wait_deployment_iteration diff --git a/k8s/scope/build_context b/k8s/scope/build_context index 8174e106..8328eab6 100755 --- a/k8s/scope/build_context +++ b/k8s/scope/build_context @@ -208,21 +208,7 @@ K8S_MODIFIERS=$(get_config_value \ ) K8S_MODIFIERS=$(echo "$K8S_MODIFIERS" | jq .) -ALB_NAME="k8s-nullplatform-$INGRESS_VISIBILITY" - -if [ "$INGRESS_VISIBILITY" = "internet-facing" ]; then - ALB_NAME=$(get_config_value \ - --provider '.providers["scope-configurations"].networking.balancer_public_name' \ - --provider '.providers["container-orchestration"].balancer.public_name' \ - --default "$ALB_NAME" - ) -else - ALB_NAME=$(get_config_value \ - --provider '.providers["scope-configurations"].networking.balancer_private_name' \ - --provider '.providers["container-orchestration"].balancer.private_name' \ - --default "$ALB_NAME" - ) -fi +source "$SCRIPT_DIR/networking/resolve_balancer" NAMESPACE_SLUG=$(echo "$CONTEXT" | jq -r .namespace.slug) APPLICATION_SLUG=$(echo "$CONTEXT" | jq -r .application.slug) diff --git a/k8s/scope/networking/resolve_balancer b/k8s/scope/networking/resolve_balancer new file mode 100755 index 00000000..feef6789 --- /dev/null +++ b/k8s/scope/networking/resolve_balancer @@ -0,0 +1,249 @@ +#!/bin/bash + +# Resolves the ALB name to use for the scope's ingress. +# +# When additional balancers are configured, queries AWS to find the ALB +# with the fewest listener rules, distributing scopes across ALBs evenly. +# +# At deployment time, instead of recalculating, the script looks up the ALB +# already assigned to this scope (via existing K8s ingress or Route53 record) +# to ensure consistency between DNS and ingress routing. +# +# Inputs (env vars): +# INGRESS_VISIBILITY - "internet-facing" or "internal" +# CONTEXT - JSON with provider configuration +# REGION - AWS region (for elbv2 API calls) +# +# Outputs (env vars): +# ALB_NAME - The resolved ALB name to use + +_RESOLVE_BALANCER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if ! type -t log >/dev/null 2>&1; then + source "$_RESOLVE_BALANCER_DIR/../../logging" +fi + +if ! type -t get_config_value >/dev/null 2>&1; then + source "$_RESOLVE_BALANCER_DIR/../../utils/get_config_value" +fi + +# Queries AWS ELBv2 to count listener rules on an ALB's HTTPS (443) listener. +# The default rule is excluded since it always exists. +# Usage: get_alb_rule_count +# Returns: integer rule count on stdout, non-zero exit on failure +if ! type -t get_alb_rule_count >/dev/null 2>&1; then +get_alb_rule_count() { + local alb_name="$1" + + local alb_arn + alb_arn=$(aws elbv2 describe-load-balancers \ + --names "$alb_name" \ + --region "$REGION" \ + --query 'LoadBalancers[0].LoadBalancerArn' \ + --output text 2>/dev/null) || return 1 + + if [ -z "$alb_arn" ] || [ "$alb_arn" = "None" ]; then + return 1 + fi + + local listener_arn + listener_arn=$(aws elbv2 describe-listeners \ + --load-balancer-arn "$alb_arn" \ + --region "$REGION" \ + --query 'Listeners[?Port==`443`].ListenerArn | [0]' \ + --output text 2>/dev/null) || return 1 + + if [ -z "$listener_arn" ] || [ "$listener_arn" = "None" ]; then + return 1 + fi + + local rules_json + rules_json=$(aws elbv2 describe-rules \ + --listener-arn "$listener_arn" \ + --region "$REGION" \ + --output json 2>/dev/null) || return 1 + + # Exclude the default rule (always present, not a routing rule) + echo "$rules_json" | jq '[.Rules[] | select(.IsDefault == false)] | length' +} +fi + +# Looks up the ALB name from an existing Kubernetes ingress for this scope. +# Returns the ALB name on stdout, or non-zero exit if no ingress found. +if ! type -t get_alb_from_ingress >/dev/null 2>&1; then +get_alb_from_ingress() { + local scope_id="$1" + local namespace="$2" + + local alb + alb=$(kubectl get ingress -l "scope_id=$scope_id" -n "$namespace" \ + -o jsonpath='{.items[0].metadata.annotations.alb\.ingress\.kubernetes\.io/load-balancer-name}' 2>/dev/null) || return 1 + + if [ -z "$alb" ] || [ "$alb" = "null" ]; then + return 1 + fi + + echo "$alb" +} +fi + +# Looks up the Route53 A-record alias for the scope domain, then resolves +# the ALB DNS name back to an ALB name. +# Returns the ALB name on stdout, or non-zero exit if not found. +if ! type -t get_alb_from_route53 >/dev/null 2>&1; then +get_alb_from_route53() { + local domain="$1" + local region="$2" + + local zone_id + zone_id=$(echo "$CONTEXT" | jq -r ' + .providers["cloud-providers"].networking.hosted_public_zone_id // + .providers["cloud-providers"].networking.hosted_zone_id // + empty + ') || return 1 + + if [ -z "$zone_id" ]; then + return 1 + fi + + local alias_dns + alias_dns=$(aws route53 list-resource-record-sets \ + --hosted-zone-id "$zone_id" \ + --query "ResourceRecordSets[?Name=='${domain}.' && Type=='A'].AliasTarget.DNSName | [0]" \ + --output text \ + --region "$region" 2>/dev/null) || return 1 + + if [ -z "$alias_dns" ] || [ "$alias_dns" = "None" ]; then + return 1 + fi + + # Strip trailing dot from DNS name + alias_dns="${alias_dns%.}" + + # Reverse-lookup: find ALB name from its DNS name + local alb_name + alb_name=$(aws elbv2 describe-load-balancers \ + --query "LoadBalancers[?DNSName=='${alias_dns}'].LoadBalancerName | [0]" \ + --output text \ + --region "$region" 2>/dev/null) || return 1 + + if [ -z "$alb_name" ] || [ "$alb_name" = "None" ]; then + return 1 + fi + + echo "$alb_name" +} +fi + +# ============================================================================= +# Main logic +# ============================================================================= + +# Resolve the base ALB name from configuration +ALB_NAME="k8s-nullplatform-$INGRESS_VISIBILITY" + +if [ "$INGRESS_VISIBILITY" = "internet-facing" ]; then + ALB_NAME=$(get_config_value \ + --provider '.providers["scope-configurations"].networking.balancer_public_name' \ + --provider '.providers["container-orchestration"].balancer.public_name' \ + --default "$ALB_NAME" + ) +else + ALB_NAME=$(get_config_value \ + --provider '.providers["scope-configurations"].networking.balancer_private_name' \ + --provider '.providers["container-orchestration"].balancer.private_name' \ + --default "$ALB_NAME" + ) +fi + +# Check for additional balancers +ADDITIONAL_BALANCERS="" +if [ "$INGRESS_VISIBILITY" = "internet-facing" ]; then + ADDITIONAL_BALANCERS=$(get_config_value \ + --provider '.providers["scope-configurations"].networking.additional_public_balancers' \ + --default "" + ) +else + ADDITIONAL_BALANCERS=$(get_config_value \ + --provider '.providers["scope-configurations"].networking.additional_private_balancers' \ + --default "" + ) +fi + +# When additional balancers are configured, resolve which ALB to use +if [ -n "$ADDITIONAL_BALANCERS" ] && [ "$ADDITIONAL_BALANCERS" != "null" ] && [ "$ADDITIONAL_BALANCERS" != "[]" ]; then + + RESOLVED=false + + # At deployment time, look up the ALB already assigned to this scope + # to ensure consistency with the Route53 record created at scope time. + DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.deployment.id // empty') + + if [ -n "$DEPLOYMENT_ID" ]; then + SCOPE_ID=$(echo "$CONTEXT" | jq -r '.scope.id') + K8S_NS=$(echo "$CONTEXT" | jq -r ' + .providers["scope-configurations"].cluster.namespace // + .providers["container-orchestration"].cluster.namespace // + "nullplatform" + ') + SCOPE_DOMAIN_VAL=$(echo "$CONTEXT" | jq -r '.scope.domain') + + EXISTING_ALB="" + + # Try ingress first (fastest, covers 2nd+ deployments) + EXISTING_ALB=$(get_alb_from_ingress "$SCOPE_ID" "$K8S_NS" 2>/dev/null) || true + + if [ -n "$EXISTING_ALB" ]; then + log debug "📋 Found ALB '$EXISTING_ALB' from existing ingress for scope $SCOPE_ID" + else + # Fall back to Route53 record (covers first deployment after scope creation) + EXISTING_ALB=$(get_alb_from_route53 "$SCOPE_DOMAIN_VAL" "$REGION" 2>/dev/null) || true + + if [ -n "$EXISTING_ALB" ]; then + log debug "📋 Found ALB '$EXISTING_ALB' from Route53 record for $SCOPE_DOMAIN_VAL" + fi + fi + + if [ -n "$EXISTING_ALB" ]; then + log info "📝 Using existing ALB '$EXISTING_ALB' (consistent with DNS)" + ALB_NAME="$EXISTING_ALB" + RESOLVED=true + else + log warn "⚠️ Could not determine existing ALB from infrastructure, recalculating" + fi + fi + + # Scope creation time (or deployment fallback): pick the ALB with fewest rules + if [ "$RESOLVED" = false ]; then + log debug "🔍 Additional balancers configured, resolving least-loaded ALB..." + + CANDIDATES=$(echo "$ADDITIONAL_BALANCERS" | jq -r --arg base "$ALB_NAME" '[$base] + . | .[]') + + log debug "📋 Candidate balancers: $(echo "$CANDIDATES" | paste -sd ',' - | sed 's/,/, /g')" + + MIN_RULES=-1 + BEST_ALB="$ALB_NAME" + + for CANDIDATE in $CANDIDATES; do + RULE_COUNT=$(get_alb_rule_count "$CANDIDATE" 2>/dev/null) || { + log warn "⚠️ Could not query rules for ALB '$CANDIDATE', skipping" + continue + } + + log debug "📋 ALB '$CANDIDATE': $RULE_COUNT rules" + + if [ "$MIN_RULES" -eq -1 ] || [ "$RULE_COUNT" -lt "$MIN_RULES" ]; then + MIN_RULES=$RULE_COUNT + BEST_ALB="$CANDIDATE" + fi + done + + if [ "$BEST_ALB" != "$ALB_NAME" ]; then + log info "📝 Selected ALB '$BEST_ALB' ($MIN_RULES rules) over default '$ALB_NAME'" + fi + + ALB_NAME="$BEST_ALB" + fi +fi + +export ALB_NAME diff --git a/k8s/scope/tests/networking/resolve_balancer.bats b/k8s/scope/tests/networking/resolve_balancer.bats new file mode 100644 index 00000000..e4581125 --- /dev/null +++ b/k8s/scope/tests/networking/resolve_balancer.bats @@ -0,0 +1,471 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for networking/resolve_balancer +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + source "$PROJECT_ROOT/k8s/utils/get_config_value" + + export SCRIPT="$PROJECT_ROOT/k8s/scope/networking/resolve_balancer" + export REGION="us-east-1" + + # Default: no aws/kubectl commands needed (no additional balancers) + aws() { return 1; } + export -f aws + kubectl() { return 1; } + export -f kubectl + + # Base CONTEXT — scope creation (no deployment field) + export CONTEXT='{ + "scope": { + "id": "test-scope-123", + "domain": "test.nullapps.io" + }, + "providers": { + "scope-configurations": {}, + "cloud-providers": { + "networking": { + "hosted_public_zone_id": "Z1234567890", + "hosted_zone_id": "Z0987654321" + } + }, + "container-orchestration": { + "cluster": { + "namespace": "test-ns" + }, + "balancer": { + "public_name": "co-balancer-public", + "private_name": "co-balancer-private" + } + } + } + }' +} + +teardown() { + unset -f aws + unset -f kubectl + unset -f log + unset -f get_config_value + unset -f get_alb_rule_count + unset -f get_alb_from_ingress + unset -f get_alb_from_route53 + unset ALB_NAME + unset ADDITIONAL_BALANCERS +} + +# Helper: add deployment field to CONTEXT (makes it a deployment context) +add_deployment_context() { + export CONTEXT=$(echo "$CONTEXT" | jq '. + {deployment: {id: "deploy-456"}}') +} + +# ============================================================================= +# Default ALB name (no provider config) +# ============================================================================= +@test "resolve_balancer: uses default ALB name when no provider config (public)" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT='{ "providers": {} }' + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "k8s-nullplatform-internet-facing" +} + +@test "resolve_balancer: uses default ALB name when no provider config (private)" { + export INGRESS_VISIBILITY="internal" + export CONTEXT='{ "providers": {} }' + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "k8s-nullplatform-internal" +} + +# ============================================================================= +# Provider overrides - container-orchestration +# ============================================================================= +@test "resolve_balancer: resolves public ALB from container-orchestration provider" { + export INGRESS_VISIBILITY="internet-facing" + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "co-balancer-public" +} + +@test "resolve_balancer: resolves private ALB from container-orchestration provider" { + export INGRESS_VISIBILITY="internal" + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "co-balancer-private" +} + +# ============================================================================= +# Provider overrides - scope-configurations takes priority +# ============================================================================= +@test "resolve_balancer: scope-configurations overrides container-orchestration (public)" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq '.providers["scope-configurations"].networking.balancer_public_name = "scope-alb-public"') + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "scope-alb-public" +} + +@test "resolve_balancer: scope-configurations overrides container-orchestration (private)" { + export INGRESS_VISIBILITY="internal" + export CONTEXT=$(echo "$CONTEXT" | jq '.providers["scope-configurations"].networking.balancer_private_name = "scope-alb-private"') + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "scope-alb-private" +} + +# ============================================================================= +# Scope creation: additional balancers — least-loaded selection +# ============================================================================= +@test "resolve_balancer: selects ALB with fewest rules from candidates (public)" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] + ') + + get_alb_rule_count() { + case "$1" in + co-balancer-public) echo "45" ;; + alb-extra-1) echo "12" ;; + alb-extra-2) echo "30" ;; + esac + } + export -f get_alb_rule_count + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "alb-extra-1" +} + +@test "resolve_balancer: selects ALB with fewest rules from candidates (private)" { + export INGRESS_VISIBILITY="internal" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_private_balancers = ["alb-priv-extra-1", "alb-priv-extra-2"] + ') + + get_alb_rule_count() { + case "$1" in + co-balancer-private) echo "50" ;; + alb-priv-extra-1) echo "20" ;; + alb-priv-extra-2) echo "5" ;; + esac + } + export -f get_alb_rule_count + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "alb-priv-extra-2" +} + +@test "resolve_balancer: keeps default ALB when it has fewest rules" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + + get_alb_rule_count() { + case "$1" in + co-balancer-public) echo "5" ;; + alb-extra-1) echo "30" ;; + esac + } + export -f get_alb_rule_count + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "co-balancer-public" +} + +@test "resolve_balancer: logs selected ALB when different from default" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + + get_alb_rule_count() { + case "$1" in + co-balancer-public) echo "45" ;; + alb-extra-1) echo "12" ;; + esac + } + export -f get_alb_rule_count + + run bash -c 'source "$SCRIPT"' + + assert_contains "$output" "📝 Selected ALB 'alb-extra-1' (12 rules) over default 'co-balancer-public'" +} + +@test "resolve_balancer: logs candidate balancers list" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] + ') + + get_alb_rule_count() { echo "10"; } + export -f get_alb_rule_count + + run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"' + + assert_contains "$output" "🔍 Additional balancers configured, resolving least-loaded ALB..." + assert_contains "$output" "📋 Candidate balancers: co-balancer-public, alb-extra-1, alb-extra-2" +} + +# ============================================================================= +# AWS API failure handling +# ============================================================================= +@test "resolve_balancer: skips candidate when AWS query fails" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] + ') + + get_alb_rule_count() { + case "$1" in + co-balancer-public) echo "45" ;; + alb-extra-1) return 1 ;; + alb-extra-2) echo "20" ;; + esac + } + export -f get_alb_rule_count + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "alb-extra-2" +} + +@test "resolve_balancer: warns when a candidate fails" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-broken"] + ') + + get_alb_rule_count() { + case "$1" in + co-balancer-public) echo "10" ;; + alb-broken) return 1 ;; + esac + } + export -f get_alb_rule_count + + run bash -c 'source "$SCRIPT"' + + assert_contains "$output" "⚠️ Could not query rules for ALB 'alb-broken', skipping" +} + +@test "resolve_balancer: keeps default when all candidates fail" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-broken-1", "alb-broken-2"] + ') + + get_alb_rule_count() { return 1; } + export -f get_alb_rule_count + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "co-balancer-public" +} + +# ============================================================================= +# No additional balancers — no AWS calls +# ============================================================================= +@test "resolve_balancer: does not call AWS when no additional balancers configured" { + export INGRESS_VISIBILITY="internet-facing" + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "co-balancer-public" +} + +@test "resolve_balancer: handles empty additional balancers array gracefully" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = [] + ') + + get_alb_rule_count() { echo "10"; } + export -f get_alb_rule_count + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "co-balancer-public" +} + +# ============================================================================= +# Tie-breaking: first candidate with fewest rules wins +# ============================================================================= +@test "resolve_balancer: picks first candidate on tie" { + export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] + ') + + get_alb_rule_count() { + case "$1" in + co-balancer-public) echo "10" ;; + alb-extra-1) echo "10" ;; + alb-extra-2) echo "10" ;; + esac + } + export -f get_alb_rule_count + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "co-balancer-public" +} + +# ============================================================================= +# Deployment time: ALB lookup from existing infrastructure +# ============================================================================= +@test "resolve_balancer: deployment uses ALB from existing ingress" { + export INGRESS_VISIBILITY="internet-facing" + add_deployment_context + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + + get_alb_from_ingress() { echo "alb-extra-1"; } + export -f get_alb_from_ingress + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "alb-extra-1" +} + +@test "resolve_balancer: deployment falls back to Route53 when no ingress exists" { + export INGRESS_VISIBILITY="internet-facing" + add_deployment_context + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + + get_alb_from_ingress() { return 1; } + export -f get_alb_from_ingress + get_alb_from_route53() { echo "alb-extra-1"; } + export -f get_alb_from_route53 + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "alb-extra-1" +} + +@test "resolve_balancer: deployment falls back to calculation when no infrastructure found" { + export INGRESS_VISIBILITY="internet-facing" + add_deployment_context + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + + get_alb_from_ingress() { return 1; } + export -f get_alb_from_ingress + get_alb_from_route53() { return 1; } + export -f get_alb_from_route53 + get_alb_rule_count() { + case "$1" in + co-balancer-public) echo "45" ;; + alb-extra-1) echo "10" ;; + esac + } + export -f get_alb_rule_count + + source "$SCRIPT" + + assert_equal "$ALB_NAME" "alb-extra-1" +} + +@test "resolve_balancer: deployment logs when using ALB from ingress" { + export INGRESS_VISIBILITY="internet-facing" + add_deployment_context + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + + get_alb_from_ingress() { echo "alb-from-ingress"; } + export -f get_alb_from_ingress + + run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"' + + assert_contains "$output" "📋 Found ALB 'alb-from-ingress' from existing ingress for scope test-scope-123" + assert_contains "$output" "📝 Using existing ALB 'alb-from-ingress' (consistent with DNS)" +} + +@test "resolve_balancer: deployment logs when using ALB from Route53" { + export INGRESS_VISIBILITY="internet-facing" + add_deployment_context + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + + get_alb_from_ingress() { return 1; } + export -f get_alb_from_ingress + get_alb_from_route53() { echo "alb-from-dns"; } + export -f get_alb_from_route53 + + run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"' + + assert_contains "$output" "📋 Found ALB 'alb-from-dns' from Route53 record for test.nullapps.io" + assert_contains "$output" "📝 Using existing ALB 'alb-from-dns' (consistent with DNS)" +} + +@test "resolve_balancer: deployment warns when falling back to calculation" { + export INGRESS_VISIBILITY="internet-facing" + add_deployment_context + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + + get_alb_from_ingress() { return 1; } + export -f get_alb_from_ingress + get_alb_from_route53() { return 1; } + export -f get_alb_from_route53 + get_alb_rule_count() { echo "10"; } + export -f get_alb_rule_count + + run bash -c 'source "$SCRIPT"' + + assert_contains "$output" "⚠️ Could not determine existing ALB from infrastructure, recalculating" +} + +@test "resolve_balancer: scope creation always calculates even when ingress exists" { + export INGRESS_VISIBILITY="internet-facing" + # No deployment in context — this is scope creation + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + ') + + # Even though ingress mock returns a value, scope creation should calculate + get_alb_from_ingress() { echo "old-alb-from-ingress"; } + export -f get_alb_from_ingress + get_alb_rule_count() { + case "$1" in + co-balancer-public) echo "45" ;; + alb-extra-1) echo "10" ;; + esac + } + export -f get_alb_rule_count + + source "$SCRIPT" + + # Should use the calculation result, not the ingress value + assert_equal "$ALB_NAME" "alb-extra-1" +} + +@test "resolve_balancer: deployment without additional balancers uses base ALB" { + export INGRESS_VISIBILITY="internet-facing" + add_deployment_context + + source "$SCRIPT" + + # No additional balancers = no lookup needed, uses base ALB + assert_equal "$ALB_NAME" "co-balancer-public" +} From a2e5b7fbf46700cd047b4e7a0c2fad91e7ba768e Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Thu, 9 Apr 2026 09:52:06 -0300 Subject: [PATCH 2/4] Simplify resolve_balancer and add container-orchestration provider --- CHANGELOG.md | 1 + k8s/scope/networking/resolve_balancer | 116 ++--- .../tests/networking/resolve_balancer.bats | 420 +++++++++--------- 3 files changed, 235 insertions(+), 302 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 578eb022..484a128d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ensure deployment-time ALB consistency with DNS by looking up the existing ALB from K8s ingress or Route53 record instead of recalculating - Add 25 BATS tests for the resolve_balancer module - **New AWS permission required** (only when using additional balancers): `route53:ListResourceRecordSets` — used at deployment time to look up the ALB assigned during scope creation +- Add support for multiple ALBs ## [1.10.1] - 2026-02-13 - Hotfix on wait_deployment_iteration diff --git a/k8s/scope/networking/resolve_balancer b/k8s/scope/networking/resolve_balancer index feef6789..a9a61628 100755 --- a/k8s/scope/networking/resolve_balancer +++ b/k8s/scope/networking/resolve_balancer @@ -2,12 +2,13 @@ # Resolves the ALB name to use for the scope's ingress. # -# When additional balancers are configured, queries AWS to find the ALB -# with the fewest listener rules, distributing scopes across ALBs evenly. -# -# At deployment time, instead of recalculating, the script looks up the ALB -# already assigned to this scope (via existing K8s ingress or Route53 record) -# to ensure consistency between DNS and ingress routing. +# Resolution priority: +# 1. Route53 — if a DNS record already exists for the scope domain, +# use the ALB it points to (ensures DNS/ingress consistency) +# 2. Load balancing — when additional balancers are configured, pick +# the ALB with the fewest HTTPS listener rules +# 3. Provider config — base ALB from scope-configurations or +# container-orchestration provider # # Inputs (env vars): # INGRESS_VISIBILITY - "internet-facing" or "internal" @@ -31,7 +32,6 @@ fi # The default rule is excluded since it always exists. # Usage: get_alb_rule_count # Returns: integer rule count on stdout, non-zero exit on failure -if ! type -t get_alb_rule_count >/dev/null 2>&1; then get_alb_rule_count() { local alb_name="$1" @@ -66,31 +66,10 @@ get_alb_rule_count() { # Exclude the default rule (always present, not a routing rule) echo "$rules_json" | jq '[.Rules[] | select(.IsDefault == false)] | length' } -fi - -# Looks up the ALB name from an existing Kubernetes ingress for this scope. -# Returns the ALB name on stdout, or non-zero exit if no ingress found. -if ! type -t get_alb_from_ingress >/dev/null 2>&1; then -get_alb_from_ingress() { - local scope_id="$1" - local namespace="$2" - - local alb - alb=$(kubectl get ingress -l "scope_id=$scope_id" -n "$namespace" \ - -o jsonpath='{.items[0].metadata.annotations.alb\.ingress\.kubernetes\.io/load-balancer-name}' 2>/dev/null) || return 1 - - if [ -z "$alb" ] || [ "$alb" = "null" ]; then - return 1 - fi - - echo "$alb" -} -fi # Looks up the Route53 A-record alias for the scope domain, then resolves # the ALB DNS name back to an ALB name. # Returns the ALB name on stdout, or non-zero exit if not found. -if ! type -t get_alb_from_route53 >/dev/null 2>&1; then get_alb_from_route53() { local domain="$1" local region="$2" @@ -133,7 +112,6 @@ get_alb_from_route53() { echo "$alb_name" } -fi # ============================================================================= # Main logic @@ -156,65 +134,35 @@ else ) fi -# Check for additional balancers -ADDITIONAL_BALANCERS="" -if [ "$INGRESS_VISIBILITY" = "internet-facing" ]; then - ADDITIONAL_BALANCERS=$(get_config_value \ - --provider '.providers["scope-configurations"].networking.additional_public_balancers' \ - --default "" - ) -else - ADDITIONAL_BALANCERS=$(get_config_value \ - --provider '.providers["scope-configurations"].networking.additional_private_balancers' \ - --default "" - ) -fi - -# When additional balancers are configured, resolve which ALB to use -if [ -n "$ADDITIONAL_BALANCERS" ] && [ "$ADDITIONAL_BALANCERS" != "null" ] && [ "$ADDITIONAL_BALANCERS" != "[]" ]; then - - RESOLVED=false - - # At deployment time, look up the ALB already assigned to this scope - # to ensure consistency with the Route53 record created at scope time. - DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.deployment.id // empty') - - if [ -n "$DEPLOYMENT_ID" ]; then - SCOPE_ID=$(echo "$CONTEXT" | jq -r '.scope.id') - K8S_NS=$(echo "$CONTEXT" | jq -r ' - .providers["scope-configurations"].cluster.namespace // - .providers["container-orchestration"].cluster.namespace // - "nullplatform" - ') - SCOPE_DOMAIN_VAL=$(echo "$CONTEXT" | jq -r '.scope.domain') - - EXISTING_ALB="" +# Priority 1: Check Route53 for an existing DNS record +SCOPE_DOMAIN_VAL=$(echo "$CONTEXT" | jq -r '.scope.domain // empty') +EXISTING_ALB="" - # Try ingress first (fastest, covers 2nd+ deployments) - EXISTING_ALB=$(get_alb_from_ingress "$SCOPE_ID" "$K8S_NS" 2>/dev/null) || true - - if [ -n "$EXISTING_ALB" ]; then - log debug "📋 Found ALB '$EXISTING_ALB' from existing ingress for scope $SCOPE_ID" - else - # Fall back to Route53 record (covers first deployment after scope creation) - EXISTING_ALB=$(get_alb_from_route53 "$SCOPE_DOMAIN_VAL" "$REGION" 2>/dev/null) || true - - if [ -n "$EXISTING_ALB" ]; then - log debug "📋 Found ALB '$EXISTING_ALB' from Route53 record for $SCOPE_DOMAIN_VAL" - fi - fi +if [ -n "$SCOPE_DOMAIN_VAL" ]; then + EXISTING_ALB=$(get_alb_from_route53 "$SCOPE_DOMAIN_VAL" "$REGION" 2>/dev/null) || true +fi - if [ -n "$EXISTING_ALB" ]; then - log info "📝 Using existing ALB '$EXISTING_ALB' (consistent with DNS)" - ALB_NAME="$EXISTING_ALB" - RESOLVED=true - else - log warn "⚠️ Could not determine existing ALB from infrastructure, recalculating" - fi +if [ -n "$EXISTING_ALB" ]; then + log info "📝 Using ALB '$EXISTING_ALB' from Route53 record for $SCOPE_DOMAIN_VAL" + ALB_NAME="$EXISTING_ALB" +else + # Priority 2: If additional balancers configured, pick the least-loaded one + ADDITIONAL_BALANCERS="" + if [ "$INGRESS_VISIBILITY" = "internet-facing" ]; then + ADDITIONAL_BALANCERS=$(get_config_value \ + --provider '.providers["scope-configurations"].networking.additional_public_balancers' \ + --provider '.providers["container-orchestration"].balancer.additional_public_names' \ + --default "" + ) + else + ADDITIONAL_BALANCERS=$(get_config_value \ + --provider '.providers["scope-configurations"].networking.additional_private_balancers' \ + --provider '.providers["container-orchestration"].balancer.additional_private_names' \ + --default "" + ) fi - # Scope creation time (or deployment fallback): pick the ALB with fewest rules - if [ "$RESOLVED" = false ]; then + if [ -n "$ADDITIONAL_BALANCERS" ] && [ "$ADDITIONAL_BALANCERS" != "null" ] && [ "$ADDITIONAL_BALANCERS" != "[]" ]; then log debug "🔍 Additional balancers configured, resolving least-loaded ALB..." CANDIDATES=$(echo "$ADDITIONAL_BALANCERS" | jq -r --arg base "$ALB_NAME" '[$base] + . | .[]') diff --git a/k8s/scope/tests/networking/resolve_balancer.bats b/k8s/scope/tests/networking/resolve_balancer.bats index e4581125..60ba2ace 100644 --- a/k8s/scope/tests/networking/resolve_balancer.bats +++ b/k8s/scope/tests/networking/resolve_balancer.bats @@ -13,13 +13,14 @@ setup() { export SCRIPT="$PROJECT_ROOT/k8s/scope/networking/resolve_balancer" export REGION="us-east-1" - # Default: no aws/kubectl commands needed (no additional balancers) + # Default: aws returns failure (no Route53 record, no ALBs) aws() { return 1; } export -f aws - kubectl() { return 1; } - export -f kubectl - # Base CONTEXT — scope creation (no deployment field) + # Temp file for tracking ALB rule counts in mocks + export MOCK_RULES_FILE="$(mktemp)" + + # Base CONTEXT export CONTEXT='{ "scope": { "id": "test-scope-123", @@ -48,19 +49,109 @@ setup() { teardown() { unset -f aws - unset -f kubectl unset -f log unset -f get_config_value - unset -f get_alb_rule_count - unset -f get_alb_from_ingress - unset -f get_alb_from_route53 + rm -f "$MOCK_RULES_FILE" unset ALB_NAME unset ADDITIONAL_BALANCERS } -# Helper: add deployment field to CONTEXT (makes it a deployment context) -add_deployment_context() { - export CONTEXT=$(echo "$CONTEXT" | jq '. + {deployment: {id: "deploy-456"}}') +# ============================================================================= +# Mock helpers +# ============================================================================= + +# Sets up aws mock that returns a Route53 record pointing to a specific ALB. +mock_route53_alb() { + local alb_name="$1" + local alb_dns="${alb_name}-123.us-east-1.elb.amazonaws.com" + + eval "aws() { + case \"\$*\" in + *list-resource-record-sets*) + echo '${alb_dns}.' + return 0 + ;; + *describe-load-balancers*) + echo '${alb_name}' + return 0 + ;; + *) + return 1 + ;; + esac + } + export -f aws" +} + +# Sets up aws mock with no Route53 record but with rule counts for ALBs. +# Write rule counts to MOCK_RULES_FILE as "alb_name count" lines. +# The mock returns the ALB ARN with the name embedded so describe-rules +# can look up the correct rule count. +mock_alb_rules() { + > "$MOCK_RULES_FILE" + for pair in "$@"; do + echo "$pair" >> "$MOCK_RULES_FILE" + done + local rules_file="$MOCK_RULES_FILE" + + eval "aws() { + case \"\$*\" in + *list-resource-record-sets*) + echo 'None' + return 0 + ;; + *describe-load-balancers*--names*) + local name='' + local prev='' + for arg in \"\$@\"; do + if [ \"\$prev\" = '--names' ]; then name=\"\$arg\"; fi + prev=\"\$arg\" + done + if ! grep -q \"^\${name} \" '${rules_file}' 2>/dev/null; then + return 1 + fi + echo \"arn:aws:elasticloadbalancing:us-east-1:123:loadbalancer/app/\${name}/abc\" + return 0 + ;; + *describe-listeners*) + local lb_arn='' + local prev='' + for arg in \"\$@\"; do + if [ \"\$prev\" = '--load-balancer-arn' ]; then lb_arn=\"\$arg\"; fi + prev=\"\$arg\" + done + local alb_name=\$(echo \"\$lb_arn\" | sed 's|.*/app/||;s|/.*||') + echo \"arn:aws:elasticloadbalancing:us-east-1:123:listener/app/\${alb_name}/abc/def\" + return 0 + ;; + *describe-rules*) + local listener_arn='' + local prev='' + for arg in \"\$@\"; do + if [ \"\$prev\" = '--listener-arn' ]; then listener_arn=\"\$arg\"; fi + prev=\"\$arg\" + done + local alb_name=\$(echo \"\$listener_arn\" | sed 's|.*/app/||;s|/.*||') + local count=\$(grep \"^\${alb_name} \" '${rules_file}' | awk '{print \$2}') + if [ -z \"\$count\" ]; then + return 1 + fi + local rules='{\"Rules\": [{\"IsDefault\": true}' + local i=0 + while [ \$i -lt \$count ]; do + rules=\"\${rules}, {\\\"IsDefault\\\": false}\" + i=\$((i + 1)) + done + rules=\"\${rules}]}\" + echo \"\$rules\" + return 0 + ;; + *) + return 1 + ;; + esac + } + export -f aws" } # ============================================================================= @@ -125,151 +216,91 @@ add_deployment_context() { } # ============================================================================= -# Scope creation: additional balancers — least-loaded selection +# Additional balancers from container-orchestration provider # ============================================================================= -@test "resolve_balancer: selects ALB with fewest rules from candidates (public)" { +@test "resolve_balancer: reads additional public balancers from container-orchestration" { export INGRESS_VISIBILITY="internet-facing" export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] + .providers["container-orchestration"].balancer.additional_public_names = ["co-extra-1"] ') - - get_alb_rule_count() { - case "$1" in - co-balancer-public) echo "45" ;; - alb-extra-1) echo "12" ;; - alb-extra-2) echo "30" ;; - esac - } - export -f get_alb_rule_count + mock_alb_rules "co-balancer-public 45" "co-extra-1 10" source "$SCRIPT" - assert_equal "$ALB_NAME" "alb-extra-1" + assert_equal "$ALB_NAME" "co-extra-1" } -@test "resolve_balancer: selects ALB with fewest rules from candidates (private)" { +@test "resolve_balancer: reads additional private balancers from container-orchestration" { export INGRESS_VISIBILITY="internal" export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_private_balancers = ["alb-priv-extra-1", "alb-priv-extra-2"] + .providers["container-orchestration"].balancer.additional_private_names = ["co-priv-extra-1"] ') - - get_alb_rule_count() { - case "$1" in - co-balancer-private) echo "50" ;; - alb-priv-extra-1) echo "20" ;; - alb-priv-extra-2) echo "5" ;; - esac - } - export -f get_alb_rule_count + mock_alb_rules "co-balancer-private 45" "co-priv-extra-1 10" source "$SCRIPT" - assert_equal "$ALB_NAME" "alb-priv-extra-2" + assert_equal "$ALB_NAME" "co-priv-extra-1" } -@test "resolve_balancer: keeps default ALB when it has fewest rules" { +@test "resolve_balancer: scope-configurations additional balancers override container-orchestration" { export INGRESS_VISIBILITY="internet-facing" export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + .providers["scope-configurations"].networking.additional_public_balancers = ["scope-extra-1"] | + .providers["container-orchestration"].balancer.additional_public_names = ["co-extra-1"] ') - - get_alb_rule_count() { - case "$1" in - co-balancer-public) echo "5" ;; - alb-extra-1) echo "30" ;; - esac - } - export -f get_alb_rule_count + mock_alb_rules "co-balancer-public 45" "scope-extra-1 10" "co-extra-1 5" source "$SCRIPT" - assert_equal "$ALB_NAME" "co-balancer-public" + # scope-configurations wins — co-extra-1 is not even a candidate + assert_equal "$ALB_NAME" "scope-extra-1" } -@test "resolve_balancer: logs selected ALB when different from default" { +# ============================================================================= +# Priority 1: Route53 lookup takes precedence over everything +# ============================================================================= +@test "resolve_balancer: uses ALB from Route53 when record exists" { export INGRESS_VISIBILITY="internet-facing" - export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] - ') - - get_alb_rule_count() { - case "$1" in - co-balancer-public) echo "45" ;; - alb-extra-1) echo "12" ;; - esac - } - export -f get_alb_rule_count + mock_route53_alb "alb-from-dns" - run bash -c 'source "$SCRIPT"' + source "$SCRIPT" - assert_contains "$output" "📝 Selected ALB 'alb-extra-1' (12 rules) over default 'co-balancer-public'" + assert_equal "$ALB_NAME" "alb-from-dns" } -@test "resolve_balancer: logs candidate balancers list" { +@test "resolve_balancer: Route53 takes priority over additional balancers config" { export INGRESS_VISIBILITY="internet-facing" export CONTEXT=$(echo "$CONTEXT" | jq ' .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] ') + mock_route53_alb "alb-from-dns" - get_alb_rule_count() { echo "10"; } - export -f get_alb_rule_count - - run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"' + source "$SCRIPT" - assert_contains "$output" "🔍 Additional balancers configured, resolving least-loaded ALB..." - assert_contains "$output" "📋 Candidate balancers: co-balancer-public, alb-extra-1, alb-extra-2" + assert_equal "$ALB_NAME" "alb-from-dns" } -# ============================================================================= -# AWS API failure handling -# ============================================================================= -@test "resolve_balancer: skips candidate when AWS query fails" { +@test "resolve_balancer: Route53 takes priority over provider config" { export INGRESS_VISIBILITY="internet-facing" - export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] - ') - - get_alb_rule_count() { - case "$1" in - co-balancer-public) echo "45" ;; - alb-extra-1) return 1 ;; - alb-extra-2) echo "20" ;; - esac - } - export -f get_alb_rule_count + export CONTEXT=$(echo "$CONTEXT" | jq '.providers["scope-configurations"].networking.balancer_public_name = "scope-alb-public"') + mock_route53_alb "alb-from-dns" source "$SCRIPT" - assert_equal "$ALB_NAME" "alb-extra-2" + assert_equal "$ALB_NAME" "alb-from-dns" } -@test "resolve_balancer: warns when a candidate fails" { +@test "resolve_balancer: logs when using Route53 ALB" { export INGRESS_VISIBILITY="internet-facing" - export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-broken"] - ') - - get_alb_rule_count() { - case "$1" in - co-balancer-public) echo "10" ;; - alb-broken) return 1 ;; - esac - } - export -f get_alb_rule_count + mock_route53_alb "alb-from-dns" run bash -c 'source "$SCRIPT"' - assert_contains "$output" "⚠️ Could not query rules for ALB 'alb-broken', skipping" + assert_contains "$output" "📝 Using ALB 'alb-from-dns' from Route53 record for test.nullapps.io" } -@test "resolve_balancer: keeps default when all candidates fail" { +@test "resolve_balancer: falls through to config when Route53 has no record" { export INGRESS_VISIBILITY="internet-facing" - export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-broken-1", "alb-broken-2"] - ') - - get_alb_rule_count() { return 1; } - export -f get_alb_rule_count source "$SCRIPT" @@ -277,195 +308,148 @@ add_deployment_context() { } # ============================================================================= -# No additional balancers — no AWS calls +# Priority 2: additional balancers — least-loaded selection # ============================================================================= -@test "resolve_balancer: does not call AWS when no additional balancers configured" { +@test "resolve_balancer: selects ALB with fewest rules from candidates (public)" { export INGRESS_VISIBILITY="internet-facing" + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] + ') + mock_alb_rules "co-balancer-public 45" "alb-extra-1 12" "alb-extra-2 30" source "$SCRIPT" - assert_equal "$ALB_NAME" "co-balancer-public" + assert_equal "$ALB_NAME" "alb-extra-1" } -@test "resolve_balancer: handles empty additional balancers array gracefully" { - export INGRESS_VISIBILITY="internet-facing" +@test "resolve_balancer: selects ALB with fewest rules from candidates (private)" { + export INGRESS_VISIBILITY="internal" export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = [] + .providers["scope-configurations"].networking.additional_private_balancers = ["alb-priv-extra-1", "alb-priv-extra-2"] ') - - get_alb_rule_count() { echo "10"; } - export -f get_alb_rule_count + mock_alb_rules "co-balancer-private 50" "alb-priv-extra-1 20" "alb-priv-extra-2 5" source "$SCRIPT" - assert_equal "$ALB_NAME" "co-balancer-public" + assert_equal "$ALB_NAME" "alb-priv-extra-2" } -# ============================================================================= -# Tie-breaking: first candidate with fewest rules wins -# ============================================================================= -@test "resolve_balancer: picks first candidate on tie" { +@test "resolve_balancer: keeps default ALB when it has fewest rules" { export INGRESS_VISIBILITY="internet-facing" export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] ') - - get_alb_rule_count() { - case "$1" in - co-balancer-public) echo "10" ;; - alb-extra-1) echo "10" ;; - alb-extra-2) echo "10" ;; - esac - } - export -f get_alb_rule_count + mock_alb_rules "co-balancer-public 5" "alb-extra-1 30" source "$SCRIPT" assert_equal "$ALB_NAME" "co-balancer-public" } -# ============================================================================= -# Deployment time: ALB lookup from existing infrastructure -# ============================================================================= -@test "resolve_balancer: deployment uses ALB from existing ingress" { +@test "resolve_balancer: logs selected ALB when different from default" { export INGRESS_VISIBILITY="internet-facing" - add_deployment_context export CONTEXT=$(echo "$CONTEXT" | jq ' .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] ') + mock_alb_rules "co-balancer-public 45" "alb-extra-1 12" - get_alb_from_ingress() { echo "alb-extra-1"; } - export -f get_alb_from_ingress - - source "$SCRIPT" + run bash -c 'source "$SCRIPT"' - assert_equal "$ALB_NAME" "alb-extra-1" + assert_contains "$output" "📝 Selected ALB 'alb-extra-1' (12 rules) over default 'co-balancer-public'" } -@test "resolve_balancer: deployment falls back to Route53 when no ingress exists" { +@test "resolve_balancer: logs candidate balancers list" { export INGRESS_VISIBILITY="internet-facing" - add_deployment_context export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] ') + mock_alb_rules "co-balancer-public 10" "alb-extra-1 10" "alb-extra-2 10" - get_alb_from_ingress() { return 1; } - export -f get_alb_from_ingress - get_alb_from_route53() { echo "alb-extra-1"; } - export -f get_alb_from_route53 - - source "$SCRIPT" + run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"' - assert_equal "$ALB_NAME" "alb-extra-1" + assert_contains "$output" "🔍 Additional balancers configured, resolving least-loaded ALB..." + assert_contains "$output" "📋 Candidate balancers: co-balancer-public, alb-extra-1, alb-extra-2" } -@test "resolve_balancer: deployment falls back to calculation when no infrastructure found" { +# ============================================================================= +# AWS API failure handling +# ============================================================================= +@test "resolve_balancer: skips candidate when rule count query fails" { export INGRESS_VISIBILITY="internet-facing" - add_deployment_context export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] ') - - get_alb_from_ingress() { return 1; } - export -f get_alb_from_ingress - get_alb_from_route53() { return 1; } - export -f get_alb_from_route53 - get_alb_rule_count() { - case "$1" in - co-balancer-public) echo "45" ;; - alb-extra-1) echo "10" ;; - esac - } - export -f get_alb_rule_count + # alb-extra-1 not in mock → describe-load-balancers returns 1 + mock_alb_rules "co-balancer-public 45" "alb-extra-2 20" source "$SCRIPT" - assert_equal "$ALB_NAME" "alb-extra-1" + assert_equal "$ALB_NAME" "alb-extra-2" } -@test "resolve_balancer: deployment logs when using ALB from ingress" { +@test "resolve_balancer: warns when a candidate fails" { export INGRESS_VISIBILITY="internet-facing" - add_deployment_context export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-broken"] ') + mock_alb_rules "co-balancer-public 10" - get_alb_from_ingress() { echo "alb-from-ingress"; } - export -f get_alb_from_ingress - - run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"' + run bash -c 'source "$SCRIPT"' - assert_contains "$output" "📋 Found ALB 'alb-from-ingress' from existing ingress for scope test-scope-123" - assert_contains "$output" "📝 Using existing ALB 'alb-from-ingress' (consistent with DNS)" + assert_contains "$output" "⚠️ Could not query rules for ALB 'alb-broken', skipping" } -@test "resolve_balancer: deployment logs when using ALB from Route53" { +@test "resolve_balancer: keeps default when all candidates fail" { export INGRESS_VISIBILITY="internet-facing" - add_deployment_context export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-broken-1", "alb-broken-2"] ') + aws() { + case "$*" in + *list-resource-record-sets*) echo "None"; return 0 ;; + *) return 1 ;; + esac + } + export -f aws - get_alb_from_ingress() { return 1; } - export -f get_alb_from_ingress - get_alb_from_route53() { echo "alb-from-dns"; } - export -f get_alb_from_route53 - - run bash -c 'export LOG_LEVEL=debug; source "$SCRIPT"' + source "$SCRIPT" - assert_contains "$output" "📋 Found ALB 'alb-from-dns' from Route53 record for test.nullapps.io" - assert_contains "$output" "📝 Using existing ALB 'alb-from-dns' (consistent with DNS)" + assert_equal "$ALB_NAME" "co-balancer-public" } -@test "resolve_balancer: deployment warns when falling back to calculation" { +# ============================================================================= +# No additional balancers — no AWS calls for rule counts +# ============================================================================= +@test "resolve_balancer: does not calculate when no additional balancers configured" { export INGRESS_VISIBILITY="internet-facing" - add_deployment_context - export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] - ') - - get_alb_from_ingress() { return 1; } - export -f get_alb_from_ingress - get_alb_from_route53() { return 1; } - export -f get_alb_from_route53 - get_alb_rule_count() { echo "10"; } - export -f get_alb_rule_count - run bash -c 'source "$SCRIPT"' + source "$SCRIPT" - assert_contains "$output" "⚠️ Could not determine existing ALB from infrastructure, recalculating" + assert_equal "$ALB_NAME" "co-balancer-public" } -@test "resolve_balancer: scope creation always calculates even when ingress exists" { +@test "resolve_balancer: handles empty additional balancers array gracefully" { export INGRESS_VISIBILITY="internet-facing" - # No deployment in context — this is scope creation export CONTEXT=$(echo "$CONTEXT" | jq ' - .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1"] + .providers["scope-configurations"].networking.additional_public_balancers = [] ') - # Even though ingress mock returns a value, scope creation should calculate - get_alb_from_ingress() { echo "old-alb-from-ingress"; } - export -f get_alb_from_ingress - get_alb_rule_count() { - case "$1" in - co-balancer-public) echo "45" ;; - alb-extra-1) echo "10" ;; - esac - } - export -f get_alb_rule_count - source "$SCRIPT" - # Should use the calculation result, not the ingress value - assert_equal "$ALB_NAME" "alb-extra-1" + assert_equal "$ALB_NAME" "co-balancer-public" } -@test "resolve_balancer: deployment without additional balancers uses base ALB" { +# ============================================================================= +# Tie-breaking: first candidate with fewest rules wins +# ============================================================================= +@test "resolve_balancer: picks first candidate on tie" { export INGRESS_VISIBILITY="internet-facing" - add_deployment_context + export CONTEXT=$(echo "$CONTEXT" | jq ' + .providers["scope-configurations"].networking.additional_public_balancers = ["alb-extra-1", "alb-extra-2"] + ') + mock_alb_rules "co-balancer-public 10" "alb-extra-1 10" "alb-extra-2 10" source "$SCRIPT" - # No additional balancers = no lookup needed, uses base ALB assert_equal "$ALB_NAME" "co-balancer-public" } From d4204e59f02c0ba5533bf2ac03a008c3a8e46a82 Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Thu, 9 Apr 2026 13:58:36 -0300 Subject: [PATCH 3/4] Fix changelog --- CHANGELOG.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 484a128d..dd402e22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add unit tests for **k8s/backup** module (backup_templates and s3 operations) - Add ALB capacity validation on scope creation. Requires additional AWS permissions: `elasticloadbalancing:DescribeLoadBalancers`, `elasticloadbalancing:DescribeListeners`, `elasticloadbalancing:DescribeRules` - Add ALB target group capacity validation on deployment. Requires additional AWS permission: `elasticloadbalancing:DescribeTargetGroups` -- Extract ALB resolution into dedicated **k8s/scope/networking/resolve_balancer** script -- Add support for distributing scopes across multiple ALBs via `additional_public_balancers` and `additional_private_balancers` provider configuration (selects the ALB with fewest listener rules) -- Ensure deployment-time ALB consistency with DNS by looking up the existing ALB from K8s ingress or Route53 record instead of recalculating -- Add 25 BATS tests for the resolve_balancer module -- **New AWS permission required** (only when using additional balancers): `route53:ListResourceRecordSets` — used at deployment time to look up the ALB assigned during scope creation - Add support for multiple ALBs ## [1.10.1] - 2026-02-13 From 5dceeb14ce37b5476f0928a3984c8d2397699df1 Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Thu, 9 Apr 2026 14:16:40 -0300 Subject: [PATCH 4/4] Fix Route53 lookup for large zones --- k8s/scope/networking/resolve_balancer | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/k8s/scope/networking/resolve_balancer b/k8s/scope/networking/resolve_balancer index a9a61628..559cb870 100755 --- a/k8s/scope/networking/resolve_balancer +++ b/k8s/scope/networking/resolve_balancer @@ -85,28 +85,36 @@ get_alb_from_route53() { return 1 fi + # Use --start-record-name to jump directly to the record instead of + # scanning from the beginning (list-resource-record-sets does not + # auto-paginate and returns max 300 per page). local alias_dns alias_dns=$(aws route53 list-resource-record-sets \ --hosted-zone-id "$zone_id" \ + --start-record-name "${domain}." \ + --start-record-type "A" \ + --max-items 1 \ --query "ResourceRecordSets[?Name=='${domain}.' && Type=='A'].AliasTarget.DNSName | [0]" \ --output text \ --region "$region" 2>/dev/null) || return 1 - if [ -z "$alias_dns" ] || [ "$alias_dns" = "None" ]; then + if [ -z "$alias_dns" ] || [ "$alias_dns" = "None" ] || [ "$alias_dns" = "null" ]; then return 1 fi # Strip trailing dot from DNS name alias_dns="${alias_dns%.}" - # Reverse-lookup: find ALB name from its DNS name + # Reverse-lookup: find ALB name from its DNS name (case-insensitive match) local alb_name alb_name=$(aws elbv2 describe-load-balancers \ - --query "LoadBalancers[?DNSName=='${alias_dns}'].LoadBalancerName | [0]" \ - --output text \ - --region "$region" 2>/dev/null) || return 1 + --region "$region" \ + --output json 2>/dev/null \ + | jq -r --arg dns "$alias_dns" \ + '.LoadBalancers[] | select(.DNSName | ascii_downcase == ($dns | ascii_downcase)) | .LoadBalancerName' \ + | head -1) || return 1 - if [ -z "$alb_name" ] || [ "$alb_name" = "None" ]; then + if [ -z "$alb_name" ]; then return 1 fi