Switch from AWS ELB to NLB to fix surge queue length

CHANGELOG.md

@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 -   Lower long poll timeout from 60s -> 40s
 -   **Bolt** Moved additional project roots to Bolt.toml
 -   **types** Support multiple project roots for reusing Protobuf types
+-   **Infra** Switch from AWS ELB to NLB to work around surge queue length limitation
 
 ### Security
 

infra/tf/dns/dns.tf

@@ -67,10 +67,6 @@ locals {
 	])
 }
 
-output "test" {
-	value = local.records
-}
-
 resource "cloudflare_record" "main" {
 	for_each = {
 		for record in local.records:

infra/tf/k8s_cluster_aws/load_balancer_controller.tf

@@ -0,0 +1,74 @@
+locals {
+	eks_oidc_issuer_url = replace(module.eks.cluster_oidc_issuer_url, "https://", "")
+}
+
+# See https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html
+
+data "http" "iam_policy" {
+	url = "https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.6.2/docs/install/iam_policy.json"
+}
+
+resource "aws_iam_policy" "load_balancer_controller_policy" {
+	name = "${local.name}-AWSLoadBalancerControllerIAMPolicy"
+	description = "IAM policy for AWS Load Balancer Controller"
+	policy = data.http.iam_policy.body
+}
+
+resource "aws_iam_role" "eks_load_balancer_role" {
+	name = "${local.name}-AmazonEKSLoadBalancerControllerRole"
+
+	assume_role_policy = jsonencode({
+		Version = "2012-10-17",
+		Statement = [
+			{
+				Effect = "Allow",
+				Principal = {
+					Federated = module.eks.oidc_provider_arn
+				},
+				Action = "sts:AssumeRoleWithWebIdentity",
+				Condition = {
+					StringEquals = {
+						"${local.eks_oidc_issuer_url}:aud" = "sts.amazonaws.com"
+						"${local.eks_oidc_issuer_url}:sub" = "system:serviceaccount:kube-system:aws-load-balancer-controller"
+					}
+				}
+			}
+		]
+	})
+}
+
+resource "aws_iam_role_policy_attachment" "eks_load_balancer_policy_attachment" {
+	role = aws_iam_role.eks_load_balancer_role.name
+	policy_arn = aws_iam_policy.load_balancer_controller_policy.arn
+}
+
+resource "kubernetes_service_account" "aws_load_balancer_controller" {
+	metadata {
+		name = "aws-load-balancer-controller"
+		namespace = "kube-system"
+		annotations = {
+			"eks.amazonaws.com/role-arn" = aws_iam_role.eks_load_balancer_role.arn
+		}
+	}
+}
+
+resource "helm_release" "load_balancer_controller" {
+	name = "aws-load-balancer-controller"
+	repository = "https://aws.github.io/eks-charts"
+	chart = "aws-load-balancer-controller"
+	# repository = "oci://public.ecr.aws/eks-charts"
+	# chart = "aws-load-balancer-controller"
+	namespace = "kube-system"
+	# Corresponds to load balancer controller version 2.6.2
+	version = "v1.6.2"
+
+	values = [yamlencode({
+		clusterName = module.eks.cluster_name
+		vpcId = module.vpc.vpc_id
+		serviceAccount = {
+			create = false
+			name = kubernetes_service_account.aws_load_balancer_controller.metadata.0.name
+		}
+	})]
+}
+

infra/tf/k8s_infra/traefik.tf

@@ -76,6 +76,17 @@ resource "helm_release" "traefik" {
 			# }
 		}
 
+		service = {
+			enabled = true
+			annotations = var.deploy_method_cluster ? {
+				# See: https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html
+				"service.beta.kubernetes.io/aws-load-balancer-type" = "external"
+				# Removes the need for an extra network hop: https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/service/nlb/#ip-mode
+				"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "ip"
+				"service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing"
+			} : {}
+		}
+
 		autoscaling = {
 			enabled = var.deploy_method_cluster
 			minReplicas = local.service_traefik.count

infra/tf/k8s_infra/traefik_tunnel.tf

@@ -122,6 +122,17 @@ resource "helm_release" "traefik_tunnel" {
 			}
 		}
 
+		service = {
+			enabled = true
+			annotations = var.deploy_method_cluster ? {
+				# See: https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html
+				"service.beta.kubernetes.io/aws-load-balancer-type" = "external"
+				# Removes the need for an extra network hop: https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/service/nlb/#ip-mode
+				"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "ip"
+				"service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing"
+			} : {}
+		}
+
 		metrics = {
 			prometheus = {
 				addEntryPointsLabels = false

infra/tf/redis_aws/outputs.tf

@@ -1,8 +1,3 @@
-output "test" {
-	value = aws_elasticache_replication_group.main
-	sensitive = true
-}
-
 output "host" {
 	value = merge(
 		{