Merged in dev/fabien/MPC-4786-NodePort-for-ingress-nginx (pull request …

…elastic#517)

MPC-4786: use NodePorts for ingress-nginx

* use NodePorts for ingress-nginx


Approved-by: Gideon Avida

aws/ams-cluster-v1-tf/eks.tf

@@ -130,6 +130,41 @@ resource "aws_security_group" "worker_ssh" {
   }
 }
 
+resource "aws_security_group" "worker_alb" {
+  vpc_id = aws_vpc.main.id
+  ingress {
+    from_port = aws_lb_target_group.eks_target_group.port
+    protocol  = "tcp"
+    to_port   = aws_lb_target_group.eks_target_group.port
+    security_groups = [
+      aws_security_group.alb.id
+    ]
+  }
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+}
+
+resource "aws_lb_target_group" "eks_target_group" {
+  name        = "${local.eks_cluster_name}-eks-tg"
+  port        = 32443
+  protocol    = "HTTPS"
+  vpc_id      = aws_vpc.main.id
+  target_type = "instance"
+  health_check {
+    port     = "traffic-port"
+    timeout  = 2
+    interval = 5
+    enabled  = true
+    protocol = "HTTPS"
+    path     = "/healthz"
+  }
+}
+
 module "eks" {
   source  = "registry.terraform.io/terraform-aws-modules/eks/aws"
   version = "17.23.0"
@@ -147,7 +182,8 @@ module "eks" {
   worker_additional_security_group_ids = [
     aws_security_group.postgres_clients.id,
     aws_security_group.redis_clients.id,
-    aws_security_group.worker_ssh.id
+    aws_security_group.worker_ssh.id,
+    aws_security_group.worker_alb.id
   ]
 
   cluster_encryption_config = [
@@ -159,6 +195,9 @@ module "eks" {
 
   worker_groups_launch_template = [
     {
+      target_group_arns = [
+        aws_lb_target_group.eks_target_group.arn
+      ]
       override_instance_types = var.eks_worker_instance_types
       asg_min_size            = var.eks_worker_asg_min_instances
       asg_max_size            = var.eks_worker_asg_max_instances
@@ -439,18 +478,15 @@ resource "helm_release" "ingress_nginx" {
           service = {
             # we don't want no stinky http
             enableHttp = false
+            type       = "NodePort"
+            nodePorts = {
+              http  = ""
+              https = aws_lb_target_group.eks_target_group.port
+              tcp   = {}
+              udp   = {}
+            }
             annotations = merge(
               {
-                "service.beta.kubernetes.io/aws-load-balancer-backend-protocol"                  = "tcp"
-                "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout"           = 3600
-                "service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled" = true
-                "service.beta.kubernetes.io/aws-load-balancer-type"                              = "nlb"
-                "service.beta.kubernetes.io/aws-load-balancer-subnets"                           = join(",", aws_subnet.eks.*.id)
-                "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags"          = join(",", [for k, v in local.default_tags : "${k}=${v}"])
-                # this one doesn't appear to work?
-                #"service.beta.kubernetes.io/aws-load-balancer-proxy-protocol"                    = "*"
-                # might be (again) AWS Controller vs ingress-nginx in-tree controller
-                #"service.beta.kubernetes.io/aws-load-balancer-target-group-attributes"           = "proxy_protocol_v2.enabled=true"
               },
               # overrides per stack if needed
               var.eks_ingress_nginx_annotations
@@ -1203,42 +1239,6 @@ resource "kubernetes_manifest" "cluster_issuer" {
   ]
 }
 
-data "aws_lb" "nlb" {
-  tags = merge(
-    local.default_tags,
-    {
-      "kubernetes.io/service-name"      = "ingress-nginx/ingress-nginx-controller"
-      "kubernetes.io/cluster/tf-fabien" = "owned"
-    }
-  )
-}
-
-data "aws_network_interfaces" "nlb_interfaces" {
-  filter {
-    name   = "description"
-    values = ["ELB ${data.aws_lb.nlb.arn_suffix}"]
-  }
-}
-
-data "aws_network_interface" "nlb_interface" {
-  for_each = toset(data.aws_network_interfaces.nlb_interfaces.ids)
-  id       = each.key
-}
-
-resource "aws_lb_target_group" "eks_target_group" {
-  name        = "${local.eks_cluster_name}-eks-tg"
-  port        = 443
-  protocol    = "HTTPS"
-  vpc_id      = aws_vpc.main.id
-  target_type = "ip"
-}
-
-resource "aws_lb_target_group_attachment" "eks_target_group" {
-  for_each         = toset(values(data.aws_network_interface.nlb_interface).*.private_ip)
-  target_group_arn = aws_lb_target_group.eks_target_group.arn
-  target_id        = each.key
-}
-
 # default is 100 rules per ALB
 # if the rules_count gets over it, we need to set the quota
 locals {