eks.tf

locals {
  # we use /16 for VPCs
  # this grabs (arbitrarily) a /20 off the /16 for EKS
  eks_vpc_chunk = cidrsubnet(var.vpc_cidr, 4, 4)
  # then we split that in 4 subnets
  eks_subnets = cidrsubnets(local.eks_vpc_chunk, 2, 2, 2, 2)
  # keep it simple
  eks_cluster_name = "tf-${var.stack_name}"
  engageli_sso_role_regex = join("|", concat(
    ["DevOps"],
    contains(["test", "dev"], var.stack_env) ? ["QaAdmin"] : [],
    var.stack_env == "dev" ? ["DevAdmin"] : []
    )
  )
  helm_charts_versions = {
    calico                       = "v3.22.0"
    cluster_autoscaler           = "9.16.2"
    cert_manager                 = "1.6.3"
    external_secrets             = "0.5.6"
    ingress_nginx                = "4.0.10"
    grafana_agent_operator       = "0.1.5"
    kube_state_metrics           = "4.4.1"
    metrics_server               = "0.6.1-1"
    node_exporter                = "3.0.1"
    cloudwatch_exporter          = "0.14.3-1"
    redis_exporter               = "1.43.0-1"
    postgres_exporter            = "0.10.1-2"
    telepresence_traffic_manager = "2.6.8-3"
    kubernetes_secret_generator  = "3.4.0-1"
  }

  k8s_default_labels = { for k, v in local.default_tags : k => v if k != "Terraform" }
}

data "aws_iam_roles" "current_account_sso_roles" {
  name_regex  = format(".+(%s).+", local.engageli_sso_role_regex)
  path_prefix = "/aws-reserved/sso.amazonaws.com/"
}

locals {
  # roles allowed to connect as cluster admins
  # for some godforsaken reason, the "full" form of the ARN for SSO roles needs to be used in order to grant
  # the eks:DescribeCluster permission
  # but the shorter form is what is needed for EKS itself to grant access to the cluster
  # I'm sure there's a reason for that somewhere and I'll get to it eventually, but in the meantime:
  eks_admin_roles_full_arn = concat(
    [
      "arn:aws:iam::${var.aws_account}:role/admin-role",
      "arn:aws:iam::569129334545:role/aws-reserved/sso.amazonaws.com/us-west-2/AWSReservedSSO_DevOps_7c4ac33af1eab0a2",
    ],
    contains(["test", "dev"], var.stack_env) ? [
      "arn:aws:iam::569129334545:role/aws-reserved/sso.amazonaws.com/us-west-2/AWSReservedSSO_QaAdmin_a69e8fe5bfd56b9b",
    ] : [],
    var.stack_env == "dev" ? [
      "arn:aws:iam::569129334545:role/aws-reserved/sso.amazonaws.com/us-west-2/AWSReservedSSO_DevAdmin_e63ce09bedcdae27"
    ] : [],
    tolist(data.aws_iam_roles.current_account_sso_roles.arns)
  )
  eks_admin_roles_trimmed_arn = [
    for parts in [for arn in local.eks_admin_roles_full_arn : split("/", arn)] :
    format("%s/%s", parts[0], element(parts, length(parts) - 1))
  ]
  # now we create a map of the actual admin roles to pass to the eks module
  eks_admin_roles = [
    for role in local.eks_admin_roles_trimmed_arn : {
      rolearn  = role
      username = "admin"
      groups   = ["system:masters"]
    }
  ]
}

resource "aws_subnet" "eks" {
  count                = var.num_zones
  vpc_id               = aws_vpc.main.id
  cidr_block           = local.eks_subnets[count.index]
  availability_zone_id = data.aws_availability_zones.available.zone_ids[count.index]
  # this is added for EKS managed nodes
  # we probably do NOT want to have a fully private EKS cluster because then the nodes
  # need to use NAT to reach the internet, and NAT gw are _very_ expensive
  map_public_ip_on_launch = true
  tags = {
    Name                                              = "tf-${var.stack_name}-eks-subnet-${count.index}"
    "kubernetes.io/cluster/${local.eks_cluster_name}" = "shared"
  }
}

resource "aws_route_table_association" "eks" {
  count          = var.num_zones
  subnet_id      = aws_subnet.eks[count.index].id
  route_table_id = aws_route_table.main.id
}

resource "aws_kms_key" "eks" {
  description             = "EKS Secret Encryption Key for ${local.eks_cluster_name}"
  deletion_window_in_days = 7
  enable_key_rotation     = true
}

resource "aws_lb_target_group" "eks_target_group" {
  name        = "${local.eks_cluster_name}-eks-tg"
  port        = 32443
  protocol    = "HTTPS"
  vpc_id      = aws_vpc.main.id
  target_type = "instance"
  health_check {
    port     = "traffic-port"
    timeout  = 2
    interval = 5
    enabled  = true
    protocol = "HTTPS"
    path     = "/healthz"
  }
}

data "aws_ami" "eks_default_bottlerocket" {
  most_recent = true
  owners      = ["amazon"]

  filter {
    name   = "name"
    values = ["bottlerocket-aws-k8s-${var.eks_cluster_version}-x86_64-*"]
  }
}

module "eks" {
  source  = "registry.terraform.io/terraform-aws-modules/eks/aws"
  version = "18.26.6"

  cluster_version = var.eks_cluster_version
  cluster_name    = local.eks_cluster_name
  vpc_id          = aws_vpc.main.id
  subnet_ids      = aws_subnet.eks.*.id

  # required for in-place upgrade from 17.24
  prefix_separator                   = ""
  iam_role_name                      = local.eks_cluster_name
  cluster_security_group_name        = local.eks_cluster_name
  cluster_security_group_description = "EKS cluster security group."

  cluster_endpoint_private_access      = true
  cluster_endpoint_public_access       = true
  cluster_endpoint_public_access_cidrs = local.management_ips

  cluster_security_group_additional_rules = {
    egress_nodes_ephemeral_ports_tcp = {
      description                = "TF managed: allow all egress"
      protocol                   = -1
      from_port                  = 0
      to_port                    = 0
      type                       = "egress"
      source_node_security_group = true
    }
  }

  node_security_group_additional_rules = {
    ingress_alb = {
      from_port                = aws_lb_target_group.eks_target_group.port
      protocol                 = "tcp"
      to_port                  = aws_lb_target_group.eks_target_group.port
      type                     = "ingress"
      source_security_group_id = aws_security_group.alb.id
    }
    ingress_ems = {
      from_port   = 5000
      protocol    = "udp"
      to_port     = 65000
      type        = "ingress"
      cidr_blocks = ["0.0.0.0/0"]
    }
    ingress_node_all = {
      description = "TF managed: Node to node all ports/protocols"
      protocol    = "-1"
      from_port   = 0
      to_port     = 0
      type        = "ingress"
      self        = true
    }
    ingress_cluster_all = {
      description                   = "Cluster to node all ports/protocols"
      protocol                      = "-1"
      from_port                     = 0
      to_port                       = 0
      type                          = "ingress"
      source_cluster_security_group = true
    }
    egress_all = {
      description      = "Node all egress"
      protocol         = "-1"
      from_port        = 0
      to_port          = 0
      type             = "egress"
      cidr_blocks      = ["0.0.0.0/0"]
      ipv6_cidr_blocks = ["::/0"]
    }
  }

  cluster_encryption_config = [
    {
      provider_key_arn = aws_kms_key.eks.arn
      resources        = ["secrets"]
    }
  ]

  # create a launch template per instance type
  # cluster autoscaler only works with instance types of similar capacity
  # so using a t3.small and a c5.24xlarge in the same launch template has no effect,
  # only the first instance type is used to calculate scaling capacity
  self_managed_node_group_defaults = {}
  self_managed_node_groups = {
    for type, config in var.eks_worker_instance_types :
    format("%s-eks-%s", local.eks_cluster_name, type) => {
      target_group_arns = [
        aws_lb_target_group.eks_target_group.arn
      ]
      platform = "bottlerocket"
      ami_id   = data.aws_ami.eks_default_bottlerocket.id
      vpc_security_group_ids = [
        aws_security_group.postgres_clients.id,
        aws_security_group.redis_clients.id,
        aws_security_group.ems_redis_clients.id,
      ]
      instance_type     = type
      desired_capacity  = config.min
      min_size          = config.min
      max_size          = config.max
      enable_monitoring = false
      # broken with Bottlerocket
      #      kubelet_extra_args = format("--node-labels=%s",
      #        join(",",
      #          [for k, v in local.default_tags : "${k}=${v}"]
      #        )
      #      )
      metadata_options = {
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
        instance_metadata_tags      = "disabled"
      }
      public_ip        = true
      key_name         = var.key_name
      use_default_tags = true
      tags = {
        Role = "EKS"
        # check if this can be set only on the asg? is it still required? do we go to karpenter instead of
        # cluster-autoscaler
        "k8s.io/cluster-autoscaler/enabled"                   = true
        "k8s.io/cluster-autoscaler/${local.eks_cluster_name}" = "owned"
      }

      iam_role_additional_policies = [
        "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
      ]
    }
  }

  create_iam_role             = true
  create_cloudwatch_log_group = true
  create_aws_auth_configmap   = true
  manage_aws_auth_configmap   = true
  aws_auth_roles              = local.eks_admin_roles

  enable_irsa = true

  depends_on = [
    aws_route.main,
    aws_route_table_association.rta,
    aws_route_table_association.eks,
  ]
}

resource "aws_iam_role_policy_attachment" "eks_role_session_manager" {
  role       = module.eks.cluster_iam_role_name
  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
}

data "aws_iam_policy_document" "cluster_autoscaler" {
  statement {
    sid    = "clusterAutoscalerAll"
    effect = "Allow"

    actions = [
      "autoscaling:DescribeAutoScalingGroups",
      "autoscaling:DescribeAutoScalingInstances",
      "autoscaling:DescribeLaunchConfigurations",
      "autoscaling:DescribeTags",
      "ec2:DescribeInstanceTypes",
      "ec2:DescribeLaunchTemplateVersions",
    ]

    resources = ["*"]
  }

  statement {
    sid    = "clusterAutoscalerOwn"
    effect = "Allow"

    actions = [
      "autoscaling:SetDesiredCapacity",
      "autoscaling:TerminateInstanceInAutoScalingGroup",
      "autoscaling:UpdateAutoScalingGroup",
    ]

    resources = ["*"]

    condition {
      test     = "StringEquals"
      variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/${local.eks_cluster_name}"
      values   = ["owned"]
    }

    condition {
      test     = "StringEquals"
      variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled"
      values   = ["true"]
    }
  }
}

data "aws_iam_policy_document" "autoscaling_role" {
  version = "2012-10-17"
  statement {
    effect = "Allow"
    principals {
      type        = "Federated"
      identifiers = [module.eks.oidc_provider_arn]
    }
    actions = [
      "sts:AssumeRoleWithWebIdentity"
    ]
    condition {
      test     = "StringEquals"
      variable = "${replace(module.eks.cluster_oidc_issuer_url, "https://", "")}:sub"
      values = [
        "system:serviceaccount:kube-system:cluster-autoscaler"
      ]
    }
  }
}

resource "aws_iam_role" "autoscaling_role" {
  name               = "${local.eks_cluster_name}-autoscaling-role"
  assume_role_policy = data.aws_iam_policy_document.autoscaling_role.json

  inline_policy {
    name   = "${local.eks_cluster_name}-autoscaling-policy"
    policy = data.aws_iam_policy_document.cluster_autoscaler.json
  }
}

### start of helm block
resource "helm_release" "cluster_autoscaler" {
  repository  = "https://kubernetes.github.io/autoscaler"
  chart       = "cluster-autoscaler"
  name        = "cluster-autoscaler"
  namespace   = "kube-system"
  version     = local.helm_charts_versions.cluster_autoscaler
  wait        = true
  atomic      = true
  max_history = 10
  values = [
    yamlencode(
      {
        awsRegion = var.region
        autoDiscovery = {
          enabled     = true
          clusterName = local.eks_cluster_name
        }
        rbac = {
          create = true
          serviceAccount = {
            name = "cluster-autoscaler"
            annotations = {
              "eks.amazonaws.com/role-arn" = aws_iam_role.autoscaling_role.arn
            }
          }
        }
      }
    )
  ]
  depends_on = [
    module.eks
  ]
}

# monitoring

data "aws_iam_policy_document" "cloudwatch_metrics_read" {
  statement {
    sid       = "CWMetrics"
    effect    = "Allow"
    resources = ["*"]

    actions = [
      "tag:GetResources",
      "cloudwatch:listMetrics",
      "cloudwatch:getMetricStatistics",
      "cloudwatch:getMetricData",
    ]
  }
}

data "aws_iam_policy_document" "eks_monitoring_assume" {
  version = "2012-10-17"
  statement {
    effect = "Allow"
    principals {
      type        = "Federated"
      identifiers = [module.eks.oidc_provider_arn]
    }
    actions = [
      "sts:AssumeRoleWithWebIdentity"
    ]
    condition {
      test     = "StringEquals"
      variable = "${replace(module.eks.cluster_oidc_issuer_url, "https://", "")}:sub"
      values = [
        "system:serviceaccount:monitoring:aws-access"
      ]
    }
  }
}

resource "aws_iam_role" "eks_monitoring" {
  name               = "${local.eks_cluster_name}-eks-monitoring"
  assume_role_policy = data.aws_iam_policy_document.eks_monitoring_assume.json
  inline_policy {
    name   = "cloudwatch-metrics-read"
    policy = data.aws_iam_policy_document.cloudwatch_metrics_read.json
  }
  inline_policy {
    name   = "postgres-access"
    policy = data.aws_iam_policy_document.postgres_access.json
  }
}

resource "kubernetes_namespace_v1" "monitoring" {
  metadata {
    name = "monitoring"
  }
}

resource "kubernetes_service_account_v1" "monitoring" {
  metadata {
    name      = "aws-access"
    labels    = local.k8s_default_labels
    namespace = kubernetes_namespace_v1.monitoring.metadata.0.name
    annotations = {
      "eks.amazonaws.com/role-arn" = aws_iam_role.eks_monitoring.arn
    }
  }
}

resource "kubernetes_manifest" "monitoring_secret" {
  manifest = {
    apiVersion = "external-secrets.io/v1beta1"
    kind       = "ExternalSecret"
    metadata = {
      namespace = kubernetes_namespace_v1.monitoring.metadata.0.name
      name      = "global-secrets"
      labels    = local.k8s_default_labels
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        name = kubernetes_manifest.external_secrets_secret_store.manifest.metadata.name
        kind = kubernetes_manifest.external_secrets_secret_store.manifest.kind
      }
      target = {
        name           = "global-secrets"
        creationPolicy = "Owner"
        template = {
          engineVersion = "v2"
          data = {
            MAIN_REDIS_PASSWORD = "{{ .main_redis_password }}"
            EMS_REDIS_PASSWORD  = "{{ .ems_redis_password }}"
          }
        }
      }
      data = [
        {
          secretKey = "main_redis_password"
          remoteRef = {
            key = aws_secretsmanager_secret_version.redis_pw.arn
          }
        },
        {
          secretKey = "ems_redis_password"
          remoteRef = {
            key = aws_secretsmanager_secret_version.ems_redis_pw.arn
          }
        }
      ]
    }
  }
}

resource "kubernetes_config_map_v1" "monitoring_config_map" {
  metadata {
    name      = "global-configs"
    labels    = local.k8s_default_labels
    namespace = kubernetes_namespace_v1.monitoring.metadata.0.name
  }
  data = {
    STACK_FQDN     = local.stack_fqdn
    PG_HOST        = aws_db_instance.pg_master.address
    AWS_REGION     = var.region
    MAIN_REDIS_URL = "rediss://${aws_elasticache_replication_group.redis.primary_endpoint_address}"
    EMS_REDIS_URL  = "rediss://${aws_elasticache_replication_group.ems_redis.primary_endpoint_address}"
  }
}

resource "helm_release" "ingress_nginx" {
  repository       = "https://kubernetes.github.io/ingress-nginx"
  name             = "ingress-nginx"
  chart            = "ingress-nginx"
  namespace        = "ingress-nginx"
  create_namespace = true
  version          = local.helm_charts_versions.ingress_nginx
  wait             = true
  atomic           = true
  max_history      = 10
  values = [
    yamlencode(
      {
        controller = {
          # config map overrides
          # https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
          # we use the defaults values for now so there's nothing to merge
          config = merge(
            {
              use-gzip               = true
              max-worker-connections = 262144
              log-format-escape-json = true
              log-format-upstream = jsonencode(
                {
                  namespace              = "$namespace"
                  ingress_name           = "$ingress_name"
                  service_name           = "$service_name"
                  remote_addr            = "$remote_addr"
                  remote_user            = "$remote_user"
                  time_local             = "$time_local"
                  request                = "$request"
                  status                 = "$status"
                  body_bytes_sent        = "$body_bytes_sent"
                  http_referer           = "$http_referer"
                  http_user_agent        = "$http_user_agent"
                  http_x_forwarded_for   = "$http_x_forwarded_for"
                  upstream_connect_time  = "$upstream_connect_time"
                  upstream_header_time   = "$upstream_header_time"
                  upstream_response_time = "$upstream_response_time"
                  request_time           = "$request_time"
                }
              )
            },
          var.eks_ingress_nginx_config_map)
          service = {
            # we don't want no stinky http
            enableHttp = false
            type       = "NodePort"
            nodePorts = {
              http  = ""
              https = aws_lb_target_group.eks_target_group.port
              tcp   = {}
              udp   = {}
            }
            annotations = merge(
              {
              },
              # overrides per stack if needed
              var.eks_ingress_nginx_annotations
            )
          }
        }
      }
    )
  ]
  depends_on = [
    module.eks
  ]
}

resource "helm_release" "cert_manager" {
  repository       = "https://charts.jetstack.io"
  name             = "cert-manager"
  chart            = "cert-manager"
  namespace        = "cert-manager"
  create_namespace = true
  version          = local.helm_charts_versions.cert_manager
  wait             = true
  atomic           = true
  max_history      = 10
  values = [
    yamlencode(
      {
        installCRDs = true
        prometheus = {
          servicemonitor = {
            enabled = true
            interval = "30s"
            scrapeTimeout = "25s"
          }
        }
        resources = {
          requests = {
            cpu = "50m"
            memory = "96Mi"
          }
          limits = {
            cpu = "50m"
            memory = "96Mi"
          }
        }
        webhook = {
          resources = {
            requests = {
              cpu    = "50m"
              memory = "64Mi"
            }
            limits = {
              cpu    = "50m"
              memory = "64Mi"
            }
          }
        }
        cainjector = {
          resources = {
            requests = {
              cpu    = "50m"
              memory = "96Mi"
            }
            limits = {
              cpu    = "50m"
              memory = "96Mi"
            }
          }
        }
      }
    )
  ]

  depends_on = [
    module.eks
  ]
}

resource "helm_release" "grafana_agent_operator" {
  repository  = "https://grafana.github.io/helm-charts"
  chart       = "grafana-agent-operator"
  name        = "grafana-agent-operator"
  namespace   = kubernetes_namespace_v1.monitoring.metadata.0.name
  version     = local.helm_charts_versions.grafana_agent_operator
  wait        = true
  atomic      = true
  max_history = 10
  depends_on = [
    module.eks
  ]
}

resource "helm_release" "external_secrets" {
  repository       = "https://external-secrets.github.io/external-secrets/"
  chart            = "external-secrets"
  name             = "external-secrets"
  namespace        = "external-secrets"
  create_namespace = true
  version          = local.helm_charts_versions.external_secrets
  wait             = true
  atomic           = true
  max_history      = 10
  values = [
    yamlencode(
      {
        serviceAccount = {
          name = "external-secrets"
          annotations = {
            "eks.amazonaws.com/role-arn" = aws_iam_role.external_secrets.arn
          }
        }
      }
    )
  ]
  depends_on = [
    module.eks
  ]
}

resource "helm_release" "kube_state_metrics" {
  repository  = "https://prometheus-community.github.io/helm-charts"
  chart       = "kube-state-metrics"
  name        = "kube-state-metrics"
  namespace   = kubernetes_namespace_v1.monitoring.metadata.0.name
  version     = local.helm_charts_versions.kube_state_metrics
  wait        = true
  atomic      = true
  max_history = 10
  # there is no namespace selector within the kube-state-metrics template
  # so we create a manifest and apply it separately
  # TODO PR in the project to fix that!
  #
  # trimming a few little used resources as they'll just eat timeseries for nothing
  values = [
    yamlencode(
      {
        customLabels = {
          app = "kube-state-metrics"
        }
      }
    )
  ]
  depends_on = [
    module.eks
  ]
}

resource "kubernetes_manifest" "kube_state_metrics_monitor" {
  manifest = {
    apiVersion = "monitoring.coreos.com/v1"
    kind       = "ServiceMonitor"
    metadata = {
      name      = "kube-state-metrics-monitor"
      namespace = helm_release.kube_state_metrics.namespace
      labels    = local.k8s_default_labels
    }
    spec = {
      endpoints = [
        {
          port = "http"
          metricRelabelings = [
            {
              action = "keep"
              regex = join("|",
                [
                  "kube_endpoint_address_available",
                  "kube_endpoint_address_not_ready",
                  "kube_pod_container_status_last_terminated_reason",
                  "kube_pod_container_status_restarts_total",
                  "kube_replicaset_spec_replicas",
                  "kube_replicaset_status_ready_replicas",
                  "kube_node_status_allocatable",
                  "kube_node_status_capacity"
                ]
              )
              sourceLabels = [
                "__name__"
              ]
            }
          ]
        }
      ]
      namespaceSelector = {
        matchNames = [
          helm_release.kube_state_metrics.namespace
        ]
      }
      selector = {
        matchLabels = {
          # quoted because terraform chokes on '/'
          "app.kubernetes.io/name" = "kube-state-metrics"
        }
      }
    }
  }
}

resource "helm_release" "calico" {
  repository       = "https://docs.projectcalico.org/charts"
  chart            = "tigera-operator"
  name             = "tigera-operator"
  namespace        = "calico"
  create_namespace = true
  version          = local.helm_charts_versions.calico
  wait             = true
  atomic           = true
  max_history      = 3
  values = [
    yamlencode(
      {
        installation = {
          kubernetesProvider = "EKS"
        }
      }
    )
  ]
  depends_on = [
    module.eks
  ]
}

# we need to manually run a kubectl patch for calico, so we need to make sure the kubeconfig is updated
resource "null_resource" "update_kubeconfig" {
  triggers = {
    endpoint = data.aws_eks_cluster.eks.endpoint
    profile  = var.target_profile
  }
  provisioner "local-exec" {
    command = <<-EOT
    ${path.module}/../misc/update-kubeconfig.sh ${var.region} ${local.eks_cluster_name} ${self.triggers.profile}
EOT
  }
  depends_on = [
    helm_release.calico
  ]
}

# terraform doesn't support kubectl patch
# https://github.com/hashicorp/terraform-provider-kubernetes/issues/723
resource "null_resource" "enable_calico_wireguard" {
  triggers = {
    endpoint = data.aws_eks_cluster.eks.endpoint
    kubeconfig = "${path.module}/kubeconfig-${local.eks_cluster_name}"
  }
  provisioner "local-exec" {
    command = <<-EOT
      until \
        kubectl \
          --kubeconfig ${self.triggers.kubeconfig} \
          get felixconfiguration default; do
        sleep 5;
      done
      kubectl \
        --kubeconfig ${self.triggers.kubeconfig} \
        patch felixconfiguration default \
        --type='merge' \
        --patch '${jsonencode(
    {
      spec = {
        wireguardEnabled       = true
        wireguardListeningPort = 51825
      }
    }
)}'
    EOT
}
depends_on = [
  null_resource.update_kubeconfig
]
}

resource "helm_release" "metrics_server" {
  chart       = "helm-charts/metrics-server"
  name        = "metrics-server"
  version     = local.helm_charts_versions.metrics_server
  wait        = true
  atomic      = true
  max_history = 10
  values      = []

  depends_on = [
    module.eks
  ]
}

resource "helm_release" "node_exporter" {
  repository  = "https://prometheus-community.github.io/helm-charts"
  chart       = "prometheus-node-exporter"
  name        = "node-exporter"
  namespace   = kubernetes_namespace_v1.monitoring.metadata.0.name
  version     = local.helm_charts_versions.node_exporter
  wait        = true
  atomic      = true
  max_history = 10
  values = [
    yamlencode(
      {
        fullnameOverride = "node-exporter"
      }
    )
  ]
  depends_on = [
    module.eks
  ]
}

resource "helm_release" "cloudwatch_exporter" {
  chart       = "helm-charts/cloudwatch-exporter"
  name        = "cloudwatch-exporter"
  namespace   = kubernetes_namespace_v1.monitoring.metadata.0.name
  version     = local.helm_charts_versions.cloudwatch_exporter
  wait        = true
  atomic      = true
  max_history = 10
  values = [
    yamlencode(
      {
        stack_name           = var.stack_name
        region               = var.region
        service_account_name = kubernetes_service_account_v1.monitoring.metadata.0.name
      }
    )
  ]
  depends_on = [
    module.eks
  ]
}

resource "helm_release" "redis_exporter" {
  chart       = "helm-charts/redis-exporter"
  name        = "redis-exporter"
  namespace   = kubernetes_namespace_v1.monitoring.metadata.0.name
  version     = local.helm_charts_versions.redis_exporter
  wait        = true
  atomic      = true
  max_history = 10
  values = [
    yamlencode(
      {
        service_account_name = kubernetes_service_account_v1.monitoring.metadata.0.name
      }
    )
  ]
  depends_on = [
    module.eks
  ]
}

resource "helm_release" "postgres_exporter" {
  chart       = "helm-charts/postgres-exporter"
  name        = "master-postgres-exporter"
  namespace   = kubernetes_namespace_v1.monitoring.metadata.0.name
  version     = local.helm_charts_versions.postgres_exporter
  wait        = true
  atomic      = true
  max_history = 10
  values = [
    yamlencode(
      {
        service_account_name = kubernetes_service_account_v1.monitoring.metadata.0.name
      }
    )
  ]
  depends_on = [
    module.eks
  ]
}

resource "helm_release" "telepresence_traffic_manager" {
  count       = local.eks_telepresence_enabled ? 1 : 0
  chart       = "helm-charts/telepresence"
  name        = "traffic-manager"
  version     = local.helm_charts_versions.telepresence_traffic_manager
  wait        = true
  atomic      = true
  max_history = 10
  values      = []

  depends_on = [
    helm_release.cert_manager,
    module.eks
  ]
}

resource "helm_release" "kubernetes_secret_generator" {
  chart       = "helm-charts/kubernetes-secret-generator"
  name        = "kubernetes-secret-generator"
  version     = local.helm_charts_versions.kubernetes_secret_generator
  wait        = true
  atomic      = true
  max_history = 10
  values      = []

  depends_on = [
    module.eks
  ]
}

### end of helm block

### engageli k8s modules block
module "engageli_kubernetes_namespaces" {
  source    = "./tfmodules/k8s_namespace"
  for_each  = local.engageli_kubernetes_apps
  namespace = each.key
  labels    = local.k8s_default_labels
  global_configmaps = {
    STACK_FQDN = local.stack_fqdn
    PG_HOST    = aws_db_instance.pg_master.address
    AWS_REGION = var.region
    NAMESPACE  = each.key
  }
  global_secrets = {
    REDIS_URL     = "rediss://default:${aws_secretsmanager_secret_version.redis_pw.secret_string}@${aws_elasticache_replication_group.redis.primary_endpoint_address}"
    EMS_REDIS_URL = "rediss://default:${aws_secretsmanager_secret_version.ems_redis_pw.secret_string}@${aws_elasticache_replication_group.ems_redis.primary_endpoint_address}"
  }
  apps = each.value
  # not liking much passing this around
  stack_fqdn = local.stack_fqdn
  depends_on = [
    module.eks,
    helm_release.ingress_nginx
  ]
}

### end of engageli k8s modules block

### grafana-agent resources block

resource "kubernetes_manifest" "grafana_agent" {
  manifest = {
    apiVersion = "monitoring.grafana.com/v1alpha1"
    kind       = "GrafanaAgent"
    metadata = {
      name      = "grafana-agent"
      namespace = helm_release.grafana_agent_operator.namespace
      labels = {
        app = "grafana-agent"
      }
    }
    spec = {
      logLevel           = "info"
      serviceAccountName = "grafana-agent"
      metrics = {
        instanceSelector = {
          matchLabels = {
            agent = "grafana-agent-metrics"
          }
        }
        scrapeInterval           = "30s"
        metricsExternalLabelName = ""
        externalLabels = {
          engageli_stack_name = var.stack_name
          engageli_stack_env  = var.stack_env
          engageli_region     = var.region
          # can't set the actual build value, we don't have access directly from tf
          engageli_version = var.sw_version
        }
      }
      logs = {
        logsExternalLabelName = ""
        instanceSelector = {
          matchLabels = {
            agent = "grafana-agent-logs"
          }
        }
      }
    }
  }
}

resource "kubernetes_service_account_v1" "grafana_agent" {
  metadata {
    name      = "grafana-agent"
    namespace = helm_release.grafana_agent_operator.namespace
    labels    = local.k8s_default_labels
  }
}

resource "kubernetes_cluster_role_v1" "grafana-agent" {
  metadata {
    name   = "grafana-agent"
    labels = local.k8s_default_labels
  }
  rule {
    api_groups = [""]
    verbs = [
      "get",
      "list",
      "watch"
    ]
    resources = [
      "nodes",
      "nodes/proxy",
      "nodes/metrics",
      "services",
      "endpoints",
      "pods"
    ]
  }
  rule {
    api_groups = [
      "networking.k8s.io"
    ]
    verbs = [
      "get",
      "list",
      "watch"
    ]
    resources = [
      "ingresses"
    ]
  }
  rule {
    non_resource_urls = [
      "/metrics",
      "/metrics/cadvisor"
    ]
    verbs = [
      "get"
    ]
  }
}

resource "kubernetes_cluster_role_binding_v1" "grafana_agent" {
  metadata {
    name   = "grafana-agent"
    labels = local.k8s_default_labels
  }
  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind      = "ClusterRole"
    name      = kubernetes_cluster_role_v1.grafana-agent.metadata.0.name
  }
  subject {
    kind      = "ServiceAccount"
    name      = kubernetes_service_account_v1.grafana_agent.metadata.0.name
    namespace = helm_release.grafana_agent_operator.namespace
  }
}

resource "kubernetes_manifest" "grafana_agent_metrics_instance" {
  manifest = {
    apiVersion = "monitoring.grafana.com/v1alpha1"
    kind       = "MetricsInstance"
    metadata = {
      name      = "grafana-agent-metrics-instance"
      namespace = helm_release.grafana_agent_operator.namespace
      labels = merge(local.k8s_default_labels, {
        agent = "grafana-agent-metrics"
        app   = "grafana-agent"
      })
    }
    spec = {
      remoteWrite = [
        {
          url = "https://prometheus-prod-10-prod-us-central-0.grafana.net/api/prom/push"
          basicAuth = {
            username = {
              name = "grafana-cloud-credentials"
              key  = "prom_username"
            }
            password = {
              name = "grafana-cloud-credentials"
              key  = "api_key"
            }
          }
        }
      ]
      # Supply an empty namespace selector to look in all namespaces. Remove
      # this to only look in the same namespace as the MetricsInstance CR
      serviceMonitorNamespaceSelector = {}
      serviceMonitorSelector          = {}
      # Supply an empty namespace selector to look in all namespaces. Remove
      # this to only look in the same namespace as the MetricsInstance CR.
      #podMonitorNamespaceSelector = {}
      #podMonitorSelector = {}
      # Supply an empty namespace selector to look in all namespaces. Remove
      # this to only look in the same namespace as the MetricsInstance CR.
      #probeNamespaceSelector = {}
      #probeSelector = {}
    }
  }
}

resource "kubernetes_manifest" "grafana_agent_kubelet_monitor" {
  manifest = {
    apiVersion = "monitoring.coreos.com/v1"
    kind       = "ServiceMonitor"
    metadata = {
      labels    = local.k8s_default_labels
      name      = "kubelet-monitor"
      namespace = helm_release.grafana_agent_operator.namespace
    }
    spec = {
      endpoints = [
        {
          bearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
          honorLabels     = true
          metricRelabelings = [
            {
              action = "keep"
              # FIXME make this a var or a local and join it like the node_exporter stuff
              regex = join("|",
                [
                  "kubelet_cgroup_manager_duration_seconds_count",
                  "kubelet_node_config_error",
                  "kubelet_node_name",
                  "kubelet_pleg_relist_duration_seconds_count",
                  "kubelet_pod_start_duration_seconds_count",
                  "kubelet_pod_worker_duration_seconds_count",
                  "kubelet_running_container_count",
                  "kubelet_running_containers",
                  "kubelet_running_pod_count",
                  "kubelet_running_pods",
                  "kubelet_volume_stats_available_bytes",
                  "kubelet_volume_stats_capacity_bytes",
                  "kubelet_volume_stats_inodes",
                  "kubelet_volume_stats_inodes_used",
                  "process_cpu_seconds_total",
                  "process_resident_memory_bytes"
                ]
              )
              sourceLabels = [
                "__name__"
              ]
            },
            {
              action      = "replace"
              targetLabel = "job"
              replacement = "kubelet"
            }
          ]
          port = "https-metrics"
          relabelings = [
            {
              sourceLabels = [
                "__metrics_path__"
              ]
              targetLabel = "metrics_path"
            }
          ]
          scheme = "https"
          tlsConfig = {
            insecureSkipVerify = true
          }
        }
      ]
      namespaceSelector = {
        matchNames = [
          "default"
        ]
      }
      selector = {
        matchLabels = {
          # quoted because terraform chokes on '/'
          "app.kubernetes.io/name" = "kubelet"
        }
      }
    }
  }
}

resource "kubernetes_manifest" "grafana_agent_cadvisor_monitor" {
  manifest = {
    apiVersion = "monitoring.coreos.com/v1"
    kind       = "ServiceMonitor"
    metadata = {
      labels = {
        app      = "kubernetes-cadvisor"
      }
      name      = "cadvisor-monitor"
      namespace = helm_release.grafana_agent_operator.namespace
    }
    spec = {
      endpoints = [
        {
          bearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
          honorLabels     = true
          honorTimestamps = false
          metricRelabelings = [
            {
              action = "keep"
              regex = join("|",
                [
                  "cadvisor_version_info",
                  "container_cpu_cfs_periods_total",
                  "container_cpu_cfs_throttled_periods_total",
                  "container_cpu_cfs_throttled_seconds_total",
                  "container_cpu_system_seconds_total",
                  "container_cpu_usage_seconds_total",
                  "container_memory_cache",
                  "container_memory_rss",
                  "container_memory_swap",
                  "container_memory_working_set_bytes",
                  "container_network_receive_bytes_total",
                  "container_network_receive_packets_dropped_total",
                  "container_network_receive_packets_total",
                  "container_network_transmit_bytes_total",
                  "container_network_transmit_packets_dropped_total",
                  "container_network_transmit_packets_total",
                  # this one only comes with cadvisor 0.40+, but let's get ready
                  "container_oom_events_total",
                  "container_start_time_seconds"
                ]
              )
              sourceLabels = [
                "__name__"
              ]
            },
            {
              action      = "replace"
              targetLabel = "job"
              replacement = "cadvisor"
            }
          ]
          port = "https-metrics"
          path = "/metrics/cadvisor"
          relabelings = [
            {
              sourceLabels = [
                "__metrics_path__"
              ]
              targetLabel = "metrics_path"
            }
          ]
          scheme = "https"
          tlsConfig = {
            insecureSkipVerify = true
          }
        }
      ]
      namespaceSelector = {
        matchNames = [
          "default"
        ]
      }
      selector = {
        matchLabels = {
          # quoted because terraform chokes on '/'
          "app.kubernetes.io/name" = "kubelet"
        }
      }
    }
  }
}

resource "kubernetes_manifest" "grafana_agent_logs_instance" {
  manifest = {
    apiVersion = "monitoring.grafana.com/v1alpha1"
    kind       = "LogsInstance"
    metadata = {
      name      = "grafana-agent-logs-instance"
      namespace = helm_release.grafana_agent_operator.namespace
      labels = {
        agent = "grafana-agent-logs"
        app   = "grafana-agent"
      }
    }
    spec = {
      clients = [
        {
          url = "https://logs-prod-us-central1.grafana.net/loki/api/v1/push"
          basicAuth = {
            username = {
              name = "grafana-cloud-credentials"
              key  = "loki_username"
            }
            password = {
              name = "grafana-cloud-credentials"
              key  = "api_key"
            }
          }
          externalLabels = {
            engageli_stack_name = var.stack_name
            engageli_stack_env  = var.stack_env
            engageli_region     = var.region
            # can't set the actual build value, we don't have access directly from tf
            engageli_version = var.sw_version
          }
        }
      ]
      # Supply an empty namespace selector to look in all namespaces. Remove
      # this to only look in the same namespace as the LogsInstance CR
      podLogsNamespaceSelector = {}
      podLogsSelector = {
        matchLabels = {
          instance = "primary"
        }
      }
    }
  }
}

resource "kubernetes_manifest" "grafana_agent_pod_logs" {
  manifest = {
    apiVersion = "monitoring.grafana.com/v1alpha1"
    kind       = "PodLogs"
    metadata = {
      name      = "grafana-agent-pod-logs"
      namespace = helm_release.grafana_agent_operator.namespace
      labels = {
        app      = "grafana-agent"
        instance = "primary"
      }
    }
    spec = {
      pipelineStages = [
        {
          docker = {}
        }
      ]
      namespaceSelector = {
        any = true
      }
      selector = {
        matchLabels = {}
      }
    }
  }
}

resource "kubernetes_manifest" "grafana_cloud_credentials" {
  manifest = {
    apiVersion = "external-secrets.io/v1beta1"
    kind       = "ExternalSecret"
    metadata = {
      namespace = helm_release.grafana_agent_operator.namespace
      name      = "grafana-cloud-credentials"
      labels    = local.k8s_default_labels
    }
    spec = {
      refreshInterval = "1h"
      secretStoreRef = {
        name = kubernetes_manifest.external_secrets_secret_store.manifest.metadata.name
        kind = kubernetes_manifest.external_secrets_secret_store.manifest.kind
      }
      target = {
        name           = "grafana-cloud-credentials"
        creationPolicy = "Owner"
        template = {
          engineVersion = "v2"
          data = {
            api_key = "{{ .api_key }}"
            # hardcoded, should be at least as stable as the secret arn...
            prom_username = "230842"
            loki_username = "114392"
          }
        }
      }
      data = [
        {
          secretKey = "api_key"
          remoteRef = {
            key = "arn:aws:secretsmanager:${var.region}:316947578945:secret:grafana-cloud-api-key-FhDjTg"
          }
        }
      ]
    }
  }
  depends_on = [
    helm_release.external_secrets
  ]
}

### end of grafana-agent resources block

### external-secrets resources block

# TODO maybe get some source of truth for the mapping of secret to KMS?
# the ones we define in tf are probably going to be fine, but the external-external ones
# will require us to
data "aws_iam_policy_document" "external_secrets" {
  statement {
    sid    = "secretsmanager"
    effect = "Allow"
    actions = [
      "secretsmanager:GetResourcePolicy",
      "secretsmanager:GetSecretValue",
      "secretsmanager:DescribeSecret",
      "secretsmanager:ListSecretVersionIds"
    ]
    resources = [
      "arn:aws:secretsmanager:*:316947578945:secret:grafana-cloud-api-key-FhDjTg",
      "arn:aws:secretsmanager:*:${var.aws_account}:secret:tf-${var.stack_name}*"
    ]
  }
  statement {
    sid    = "kms"
    effect = "Allow"
    actions = [
      "kms:Decrypt"
    ]
    resources = [
      # specific KMS we use
      "arn:aws:kms:*:316947578945:key/mrk-ccac6208481d4bbf82fe464a98291d3b"
    ]
  }
}

data "aws_iam_policy_document" "external_secrets_assume" {
  version = "2012-10-17"
  statement {
    effect = "Allow"
    principals {
      type        = "Federated"
      identifiers = [module.eks.oidc_provider_arn]
    }
    actions = [
      "sts:AssumeRoleWithWebIdentity"
    ]
    condition {
      test     = "StringEquals"
      variable = "${replace(module.eks.cluster_oidc_issuer_url, "https://", "")}:sub"
      values = [
        "system:serviceaccount:external-secrets:external-secrets"
      ]
    }
  }
}

resource "aws_iam_role" "external_secrets" {
  name               = "${local.eks_cluster_name}-external-secrets-role"
  assume_role_policy = data.aws_iam_policy_document.external_secrets_assume.json
  inline_policy {
    name   = "${local.eks_cluster_name}-external-secrets-policy"
    policy = data.aws_iam_policy_document.external_secrets.json
  }
}

resource "kubernetes_manifest" "external_secrets_secret_store" {
  manifest = {
    apiVersion = "external-secrets.io/v1beta1"
    kind       = "ClusterSecretStore"
    metadata = {
      name   = "stack"
      labels = local.k8s_default_labels
    }
    spec = {
      provider = {
        aws = {
          service = "SecretsManager"
          region  = var.region
        }
      }
    }
  }
  depends_on = [
    helm_release.external_secrets
  ]
}

### end of external-secrets block

resource "kubernetes_manifest" "node_exporter_monitor" {
  manifest = {
    apiVersion = "monitoring.coreos.com/v1"
    kind       = "ServiceMonitor"
    metadata = {
      name      = "node-exporter-monitor"
      namespace = helm_release.node_exporter.namespace
      labels    = local.k8s_default_labels
    }
    spec = {
      endpoints = [
        {
          port   = "metrics"
          scheme = "http"
          metricRelabelings = [
            {
              action = "keep"
              regex = join("|",
                [
                  "node_cpu_seconds_total",
                  "node_load.*",
                  "node_memory_Mem.*",
                  "node_network_receive_packets_dropped_total",
                  "node_network_receive_packets_total",
                  "node_network_transmit_bytes_total",
                  "node_network_transmit_bytes_total",
                  "node_network_transmit_packets_dropped_total",
                  "node_network_transmit_packets_total"
                ]
              )
              sourceLabels = [
                "__name__"
              ]
            }
          ]
        }
      ]
      namespaceSelector = {
        matchNames = [
          helm_release.node_exporter.namespace
        ]
      }
      selector = {
        matchLabels = {
          app     = "prometheus-node-exporter"
          release = "node-exporter"
        }
      }
    }
  }
}

resource "kubernetes_manifest" "cluster_issuer" {
  manifest = {
    apiVersion = "cert-manager.io/v1"
    kind       = "ClusterIssuer"
    metadata = {
      name   = "self-signed-cluster-issuer"
      labels = local.k8s_default_labels
    }
    spec = {
      selfSigned = {}
    }
  }
  depends_on = [
    helm_release.cert_manager
  ]
}


# default is 100 rules per ALB
# if the rules_count gets over it, we need to set the quota
locals {
  #
  # note that the check on config.replicas > 0 means that if you update replicas without terraform,
  # no ALB rules will map to the pods
  # this allows us to postpone dealing with quotas specifically for this
  #
  # the repetition between this and k8s_namespace/main.tf offends my DRY sense, but I see no clean way to fix it
  #
  rules_paths = flatten(
    [
      for namespace, apps in local.engageli_kubernetes_apps :
      [
        for app, config in apps : [
          for path in config.paths : [
            path.path == "" ? "/${namespace}/${app}*" : (path.sandbox_prefixed_path ? "/${namespace}${path.path}*" : "${path.path}*")
          ]
        ]
        if config.create_ingress && config.replicas > 0
      ]
    ]
  )
  rules_priority_map = { for index, path in sort(local.rules_paths) : path => index }
  # +20 as an arbitrary buffer
  rules_count = length(local.rules_paths) + 20
}

data "aws_servicequotas_service" "alb" {
  service_name = "Elastic Load Balancing (ELB)"
}

data "aws_servicequotas_service_quota" "alb_rules" {
  service_code = data.aws_servicequotas_service.alb.service_code
  quota_name   = "Rules per Application Load Balancer"
}

# quotas are per-account, so if we already have something set that's higher than what our current ALB needs,
# don't touch it
resource "aws_servicequotas_service_quota" "alb_rules" {
  quota_code   = data.aws_servicequotas_service_quota.alb_rules.quota_code
  service_code = data.aws_servicequotas_service.alb.service_code
  value        = max(local.rules_count, data.aws_servicequotas_service_quota.alb_rules.value)
}

# this can balloon pretty quick, as we have one rule per service and one service per namespace
#
resource "aws_lb_listener_rule" "alb_to_k8s_rule" {
  for_each     = local.rules_priority_map
  listener_arn = aws_lb_listener.https.arn
  # can't have priority 0
  priority = 1 + each.value
  action {
    type             = "forward"
    target_group_arn = aws_lb_target_group.eks_target_group.arn
  }
  condition {
    path_pattern {
      values = [each.key]
    }
  }
}

locals {
  engageli_kubernetes_apps_defaults = {
    paths = [
      {
        path                  = ""
        sandbox_prefixed_path = true
      }
    ]
    replicas                = 0
    ingress_annotations     = {}
    config_map              = {}
    secret                  = {}
    create_ingress          = true
    host_network            = false
    image_registry          = var.engageli_image_registry_default
    image_repository_suffix = local.image_repository_suffix
    image_tag               = var.engageli_image_tag_default
    iam_role_arn            = ""
    resources = {
      limits = {
        cpu    = "1"
        memory = "512Mi"
      }
      requests = {
        cpu    = "100m"
        memory = "512Mi"
      }
    }
  }

  engageli_kubernetes_sandbox_apps_defaults = {
    collaboration-service = {}
    portal-service        = {}
    backend1 = {
      replicas     = var.eks_backend1_enabled ? 1 : 0
      iam_role_arn = aws_iam_role.engageli_service.arn
      config_map = {
        SENTRY_FLAG = var.stack_env == "dev" ? "" : "--sentry"
      }
      secret = {
        EMS_AUTH_SECRET   = aws_secretsmanager_secret.ems_auth_secret.arn
        EMS_STREAM_SECRET = aws_secretsmanager_secret.ems_stream_secret.arn
      }
    }
  }

  # TODO per sandbox secrets or iam_role_arn getting assigned here (eventually)
  engageli_kubernetes_sandbox_apps = {
    for namespace in toset(var.sandboxes) :
    namespace => {
      for app, config in merge(
        local.engageli_kubernetes_sandbox_apps_defaults,
        var.engageli_kubernetes_sandbox_apps_overrides
      ) : app => config
    }
  }

  # EMS namespace is special, in that its apps depends on external variables like ems_num_clusters
  engageli_kubernetes_ems_apps = {
    ems = merge(
      {
        for i in range(0, var.emsmgr_enabled ? 1 : 0) :
        "emsmgr" => {
          paths = [
            {
              path                  = "/api/emsmgr/"
              sandbox_prefixed_path = false
            }
          ]
          replicas     = 1
          iam_role_arn = aws_iam_role.emsmgr_role.arn
          secret = {
            EMS_AUTH_SECRET            = aws_secretsmanager_secret.ems_auth_secret.arn
            EMS_STREAM_SECRET          = aws_secretsmanager_secret.ems_stream_secret.arn
            EMS_ROUTER_API_AUTH_SECRET = aws_secretsmanager_secret.ems_router_api_auth_secret.arn
          }
        }
      },
      {
        for i in range(0, var.ems_router_enabled ? var.ems_num_clusters : 0) :
        format("ems-router-%02d", i) => {
          paths = [
            {
              path                  = format("/ems%02d/", i)
              sandbox_prefixed_path = false
            },
            {
              path                  = format("/api/ems-router-%02d/", i)
              sandbox_prefixed_path = false
            }
          ]
          replicas = 1
          # hardcoded as the app name is ems-router-XX
          image_repository = "ems-router"
          iam_role_arn     = aws_iam_role.ems_router_autoscaling_role[i].arn
          config_map = {
            EMS_CLUSTER_ID              = format("/ems%02d", i)
            EMS_ASG_NAME                = var.eks_ems_enabled ? "" : aws_autoscaling_group.ems[i].name
            EMS_ASG_MIN_SIZE            = var.ems_asg_min_size
            EMS_ASG_MAX_SIZE            = var.ems_asg_max_size
            EKS_EMS_ENABLED             = var.eks_ems_enabled
            EMSMGR_ENABLED              = var.emsmgr_enabled
            AWS_REGION                  = var.region
            EMS_NODE_SCALING_LOW_WATER  = var.ems_node_scaling_low_water
            EMS_NODE_SCALING_HIGH_WATER = var.ems_node_scaling_high_water
          }
          secret = {
            EMS_AUTH_SECRET            = aws_secretsmanager_secret.ems_auth_secret.arn
            EMS_STREAM_SECRET          = aws_secretsmanager_secret.ems_stream_secret.arn
            EMS_ROUTER_API_AUTH_SECRET = aws_secretsmanager_secret.ems_router_api_auth_secret.arn
          }
        }
      },
      {
        for i in range(0, var.eks_ems_enabled ? 1 : 0) :
        "engageli-media-server-static" => {
          paths = [
            {
              path                  = "/ems-static"
              sandbox_prefixed_path = false
            }
          ]
          replicas = 1
          ingress_annotations = {
            "nginx.ingress.kubernetes.io/backend-protocol" = "HTTPS"
          }
          image_repository = "engageli-media-server"
          config_map = {
            EMS_CLUSTER_ID       = "static"
            CLIENT_USES_HOSTNAME = false
            EMSMGR_ENABLED       = false
          }
          secret = {
            EMS_AUTH_SECRET   = aws_secretsmanager_secret.ems_auth_secret.arn
            EMS_STREAM_SECRET = aws_secretsmanager_secret.ems_stream_secret.arn
          }
          # arbitrarily lower resources, only serves a static file & only if hit from cloudfront
          # FIXME: check actual resource usage and tighten it
          resources = {
            limits = {
              cpu    = "250m"
              memory = "500Mi"
            }
            requests = {
              cpu    = "250m"
              memory = "500Mi"
            }
          }
        }
      },
      {
        for i in range(0, var.eks_ems_enabled ? var.ems_num_clusters : 0) :
        format("engageli-media-server-%02d", i) => {
          paths = [
            {
              # TODO: is the 'changeme' needed?
              path                  = var.ems_router_enabled ? format("/changeme/ems%02d/", i) : format("/ems%02d/", i)
              sandbox_prefixed_path = false
            }
          ]
          replicas         = 1
          create_ingress   = var.ems_router_enabled ? false : true
          host_network     = true
          image_repository = "engageli-media-server"
          ingress_annotations = {
            "nginx.ingress.kubernetes.io/backend-protocol"   = "HTTPS"
            "nginx.ingress.kubernetes.io/proxy-read-timeout" = "3600"
            "nginx.ingress.kubernetes.io/proxy-send-timeout" = "3600"
          }
          config_map = {
            EMS_CLUSTER_ID = format("/ems%02d", i)
            EMSMGR_ENABLED = var.emsmgr_enabled
            # directly set this into the environment - EMS on EKS will use the stack's fqdn just like EBS
            CLIENT_USES_HOSTNAME = false
            EMSMGR_FQDN          = "${var.stack_name}.${var.domain}"
          }
          secret = {
            EMS_AUTH_SECRET   = aws_secretsmanager_secret.ems_auth_secret.arn
            EMS_STREAM_SECRET = aws_secretsmanager_secret.ems_stream_secret.arn
          }
        }
      }
    )
  }

  engageli_kubernetes_apps = {
    for namespace, apps in merge(
      local.engageli_kubernetes_sandbox_apps,
      local.engageli_kubernetes_ems_apps
    ) :
    namespace => {
      for app, config in apps :
      app => merge(
        local.engageli_kubernetes_apps_defaults,
        config,
        {
          image_repository = lookup(config, "image_repository", app)
        },
        lookup(
          lookup(var.engageli_kubernetes_apps_overrides, namespace, {}),
          app,
          {}
        )
      )
    }
  }
}


output "eks_cluster" {
  value = {
    id            = module.eks.cluster_id
    arn           = module.eks.cluster_arn
    version       = module.eks.cluster_version
    oidc          = module.eks.cluster_oidc_issuer_url
    eks_vpc_chunk = local.eks_vpc_chunk
    eks_subnets   = local.eks_subnets
  }
}