From 0ffc732dc21edc62547d63fc839addfcce82fd35 Mon Sep 17 00:00:00 2001 From: maxim Date: Wed, 18 May 2022 13:48:39 +0600 Subject: [PATCH 1/5] enh: use aws-load-balancer-controller in front of ingress-nginx --- .../eks-aws-loadbalancer-controller.tf | 125 +++++++++++++++++- .../eks-ingress-nginx-controller.tf | 18 +-- terraform/layer2-k8s/helm-releases.yaml | 4 +- 3 files changed, 130 insertions(+), 17 deletions(-) diff --git a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf index db871145..e8ce2ee0 100644 --- a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf +++ b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf @@ -7,7 +7,10 @@ locals { chart_version = local.helm_releases[index(local.helm_releases.*.id, "aws-load-balancer-controller")].chart_version namespace = local.helm_releases[index(local.helm_releases.*.id, "aws-load-balancer-controller")].namespace } - aws_load_balancer_controller_values = < Date: Wed, 18 May 2022 13:53:35 +0600 Subject: [PATCH 2/5] update readme.md using terraform-docs --- terraform/layer2-k8s/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/terraform/layer2-k8s/README.md b/terraform/layer2-k8s/README.md index 652af543..86f4975c 100644 --- a/terraform/layer2-k8s/README.md +++ b/terraform/layer2-k8s/README.md @@ -21,6 +21,7 @@ | [kubernetes](#provider\_kubernetes) | 2.10.0 | | [random](#provider\_random) | 3.1.3 | | [terraform](#provider\_terraform) | n/a | +| [tls](#provider\_tls) | 3.3.0 | ## Modules @@ -86,6 +87,7 @@ | [kubectl_manifest.istio_prometheus_service_monitor_cp](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource | | [kubectl_manifest.istio_prometheus_service_monitor_dp](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource | | [kubectl_manifest.kube_prometheus_stack_operator_crds](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource | +| [kubernetes_ingress_v1.default](https://registry.terraform.io/providers/kubernetes/2.10.0/docs/resources/ingress_v1) | resource | | [kubernetes_secret.elasticsearch_certificates](https://registry.terraform.io/providers/kubernetes/2.10.0/docs/resources/secret) | resource | | [kubernetes_secret.elasticsearch_credentials](https://registry.terraform.io/providers/kubernetes/2.10.0/docs/resources/secret) | resource | | [kubernetes_secret.elasticsearch_s3_user_creds](https://registry.terraform.io/providers/kubernetes/2.10.0/docs/resources/secret) | resource | @@ -96,6 +98,11 @@ | [random_string.kibana_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [random_string.kube_prometheus_stack_grafana_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [random_string.victoria_metrics_k8s_stack_grafana_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [tls_cert_request.aws_loadbalancer_controller_webhook](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/cert_request) | resource | +| [tls_locally_signed_cert.aws_loadbalancer_controller_webhook](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/locally_signed_cert) | resource | +| [tls_private_key.aws_loadbalancer_controller_webhook](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | +| [tls_private_key.aws_loadbalancer_controller_webhook_ca](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource | +| [tls_self_signed_cert.aws_loadbalancer_controller_webhook_ca](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/self_signed_cert) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/aws/4.10.0/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster.main](https://registry.terraform.io/providers/aws/4.10.0/docs/data-sources/eks_cluster) | data source | | [aws_eks_cluster_auth.main](https://registry.terraform.io/providers/aws/4.10.0/docs/data-sources/eks_cluster_auth) | data source | From f7bc85164f7148df12405f2f7afe8d0beae0fa37 Mon Sep 17 00:00:00 2001 From: maxim Date: Wed, 18 May 2022 15:15:05 +0600 Subject: [PATCH 3/5] fix ingress annotations for ssl-redirects (301 will be used) --- terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf index e8ce2ee0..a31ca344 100644 --- a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf +++ b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf @@ -432,7 +432,7 @@ resource "kubernetes_ingress_v1" "default" { "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTP\": 80}, {\"HTTPS\": 443}]" "alb.ingress.kubernetes.io/target-type" = "ip" "alb.ingress.kubernetes.io/load-balancer-attributes" = "routing.http2.enabled=true" - "alb.ingress.kubernetes.io/actions.ssl-redirect" = "{\"Type\": \"redirect\", \"RedirectConfig\": { \"Protocol\": \"HTTPS\", \"Port\": \"443\", \"StatusCode\": \"HTTP_302\"}}" + "alb.ingress.kubernetes.io/ssl-redirect" = "443" } namespace = module.ingress_nginx_namespace[count.index].name } From b41cab48a5ec182d297de16e151179ad61ced0b0 Mon Sep 17 00:00:00 2001 From: maxim Date: Fri, 20 May 2022 17:08:21 +0600 Subject: [PATCH 4/5] automatically create dns record points to lb --- README.md | 7 +++++-- docs/FAQ.md | 7 +++++++ terraform/layer2-k8s/README.md | 1 + .../eks-aws-loadbalancer-controller.tf | 18 +++++++++++++++++- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0c44a980..6ed201f5 100644 --- a/README.md +++ b/README.md @@ -396,14 +396,17 @@ terragrunt destroy ## What to do after deployment -After applying this configuration, you will get the infrastructure described and outlined at the beginning of the document. In AWS and within the EKS cluster, the basic resources and services necessary for the operation of the EKS k8s cluster will be created. +* After applying this configuration, you will get the infrastructure described and outlined at the beginning of the document. In AWS and within the EKS cluster, the basic resources and services necessary for the operation of the EKS k8s cluster will be created (except configuration for Route53). -You can get access to the cluster using this command: +* You can get access to the cluster using this command: ```bash aws eks update-kubeconfig --name maddevs-demo-use1 --region us-east-1 ``` +* If you used default configuration and want to serve traffic for a main domain (example.com) by an application deployed into a k8s cluster, youn need to manually create DNS record in Route53 with type A + Alias +* DNS record `*.example.com` created automatically and points to Load Balancer in front of k8s cluster. + ## Update terraform version Change terraform version in this files diff --git a/docs/FAQ.md b/docs/FAQ.md index f8b01169..0ea297bf 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -304,3 +304,10 @@ kubectl delete installations.operator.tigera.io default kubectl delete ns calico-apiserver calico-system ``` 5. Restart all nodes + +## What if you don't want to use an aws-load-balancer controller in front of an ingress-nginx and want to use a cert-manager and terminate SSL on ingres-nginx side + +1. Set `nginx ` for a `nginx_ingress_ssl_terminator` variable in the layer2-k8s folder +2. Set `enabled: false` for `id: aws-load-balancer-controller` in the **layer2-k8s/helm-releases.yaml** file +3. Set `enabled: true` for `id: external-dns`, `id: cert-manager`, `id: cert-mananger-certificate`, `id:cert-manager-cluster-issuer` in the **layer2-k8s/helm-releases.yaml** file +4. Run `terraform apply` in the layer2-k8s folder diff --git a/terraform/layer2-k8s/README.md b/terraform/layer2-k8s/README.md index 86f4975c..d1bf58d1 100644 --- a/terraform/layer2-k8s/README.md +++ b/terraform/layer2-k8s/README.md @@ -59,6 +59,7 @@ | Name | Type | |------|------| +| [aws_route53_record.default_ingress](https://registry.terraform.io/providers/aws/4.10.0/docs/resources/route53_record) | resource | | [aws_s3_bucket.elastic_stack](https://registry.terraform.io/providers/aws/4.10.0/docs/resources/s3_bucket) | resource | | [aws_s3_bucket.gitlab_runner_cache](https://registry.terraform.io/providers/aws/4.10.0/docs/resources/s3_bucket) | resource | | [aws_s3_bucket_public_access_block.elastic_stack_public_access_block](https://registry.terraform.io/providers/aws/4.10.0/docs/resources/s3_bucket_public_access_block) | resource | diff --git a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf index a31ca344..13cf8aa3 100644 --- a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf +++ b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf @@ -419,7 +419,7 @@ resource "helm_release" "aws_loadbalancer_controller" { } resource "kubernetes_ingress_v1" "default" { - count = local.aws_load_balancer_controller.enabled && local.ingress_nginx.enabled ? 1 : 0 + count = local.aws_load_balancer_controller.enabled && local.ingress_nginx.enabled && var.nginx_ingress_ssl_terminator == "lb" ? 1 : 0 metadata { name = "${local.ingress_nginx.name}-controller" @@ -453,6 +453,22 @@ resource "kubernetes_ingress_v1" "default" { } } } + wait_for_load_balancer = true depends_on = [helm_release.aws_loadbalancer_controller, helm_release.ingress_nginx, module.aws_iam_aws_loadbalancer_controller, tls_locally_signed_cert.aws_loadbalancer_controller_webhook] } + +resource "aws_route53_record" "default_ingress" { + count = local.aws_load_balancer_controller.enabled && local.ingress_nginx.enabled && var.nginx_ingress_ssl_terminator == "lb" ? 1 : 0 + + zone_id = local.zone_id + name = "*.${local.domain_name}" + type = "CNAME" + ttl = 360 + + records = [kubernetes_ingress_v1.default[count.index].status.0.load_balancer.0.ingress.0.hostname] + + depends_on = [ + kubernetes_ingress_v1.default + ] +} From 53217f34f330e7e4047061f7c4a1e89dacb46404 Mon Sep 17 00:00:00 2001 From: maxim Date: Fri, 20 May 2022 17:47:35 +0600 Subject: [PATCH 5/5] fix README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ed201f5..3bb19130 100644 --- a/README.md +++ b/README.md @@ -396,7 +396,7 @@ terragrunt destroy ## What to do after deployment -* After applying this configuration, you will get the infrastructure described and outlined at the beginning of the document. In AWS and within the EKS cluster, the basic resources and services necessary for the operation of the EKS k8s cluster will be created (except configuration for Route53). +* After applying this configuration, you will get the infrastructure described and outlined at the beginning of the document. In AWS and within the EKS cluster, the basic resources and services necessary for the operation of the EKS k8s cluster will be created. * You can get access to the cluster using this command: