Update AWS EKS to v1.27

infrastructure/base/aws.tf

@@ -17,10 +17,10 @@ module "bootstrap" {
 
 # Internal module which defines the VPC
 module "vpc" {
-  source  = "./modules/aws/vpc"
-  region  = var.aws_region
-  project = var.project_name
-  tags    = local.tags
+  source              = "./modules/aws/vpc"
+  region              = var.aws_region
+  project             = var.project_name
+  tags                = local.tags
   private_subnet_tags = {
     "kubernetes.io/role/internal-elb" : 1
 
@@ -42,20 +42,23 @@ module "bastion" {
 }
 
 module "dns" {
-  source = "./modules/aws/dns"
-  domain = var.domain
+  source              = "./modules/aws/dns"
+  domain              = var.domain
   site_server_ip_list = [
     module.load_balancer.load-balancer-ip
   ]
   bastion_hostname = module.bastion.bastion_hostname
 }
 
 module "eks" {
-  source     = "./modules/aws/eks"
-  project    = var.project_name
-  vpc_id     = module.vpc.id
-  subnet_ids = module.vpc.private_subnets.*.id
-  aws_region = var.aws_region
+  source                = "./modules/aws/eks"
+  project               = var.project_name
+  vpc_id                = module.vpc.id
+  subnet_ids            = module.vpc.private_subnets.*.id
+  aws_region            = var.aws_region
+  ebs_csi_addon_version = var.ebs_csi_addon_version
+  k8s_version           = var.eks_cluster_version
+  coredns_addon_version = var.coredns_addon_version
 }
 
 module "default-node-group" {
@@ -69,7 +72,7 @@ module "default-node-group" {
   desired_size    = var.default_node_group_desired_size
   node_role_arn   = module.eks.node_role.arn
   subnet_ids      = module.vpc.private_subnets.*.id
-  labels = {
+  labels          = {
     type : "default"
   }
 }
@@ -85,7 +88,7 @@ module "data-node-group" {
   desired_size    = var.data_node_group_desired_size
   node_role_arn   = module.eks.node_role.arn
   subnet_ids      = [module.vpc.private_subnets[0].id]
-  labels = {
+  labels          = {
     type : "data"
   }
 }
@@ -125,13 +128,13 @@ resource "aws_iam_policy" "raw_s3_rw_access" {
   description = "Read + write access to the raw data S3 bucket"
 
   policy = jsonencode({
-    Version = "2012-10-17"
+    Version   = "2012-10-17"
     Statement = [
       {
         "Action" : [
           "s3:*",
         ],
-        Effect = "Allow"
+        Effect   = "Allow"
         Resource = [
           module.s3_bucket.bucket_arn,
           "${module.s3_bucket.bucket_arn}/*",
@@ -155,14 +158,14 @@ resource "aws_iam_policy" "raw_s3_read_access" {
   description = "Read access to the raw data S3 bucket"
 
   policy = jsonencode({
-    Version = "2012-10-17"
+    Version   = "2012-10-17"
     Statement = [
       {
         "Action" : [
           "s3:Get*",
           "s3:List*",
         ],
-        Effect = "Allow"
+        Effect   = "Allow"
         Resource = [
           module.s3_bucket.bucket_arn,
           "${module.s3_bucket.bucket_arn}/*",

infrastructure/base/modules/aws/eks/ebs.tf

@@ -0,0 +1,51 @@
+resource "aws_eks_addon" "aws_ebs_csi_driver" {
+  cluster_name             = aws_eks_cluster.eks_cluster.name
+  addon_name               = "aws-ebs-csi-driver"
+  addon_version            = var.ebs_csi_addon_version
+  service_account_role_arn = aws_iam_role.ebs_csi_iam_role.arn
+}
+
+data "aws_caller_identity" "current" {}
+
+data "external" "thumbprint" {
+  program = ["${path.module}/thumbprint.sh", var.aws_region]
+}
+
+resource "aws_iam_openid_connect_provider" "example" {
+  client_id_list  = ["sts.amazonaws.com"]
+  thumbprint_list = [data.external.thumbprint.result.thumbprint]
+  url             = aws_eks_cluster.eks_cluster.identity.0.oidc.0.issuer
+}
+
+resource "aws_iam_role" "ebs_csi_iam_role" {
+  name = "AmazonEKS_EBS_CSI_DriverRole"
+
+  assume_role_policy = jsonencode({
+    Version : "2012-10-17",
+    Statement : [
+      {
+        Effect : "Allow",
+        Principal : {
+          "Federated" : "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/oidc.eks.${var.aws_region}.amazonaws.com/id/${local.oicd_id}"
+        },
+        Action : "sts:AssumeRoleWithWebIdentity",
+        Condition : {
+          StringEquals : {
+            "oidc.eks.${var.aws_region}.amazonaws.com/id/${local.oicd_id}:aud" : "sts.amazonaws.com",
+            "oidc.eks.${var.aws_region}.amazonaws.com/id/${local.oicd_id}:sub" : "system:serviceaccount:kube-system:ebs-csi-controller-sa"
+          }
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "ebs-csi-service-role-AmazonEKS_EBS_CSI_DriverRole" {
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
+  role       = aws_iam_role.ebs_csi_iam_role.name
+}
+
+resource "aws_iam_role_policy_attachment" "eks-node-group-admin-AmazonEKS_EBS_CSI_DriverRole" {
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
+  role       = aws_iam_role.eks-node-group-iam-role.name
+}

infrastructure/base/modules/aws/eks/main.tf

@@ -1,6 +1,11 @@
 #
 # EKS resources
 #
+
+locals {
+  oicd_id = element(split("/", aws_eks_cluster.eks_cluster.identity.0.oidc.0.issuer), length(split("/", aws_eks_cluster.eks_cluster.identity.0.oidc.0.issuer)) - 1)
+}
+
 resource "aws_eks_cluster" "eks_cluster" {
   name     = "${replace(var.project, " ", "-")}-k8s-cluster"
   role_arn = aws_iam_role.eks-cluster-admin.arn
@@ -17,10 +22,12 @@ resource "aws_eks_cluster" "eks_cluster" {
     aws_iam_role_policy_attachment.eks-admin-AmazonEKSClusterPolicy,
     aws_iam_role_policy_attachment.eks-admin-AmazonEKSServicePolicy,
   ]
+}
 
-  lifecycle {
-    ignore_changes = [version]
-  }
+resource "aws_eks_addon" "aws_coredns" {
+  cluster_name             = aws_eks_cluster.eks_cluster.name
+  addon_name               = "coredns"
+  addon_version            = var.coredns_addon_version
 }
 
 resource "aws_security_group" "eks_cluster_security_group" {
@@ -76,16 +83,6 @@ resource "aws_iam_role_policy_attachment" "eks-admin-AmazonEKSServicePolicy" {
   role       = aws_iam_role.eks-cluster-admin.name
 }
 
-data "external" "thumbprint" {
-  program = ["${path.module}/thumbprint.sh", var.aws_region]
-}
-
-resource "aws_iam_openid_connect_provider" "example" {
-  client_id_list  = ["sts.amazonaws.com"]
-  thumbprint_list = [data.external.thumbprint.result.thumbprint]
-  url             = aws_eks_cluster.eks_cluster.identity.0.oidc.0.issuer
-}
-
 #
 # Node Group shared resources
 #
@@ -105,31 +102,11 @@ resource "aws_iam_role" "eks-node-group-iam-role" {
   })
 }
 
-data "aws_iam_policy_document" "eks-admin-ClusterAutoscaleAccessPolicy-document" {
-  source_policy_documents = [file("${path.module}/cluster-autoscale-access-policy.json")]
-}
-
-resource "aws_iam_policy" "eks-admin-ClusterAutoscaleAccessPolicy" {
-  name   = "ClusterAutoscaleAccessPolicy"
-  path   = "/"
-  policy = data.aws_iam_policy_document.eks-admin-ClusterAutoscaleAccessPolicy-document.json
-}
-
-resource "aws_iam_role_policy_attachment" "eks-admin-ClusterAutoscaleAccessPolicy" {
-  policy_arn = aws_iam_policy.eks-admin-ClusterAutoscaleAccessPolicy.arn
-  role       = aws_iam_role.eks-node-group-iam-role.name
-}
-
 resource "aws_iam_role_policy_attachment" "eks-node-group-admin-AmazonEKSWorkerNodePolicy" {
   policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
   role       = aws_iam_role.eks-node-group-iam-role.name
 }
 
-resource "aws_iam_role_policy_attachment" "eks-node-group-admin-AmazonEKS_CNI_Policy" {
-  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
-  role       = aws_iam_role.eks-node-group-iam-role.name
-}
-
 resource "aws_iam_role_policy_attachment" "eks-node-group-admin-AmazonEC2ContainerRegistryReadOnly" {
   policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
   role       = aws_iam_role.eks-node-group-iam-role.name

infrastructure/base/modules/aws/eks/variable.tf

@@ -18,9 +18,17 @@ variable "vpc_id" {
   description = "ID of the VPC."
 }
 
-
 variable "k8s_version" {
   type        = string
   description = "Version of Kubernetes to use"
-  default     = "1.21"
+}
+
+variable "ebs_csi_addon_version" {
+  type        = string
+  description = "Version of AWS EBS CRI driver to use"
+}
+
+variable "coredns_addon_version" {
+  type        = string
+  description = "Version of AWS Core DNS addon to use"
 }

infrastructure/base/modules/aws/node_group/main.tf

@@ -9,10 +9,16 @@ resource "random_id" "eks-node-group" {
   byte_length = 8
 }
 
+data "aws_ssm_parameter" "eks_ami_release_version" {
+  name = "/aws/service/eks/optimized-ami/${var.cluster.version}/amazon-linux-2/recommended/release_version"
+}
+
 resource "aws_eks_node_group" "eks-node-group" {
   cluster_name    = var.cluster_name
   node_group_name = "${var.node_group_name}-${random_id.eks-node-group.hex}"
   node_role_arn   = var.node_role_arn
+  version         = var.cluster.version
+  release_version = nonsensitive(data.aws_ssm_parameter.eks_ami_release_version.value)
   subnet_ids      = var.subnet_ids
 
   scaling_config {

infrastructure/base/variables.tf

@@ -84,6 +84,25 @@ variable "rds_backup_retention_period" {
   description = "Time in days to keep db backups"
 }
 
+#
+# EKS
+#
+
+variable "ebs_csi_addon_version" {
+  type        = string
+  description = "Version of AWS EBS CRI driver to use"
+}
+
+variable "coredns_addon_version" {
+  type        = string
+  description = "Version of AWS Core DNS addon to use"
+}
+
+variable "eks_cluster_version" {
+  type        = string
+  description = "Version of EKS (kubernetes) cluster deploy"
+}
+
 #
 # EKS default node group
 #

infrastructure/base/vars/terraform.tfvars

@@ -4,8 +4,17 @@ allowed_account_id = "622152552144"
 domain             = "landgriffon.com"
 rds_engine_version = "13.2"
 
-gcp_project_id     = "landgriffon"
-gcp_region         = "europe-west1"
-marketing_site_tag = "dev"
+gcp_project_id = "landgriffon"
+gcp_region     = "europe-west1"
+gcp_zone       = "europe-west1-b"
 
-marketing_site_google_analytics = "271375789"
+ebs_csi_addon_version           = "v1.19.0-eksbuild.1"
+coredns_addon_version           = "v1.10.1-eksbuild.1"
+eks_cluster_version             = "1.27"
+
+marketing_site_tag                           = "dev"
+marketing_site_google_analytics              = "271375789"
+marketing_site_sendgrid_api_key_subscription = ""
+marketing_site_sendgrid_api_key_contact      = ""
+
+repo_name = "landgriffon"

infrastructure/kubernetes/main.tf

@@ -32,11 +32,13 @@ resource "aws_iam_access_key" "access_key" {
 }
 
 module "k8s_infrastructure" {
-  source                = "./modules/k8s_infrastructure"
-  cluster_name          = data.terraform_remote_state.core.outputs.eks_cluster_name
-  aws_region            = var.aws_region
-  vpc_id                = data.aws_eks_cluster.cluster.vpc_config[0].vpc_id
-  deploy_metrics_server = false
+  source                   = "./modules/k8s_infrastructure"
+  cluster_name             = data.terraform_remote_state.core.outputs.eks_cluster_name
+  aws_region               = var.aws_region
+  vpc_id                   = data.aws_eks_cluster.cluster.vpc_config[0].vpc_id
+  deploy_metrics_server    = false
+  vpc_cni_addon_version    = var.vpc_cni_addon_version
+  kube_proxy_addon_version = var.kube_proxy_addon_version
 
   providers = {
     helm       = helm.aws_helm

infrastructure/kubernetes/modules/aws/node_group/main.tf

@@ -1,3 +1,7 @@
+data "aws_eks_cluster" "eks_cluster" {
+  name = var.cluster_name
+}
+
 data "aws_eks_node_groups" "node_groups" {
   cluster_name = var.cluster_name
 }
@@ -16,10 +20,17 @@ resource "random_id" "eks-node-group" {
   byte_length = 8
 }
 
+data "aws_ssm_parameter" "eks_ami_release_version" {
+  name = "/aws/service/eks/optimized-ami/${data.aws_eks_cluster.eks_cluster.version}/amazon-linux-2/recommended/release_version"
+}
+
 resource "aws_eks_node_group" "eks-node-group" {
   cluster_name    = var.cluster_name
   node_group_name = "${var.node_group_name}-${random_id.eks-node-group.hex}"
   node_role_arn   = data.aws_eks_node_group.node_group.node_role_arn
+  version         = data.aws_eks_cluster.eks_cluster.version
+  release_version = nonsensitive(data.aws_ssm_parameter.eks_ami_release_version.value)
+
   subnet_ids      = var.subnet_ids
 
   scaling_config {