Add new provider configuration to fix k8s auth issues

templates/Makefile

@@ -26,7 +26,7 @@ apply-env:
 pre-k8s:
 	@echo "Creating VPN private key..."
 	WGKEY=$(shell kubectl run -i --tty zero-k8s-utilities --image=commitdev/zero-k8s-utilities:0.0.3 --restart=Never -- wg genkey) && kubectl delete pod/zero-k8s-utilities && \
-	aws secretsmanager create-secret --name $(PROJECT)-$(ENVIRONMENT)-vpn-wg-privatekey-<% index .Params `randomSeed` %> --description "Auto-generated Wireguard VPN private key" --secret-string $$WGKEY
+	aws secretsmanager create-secret --region <% index .Params `region` %> --name $(PROJECT)-$(ENVIRONMENT)-vpn-wg-privatekey-<% index .Params `randomSeed` %> --description "Auto-generated Wireguard VPN private key" --secret-string $$WGKEY
 	@echo "Done VPN private key creation"
 
 apply-k8s-utils:
@@ -56,7 +56,8 @@ teardown-secrets:
 	aws secretsmanager list-secrets --region <% index .Params `region` %> --query "SecretList[?Tags[?Key=='rds' && Value=='$(PROJECT)-$(ENVIRONMENT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region <% index .Params `region` %> --secret-id && \
 	aws secretsmanager list-secrets --region <% index .Params `region` %> --query "SecretList[?Tags[?Key=='sendgrid' && Value=='$(PROJECT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region <% index .Params `region` %> --secret-id && \
 	aws iam delete-access-key --user-name $(PROJECT)-ci-user --access-key-id $(shell aws iam list-access-keys --user-name $(PROJECT)-ci-user --query "AccessKeyMetadata[0].AccessKeyId" | sed 's/"//g') && \
-	aws iam delete-user --user-name $(PROJECT)-ci-user
+	aws iam delete-user --user-name $(PROJECT)-ci-user && \
+	aws iam delete-role --role-name $(PROJECT)-eks-cluster-creator
 
 teardown-env:
 	cd terraform/environments/$(ENVIRONMENT) && \

templates/kubernetes/terraform/environments/dev/main.tf

@@ -8,15 +8,21 @@ terraform {
   }
 }
 
+provider "aws" {
+  region              = "<% index .Params `region` %>"
+  allowed_account_ids = ["<% index .Params `accountId` %>"]
+}
+
 # Provision kubernetes resources required to run services/applications
 module "kubernetes" {
   source = "../../modules/kubernetes"
 
   project = "<% .Name %>"
 
-  environment = "dev"
-  region      = "<% index .Params `region` %>"
-  random_seed = "<% index .Params `randomSeed` %>"
+  environment         = "dev"
+  region              = "<% index .Params `region` %>"
+  allowed_account_ids = ["<% index .Params `accountId` %>"]
+  random_seed         = "<% index .Params `randomSeed` %>"
 
   # Authenticate with the EKS cluster via the cluster id
   cluster_name = "<% .Name %>-dev-<% index .Params `region` %>"
@@ -30,14 +36,16 @@ module "kubernetes" {
   # Logging configuration
   logging_type = "<% index .Params `loggingType` %>"
 
-  # Application policy list
+  # Application policy list - This allows applications running in kubernetes to have access to AWS resources.
+  # Specify the service account name, the namespace, and the policy that should be applied.
+  # This makes use of IRSA: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/
   application_policy_list = [
     {
       service_account = "backend-service"
       namespace       = "<% .Name %>"
       policy          = data.aws_iam_policy_document.resource_access_backendservice
     }
-    # could be more policies defined here (if have)
+    # Add additional mappings here
   ]
 
   # Wireguard configuration

templates/kubernetes/terraform/environments/prod/main.tf

@@ -9,17 +9,21 @@ terraform {
 }
 
 provider "aws" {
-  region = "<% index .Params `region` %>"
+  region              = "<% index .Params `region` %>"
+  allowed_account_ids = ["<% index .Params `accountId` %>"]
 }
 
+
 # Provision kubernetes resources required to run services/applications
 module "kubernetes" {
   source = "../../modules/kubernetes"
 
   project = "<% .Name %>"
 
-  environment = "prod"
-  region      = "<% index .Params `region` %>"
+  environment         = "prod"
+  region              = "<% index .Params `region` %>"
+  allowed_account_ids = ["<% index .Params `accountId` %>"]
+  random_seed         = "<% index .Params `randomSeed` %>"
 
   # Authenticate with the EKS cluster via the cluster id
   cluster_name = "<% .Name %>-prod-<% index .Params `region` %>"
@@ -33,16 +37,19 @@ module "kubernetes" {
   # Logging configuration
   logging_type = "<% index .Params `loggingType` %>"
 
-  # Application policy list
+  # Application policy list - This allows applications running in kubernetes to have access to AWS resources.
+  # Specify the service account name, the namespace, and the policy that should be applied.
+  # This makes use of IRSA: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/
   application_policy_list = [
     {
       service_account = "backend-service"
       namespace       = "<% .Name %>"
       policy          = data.aws_iam_policy_document.resource_access_backendservice
     }
-    # could be more policies defined here (if have)
+    # Add additional mappings here
   ]
 
+
   # Wireguard configuration
   vpn_server_address = "10.10.99.0/24"
   vpn_client_publickeys = [

templates/kubernetes/terraform/environments/stage/main.tf

@@ -18,9 +18,10 @@ module "kubernetes" {
 
   project = "<% .Name %>"
 
-  environment = "stage"
-  region      = "<% index .Params `region` %>"
-  random_seed = "<% index .Params `randomSeed` %>"
+  environment         = "stage"
+  region              = "<% index .Params `region` %>"
+  allowed_account_ids = ["<% index .Params `accountId` %>"]
+  random_seed         = "<% index .Params `randomSeed` %>"
 
   # Authenticate with the EKS cluster via the cluster id
   cluster_name = "<% .Name %>-stage-<% index .Params `region` %>"
@@ -34,16 +35,19 @@ module "kubernetes" {
   # Logging configuration
   logging_type = "<% index .Params `loggingType` %>"
 
-  # Application policy list
+  # Application policy list - This allows applications running in kubernetes to have access to AWS resources.
+  # Specify the service account name, the namespace, and the policy that should be applied.
+  # This makes use of IRSA: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/
   application_policy_list = [
     {
       service_account = "backend-service"
       namespace       = "<% .Name %>"
       policy          = data.aws_iam_policy_document.resource_access_backendservice
     }
-    # could be more policies defined here (if have)
+    # Add additional mappings here
   ]
 
+
   # Wireguard configuration
   vpn_server_address = "10.10.199.0/24"
   vpn_client_publickeys = [

templates/kubernetes/terraform/modules/kubernetes/backend_service.tf

@@ -1,5 +1,5 @@
 data "aws_secretsmanager_secret" "cf_keypair" {
-  name = "{var.project}_cf_keypair"
+  name = "${var.project}_cf_keypair"
 }
 
 data "aws_secretsmanager_secret_version" "cf_keypair" {

templates/kubernetes/terraform/modules/kubernetes/provider.tf

@@ -1,27 +1,38 @@
 data "aws_caller_identity" "current" {}
 
+
+# Created by bootstrap/secrets
+data "aws_iam_role" "eks_cluster_creator" {
+  name = "${var.project}-eks-cluster-creator"
+}
+
+# Used only for EKS creation to tie "cluster creator" to a role instead of the user who runs terraform
+# This allows us to rely on credentials pulled from the EKS cluster instead of the user's local kube config
+provider "aws" {
+  alias = "for_eks"
+
+  region              = var.region
+  allowed_account_ids = var.allowed_account_ids
+
+  assume_role {
+    role_arn = data.aws_iam_role.eks_cluster_creator.arn
+  }
+}
+
 data "aws_eks_cluster" "cluster" {
-  name = var.cluster_name
+  provider = aws.for_eks
+  name     = var.cluster_name
 }
 
-data "aws_eks_cluster_auth" "cluster_auth" {
-  name = data.aws_eks_cluster.cluster.name
+data "aws_eks_cluster_auth" "cluster" {
+  provider = aws.for_eks
+  name     = var.cluster_name
 }
 
 provider "kubernetes" {
-  ## This is a workaround because aws-eks-cluster-auth will default to us-east-1
-  ## leading to an invalid token to access the cluster
-  exec {
-    api_version = "client.authentication.k8s.io/v1alpha1"
-    command     = "aws"
-    args = [
-      "eks",
-      "get-token",
-      "--region",
-      var.region,
-      "--cluster-name",
-      var.cluster_name,
-      "--role",
-    "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.project}-kubernetes-admin-${var.environment}"]
-  }
+  host                   = data.aws_eks_cluster.cluster.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
+  token                  = data.aws_eks_cluster_auth.cluster.token
+  load_config_file       = false
+  version                = "~> 1.11"
 }

templates/kubernetes/terraform/modules/kubernetes/variables.tf

@@ -6,6 +6,11 @@ variable "project" {
   description = "The name of the project"
 }
 
+variable "allowed_account_ids" {
+  description = "The IDs of AWS accounts for this project, to protect against mistakenly applying to the wrong env"
+  type        = list(string)
+}
+
 variable "environment" {
   description = "Environment"
 }

templates/kubernetes/terraform/modules/kubernetes/vpn.tf

@@ -65,11 +65,11 @@ resource "kubernetes_namespace" "vpn_namespace" {
 resource "kubernetes_secret" "vpn_private_key" {
   metadata {
     name      = "wg-secret"
-    namespace = local.namespace
+    namespace = kubernetes_namespace.vpn_namespace.metadata[0].name
   }
 
   data = {
-    privatekey = jsondecode(data.aws_secretsmanager_secret_version.vpn_private_key.secret_string)["key"]
+    privatekey = data.aws_secretsmanager_secret_version.vpn_private_key.secret_string
   }
 
   type = "Opaque"
@@ -78,7 +78,7 @@ resource "kubernetes_secret" "vpn_private_key" {
 resource "kubernetes_config_map" "vpn_configmap" {
   metadata {
     name      = "wg-configmap"
-    namespace = local.namespace
+    namespace = kubernetes_namespace.vpn_namespace.metadata[0].name
   }
 
   data = {
@@ -89,7 +89,7 @@ resource "kubernetes_config_map" "vpn_configmap" {
 resource "kubernetes_service" "wireguard" {
   metadata {
     name      = "wireguard"
-    namespace = local.namespace
+    namespace = kubernetes_namespace.vpn_namespace.metadata[0].name
 
     labels = {
       app = "wireguard"
@@ -121,7 +121,7 @@ resource "kubernetes_service" "wireguard" {
 resource "kubernetes_deployment" "wireguard" {
   metadata {
     name      = "wireguard"
-    namespace = local.namespace
+    namespace = kubernetes_namespace.vpn_namespace.metadata[0].name
   }
 
   spec {

templates/terraform/bootstrap/secrets/eks_creator_user.tf

@@ -0,0 +1,80 @@
+
+#
+# EKS Cluster Creator Role
+# This has to be created first because it is used by the aws provider in the main terraform, so it can't be created by
+# that same terraform due to a chicken-and-egg situation.
+
+# Cluster creator role
+resource "aws_iam_role" "eks_cluster_creator" {
+  name               = "${local.project}-eks-cluster-creator"
+  assume_role_policy = data.aws_iam_policy_document.assumerole_root_only_policy.json
+  description        = "EKS cluster creator role"
+}
+
+# Trust relationship
+data "aws_iam_policy_document" "assumerole_root_only_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "AWS"
+      identifiers = [local.aws_account_id]
+    }
+  }
+}
+
+# Attach AWS managed policy for EKS
+resource "aws_iam_role_policy_attachment" "eks_cluster_creator_managed" {
+  role       = aws_iam_role.eks_cluster_creator.id
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
+}
+
+# Attach additional permissions
+resource "aws_iam_role_policy" "eks_cluster_creator" {
+  name = "manage_eks"
+  role = aws_iam_role.eks_cluster_creator.id
+
+  policy = data.aws_iam_policy_document.eks_manage.json
+}
+
+# Allow the cluster creator role to create a cluster
+data "aws_iam_policy_document" "eks_manage" {
+  statement {
+    actions = [
+      "eks:*",
+      "ec2:*",
+      "autoscaling:*",
+      "iam:CreateOpenIDConnectProvider",
+      "iam:DeleteOpenIDConnectProvider",
+      "iam:GetOpenIDConnectProvider",
+      "iam:ListOpenIDConnectProviders",
+      "iam:CreateInstanceProfile",
+      "iam:DeleteInstanceProfile",
+      "iam:GetInstanceProfile",
+      "iam:ListInstanceProfiles",
+      "iam:AddRoleToInstanceProfile",
+      "iam:RemoveRoleFromInstanceProfile",
+      "iam:ListInstanceProfilesForRole",
+    ]
+    resources = ["*"]
+  }
+
+  statement {
+    actions = [
+      "iam:GetRole",
+      "iam:PassRole",
+      "iam:CreateRole",
+      "iam:DeleteRole",
+      "iam:TagRole",
+      "iam:UntagRole",
+      "iam:AttachRolePolicy",
+      "iam:DetachRolePolicy",
+      "iam:ListAttachedRolePolicies",
+      "iam:ListRolePolicies"
+    ]
+    resources = [
+      "arn:aws:iam::${local.aws_account_id}:role/${local.project}-*",
+      "arn:aws:iam::${local.aws_account_id}:role/k8s-${local.project}-*",
+    ]
+  }
+}