Create a private GKE cluster with NAT routing

gke-private-cluster/README.md

@@ -0,0 +1,30 @@
+GKE Private Cluster with Workload-Identity enabled
+
+Set up a zonal cluster with private node IPs behind a NAT router.
+
+This is a more secure configuration than the default public one.
+The NAT router enables nodes to pull images from external docker repositories and 
+allows workloads to access the internet, while protecting both from direct access 
+from the internet.
+
+To make the setup easier to read, only repeated values have been extracted in variables.tf
+
+```bash
+terraform apply
+
+gcloud config set compute/region us-central1
+gcloud config set compute/zone   us-central1-a
+
+gcloud container clusters get-credentials apps
+# Now can use kubectl, helm etc. in this cluster
+kubectl version
+kubectl get all
+
+# Inspect some of the deployed objects
+gcloud container clusters describe apps
+gcloud container networks subnets describe subnet
+gcloud compute routers describe router
+
+# Delete everything
+terraform destroy
+```

gke-private-cluster/gke.tf

@@ -0,0 +1,67 @@
+data "google_container_engine_versions" "k8s_versions" {
+  project        = var.project
+  location       = var.zone
+  version_prefix = "1.14.8-"
+}
+
+resource "google_container_cluster" "apps" {
+  provider           = google-beta
+  name               = "apps"
+  min_master_version = data.google_container_engine_versions.k8s_versions.latest_master_version
+  node_version       = data.google_container_engine_versions.k8s_versions.latest_node_version
+  # Set location = var.region for a regional cluster
+  location                  = var.zone
+  network                   = google_compute_network.main.self_link
+  subnetwork                = google_compute_subnetwork.subnet.self_link
+  default_max_pods_per_node = 32
+  ip_allocation_policy {
+    cluster_secondary_range_name  = google_compute_subnetwork.subnet.secondary_ip_range[0].range_name # "pods"
+    services_secondary_range_name = google_compute_subnetwork.subnet.secondary_ip_range[1].range_name # "services"
+  }
+  private_cluster_config {
+    enable_private_endpoint = false
+    enable_private_nodes    = true
+    master_ipv4_cidr_block  = cidrsubnet("172.16.0.0/24", 4, 1) # 172.16.0.16/28
+  }
+  workload_identity_config {
+    identity_namespace = "${data.google_client_config.current.project}.svc.id.goog"
+  }
+  addons_config {
+    network_policy_config {
+      disabled = true
+    }
+    istio_config {
+      disabled = true
+      auth     = "AUTH_NONE"
+    }
+  }
+  release_channel {
+    channel = "REGULAR"
+  }
+  initial_node_count       = 1
+  remove_default_node_pool = true
+  node_config {
+    workload_metadata_config {
+      node_metadata = "GKE_METADATA_SERVER"
+    }
+  }
+}
+
+resource google_container_node_pool "apps" {
+  provider           = google-beta
+  name               = "apps"
+  cluster            = google_container_cluster.apps.name
+  initial_node_count = 1
+  autoscaling {
+    max_node_count = 3
+    min_node_count = 1
+  }
+  max_pods_per_node = 64
+  node_config {
+    machine_type = var.apps_machine_type
+    preemptible  = false
+    workload_metadata_config {
+      node_metadata = "GKE_METADATA_SERVER"
+    }
+  }
+}

gke-private-cluster/network.tf

@@ -0,0 +1,54 @@
+resource "google_compute_network" "main" {
+  name                    = "main"
+  project                 = data.google_client_config.current.project
+  routing_mode            = "REGIONAL"
+  auto_create_subnetworks = false
+}
+
+# Create a subnet for the cluster
+resource "google_compute_subnetwork" "subnet" {
+  name                     = "subnet"
+  project                  = data.google_client_config.current.project
+  region                   = var.region
+  network                  = google_compute_network.main.self_link
+  private_ip_google_access = true
+  ip_cidr_range            = cidrsubnet("10.10.0.0/18", 4, 1) # 10.10.4.0/22
+  secondary_ip_range {
+    ip_cidr_range = cidrsubnet("10.0.0.0/16", 4, 1) # 10.0.16.0/20
+    range_name    = "pods"
+  }
+  secondary_ip_range {
+    ip_cidr_range = cidrsubnet("10.1.0.0/17", 4, 1) # 10.1.8.0/21
+    range_name    = "services"
+  }
+}
+
+resource "google_compute_router" "router" {
+  name    = "router"
+  project = data.google_client_config.current.project
+  region  = google_compute_subnetwork.subnet.region
+  network = google_compute_network.main.self_link
+}
+
+
+resource "google_compute_router_nat" "router_nat" {
+  depends_on             = [google_compute_subnetwork.subnet]
+  name                   = "router-nat"
+  nat_ip_allocate_option = "AUTO_ONLY"
+  router                 = google_compute_router.router.name
+  region                 = google_compute_router.router.region
+  # this would cover the nodes, pods and services
+  #source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+  source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
+  # Nodes
+  subnetwork {
+    name                    = google_compute_subnetwork.subnet.self_link
+    source_ip_ranges_to_nat = ["PRIMARY_IP_RANGE"]
+  }
+  # Pods
+  subnetwork {
+    name                     = google_compute_subnetwork.subnet.self_link
+    source_ip_ranges_to_nat  = ["LIST_OF_SECONDARY_IP_RANGES"]
+    secondary_ip_range_names = [google_container_cluster.apps.ip_allocation_policy.0.cluster_secondary_range_name]
+  }
+}

gke-private-cluster/outputs.tf

@@ -0,0 +1,19 @@
+output "master_ipv4_cidr" {
+  value = google_container_cluster.apps.private_cluster_config[0].master_ipv4_cidr_block
+}
+
+output "network_ipv4_range" {
+  value = google_compute_network.main.ipv4_range
+}
+
+output "subnet_ip_cidr_range" {
+  value = google_compute_subnetwork.subnet.ip_cidr_range
+}
+
+output "cluster_ipv4_cidr" {
+  value = google_container_cluster.apps.cluster_ipv4_cidr
+}
+
+output "services_ipv4_cidr" {
+  value = google_container_cluster.apps.services_ipv4_cidr
+}

gke-private-cluster/provider.tf

@@ -0,0 +1,24 @@
+# https://www.terraform.io/docs/providers/google/provider_reference.html
+provider "google" {
+  credentials = file(var.terraform-gcp-sa-file)
+  region      = var.region
+  zone        = var.zone
+  project     = var.project
+}
+
+provider "google-beta" {
+  credentials = file(var.terraform-gcp-sa-file)
+  region      = var.region
+  zone        = var.zone
+  project     = var.project
+}
+
+provider "kubernetes" {
+  alias                  = "apps"
+  host                   = google_container_cluster.apps.endpoint
+  token                  = data.google_client_config.current.access_token
+  cluster_ca_certificate = base64decode(google_container_cluster.apps.master_auth[0].cluster_ca_certificate)
+  load_config_file       = false
+}
+
+data "google_client_config" "current" {}

gke-private-cluster/variables.tf

@@ -0,0 +1,20 @@
+variable "terraform-gcp-sa-file" {
+  description = "Path to Terraform Service Account json file"
+}
+
+variable "project" {
+  description = "Project id to deploy infrastructure under"
+}
+
+variable "region" {
+  default = "us-central1"
+}
+
+variable "zone" {
+  default = "us-central1-a"
+}
+
+variable "apps_machine_type" {
+  default     = "e2-small"
+  description = "Machine type for the apps cluster"
+}