From a1793cda55e70eb8ef3f270efd7afc62e8226c1f Mon Sep 17 00:00:00 2001 From: Naveen Mandadhi Date: Mon, 1 Apr 2024 14:36:16 +0000 Subject: [PATCH] feat: add island-cluster submodule and an example --- .../island_cluster_with_vm_router/README.md | 3 + .../island_cluster_with_vm_router/main.tf | 16 ++ .../terraform.tfvars | 28 +++ .../island_cluster_with_vm_router/vars.tf | 45 ++++ modules/island-cluster/README.md | 62 +++++ modules/island-cluster/main.tf | 219 ++++++++++++++++++ modules/island-cluster/outputs.tf | 3 + modules/island-cluster/providers.tf | 20 ++ modules/island-cluster/router.tf | 35 +++ modules/island-cluster/vars.tf | 48 ++++ 10 files changed, 479 insertions(+) create mode 100644 examples/island_cluster_with_vm_router/README.md create mode 100644 examples/island_cluster_with_vm_router/main.tf create mode 100644 examples/island_cluster_with_vm_router/terraform.tfvars create mode 100644 examples/island_cluster_with_vm_router/vars.tf create mode 100644 modules/island-cluster/README.md create mode 100644 modules/island-cluster/main.tf create mode 100644 modules/island-cluster/outputs.tf create mode 100644 modules/island-cluster/providers.tf create mode 100644 modules/island-cluster/router.tf create mode 100644 modules/island-cluster/vars.tf diff --git a/examples/island_cluster_with_vm_router/README.md b/examples/island_cluster_with_vm_router/README.md new file mode 100644 index 0000000000..f0147e4cf8 --- /dev/null +++ b/examples/island_cluster_with_vm_router/README.md @@ -0,0 +1,3 @@ +# GKE island cluster using VM as router + +This uses `island-cluster` submodule to provison a cluster in an island VPC allowing reuse of the IP address space for multiple clusters in the same project. \ No newline at end of file diff --git a/examples/island_cluster_with_vm_router/main.tf b/examples/island_cluster_with_vm_router/main.tf new file mode 100644 index 0000000000..9e8ae95ff9 --- /dev/null +++ b/examples/island_cluster_with_vm_router/main.tf @@ -0,0 +1,16 @@ +module "gke-island-cluster" { + source = "../../modules/island-cluster" + + project_id = var.project_id + region = var.region + random_suffix = "b8701ba0" + node_locations = var.node_locations + subnet_cidr = var.subnet_cidr + secondary_ranges = var.secondary_ranges + master_authorized_networks = var.master_authorized_networks + router_machine_type = var.router_machine_type + primary_subnet = var.primary_subnet + primary_net_cidrs = var.primary_net_cidrs + psc_subnet_cidr = var.psc_subnet_cidr + proxy_subnet_cidr = var.proxy_subnet_cidr +} diff --git a/examples/island_cluster_with_vm_router/terraform.tfvars b/examples/island_cluster_with_vm_router/terraform.tfvars new file mode 100644 index 0000000000..ec3c204171 --- /dev/null +++ b/examples/island_cluster_with_vm_router/terraform.tfvars @@ -0,0 +1,28 @@ +project_id = "" +region = "us-central1" +node_locations = [ + "us-central1-a", + "us-central1-b", + "us-central1-f" +] +subnet_cidr = "100.64.0.0/20" +router_machine_type = "n2-highcpu-4" +primary_subnet = "projects//regions//subnetworks/" +secondary_ranges = { + pods = "100.64.64.0/18" + services = "100.64.128.0/20" + master_cidr = "100.64.144.0/28" +} +proxy_subnet_cidr = "100.64.168.0/24" +psc_subnet_cidr = "100.64.192.0/24" +master_authorized_networks = [ + { + cidr_block = "100.64.0.0/10" + display_name = "cluster net" + } +] +primary_net_cidrs = [ + "10.0.0.0/8", + "192.168.0.0/16", + "172.16.0.0/12" +] \ No newline at end of file diff --git a/examples/island_cluster_with_vm_router/vars.tf b/examples/island_cluster_with_vm_router/vars.tf new file mode 100644 index 0000000000..7f90e43517 --- /dev/null +++ b/examples/island_cluster_with_vm_router/vars.tf @@ -0,0 +1,45 @@ +variable "project_id" { + type = string +} + +variable "region" { + type = string +} + +variable "node_locations" { + type = list(string) +} + +variable "primary_subnet" { + type = string + description = "subnet of the existing network for the router VM NIC" +} + +variable "subnet_cidr" { + type = string +} + +variable "psc_subnet_cidr" { + type = string +} + +variable "proxy_subnet_cidr" { + type = string +} + +variable "secondary_ranges" { + type = map(string) +} + +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +} + +variable "primary_net_cidrs" { + type = list(string) +} + +variable "router_machine_type" { + type = string +} diff --git a/modules/island-cluster/README.md b/modules/island-cluster/README.md new file mode 100644 index 0000000000..c925b3ce67 --- /dev/null +++ b/modules/island-cluster/README.md @@ -0,0 +1,62 @@ +# GKE island cluster using VM as router + +This submodule provisions a cluster in an island VPC allowing reuse of the IP address space for multiple clusters in the same project. + +1. An appliance(VM as router) with multiple NICs is used to establish connectivity between the island VPC and the existing network. +2. Outbound connections will go through the router. +3. For inbound connections, use Private Service Connect. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | ~> 1.6 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | +| [google-beta](#provider\_google-beta) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [gke](#module\_gke) | terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster | ~> 30.0 | +| [net](#module\_net) | terraform-google-modules/network/google | ~> 9.0 | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_gke_hub_membership.primary](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_gke_hub_membership) | resource | +| [google_compute_firewall.proxy_snet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource | +| [google_compute_firewall.psc_snet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource | +| [google_compute_instance.vm](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource | +| [google_project_iam_member.gke-dev](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_service_account.gke-access-sa](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource | +| [google_service_account.gke-sa](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [master\_authorized\_networks](#input\_master\_authorized\_networks) | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | n/a | yes | +| [node\_locations](#input\_node\_locations) | n/a | `list(string)` | n/a | yes | +| [primary\_net\_cidrs](#input\_primary\_net\_cidrs) | n/a | `list(string)` | n/a | yes | +| [primary\_subnet](#input\_primary\_subnet) | n/a | `string` | n/a | yes | +| [project\_id](#input\_project\_id) | n/a | `string` | n/a | yes | +| [proxy\_subnet\_cidr](#input\_proxy\_subnet\_cidr) | n/a | `string` | n/a | yes | +| [psc\_subnet\_cidr](#input\_psc\_subnet\_cidr) | n/a | `string` | n/a | yes | +| [random\_suffix](#input\_random\_suffix) | n/a | `string` | n/a | yes | +| [region](#input\_region) | n/a | `string` | n/a | yes | +| [router\_machine\_type](#input\_router\_machine\_type) | n/a | `string` | n/a | yes | +| [secondary\_ranges](#input\_secondary\_ranges) | n/a | `map(string)` | n/a | yes | +| [subnet\_cidr](#input\_subnet\_cidr) | n/a | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cluster\_id](#output\_cluster\_id) | n/a | \ No newline at end of file diff --git a/modules/island-cluster/main.tf b/modules/island-cluster/main.tf new file mode 100644 index 0000000000..40cbb94de7 --- /dev/null +++ b/modules/island-cluster/main.tf @@ -0,0 +1,219 @@ +resource "google_service_account" "gke-sa" { + account_id = "gke-sa-${var.random_suffix}" +} + +resource "google_service_account" "gke-access-sa" { + account_id = "gke-access-sa-${var.random_suffix}" +} + +resource "google_project_iam_member" "gke-dev" { + project = var.project_id + member = "serviceAccount:${google_service_account.gke-access-sa.email}" + role = "roles/container.developer" +} + +module "net" { + source = "terraform-google-modules/network/google" + version = "~> 9.0" + + network_name = "gke-net-${var.random_suffix}" + routing_mode = "GLOBAL" + project_id = var.project_id + delete_default_internet_gateway_routes = true + + subnets = [ + { + subnet_name = "${var.region}-snet-${var.random_suffix}" + subnet_ip = var.subnet_cidr + subnet_region = var.region + subnet_private_access = "true" + }, + { + subnet_name = "${var.region}-proxy-snet-${var.random_suffix}" + subnet_ip = var.proxy_subnet_cidr + subnet_region = var.region + purpose = "REGIONAL_MANAGED_PROXY" + role = "ACTIVE" + }, + { + subnet_name = "${var.region}-psc-snet-${var.random_suffix}" + subnet_ip = var.psc_subnet_cidr + subnet_region = var.region + subnet_private_access = "true" + purpose = "PRIVATE_SERVICE_CONNECT" + } + ] + + secondary_ranges = { + "${var.region}-snet-${var.random_suffix}" = [ + { + range_name = "${var.region}-snet-pods-${var.random_suffix}" + ip_cidr_range = var.secondary_ranges["pods"] + }, + { + range_name = "${var.region}-snet-services-${var.random_suffix}" + ip_cidr_range = var.secondary_ranges["services"] + }, + ] + } + + routes = flatten([ + [for k, v in var.primary_net_cidrs : + { + name = "egress-gke-${k}-${var.random_suffix}" + description = "egress through the router for range ${v}" + destination_range = v + tags = "gke-${var.random_suffix}" + next_hop_instance = google_compute_instance.vm.self_link + priority = 100 + } + ], + [ + { + name = "default-igw-${var.random_suffix}" + description = "internet through the router" + destination_range = "0.0.0.0/0" + tags = "gke-${var.random_suffix}" + next_hop_instance = google_compute_instance.vm.self_link + priority = 100 + } + ] + ]) + + firewall_rules = [ + { + name = "iap-fw-${var.random_suffix}" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + ports = ["22"] + } + ] + ranges = ["35.235.240.0/20"] + }, + { + name = "tcp-primary-fw-${var.random_suffix}" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + } + ] + ranges = [ + var.subnet_cidr, + var.secondary_ranges["pods"] + ] + }, + ] +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster" + version = "~> 30.0" + + depends_on = [google_compute_instance.vm] + + name = "gke-test-${var.random_suffix}" + project_id = var.project_id + region = var.region + release_channel = "RAPID" + zones = var.node_locations + network = module.net.network_name + subnetwork = "${var.region}-snet-${var.random_suffix}" + ip_range_pods = "${var.region}-snet-pods-${var.random_suffix}" + ip_range_services = "${var.region}-snet-services-${var.random_suffix}" + http_load_balancing = true + horizontal_pod_autoscaling = true + enable_private_endpoint = true + enable_private_nodes = true + datapath_provider = "ADVANCED_DATAPATH" + monitoring_enable_managed_prometheus = false + enable_shielded_nodes = true + master_global_access_enabled = false + master_ipv4_cidr_block = var.secondary_ranges["master_cidr"] + master_authorized_networks = var.master_authorized_networks + deletion_protection = false + remove_default_node_pool = true + disable_default_snat = true + gateway_api_channel = "CHANNEL_STANDARD" + + node_pools = [ + { + name = "default-${var.random_suffix}" + machine_type = "e2-highcpu-2" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = true + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = google_service_account.gke-sa.email + initial_node_count = 1 + enable_secure_boot = true + }, + ] + + node_pools_tags = { + all = ["gke-${var.random_suffix}"] + } + + node_pools_oauth_scopes = { + all = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + ] + } + + timeouts = { + create = "10m" + update = "10m" + delete = "10m" + } +} + +resource "google_gke_hub_membership" "primary" { + provider = google-beta + + project = var.project_id + membership_id = "${var.project_id}-${module.gke.name}" + location = var.region + + endpoint { + gke_cluster { + resource_link = "//container.googleapis.com/${module.gke.cluster_id}" + } + } + authority { + issuer = "https://container.googleapis.com/v1/${module.gke.cluster_id}" + } +} + +resource "google_compute_firewall" "psc_snet" { + name = "allow-psc-${var.random_suffix}" + direction = "INGRESS" + allow { + protocol = "tcp" + } + source_ranges = [var.psc_subnet_cidr] + target_service_accounts = [google_service_account.gke-sa.email] + network = module.net.network_id + project = var.project_id +} + +resource "google_compute_firewall" "proxy_snet" { + name = "allow-proxy-${var.random_suffix}" + direction = "INGRESS" + allow { + protocol = "tcp" + } + source_ranges = [var.proxy_subnet_cidr] + target_service_accounts = [google_service_account.gke-sa.email] + network = module.net.network_id + project = var.project_id +} diff --git a/modules/island-cluster/outputs.tf b/modules/island-cluster/outputs.tf new file mode 100644 index 0000000000..d0ca527bc3 --- /dev/null +++ b/modules/island-cluster/outputs.tf @@ -0,0 +1,3 @@ +output "cluster_id" { + value = module.gke.cluster_id +} \ No newline at end of file diff --git a/modules/island-cluster/providers.tf b/modules/island-cluster/providers.tf new file mode 100644 index 0000000000..049a688d80 --- /dev/null +++ b/modules/island-cluster/providers.tf @@ -0,0 +1,20 @@ +terraform { + required_version = "~> 1.6" + + required_providers { + google = { + source = "hashicorp/google" + } + google-beta = { + source = "hashicorp/google-beta" + } + } +} + +provider "google" { + project = var.project_id +} + +provider "google-beta" { + project = var.project_id +} \ No newline at end of file diff --git a/modules/island-cluster/router.tf b/modules/island-cluster/router.tf new file mode 100644 index 0000000000..7b9c219584 --- /dev/null +++ b/modules/island-cluster/router.tf @@ -0,0 +1,35 @@ +resource "google_compute_instance" "vm" { + project = var.project_id + zone = var.node_locations[0] + name = "router-${var.random_suffix}" + machine_type = var.router_machine_type + allow_stopping_for_update = true + boot_disk { + initialize_params { + image = "debian-cloud/debian-12" + } + } + can_ip_forward = true + shielded_instance_config { + enable_secure_boot = true + } + network_interface { + subnetwork = var.primary_subnet + } + network_interface { + subnetwork = module.net.subnets["${var.region}/${var.region}-snet-${var.random_suffix}"]["self_link"] + } + metadata_startup_script = <<-EOT + #!/bin/bash + set -ex + sudo apt-get update + echo "net.ipv4.ip_forward = 1" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p + sudo iptables -A FORWARD -i ens5 -o ens4 -j ACCEPT + sudo iptables -A FORWARD -i ens4 -o ens5 -m state --state ESTABLISHED,RELATED -j ACCEPT + GWY_URL="http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/1/gateway" + GWY_IP=$(curl $${GWY_URL} -H "Metadata-Flavor: Google") + sudo ip route add ${var.secondary_ranges["pods"]} via $${GWY_IP} dev ens5 + sudo iptables -t nat -A POSTROUTING -o ens4 -s 0.0.0.0/0 -j MASQUERADE + EOT +} diff --git a/modules/island-cluster/vars.tf b/modules/island-cluster/vars.tf new file mode 100644 index 0000000000..2958815770 --- /dev/null +++ b/modules/island-cluster/vars.tf @@ -0,0 +1,48 @@ +variable "project_id" { + type = string +} + +variable "region" { + type = string +} + +variable "node_locations" { + type = list(string) +} + +variable "subnet_cidr" { + type = string +} + +variable "psc_subnet_cidr" { + type = string +} + +variable "proxy_subnet_cidr" { + type = string +} + +variable "secondary_ranges" { + type = map(string) +} + +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +} + +variable "primary_subnet" { + type = string +} + +variable "primary_net_cidrs" { + type = list(string) +} + +variable "router_machine_type" { + type = string +} + +variable "random_suffix" { + type = string +}