diff --git a/README.md b/README.md index 03b9347..0e43910 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,13 @@ module "gke" { subnet_network = "10.1.0.0/20" regional = false zones = ["europe-central2-a"] - node_pools = [ - { - name = "default-pool" + node_pools = { + default-pool = { disk_size_gb = 50 max_count = 3 preemptible = true } - ] + } } ``` @@ -40,6 +39,32 @@ By default, it creates a "private" GKE cluster, but this can be changed setting This module is based on opinionated google modules, but combines several modules into "one module to rule them all". It uses the `private-cluster-update-variant` submodule of GKE - the version which can creates private cluster and - in case of node pool changes - creates new pool before deleting the old one, which minimizes the downtime of the live system. +## Node pools variable +The `node_pools` is a map of objects, where the node pool name is a key. Possible values: + +| Name | Description | Default | +|------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| image_type | The image type used to create the nodes. The possible values are COS_CONTAINERD, COS, UBUNTU_CONTAINERD, UBUNTU. NOTE : COS AND UBUNTU are deprecated as of GKE 1.24 | COS_CONTAINERD | +| machine_type | The name of a Google Compute Engine machine type. | e2-medium | +| min_cpu_platform | Minimum CPU platform to be used by this instance. The instance may be scheduled on the specified or newer CPU platform. Applicable values are the friendly names of CPU platforms, such as Intel Haswell. See the [official documentation](https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform) for more information. | "" | +| local_ssd_count | Number of local SSDs to use to back ephemeral storage. Uses NVMe interfaces. Each local SSD is 375 GB in size. If zero, it means to disable using local SSDs as ephemeral storage. | 0 | +| disk_size_gb | Size of the disk attached to each node, specified in GB. | 100 | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard', 'pd-ssd' or 'pd-balanced'). | pd-standard | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | +| spot | A boolean that represents whether the underlying node VMs are spot. | false | +| labels | The Kubernetes labels (key/value pairs) to be applied to each node. The kubernetes.io/ and k8s.io/ prefixes are reserved by Kubernetes Core components and cannot be specified. | {} | +| oauth_scopes | The set of Google API scopes to be made available on all of the node VMs under the "default" service account. Use the "https://www.googleapis.com/auth/cloud-platform" scope to grant access to all APIs. It is recommended that you set service_account to a non-default service account and grant IAM roles to that service account for only the resources that it needs. | var.default_node_pools_oauth_scopes | +| service_account | The service account to be used by the Node VMs. If not specified, the "default" service account is used. | null | +| taint | A list of Kubernetes taints to apply to nodes. Each taint consist of `key`, `value` and the effect, which must be one of NO_SCHEDULE, PREFER_NO_SCHEDULE, and NO_EXECUTE. | [] | + + + + +## Autopilot support +This module supports the autopilot GKE cluster. To create the autopilot cluster: +- set the `enable_autopilot` to `true` +- set the `node_pools` variable to empty list `[]` + ## Requirements @@ -49,17 +74,17 @@ No requirements. | Name | Version | |------|---------| -| [google](#provider\_google) | 4.76.0 | -| [google-beta](#provider\_google-beta) | 4.76.0 | +| [google](#provider\_google) | 4.84.0 | +| [google-beta](#provider\_google-beta) | 4.84.0 | ## Modules | Name | Source | Version | |------|--------|---------| -| [cloud\_nat](#module\_cloud\_nat) | registry.terraform.io/terraform-google-modules/cloud-nat/google | 2.2.0 | -| [network](#module\_network) | registry.terraform.io/terraform-google-modules/network/google | 5.0.0 | -| [project](#module\_project) | registry.terraform.io/terraform-google-modules/project-factory/google | 13.0.0 | -| [project\_services](#module\_project\_services) | registry.terraform.io/terraform-google-modules/project-factory/google//modules/project_services | 13.0.0 | +| [cloud\_nat](#module\_cloud\_nat) | registry.terraform.io/terraform-google-modules/cloud-nat/google | 4.0.0 | +| [network](#module\_network) | registry.terraform.io/terraform-google-modules/network/google | 7.1.0 | +| [project](#module\_project) | registry.terraform.io/terraform-google-modules/project-factory/google | 14.2.1 | +| [project\_services](#module\_project\_services) | terraform-google-modules/project-factory/google//modules/project_services | 14.2.1 | ## Resources @@ -75,12 +100,12 @@ No requirements. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [activate\_apis](#input\_activate\_apis) | List of Google APIs activated in new or existing project. | `list(string)` | 
[
  "compute.googleapis.com",
  "container.googleapis.com"
]
| no | -| [additional\_node\_pool\_oauth\_scopes](#input\_additional\_node\_pool\_oauth\_scopes) | Node pool oauth scopes added to specified node pool in addition to default\_node\_pool\_oauth\_scopes. It's referenced by node\_pool `name` | `map(list(string))` | 
{
  "default-node-pool": []
}
| no | | [billing\_account](#input\_billing\_account) | YOU NEED TO HAVE PERMISSION TO BILLING ACCOUNT, The billing account to witch the new project should be connected. Required if `create_project` set to `true`. | `string` | `""` | no | | [create\_project](#input\_create\_project) | Defines if create the project. All resources are created this project. If `false` - the project\_id is required. | `bool` | `false` | no | -| [default\_node\_pools\_oauth\_scopes](#input\_default\_node\_pools\_oauth\_scopes) | Default node pool oauth scopes added to all node pools | `list(string)` | 
[
  "https://www.googleapis.com/auth/devstorage.read_only",
  "https://www.googleapis.com/auth/cloud-platform",
  "https://www.googleapis.com/auth/logging.write",
  "https://www.googleapis.com/auth/monitoring",
  "https://www.googleapis.com/auth/compute"
]
| no | +| [default\_node\_pools\_oauth\_scopes](#input\_default\_node\_pools\_oauth\_scopes) | Default node pool oauth scopes added to all node pools | `list(string)` | 
[
  "https://www.googleapis.com/auth/cloud-platform"
]
| no | | [default\_pool\_machine\_type](#input\_default\_pool\_machine\_type) | In some cases the GKE won't be created unless the default pool uses specific machine type (for example confidential nodes) so we have to set the type even if the default pool is removed. | `string` | `"e2-small"` | no | | [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | Whether project services will be disabled when the resources are destroyed. | `bool` | `true` | no | +| [enable\_autopilot](#input\_enable\_autopilot) | Whether to enable Autopilot feature | `bool` | `null` | no | | [enable\_confidential\_nodes](#input\_enable\_confidential\_nodes) | Whether to enable confidential nodes. | `bool` | `false` | no | | [enable\_private\_endpoint](#input\_enable\_private\_endpoint) | Defines if create private endpoint. It disables the public endpoint so the cluster is accessible only from VPC. | `bool` | `false` | no | | [enable\_private\_nodes](#input\_enable\_private\_nodes) | Defines if use private nodes. Implies creation of cloud NAT service so nodes and pods can access public internet. | `bool` | `true` | no | @@ -88,8 +113,7 @@ No requirements. | [k8s\_network\_base](#input\_k8s\_network\_base) | The IP CIDR base for pods and services secondary networks. Must not overlap with `subnet_network`. Must be a `/16` network. | `string` | n/a | yes | | [master\_authorized\_networks](#input\_master\_authorized\_networks) | Allows accessing masters only from defined networks. If `enable_private_endpoint` is `true` it must not be any public CIDR block. | `list(map(string))` | 
[
  {
    "cidr_block": "0.0.0.0/0",
    "display_name": "ALL"
  }
]
| no | | [master\_ipv4\_cidr\_block](#input\_master\_ipv4\_cidr\_block) | The /28 CIDR block for masters when using private cluster. | `string` | `"172.16.0.0/28"` | no | -| [node\_pools](#input\_node\_pools) | List of node pools. For parameter details refer to node\_pool variable table below | `list(any)` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| [node\_pools\_labels](#input\_node\_pools\_labels) | List of node pools labels. https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/21.1.0/submodules/private-cluster-update-variant?tab=inputs#:~:text=default%2Dnode%2Dpool%22%20%7D%20%5D-,node_pools_labels,-map(map(string | `map(map(string))` | 
{
  "default-node-pool": {
    "node.pool/name": "default-node-pool"
  }
}
| no | +| [node\_pools](#input\_node\_pools) | The object which describes the node pools. The structure is described in the README file. | `map(map(any))` | `{}` | no | | [org\_id](#input\_org\_id) | GCP organization id. Required if `create_project` is `true`. | `string` | `""` | no | | [platform\_name](#input\_platform\_name) | The name of the platform. Many resource names are based on this (VPC, subnet, GKE cluster etc). | `string` | n/a | yes | | [project\_id](#input\_project\_id) | Existing project id. Required if `create_project` set to `false` | `string` | `""` | no | @@ -98,6 +122,7 @@ No requirements. | [regional](#input\_regional) | Defines the type of the GKE cluster. If `true` - the cluster is created as `regional`. Otherwise - as `zonal`. | `bool` | `true` | no | | [release\_channel](#input\_release\_channel) | The GKE release channel. | `string` | `"UNSPECIFIED"` | no | | [subnet\_network](#input\_subnet\_network) | The IP CIDR of the network for the GKE nodes. Must not overlap with `k8s_network_base`. | `string` | n/a | yes | +| [subnet\_private\_access](#input\_subnet\_private\_access) | Whether to enable google private IP access for the subnet | `bool` | `true` | no | | [zones](#input\_zones) | List of zones for `zonal` cluster. Required if `regional` set to `false`. | `list(string)` | `[]` | no | ## Outputs @@ -110,10 +135,12 @@ No requirements. | [gke\_endpoint](#output\_gke\_endpoint) | The kubernetes endpoint | | [gke\_location](#output\_gke\_location) | Location of the GKE cluster. Region if cluster is regional, zone if zonal | | [gke\_zones](#output\_gke\_zones) | List of zones where the cluster lives | -| [nat\_ip](#output\_nat\_ip) | n/a | +| [nat\_ip](#output\_nat\_ip) | The IP address allocated for NAT | | [project\_id](#output\_project\_id) | ID of the project containing the cluster | +| [subnetwork\_name](#output\_subnetwork\_name) | Name of the subnetwork | | [vpc\_id](#output\_vpc\_id) | VPC (network) ID | | [vpc\_name](#output\_vpc\_name) | Name of the created VPC (network) | +| [vpc\_self\_link](#output\_vpc\_self\_link) | VPC (network) self link | ## node_pools variable diff --git a/examples/terraform/private-cluster-existing-project/main.tf b/examples/terraform/private-cluster-existing-project/main.tf index ab34039..7905149 100644 --- a/examples/terraform/private-cluster-existing-project/main.tf +++ b/examples/terraform/private-cluster-existing-project/main.tf @@ -8,12 +8,11 @@ module "gke" { k8s_network_base = "10.100.0.0/16" regional = false zones = ["europe-central2-a"] - node_pools = [ - { - name = "default-pool" + node_pools = { + default-pool = { disk_size_gb = 50 max_count = 3 preemptible = true } - ] + } } diff --git a/examples/terraform/private-cluster-new-project/main.tf b/examples/terraform/private-cluster-new-project/main.tf index fcf9389..2fca35e 100644 --- a/examples/terraform/private-cluster-new-project/main.tf +++ b/examples/terraform/private-cluster-new-project/main.tf @@ -9,12 +9,22 @@ module "gke" { subnet_network = "10.1.0.0/20" regional = false zones = ["europe-central2-a"] - node_pools = [ - { - name = "default-pool" + node_pools = { + default-pool = { disk_size_gb = 50 max_count = 3 - preemptible = true + labels = { + "node.pool/name" = "default" + } + oauth_scopes = ["https://www.googleapis.com/auth/compute"] + spot = true + taint = [ + { + key = "test" + value = "test" + effect = "NO_SCHEDULE" + } + ] } - ] + } } diff --git a/examples/terraform/public-cluster-existing-project/main.tf b/examples/terraform/public-cluster-existing-project/main.tf index bd8d368..6f98446 100644 --- a/examples/terraform/public-cluster-existing-project/main.tf +++ b/examples/terraform/public-cluster-existing-project/main.tf @@ -9,12 +9,11 @@ module "gke" { regional = false zones = ["europe-central2-a"] enable_private_nodes = false - node_pools = [ - { - name = "default-pool" + node_pools = { + default-pool = { disk_size_gb = 50 max_count = 3 preemptible = true } - ] + } } diff --git a/examples/terraform/public-cluster-new-project/main.tf b/examples/terraform/public-cluster-new-project/main.tf index 1f87e7a..acf0124 100644 --- a/examples/terraform/public-cluster-new-project/main.tf +++ b/examples/terraform/public-cluster-new-project/main.tf @@ -10,12 +10,12 @@ module "gke" { regional = false zones = ["europe-central2-a"] enable_private_nodes = false - node_pools = [ - { + node_pools = { + default-pool = { name = "default-pool" disk_size_gb = 50 max_count = 3 preemptible = true } - ] + } } diff --git a/locals.tf b/locals.tf index d9d5db2..0e74dc9 100644 --- a/locals.tf +++ b/locals.tf @@ -1,16 +1,13 @@ locals { - project_id = var.create_project ? module.project.0.project_id : var.project_id - project_name = var.project_name != "" ? var.project_name : var.platform_name - subnet_name = "${var.platform_name}-subnet" - router = "${var.platform_name}-router" - cloud_nat_name = "${var.platform_name}-cloud-nat" - pods_network_name = "${local.subnet_name}-pods" - services_network_name = "${local.subnet_name}-services" - pods_ip_range = cidrsubnet(var.k8s_network_base, 4, 1) - services_ip_range = cidrsubnet(var.k8s_network_base, 4, 2) - location = var.regional ? var.region : var.zones.0 - node_pool_names = [for np in toset(var.node_pools) : np.name] - node_pools = zipmap(local.node_pool_names, tolist(toset(var.node_pools))) - node_locations = var.regional ? var.zones : slice(var.zones, 1, length(var.zones)) - node_pool_oauth_scopes = { for key, value in var.additional_node_pool_oauth_scopes : key => distinct(concat(value, var.default_node_pools_oauth_scopes)) } + project_id = var.create_project ? module.project.0.project_id : var.project_id + project_name = var.project_name != "" ? var.project_name : var.platform_name + subnet_name = "${var.platform_name}-subnet" + router = "${var.platform_name}-router" + cloud_nat_name = "${var.platform_name}-cloud-nat" + pods_network_name = "${local.subnet_name}-pods" + services_network_name = "${local.subnet_name}-services" + pods_ip_range = cidrsubnet(var.k8s_network_base, 4, 1) + services_ip_range = cidrsubnet(var.k8s_network_base, 4, 2) + location = var.regional ? var.region : var.zones.0 + node_locations = var.regional ? (length(var.zones) != 0 ? var.zones : null) : slice(var.zones, 1, length(var.zones)) } diff --git a/main.tf b/main.tf index 982c775..bb5e383 100644 --- a/main.tf +++ b/main.tf @@ -1,6 +1,6 @@ module "project" { source = "registry.terraform.io/terraform-google-modules/project-factory/google" - version = "13.0.0" + version = "14.2.1" billing_account = var.billing_account name = var.platform_name org_id = var.org_id @@ -10,8 +10,8 @@ module "project" { } module "project_services" { - source = "registry.terraform.io/terraform-google-modules/project-factory/google//modules/project_services" - version = "13.0.0" + source = "terraform-google-modules/project-factory/google//modules/project_services" + version = "14.2.1" project_id = var.project_id activate_apis = var.activate_apis disable_services_on_destroy = var.disable_services_on_destroy @@ -20,15 +20,16 @@ module "project_services" { module "network" { source = "registry.terraform.io/terraform-google-modules/network/google" - version = "5.0.0" + version = "7.1.0" network_name = var.platform_name project_id = local.project_id auto_create_subnetworks = false subnets = [ { - subnet_name = local.subnet_name - subnet_ip = var.subnet_network - subnet_region = var.region + subnet_name = local.subnet_name + subnet_ip = var.subnet_network + subnet_region = var.region + subnet_private_access = var.subnet_private_access } ] secondary_ranges = { @@ -60,7 +61,7 @@ resource "google_compute_address" "cloud_nat_address" { module "cloud_nat" { source = "registry.terraform.io/terraform-google-modules/cloud-nat/google" - version = "2.2.0" + version = "4.0.0" project_id = local.project_id region = var.region network = module.network.network_name @@ -81,7 +82,8 @@ resource "google_container_cluster" "gke" { node_locations = local.node_locations network = module.network.network_self_link subnetwork = local.subnet_name - remove_default_node_pool = true + remove_default_node_pool = var.enable_autopilot == null ? true : null + enable_autopilot = var.enable_autopilot initial_node_count = 1 node_config { machine_type = var.default_pool_machine_type @@ -95,11 +97,15 @@ resource "google_container_cluster" "gke" { depends_on = [ module.network.subnets ] + ip_allocation_policy { + cluster_secondary_range_name = local.pods_network_name + services_secondary_range_name = local.services_network_name + } } resource "google_container_node_pool" "pools" { provider = google-beta - for_each = local.node_pools + for_each = var.node_pools location = local.location project = local.project_id cluster = google_container_cluster.gke.name @@ -129,8 +135,10 @@ resource "google_container_node_pool" "pools" { disk_type = lookup(each.value, "disk_type", "pd-standard") preemptible = lookup(each.value, "preemptible", false) spot = lookup(each.value, "spot", false) - labels = lookup(var.node_pools_labels, each.value["name"], {}) - oauth_scopes = lookup(local.node_pool_oauth_scopes, each.value["name"], []) + labels = lookup(each.value, "labels", {}) + oauth_scopes = lookup(each.value, "oauth_scopes", var.default_node_pools_oauth_scopes) + service_account = lookup(each.value, "service_account", null) + taint = lookup(each.value, "taint", []) dynamic "guest_accelerator" { for_each = lookup(each.value, "guest_accelerator", null) != null ? [1] : [] @@ -159,4 +167,3 @@ resource "google_container_registry" "registry" { project = local.project_id location = var.gcr_location } - diff --git a/outputs.tf b/outputs.tf index 82fb499..7c50c0f 100644 --- a/outputs.tf +++ b/outputs.tf @@ -34,10 +34,19 @@ output "vpc_id" { value = module.network.network_id description = "VPC (network) ID" } +output "vpc_self_link" { + value = module.network.network_self_link + description = "VPC (network) self link" +} output "gke_zones" { value = google_container_cluster.gke.node_locations description = "List of zones where the cluster lives" } output "nat_ip" { - value = google_compute_address.cloud_nat_address.*.address + value = google_compute_address.cloud_nat_address.*.address + description = "The IP address allocated for NAT" +} +output "subnetwork_name" { + value = module.network.subnets_names.0 + description = "Name of the subnetwork" } diff --git a/variables.tf b/variables.tf index 5baa98b..e61a401 100644 --- a/variables.tf +++ b/variables.tf @@ -22,7 +22,7 @@ variable "project_id" { default = "" description = "Existing project id. Required if `create_project` set to `false`" validation { - condition = can(regex("^[a-z]{1}[0-9a-z-]{5,29}$", var.project_id)) + condition = (var.project_id == "" || can(regex("^[a-z]{1}[0-9a-z-]{5,29}$", var.project_id))) error_message = "The project id must be 6 to 30 characters in length, can only contain lowercase letters, numbers, and hyphens" } } @@ -31,7 +31,7 @@ variable "project_name" { default = "" description = "The name of the created project. Defaults to `platform_name` if not set." validation { - condition = length(var.project_name) < 25 && length(var.project_name) > 4 + condition = (var.project_name == "" || length(var.project_name) < 25 && length(var.project_name) > 4) error_message = "The project name should contain only 25 characters. Last 5 characters up to 30 total are generated" } } @@ -70,24 +70,11 @@ variable "zones" { default = [] description = "List of zones for `zonal` cluster. Required if `regional` set to `false`." } -variable "node_pools" { - type = list(any) - default = [ - { - name = "default-node-pool" - }, - ] - description = "List of node pools. For parameter details refer to node_pool variable table below" -} -variable "node_pools_labels" { - type = map(map(string)) - default = { - "default-node-pool" = { - "node.pool/name" = "default-node-pool" - }, - } - description = "List of node pools labels. https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/21.1.0/submodules/private-cluster-update-variant?tab=inputs#:~:text=default%2Dnode%2Dpool%22%20%7D%20%5D-,node_pools_labels,-map(map(string" +variable "node_pools" { + type = map(map(any)) + default = {} + description = "The object which describes the node pools. The structure is described in the README file." } variable "master_ipv4_cidr_block" { @@ -144,22 +131,22 @@ variable "default_pool_machine_type" { description = "In some cases the GKE won't be created unless the default pool uses specific machine type (for example confidential nodes) so we have to set the type even if the default pool is removed." } -variable "additional_node_pool_oauth_scopes" { - type = map(list(string)) - default = { - default-node-pool = [] - } - description = "Node pool oauth scopes added to specified node pool in addition to default_node_pool_oauth_scopes. It's referenced by node_pool `name`" -} - variable "default_node_pools_oauth_scopes" { type = list(string) default = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring", - "https://www.googleapis.com/auth/compute" + "https://www.googleapis.com/auth/cloud-platform" ] description = "Default node pool oauth scopes added to all node pools" } + +variable "enable_autopilot" { + type = bool + default = null + description = "Whether to enable Autopilot feature" +} + +variable "subnet_private_access" { + type = bool + default = true + description = "Whether to enable google private IP access for the subnet" +}