Enabled creation of autoscaling nodepools and regional clusters (#2813)

Added options to the GKE terraform install for creating a regional cluster
and enabling nodepool autoscaling.

build/Makefile

@@ -43,6 +43,7 @@ KUBECONFIG ?= ~/.kube/config
 # The (gcloud) test cluster that is being worked against
 GCP_CLUSTER_NAME ?= test-cluster
 GCP_CLUSTER_ZONE ?= us-west1-c
+GCP_CLUSTER_LOCATION ?= $(GCP_CLUSTER_ZONE)
 GCP_BUCKET_CHARTS ?= agones-chart
 # the profile to use when developing on minikube
 MINIKUBE_PROFILE ?= agones

build/README.md

@@ -199,7 +199,10 @@ See the table below for available customizations :
 | Parameter                                      | Description                                                                           | Default         |
 |------------------------------------------------|---------------------------------------------------------------------------------------|-----------------|
 | `GCP_CLUSTER_NAME`                             | The name of the cluster                                                               | `test-cluster`  |
-| `GCP_CLUSTER_ZONE`                             | The name of the Google Compute Engine zone in which the cluster will resides.         | `us-west1-c`    |
+| `GCP_CLUSTER_ZONE` or `GCP_CLUSTER_LOCATION`   | The name of the Google Compute Engine zone/location in which the cluster will resides | `us-west1-c`    |
+| `GCP_CLUSTER_NODEPOOL_AUTOSCALE`               | Whether or not to enable autoscaling on game server nodepool                          | `false`         |
+| `GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT`           | The number of minimum nodes if autoscale is enabled                                   | `1`             |
+| `GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT`           | The number of maximum nodes if autoscale is enabled                                   | `5`             |
 | `GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT`        | The number of nodes to create in this cluster.                                        | `4`             |
 | `GCP_CLUSTER_NODEPOOL_MACHINETYPE`             | The name of a Google Compute Engine machine type.                                     | `e2-standard-4` |
 | `GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING`    | Whether or not to enable image streaming for the `"default"` node pool in the cluster | `true`          |

build/includes/google-cloud.mk

@@ -27,10 +27,17 @@ gcloud-init: ensure-build-config
 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4
 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= e2-standard-4
 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING ?= true
+gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_AUTOSCALE ?= false
+gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT ?= 1
+gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT ?= 5
 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT ?= 0
 gcloud-test-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE ?= e2-standard-4
 gcloud-test-cluster: $(ensure-build-image)
-	$(MAKE) gcloud-terraform-cluster GCP_TF_CLUSTER_NAME="$(GCP_CLUSTER_NAME)" GCP_CLUSTER_ZONE="$(GCP_CLUSTER_ZONE)" \
+	$(MAKE) gcloud-terraform-cluster GCP_TF_CLUSTER_NAME="$(GCP_CLUSTER_NAME)" \
+		GCP_CLUSTER_LOCATION="$(GCP_CLUSTER_LOCATION)" \
+		GCP_CLUSTER_AUTOSCALE="$(GCP_CLUSTER_NODEPOOL_AUTOSCALE)" \
+		GCP_CLUSTER_MIN_NODECOUNT="$(GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT)" \
+		GCP_CLUSTER_MAX_NODECOUNT="$(GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT)" \
 		GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT="$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT)" \
 		GCP_CLUSTER_NODEPOOL_MACHINETYPE="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \
 		GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING="$(GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING)" \
@@ -73,9 +80,7 @@ clean-gcloud-prow-build-cluster: $(ensure-build-image)
 # Pulls down authentication information for kubectl against a cluster, name can be specified through GCP_CLUSTER_NAME
 # (defaults to 'test-cluster')
 gcloud-auth-cluster: $(ensure-build-image)
-	docker run --rm $(common_mounts) $(build_tag) gcloud config set container/cluster $(GCP_CLUSTER_NAME)
-	docker run --rm $(common_mounts) $(build_tag) gcloud config set compute/zone $(GCP_CLUSTER_ZONE)
-	docker run --rm $(common_mounts) $(build_tag) gcloud container clusters get-credentials $(GCP_CLUSTER_NAME)
+	docker run --rm $(common_mounts) $(build_tag) gcloud container clusters get-credentials $(GCP_CLUSTER_NAME) --zone  $(GCP_CLUSTER_LOCATION)
 
 # authenticate our docker configuration so that you can do a docker push directly
 # to the gcr.io repository

build/includes/terraform.mk

@@ -38,6 +38,9 @@ terraform-clean:
 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4
 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= e2-standard-4
 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING ?= true
+gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_AUTOSCALE ?= false
+gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT ?= 1
+gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT ?= 5
 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT ?= 0
 gcloud-terraform-cluster: GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE ?= e2-standard-4
 gcloud-terraform-cluster: AGONES_VERSION ?= ''
@@ -53,8 +56,12 @@ gcloud-terraform-cluster:
 		-var name=$(GCP_TF_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \
 		-var values_file="" \
 		-var feature_gates=$(FEATURE_GATES) \
-		-var zone="$(GCP_CLUSTER_ZONE)" -var project="$(GCP_PROJECT)" \
+		-var project="$(GCP_PROJECT)" \
+		-var location="$(GCP_CLUSTER_LOCATION)" \
 		-var log_level="$(LOG_LEVEL)" \
+		-var autoscale=$(GCP_CLUSTER_NODEPOOL_AUTOSCALE) \
+		-var min_node_count=$(GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT) \
+		-var max_node_count=$(GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT) \
 		-var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT) \
 		-var enable_image_streaming=$(GCP_CLUSTER_NODEPOOL_ENABLEIMAGESTREAMING) \
 		-var windows_node_count=$(GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT) \
@@ -66,6 +73,9 @@ gcloud-terraform-cluster:
 # Unifies previous `make gcloud-test-cluster` and `make install` targets
 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT ?= 4
 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_MACHINETYPE ?= e2-standard-4
+gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_AUTOSCALE ?= false
+gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT ?= 1
+gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT ?= 5
 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT ?= 0
 gcloud-terraform-install: GCP_CLUSTER_NODEPOOL_WINDOWSMACHINETYPE ?= e2-standard-4
 gcloud-terraform-install: ALWAYS_PULL_SIDECAR := true
@@ -87,8 +97,12 @@ gcloud-terraform-install:
 		-var crd_cleanup="$(CRD_CLEANUP)" \
 		-var chart="../../../install/helm/agones/" \
 		-var name=$(GCP_TF_CLUSTER_NAME) -var machine_type="$(GCP_CLUSTER_NODEPOOL_MACHINETYPE)" \
-		-var zone=$(GCP_CLUSTER_ZONE) -var project=$(GCP_PROJECT) \
+		-var project=$(GCP_PROJECT) \
+		-var location=$(GCP_CLUSTER_LOCATION) \
 		-var log_level=$(LOG_LEVEL) \
+		-var autoscale=$(GCP_CLUSTER_NODEPOOL_AUTOSCALE) \
+		-var min_node_count=$(GCP_CLUSTER_NODEPOOL_MIN_NODECOUNT) \
+		-var max_node_count=$(GCP_CLUSTER_NODEPOOL_MAX_NODECOUNT) \
 		-var feature_gates=$(FEATURE_GATES) \
 		-var node_count=$(GCP_CLUSTER_NODEPOOL_INITIALNODECOUNT) \
 		-var windows_node_count=$(GCP_CLUSTER_NODEPOOL_WINDOWSINITIALNODECOUNT) \

build/terraform/gke/module.tf

@@ -63,9 +63,14 @@ variable "ping_service_type" {
   default = "LoadBalancer"
 }
 
-variable "zone" {
+variable "location" {
   default     = "us-west1-c"
-  description = "The GCP zone to create the cluster in"
+  description = "The GCP location to create the cluster in"
+}
+
+variable "zone" {
+  default     = ""
+  description = "The GCP zone to create the cluster in (deprecated, use `location`)"
 }
 
 variable "pull_policy" {
@@ -88,6 +93,18 @@ variable "log_level" {
   default = "info"
 }
 
+variable "autoscale" {
+  default = "false"
+}
+
+variable "min_node_count" {
+  default = "1"
+}
+
+variable "max_node_count" {
+  default = "5"
+}
+
 // Note: This is the number of gameserver nodes. The Agones module will automatically create an additional
 // two node pools with 1 node each for "agones-system" and "agones-metrics".
 variable "node_count" {
@@ -116,6 +133,7 @@ module "gke_cluster" {
 
   cluster = {
     "name"                    = var.name
+    "location"		      = var.location
     "zone"                    = var.zone
     "machineType"             = var.machine_type
     "initialNodeCount"        = var.node_count
@@ -124,6 +142,9 @@ module "gke_cluster" {
     "windowsInitialNodeCount" = var.windows_node_count
     "project"                 = var.project
     "network"                 = var.network
+    "autoscale"		      = var.autoscale
+    "minNodeCount"	      = var.min_node_count
+    "maxNodeCount"	      = var.max_node_count
   }
 }
 

examples/terraform-submodules/gke/module.tf

@@ -53,9 +53,14 @@ variable "enable_image_streaming" {
   default = "true"
 }
 
-variable "zone" {
+variable "location" {
   default     = "us-west1-c"
-  description = "The GCP zone to create the cluster in"
+  description = "The GCP location to create the cluster in"
+}
+
+variable "zone" {
+  default     = ""
+  description = "The GCP zone to create the cluster in (deprecated, use `location`)"
 }
 
 variable "network" {
@@ -84,6 +89,18 @@ variable "windows_machine_type" {
   default = "e2-standard-4"
 }
 
+variable "autoscale" {
+  default = "false"
+}
+
+variable "min_node_count" {
+  default = "1"
+}
+
+variable "max_node_count" {
+  default = "5"
+}
+
 module "gke_cluster" {
   // ***************************************************************************************************
   // Update ?ref= to the agones release you are installing. For example, ?ref=release-1.17.0 corresponds
@@ -93,6 +110,7 @@ module "gke_cluster" {
 
   cluster = {
     "name"                    = var.name
+    "location"		      = var.location
     "zone"                    = var.zone
     "machineType"             = var.machine_type
     "initialNodeCount"        = var.node_count
@@ -102,6 +120,9 @@ module "gke_cluster" {
     "subnetwork"              = var.subnetwork
     "windowsInitialNodeCount" = var.windows_node_count
     "windowsMachineType"      = var.windows_machine_type
+    "autoscale"		      = var.autoscale
+    "mindNodeCount"	      = var.min_node_count
+    "maxNodeCount"	      = var.max_node_count
   }
 }
 

install/terraform/modules/gke/cluster.tf

@@ -23,7 +23,8 @@ data "google_client_config" "default" {}
 # Set values to default if not key was not set in original map
 locals {
   project                 = lookup(var.cluster, "project", "agones")
-  zone                    = lookup(var.cluster, "zone", "us-west1-c")
+  location		  = lookup(var.cluster, "location", "us-west1-c")
+  zone                    = lookup(var.cluster, "zone", "")
   name                    = lookup(var.cluster, "name", "test-cluster")
   machineType             = lookup(var.cluster, "machineType", "e2-standard-4")
   initialNodeCount        = lookup(var.cluster, "initialNodeCount", "4")
@@ -33,6 +34,9 @@ locals {
   kubernetesVersion       = lookup(var.cluster, "kubernetesVersion", "1.23")
   windowsInitialNodeCount = lookup(var.cluster, "windowsInitialNodeCount", "0")
   windowsMachineType      = lookup(var.cluster, "windowsMachineType", "e2-standard-4")
+  autoscale 		  = lookup(var.cluster, "autoscale", false)
+  minNodeCount		  = lookup(var.cluster, "minNodeCount", "1")
+  maxNodeCount		  = lookup(var.cluster, "maxNodeCount", "5")
 }
 
 # echo command used for debugging purpose
@@ -56,7 +60,7 @@ resource "null_resource" "test-setting-variables" {
 
 resource "google_container_cluster" "primary" {
   name       = local.name
-  location   = local.zone
+  location   = local.zone != "" ? local.zone : local.location
   project    = local.project
   network    = local.network
   subnetwork = local.subnetwork
@@ -65,9 +69,17 @@ resource "google_container_cluster" "primary" {
 
   node_pool {
     name       = "default"
-    node_count = local.initialNodeCount
+    node_count = local.autoscale ? null : local.initialNodeCount
     version    = local.kubernetesVersion
 
+    dynamic "autoscaling" {
+      for_each = local.autoscale ? [1] : []
+      content {
+      	min_node_count = local.minNodeCount
+	max_node_count = local.maxNodeCount
+      }
+    }
+
     management {
       auto_upgrade = false
     }

install/terraform/modules/gke/variables.tf

@@ -30,7 +30,7 @@ variable "cluster" {
   type        = map
 
   default = {
-    "zone"                    = "us-west1-c"
+    "location"		      = "us-west1-c"
     "name"                    = "test-cluster"
     "machineType"             = "e2-standard-4"
     "initialNodeCount"        = "4"
@@ -40,6 +40,9 @@ variable "cluster" {
     "kubernetesVersion"       = "1.23"
     "windowsInitialNodeCount" = "0"
     "windowsMachineType"      = "e2-standard-4"
+    "autoscale"		      = false
+    "minNodeCount"	      = "1"
+    "maxNodeCount"	      = "5"
   }
 }
 

site/content/en/docs/Installation/Terraform/gke.md

@@ -86,8 +86,14 @@ Configurable parameters:
 - machine_type - machine type for hosting game servers (default is "e2-standard-4")
 - node_count - count of game server nodes for the default node pool (default is "4") 
 - enable_image_streaming - whether or not to enable image streaming for the `"default"` node pool (default is true) 
+{{% feature expiryVersion="1.28" %}}
 - zone - the name of the [zone](https://cloud.google.com/compute/docs/regions-zones) you want your cluster to be
   created in (default is "us-west1-c")
+{{% /feature %}}
+{{% feature publishVersion="1.28" %}}
+- zone - (Deprecated, use location) the name of the [zone](https://cloud.google.com/compute/docs/regions-zones) you want your cluster to be
+  created in (default is "us-west1-c")
+{{% /feature %}}
 - network - the name of the VPC network you want your cluster and firewall rules to be connected to (default is "default")
 - subnetwork - the name of the subnetwork in which the cluster's instances are launched. (required when using non default network)
 - log_level - possible values: Fatal, Error, Warn, Info, Debug (default is "info")
@@ -96,6 +102,12 @@ Configurable parameters:
 - gameserver_maxPort - the upper bound of the port range which gameservers will listen on (default is "8000")
 - gameserver_namespaces - a list of namespaces which will be used to run gameservers (default is `["default"]`). For example `["default", "xbox-gameservers", "mobile-gameservers"]`
 - force_update - whether or not to force the replacement/update of resource (default is true, false may be required to prevent immutability errors when updating the configuration)
+{{% feature publishVersion="1.28" %}}
+- location - the name of the [location](https://cloud.google.com/compute/docs/regions-zones) you want your cluster  to eb created in (default is "us-west1-c")
+- autoscale - whether you want to enable autoscale for the gameserver nodepool (default is false)
+- min_node_count - the minimum number of nodes for a nodepool when autoscale is enabled (default is "1")
+- max_node_count - the maximum number of nodes for a nodepool when autoscale is enabled (default is "5")
+{{% /feature %}}
 
 {{% alert title="Warning" color="warning"%}}
 On the lines that read `source = "git::https://github.com/googleforgames/agones.git//install/terraform/modules/gke/?ref=main"`