feat: Add support for Logging Variant to enable max throughput option (…

…#1616)

Co-authored-by: Eric Zhao <zhao.yichao15@gmail.com>
Co-authored-by: Eric Zhao <zhaoe@google.com>

README.md

@@ -65,6 +65,7 @@ module "gke" {
       image_type                = "COS_CONTAINERD"
       enable_gcfs               = false
       enable_gvnic              = false
+      logging_variant           = "DEFAULT"
       auto_repair               = true
       auto_upgrade              = true
       service_account           = "project-service-account@<PROJECT ID>.iam.gserviceaccount.com"
@@ -279,6 +280,7 @@ The node_pools variable takes the following parameters:
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
 | key | The key required for the taint | | Required |
+| logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |
 | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional |
 | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional |

autogen/main/README.md

@@ -111,6 +111,7 @@ module "gke" {
       image_type                = "COS_CONTAINERD"
       enable_gcfs               = false
       enable_gvnic              = false
+      logging_variant           = "DEFAULT"
       auto_repair               = true
       auto_upgrade              = true
       service_account           = "project-service-account@<PROJECT ID>.iam.gserviceaccount.com"
@@ -208,6 +209,7 @@ The node_pools variable takes the following parameters:
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
 | key | The key required for the taint | | Required |
+| logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |
 {% if beta_cluster %}
 | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional |

autogen/main/cluster.tf.tmpl

@@ -427,6 +427,8 @@ resource "google_container_cluster" "primary" {
         lookup(local.node_pools_tags, var.node_pools[0].name, []),
       )
 
+      logging_variant = lookup(var.node_pools[0], "logging_variant", "DEFAULT")
+
       dynamic "workload_metadata_config" {
         for_each = local.cluster_node_metadata_config
 
@@ -796,6 +798,8 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")

cluster.tf

@@ -305,6 +305,8 @@ resource "google_container_cluster" "primary" {
         lookup(local.node_pools_tags, var.node_pools[0].name, []),
       )
 
+      logging_variant = lookup(var.node_pools[0], "logging_variant", "DEFAULT")
+
       dynamic "workload_metadata_config" {
         for_each = local.cluster_node_metadata_config
 
@@ -500,6 +502,8 @@ resource "google_container_node_pool" "pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")
@@ -686,6 +690,8 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")

modules/beta-private-cluster-update-variant/README.md

@@ -97,6 +97,7 @@ module "gke" {
       image_type                = "COS_CONTAINERD"
       enable_gcfs               = false
       enable_gvnic              = false
+      logging_variant           = "DEFAULT"
       auto_repair               = true
       auto_upgrade              = true
       service_account           = "project-service-account@<PROJECT ID>.iam.gserviceaccount.com"
@@ -344,6 +345,7 @@ The node_pools variable takes the following parameters:
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
 | key | The key required for the taint | | Required |
+| logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |
 | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional |
 | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional |

modules/beta-private-cluster-update-variant/cluster.tf

@@ -362,6 +362,8 @@ resource "google_container_cluster" "primary" {
         lookup(local.node_pools_tags, var.node_pools[0].name, []),
       )
 
+      logging_variant = lookup(var.node_pools[0], "logging_variant", "DEFAULT")
+
       dynamic "workload_metadata_config" {
         for_each = local.cluster_node_metadata_config
 
@@ -692,6 +694,8 @@ resource "google_container_node_pool" "pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")
@@ -918,6 +922,8 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")

modules/beta-private-cluster/README.md

@@ -75,6 +75,7 @@ module "gke" {
       image_type                = "COS_CONTAINERD"
       enable_gcfs               = false
       enable_gvnic              = false
+      logging_variant           = "DEFAULT"
       auto_repair               = true
       auto_upgrade              = true
       service_account           = "project-service-account@<PROJECT ID>.iam.gserviceaccount.com"
@@ -322,6 +323,7 @@ The node_pools variable takes the following parameters:
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
 | key | The key required for the taint | | Required |
+| logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |
 | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional |
 | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional |

modules/beta-private-cluster/cluster.tf

@@ -362,6 +362,8 @@ resource "google_container_cluster" "primary" {
         lookup(local.node_pools_tags, var.node_pools[0].name, []),
       )
 
+      logging_variant = lookup(var.node_pools[0], "logging_variant", "DEFAULT")
+
       dynamic "workload_metadata_config" {
         for_each = local.cluster_node_metadata_config
 
@@ -598,6 +600,8 @@ resource "google_container_node_pool" "pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")
@@ -823,6 +827,8 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")

modules/beta-public-cluster-update-variant/README.md

@@ -91,6 +91,7 @@ module "gke" {
       image_type                = "COS_CONTAINERD"
       enable_gcfs               = false
       enable_gvnic              = false
+      logging_variant           = "DEFAULT"
       auto_repair               = true
       auto_upgrade              = true
       service_account           = "project-service-account@<PROJECT ID>.iam.gserviceaccount.com"
@@ -331,6 +332,7 @@ The node_pools variable takes the following parameters:
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
 | key | The key required for the taint | | Required |
+| logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |
 | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional |
 | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional |

modules/beta-public-cluster-update-variant/cluster.tf

@@ -362,6 +362,8 @@ resource "google_container_cluster" "primary" {
         lookup(local.node_pools_tags, var.node_pools[0].name, []),
       )
 
+      logging_variant = lookup(var.node_pools[0], "logging_variant", "DEFAULT")
+
       dynamic "workload_metadata_config" {
         for_each = local.cluster_node_metadata_config
 
@@ -673,6 +675,8 @@ resource "google_container_node_pool" "pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")
@@ -899,6 +903,8 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")

modules/beta-public-cluster/README.md

@@ -69,6 +69,7 @@ module "gke" {
       image_type                = "COS_CONTAINERD"
       enable_gcfs               = false
       enable_gvnic              = false
+      logging_variant           = "DEFAULT"
       auto_repair               = true
       auto_upgrade              = true
       service_account           = "project-service-account@<PROJECT ID>.iam.gserviceaccount.com"
@@ -309,6 +310,7 @@ The node_pools variable takes the following parameters:
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
 | key | The key required for the taint | | Required |
+| logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |
 | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional |
 | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional |

modules/beta-public-cluster/cluster.tf

@@ -362,6 +362,8 @@ resource "google_container_cluster" "primary" {
         lookup(local.node_pools_tags, var.node_pools[0].name, []),
       )
 
+      logging_variant = lookup(var.node_pools[0], "logging_variant", "DEFAULT")
+
       dynamic "workload_metadata_config" {
         for_each = local.cluster_node_metadata_config
 
@@ -579,6 +581,8 @@ resource "google_container_node_pool" "pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")
@@ -804,6 +808,8 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")

modules/private-cluster-update-variant/README.md

@@ -93,6 +93,7 @@ module "gke" {
       image_type                = "COS_CONTAINERD"
       enable_gcfs               = false
       enable_gvnic              = false
+      logging_variant           = "DEFAULT"
       auto_repair               = true
       auto_upgrade              = true
       service_account           = "project-service-account@<PROJECT ID>.iam.gserviceaccount.com"
@@ -314,6 +315,7 @@ The node_pools variable takes the following parameters:
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
 | key | The key required for the taint | | Required |
+| logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |
 | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional |
 | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional |

modules/private-cluster-update-variant/cluster.tf

@@ -305,6 +305,8 @@ resource "google_container_cluster" "primary" {
         lookup(local.node_pools_tags, var.node_pools[0].name, []),
       )
 
+      logging_variant = lookup(var.node_pools[0], "logging_variant", "DEFAULT")
+
       dynamic "workload_metadata_config" {
         for_each = local.cluster_node_metadata_config
 
@@ -613,6 +615,8 @@ resource "google_container_node_pool" "pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")
@@ -800,6 +804,8 @@ resource "google_container_node_pool" "windows_pools" {
       local.node_pools_tags[each.value["name"]],
     )
 
+    logging_variant = lookup(each.value, "logging_variant", "DEFAULT")
+
     local_ssd_count = lookup(each.value, "local_ssd_count", 0)
     disk_size_gb    = lookup(each.value, "disk_size_gb", 100)
     disk_type       = lookup(each.value, "disk_type", "pd-standard")

modules/private-cluster/README.md

@@ -71,6 +71,7 @@ module "gke" {
       image_type                = "COS_CONTAINERD"
       enable_gcfs               = false
       enable_gvnic              = false
+      logging_variant           = "DEFAULT"
       auto_repair               = true
       auto_upgrade              = true
       service_account           = "project-service-account@<PROJECT ID>.iam.gserviceaccount.com"
@@ -292,6 +293,7 @@ The node_pools variable takes the following parameters:
 | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional |
 | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional |
 | key | The key required for the taint | | Required |
+| logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT.  | DEFAULT | Optional |
 | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume.  | 0 | Optional |
 | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional |
 | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional |