Skip to content

Commit

Permalink
fix: Create separate firewall rule for egress to TPUs (#1126)
Browse files Browse the repository at this point in the history 
BREAKING CHANGE: TPU firewall rule split into a separate resource
  • Loading branch information
@tomasgareau
tomasgareau committed Jan 13, 2022
1 parent b8b8547 commit 99cfd98
Show file tree
Hide file tree
Showing 5 changed files with 185 additions and 20 deletions.
47 changes: 39 additions & 8 deletions autogen/main/firewall.tf.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,50 @@ resource "google_compute_firewall" "intra_egress" {
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
{% if beta_cluster %}
destination_ranges = compact([
local.cluster_endpoint_for_nodes,
local.cluster_subnet_cidr,
local.cluster_alias_ranges_cidr[var.ip_range_pods],
google_container_cluster.primary.tpu_ipv4_cidr_block,
])
{% else %}
destination_ranges = [
local.cluster_endpoint_for_nodes,
local.cluster_subnet_cidr,
local.cluster_alias_ranges_cidr[var.ip_range_pods],
]

# Allow all possible protocols
allow { protocol = "tcp" }
allow { protocol = "udp" }
allow { protocol = "icmp" }
allow { protocol = "sctp" }
allow { protocol = "esp" }
allow { protocol = "ah" }

{% if not private_cluster %}
depends_on = [
google_container_cluster.primary,
]
{% endif %}
}


{% if beta_cluster %}
/******************************************
Allow egress to the TPU IPv4 CIDR block

This rule is defined separately from the
intra_egress rule above since it requires
an output from the google_container_cluster
resource.

https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
*****************************************/
resource "google_compute_firewall" "tpu_egress" {
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
project = local.network_project_id
network = var.network
priority = var.firewall_priority
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]

# Allow all possible protocols
allow { protocol = "tcp" }
Expand All @@ -65,6 +95,7 @@ resource "google_compute_firewall" "intra_egress" {
}


{% endif %}
/******************************************
Allow GKE master to hit non 443 ports for
Webhooks/Admission Controllers
Expand Down
38 changes: 35 additions & 3 deletions modules/beta-private-cluster-update-variant/firewall.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,44 @@ resource "google_compute_firewall" "intra_egress" {
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
destination_ranges = compact([
destination_ranges = [
local.cluster_endpoint_for_nodes,
local.cluster_subnet_cidr,
local.cluster_alias_ranges_cidr[var.ip_range_pods],
google_container_cluster.primary.tpu_ipv4_cidr_block,
])
]

# Allow all possible protocols
allow { protocol = "tcp" }
allow { protocol = "udp" }
allow { protocol = "icmp" }
allow { protocol = "sctp" }
allow { protocol = "esp" }
allow { protocol = "ah" }

}


/******************************************
Allow egress to the TPU IPv4 CIDR block
This rule is defined separately from the
intra_egress rule above since it requires
an output from the google_container_cluster
resource.
https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
*****************************************/
resource "google_compute_firewall" "tpu_egress" {
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
project = local.network_project_id
network = var.network
priority = var.firewall_priority
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]

# Allow all possible protocols
allow { protocol = "tcp" }
Expand Down
38 changes: 35 additions & 3 deletions modules/beta-private-cluster/firewall.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,44 @@ resource "google_compute_firewall" "intra_egress" {
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
destination_ranges = compact([
destination_ranges = [
local.cluster_endpoint_for_nodes,
local.cluster_subnet_cidr,
local.cluster_alias_ranges_cidr[var.ip_range_pods],
google_container_cluster.primary.tpu_ipv4_cidr_block,
])
]

# Allow all possible protocols
allow { protocol = "tcp" }
allow { protocol = "udp" }
allow { protocol = "icmp" }
allow { protocol = "sctp" }
allow { protocol = "esp" }
allow { protocol = "ah" }

}


/******************************************
Allow egress to the TPU IPv4 CIDR block
This rule is defined separately from the
intra_egress rule above since it requires
an output from the google_container_cluster
resource.
https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
*****************************************/
resource "google_compute_firewall" "tpu_egress" {
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
project = local.network_project_id
network = var.network
priority = var.firewall_priority
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]

# Allow all possible protocols
allow { protocol = "tcp" }
Expand Down
41 changes: 38 additions & 3 deletions modules/beta-public-cluster-update-variant/firewall.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,47 @@ resource "google_compute_firewall" "intra_egress" {
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
destination_ranges = compact([
destination_ranges = [
local.cluster_endpoint_for_nodes,
local.cluster_subnet_cidr,
local.cluster_alias_ranges_cidr[var.ip_range_pods],
google_container_cluster.primary.tpu_ipv4_cidr_block,
])
]

# Allow all possible protocols
allow { protocol = "tcp" }
allow { protocol = "udp" }
allow { protocol = "icmp" }
allow { protocol = "sctp" }
allow { protocol = "esp" }
allow { protocol = "ah" }

depends_on = [
google_container_cluster.primary,
]
}


/******************************************
Allow egress to the TPU IPv4 CIDR block
This rule is defined separately from the
intra_egress rule above since it requires
an output from the google_container_cluster
resource.
https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
*****************************************/
resource "google_compute_firewall" "tpu_egress" {
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
project = local.network_project_id
network = var.network
priority = var.firewall_priority
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]

# Allow all possible protocols
allow { protocol = "tcp" }
Expand Down
41 changes: 38 additions & 3 deletions modules/beta-public-cluster/firewall.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,47 @@ resource "google_compute_firewall" "intra_egress" {
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
destination_ranges = compact([
destination_ranges = [
local.cluster_endpoint_for_nodes,
local.cluster_subnet_cidr,
local.cluster_alias_ranges_cidr[var.ip_range_pods],
google_container_cluster.primary.tpu_ipv4_cidr_block,
])
]

# Allow all possible protocols
allow { protocol = "tcp" }
allow { protocol = "udp" }
allow { protocol = "icmp" }
allow { protocol = "sctp" }
allow { protocol = "esp" }
allow { protocol = "ah" }

depends_on = [
google_container_cluster.primary,
]
}


/******************************************
Allow egress to the TPU IPv4 CIDR block
This rule is defined separately from the
intra_egress rule above since it requires
an output from the google_container_cluster
resource.
https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1124
*****************************************/
resource "google_compute_firewall" "tpu_egress" {
count = var.add_cluster_firewall_rules && var.enable_tpu ? 1 : 0
name = "gke-${substr(var.name, 0, min(25, length(var.name)))}-tpu-egress"
description = "Managed by terraform gke module: Allow pods to communicate with TPUs"
project = local.network_project_id
network = var.network
priority = var.firewall_priority
direction = "EGRESS"

target_tags = [local.cluster_network_tag]
destination_ranges = [google_container_cluster.primary.tpu_ipv4_cidr_block]

# Allow all possible protocols
allow { protocol = "tcp" }
Expand Down

0 comments on commit 99cfd98

Please sign in to comment.