GCE: support egress specification

pkg/model/gcemodel/network.go

@@ -17,6 +17,8 @@ limitations under the License.
 package gcemodel
 
 import (
+	"fmt"
+
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
@@ -81,23 +83,49 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	// Create a CloudNAT for private topology.
-	if b.Cluster.Spec.Topology.Masters == kops.TopologyPrivate {
-		var hasPrivateSubnet bool
-		for _, subnet := range b.Cluster.Spec.Subnets {
-			if subnet.Type == kops.SubnetTypePrivate {
-				hasPrivateSubnet = true
-				break
+	{
+		// We only consider private subnets.
+		// Then if we are creating subnet, we will create a NAT gateway tied to those subnets.
+		// This can be over-ridden by specifying "external", in which case we will not create a NAT gateway.
+		// If we are reusing an existing subnet, we assume that the NAT gateway is already configured.
+
+		var subnetworks []*gcetasks.Subnet
+
+		for i := range b.Cluster.Spec.Subnets {
+			subnet := &b.Cluster.Spec.Subnets[i]
+			// Only need to deal with private subnets
+			if subnet.Type != kops.SubnetTypePrivate {
+				continue
+			}
+
+			// If we're in an existing subnet, we assume egress is already configured.
+			if subnet.ProviderID != "" {
+				continue
+			}
+
+			switch subnet.Egress {
+			case kops.EgressExternal:
+				// User has request we ignore this
+				continue
+
+			case kops.EgressNatGateway, "":
+				// OK, should create
+				subnetworks = append(subnetworks, b.LinkToSubnet(subnet))
+
+			default:
+				return fmt.Errorf("egress mode %q is not supported", subnet.Egress)
 			}
 		}
 
-		if hasPrivateSubnet {
+		if len(subnetworks) != 0 {
 			r := &gcetasks.Router{
 				Name:                          s(b.SafeObjectName("nat")),
 				Lifecycle:                     b.Lifecycle,
 				Network:                       b.LinkToNetwork(),
 				Region:                        s(b.Region),
 				NATIPAllocationOption:         s(gcetasks.NATIPAllocationOptionAutoOnly),
-				SourceSubnetworkIPRangesToNAT: s(gcetasks.SourceSubnetworkIPRangesAll),
+				SourceSubnetworkIPRangesToNAT: s(gcetasks.SourceSubnetworkIPRangesSpecificSubnets),
+				Subnetworks:                   subnetworks,
 			}
 			c.AddTask(r)
 		}

tests/integration/update_cluster/minimal_gce_private/kubernetes.tf

@@ -513,7 +513,11 @@ resource "google_compute_router_nat" "nat-minimal-gce-private-example-com" {
   nat_ip_allocate_option             = "AUTO_ONLY"
   region                             = "us-test1"
   router                             = "nat-minimal-gce-private-example-com"
-  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+  source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS"
+  subnetwork {
+    name                    = google_compute_subnetwork.us-test1-minimal-gce-private-example-com.name
+    source_ip_ranges_to_nat = ["ALL_IP_RANGES"]
+  }
 }
 
 resource "google_compute_subnetwork" "us-test1-minimal-gce-private-example-com" {

upup/pkg/fi/cloudup/gcetasks/router.go

@@ -19,6 +19,7 @@ package gcetasks
 import (
 	"fmt"
 	"reflect"
+	"strings"
 
 	compute "google.golang.org/api/compute/v1"
 	"k8s.io/klog/v2"
@@ -31,8 +32,14 @@ import (
 const (
 	// NATIPAllocationOptionAutoOnly is specified when NAT IPs are allocated by Google Cloud.
 	NATIPAllocationOptionAutoOnly = "AUTO_ONLY"
+
 	// SourceSubnetworkIPRangesAll is specified when all of the IP ranges in every subnetwork are allowed to be NAT-ed.
 	SourceSubnetworkIPRangesAll = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+	// SourceSubnetworkIPRangesSpecificSubnets is specified when we should NAT only specific listed subnets.
+	SourceSubnetworkIPRangesSpecificSubnets = "LIST_OF_SUBNETWORKS"
+
+	// subnetNatAllIPRanges specifies that we should NAT all IP ranges in the subnet.
+	subnetNatAllIPRanges = "ALL_IP_RANGES"
 )
 
 // +kops:fitask
@@ -47,6 +54,8 @@ type Router struct {
 
 	NATIPAllocationOption         *string
 	SourceSubnetworkIPRangesToNAT *string
+
+	Subnetworks []*Subnet
 }
 
 var _ fi.CompareWithID = &Router{}
@@ -65,7 +74,7 @@ func (r *Router) Find(c *fi.Context) (*Router, error) {
 		if gce.IsNotFound(err) {
 			return nil, nil
 		}
-		return nil, fmt.Errorf("error listing Routers: %v", err)
+		return nil, fmt.Errorf("error listing Routers: %w", err)
 	}
 
 	if len(found.Nats) != 1 {
@@ -77,14 +86,25 @@ func (r *Router) Find(c *fi.Context) (*Router, error) {
 		klog.Warningf("SelfLink did not match URL: %q vs %q", a, e)
 	}
 
-	return &Router{
+	actual := &Router{
 		Name:                          &found.Name,
 		Lifecycle:                     r.Lifecycle,
 		Network:                       &Network{Name: fi.String(lastComponent(found.Network))},
 		Region:                        fi.String(lastComponent(found.Region)),
 		NATIPAllocationOption:         &nat.NatIpAllocateOption,
 		SourceSubnetworkIPRangesToNAT: &nat.SourceSubnetworkIpRangesToNat,
-	}, nil
+	}
+
+	for _, subnet := range nat.Subnetworks {
+		if strings.Join(subnet.SourceIpRangesToNat, ",") != subnetNatAllIPRanges {
+			klog.Warningf("ignoring NAT router %q with nats.subnetworks.sourceIpRangesToNat != %s", found.SelfLink, subnetNatAllIPRanges)
+			return nil, nil
+		}
+
+		subnetName := lastComponent(subnet.Name)
+		actual.Subnetworks = append(actual.Subnetworks, &Subnet{Name: &subnetName})
+	}
+	return actual, nil
 
 }
 
@@ -128,6 +148,7 @@ func (*Router) CheckChanges(a, e, changes *Router) error {
 func (*Router) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Router) error {
 	cloud := t.Cloud
 	project := cloud.Project()
+	region := fi.StringValue(e.Region)
 
 	if a == nil {
 		klog.V(2).Infof("Creating Cloud NAT Gateway %v", e.Name)
@@ -142,8 +163,19 @@ func (*Router) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Router) error {
 				},
 			},
 		}
-		if _, err := t.Cloud.Compute().Routers().Insert(t.Cloud.Project(), *e.Region, router); err != nil {
-			return fmt.Errorf("error creating Router: %v", err)
+
+		for _, subnet := range e.Subnetworks {
+			router.Nats[0].Subnetworks = append(router.Nats[0].Subnetworks, &compute.RouterNatSubnetworkToNat{
+				Name:                subnet.URL(project, region),
+				SourceIpRangesToNat: []string{subnetNatAllIPRanges},
+			})
+		}
+		op, err := t.Cloud.Compute().Routers().Insert(project, region, router)
+		if err != nil {
+			return fmt.Errorf("error creating Router: %w", err)
+		}
+		if err := t.Cloud.WaitForOp(op); err != nil {
+			return fmt.Errorf("error waiting for router creation: %w", err)
 		}
 	} else {
 		if !reflect.DeepEqual(changes, &Router{}) {
@@ -155,11 +187,17 @@ func (*Router) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Router) error {
 }
 
 type terraformRouterNat struct {
-	Name                          *string `json:"name,omitempty" cty:"name"`
-	Region                        *string `json:"region,omitempty" cty:"region"`
-	Router                        *string `json:"router,omitempty" cty:"router"`
-	NATIPAllocateOption           *string `json:"nat_ip_allocate_option,omitempty" cty:"nat_ip_allocate_option"`
-	SourceSubnetworkIPRangesToNat *string `json:"source_subnetwork_ip_ranges_to_nat,omitempty" cty:"source_subnetwork_ip_ranges_to_nat"`
+	Name                          *string                         `json:"name,omitempty" cty:"name"`
+	Region                        *string                         `json:"region,omitempty" cty:"region"`
+	Router                        *string                         `json:"router,omitempty" cty:"router"`
+	NATIPAllocateOption           *string                         `json:"nat_ip_allocate_option,omitempty" cty:"nat_ip_allocate_option"`
+	SourceSubnetworkIPRangesToNat *string                         `json:"source_subnetwork_ip_ranges_to_nat,omitempty" cty:"source_subnetwork_ip_ranges_to_nat"`
+	Subnetworks                   []*terraformRouterNatSubnetwork `json:"subnetwork,omitempty" cty:"subnetwork"`
+}
+
+type terraformRouterNatSubnetwork struct {
+	Name                *terraformWriter.Literal `json:"name,omitempty" cty:"name"`
+	SourceIPRangesToNat []string                 `json:"source_ip_ranges_to_nat,omitempty" cty:"source_ip_ranges_to_nat"`
 }
 
 type terraformRouter struct {
@@ -188,6 +226,12 @@ func (*Router) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Rout
 		NATIPAllocateOption:           e.NATIPAllocationOption,
 		SourceSubnetworkIPRangesToNat: e.SourceSubnetworkIPRangesToNAT,
 	}
+	for _, subnet := range e.Subnetworks {
+		trn.Subnetworks = append(trn.Subnetworks, &terraformRouterNatSubnetwork{
+			Name:                subnet.TerraformLink(),
+			SourceIPRangesToNat: []string{subnetNatAllIPRanges},
+		})
+	}
 	return t.RenderResource("google_compute_router_nat", *e.Name, trn)
 }