From b80ab99805110c77be953a4a4bc719a0c6111167 Mon Sep 17 00:00:00 2001 From: Bob Fournier Date: Wed, 1 May 2024 11:28:00 -0400 Subject: [PATCH] Create an Internal Load Balancer if configured Provide the ability to configure the types of Load Balancers to be created (Internal and/or External). By default, an External Proxy Load Balancer will be created per the current implementation. If set for an Internal Load Balancer, an Internal Passthrough Load Balancer will be created using resources in the specified region. --- api/v1beta1/labels.go | 3 + api/v1beta1/types.go | 62 ++ api/v1beta1/zz_generated.deepcopy.go | 55 ++ cloud/scope/cluster.go | 16 +- .../compute/loadbalancers/reconcile.go | 547 ++++++++++++++--- .../compute/loadbalancers/reconcile_test.go | 560 +++++++++++++++++- .../services/compute/loadbalancers/service.go | 51 +- ...tructure.cluster.x-k8s.io_gcpclusters.yaml | 42 ++ ....cluster.x-k8s.io_gcpclustertemplates.yaml | 22 + ...e.cluster.x-k8s.io_gcpmanagedclusters.yaml | 42 ++ controllers/gcpcluster_controller.go | 3 +- 11 files changed, 1303 insertions(+), 100 deletions(-) diff --git a/api/v1beta1/labels.go b/api/v1beta1/labels.go index 264f70f99..eded27abf 100644 --- a/api/v1beta1/labels.go +++ b/api/v1beta1/labels.go @@ -111,6 +111,9 @@ const ( // APIServerRoleTagValue describes the value for the apiserver role. APIServerRoleTagValue = "apiserver" + + // InternalRoleTagValue describes the value for the internal role. + InternalRoleTagValue = "api-internal" ) // ClusterTagKey generates the key for resources associated with a cluster. diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 763916cb2..7a760368b 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -85,6 +85,26 @@ type Network struct { // created for the API Server. // +optional APIServerForwardingRule *string `json:"apiServerForwardingRule,omitempty"` + + // APIInternalAddress is the IPV4 regional address assigned to the + // internal Load Balancer. + // +optional + APIInternalAddress *string `json:"apiInternalIpAddress,omitempty"` + + // APIInternalHealthCheck is the full reference to the health check + // created for the internal Load Balancer. + // +optional + APIInternalHealthCheck *string `json:"apiInternalHealthCheck,omitempty"` + + // APIInternalBackendService is the full reference to the backend service + // created for the internal Load Balancer. + // +optional + APIInternalBackendService *string `json:"apiInternalBackendService,omitempty"` + + // APIInternalForwardingRule is the full reference to the forwarding rule + // created for the internal Load Balancer. + // +optional + APIInternalForwardingRule *string `json:"apiInternalForwardingRule,omitempty"` } // NetworkSpec encapsulates all things related to a GCP network. @@ -114,6 +134,24 @@ type NetworkSpec struct { LoadBalancerBackendPort *int32 `json:"loadBalancerBackendPort,omitempty"` } +// LoadBalancerType defines the Load Balancer that should be created. +type LoadBalancerType string + +var ( + // ExternalLoadBalancerOnly creates a Global External Proxy Load Balancer + // to manage traffic to backends in multiple regions. This is the default Load + // Balancer and will be created if no LoadBalancerType is defined. + ExternalLoadBalancerOnly = LoadBalancerType("EXTERNAL_ONLY") + + // InternalLoadBalancerOnly creates a Regional Internal Passthrough Load + // Balancer to manage traffic to backends in the configured region. + InternalLoadBalancerOnly = LoadBalancerType("INTERNAL_ONLY") + + // DualLoadBalancer creates both External and Internal Load Balancers to provide + // separate endpoints for managing both external and internal traffic. + DualLoadBalancer = LoadBalancerType("Dual") +) + // LoadBalancerSpec contains configuration for one or more LoadBalancers. type LoadBalancerSpec struct { // APIServerInstanceGroupTagOverride overrides the default setting for the @@ -123,6 +161,15 @@ type LoadBalancerSpec struct { // +kubebuilder:validation:Pattern=`(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)` // +optional APIServerInstanceGroupTagOverride *string `json:"apiServerInstanceGroupTagOverride,omitempty"` + + // LoadBalancerType defines the type of Load Balancer that should be created. + // If not set, a Global External Proxy Load Balancer will be created by default. + // +optional + LoadBalancerType *LoadBalancerType `json:"loadBalancerType,omitempty"` + + // InternalLoadBalancer is the configuration for an Internal Passthrough Network Load Balancer. + // +optional + InternalLoadBalancer *LoadBalancer `json:"internalLoadBalancer,omitempty"` } // SubnetSpec configures an GCP Subnet. @@ -278,3 +325,18 @@ type ObjectReference struct { // +kubebuilder:validation:Required Name string `json:"name"` } + +// LoadBalancer specifies the configuration of a LoadBalancer. +type LoadBalancer struct { + // Name is the name of the Load Balancer. If not set a default name + // will be used. For an Internal Load Balancer service the default + // name is "api-internal". + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:Pattern=`(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)` + // +optional + Name *string `json:"name,omitempty"` + + // Subnet is the name of the subnet to use for a regional Load Balancer. + Subnet *string `json:"subnet,omitempty"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index d3e20f307..c05c5bfab 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -674,6 +674,31 @@ func (in Labels) DeepCopy() Labels { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) { + *out = *in + if in.Name != nil { + in, out := &in.Name, &out.Name + *out = new(string) + **out = **in + } + if in.Subnet != nil { + in, out := &in.Subnet, &out.Subnet + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer. +func (in *LoadBalancer) DeepCopy() *LoadBalancer { + if in == nil { + return nil + } + out := new(LoadBalancer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = *in @@ -682,6 +707,16 @@ func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = new(string) **out = **in } + if in.LoadBalancerType != nil { + in, out := &in.LoadBalancerType, &out.LoadBalancerType + *out = new(LoadBalancerType) + **out = **in + } + if in.InternalLoadBalancer != nil { + in, out := &in.InternalLoadBalancer, &out.InternalLoadBalancer + *out = new(LoadBalancer) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerSpec. @@ -781,6 +816,26 @@ func (in *Network) DeepCopyInto(out *Network) { *out = new(string) **out = **in } + if in.APIInternalAddress != nil { + in, out := &in.APIInternalAddress, &out.APIInternalAddress + *out = new(string) + **out = **in + } + if in.APIInternalHealthCheck != nil { + in, out := &in.APIInternalHealthCheck, &out.APIInternalHealthCheck + *out = new(string) + **out = **in + } + if in.APIInternalBackendService != nil { + in, out := &in.APIInternalBackendService, &out.APIInternalBackendService + *out = new(string) + **out = **in + } + if in.APIInternalForwardingRule != nil { + in, out := &in.APIInternalForwardingRule, &out.APIInternalForwardingRule + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 6f7cb0a98..b32cac9f1 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -290,18 +290,18 @@ func (s *ClusterScope) FirewallRulesSpec() []*compute.Firewall { // ANCHOR: ClusterControlPlaneSpec // AddressSpec returns google compute address spec. -func (s *ClusterScope) AddressSpec() *compute.Address { +func (s *ClusterScope) AddressSpec(lbname string) *compute.Address { return &compute.Address{ - Name: fmt.Sprintf("%s-%s", s.Name(), infrav1.APIServerRoleTagValue), + Name: fmt.Sprintf("%s-%s", s.Name(), lbname), AddressType: "EXTERNAL", IpVersion: "IPV4", } } // BackendServiceSpec returns google compute backend-service spec. -func (s *ClusterScope) BackendServiceSpec() *compute.BackendService { +func (s *ClusterScope) BackendServiceSpec(lbname string) *compute.BackendService { return &compute.BackendService{ - Name: fmt.Sprintf("%s-%s", s.Name(), infrav1.APIServerRoleTagValue), + Name: fmt.Sprintf("%s-%s", s.Name(), lbname), LoadBalancingScheme: "EXTERNAL", PortName: "apiserver", Protocol: "TCP", @@ -310,14 +310,14 @@ func (s *ClusterScope) BackendServiceSpec() *compute.BackendService { } // ForwardingRuleSpec returns google compute forwarding-rule spec. -func (s *ClusterScope) ForwardingRuleSpec() *compute.ForwardingRule { +func (s *ClusterScope) ForwardingRuleSpec(lbname string) *compute.ForwardingRule { port := int32(443) if c := s.Cluster.Spec.ClusterNetwork; c != nil { port = ptr.Deref(c.APIServerPort, 443) } portRange := fmt.Sprintf("%d-%d", port, port) return &compute.ForwardingRule{ - Name: fmt.Sprintf("%s-%s", s.Name(), infrav1.APIServerRoleTagValue), + Name: fmt.Sprintf("%s-%s", s.Name(), lbname), IPProtocol: "TCP", LoadBalancingScheme: "EXTERNAL", PortRange: portRange, @@ -325,9 +325,9 @@ func (s *ClusterScope) ForwardingRuleSpec() *compute.ForwardingRule { } // HealthCheckSpec returns google compute health-check spec. -func (s *ClusterScope) HealthCheckSpec() *compute.HealthCheck { +func (s *ClusterScope) HealthCheckSpec(lbname string) *compute.HealthCheck { return &compute.HealthCheck{ - Name: fmt.Sprintf("%s-%s", s.Name(), infrav1.APIServerRoleTagValue), + Name: fmt.Sprintf("%s-%s", s.Name(), lbname), Type: "HTTPS", HttpsHealthCheck: &compute.HTTPSHealthCheck{ Port: 6443, diff --git a/cloud/services/compute/loadbalancers/reconcile.go b/cloud/services/compute/loadbalancers/reconcile.go index e6b723ec0..5f0d9c4c1 100644 --- a/cloud/services/compute/loadbalancers/reconcile.go +++ b/cloud/services/compute/loadbalancers/reconcile.go @@ -18,72 +18,235 @@ package loadbalancers import ( "context" + "errors" + "strings" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" "google.golang.org/api/compute/v1" - "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-gcp/cloud/gcperrors" "sigs.k8s.io/controller-runtime/pkg/log" ) -// Reconcile reconcile cluster control-plane loadbalancer compoenents. +// loadBalancingMode describes the load balancing mode that the backend performs. +type loadBalancingMode string + +const ( + // Utilization determines how the traffic load is spread based on the + // utilization of instances. + loadBalancingModeUtilization = loadBalancingMode("UTILIZATION") + + // Connection determines how the traffic load is spread based on the + // total number of connections that a backend can handle. This is + // only mode available for passthrough Load Balancers. + loadBalancingModeConnection = loadBalancingMode("CONNECTION") + + loadBalanceTrafficInternal = "INTERNAL" +) + +// Reconcile reconcile cluster control-plane loadbalancer components. func (s *Service) Reconcile(ctx context.Context) error { log := log.FromContext(ctx) log.Info("Reconciling loadbalancer resources") + + // Creates instance groups used by load balancer(s) instancegroups, err := s.createOrGetInstanceGroups(ctx) if err != nil { return err } - healthcheck, err := s.createOrGetHealthCheck(ctx) + lbSpec := s.scope.LoadBalancer() + lbType := ptr.Deref(lbSpec.LoadBalancerType, infrav1.ExternalLoadBalancerOnly) + // Create a Global External Proxy Load Balancer by default + if lbType == infrav1.ExternalLoadBalancerOnly || lbType == infrav1.DualLoadBalancer { + if err = s.createExternalLoadBalancer(ctx, lbType, instancegroups); err != nil { + return err + } + } + + // Create a Regional Internal Passthrough Load Balancer if configured + if lbType == infrav1.DualLoadBalancer || lbType == infrav1.InternalLoadBalancerOnly { + name := infrav1.InternalRoleTagValue + if lbSpec.InternalLoadBalancer != nil { + name = ptr.Deref(lbSpec.InternalLoadBalancer.Name, infrav1.InternalRoleTagValue) + } + if err = s.createInternalLoadBalancer(ctx, name, lbType, instancegroups); err != nil { + return err + } + } + + return nil +} + +// Delete deletes cluster control-plane loadbalancer components. +func (s *Service) Delete(ctx context.Context) error { + var externalLBErr, internalLBErr error + lbSpec := s.scope.LoadBalancer() + lbType := ptr.Deref(lbSpec.LoadBalancerType, infrav1.ExternalLoadBalancerOnly) + if lbType == infrav1.ExternalLoadBalancerOnly || lbType == infrav1.DualLoadBalancer { + externalLBErr = s.deleteExternalLoadBalancer(ctx) + } + + if lbType == infrav1.DualLoadBalancer || lbType == infrav1.InternalLoadBalancerOnly { + name := infrav1.InternalRoleTagValue + if lbSpec.InternalLoadBalancer != nil { + name = ptr.Deref(lbSpec.InternalLoadBalancer.Name, infrav1.InternalRoleTagValue) + } + internalLBErr = s.deleteInternalLoadBalancer(ctx, name) + } + if externalLBErr != nil { + s.deleteInstanceGroups(ctx) + return externalLBErr + } + if internalLBErr != nil { + s.deleteInstanceGroups(ctx) + return internalLBErr + } + + // Delete shared instancegroups last + return s.deleteInstanceGroups(ctx) +} + +func (s *Service) deleteExternalLoadBalancer(ctx context.Context) error { + log := log.FromContext(ctx) + log.Info("Deleting external loadbalancer resources") + name := infrav1.APIServerRoleTagValue + if err := s.deleteForwardingRule(ctx, name); err != nil { + return err + } + s.scope.Network().APIServerForwardingRule = nil + + if err := s.deleteAddress(ctx, name); err != nil { + return err + } + s.scope.Network().APIServerAddress = nil + + if err := s.deleteTargetTCPProxy(ctx); err != nil { + return err + } + s.scope.Network().APIServerTargetProxy = nil + + if err := s.deleteBackendService(ctx, name); err != nil { + return err + } + s.scope.Network().APIServerBackendService = nil + + if err := s.deleteHealthCheck(ctx, name); err != nil { + return err + } + s.scope.Network().APIServerHealthCheck = nil + + return nil +} + +func (s *Service) deleteInternalLoadBalancer(ctx context.Context, name string) error { + log := log.FromContext(ctx) + log.Info("Deleting internal loadbalancer resources") + if err := s.deleteRegionalForwardingRule(ctx, name); err != nil { + return err + } + s.scope.Network().APIInternalForwardingRule = nil + + if err := s.deleteInternalAddress(ctx, name); err != nil { + return err + } + s.scope.Network().APIInternalAddress = nil + + if err := s.deleteRegionalBackendService(ctx, name); err != nil { + return err + } + s.scope.Network().APIInternalBackendService = nil + + if err := s.deleteRegionalHealthCheck(ctx, name); err != nil { + return err + } + s.scope.Network().APIInternalHealthCheck = nil + + return nil +} + +// createExternalLoadBalancer creates the components for a Global External Proxy LoadBalancer. +func (s *Service) createExternalLoadBalancer(ctx context.Context, lbType infrav1.LoadBalancerType, instancegroups []*compute.InstanceGroup) error { + name := infrav1.APIServerRoleTagValue + healthcheck, err := s.createOrGetHealthCheck(ctx, name) if err != nil { return err } + s.scope.Network().APIServerHealthCheck = ptr.To[string](healthcheck.SelfLink) - backendsvc, err := s.createOrGetBackendService(ctx, instancegroups, healthcheck) + // If an Internal LoadBalancer is being created, the BalancingMode must match the Internal LB + mode := loadBalancingModeUtilization + if lbType == infrav1.DualLoadBalancer { + mode = loadBalancingModeConnection + } + backendsvc, err := s.createOrGetBackendService(ctx, name, mode, instancegroups, healthcheck) if err != nil { return err } + s.scope.Network().APIServerBackendService = ptr.To[string](backendsvc.SelfLink) + // Create TargetTCPProxy for Proxy Load Balancer target, err := s.createOrGetTargetTCPProxy(ctx, backendsvc) if err != nil { return err } + s.scope.Network().APIServerTargetProxy = ptr.To[string](target.SelfLink) - addr, err := s.createOrGetAddress(ctx) + addr, err := s.createOrGetAddress(ctx, name) if err != nil { return err } + s.scope.Network().APIServerAddress = ptr.To[string](addr.SelfLink) + endpoint := s.scope.ControlPlaneEndpoint() + endpoint.Host = addr.Address + s.scope.SetControlPlaneEndpoint(endpoint) - return s.createForwardingRule(ctx, target, addr) -} - -// Delete delete cluster control-plane loadbalancer compoenents. -func (s *Service) Delete(ctx context.Context) error { - log := log.FromContext(ctx) - log.Info("Deleting loadbalancer resources") - if err := s.deleteForwardingRule(ctx); err != nil { + forwarding, err := s.createOrGetForwardingRule(ctx, name, target, addr) + if err != nil { return err } + s.scope.Network().APIServerForwardingRule = ptr.To[string](forwarding.SelfLink) + + return nil +} - if err := s.deleteAddress(ctx); err != nil { +// createInternalLoadBalancer creates the components for a Regional Internal Passthrough LoadBalancer. +// Since this is a passthrough LoadBalancer the TargetTCPProxy resource is not created. +func (s *Service) createInternalLoadBalancer(ctx context.Context, name string, lbType infrav1.LoadBalancerType, instancegroups []*compute.InstanceGroup) error { + healthcheck, err := s.createOrGetRegionalHealthCheck(ctx, name) + if err != nil { return err } + s.scope.Network().APIInternalHealthCheck = ptr.To[string](healthcheck.SelfLink) - if err := s.deleteTargetTCPProxy(ctx); err != nil { + backendsvc, err := s.createOrGetRegionalBackendService(ctx, name, instancegroups, healthcheck) + if err != nil { return err } + s.scope.Network().APIInternalBackendService = ptr.To[string](backendsvc.SelfLink) - if err := s.deleteBackendService(ctx); err != nil { + // Create an address on internal subnet. + addr, err := s.createOrGetInternalAddress(ctx, name) + if err != nil { return err } + s.scope.Network().APIInternalAddress = ptr.To[string](addr.Address) + if lbType == infrav1.InternalLoadBalancerOnly { + // If only creating an internal Load Balancer, set the control plane endpoint + endpoint := s.scope.ControlPlaneEndpoint() + endpoint.Host = addr.Address + s.scope.SetControlPlaneEndpoint(endpoint) + } - if err := s.deleteHealthCheck(ctx); err != nil { + // Create a regional forwarding rule to the backend service + forwarding, err := s.createOrGetRegionalForwardingRule(ctx, name, backendsvc, addr) + if err != nil { return err } + s.scope.Network().APIInternalForwardingRule = ptr.To[string](forwarding.SelfLink) - return s.deleteInstanceGroups(ctx) + return nil } func (s *Service) createOrGetInstanceGroups(ctx context.Context) ([]*compute.InstanceGroup, error) { @@ -130,11 +293,12 @@ func (s *Service) createOrGetInstanceGroups(ctx context.Context) ([]*compute.Ins return groups, nil } -func (s *Service) createOrGetHealthCheck(ctx context.Context) (*compute.HealthCheck, error) { +func (s *Service) createOrGetHealthCheck(ctx context.Context, lbname string) (*compute.HealthCheck, error) { log := log.FromContext(ctx) - healthcheckSpec := s.scope.HealthCheckSpec() + healthcheckSpec := s.scope.HealthCheckSpec(lbname) log.V(2).Info("Looking for healthcheck", "name", healthcheckSpec.Name) - healthcheck, err := s.healthchecks.Get(ctx, meta.GlobalKey(healthcheckSpec.Name)) + key := meta.GlobalKey(healthcheckSpec.Name) + healthcheck, err := s.healthchecks.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for healthcheck", "name", healthcheckSpec.Name) @@ -142,35 +306,68 @@ func (s *Service) createOrGetHealthCheck(ctx context.Context) (*compute.HealthCh } log.V(2).Info("Creating a healthcheck", "name", healthcheckSpec.Name) - if err := s.healthchecks.Insert(ctx, meta.GlobalKey(healthcheckSpec.Name), healthcheckSpec); err != nil { + if err := s.healthchecks.Insert(ctx, key, healthcheckSpec); err != nil { log.Error(err, "Error creating a healthcheck", "name", healthcheckSpec.Name) return nil, err } - healthcheck, err = s.healthchecks.Get(ctx, meta.GlobalKey(healthcheckSpec.Name)) + healthcheck, err = s.healthchecks.Get(ctx, key) + if err != nil { + return nil, err + } + } + + return healthcheck, nil +} + +func (s *Service) createOrGetRegionalHealthCheck(ctx context.Context, lbname string) (*compute.HealthCheck, error) { + log := log.FromContext(ctx) + healthcheckSpec := s.scope.HealthCheckSpec(lbname) + healthcheckSpec.Region = s.scope.Region() + log.V(2).Info("Looking for regional healthcheck", "name", healthcheckSpec.Name) + key := meta.RegionalKey(healthcheckSpec.Name, s.scope.Region()) + healthcheck, err := s.regionalhealthchecks.Get(ctx, key) + if err != nil { + if !gcperrors.IsNotFound(err) { + log.Error(err, "Error looking for regional healthcheck", "name", healthcheckSpec.Name) + return nil, err + } + + log.V(2).Info("Creating a regional healthcheck", "name", healthcheckSpec.Name) + if err := s.regionalhealthchecks.Insert(ctx, key, healthcheckSpec); err != nil { + log.Error(err, "Error creating a regional healthcheck", "name", healthcheckSpec.Name) + return nil, err + } + + healthcheck, err = s.regionalhealthchecks.Get(ctx, key) if err != nil { return nil, err } } - s.scope.Network().APIServerHealthCheck = ptr.To[string](healthcheck.SelfLink) return healthcheck, nil } -func (s *Service) createOrGetBackendService(ctx context.Context, instancegroups []*compute.InstanceGroup, healthcheck *compute.HealthCheck) (*compute.BackendService, error) { +func (s *Service) createOrGetBackendService(ctx context.Context, lbname string, mode loadBalancingMode, instancegroups []*compute.InstanceGroup, healthcheck *compute.HealthCheck) (*compute.BackendService, error) { log := log.FromContext(ctx) backends := make([]*compute.Backend, 0, len(instancegroups)) for _, group := range instancegroups { - backends = append(backends, &compute.Backend{ - BalancingMode: "UTILIZATION", + be := &compute.Backend{ + BalancingMode: string(mode), Group: group.SelfLink, - }) + } + if mode == loadBalancingModeConnection { + be.MaxConnections = int64(2 ^ 32) + } + backends = append(backends, be) } - backendsvcSpec := s.scope.BackendServiceSpec() + backendsvcSpec := s.scope.BackendServiceSpec(lbname) backendsvcSpec.Backends = backends backendsvcSpec.HealthChecks = []string{healthcheck.SelfLink} - backendsvc, err := s.backendservices.Get(ctx, meta.GlobalKey(backendsvcSpec.Name)) + + key := meta.GlobalKey(backendsvcSpec.Name) + backendsvc, err := s.backendservices.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for backendservice", "name", backendsvcSpec.Name) @@ -178,12 +375,12 @@ func (s *Service) createOrGetBackendService(ctx context.Context, instancegroups } log.V(2).Info("Creating a backendservice", "name", backendsvcSpec.Name) - if err := s.backendservices.Insert(ctx, meta.GlobalKey(backendsvcSpec.Name), backendsvcSpec); err != nil { + if err := s.backendservices.Insert(ctx, key, backendsvcSpec); err != nil { log.Error(err, "Error creating a backendservice", "name", backendsvcSpec.Name) return nil, err } - backendsvc, err = s.backendservices.Get(ctx, meta.GlobalKey(backendsvcSpec.Name)) + backendsvc, err = s.backendservices.Get(ctx, key) if err != nil { return nil, err } @@ -192,13 +389,68 @@ func (s *Service) createOrGetBackendService(ctx context.Context, instancegroups if len(backendsvc.Backends) != len(backendsvcSpec.Backends) { log.V(2).Info("Updating a backendservice", "name", backendsvcSpec.Name) backendsvc.Backends = backendsvcSpec.Backends - if err := s.backendservices.Update(ctx, meta.GlobalKey(backendsvcSpec.Name), backendsvc); err != nil { + if err := s.backendservices.Update(ctx, key, backendsvc); err != nil { log.Error(err, "Error updating a backendservice", "name", backendsvcSpec.Name) return nil, err } } - s.scope.Network().APIServerBackendService = ptr.To[string](backendsvc.SelfLink) + return backendsvc, nil +} + +// createOrGetRegionalBackendService is used for internal passthrough load balancers. +func (s *Service) createOrGetRegionalBackendService(ctx context.Context, lbname string, instancegroups []*compute.InstanceGroup, healthcheck *compute.HealthCheck) (*compute.BackendService, error) { + log := log.FromContext(ctx) + backends := make([]*compute.Backend, 0, len(instancegroups)) + for _, group := range instancegroups { + be := &compute.Backend{ + // Always use connection mode for passthrough load balancer + BalancingMode: string(loadBalancingModeConnection), + Group: group.SelfLink, + } + backends = append(backends, be) + } + + backendsvcSpec := s.scope.BackendServiceSpec(lbname) + backendsvcSpec.Backends = backends + backendsvcSpec.HealthChecks = []string{healthcheck.SelfLink} + backendsvcSpec.Region = s.scope.Region() + backendsvcSpec.LoadBalancingScheme = string(loadBalanceTrafficInternal) + backendsvcSpec.PortName = "" + network := s.scope.Network() + if network.SelfLink != nil { + backendsvcSpec.Network = *network.SelfLink + } + + key := meta.RegionalKey(backendsvcSpec.Name, s.scope.Region()) + backendsvc, err := s.regionalbackendservices.Get(ctx, key) + if err != nil { + if !gcperrors.IsNotFound(err) { + log.Error(err, "Error looking for regional backendservice", "name", backendsvcSpec.Name) + return nil, err + } + + log.V(2).Info("Creating a regional backendservice", "name", backendsvcSpec.Name) + if err := s.regionalbackendservices.Insert(ctx, key, backendsvcSpec); err != nil { + log.Error(err, "Error creating a regional backendservice", "name", backendsvcSpec.Name) + return nil, err + } + + backendsvc, err = s.regionalbackendservices.Get(ctx, key) + if err != nil { + return nil, err + } + } + + if len(backendsvc.Backends) != len(backendsvcSpec.Backends) { + log.V(2).Info("Updating a regional backendservice", "name", backendsvcSpec.Name) + backendsvc.Backends = backendsvcSpec.Backends + if err := s.regionalbackendservices.Update(ctx, key, backendsvc); err != nil { + log.Error(err, "Error updating a regional backendservice", "name", backendsvcSpec.Name) + return nil, err + } + } + return backendsvc, nil } @@ -206,7 +458,8 @@ func (s *Service) createOrGetTargetTCPProxy(ctx context.Context, service *comput log := log.FromContext(ctx) targetSpec := s.scope.TargetTCPProxySpec() targetSpec.Service = service.SelfLink - target, err := s.targettcpproxies.Get(ctx, meta.GlobalKey(targetSpec.Name)) + key := meta.GlobalKey(targetSpec.Name) + target, err := s.targettcpproxies.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for targettcpproxy", "name", targetSpec.Name) @@ -214,26 +467,27 @@ func (s *Service) createOrGetTargetTCPProxy(ctx context.Context, service *comput } log.V(2).Info("Creating a targettcpproxy", "name", targetSpec.Name) - if err := s.targettcpproxies.Insert(ctx, meta.GlobalKey(targetSpec.Name), targetSpec); err != nil { + if err := s.targettcpproxies.Insert(ctx, key, targetSpec); err != nil { log.Error(err, "Error creating a targettcpproxy", "name", targetSpec.Name) return nil, err } - target, err = s.targettcpproxies.Get(ctx, meta.GlobalKey(targetSpec.Name)) + target, err = s.targettcpproxies.Get(ctx, key) if err != nil { return nil, err } } - s.scope.Network().APIServerTargetProxy = ptr.To[string](target.SelfLink) return target, nil } -func (s *Service) createOrGetAddress(ctx context.Context) (*compute.Address, error) { +// createOrGetAddress is used to obtain a Global address. +func (s *Service) createOrGetAddress(ctx context.Context, lbname string) (*compute.Address, error) { log := log.FromContext(ctx) - addrSpec := s.scope.AddressSpec() + addrSpec := s.scope.AddressSpec(lbname) log.V(2).Info("Looking for address", "name", addrSpec.Name) - addr, err := s.addresses.Get(ctx, meta.GlobalKey(addrSpec.Name)) + key := meta.GlobalKey(addrSpec.Name) + addr, err := s.addresses.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for address", "name", addrSpec.Name) @@ -241,57 +495,133 @@ func (s *Service) createOrGetAddress(ctx context.Context) (*compute.Address, err } log.V(2).Info("Creating an address", "name", addrSpec.Name) - if err := s.addresses.Insert(ctx, meta.GlobalKey(addrSpec.Name), addrSpec); err != nil { + if err := s.addresses.Insert(ctx, key, addrSpec); err != nil { log.Error(err, "Error creating an address", "name", addrSpec.Name) return nil, err } - addr, err = s.addresses.Get(ctx, meta.GlobalKey(addrSpec.Name)) + addr, err = s.addresses.Get(ctx, key) if err != nil { return nil, err } } - s.scope.Network().APIServerAddress = ptr.To[string](addr.SelfLink) - endpoint := s.scope.ControlPlaneEndpoint() - endpoint.Host = addr.Address - s.scope.SetControlPlaneEndpoint(endpoint) return addr, nil } -func (s *Service) createForwardingRule(ctx context.Context, target *compute.TargetTcpProxy, addr *compute.Address) error { +// createOrGetInternalAddress is used to obtain an internal address. +func (s *Service) createOrGetInternalAddress(ctx context.Context, lbname string) (*compute.Address, error) { log := log.FromContext(ctx) - spec := s.scope.ForwardingRuleSpec() - key := meta.GlobalKey(spec.Name) - spec.IPAddress = addr.SelfLink + addrSpec := s.scope.AddressSpec(lbname) + addrSpec.AddressType = string(loadBalanceTrafficInternal) + addrSpec.Region = s.scope.Region() + subnet, err := s.getSubnet(ctx) + if err != nil { + log.Error(err, "Error getting subnet for Internal Load Balancer") + return nil, err + } + addrSpec.Subnetwork = subnet.SelfLink + addrSpec.Purpose = "GCE_ENDPOINT" + log.V(2).Info("Looking for internal address", "name", addrSpec.Name) + key := meta.RegionalKey(addrSpec.Name, s.scope.Region()) + addr, err := s.internaladdresses.Get(ctx, key) + if err != nil { + if !gcperrors.IsNotFound(err) { + log.Error(err, "Error looking for internal address", "name", addrSpec.Name) + return nil, err + } + + log.V(2).Info("Creating an internal address", "name", addrSpec.Name) + if err := s.internaladdresses.Insert(ctx, key, addrSpec); err != nil { + log.Error(err, "Error creating an internal address", "name", addrSpec.Name) + return nil, err + } + + addr, err = s.internaladdresses.Get(ctx, key) + if err != nil { + return nil, err + } + } + + return addr, nil +} + +// createOrGetForwardingRule is used obtain a Global ForwardingRule. +func (s *Service) createOrGetForwardingRule(ctx context.Context, lbname string, target *compute.TargetTcpProxy, addr *compute.Address) (*compute.ForwardingRule, error) { + log := log.FromContext(ctx) + spec := s.scope.ForwardingRuleSpec(lbname) spec.Target = target.SelfLink + spec.IPAddress = addr.SelfLink + + key := meta.GlobalKey(spec.Name) log.V(2).Info("Looking for forwardingrule", "name", spec.Name) forwarding, err := s.forwardingrules.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for forwardingrule", "name", spec.Name) - return err + return nil, err } log.V(2).Info("Creating a forwardingrule", "name", spec.Name) if err := s.forwardingrules.Insert(ctx, key, spec); err != nil { log.Error(err, "Error creating a forwardingrule", "name", spec.Name) - return err + return nil, err } forwarding, err = s.forwardingrules.Get(ctx, key) if err != nil { - return err + return nil, err } } - s.scope.Network().APIServerForwardingRule = ptr.To[string](forwarding.SelfLink) - return nil + return forwarding, nil } -func (s *Service) deleteForwardingRule(ctx context.Context) error { +// createOrGetRegionalForwardingRule is used to obtain a Regional ForwardingRule. +func (s *Service) createOrGetRegionalForwardingRule(ctx context.Context, lbname string, backendSvc *compute.BackendService, addr *compute.Address) (*compute.ForwardingRule, error) { log := log.FromContext(ctx) - spec := s.scope.ForwardingRuleSpec() + spec := s.scope.ForwardingRuleSpec(lbname) + spec.LoadBalancingScheme = string(loadBalanceTrafficInternal) + spec.Region = s.scope.Region() + spec.BackendService = backendSvc.SelfLink + // Ports are used instead or PortRange for passthrough Load Balancer + spec.Ports = []string{"6443", "22623"} + spec.PortRange = "" + subnet, err := s.getSubnet(ctx) + if err != nil { + log.Error(err, "Error getting subnet for regional forwardingrule") + return nil, err + } + spec.Subnetwork = subnet.SelfLink + spec.IPAddress = addr.SelfLink + + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Looking for regional forwardingrule", "name", spec.Name) + forwarding, err := s.regionalforwardingrules.Get(ctx, key) + if err != nil { + if !gcperrors.IsNotFound(err) { + log.Error(err, "Error looking for regional forwardingrule", "name", spec.Name) + return nil, err + } + + log.V(2).Info("Creating a regional forwardingrule", "name", spec.Name) + if err := s.regionalforwardingrules.Insert(ctx, key, spec); err != nil { + log.Error(err, "Error creating a regional forwardingrule", "name", spec.Name) + return nil, err + } + + forwarding, err = s.regionalforwardingrules.Get(ctx, key) + if err != nil { + return nil, err + } + } + + return forwarding, nil +} + +func (s *Service) deleteForwardingRule(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.ForwardingRuleSpec(lbname) key := meta.GlobalKey(spec.Name) log.V(2).Info("Deleting a forwardingrule", "name", spec.Name) if err := s.forwardingrules.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { @@ -299,20 +629,43 @@ func (s *Service) deleteForwardingRule(ctx context.Context) error { return err } - s.scope.Network().APIServerForwardingRule = nil return nil } -func (s *Service) deleteAddress(ctx context.Context) error { +func (s *Service) deleteRegionalForwardingRule(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.ForwardingRuleSpec(lbname) + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Deleting a regional forwardingrule", "name", spec.Name) + if err := s.regionalforwardingrules.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { + log.Error(err, "Error updating a regional forwardingrule", "name", spec.Name) + return err + } + + return nil +} + +func (s *Service) deleteAddress(ctx context.Context, lbname string) error { log := log.FromContext(ctx) - spec := s.scope.AddressSpec() + spec := s.scope.AddressSpec(lbname) key := meta.GlobalKey(spec.Name) log.V(2).Info("Deleting a address", "name", spec.Name) if err := s.addresses.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { return err } - s.scope.Network().APIServerAddress = nil + return nil +} + +func (s *Service) deleteInternalAddress(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.AddressSpec(lbname) + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Deleting an internal address", "name", spec.Name) + if err := s.internaladdresses.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { + return err + } + return nil } @@ -326,13 +679,12 @@ func (s *Service) deleteTargetTCPProxy(ctx context.Context) error { return err } - s.scope.Network().APIServerTargetProxy = nil return nil } -func (s *Service) deleteBackendService(ctx context.Context) error { +func (s *Service) deleteBackendService(ctx context.Context, lbname string) error { log := log.FromContext(ctx) - spec := s.scope.BackendServiceSpec() + spec := s.scope.BackendServiceSpec(lbname) key := meta.GlobalKey(spec.Name) log.V(2).Info("Deleting a backendservice", "name", spec.Name) if err := s.backendservices.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { @@ -340,13 +692,25 @@ func (s *Service) deleteBackendService(ctx context.Context) error { return err } - s.scope.Network().APIServerBackendService = nil return nil } -func (s *Service) deleteHealthCheck(ctx context.Context) error { +func (s *Service) deleteRegionalBackendService(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.BackendServiceSpec(lbname) + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Deleting a regional backendservice", "name", spec.Name) + if err := s.regionalbackendservices.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { + log.Error(err, "Error deleting a regional backendservice", "name", spec.Name) + return err + } + + return nil +} + +func (s *Service) deleteHealthCheck(ctx context.Context, lbname string) error { log := log.FromContext(ctx) - spec := s.scope.HealthCheckSpec() + spec := s.scope.HealthCheckSpec(lbname) key := meta.GlobalKey(spec.Name) log.V(2).Info("Deleting a healthcheck", "name", spec.Name) if err := s.healthchecks.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { @@ -354,7 +718,19 @@ func (s *Service) deleteHealthCheck(ctx context.Context) error { return err } - s.scope.Network().APIServerHealthCheck = nil + return nil +} + +func (s *Service) deleteRegionalHealthCheck(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.HealthCheckSpec(lbname) + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Deleting a regional healthcheck", "name", spec.Name) + if err := s.regionalhealthchecks.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { + log.Error(err, "Error deleting a regional healthcheck", "name", spec.Name) + return err + } + return nil } @@ -376,3 +752,36 @@ func (s *Service) deleteInstanceGroups(ctx context.Context) error { return nil } + +// getSubnet gets the subnet to use for an internal Load Balancer. +func (s *Service) getSubnet(ctx context.Context) (*compute.Subnetwork, error) { + log := log.FromContext(ctx) + cfgSubnet := "" + lbSpec := s.scope.LoadBalancer() + if lbSpec.InternalLoadBalancer != nil { + cfgSubnet = ptr.Deref(lbSpec.InternalLoadBalancer.Subnet, "") + } + for _, subnetSpec := range s.scope.SubnetSpecs() { + log.V(2).Info("Looking for subnet for load balancer", "name", subnetSpec.Name) + region := subnetSpec.Region + if region == "" { + region = s.scope.Region() + } + + subnetKey := meta.RegionalKey(subnetSpec.Name, region) + subnet, err := s.subnets.Get(ctx, subnetKey) + if err != nil { + return nil, err + } + // Return subnet that matches configuration or first one if not configured + if cfgSubnet == "" || strings.HasSuffix(subnet.Name, cfgSubnet) { + return subnet, nil + } + } + + return nil, errors.New("could not find subnet") +} + +func isExternalLoadBalancer(name string) bool { + return strings.Contains(name, infrav1.APIServerRoleTagValue) +} diff --git a/cloud/services/compute/loadbalancers/reconcile_test.go b/cloud/services/compute/loadbalancers/reconcile_test.go index d837badb7..6518d9b68 100644 --- a/cloud/services/compute/loadbalancers/reconcile_test.go +++ b/cloud/services/compute/loadbalancers/reconcile_test.go @@ -28,12 +28,15 @@ import ( "google.golang.org/api/googleapi" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-gcp/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +var lbTypeInternal = infrav1.InternalLoadBalancerOnly + func init() { _ = clusterv1.AddToScheme(scheme.Scheme) _ = infrav1.AddToScheme(scheme.Scheme) @@ -60,6 +63,16 @@ func getBaseClusterScope() (*scope.ClusterScope, error) { Spec: infrav1.GCPClusterSpec{ Project: "my-proj", Region: "us-central1", + Network: infrav1.NetworkSpec{ + Subnets: infrav1.Subnets{ + infrav1.SubnetSpec{ + Name: "control-plane", + CidrBlock: "10.0.0.1/28", + Region: "us-central1", + Purpose: ptr.To[string]("INTERNAL_HTTPS_LOAD_BALANCER"), + }, + }, + }, }, Status: infrav1.GCPClusterStatus{ FailureDomains: clusterv1.FailureDomains{ @@ -106,9 +119,8 @@ func TestService_createOrGetInstanceGroup(t *testing.T) { { name: "instanceGroup name is overridden (should create instanceGroup)", scope: func(s *scope.ClusterScope) Scope { - tagOverride := "master" s.GCPCluster.Spec.LoadBalancer = infrav1.LoadBalancerSpec{ - APIServerInstanceGroupTagOverride: &tagOverride, + APIServerInstanceGroupTagOverride: ptr.To[string]("master"), } return s }, @@ -143,7 +155,6 @@ func TestService_createOrGetInstanceGroup(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx := context.TODO() - clusterScope, err := getBaseClusterScope() if err != nil { t.Fatal(err) @@ -155,10 +166,551 @@ func TestService_createOrGetInstanceGroup(t *testing.T) { t.Errorf("Service s.createOrGetInstanceGroups() error = %v, wantErr %v", err, tt.wantErr) return } - if d := cmp.Diff(tt.want, got); d != "" { t.Errorf("Service s.createOrGetInstanceGroups() mismatch (-want +got):\n%s", d) } }) } } + +func TestService_createOrGetHealthCheck(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + mockHealthChecks *cloud.MockHealthChecks + want *compute.HealthCheck + wantErr bool + }{ + { + name: "health check does not exist for external load balancer (should create healthcheck)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.APIServerRoleTagValue, + mockHealthChecks: &cloud.MockHealthChecks{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockHealthChecksObj{}, + }, + want: &compute.HealthCheck{ + CheckIntervalSec: 10, + HealthyThreshold: 5, + HttpsHealthCheck: &compute.HTTPSHealthCheck{Port: 6443, PortSpecification: "USE_FIXED_PORT", RequestPath: "/readyz"}, + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/healthChecks/my-cluster-apiserver", + TimeoutSec: 5, + Type: "HTTPS", + UnhealthyThreshold: 3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.healthchecks = tt.mockHealthChecks + got, err := s.createOrGetHealthCheck(ctx, tt.lbName) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetHealthChecks() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetHealthCheck() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetRegionalHealthCheck(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + mockHealthChecks *cloud.MockRegionHealthChecks + want *compute.HealthCheck + wantErr bool + }{ + { + name: "regional health check does not exist for internal load balancer (should create healthcheck)", + scope: func(s *scope.ClusterScope) Scope { + s.GCPCluster.Spec.LoadBalancer = infrav1.LoadBalancerSpec{ + LoadBalancerType: &lbTypeInternal, + } + return s + }, + lbName: infrav1.InternalRoleTagValue, + mockHealthChecks: &cloud.MockRegionHealthChecks{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockRegionHealthChecksObj{}, + }, + want: &compute.HealthCheck{ + CheckIntervalSec: 10, + HealthyThreshold: 5, + HttpsHealthCheck: &compute.HTTPSHealthCheck{Port: 6443, PortSpecification: "USE_FIXED_PORT", RequestPath: "/readyz"}, + Name: "my-cluster-api-internal", + Region: "us-central1", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/healthChecks/my-cluster-api-internal", + TimeoutSec: 5, + Type: "HTTPS", + UnhealthyThreshold: 3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.regionalhealthchecks = tt.mockHealthChecks + got, err := s.createOrGetRegionalHealthCheck(ctx, tt.lbName) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetRegionalHealthChecks() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrRegionalGetHealthCheck() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetBackendService(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + healthCheck *compute.HealthCheck + instanceGroups []*compute.InstanceGroup + mockBackendService *cloud.MockBackendServices + want *compute.BackendService + wantErr bool + }{ + { + name: "backend service does not exist for external load balancer (should create backendservice)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.APIServerRoleTagValue, + healthCheck: &compute.HealthCheck{ + HttpsHealthCheck: &compute.HTTPSHealthCheck{Port: 6443, PortSpecification: "USE_FIXED_PORT", RequestPath: "/readyz"}, + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/healthChecks/my-cluster-apiserver", + }, + instanceGroups: []*compute.InstanceGroup{ + { + Name: "my-cluster-master-us-central1-a", + NamedPorts: []*compute.NamedPort{{Name: "apiserver", Port: 6443}}, + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/zones/us-central1-a/instanceGroups/my-cluster-master-us-central1-a", + }, + }, + mockBackendService: &cloud.MockBackendServices{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockBackendServicesObj{}, + }, + want: &compute.BackendService{ + Backends: []*compute.Backend{ + { + BalancingMode: "UTILIZATION", + Group: "https://www.googleapis.com/compute/v1/projects/proj-id/zones/us-central1-a/instanceGroups/my-cluster-master-us-central1-a", + }, + }, + HealthChecks: []string{ + "https://www.googleapis.com/compute/v1/projects/proj-id/global/healthChecks/my-cluster-apiserver", + }, + LoadBalancingScheme: "EXTERNAL", + Name: "my-cluster-apiserver", + PortName: "apiserver", + Protocol: "TCP", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/backendServices/my-cluster-apiserver", + TimeoutSec: 600, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.backendservices = tt.mockBackendService + mode := loadBalancingModeUtilization + got, err := s.createOrGetBackendService(ctx, tt.lbName, mode, tt.instanceGroups, tt.healthCheck) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetBackendService() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetBackendService() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetRegionalBackendService(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + healthCheck *compute.HealthCheck + instanceGroups []*compute.InstanceGroup + mockBackendService *cloud.MockRegionBackendServices + want *compute.BackendService + wantErr bool + }{ + { + name: "regional backend service does not exist for internal load balancer (should create regional backendservice)", + scope: func(s *scope.ClusterScope) Scope { + s.GCPCluster.Spec.LoadBalancer = infrav1.LoadBalancerSpec{ + LoadBalancerType: &lbTypeInternal, + } + s.GCPCluster.Status.Network.SelfLink = ptr.To[string]("https://www.googleapis.com/compute/v1/projects/openshift-dev-installer/global/networks/bfournie-capg-test-5jp2d-network") + return s + }, + lbName: infrav1.InternalRoleTagValue, + healthCheck: &compute.HealthCheck{ + HttpsHealthCheck: &compute.HTTPSHealthCheck{Port: 6443, PortSpecification: "USE_FIXED_PORT", RequestPath: "/readyz"}, + Name: "my-cluster-api-internal", + Region: "us-central1", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/healthChecks/my-cluster-api-internal", + }, + instanceGroups: []*compute.InstanceGroup{ + { + Name: "my-cluster-apiserver-us-central1-a", + NamedPorts: []*compute.NamedPort{{Name: "apiserver", Port: 6443}}, + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/zones/us-central1-a/instanceGroups/my-cluster-master-us-central1-a", + }, + }, + mockBackendService: &cloud.MockRegionBackendServices{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockRegionBackendServicesObj{}, + }, + want: &compute.BackendService{ + Backends: []*compute.Backend{ + { + BalancingMode: "CONNECTION", + Group: "https://www.googleapis.com/compute/v1/projects/proj-id/zones/us-central1-a/instanceGroups/my-cluster-master-us-central1-a", + }, + }, + HealthChecks: []string{ + "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/healthChecks/my-cluster-api-internal", + }, + LoadBalancingScheme: "INTERNAL", + Name: "my-cluster-api-internal", + Network: "https://www.googleapis.com/compute/v1/projects/openshift-dev-installer/global/networks/bfournie-capg-test-5jp2d-network", + PortName: "", + Protocol: "TCP", + Region: "us-central1", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/backendServices/my-cluster-api-internal", + TimeoutSec: 600, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.regionalbackendservices = tt.mockBackendService + got, err := s.createOrGetRegionalBackendService(ctx, tt.lbName, tt.instanceGroups, tt.healthCheck) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetRegionalBackendService() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetRegionalBackendService() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetAddress(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + mockAddress *cloud.MockGlobalAddresses + want *compute.Address + wantErr bool + }{ + { + name: "address does not exist for external load balancer (should create address)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.APIServerRoleTagValue, + mockAddress: &cloud.MockGlobalAddresses{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockGlobalAddressesObj{}, + }, + want: &compute.Address{ + IpVersion: "IPV4", + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/addresses/my-cluster-apiserver", + AddressType: "EXTERNAL", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.addresses = tt.mockAddress + got, err := s.createOrGetAddress(ctx, tt.lbName) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetAddress() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetAddress() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetInternalAddress(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + mockAddress *cloud.MockAddresses + mockSubnetworks *cloud.MockSubnetworks + want *compute.Address + wantErr bool + }{ + { + name: "address does not exist for internal load balancer (should create address)", + scope: func(s *scope.ClusterScope) Scope { + lbType := infrav1.InternalLoadBalancerOnly + s.GCPCluster.Spec.LoadBalancer = infrav1.LoadBalancerSpec{ + LoadBalancerType: &lbType, + } + return s + }, + lbName: infrav1.InternalRoleTagValue, + mockAddress: &cloud.MockAddresses{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockAddressesObj{}, + }, + mockSubnetworks: &cloud.MockSubnetworks{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "my-proj"}, + Objects: map[meta.Key]*cloud.MockSubnetworksObj{ + *meta.RegionalKey("control-plane", "us-central1"): {}, + }, + }, + want: &compute.Address{ + IpVersion: "IPV4", + Name: "my-cluster-api-internal", + Region: "us-central1", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-api-internal", + AddressType: "INTERNAL", + Purpose: "GCE_ENDPOINT", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.internaladdresses = tt.mockAddress + s.subnets = tt.mockSubnetworks + got, err := s.createOrGetInternalAddress(ctx, tt.lbName) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetInternalAddress() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetInternalAddress() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetTargetTCPProxy(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + backendService *compute.BackendService + mockTargetTCPProxy *cloud.MockTargetTcpProxies + want *compute.TargetTcpProxy + wantErr bool + }{ + { + name: "target tcp proxy does not exist for external load balancer (should create target tp proxy)", + scope: func(s *scope.ClusterScope) Scope { return s }, + backendService: &compute.BackendService{ + Name: "my-cluster-api-internal", + }, + mockTargetTCPProxy: &cloud.MockTargetTcpProxies{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockTargetTcpProxiesObj{}, + }, + want: &compute.TargetTcpProxy{ + Name: "my-cluster-apiserver", + ProxyHeader: "NONE", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/targetTcpProxies/my-cluster-apiserver", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.targettcpproxies = tt.mockTargetTCPProxy + got, err := s.createOrGetTargetTCPProxy(ctx, tt.backendService) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetTargetTCPProxy() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetTargetTCPProxy() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetForwardingRule(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + backendService *compute.BackendService + targetTcpproxy *compute.TargetTcpProxy + address *compute.Address + mockForwardingRule *cloud.MockGlobalForwardingRules + want *compute.ForwardingRule + wantErr bool + }{ + { + name: "forwarding rule does not exist for external load balancer (should create forwardingrule)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.APIServerRoleTagValue, + address: &compute.Address{ + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-apiserver", + }, + backendService: &compute.BackendService{}, + targetTcpproxy: &compute.TargetTcpProxy{ + Name: "my-cluster-apiserver", + }, + mockForwardingRule: &cloud.MockGlobalForwardingRules{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockGlobalForwardingRulesObj{}, + }, + want: &compute.ForwardingRule{ + IPAddress: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-apiserver", + IPProtocol: "TCP", + LoadBalancingScheme: "EXTERNAL", + PortRange: "443-443", + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/forwardingRules/my-cluster-apiserver", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.forwardingrules = tt.mockForwardingRule + var fwdRule *compute.ForwardingRule + fwdRule, err = s.createOrGetForwardingRule(ctx, tt.lbName, tt.targetTcpproxy, tt.address) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetForwardingRule() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, fwdRule); d != "" { + t.Errorf("Service s.createOrGetForwardingRule() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetRegionalForwardingRule(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + backendService *compute.BackendService + targetTcpproxy *compute.TargetTcpProxy + address *compute.Address + mockSubnetworks *cloud.MockSubnetworks + mockForwardingRule *cloud.MockForwardingRules + want *compute.ForwardingRule + wantErr bool + }{ + { + name: "regional forwarding rule does not exist for internal load balancer (should create forwardingrule)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.InternalRoleTagValue, + address: &compute.Address{ + Name: "my-cluster-api-internal", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-api-internal", + }, + backendService: &compute.BackendService{ + Name: "my-cluster-api-internal", + }, + targetTcpproxy: &compute.TargetTcpProxy{}, + mockSubnetworks: &cloud.MockSubnetworks{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "my-proj"}, + Objects: map[meta.Key]*cloud.MockSubnetworksObj{ + *meta.RegionalKey("control-plane", "us-central1"): {}, + }, + }, + mockForwardingRule: &cloud.MockForwardingRules{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockForwardingRulesObj{}, + }, + want: &compute.ForwardingRule{ + IPAddress: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-api-internal", + IPProtocol: "TCP", + LoadBalancingScheme: "INTERNAL", + Ports: []string{"6443", "22623"}, + Region: "us-central1", + Name: "my-cluster-api-internal", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/forwardingRules/my-cluster-api-internal", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.regionalforwardingrules = tt.mockForwardingRule + var fwdRule *compute.ForwardingRule + s.subnets = tt.mockSubnetworks + fwdRule, err = s.createOrGetRegionalForwardingRule(ctx, tt.lbName, tt.backendService, tt.address) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetRegionalForwardingRule() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, fwdRule); d != "" { + t.Errorf("Service s.createOrGetRegionalForwardingRule() mismatch (-want +got):\n%s", d) + } + }) + } +} diff --git a/cloud/services/compute/loadbalancers/service.go b/cloud/services/compute/loadbalancers/service.go index 8a1067054..276fd2515 100644 --- a/cloud/services/compute/loadbalancers/service.go +++ b/cloud/services/compute/loadbalancers/service.go @@ -65,26 +65,36 @@ type targettcpproxiesInterface interface { Delete(ctx context.Context, key *meta.Key, options ...k8scloud.Option) error } +type subnetsInterface interface { + Get(ctx context.Context, key *meta.Key, options ...k8scloud.Option) (*compute.Subnetwork, error) +} + // Scope is an interfaces that hold used methods. type Scope interface { cloud.Cluster - AddressSpec() *compute.Address - BackendServiceSpec() *compute.BackendService - ForwardingRuleSpec() *compute.ForwardingRule - HealthCheckSpec() *compute.HealthCheck + AddressSpec(name string) *compute.Address + BackendServiceSpec(name string) *compute.BackendService + ForwardingRuleSpec(name string) *compute.ForwardingRule + HealthCheckSpec(name string) *compute.HealthCheck InstanceGroupSpec(zone string) *compute.InstanceGroup TargetTCPProxySpec() *compute.TargetTcpProxy + SubnetSpecs() []*compute.Subnetwork } // Service implements loadbalancers reconciler. type Service struct { - scope Scope - addresses addressesInterface - backendservices backendservicesInterface - forwardingrules forwardingrulesInterface - healthchecks healthchecksInterface - instancegroups instancegroupsInterface - targettcpproxies targettcpproxiesInterface + scope Scope + addresses addressesInterface + internaladdresses addressesInterface + backendservices backendservicesInterface + regionalbackendservices backendservicesInterface + forwardingrules forwardingrulesInterface + regionalforwardingrules forwardingrulesInterface + healthchecks healthchecksInterface + regionalhealthchecks healthchecksInterface + instancegroups instancegroupsInterface + targettcpproxies targettcpproxiesInterface + subnets subnetsInterface } var _ cloud.Reconciler = &Service{} @@ -92,12 +102,17 @@ var _ cloud.Reconciler = &Service{} // New returns Service from given scope. func New(scope Scope) *Service { return &Service{ - scope: scope, - addresses: scope.Cloud().GlobalAddresses(), - backendservices: scope.Cloud().BackendServices(), - forwardingrules: scope.Cloud().GlobalForwardingRules(), - healthchecks: scope.Cloud().HealthChecks(), - instancegroups: scope.Cloud().InstanceGroups(), - targettcpproxies: scope.Cloud().TargetTcpProxies(), + scope: scope, + addresses: scope.Cloud().GlobalAddresses(), + internaladdresses: scope.Cloud().Addresses(), + backendservices: scope.Cloud().BackendServices(), + regionalbackendservices: scope.Cloud().RegionBackendServices(), + forwardingrules: scope.Cloud().GlobalForwardingRules(), + regionalforwardingrules: scope.Cloud().ForwardingRules(), + healthchecks: scope.Cloud().HealthChecks(), + regionalhealthchecks: scope.Cloud().RegionHealthChecks(), + instancegroups: scope.Cloud().InstanceGroups(), + targettcpproxies: scope.Cloud().TargetTcpProxies(), + subnets: scope.Cloud().Subnetworks(), } } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml index 10fe9b1dc..bcdd9d22e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml @@ -118,6 +118,28 @@ spec: maxLength: 16 pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) type: string + internalLoadBalancer: + description: InternalLoadBalancer is the configuration for an + Internal Passthrough Network Load Balancer. + properties: + name: + description: |- + Name is the name of the Load Balancer. If not set a default name + will be used. For an Internal Load Balancer service the default + name is "api-internal". + maxLength: 32 + pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) + type: string + subnet: + description: Subnet is the name of the subnet to use for a + regional Load Balancer. + type: string + type: object + loadBalancerType: + description: |- + LoadBalancerType defines the type of Load Balancer that should be created. + If not set, a Global External Proxy Load Balancer will be created by default. + type: string type: object network: description: NetworkSpec encapsulates all things related to GCP network. @@ -298,6 +320,26 @@ spec: network: description: Network encapsulates GCP networking resources. properties: + apiInternalBackendService: + description: |- + APIInternalBackendService is the full reference to the backend service + created for the internal Load Balancer. + type: string + apiInternalForwardingRule: + description: |- + APIInternalForwardingRule is the full reference to the forwarding rule + created for the internal Load Balancer. + type: string + apiInternalHealthCheck: + description: |- + APIInternalHealthCheck is the full reference to the health check + created for the internal Load Balancer. + type: string + apiInternalIpAddress: + description: |- + APIInternalAddress is the IPV4 regional address assigned to the + internal Load Balancer. + type: string apiServerBackendService: description: |- APIServerBackendService is the full reference to the backend service diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml index 05179c5c3..ed1f3786f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml @@ -134,6 +134,28 @@ spec: maxLength: 16 pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) type: string + internalLoadBalancer: + description: InternalLoadBalancer is the configuration + for an Internal Passthrough Network Load Balancer. + properties: + name: + description: |- + Name is the name of the Load Balancer. If not set a default name + will be used. For an Internal Load Balancer service the default + name is "api-internal". + maxLength: 32 + pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) + type: string + subnet: + description: Subnet is the name of the subnet to use + for a regional Load Balancer. + type: string + type: object + loadBalancerType: + description: |- + LoadBalancerType defines the type of Load Balancer that should be created. + If not set, a Global External Proxy Load Balancer will be created by default. + type: string type: object network: description: NetworkSpec encapsulates all things related to diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml index 7faef1da9..ebaa5f5ae 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml @@ -113,6 +113,28 @@ spec: maxLength: 16 pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) type: string + internalLoadBalancer: + description: InternalLoadBalancer is the configuration for an + Internal Passthrough Network Load Balancer. + properties: + name: + description: |- + Name is the name of the Load Balancer. If not set a default name + will be used. For an Internal Load Balancer service the default + name is "api-internal". + maxLength: 32 + pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) + type: string + subnet: + description: Subnet is the name of the subnet to use for a + regional Load Balancer. + type: string + type: object + loadBalancerType: + description: |- + LoadBalancerType defines the type of Load Balancer that should be created. + If not set, a Global External Proxy Load Balancer will be created by default. + type: string type: object network: description: NetworkSpec encapsulates all things related to the GCP @@ -340,6 +362,26 @@ spec: network: description: Network encapsulates GCP networking resources. properties: + apiInternalBackendService: + description: |- + APIInternalBackendService is the full reference to the backend service + created for the internal Load Balancer. + type: string + apiInternalForwardingRule: + description: |- + APIInternalForwardingRule is the full reference to the forwarding rule + created for the internal Load Balancer. + type: string + apiInternalHealthCheck: + description: |- + APIInternalHealthCheck is the full reference to the health check + created for the internal Load Balancer. + type: string + apiInternalIpAddress: + description: |- + APIInternalAddress is the IPV4 regional address assigned to the + internal Load Balancer. + type: string apiServerBackendService: description: |- APIServerBackendService is the full reference to the backend service diff --git a/controllers/gcpcluster_controller.go b/controllers/gcpcluster_controller.go index 58dcd5f01..f5eddfe1b 100644 --- a/controllers/gcpcluster_controller.go +++ b/controllers/gcpcluster_controller.go @@ -200,8 +200,9 @@ func (r *GCPClusterReconciler) reconcile(ctx context.Context, clusterScope *scop reconcilers := []cloud.Reconciler{ networks.New(clusterScope), firewalls.New(clusterScope), - loadbalancers.New(clusterScope), + // Reconcile subnets before loadbalancers since subnet is needed for internal LB subnets.New(clusterScope), + loadbalancers.New(clusterScope), } for _, r := range reconcilers {