diff --git a/api/v1beta1/labels.go b/api/v1beta1/labels.go index 264f70f99..eded27abf 100644 --- a/api/v1beta1/labels.go +++ b/api/v1beta1/labels.go @@ -111,6 +111,9 @@ const ( // APIServerRoleTagValue describes the value for the apiserver role. APIServerRoleTagValue = "apiserver" + + // InternalRoleTagValue describes the value for the internal role. + InternalRoleTagValue = "api-internal" ) // ClusterTagKey generates the key for resources associated with a cluster. diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 763916cb2..7a760368b 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -85,6 +85,26 @@ type Network struct { // created for the API Server. // +optional APIServerForwardingRule *string `json:"apiServerForwardingRule,omitempty"` + + // APIInternalAddress is the IPV4 regional address assigned to the + // internal Load Balancer. + // +optional + APIInternalAddress *string `json:"apiInternalIpAddress,omitempty"` + + // APIInternalHealthCheck is the full reference to the health check + // created for the internal Load Balancer. + // +optional + APIInternalHealthCheck *string `json:"apiInternalHealthCheck,omitempty"` + + // APIInternalBackendService is the full reference to the backend service + // created for the internal Load Balancer. + // +optional + APIInternalBackendService *string `json:"apiInternalBackendService,omitempty"` + + // APIInternalForwardingRule is the full reference to the forwarding rule + // created for the internal Load Balancer. + // +optional + APIInternalForwardingRule *string `json:"apiInternalForwardingRule,omitempty"` } // NetworkSpec encapsulates all things related to a GCP network. @@ -114,6 +134,24 @@ type NetworkSpec struct { LoadBalancerBackendPort *int32 `json:"loadBalancerBackendPort,omitempty"` } +// LoadBalancerType defines the Load Balancer that should be created. +type LoadBalancerType string + +var ( + // ExternalLoadBalancerOnly creates a Global External Proxy Load Balancer + // to manage traffic to backends in multiple regions. This is the default Load + // Balancer and will be created if no LoadBalancerType is defined. + ExternalLoadBalancerOnly = LoadBalancerType("EXTERNAL_ONLY") + + // InternalLoadBalancerOnly creates a Regional Internal Passthrough Load + // Balancer to manage traffic to backends in the configured region. + InternalLoadBalancerOnly = LoadBalancerType("INTERNAL_ONLY") + + // DualLoadBalancer creates both External and Internal Load Balancers to provide + // separate endpoints for managing both external and internal traffic. + DualLoadBalancer = LoadBalancerType("Dual") +) + // LoadBalancerSpec contains configuration for one or more LoadBalancers. type LoadBalancerSpec struct { // APIServerInstanceGroupTagOverride overrides the default setting for the @@ -123,6 +161,15 @@ type LoadBalancerSpec struct { // +kubebuilder:validation:Pattern=`(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)` // +optional APIServerInstanceGroupTagOverride *string `json:"apiServerInstanceGroupTagOverride,omitempty"` + + // LoadBalancerType defines the type of Load Balancer that should be created. + // If not set, a Global External Proxy Load Balancer will be created by default. + // +optional + LoadBalancerType *LoadBalancerType `json:"loadBalancerType,omitempty"` + + // InternalLoadBalancer is the configuration for an Internal Passthrough Network Load Balancer. + // +optional + InternalLoadBalancer *LoadBalancer `json:"internalLoadBalancer,omitempty"` } // SubnetSpec configures an GCP Subnet. @@ -278,3 +325,18 @@ type ObjectReference struct { // +kubebuilder:validation:Required Name string `json:"name"` } + +// LoadBalancer specifies the configuration of a LoadBalancer. +type LoadBalancer struct { + // Name is the name of the Load Balancer. If not set a default name + // will be used. For an Internal Load Balancer service the default + // name is "api-internal". + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:Pattern=`(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)` + // +optional + Name *string `json:"name,omitempty"` + + // Subnet is the name of the subnet to use for a regional Load Balancer. + Subnet *string `json:"subnet,omitempty"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index d3e20f307..c05c5bfab 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -674,6 +674,31 @@ func (in Labels) DeepCopy() Labels { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) { + *out = *in + if in.Name != nil { + in, out := &in.Name, &out.Name + *out = new(string) + **out = **in + } + if in.Subnet != nil { + in, out := &in.Subnet, &out.Subnet + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer. +func (in *LoadBalancer) DeepCopy() *LoadBalancer { + if in == nil { + return nil + } + out := new(LoadBalancer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = *in @@ -682,6 +707,16 @@ func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = new(string) **out = **in } + if in.LoadBalancerType != nil { + in, out := &in.LoadBalancerType, &out.LoadBalancerType + *out = new(LoadBalancerType) + **out = **in + } + if in.InternalLoadBalancer != nil { + in, out := &in.InternalLoadBalancer, &out.InternalLoadBalancer + *out = new(LoadBalancer) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerSpec. @@ -781,6 +816,26 @@ func (in *Network) DeepCopyInto(out *Network) { *out = new(string) **out = **in } + if in.APIInternalAddress != nil { + in, out := &in.APIInternalAddress, &out.APIInternalAddress + *out = new(string) + **out = **in + } + if in.APIInternalHealthCheck != nil { + in, out := &in.APIInternalHealthCheck, &out.APIInternalHealthCheck + *out = new(string) + **out = **in + } + if in.APIInternalBackendService != nil { + in, out := &in.APIInternalBackendService, &out.APIInternalBackendService + *out = new(string) + **out = **in + } + if in.APIInternalForwardingRule != nil { + in, out := &in.APIInternalForwardingRule, &out.APIInternalForwardingRule + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 6f7cb0a98..b32cac9f1 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -290,18 +290,18 @@ func (s *ClusterScope) FirewallRulesSpec() []*compute.Firewall { // ANCHOR: ClusterControlPlaneSpec // AddressSpec returns google compute address spec. -func (s *ClusterScope) AddressSpec() *compute.Address { +func (s *ClusterScope) AddressSpec(lbname string) *compute.Address { return &compute.Address{ - Name: fmt.Sprintf("%s-%s", s.Name(), infrav1.APIServerRoleTagValue), + Name: fmt.Sprintf("%s-%s", s.Name(), lbname), AddressType: "EXTERNAL", IpVersion: "IPV4", } } // BackendServiceSpec returns google compute backend-service spec. -func (s *ClusterScope) BackendServiceSpec() *compute.BackendService { +func (s *ClusterScope) BackendServiceSpec(lbname string) *compute.BackendService { return &compute.BackendService{ - Name: fmt.Sprintf("%s-%s", s.Name(), infrav1.APIServerRoleTagValue), + Name: fmt.Sprintf("%s-%s", s.Name(), lbname), LoadBalancingScheme: "EXTERNAL", PortName: "apiserver", Protocol: "TCP", @@ -310,14 +310,14 @@ func (s *ClusterScope) BackendServiceSpec() *compute.BackendService { } // ForwardingRuleSpec returns google compute forwarding-rule spec. -func (s *ClusterScope) ForwardingRuleSpec() *compute.ForwardingRule { +func (s *ClusterScope) ForwardingRuleSpec(lbname string) *compute.ForwardingRule { port := int32(443) if c := s.Cluster.Spec.ClusterNetwork; c != nil { port = ptr.Deref(c.APIServerPort, 443) } portRange := fmt.Sprintf("%d-%d", port, port) return &compute.ForwardingRule{ - Name: fmt.Sprintf("%s-%s", s.Name(), infrav1.APIServerRoleTagValue), + Name: fmt.Sprintf("%s-%s", s.Name(), lbname), IPProtocol: "TCP", LoadBalancingScheme: "EXTERNAL", PortRange: portRange, @@ -325,9 +325,9 @@ func (s *ClusterScope) ForwardingRuleSpec() *compute.ForwardingRule { } // HealthCheckSpec returns google compute health-check spec. -func (s *ClusterScope) HealthCheckSpec() *compute.HealthCheck { +func (s *ClusterScope) HealthCheckSpec(lbname string) *compute.HealthCheck { return &compute.HealthCheck{ - Name: fmt.Sprintf("%s-%s", s.Name(), infrav1.APIServerRoleTagValue), + Name: fmt.Sprintf("%s-%s", s.Name(), lbname), Type: "HTTPS", HttpsHealthCheck: &compute.HTTPSHealthCheck{ Port: 6443, diff --git a/cloud/services/compute/loadbalancers/reconcile.go b/cloud/services/compute/loadbalancers/reconcile.go index e6b723ec0..5f0d9c4c1 100644 --- a/cloud/services/compute/loadbalancers/reconcile.go +++ b/cloud/services/compute/loadbalancers/reconcile.go @@ -18,72 +18,235 @@ package loadbalancers import ( "context" + "errors" + "strings" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" "google.golang.org/api/compute/v1" - "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-gcp/cloud/gcperrors" "sigs.k8s.io/controller-runtime/pkg/log" ) -// Reconcile reconcile cluster control-plane loadbalancer compoenents. +// loadBalancingMode describes the load balancing mode that the backend performs. +type loadBalancingMode string + +const ( + // Utilization determines how the traffic load is spread based on the + // utilization of instances. + loadBalancingModeUtilization = loadBalancingMode("UTILIZATION") + + // Connection determines how the traffic load is spread based on the + // total number of connections that a backend can handle. This is + // only mode available for passthrough Load Balancers. + loadBalancingModeConnection = loadBalancingMode("CONNECTION") + + loadBalanceTrafficInternal = "INTERNAL" +) + +// Reconcile reconcile cluster control-plane loadbalancer components. func (s *Service) Reconcile(ctx context.Context) error { log := log.FromContext(ctx) log.Info("Reconciling loadbalancer resources") + + // Creates instance groups used by load balancer(s) instancegroups, err := s.createOrGetInstanceGroups(ctx) if err != nil { return err } - healthcheck, err := s.createOrGetHealthCheck(ctx) + lbSpec := s.scope.LoadBalancer() + lbType := ptr.Deref(lbSpec.LoadBalancerType, infrav1.ExternalLoadBalancerOnly) + // Create a Global External Proxy Load Balancer by default + if lbType == infrav1.ExternalLoadBalancerOnly || lbType == infrav1.DualLoadBalancer { + if err = s.createExternalLoadBalancer(ctx, lbType, instancegroups); err != nil { + return err + } + } + + // Create a Regional Internal Passthrough Load Balancer if configured + if lbType == infrav1.DualLoadBalancer || lbType == infrav1.InternalLoadBalancerOnly { + name := infrav1.InternalRoleTagValue + if lbSpec.InternalLoadBalancer != nil { + name = ptr.Deref(lbSpec.InternalLoadBalancer.Name, infrav1.InternalRoleTagValue) + } + if err = s.createInternalLoadBalancer(ctx, name, lbType, instancegroups); err != nil { + return err + } + } + + return nil +} + +// Delete deletes cluster control-plane loadbalancer components. +func (s *Service) Delete(ctx context.Context) error { + var externalLBErr, internalLBErr error + lbSpec := s.scope.LoadBalancer() + lbType := ptr.Deref(lbSpec.LoadBalancerType, infrav1.ExternalLoadBalancerOnly) + if lbType == infrav1.ExternalLoadBalancerOnly || lbType == infrav1.DualLoadBalancer { + externalLBErr = s.deleteExternalLoadBalancer(ctx) + } + + if lbType == infrav1.DualLoadBalancer || lbType == infrav1.InternalLoadBalancerOnly { + name := infrav1.InternalRoleTagValue + if lbSpec.InternalLoadBalancer != nil { + name = ptr.Deref(lbSpec.InternalLoadBalancer.Name, infrav1.InternalRoleTagValue) + } + internalLBErr = s.deleteInternalLoadBalancer(ctx, name) + } + if externalLBErr != nil { + s.deleteInstanceGroups(ctx) + return externalLBErr + } + if internalLBErr != nil { + s.deleteInstanceGroups(ctx) + return internalLBErr + } + + // Delete shared instancegroups last + return s.deleteInstanceGroups(ctx) +} + +func (s *Service) deleteExternalLoadBalancer(ctx context.Context) error { + log := log.FromContext(ctx) + log.Info("Deleting external loadbalancer resources") + name := infrav1.APIServerRoleTagValue + if err := s.deleteForwardingRule(ctx, name); err != nil { + return err + } + s.scope.Network().APIServerForwardingRule = nil + + if err := s.deleteAddress(ctx, name); err != nil { + return err + } + s.scope.Network().APIServerAddress = nil + + if err := s.deleteTargetTCPProxy(ctx); err != nil { + return err + } + s.scope.Network().APIServerTargetProxy = nil + + if err := s.deleteBackendService(ctx, name); err != nil { + return err + } + s.scope.Network().APIServerBackendService = nil + + if err := s.deleteHealthCheck(ctx, name); err != nil { + return err + } + s.scope.Network().APIServerHealthCheck = nil + + return nil +} + +func (s *Service) deleteInternalLoadBalancer(ctx context.Context, name string) error { + log := log.FromContext(ctx) + log.Info("Deleting internal loadbalancer resources") + if err := s.deleteRegionalForwardingRule(ctx, name); err != nil { + return err + } + s.scope.Network().APIInternalForwardingRule = nil + + if err := s.deleteInternalAddress(ctx, name); err != nil { + return err + } + s.scope.Network().APIInternalAddress = nil + + if err := s.deleteRegionalBackendService(ctx, name); err != nil { + return err + } + s.scope.Network().APIInternalBackendService = nil + + if err := s.deleteRegionalHealthCheck(ctx, name); err != nil { + return err + } + s.scope.Network().APIInternalHealthCheck = nil + + return nil +} + +// createExternalLoadBalancer creates the components for a Global External Proxy LoadBalancer. +func (s *Service) createExternalLoadBalancer(ctx context.Context, lbType infrav1.LoadBalancerType, instancegroups []*compute.InstanceGroup) error { + name := infrav1.APIServerRoleTagValue + healthcheck, err := s.createOrGetHealthCheck(ctx, name) if err != nil { return err } + s.scope.Network().APIServerHealthCheck = ptr.To[string](healthcheck.SelfLink) - backendsvc, err := s.createOrGetBackendService(ctx, instancegroups, healthcheck) + // If an Internal LoadBalancer is being created, the BalancingMode must match the Internal LB + mode := loadBalancingModeUtilization + if lbType == infrav1.DualLoadBalancer { + mode = loadBalancingModeConnection + } + backendsvc, err := s.createOrGetBackendService(ctx, name, mode, instancegroups, healthcheck) if err != nil { return err } + s.scope.Network().APIServerBackendService = ptr.To[string](backendsvc.SelfLink) + // Create TargetTCPProxy for Proxy Load Balancer target, err := s.createOrGetTargetTCPProxy(ctx, backendsvc) if err != nil { return err } + s.scope.Network().APIServerTargetProxy = ptr.To[string](target.SelfLink) - addr, err := s.createOrGetAddress(ctx) + addr, err := s.createOrGetAddress(ctx, name) if err != nil { return err } + s.scope.Network().APIServerAddress = ptr.To[string](addr.SelfLink) + endpoint := s.scope.ControlPlaneEndpoint() + endpoint.Host = addr.Address + s.scope.SetControlPlaneEndpoint(endpoint) - return s.createForwardingRule(ctx, target, addr) -} - -// Delete delete cluster control-plane loadbalancer compoenents. -func (s *Service) Delete(ctx context.Context) error { - log := log.FromContext(ctx) - log.Info("Deleting loadbalancer resources") - if err := s.deleteForwardingRule(ctx); err != nil { + forwarding, err := s.createOrGetForwardingRule(ctx, name, target, addr) + if err != nil { return err } + s.scope.Network().APIServerForwardingRule = ptr.To[string](forwarding.SelfLink) + + return nil +} - if err := s.deleteAddress(ctx); err != nil { +// createInternalLoadBalancer creates the components for a Regional Internal Passthrough LoadBalancer. +// Since this is a passthrough LoadBalancer the TargetTCPProxy resource is not created. +func (s *Service) createInternalLoadBalancer(ctx context.Context, name string, lbType infrav1.LoadBalancerType, instancegroups []*compute.InstanceGroup) error { + healthcheck, err := s.createOrGetRegionalHealthCheck(ctx, name) + if err != nil { return err } + s.scope.Network().APIInternalHealthCheck = ptr.To[string](healthcheck.SelfLink) - if err := s.deleteTargetTCPProxy(ctx); err != nil { + backendsvc, err := s.createOrGetRegionalBackendService(ctx, name, instancegroups, healthcheck) + if err != nil { return err } + s.scope.Network().APIInternalBackendService = ptr.To[string](backendsvc.SelfLink) - if err := s.deleteBackendService(ctx); err != nil { + // Create an address on internal subnet. + addr, err := s.createOrGetInternalAddress(ctx, name) + if err != nil { return err } + s.scope.Network().APIInternalAddress = ptr.To[string](addr.Address) + if lbType == infrav1.InternalLoadBalancerOnly { + // If only creating an internal Load Balancer, set the control plane endpoint + endpoint := s.scope.ControlPlaneEndpoint() + endpoint.Host = addr.Address + s.scope.SetControlPlaneEndpoint(endpoint) + } - if err := s.deleteHealthCheck(ctx); err != nil { + // Create a regional forwarding rule to the backend service + forwarding, err := s.createOrGetRegionalForwardingRule(ctx, name, backendsvc, addr) + if err != nil { return err } + s.scope.Network().APIInternalForwardingRule = ptr.To[string](forwarding.SelfLink) - return s.deleteInstanceGroups(ctx) + return nil } func (s *Service) createOrGetInstanceGroups(ctx context.Context) ([]*compute.InstanceGroup, error) { @@ -130,11 +293,12 @@ func (s *Service) createOrGetInstanceGroups(ctx context.Context) ([]*compute.Ins return groups, nil } -func (s *Service) createOrGetHealthCheck(ctx context.Context) (*compute.HealthCheck, error) { +func (s *Service) createOrGetHealthCheck(ctx context.Context, lbname string) (*compute.HealthCheck, error) { log := log.FromContext(ctx) - healthcheckSpec := s.scope.HealthCheckSpec() + healthcheckSpec := s.scope.HealthCheckSpec(lbname) log.V(2).Info("Looking for healthcheck", "name", healthcheckSpec.Name) - healthcheck, err := s.healthchecks.Get(ctx, meta.GlobalKey(healthcheckSpec.Name)) + key := meta.GlobalKey(healthcheckSpec.Name) + healthcheck, err := s.healthchecks.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for healthcheck", "name", healthcheckSpec.Name) @@ -142,35 +306,68 @@ func (s *Service) createOrGetHealthCheck(ctx context.Context) (*compute.HealthCh } log.V(2).Info("Creating a healthcheck", "name", healthcheckSpec.Name) - if err := s.healthchecks.Insert(ctx, meta.GlobalKey(healthcheckSpec.Name), healthcheckSpec); err != nil { + if err := s.healthchecks.Insert(ctx, key, healthcheckSpec); err != nil { log.Error(err, "Error creating a healthcheck", "name", healthcheckSpec.Name) return nil, err } - healthcheck, err = s.healthchecks.Get(ctx, meta.GlobalKey(healthcheckSpec.Name)) + healthcheck, err = s.healthchecks.Get(ctx, key) + if err != nil { + return nil, err + } + } + + return healthcheck, nil +} + +func (s *Service) createOrGetRegionalHealthCheck(ctx context.Context, lbname string) (*compute.HealthCheck, error) { + log := log.FromContext(ctx) + healthcheckSpec := s.scope.HealthCheckSpec(lbname) + healthcheckSpec.Region = s.scope.Region() + log.V(2).Info("Looking for regional healthcheck", "name", healthcheckSpec.Name) + key := meta.RegionalKey(healthcheckSpec.Name, s.scope.Region()) + healthcheck, err := s.regionalhealthchecks.Get(ctx, key) + if err != nil { + if !gcperrors.IsNotFound(err) { + log.Error(err, "Error looking for regional healthcheck", "name", healthcheckSpec.Name) + return nil, err + } + + log.V(2).Info("Creating a regional healthcheck", "name", healthcheckSpec.Name) + if err := s.regionalhealthchecks.Insert(ctx, key, healthcheckSpec); err != nil { + log.Error(err, "Error creating a regional healthcheck", "name", healthcheckSpec.Name) + return nil, err + } + + healthcheck, err = s.regionalhealthchecks.Get(ctx, key) if err != nil { return nil, err } } - s.scope.Network().APIServerHealthCheck = ptr.To[string](healthcheck.SelfLink) return healthcheck, nil } -func (s *Service) createOrGetBackendService(ctx context.Context, instancegroups []*compute.InstanceGroup, healthcheck *compute.HealthCheck) (*compute.BackendService, error) { +func (s *Service) createOrGetBackendService(ctx context.Context, lbname string, mode loadBalancingMode, instancegroups []*compute.InstanceGroup, healthcheck *compute.HealthCheck) (*compute.BackendService, error) { log := log.FromContext(ctx) backends := make([]*compute.Backend, 0, len(instancegroups)) for _, group := range instancegroups { - backends = append(backends, &compute.Backend{ - BalancingMode: "UTILIZATION", + be := &compute.Backend{ + BalancingMode: string(mode), Group: group.SelfLink, - }) + } + if mode == loadBalancingModeConnection { + be.MaxConnections = int64(2 ^ 32) + } + backends = append(backends, be) } - backendsvcSpec := s.scope.BackendServiceSpec() + backendsvcSpec := s.scope.BackendServiceSpec(lbname) backendsvcSpec.Backends = backends backendsvcSpec.HealthChecks = []string{healthcheck.SelfLink} - backendsvc, err := s.backendservices.Get(ctx, meta.GlobalKey(backendsvcSpec.Name)) + + key := meta.GlobalKey(backendsvcSpec.Name) + backendsvc, err := s.backendservices.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for backendservice", "name", backendsvcSpec.Name) @@ -178,12 +375,12 @@ func (s *Service) createOrGetBackendService(ctx context.Context, instancegroups } log.V(2).Info("Creating a backendservice", "name", backendsvcSpec.Name) - if err := s.backendservices.Insert(ctx, meta.GlobalKey(backendsvcSpec.Name), backendsvcSpec); err != nil { + if err := s.backendservices.Insert(ctx, key, backendsvcSpec); err != nil { log.Error(err, "Error creating a backendservice", "name", backendsvcSpec.Name) return nil, err } - backendsvc, err = s.backendservices.Get(ctx, meta.GlobalKey(backendsvcSpec.Name)) + backendsvc, err = s.backendservices.Get(ctx, key) if err != nil { return nil, err } @@ -192,13 +389,68 @@ func (s *Service) createOrGetBackendService(ctx context.Context, instancegroups if len(backendsvc.Backends) != len(backendsvcSpec.Backends) { log.V(2).Info("Updating a backendservice", "name", backendsvcSpec.Name) backendsvc.Backends = backendsvcSpec.Backends - if err := s.backendservices.Update(ctx, meta.GlobalKey(backendsvcSpec.Name), backendsvc); err != nil { + if err := s.backendservices.Update(ctx, key, backendsvc); err != nil { log.Error(err, "Error updating a backendservice", "name", backendsvcSpec.Name) return nil, err } } - s.scope.Network().APIServerBackendService = ptr.To[string](backendsvc.SelfLink) + return backendsvc, nil +} + +// createOrGetRegionalBackendService is used for internal passthrough load balancers. +func (s *Service) createOrGetRegionalBackendService(ctx context.Context, lbname string, instancegroups []*compute.InstanceGroup, healthcheck *compute.HealthCheck) (*compute.BackendService, error) { + log := log.FromContext(ctx) + backends := make([]*compute.Backend, 0, len(instancegroups)) + for _, group := range instancegroups { + be := &compute.Backend{ + // Always use connection mode for passthrough load balancer + BalancingMode: string(loadBalancingModeConnection), + Group: group.SelfLink, + } + backends = append(backends, be) + } + + backendsvcSpec := s.scope.BackendServiceSpec(lbname) + backendsvcSpec.Backends = backends + backendsvcSpec.HealthChecks = []string{healthcheck.SelfLink} + backendsvcSpec.Region = s.scope.Region() + backendsvcSpec.LoadBalancingScheme = string(loadBalanceTrafficInternal) + backendsvcSpec.PortName = "" + network := s.scope.Network() + if network.SelfLink != nil { + backendsvcSpec.Network = *network.SelfLink + } + + key := meta.RegionalKey(backendsvcSpec.Name, s.scope.Region()) + backendsvc, err := s.regionalbackendservices.Get(ctx, key) + if err != nil { + if !gcperrors.IsNotFound(err) { + log.Error(err, "Error looking for regional backendservice", "name", backendsvcSpec.Name) + return nil, err + } + + log.V(2).Info("Creating a regional backendservice", "name", backendsvcSpec.Name) + if err := s.regionalbackendservices.Insert(ctx, key, backendsvcSpec); err != nil { + log.Error(err, "Error creating a regional backendservice", "name", backendsvcSpec.Name) + return nil, err + } + + backendsvc, err = s.regionalbackendservices.Get(ctx, key) + if err != nil { + return nil, err + } + } + + if len(backendsvc.Backends) != len(backendsvcSpec.Backends) { + log.V(2).Info("Updating a regional backendservice", "name", backendsvcSpec.Name) + backendsvc.Backends = backendsvcSpec.Backends + if err := s.regionalbackendservices.Update(ctx, key, backendsvc); err != nil { + log.Error(err, "Error updating a regional backendservice", "name", backendsvcSpec.Name) + return nil, err + } + } + return backendsvc, nil } @@ -206,7 +458,8 @@ func (s *Service) createOrGetTargetTCPProxy(ctx context.Context, service *comput log := log.FromContext(ctx) targetSpec := s.scope.TargetTCPProxySpec() targetSpec.Service = service.SelfLink - target, err := s.targettcpproxies.Get(ctx, meta.GlobalKey(targetSpec.Name)) + key := meta.GlobalKey(targetSpec.Name) + target, err := s.targettcpproxies.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for targettcpproxy", "name", targetSpec.Name) @@ -214,26 +467,27 @@ func (s *Service) createOrGetTargetTCPProxy(ctx context.Context, service *comput } log.V(2).Info("Creating a targettcpproxy", "name", targetSpec.Name) - if err := s.targettcpproxies.Insert(ctx, meta.GlobalKey(targetSpec.Name), targetSpec); err != nil { + if err := s.targettcpproxies.Insert(ctx, key, targetSpec); err != nil { log.Error(err, "Error creating a targettcpproxy", "name", targetSpec.Name) return nil, err } - target, err = s.targettcpproxies.Get(ctx, meta.GlobalKey(targetSpec.Name)) + target, err = s.targettcpproxies.Get(ctx, key) if err != nil { return nil, err } } - s.scope.Network().APIServerTargetProxy = ptr.To[string](target.SelfLink) return target, nil } -func (s *Service) createOrGetAddress(ctx context.Context) (*compute.Address, error) { +// createOrGetAddress is used to obtain a Global address. +func (s *Service) createOrGetAddress(ctx context.Context, lbname string) (*compute.Address, error) { log := log.FromContext(ctx) - addrSpec := s.scope.AddressSpec() + addrSpec := s.scope.AddressSpec(lbname) log.V(2).Info("Looking for address", "name", addrSpec.Name) - addr, err := s.addresses.Get(ctx, meta.GlobalKey(addrSpec.Name)) + key := meta.GlobalKey(addrSpec.Name) + addr, err := s.addresses.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for address", "name", addrSpec.Name) @@ -241,57 +495,133 @@ func (s *Service) createOrGetAddress(ctx context.Context) (*compute.Address, err } log.V(2).Info("Creating an address", "name", addrSpec.Name) - if err := s.addresses.Insert(ctx, meta.GlobalKey(addrSpec.Name), addrSpec); err != nil { + if err := s.addresses.Insert(ctx, key, addrSpec); err != nil { log.Error(err, "Error creating an address", "name", addrSpec.Name) return nil, err } - addr, err = s.addresses.Get(ctx, meta.GlobalKey(addrSpec.Name)) + addr, err = s.addresses.Get(ctx, key) if err != nil { return nil, err } } - s.scope.Network().APIServerAddress = ptr.To[string](addr.SelfLink) - endpoint := s.scope.ControlPlaneEndpoint() - endpoint.Host = addr.Address - s.scope.SetControlPlaneEndpoint(endpoint) return addr, nil } -func (s *Service) createForwardingRule(ctx context.Context, target *compute.TargetTcpProxy, addr *compute.Address) error { +// createOrGetInternalAddress is used to obtain an internal address. +func (s *Service) createOrGetInternalAddress(ctx context.Context, lbname string) (*compute.Address, error) { log := log.FromContext(ctx) - spec := s.scope.ForwardingRuleSpec() - key := meta.GlobalKey(spec.Name) - spec.IPAddress = addr.SelfLink + addrSpec := s.scope.AddressSpec(lbname) + addrSpec.AddressType = string(loadBalanceTrafficInternal) + addrSpec.Region = s.scope.Region() + subnet, err := s.getSubnet(ctx) + if err != nil { + log.Error(err, "Error getting subnet for Internal Load Balancer") + return nil, err + } + addrSpec.Subnetwork = subnet.SelfLink + addrSpec.Purpose = "GCE_ENDPOINT" + log.V(2).Info("Looking for internal address", "name", addrSpec.Name) + key := meta.RegionalKey(addrSpec.Name, s.scope.Region()) + addr, err := s.internaladdresses.Get(ctx, key) + if err != nil { + if !gcperrors.IsNotFound(err) { + log.Error(err, "Error looking for internal address", "name", addrSpec.Name) + return nil, err + } + + log.V(2).Info("Creating an internal address", "name", addrSpec.Name) + if err := s.internaladdresses.Insert(ctx, key, addrSpec); err != nil { + log.Error(err, "Error creating an internal address", "name", addrSpec.Name) + return nil, err + } + + addr, err = s.internaladdresses.Get(ctx, key) + if err != nil { + return nil, err + } + } + + return addr, nil +} + +// createOrGetForwardingRule is used obtain a Global ForwardingRule. +func (s *Service) createOrGetForwardingRule(ctx context.Context, lbname string, target *compute.TargetTcpProxy, addr *compute.Address) (*compute.ForwardingRule, error) { + log := log.FromContext(ctx) + spec := s.scope.ForwardingRuleSpec(lbname) spec.Target = target.SelfLink + spec.IPAddress = addr.SelfLink + + key := meta.GlobalKey(spec.Name) log.V(2).Info("Looking for forwardingrule", "name", spec.Name) forwarding, err := s.forwardingrules.Get(ctx, key) if err != nil { if !gcperrors.IsNotFound(err) { log.Error(err, "Error looking for forwardingrule", "name", spec.Name) - return err + return nil, err } log.V(2).Info("Creating a forwardingrule", "name", spec.Name) if err := s.forwardingrules.Insert(ctx, key, spec); err != nil { log.Error(err, "Error creating a forwardingrule", "name", spec.Name) - return err + return nil, err } forwarding, err = s.forwardingrules.Get(ctx, key) if err != nil { - return err + return nil, err } } - s.scope.Network().APIServerForwardingRule = ptr.To[string](forwarding.SelfLink) - return nil + return forwarding, nil } -func (s *Service) deleteForwardingRule(ctx context.Context) error { +// createOrGetRegionalForwardingRule is used to obtain a Regional ForwardingRule. +func (s *Service) createOrGetRegionalForwardingRule(ctx context.Context, lbname string, backendSvc *compute.BackendService, addr *compute.Address) (*compute.ForwardingRule, error) { log := log.FromContext(ctx) - spec := s.scope.ForwardingRuleSpec() + spec := s.scope.ForwardingRuleSpec(lbname) + spec.LoadBalancingScheme = string(loadBalanceTrafficInternal) + spec.Region = s.scope.Region() + spec.BackendService = backendSvc.SelfLink + // Ports are used instead or PortRange for passthrough Load Balancer + spec.Ports = []string{"6443", "22623"} + spec.PortRange = "" + subnet, err := s.getSubnet(ctx) + if err != nil { + log.Error(err, "Error getting subnet for regional forwardingrule") + return nil, err + } + spec.Subnetwork = subnet.SelfLink + spec.IPAddress = addr.SelfLink + + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Looking for regional forwardingrule", "name", spec.Name) + forwarding, err := s.regionalforwardingrules.Get(ctx, key) + if err != nil { + if !gcperrors.IsNotFound(err) { + log.Error(err, "Error looking for regional forwardingrule", "name", spec.Name) + return nil, err + } + + log.V(2).Info("Creating a regional forwardingrule", "name", spec.Name) + if err := s.regionalforwardingrules.Insert(ctx, key, spec); err != nil { + log.Error(err, "Error creating a regional forwardingrule", "name", spec.Name) + return nil, err + } + + forwarding, err = s.regionalforwardingrules.Get(ctx, key) + if err != nil { + return nil, err + } + } + + return forwarding, nil +} + +func (s *Service) deleteForwardingRule(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.ForwardingRuleSpec(lbname) key := meta.GlobalKey(spec.Name) log.V(2).Info("Deleting a forwardingrule", "name", spec.Name) if err := s.forwardingrules.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { @@ -299,20 +629,43 @@ func (s *Service) deleteForwardingRule(ctx context.Context) error { return err } - s.scope.Network().APIServerForwardingRule = nil return nil } -func (s *Service) deleteAddress(ctx context.Context) error { +func (s *Service) deleteRegionalForwardingRule(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.ForwardingRuleSpec(lbname) + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Deleting a regional forwardingrule", "name", spec.Name) + if err := s.regionalforwardingrules.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { + log.Error(err, "Error updating a regional forwardingrule", "name", spec.Name) + return err + } + + return nil +} + +func (s *Service) deleteAddress(ctx context.Context, lbname string) error { log := log.FromContext(ctx) - spec := s.scope.AddressSpec() + spec := s.scope.AddressSpec(lbname) key := meta.GlobalKey(spec.Name) log.V(2).Info("Deleting a address", "name", spec.Name) if err := s.addresses.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { return err } - s.scope.Network().APIServerAddress = nil + return nil +} + +func (s *Service) deleteInternalAddress(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.AddressSpec(lbname) + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Deleting an internal address", "name", spec.Name) + if err := s.internaladdresses.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { + return err + } + return nil } @@ -326,13 +679,12 @@ func (s *Service) deleteTargetTCPProxy(ctx context.Context) error { return err } - s.scope.Network().APIServerTargetProxy = nil return nil } -func (s *Service) deleteBackendService(ctx context.Context) error { +func (s *Service) deleteBackendService(ctx context.Context, lbname string) error { log := log.FromContext(ctx) - spec := s.scope.BackendServiceSpec() + spec := s.scope.BackendServiceSpec(lbname) key := meta.GlobalKey(spec.Name) log.V(2).Info("Deleting a backendservice", "name", spec.Name) if err := s.backendservices.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { @@ -340,13 +692,25 @@ func (s *Service) deleteBackendService(ctx context.Context) error { return err } - s.scope.Network().APIServerBackendService = nil return nil } -func (s *Service) deleteHealthCheck(ctx context.Context) error { +func (s *Service) deleteRegionalBackendService(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.BackendServiceSpec(lbname) + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Deleting a regional backendservice", "name", spec.Name) + if err := s.regionalbackendservices.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { + log.Error(err, "Error deleting a regional backendservice", "name", spec.Name) + return err + } + + return nil +} + +func (s *Service) deleteHealthCheck(ctx context.Context, lbname string) error { log := log.FromContext(ctx) - spec := s.scope.HealthCheckSpec() + spec := s.scope.HealthCheckSpec(lbname) key := meta.GlobalKey(spec.Name) log.V(2).Info("Deleting a healthcheck", "name", spec.Name) if err := s.healthchecks.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { @@ -354,7 +718,19 @@ func (s *Service) deleteHealthCheck(ctx context.Context) error { return err } - s.scope.Network().APIServerHealthCheck = nil + return nil +} + +func (s *Service) deleteRegionalHealthCheck(ctx context.Context, lbname string) error { + log := log.FromContext(ctx) + spec := s.scope.HealthCheckSpec(lbname) + key := meta.RegionalKey(spec.Name, s.scope.Region()) + log.V(2).Info("Deleting a regional healthcheck", "name", spec.Name) + if err := s.regionalhealthchecks.Delete(ctx, key); err != nil && !gcperrors.IsNotFound(err) { + log.Error(err, "Error deleting a regional healthcheck", "name", spec.Name) + return err + } + return nil } @@ -376,3 +752,36 @@ func (s *Service) deleteInstanceGroups(ctx context.Context) error { return nil } + +// getSubnet gets the subnet to use for an internal Load Balancer. +func (s *Service) getSubnet(ctx context.Context) (*compute.Subnetwork, error) { + log := log.FromContext(ctx) + cfgSubnet := "" + lbSpec := s.scope.LoadBalancer() + if lbSpec.InternalLoadBalancer != nil { + cfgSubnet = ptr.Deref(lbSpec.InternalLoadBalancer.Subnet, "") + } + for _, subnetSpec := range s.scope.SubnetSpecs() { + log.V(2).Info("Looking for subnet for load balancer", "name", subnetSpec.Name) + region := subnetSpec.Region + if region == "" { + region = s.scope.Region() + } + + subnetKey := meta.RegionalKey(subnetSpec.Name, region) + subnet, err := s.subnets.Get(ctx, subnetKey) + if err != nil { + return nil, err + } + // Return subnet that matches configuration or first one if not configured + if cfgSubnet == "" || strings.HasSuffix(subnet.Name, cfgSubnet) { + return subnet, nil + } + } + + return nil, errors.New("could not find subnet") +} + +func isExternalLoadBalancer(name string) bool { + return strings.Contains(name, infrav1.APIServerRoleTagValue) +} diff --git a/cloud/services/compute/loadbalancers/reconcile_test.go b/cloud/services/compute/loadbalancers/reconcile_test.go index d837badb7..6518d9b68 100644 --- a/cloud/services/compute/loadbalancers/reconcile_test.go +++ b/cloud/services/compute/loadbalancers/reconcile_test.go @@ -28,12 +28,15 @@ import ( "google.golang.org/api/googleapi" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-gcp/cloud/scope" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +var lbTypeInternal = infrav1.InternalLoadBalancerOnly + func init() { _ = clusterv1.AddToScheme(scheme.Scheme) _ = infrav1.AddToScheme(scheme.Scheme) @@ -60,6 +63,16 @@ func getBaseClusterScope() (*scope.ClusterScope, error) { Spec: infrav1.GCPClusterSpec{ Project: "my-proj", Region: "us-central1", + Network: infrav1.NetworkSpec{ + Subnets: infrav1.Subnets{ + infrav1.SubnetSpec{ + Name: "control-plane", + CidrBlock: "10.0.0.1/28", + Region: "us-central1", + Purpose: ptr.To[string]("INTERNAL_HTTPS_LOAD_BALANCER"), + }, + }, + }, }, Status: infrav1.GCPClusterStatus{ FailureDomains: clusterv1.FailureDomains{ @@ -106,9 +119,8 @@ func TestService_createOrGetInstanceGroup(t *testing.T) { { name: "instanceGroup name is overridden (should create instanceGroup)", scope: func(s *scope.ClusterScope) Scope { - tagOverride := "master" s.GCPCluster.Spec.LoadBalancer = infrav1.LoadBalancerSpec{ - APIServerInstanceGroupTagOverride: &tagOverride, + APIServerInstanceGroupTagOverride: ptr.To[string]("master"), } return s }, @@ -143,7 +155,6 @@ func TestService_createOrGetInstanceGroup(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx := context.TODO() - clusterScope, err := getBaseClusterScope() if err != nil { t.Fatal(err) @@ -155,10 +166,551 @@ func TestService_createOrGetInstanceGroup(t *testing.T) { t.Errorf("Service s.createOrGetInstanceGroups() error = %v, wantErr %v", err, tt.wantErr) return } - if d := cmp.Diff(tt.want, got); d != "" { t.Errorf("Service s.createOrGetInstanceGroups() mismatch (-want +got):\n%s", d) } }) } } + +func TestService_createOrGetHealthCheck(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + mockHealthChecks *cloud.MockHealthChecks + want *compute.HealthCheck + wantErr bool + }{ + { + name: "health check does not exist for external load balancer (should create healthcheck)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.APIServerRoleTagValue, + mockHealthChecks: &cloud.MockHealthChecks{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockHealthChecksObj{}, + }, + want: &compute.HealthCheck{ + CheckIntervalSec: 10, + HealthyThreshold: 5, + HttpsHealthCheck: &compute.HTTPSHealthCheck{Port: 6443, PortSpecification: "USE_FIXED_PORT", RequestPath: "/readyz"}, + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/healthChecks/my-cluster-apiserver", + TimeoutSec: 5, + Type: "HTTPS", + UnhealthyThreshold: 3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.healthchecks = tt.mockHealthChecks + got, err := s.createOrGetHealthCheck(ctx, tt.lbName) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetHealthChecks() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetHealthCheck() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetRegionalHealthCheck(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + mockHealthChecks *cloud.MockRegionHealthChecks + want *compute.HealthCheck + wantErr bool + }{ + { + name: "regional health check does not exist for internal load balancer (should create healthcheck)", + scope: func(s *scope.ClusterScope) Scope { + s.GCPCluster.Spec.LoadBalancer = infrav1.LoadBalancerSpec{ + LoadBalancerType: &lbTypeInternal, + } + return s + }, + lbName: infrav1.InternalRoleTagValue, + mockHealthChecks: &cloud.MockRegionHealthChecks{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockRegionHealthChecksObj{}, + }, + want: &compute.HealthCheck{ + CheckIntervalSec: 10, + HealthyThreshold: 5, + HttpsHealthCheck: &compute.HTTPSHealthCheck{Port: 6443, PortSpecification: "USE_FIXED_PORT", RequestPath: "/readyz"}, + Name: "my-cluster-api-internal", + Region: "us-central1", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/healthChecks/my-cluster-api-internal", + TimeoutSec: 5, + Type: "HTTPS", + UnhealthyThreshold: 3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.regionalhealthchecks = tt.mockHealthChecks + got, err := s.createOrGetRegionalHealthCheck(ctx, tt.lbName) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetRegionalHealthChecks() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrRegionalGetHealthCheck() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetBackendService(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + healthCheck *compute.HealthCheck + instanceGroups []*compute.InstanceGroup + mockBackendService *cloud.MockBackendServices + want *compute.BackendService + wantErr bool + }{ + { + name: "backend service does not exist for external load balancer (should create backendservice)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.APIServerRoleTagValue, + healthCheck: &compute.HealthCheck{ + HttpsHealthCheck: &compute.HTTPSHealthCheck{Port: 6443, PortSpecification: "USE_FIXED_PORT", RequestPath: "/readyz"}, + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/healthChecks/my-cluster-apiserver", + }, + instanceGroups: []*compute.InstanceGroup{ + { + Name: "my-cluster-master-us-central1-a", + NamedPorts: []*compute.NamedPort{{Name: "apiserver", Port: 6443}}, + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/zones/us-central1-a/instanceGroups/my-cluster-master-us-central1-a", + }, + }, + mockBackendService: &cloud.MockBackendServices{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockBackendServicesObj{}, + }, + want: &compute.BackendService{ + Backends: []*compute.Backend{ + { + BalancingMode: "UTILIZATION", + Group: "https://www.googleapis.com/compute/v1/projects/proj-id/zones/us-central1-a/instanceGroups/my-cluster-master-us-central1-a", + }, + }, + HealthChecks: []string{ + "https://www.googleapis.com/compute/v1/projects/proj-id/global/healthChecks/my-cluster-apiserver", + }, + LoadBalancingScheme: "EXTERNAL", + Name: "my-cluster-apiserver", + PortName: "apiserver", + Protocol: "TCP", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/backendServices/my-cluster-apiserver", + TimeoutSec: 600, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.backendservices = tt.mockBackendService + mode := loadBalancingModeUtilization + got, err := s.createOrGetBackendService(ctx, tt.lbName, mode, tt.instanceGroups, tt.healthCheck) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetBackendService() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetBackendService() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetRegionalBackendService(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + healthCheck *compute.HealthCheck + instanceGroups []*compute.InstanceGroup + mockBackendService *cloud.MockRegionBackendServices + want *compute.BackendService + wantErr bool + }{ + { + name: "regional backend service does not exist for internal load balancer (should create regional backendservice)", + scope: func(s *scope.ClusterScope) Scope { + s.GCPCluster.Spec.LoadBalancer = infrav1.LoadBalancerSpec{ + LoadBalancerType: &lbTypeInternal, + } + s.GCPCluster.Status.Network.SelfLink = ptr.To[string]("https://www.googleapis.com/compute/v1/projects/openshift-dev-installer/global/networks/bfournie-capg-test-5jp2d-network") + return s + }, + lbName: infrav1.InternalRoleTagValue, + healthCheck: &compute.HealthCheck{ + HttpsHealthCheck: &compute.HTTPSHealthCheck{Port: 6443, PortSpecification: "USE_FIXED_PORT", RequestPath: "/readyz"}, + Name: "my-cluster-api-internal", + Region: "us-central1", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/healthChecks/my-cluster-api-internal", + }, + instanceGroups: []*compute.InstanceGroup{ + { + Name: "my-cluster-apiserver-us-central1-a", + NamedPorts: []*compute.NamedPort{{Name: "apiserver", Port: 6443}}, + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/zones/us-central1-a/instanceGroups/my-cluster-master-us-central1-a", + }, + }, + mockBackendService: &cloud.MockRegionBackendServices{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockRegionBackendServicesObj{}, + }, + want: &compute.BackendService{ + Backends: []*compute.Backend{ + { + BalancingMode: "CONNECTION", + Group: "https://www.googleapis.com/compute/v1/projects/proj-id/zones/us-central1-a/instanceGroups/my-cluster-master-us-central1-a", + }, + }, + HealthChecks: []string{ + "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/healthChecks/my-cluster-api-internal", + }, + LoadBalancingScheme: "INTERNAL", + Name: "my-cluster-api-internal", + Network: "https://www.googleapis.com/compute/v1/projects/openshift-dev-installer/global/networks/bfournie-capg-test-5jp2d-network", + PortName: "", + Protocol: "TCP", + Region: "us-central1", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/backendServices/my-cluster-api-internal", + TimeoutSec: 600, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.regionalbackendservices = tt.mockBackendService + got, err := s.createOrGetRegionalBackendService(ctx, tt.lbName, tt.instanceGroups, tt.healthCheck) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetRegionalBackendService() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetRegionalBackendService() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetAddress(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + mockAddress *cloud.MockGlobalAddresses + want *compute.Address + wantErr bool + }{ + { + name: "address does not exist for external load balancer (should create address)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.APIServerRoleTagValue, + mockAddress: &cloud.MockGlobalAddresses{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockGlobalAddressesObj{}, + }, + want: &compute.Address{ + IpVersion: "IPV4", + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/addresses/my-cluster-apiserver", + AddressType: "EXTERNAL", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.addresses = tt.mockAddress + got, err := s.createOrGetAddress(ctx, tt.lbName) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetAddress() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetAddress() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetInternalAddress(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + mockAddress *cloud.MockAddresses + mockSubnetworks *cloud.MockSubnetworks + want *compute.Address + wantErr bool + }{ + { + name: "address does not exist for internal load balancer (should create address)", + scope: func(s *scope.ClusterScope) Scope { + lbType := infrav1.InternalLoadBalancerOnly + s.GCPCluster.Spec.LoadBalancer = infrav1.LoadBalancerSpec{ + LoadBalancerType: &lbType, + } + return s + }, + lbName: infrav1.InternalRoleTagValue, + mockAddress: &cloud.MockAddresses{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockAddressesObj{}, + }, + mockSubnetworks: &cloud.MockSubnetworks{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "my-proj"}, + Objects: map[meta.Key]*cloud.MockSubnetworksObj{ + *meta.RegionalKey("control-plane", "us-central1"): {}, + }, + }, + want: &compute.Address{ + IpVersion: "IPV4", + Name: "my-cluster-api-internal", + Region: "us-central1", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-api-internal", + AddressType: "INTERNAL", + Purpose: "GCE_ENDPOINT", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.internaladdresses = tt.mockAddress + s.subnets = tt.mockSubnetworks + got, err := s.createOrGetInternalAddress(ctx, tt.lbName) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetInternalAddress() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetInternalAddress() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetTargetTCPProxy(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + backendService *compute.BackendService + mockTargetTCPProxy *cloud.MockTargetTcpProxies + want *compute.TargetTcpProxy + wantErr bool + }{ + { + name: "target tcp proxy does not exist for external load balancer (should create target tp proxy)", + scope: func(s *scope.ClusterScope) Scope { return s }, + backendService: &compute.BackendService{ + Name: "my-cluster-api-internal", + }, + mockTargetTCPProxy: &cloud.MockTargetTcpProxies{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockTargetTcpProxiesObj{}, + }, + want: &compute.TargetTcpProxy{ + Name: "my-cluster-apiserver", + ProxyHeader: "NONE", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/targetTcpProxies/my-cluster-apiserver", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.targettcpproxies = tt.mockTargetTCPProxy + got, err := s.createOrGetTargetTCPProxy(ctx, tt.backendService) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetTargetTCPProxy() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, got); d != "" { + t.Errorf("Service s.createOrGetTargetTCPProxy() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetForwardingRule(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + backendService *compute.BackendService + targetTcpproxy *compute.TargetTcpProxy + address *compute.Address + mockForwardingRule *cloud.MockGlobalForwardingRules + want *compute.ForwardingRule + wantErr bool + }{ + { + name: "forwarding rule does not exist for external load balancer (should create forwardingrule)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.APIServerRoleTagValue, + address: &compute.Address{ + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-apiserver", + }, + backendService: &compute.BackendService{}, + targetTcpproxy: &compute.TargetTcpProxy{ + Name: "my-cluster-apiserver", + }, + mockForwardingRule: &cloud.MockGlobalForwardingRules{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockGlobalForwardingRulesObj{}, + }, + want: &compute.ForwardingRule{ + IPAddress: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-apiserver", + IPProtocol: "TCP", + LoadBalancingScheme: "EXTERNAL", + PortRange: "443-443", + Name: "my-cluster-apiserver", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/global/forwardingRules/my-cluster-apiserver", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.forwardingrules = tt.mockForwardingRule + var fwdRule *compute.ForwardingRule + fwdRule, err = s.createOrGetForwardingRule(ctx, tt.lbName, tt.targetTcpproxy, tt.address) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetForwardingRule() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, fwdRule); d != "" { + t.Errorf("Service s.createOrGetForwardingRule() mismatch (-want +got):\n%s", d) + } + }) + } +} + +func TestService_createOrGetRegionalForwardingRule(t *testing.T) { + tests := []struct { + name string + scope func(s *scope.ClusterScope) Scope + lbName string + backendService *compute.BackendService + targetTcpproxy *compute.TargetTcpProxy + address *compute.Address + mockSubnetworks *cloud.MockSubnetworks + mockForwardingRule *cloud.MockForwardingRules + want *compute.ForwardingRule + wantErr bool + }{ + { + name: "regional forwarding rule does not exist for internal load balancer (should create forwardingrule)", + scope: func(s *scope.ClusterScope) Scope { return s }, + lbName: infrav1.InternalRoleTagValue, + address: &compute.Address{ + Name: "my-cluster-api-internal", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-api-internal", + }, + backendService: &compute.BackendService{ + Name: "my-cluster-api-internal", + }, + targetTcpproxy: &compute.TargetTcpProxy{}, + mockSubnetworks: &cloud.MockSubnetworks{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "my-proj"}, + Objects: map[meta.Key]*cloud.MockSubnetworksObj{ + *meta.RegionalKey("control-plane", "us-central1"): {}, + }, + }, + mockForwardingRule: &cloud.MockForwardingRules{ + ProjectRouter: &cloud.SingleProjectRouter{ID: "proj-id"}, + Objects: map[meta.Key]*cloud.MockForwardingRulesObj{}, + }, + want: &compute.ForwardingRule{ + IPAddress: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/addresses/my-cluster-api-internal", + IPProtocol: "TCP", + LoadBalancingScheme: "INTERNAL", + Ports: []string{"6443", "22623"}, + Region: "us-central1", + Name: "my-cluster-api-internal", + SelfLink: "https://www.googleapis.com/compute/v1/projects/proj-id/regions/us-central1/forwardingRules/my-cluster-api-internal", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.TODO() + clusterScope, err := getBaseClusterScope() + if err != nil { + t.Fatal(err) + } + s := New(tt.scope(clusterScope)) + s.regionalforwardingrules = tt.mockForwardingRule + var fwdRule *compute.ForwardingRule + s.subnets = tt.mockSubnetworks + fwdRule, err = s.createOrGetRegionalForwardingRule(ctx, tt.lbName, tt.backendService, tt.address) + if (err != nil) != tt.wantErr { + t.Errorf("Service s.createOrGetRegionalForwardingRule() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(tt.want, fwdRule); d != "" { + t.Errorf("Service s.createOrGetRegionalForwardingRule() mismatch (-want +got):\n%s", d) + } + }) + } +} diff --git a/cloud/services/compute/loadbalancers/service.go b/cloud/services/compute/loadbalancers/service.go index 8a1067054..276fd2515 100644 --- a/cloud/services/compute/loadbalancers/service.go +++ b/cloud/services/compute/loadbalancers/service.go @@ -65,26 +65,36 @@ type targettcpproxiesInterface interface { Delete(ctx context.Context, key *meta.Key, options ...k8scloud.Option) error } +type subnetsInterface interface { + Get(ctx context.Context, key *meta.Key, options ...k8scloud.Option) (*compute.Subnetwork, error) +} + // Scope is an interfaces that hold used methods. type Scope interface { cloud.Cluster - AddressSpec() *compute.Address - BackendServiceSpec() *compute.BackendService - ForwardingRuleSpec() *compute.ForwardingRule - HealthCheckSpec() *compute.HealthCheck + AddressSpec(name string) *compute.Address + BackendServiceSpec(name string) *compute.BackendService + ForwardingRuleSpec(name string) *compute.ForwardingRule + HealthCheckSpec(name string) *compute.HealthCheck InstanceGroupSpec(zone string) *compute.InstanceGroup TargetTCPProxySpec() *compute.TargetTcpProxy + SubnetSpecs() []*compute.Subnetwork } // Service implements loadbalancers reconciler. type Service struct { - scope Scope - addresses addressesInterface - backendservices backendservicesInterface - forwardingrules forwardingrulesInterface - healthchecks healthchecksInterface - instancegroups instancegroupsInterface - targettcpproxies targettcpproxiesInterface + scope Scope + addresses addressesInterface + internaladdresses addressesInterface + backendservices backendservicesInterface + regionalbackendservices backendservicesInterface + forwardingrules forwardingrulesInterface + regionalforwardingrules forwardingrulesInterface + healthchecks healthchecksInterface + regionalhealthchecks healthchecksInterface + instancegroups instancegroupsInterface + targettcpproxies targettcpproxiesInterface + subnets subnetsInterface } var _ cloud.Reconciler = &Service{} @@ -92,12 +102,17 @@ var _ cloud.Reconciler = &Service{} // New returns Service from given scope. func New(scope Scope) *Service { return &Service{ - scope: scope, - addresses: scope.Cloud().GlobalAddresses(), - backendservices: scope.Cloud().BackendServices(), - forwardingrules: scope.Cloud().GlobalForwardingRules(), - healthchecks: scope.Cloud().HealthChecks(), - instancegroups: scope.Cloud().InstanceGroups(), - targettcpproxies: scope.Cloud().TargetTcpProxies(), + scope: scope, + addresses: scope.Cloud().GlobalAddresses(), + internaladdresses: scope.Cloud().Addresses(), + backendservices: scope.Cloud().BackendServices(), + regionalbackendservices: scope.Cloud().RegionBackendServices(), + forwardingrules: scope.Cloud().GlobalForwardingRules(), + regionalforwardingrules: scope.Cloud().ForwardingRules(), + healthchecks: scope.Cloud().HealthChecks(), + regionalhealthchecks: scope.Cloud().RegionHealthChecks(), + instancegroups: scope.Cloud().InstanceGroups(), + targettcpproxies: scope.Cloud().TargetTcpProxies(), + subnets: scope.Cloud().Subnetworks(), } } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml index 10fe9b1dc..bcdd9d22e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclusters.yaml @@ -118,6 +118,28 @@ spec: maxLength: 16 pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) type: string + internalLoadBalancer: + description: InternalLoadBalancer is the configuration for an + Internal Passthrough Network Load Balancer. + properties: + name: + description: |- + Name is the name of the Load Balancer. If not set a default name + will be used. For an Internal Load Balancer service the default + name is "api-internal". + maxLength: 32 + pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) + type: string + subnet: + description: Subnet is the name of the subnet to use for a + regional Load Balancer. + type: string + type: object + loadBalancerType: + description: |- + LoadBalancerType defines the type of Load Balancer that should be created. + If not set, a Global External Proxy Load Balancer will be created by default. + type: string type: object network: description: NetworkSpec encapsulates all things related to GCP network. @@ -298,6 +320,26 @@ spec: network: description: Network encapsulates GCP networking resources. properties: + apiInternalBackendService: + description: |- + APIInternalBackendService is the full reference to the backend service + created for the internal Load Balancer. + type: string + apiInternalForwardingRule: + description: |- + APIInternalForwardingRule is the full reference to the forwarding rule + created for the internal Load Balancer. + type: string + apiInternalHealthCheck: + description: |- + APIInternalHealthCheck is the full reference to the health check + created for the internal Load Balancer. + type: string + apiInternalIpAddress: + description: |- + APIInternalAddress is the IPV4 regional address assigned to the + internal Load Balancer. + type: string apiServerBackendService: description: |- APIServerBackendService is the full reference to the backend service diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml index 05179c5c3..ed1f3786f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpclustertemplates.yaml @@ -134,6 +134,28 @@ spec: maxLength: 16 pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) type: string + internalLoadBalancer: + description: InternalLoadBalancer is the configuration + for an Internal Passthrough Network Load Balancer. + properties: + name: + description: |- + Name is the name of the Load Balancer. If not set a default name + will be used. For an Internal Load Balancer service the default + name is "api-internal". + maxLength: 32 + pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) + type: string + subnet: + description: Subnet is the name of the subnet to use + for a regional Load Balancer. + type: string + type: object + loadBalancerType: + description: |- + LoadBalancerType defines the type of Load Balancer that should be created. + If not set, a Global External Proxy Load Balancer will be created by default. + type: string type: object network: description: NetworkSpec encapsulates all things related to diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml index 7faef1da9..ebaa5f5ae 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_gcpmanagedclusters.yaml @@ -113,6 +113,28 @@ spec: maxLength: 16 pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) type: string + internalLoadBalancer: + description: InternalLoadBalancer is the configuration for an + Internal Passthrough Network Load Balancer. + properties: + name: + description: |- + Name is the name of the Load Balancer. If not set a default name + will be used. For an Internal Load Balancer service the default + name is "api-internal". + maxLength: 32 + pattern: (^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$) + type: string + subnet: + description: Subnet is the name of the subnet to use for a + regional Load Balancer. + type: string + type: object + loadBalancerType: + description: |- + LoadBalancerType defines the type of Load Balancer that should be created. + If not set, a Global External Proxy Load Balancer will be created by default. + type: string type: object network: description: NetworkSpec encapsulates all things related to the GCP @@ -340,6 +362,26 @@ spec: network: description: Network encapsulates GCP networking resources. properties: + apiInternalBackendService: + description: |- + APIInternalBackendService is the full reference to the backend service + created for the internal Load Balancer. + type: string + apiInternalForwardingRule: + description: |- + APIInternalForwardingRule is the full reference to the forwarding rule + created for the internal Load Balancer. + type: string + apiInternalHealthCheck: + description: |- + APIInternalHealthCheck is the full reference to the health check + created for the internal Load Balancer. + type: string + apiInternalIpAddress: + description: |- + APIInternalAddress is the IPV4 regional address assigned to the + internal Load Balancer. + type: string apiServerBackendService: description: |- APIServerBackendService is the full reference to the backend service diff --git a/controllers/gcpcluster_controller.go b/controllers/gcpcluster_controller.go index 58dcd5f01..f5eddfe1b 100644 --- a/controllers/gcpcluster_controller.go +++ b/controllers/gcpcluster_controller.go @@ -200,8 +200,9 @@ func (r *GCPClusterReconciler) reconcile(ctx context.Context, clusterScope *scop reconcilers := []cloud.Reconciler{ networks.New(clusterScope), firewalls.New(clusterScope), - loadbalancers.New(clusterScope), + // Reconcile subnets before loadbalancers since subnet is needed for internal LB subnets.New(clusterScope), + loadbalancers.New(clusterScope), } for _, r := range reconcilers {