Merge pull request #4075 from nilo19/feat/multi-slb/config

feat: support load balancer choosing logic for multi-slb

pkg/consts/consts.go

@@ -241,7 +241,7 @@ const (
 
 	// ServiceAnnotationLoadBalancerMode is the annotation used on the service to specify
 	// which load balancer should be associated with the service. This is valid when using the basic
-	// load balancer or turn on the multiple standard load balancers mode, or it would be ignored.
+	// sku load balancer, or it would be ignored.
 	// 1. Default mode - service has no annotation ("service.beta.kubernetes.io/azure-load-balancer-mode")
 	//	  In this case the Loadbalancer of the primary VMSS/VMAS is selected.
 	// 2. "__auto__" mode - service is annotated with __auto__ value, this when loadbalancer from any VMSS/VMAS
@@ -315,11 +315,15 @@ const (
 	// If omitted, the default value is false
 	ServiceAnnotationDisableLoadBalancerFloatingIP = "service.beta.kubernetes.io/azure-disable-load-balancer-floating-ip"
 
-	// ServiceAnnotationAzurePIPTags sets the additional Public IPs (split by comma) besides the service's Public IP configured on LoadBalancer.
+	// ServiceAnnotationAdditionalPublicIPs sets the additional Public IPs (split by comma) besides the service's Public IP configured on LoadBalancer.
 	// These additional Public IPs would be consumed by kube-proxy to configure the iptables rules on each node. Note they would not be configured
 	// automatically on Azure LoadBalancer. Instead, they need to be configured manually (e.g. on Azure cross-region LoadBalancer by another operator).
 	ServiceAnnotationAdditionalPublicIPs = "service.beta.kubernetes.io/azure-additional-public-ips"
 
+	// ServiceAnnotationLoadBalancerConfigurations is the list of load balancer configurations the service can use.
+	// The list is separated by comma. It will be omitted if multi-slb is not used.
+	ServiceAnnotationLoadBalancerConfigurations = "service.beta.kubernetes.io/azure-load-balancer-configurations"
+
 	// ServiceTagKey is the service key applied for public IP tags.
 	ServiceTagKey       = "k8s-azure-service"
 	LegacyServiceTagKey = "service"

pkg/consts/helpers.go

@@ -142,3 +142,17 @@ func expectAttributeInSvcAnnotationBeEqualTo(annotations map[string]string, key
 	}
 	return false
 }
+
+// getLoadBalancerConfigurationsNames parse the annotation and return the names of the load balancer configurations.
+func GetLoadBalancerConfigurationsNames(service *v1.Service) []string {
+	var names []string
+	for key, lbConfig := range service.Annotations {
+		if strings.EqualFold(key, ServiceAnnotationLoadBalancerConfigurations) {
+			names = append(names, strings.Split(lbConfig, ",")...)
+		}
+	}
+	for i := range names {
+		names[i] = strings.ToLower(strings.TrimSpace(names[i]))
+	}
+	return names
+}

pkg/provider/azure.go

@@ -34,6 +34,7 @@ import (
 	"github.com/Azure/go-autorest/autorest/azure"
 
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/informers"
@@ -255,6 +256,57 @@ type Config struct {
 	// If the API is not used, the migration will be done by decoupling all nodes on the backend pool and then re-attaching
 	// node IPs, which will introduce service downtime. The downtime increases with the number of nodes in the backend pool.
 	EnableMigrateToIPBasedBackendPoolAPI bool `json:"enableMigrateToIPBasedBackendPoolAPI" yaml:"enableMigrateToIPBasedBackendPoolAPI"`
+
+	// MultipleStandardLoadBalancerConfigurations stores the properties regarding multiple standard load balancers.
+	// It will be ignored if LoadBalancerBackendPoolConfigurationType is nodeIPConfiguration.
+	// If the length is not 0, it is assumed the multiple standard load balancers mode is on. In this case,
+	// there must be one configuration named “<clustername>” or an error will be reported.
+	MultipleStandardLoadBalancerConfigurations []MultipleStandardLoadBalancerConfiguration `json:"multipleStandardLoadBalancerConfigurations,omitempty" yaml:"multipleStandardLoadBalancerConfigurations,omitempty"`
+}
+
+// MultipleStandardLoadBalancerConfiguration stores the properties regarding multiple standard load balancers.
+type MultipleStandardLoadBalancerConfiguration struct {
+	// Name of the public load balancer. There will be an internal load balancer
+	// created if needed, and the name will be `<name>-internal`. The internal lb
+	// shares the same configurations as the external one. The internal lbs
+	// are not needed to be included in `MultipleStandardLoadBalancerConfigurations`.
+	// There must be a name of “<clustername>” in the load balancer configuration list.
+	Name string `json:"name" yaml:"name"`
+
+	MultipleStandardLoadBalancerConfigurationSpec
+
+	MultipleStandardLoadBalancerConfigurationStatus
+}
+
+// MultipleStandardLoadBalancerConfigurationSpec stores the properties regarding multiple standard load balancers.
+type MultipleStandardLoadBalancerConfigurationSpec struct {
+	// This load balancer can have services placed on it. Defaults to true,
+	// can be set to false to drain and eventually remove a load balancer.
+	// This only affects services that will be using the LB. For services
+	// that is currently using the LB, they will not be affected.
+	AllowServicePlacement *bool `json:"allowServicePlacement" yaml:"allowServicePlacement"`
+
+	// A string value that must specify the name of an existing vmSet.
+	// All nodes in the given vmSet will always be added to this load balancer.
+	// A vmSet can only be the primary vmSet for a single load balancer.
+	PrimaryVMSet string `json:"primaryVMSet" yaml:"primaryVMSet"`
+
+	// Services that must match this selector can be placed on this load balancer. If not supplied,
+	// services with any labels can be created on the load balancer.
+	ServiceLabelSelector *metav1.LabelSelector `json:"serviceLabelSelector" yaml:"serviceLabelSelector"`
+
+	// Services created in namespaces with the supplied label will be allowed to select that load balancer.
+	// If not supplied, services created in any namespaces can be created on that load balancer.
+	ServiceNamespaceSelector *metav1.LabelSelector `json:"serviceNamespaceSelector" yaml:"serviceNamespaceSelector"`
+
+	// Nodes matching this selector will be preferentially added to the load balancers that
+	// they match selectors for. NodeSelector does not override primaryAgentPool for node allocation.
+	NodeSelector *metav1.LabelSelector `json:"nodeSelector" yaml:"nodeSelector"`
+}
+
+// MultipleStandardLoadBalancerConfigurationStatus stores the properties regarding multiple standard load balancers.
+type MultipleStandardLoadBalancerConfigurationStatus struct {
+	ActiveServices sets.Set[string] `json:"activeServices" yaml:"activeServices"`
 }
 
 type InitSecretConfig struct {
@@ -365,6 +417,8 @@ type Cloud struct {
 
 	*ManagedDiskController
 	*controllerCommon
+
+	multipleStandardLoadBalancerConfigurationsSynced bool
 }
 
 // NewCloud returns a Cloud with initialized clients
@@ -538,6 +592,12 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *Config,
 		}
 	}
 
+	if az.useMultipleStandardLoadBalancers() {
+		if err := az.checkEnableMultipleStandardLoadBalancers(); err != nil {
+			return err
+		}
+	}
+
 	env, err := ratelimitconfig.ParseAzureEnvironment(config.Cloud, config.ResourceManagerEndpoint, config.IdentitySystem)
 	if err != nil {
 		return err
@@ -653,6 +713,22 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *Config,
 	return nil
 }
 
+func (az *Cloud) useMultipleStandardLoadBalancers() bool {
+	return az.useStandardLoadBalancer() && len(az.MultipleStandardLoadBalancerConfigurations) > 0
+}
+
+func (az *Cloud) useSingleStandardLoadBalancer() bool {
+	return az.useStandardLoadBalancer() && len(az.MultipleStandardLoadBalancerConfigurations) == 0
+}
+
+// Multiple standard load balancer mode only supports IP-based load balancers.
+func (az *Cloud) checkEnableMultipleStandardLoadBalancers() error {
+	if az.isLBBackendPoolTypeNodeIPConfig() {
+		return fmt.Errorf("multiple standard load balancers cannot be used with backend pool type %s", consts.LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration)
+	}
+	return nil
+}
+
 func (az *Cloud) isLBBackendPoolTypeNodeIPConfig() bool {
 	return strings.EqualFold(az.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration)
 }

pkg/provider/azure_backoff.go

@@ -336,36 +336,47 @@ func (az *Cloud) ListManagedLBs(service *v1.Service, nodes []*v1.Node, clusterNa
 		return nil, nil
 	}
 
-	// return early if wantLb=false
-	if nodes == nil {
-		klog.V(4).Infof("ListManagedLBs: return all LBs in the resource group %s, including unmanaged LBs", az.getLoadBalancerResourceGroup())
-		return allLBs, nil
-	}
+	managedLBNames := sets.New[string](strings.ToLower(clusterName))
+	managedLBs := make([]network.LoadBalancer, 0)
+	if strings.EqualFold(az.LoadBalancerSku, consts.LoadBalancerSkuBasic) {
+		// return early if wantLb=false
+		if nodes == nil {
+			klog.V(4).Infof("ListManagedLBs: return all LBs in the resource group %s, including unmanaged LBs", az.getLoadBalancerResourceGroup())
+			return allLBs, nil
+		}
 
-	agentPoolLBs := make([]network.LoadBalancer, 0)
-	agentPoolVMSetNames, err := az.VMSet.GetAgentPoolVMSetNames(nodes)
-	if err != nil {
-		return nil, fmt.Errorf("ListManagedLBs: failed to get agent pool vmSet names: %w", err)
+		agentPoolVMSetNamesMap := make(map[string]bool)
+		agentPoolVMSetNames, err := az.VMSet.GetAgentPoolVMSetNames(nodes)
+		if err != nil {
+			return nil, fmt.Errorf("ListManagedLBs: failed to get agent pool vmSet names: %w", err)
+		}
+
+		if agentPoolVMSetNames != nil && len(*agentPoolVMSetNames) > 0 {
+			for _, vmSetName := range *agentPoolVMSetNames {
+				klog.V(6).Infof("ListManagedLBs: found agent pool vmSet name %s", vmSetName)
+				agentPoolVMSetNamesMap[strings.ToLower(vmSetName)] = true
+			}
+		}
+
+		for agentPoolVMSetName := range agentPoolVMSetNamesMap {
+			managedLBNames.Insert(az.mapVMSetNameToLoadBalancerName(agentPoolVMSetName, clusterName))
+		}
 	}
 
-	agentPoolVMSetNamesSet := sets.New[string]()
-	if agentPoolVMSetNames != nil && len(*agentPoolVMSetNames) > 0 {
-		for _, vmSetName := range *agentPoolVMSetNames {
-			klog.V(6).Infof("ListManagedLBs: found agent pool vmSet name %s", vmSetName)
-			agentPoolVMSetNamesSet.Insert(strings.ToLower(vmSetName))
+	if az.useMultipleStandardLoadBalancers() {
+		for _, multiSLBConfig := range az.MultipleStandardLoadBalancerConfigurations {
+			managedLBNames.Insert(multiSLBConfig.Name, fmt.Sprintf("%s%s", multiSLBConfig.Name, consts.InternalLoadBalancerNameSuffix))
 		}
 	}
 
 	for _, lb := range allLBs {
-		vmSetNameFromLBName := az.mapLoadBalancerNameToVMSet(pointer.StringDeref(lb.Name, ""), clusterName)
-		if strings.EqualFold(strings.TrimSuffix(pointer.StringDeref(lb.Name, ""), consts.InternalLoadBalancerNameSuffix), clusterName) ||
-			agentPoolVMSetNamesSet.Has(strings.ToLower(vmSetNameFromLBName)) {
-			agentPoolLBs = append(agentPoolLBs, lb)
-			klog.V(4).Infof("ListManagedLBs: found agent pool LB %s", pointer.StringDeref(lb.Name, ""))
+		if managedLBNames.Has(strings.ToLower(strings.TrimSuffix(pointer.StringDeref(lb.Name, ""), consts.InternalLoadBalancerNameSuffix))) {
+			managedLBs = append(managedLBs, lb)
+			klog.V(4).Infof("ListManagedLBs: found managed LB %s", pointer.StringDeref(lb.Name, ""))
 		}
 	}
 
-	return agentPoolLBs, nil
+	return managedLBs, nil
 }
 
 // ListLB invokes az.LoadBalancerClient.List with exponential backoff retry

pkg/provider/azure_backoff_test.go

@@ -299,13 +299,14 @@ func TestCreateOrUpdateLB(t *testing.T) {
 	}
 }
 
-func TestListAgentPoolLBs(t *testing.T) {
+func TestListManagedLBs(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
 	tests := []struct {
 		existingLBs, expectedLBs []network.LoadBalancer
 		callTimes                int
+		multiSLBConfigs          []MultipleStandardLoadBalancerConfiguration
 		clientErr                *retry.Error
 		expectedErr              error
 	}{
@@ -334,9 +335,32 @@ func TestListAgentPoolLBs(t *testing.T) {
 			},
 			callTimes: 1,
 		},
+		{
+			existingLBs: []network.LoadBalancer{
+				{Name: pointer.String("kubernetes")},
+				{Name: pointer.String("kubernetes-internal")},
+				{Name: pointer.String("lb1-internal")},
+				{Name: pointer.String("lb2")},
+			},
+			multiSLBConfigs: []MultipleStandardLoadBalancerConfiguration{
+				{Name: "kubernetes"},
+				{Name: "lb1"},
+			},
+			expectedLBs: []network.LoadBalancer{
+				{Name: pointer.String("kubernetes")},
+				{Name: pointer.String("kubernetes-internal")},
+				{Name: pointer.String("lb1-internal")},
+			},
+		},
 	}
 	for _, test := range tests {
 		az := GetTestCloud(ctrl)
+		if len(test.multiSLBConfigs) > 0 {
+			az.LoadBalancerSku = consts.LoadBalancerSkuStandard
+			az.MultipleStandardLoadBalancerConfigurations = test.multiSLBConfigs
+		} else {
+			az.LoadBalancerSku = consts.LoadBalancerSkuBasic
+		}
 
 		mockLBClient := az.LoadBalancerClient.(*mockloadbalancerclient.MockInterface)
 		mockLBClient.EXPECT().List(gomock.Any(), az.ResourceGroup).Return(test.existingLBs, test.clientErr)