feat: support node selection for multiple standard load balancers

kubernetes-sigs · Jul 13, 2023 · bd9fa86 · bd9fa86
1 parent 547ff5d
commit bd9fa86
Show file tree

Hide file tree

Showing 7 changed files with 1,068 additions and 92 deletions.
diff --git a/pkg/provider/azure.go b/pkg/provider/azure.go
@@ -309,7 +309,12 @@ type MultipleStandardLoadBalancerConfigurationSpec struct {
 
 // MultipleStandardLoadBalancerConfigurationStatus stores the properties regarding multiple standard load balancers.
 type MultipleStandardLoadBalancerConfigurationStatus struct {
+	// ActiveServices stores the services that are supposed to use the load balancer.
 	ActiveServices sets.Set[string] `json:"activeServices" yaml:"activeServices"`
+
+	// ActiveNodes stores the nodes that are supposed to be in the load balancer.
+	// It will be used in EnsureHostsInPool to make sure the given ones are in the backend pool.
+	ActiveNodes sets.Set[string] `json:"activeNodes" yaml:"activeNodes"`
 }
 
 type InitSecretConfig struct {
@@ -383,8 +388,9 @@ type Cloud struct {
 	// unmanagedNodes holds a list of nodes not managed by Azure cloud provider.
 	unmanagedNodes sets.Set[string]
 	// excludeLoadBalancerNodes holds a list of nodes that should be excluded from LoadBalancer.
-	excludeLoadBalancerNodes sets.Set[string]
-	nodePrivateIPs           map[string]sets.Set[string]
+	excludeLoadBalancerNodes   sets.Set[string]
+	nodePrivateIPs             map[string]sets.Set[string]
+	nodePrivateIPToNodeNameMap map[string]string
 	// nodeInformerSynced is for determining if the informer has synced.
 	nodeInformerSynced cache.InformerSynced
 
@@ -421,7 +427,14 @@ type Cloud struct {
 	*ManagedDiskController
 	*controllerCommon
 
+	// multipleStandardLoadBalancerConfigurationsSynced make sure the `reconcileMultipleStandardLoadBalancerConfigurations`
+	// runs only once every time the cloud provide restarts.
 	multipleStandardLoadBalancerConfigurationsSynced bool
+	// nodesWithCorrectLoadBalancerByPrimaryVMSet marks nodes that are matched with load balancers by primary vmSet.
+	nodesWithCorrectLoadBalancerByPrimaryVMSet sync.Map
+
+	multipleStandardLoadBalancersActiveServicesLock sync.Mutex
+	multipleStandardLoadBalancersActiveNodesLock    sync.Mutex
 }
 
 // NewCloud returns a Cloud with initialized clients
@@ -487,13 +500,14 @@ func (az *Cloud) configSecretMetadata(secretName, secretNamespace, cloudConfigKe
 
 func NewCloudFromSecret(ctx context.Context, clientBuilder cloudprovider.ControllerClientBuilder, secretName, secretNamespace, cloudConfigKey string) (cloudprovider.Interface, error) {
 	az := &Cloud{
-		nodeNames:                sets.New[string](),
-		nodeZones:                map[string]sets.Set[string]{},
-		nodeResourceGroups:       map[string]string{},
-		unmanagedNodes:           sets.New[string](),
-		routeCIDRs:               map[string]string{},
-		excludeLoadBalancerNodes: sets.New[string](),
-		nodePrivateIPs:           map[string]sets.Set[string]{},
+		nodeNames:                  sets.New[string](),
+		nodeZones:                  map[string]sets.Set[string]{},
+		nodeResourceGroups:         map[string]string{},
+		unmanagedNodes:             sets.New[string](),
+		routeCIDRs:                 map[string]string{},
+		excludeLoadBalancerNodes:   sets.New[string](),
+		nodePrivateIPs:             map[string]sets.Set[string]{},
+		nodePrivateIPToNodeNameMap: map[string]string{},
 	}
 
 	az.configSecretMetadata(secretName, secretNamespace, cloudConfigKey)
@@ -519,13 +533,14 @@ func NewCloudWithoutFeatureGates(ctx context.Context, configReader io.Reader, ca
 	}
 
 	az := &Cloud{
-		nodeNames:                sets.New[string](),
-		nodeZones:                map[string]sets.Set[string]{},
-		nodeResourceGroups:       map[string]string{},
-		unmanagedNodes:           sets.New[string](),
-		routeCIDRs:               map[string]string{},
-		excludeLoadBalancerNodes: sets.New[string](),
-		nodePrivateIPs:           map[string]sets.Set[string]{},
+		nodeNames:                  sets.New[string](),
+		nodeZones:                  map[string]sets.Set[string]{},
+		nodeResourceGroups:         map[string]string{},
+		unmanagedNodes:             sets.New[string](),
+		routeCIDRs:                 map[string]string{},
+		excludeLoadBalancerNodes:   sets.New[string](),
+		nodePrivateIPs:             map[string]sets.Set[string]{},
+		nodePrivateIPToNodeNameMap: map[string]string{},
 	}
 
 	err = az.InitializeCloudFromConfig(ctx, config, false, callFromCCM)
@@ -729,6 +744,24 @@ func (az *Cloud) checkEnableMultipleStandardLoadBalancers() error {
 	if az.isLBBackendPoolTypeNodeIPConfig() {
 		return fmt.Errorf("multiple standard load balancers cannot be used with backend pool type %s", consts.LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration)
 	}
+
+	names := sets.New[string]()
+	primaryVMSets := sets.New[string]()
+	for _, multiSLBConfig := range az.MultipleStandardLoadBalancerConfigurations {
+		if names.Has(multiSLBConfig.Name) {
+			return fmt.Errorf("duplicated multiple standard load balancer configuration name %s", multiSLBConfig.Name)
+		}
+		names.Insert(multiSLBConfig.Name)
+
+		if multiSLBConfig.PrimaryVMSet == "" {
+			return fmt.Errorf("multiple standard load balancer configuration %s must have primary VMSet", multiSLBConfig.Name)
+		}
+		if primaryVMSets.Has(multiSLBConfig.PrimaryVMSet) {
+			return fmt.Errorf("duplicated primary VMSet %s in multiple standard load balancer configurations %s", multiSLBConfig.PrimaryVMSet, multiSLBConfig.Name)
+		}
+		primaryVMSets.Insert(multiSLBConfig.PrimaryVMSet)
+	}
+
 	return nil
 }
 
@@ -1188,12 +1221,14 @@ func (az *Cloud) updateNodeCaches(prevNode, newNode *v1.Node) {
 		// if the node is being deleted from the cluster, exclude it from load balancers
 		if newNode == nil {
 			az.excludeLoadBalancerNodes.Insert(prevNode.ObjectMeta.Name)
+			az.nodesWithCorrectLoadBalancerByPrimaryVMSet.Delete(strings.ToLower(prevNode.ObjectMeta.Name))
 		}
 
 		// Remove from nodePrivateIPs cache.
 		for _, address := range getNodePrivateIPAddresses(prevNode) {
 			klog.V(4).Infof("removing IP address %s of the node %s", address, prevNode.Name)
 			az.nodePrivateIPs[prevNode.Name].Delete(address)
+			delete(az.nodePrivateIPToNodeNameMap, address)
 		}
 	}
 
@@ -1255,9 +1290,13 @@ func (az *Cloud) updateNodeCaches(prevNode, newNode *v1.Node) {
 			if az.nodePrivateIPs[newNode.Name] == nil {
 				az.nodePrivateIPs[newNode.Name] = sets.New[string]()
 			}
+			if az.nodePrivateIPToNodeNameMap == nil {
+				az.nodePrivateIPToNodeNameMap = make(map[string]string)
+			}
 
 			klog.V(6).Infof("adding IP address %s of the node %s", address, newNode.Name)
 			az.nodePrivateIPs[newNode.Name].Insert(address)
+			az.nodePrivateIPToNodeNameMap[address] = newNode.Name
 		}
 	}
 }
@@ -1392,3 +1431,16 @@ func isNodeReady(node *v1.Node) bool {
 	}
 	return false
 }
+
+func (az *Cloud) getActiveNodesByLoadBalancerName(lbName string) sets.Set[string] {
+	az.multipleStandardLoadBalancersActiveNodesLock.Lock()
+	defer az.multipleStandardLoadBalancersActiveNodesLock.Unlock()
+
+	for _, multiSLBConfig := range az.MultipleStandardLoadBalancerConfigurations {
+		if strings.EqualFold(strings.TrimSuffix(lbName, consts.InternalLoadBalancerNameSuffix), multiSLBConfig.Name) {
+			return multiSLBConfig.ActiveNodes
+		}
+	}
+
+	return sets.New[string]()
+}