feat: support shared load balancer health probe mode. By setting `clu…

…sterServiceLoadBalancerHealthProbeMode` to `shared`, all cluster services will share one health probe targeting the kube-proxy port 10256 and /healthz by default. The health check port and path can be configured by `clusterServiceSharedLoadBalancerHealthProbePort` and `clusterServiceSharedLoadBalancerHealthProbePort`.

pkg/consts/consts.go

@@ -556,3 +556,12 @@ const (
 
 	ServiceNameLabel = "kubernetes.io/service-name"
 )
+
+// Load Balancer health probe mode
+const (
+	ClusterServiceLoadBalancerHealthProbeModeDefault = "default"
+	ClusterServiceLoadBalancerHealthProbeModeShared  = "shared"
+	ClusterServiceLoadBalancerHealthProbeDefaultPort = 10256
+	ClusterServiceLoadBalancerHealthProbeDefaultPath = "/healthz"
+	SharedProbeName                                  = "cluster-service-shared-health-probe"
+)

pkg/provider/azure.go

@@ -278,6 +278,16 @@ type Config struct {
 	RouteUpdateIntervalInSeconds int `json:"routeUpdateIntervalInSeconds,omitempty" yaml:"routeUpdateIntervalInSeconds,omitempty"`
 	// LoadBalancerBackendPoolUpdateIntervalInSeconds is the interval for updating load balancer backend pool of local services. Default is 30 seconds.
 	LoadBalancerBackendPoolUpdateIntervalInSeconds int `json:"loadBalancerBackendPoolUpdateIntervalInSeconds,omitempty" yaml:"loadBalancerBackendPoolUpdateIntervalInSeconds,omitempty"`
+
+	// ClusterServiceLoadBalancerHealthProbeMode determines the health probe mode for cluster service load balancer.
+	// Supported values are `shared` and `default`.
+	// `default`: the health probe will be created against each port of each service by watching the backend application.
+	// `shared`: all cluster services shares one probe targeting the kube-proxy on the node (<nodeIP>/healthz:10256).
+	ClusterServiceLoadBalancerHealthProbeMode string `json:"clusterServiceLoadBalancerHealthProbeMode,omitempty" yaml:"clusterServiceLoadBalancerHealthProbeMode,omitempty"`
+	// ClusterServiceSharedLoadBalancerHealthProbePort defines the target port of the shared health probe. Default to 10256.
+	ClusterServiceSharedLoadBalancerHealthProbePort int32 `json:"clusterServiceSharedLoadBalancerHealthProbePort,omitempty" yaml:"clusterServiceSharedLoadBalancerHealthProbePort,omitempty"`
+	// ClusterServiceSharedLoadBalancerHealthProbePath defines the target path of the shared health probe. Default to `/healthz`.
+	ClusterServiceSharedLoadBalancerHealthProbePath string `json:"clusterServiceSharedLoadBalancerHealthProbePath,omitempty" yaml:"clusterServiceSharedLoadBalancerHealthProbePath,omitempty"`
 }
 
 // MultipleStandardLoadBalancerConfiguration stores the properties regarding multiple standard load balancers.
@@ -627,6 +637,26 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *Config,
 		}
 	}
 
+	if config.ClusterServiceLoadBalancerHealthProbeMode == "" {
+		config.ClusterServiceLoadBalancerHealthProbeMode = consts.ClusterServiceLoadBalancerHealthProbeModeDefault
+	} else {
+		supportedClusterServiceLoadBalancerHealthProbeModes := sets.New(
+			strings.ToLower(consts.ClusterServiceLoadBalancerHealthProbeModeDefault),
+			strings.ToLower(consts.ClusterServiceLoadBalancerHealthProbeModeShared),
+		)
+		if !supportedClusterServiceLoadBalancerHealthProbeModes.Has(strings.ToLower(config.ClusterServiceLoadBalancerHealthProbeMode)) {
+			return fmt.Errorf("clusterServiceLoadBalancerHealthProbeMode %s is not supported, supported values are %v", config.ClusterServiceLoadBalancerHealthProbeMode, supportedClusterServiceLoadBalancerHealthProbeModes.UnsortedList())
+		}
+	}
+	if strings.EqualFold(config.ClusterServiceLoadBalancerHealthProbeMode, consts.ClusterServiceLoadBalancerHealthProbeModeShared) {
+		if config.ClusterServiceSharedLoadBalancerHealthProbePort == 0 {
+			config.ClusterServiceSharedLoadBalancerHealthProbePort = consts.ClusterServiceLoadBalancerHealthProbeDefaultPort
+		}
+		if config.ClusterServiceSharedLoadBalancerHealthProbePath == "" {
+			config.ClusterServiceSharedLoadBalancerHealthProbePath = consts.ClusterServiceLoadBalancerHealthProbeDefaultPath
+		}
+	}
+
 	env, err := ratelimitconfig.ParseAzureEnvironment(config.Cloud, config.ResourceManagerEndpoint, config.IdentitySystem)
 	if err != nil {
 		return err

pkg/provider/azure_loadbalancer.go

@@ -2597,6 +2597,13 @@ func (az *Cloud) getExpectedLBRules(
 		}
 	}
 
+	var useSharedProbe bool
+	if az.useSharedLoadBalancerHealthProbeMode() &&
+		!strings.EqualFold(string(service.Spec.ExternalTrafficPolicy), string(v1.ServiceExternalTrafficPolicyLocal)) {
+		nodeEndpointHealthprobe = az.buildClusterServiceSharedProbe()
+		useSharedProbe = true
+	}
+
 	// In HA mode, lb forward traffic of all port to backend
 	// HA mode is only supported on standard loadbalancer SKU in internal mode
 	if consts.IsK8sServiceUsingInternalLoadBalancer(service) &&
@@ -2614,7 +2621,7 @@ func (az *Cloud) getExpectedLBRules(
 		if nodeEndpointHealthprobe == nil {
 			// use user customized health probe rule if any
 			for _, port := range service.Spec.Ports {
-				portprobe, err := az.buildHealthProbeRulesForPort(service, port, lbRuleName, nil)
+				portprobe, err := az.buildHealthProbeRulesForPort(service, port, lbRuleName, nil, false)
 				if err != nil {
 					klog.V(2).ErrorS(err, "error occurred when buildHealthProbeRulesForPort", "service", service.Name, "namespace", service.Namespace,
 						"rule-name", lbRuleName, "port", port.Port)
@@ -2676,7 +2683,7 @@ func (az *Cloud) getExpectedLBRules(
 					"rule-name", lbRuleName, "port", port.Port)
 			}
 			if !isNoHealthProbeRule {
-				portprobe, err := az.buildHealthProbeRulesForPort(service, port, lbRuleName, nodeEndpointHealthprobe)
+				portprobe, err := az.buildHealthProbeRulesForPort(service, port, lbRuleName, nodeEndpointHealthprobe, useSharedProbe)
 				if err != nil {
 					klog.V(2).ErrorS(err, "error occurred when buildHealthProbeRulesForPort", "service", service.Name, "namespace", service.Namespace,
 						"rule-name", lbRuleName, "port", port.Port)

pkg/provider/azure_loadbalancer_healthprobe.go

@@ -22,17 +22,37 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
+
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
 
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 )
 
+func (az *Cloud) buildClusterServiceSharedProbe() *network.Probe {
+	return &network.Probe{
+		Name: pointer.String(consts.SharedProbeName),
+		ProbePropertiesFormat: &network.ProbePropertiesFormat{
+			Protocol:          network.ProbeProtocolHTTP,
+			Port:              pointer.Int32(az.ClusterServiceSharedLoadBalancerHealthProbePort),
+			RequestPath:       pointer.String(az.ClusterServiceSharedLoadBalancerHealthProbePath),
+			IntervalInSeconds: pointer.Int32(consts.HealthProbeDefaultProbeInterval),
+			ProbeThreshold:    pointer.Int32(consts.HealthProbeDefaultNumOfProbe),
+		},
+	}
+}
+
 // buildHealthProbeRulesForPort
 // for following sku: basic loadbalancer vs standard load balancer
 // for following protocols: TCP HTTP HTTPS(SLB only)
 // return nil if no new probe is added
-func (az *Cloud) buildHealthProbeRulesForPort(serviceManifest *v1.Service, port v1.ServicePort, lbrule string, healthCheckNodePortProbe *network.Probe) (*network.Probe, error) {
+func (az *Cloud) buildHealthProbeRulesForPort(serviceManifest *v1.Service, port v1.ServicePort, lbrule string, healthCheckNodePortProbe *network.Probe, useSharedProbe bool) (*network.Probe, error) {
+	if useSharedProbe {
+		klog.V(4).Infof("skip creating health probe for port %s because the shared probe is used", port.Port)
+		return nil, nil
+	}
+
 	if port.Protocol == v1.ProtocolUDP || port.Protocol == v1.ProtocolSCTP {
 		return nil, nil
 	}

pkg/provider/azure_loadbalancer_repo.go

@@ -387,3 +387,12 @@ func isBackendPoolOnSameLB(newBackendPoolID string, existingBackendPools []strin
 
 	return true, "", nil
 }
+
+func (az *Cloud) serviceOwnsRule(service *v1.Service, rule string) bool {
+	if !strings.EqualFold(string(service.Spec.ExternalTrafficPolicy), string(v1.ServiceExternalTrafficPolicyTypeLocal)) &&
+		rule == consts.SharedProbeName {
+		return true
+	}
+	prefix := az.getRulePrefix(service)
+	return strings.HasPrefix(strings.ToUpper(rule), strings.ToUpper(prefix))
+}

pkg/provider/azure_loadbalancer_repo_test.go

@@ -407,3 +407,22 @@ func TestIsBackendPoolOnSameLB(t *testing.T) {
 		assert.Equal(t, test.expectedLBName, lbName)
 	}
 }
+
+func TestServiceOwnsRuleSharedProbe(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	for _, tc := range []struct {
+		desc string
+	}{
+		{
+			desc: "should count in the shared probe",
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			az := GetTestCloud(ctrl)
+			svc := getTestService("test", v1.ProtocolTCP, nil, false)
+			assert.True(t, az.serviceOwnsRule(&svc, consts.SharedProbeName))
+		})
+	}
+}

pkg/provider/azure_loadbalancer_test.go

@@ -3020,6 +3020,33 @@ func TestReconcileLoadBalancerRuleCommon(t *testing.T) {
 	}
 }
 
+func TestGetExpectedLBRulesSharedProbe(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	for _, tc := range []struct {
+		desc string
+	}{
+		{
+			desc: "getExpectedLBRules should return a shared rule for a cluster service when shared probe is enabled",
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			az := GetTestCloud(ctrl)
+			az.ClusterServiceLoadBalancerHealthProbeMode = consts.ClusterServiceLoadBalancerHealthProbeModeShared
+			svc := getTestService("test1", v1.ProtocolTCP, nil, false, 80, 81)
+
+			probe, lbrule, err := az.getExpectedLBRules(&svc, "frontendIPConfigID", "backendPoolID", "lbname", consts.IPVersionIPv4)
+			assert.NoError(t, err)
+			assert.Equal(t, 1, len(probe))
+			assert.Equal(t, *az.buildClusterServiceSharedProbe(), probe[0])
+			assert.Equal(t, 2, len(lbrule))
+			assert.Equal(t, "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lbname/probes/cluster-service-shared-health-probe", *lbrule[0].Probe.ID)
+			assert.Equal(t, "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lbname/probes/cluster-service-shared-health-probe", *lbrule[1].Probe.ID)
+		})
+	}
+}
+
 // getDefaultTestRules returns dualstack rules.
 func getDefaultTestRules(enableTCPReset bool) map[bool][]network.LoadBalancingRule {
 	return map[bool][]network.LoadBalancingRule{

pkg/provider/azure_standard.go

@@ -345,12 +345,6 @@ func (az *Cloud) getPublicIPName(clusterName string, service *v1.Service, isIPv6
 	return getResourceByIPFamily(pipName, isDualStack, isIPv6), nil
 }
 
-// TODO: UT
-func (az *Cloud) serviceOwnsRule(service *v1.Service, rule string) bool {
-	prefix := az.getRulePrefix(service)
-	return strings.HasPrefix(strings.ToUpper(rule), strings.ToUpper(prefix))
-}
-
 func publicIPOwnsFrontendIP(service *v1.Service, fip *network.FrontendIPConfiguration, pip *network.PublicIPAddress) bool {
 	if pip != nil &&
 		pip.ID != nil &&

pkg/provider/azure_wrap.go

@@ -97,3 +97,7 @@ func ConvertResourceGroupNameToLower(resourceID string) (string, error) {
 	resourceGroup := matches[1]
 	return strings.Replace(resourceID, resourceGroup, strings.ToLower(resourceGroup), 1), nil
 }
+
+func (az *Cloud) useSharedLoadBalancerHealthProbeMode() bool {
+	return strings.EqualFold(az.ClusterServiceLoadBalancerHealthProbeMode, consts.ClusterServiceLoadBalancerHealthProbeModeShared)
+}