Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support customization of numOfProbe and probeInterval when externaltrafficpolicy is local #4207

Merged
merged 1 commit into from
Jul 5, 2023
Merged

Support customization of numOfProbe and probeInterval when externaltrafficpolicy is local #4207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
226 changes: 6 additions & 220 deletions pkg/provider/azure_loadbalancer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2172,209 +2172,6 @@ func lbRuleConflictsWithPort(rule network.LoadBalancingRule, frontendIPConfigID
*rule.FrontendPort == port.Port
}

// buildHealthProbeRulesForPort
// for following sku: basic loadbalancer vs standard load balancer
// for following protocols: TCP HTTP HTTPS(SLB only)
func (az *Cloud) buildHealthProbeRulesForPort(serviceManifest *v1.Service, port v1.ServicePort, lbrule string) (*network.Probe, error) {
if port.Protocol == v1.ProtocolUDP || port.Protocol == v1.ProtocolSCTP {
return nil, nil
}
// protocol should be tcp, because sctp is handled in outer loop

properties := &network.ProbePropertiesFormat{}
var err error

// order - Specific Override
// port_ annotation
// global annotation

// Select Protocol
//
var protocol *string

// 1. Look up port-specific override
protocol, err = consts.GetHealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsProtocol)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsProtocol), err)
}

// 2. If not specified, look up from AppProtocol
// Note - this order is to remain compatible with previous versions
if protocol == nil {
protocol = port.AppProtocol
}

// 3. If protocol is still nil, check the global annotation
if protocol == nil {
protocol, err = consts.GetAttributeValueInSvcAnnotation(serviceManifest.Annotations, consts.ServiceAnnotationLoadBalancerHealthProbeProtocol)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.ServiceAnnotationLoadBalancerHealthProbeProtocol, err)
}
}

// 4. Finally, if protocol is still nil, default to TCP
if protocol == nil {
protocol = pointer.String(string(network.ProtocolTCP))
}

*protocol = strings.TrimSpace(*protocol)
switch {
case strings.EqualFold(*protocol, string(network.ProtocolTCP)):
properties.Protocol = network.ProbeProtocolTCP
case strings.EqualFold(*protocol, string(network.ProtocolHTTPS)):
//HTTPS probe is only supported in standard loadbalancer
//For backward compatibility,when unsupported protocol is used, fall back to tcp protocol in basic lb mode instead
if !az.useStandardLoadBalancer() {
properties.Protocol = network.ProbeProtocolTCP
} else {
properties.Protocol = network.ProbeProtocolHTTPS
}
case strings.EqualFold(*protocol, string(network.ProtocolHTTP)):
properties.Protocol = network.ProbeProtocolHTTP
default:
//For backward compatibility,when unsupported protocol is used, fall back to tcp protocol in basic lb mode instead
properties.Protocol = network.ProbeProtocolTCP
}

// Lookup or Override Health Probe Port
properties.Port = &port.NodePort

probePort, err := consts.GetHealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsPort, func(s *string) error {
if s == nil {
return nil
}
//nolint:gosec
port, err := strconv.Atoi(*s)
if err != nil {
//not a integer
for _, item := range serviceManifest.Spec.Ports {
if strings.EqualFold(item.Name, *s) {
//found the port
return nil
}
}
return fmt.Errorf("port %s not found in service", *s)
}
if port < 0 || port > 65535 {
return fmt.Errorf("port %d is out of range", port)
}
for _, item := range serviceManifest.Spec.Ports {
//nolint:gosec
if item.Port == int32(port) {
//found the port
return nil
}
}
return fmt.Errorf("port %s not found in service", *s)
})
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsPort), err)
}

if probePort != nil {
//nolint:gosec
port, err := strconv.Atoi(*probePort)
if err != nil {
//not a integer
for _, item := range serviceManifest.Spec.Ports {
if strings.EqualFold(item.Name, *probePort) {
//found the port
properties.Port = pointer.Int32(item.NodePort)
}
}
} else {
// Not need to verify probePort is in correct range again.
for _, item := range serviceManifest.Spec.Ports {
//nolint:gosec
if item.Port == int32(port) {
//found the port
properties.Port = pointer.Int32(item.NodePort)
}
}
}
}

// Select request path
if strings.EqualFold(string(properties.Protocol), string(network.ProtocolHTTPS)) || strings.EqualFold(string(properties.Protocol), string(network.ProtocolHTTP)) {
// get request path ,only used with http/https probe
path, err := consts.GetHealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsRequestPath)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsRequestPath), err)
}
if path == nil {
if path, err = consts.GetAttributeValueInSvcAnnotation(serviceManifest.Annotations, consts.ServiceAnnotationLoadBalancerHealthProbeRequestPath); err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.ServiceAnnotationLoadBalancerHealthProbeRequestPath, err)
}
}
if path == nil {
path = pointer.String(consts.HealthProbeDefaultRequestPath)
}
properties.RequestPath = path
}
// get number of probes
var numOfProbeValidator = func(val *int32) error {
//minimum number of unhealthy responses is 2. ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
const (
MinimumNumOfProbe = 2
)
if *val < MinimumNumOfProbe {
return fmt.Errorf("the minimum value of %s is %d", consts.HealthProbeParamsNumOfProbe, MinimumNumOfProbe)
}
return nil
}
numberOfProbes, err := consts.GetInt32HealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsNumOfProbe, numOfProbeValidator)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsNumOfProbe), err)
}
if numberOfProbes == nil {
if numberOfProbes, err = consts.Getint32ValueFromK8sSvcAnnotation(serviceManifest.Annotations, consts.ServiceAnnotationLoadBalancerHealthProbeNumOfProbe, numOfProbeValidator); err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.ServiceAnnotationLoadBalancerHealthProbeNumOfProbe, err)
}
}

// if numberOfProbes is not set, set it to default instead ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
if numberOfProbes == nil {
numberOfProbes = pointer.Int32(consts.HealthProbeDefaultNumOfProbe)
}

// get probe interval in seconds
var probeIntervalValidator = func(val *int32) error {
//minimum probe interval in seconds is 5. ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
const (
MinimumProbeIntervalInSecond = 5
)
if *val < 5 {
return fmt.Errorf("the minimum value of %s is %d", consts.HealthProbeParamsProbeInterval, MinimumProbeIntervalInSecond)
}
return nil
}
probeInterval, err := consts.GetInt32HealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsProbeInterval, probeIntervalValidator)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s:%w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsProbeInterval), err)
}
if probeInterval == nil {
if probeInterval, err = consts.Getint32ValueFromK8sSvcAnnotation(serviceManifest.Annotations, consts.ServiceAnnotationLoadBalancerHealthProbeInterval, probeIntervalValidator); err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.ServiceAnnotationLoadBalancerHealthProbeInterval, err)
}
}
// if probeInterval is not set, set it to default instead ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
if probeInterval == nil {
probeInterval = pointer.Int32(consts.HealthProbeDefaultProbeInterval)
}

// total probe should be less than 120 seconds ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
if (*probeInterval)*(*numberOfProbes) >= 120 {
return nil, fmt.Errorf("total probe should be less than 120, please adjust interval and number of probe accordingly")
}
properties.IntervalInSeconds = probeInterval
properties.ProbeThreshold = numberOfProbes
probe := &network.Probe{
Name: &lbrule,
ProbePropertiesFormat: properties,
}
return probe, nil
}

// buildLBRules
// for following sku: basic loadbalancer vs standard load balancer
// for following scenario: internal vs external
Expand All @@ -2396,15 +2193,18 @@ func (az *Cloud) getExpectedLBRules(
if servicehelpers.NeedsHealthCheck(service) && !(consts.IsPLSEnabled(service.Annotations) && consts.IsPLSProxyProtocolEnabled(service.Annotations)) {
podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service)
lbRuleName := az.getLoadBalancerRuleName(service, v1.ProtocolTCP, podPresencePort, isIPv6)

probeInterval, numberOfProbes, err := az.getHealthProbeConfigProbeIntervalAndNumOfProbe(service, podPresencePort)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

newly added.

if err != nil {
return nil, nil, err
}
nodeEndpointHealthprobe = &network.Probe{
Name: &lbRuleName,
ProbePropertiesFormat: &network.ProbePropertiesFormat{
RequestPath: pointer.String(podPresencePath),
Protocol: network.ProbeProtocolHTTP,
Port: pointer.Int32(podPresencePort),
IntervalInSeconds: pointer.Int32(consts.HealthProbeDefaultProbeInterval),
ProbeThreshold: pointer.Int32(consts.HealthProbeDefaultNumOfProbe),
IntervalInSeconds: probeInterval,
ProbeThreshold: numberOfProbes,
},
}
expectedProbes = append(expectedProbes, *nodeEndpointHealthprobe)
Expand Down Expand Up @@ -3480,20 +3280,6 @@ func (az *Cloud) safeDeletePublicIP(service *v1.Service, pipResourceGroup string
return nil
}

func findProbe(probes []network.Probe, probe network.Probe) bool {
for _, existingProbe := range probes {
if strings.EqualFold(pointer.StringDeref(existingProbe.Name, ""), pointer.StringDeref(probe.Name, "")) &&
pointer.Int32Deref(existingProbe.Port, 0) == pointer.Int32Deref(probe.Port, 0) &&
strings.EqualFold(string(existingProbe.Protocol), string(probe.Protocol)) &&
strings.EqualFold(pointer.StringDeref(existingProbe.RequestPath, ""), pointer.StringDeref(probe.RequestPath, "")) &&
pointer.Int32Deref(existingProbe.IntervalInSeconds, 0) == pointer.Int32Deref(probe.IntervalInSeconds, 0) &&
pointer.Int32Deref(existingProbe.ProbeThreshold, 0) == pointer.Int32Deref(probe.ProbeThreshold, 0) {
return true
}
}
return false
}

func findRule(rules []network.LoadBalancingRule, rule network.LoadBalancingRule, wantLB bool) bool {
for _, existingRule := range rules {
if strings.EqualFold(pointer.StringDeref(existingRule.Name, ""), pointer.StringDeref(rule.Name, "")) &&
Expand Down