Skip to content

Commit

Permalink
Merge pull request #4295 from MartinForReal/release-1.25-shafan4207
Browse files Browse the repository at this point in the history 
[release-1.25]Support customization of numOfProbe and probeInterval when externaltr…
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Jul 17, 2023
2 parents 0009be7 + fc8b8cd commit 224eeca
Show file tree
Hide file tree
Showing 5 changed files with 555 additions and 400 deletions.
225 changes: 6 additions & 219 deletions pkg/provider/azure_loadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2121,209 +2121,6 @@ func lbRuleConflictsWithPort(rule network.LoadBalancingRule, frontendIPConfigID
*rule.FrontendPort == port.Port
}

// buildHealthProbeRulesForPort
// for following sku: basic loadbalancer vs standard load balancer
// for following protocols: TCP HTTP HTTPS(SLB only)
func (az *Cloud) buildHealthProbeRulesForPort(serviceManifest *v1.Service, port v1.ServicePort, lbrule string) (*network.Probe, error) {
if port.Protocol == v1.ProtocolUDP || port.Protocol == v1.ProtocolSCTP {
return nil, nil
}
// protocol should be tcp, because sctp is handled in outer loop

properties := &network.ProbePropertiesFormat{}
var err error

// order - Specific Override
// port_ annotation
// global annotation

// Select Protocol
//
var protocol *string

// 1. Look up port-specific override
protocol, err = consts.GetHealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsProtocol)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsProtocol), err)
}

// 2. If not specified, look up from AppProtocol
// Note - this order is to remain compatible with previous versions
if protocol == nil {
protocol = port.AppProtocol
}

// 3. If protocol is still nil, check the global annotation
if protocol == nil {
protocol, err = consts.GetAttributeValueInSvcAnnotation(serviceManifest.Annotations, consts.ServiceAnnotationLoadBalancerHealthProbeProtocol)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.ServiceAnnotationLoadBalancerHealthProbeProtocol, err)
}
}

// 4. Finally, if protocol is still nil, default to TCP
if protocol == nil {
protocol = pointer.String(string(network.ProtocolTCP))
}

*protocol = strings.TrimSpace(*protocol)
switch {
case strings.EqualFold(*protocol, string(network.ProtocolTCP)):
properties.Protocol = network.ProbeProtocolTCP
case strings.EqualFold(*protocol, string(network.ProtocolHTTPS)):
//HTTPS probe is only supported in standard loadbalancer
//For backward compatibility,when unsupported protocol is used, fall back to tcp protocol in basic lb mode instead
if !az.useStandardLoadBalancer() {
properties.Protocol = network.ProbeProtocolTCP
} else {
properties.Protocol = network.ProbeProtocolHTTPS
}
case strings.EqualFold(*protocol, string(network.ProtocolHTTP)):
properties.Protocol = network.ProbeProtocolHTTP
default:
//For backward compatibility,when unsupported protocol is used, fall back to tcp protocol in basic lb mode instead
properties.Protocol = network.ProbeProtocolTCP
}

// Lookup or Override Health Probe Port
properties.Port = &port.NodePort

probePort, err := consts.GetHealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsPort, func(s *string) error {
if s == nil {
return nil
}
//nolint:gosec
port, err := strconv.Atoi(*s)
if err != nil {
//not a integer
for _, item := range serviceManifest.Spec.Ports {
if strings.EqualFold(item.Name, *s) {
//found the port
return nil
}
}
return fmt.Errorf("port %s not found in service", *s)
}
if port < 0 || port > 65535 {
return fmt.Errorf("port %d is out of range", port)
}
for _, item := range serviceManifest.Spec.Ports {
//nolint:gosec
if item.Port == int32(port) {
//found the port
return nil
}
}
return fmt.Errorf("port %s not found in service", *s)
})
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsPort), err)
}

if probePort != nil {
//nolint:gosec
port, err := strconv.Atoi(*probePort)
if err != nil {
//not a integer
for _, item := range serviceManifest.Spec.Ports {
if strings.EqualFold(item.Name, *probePort) {
//found the port
properties.Port = pointer.Int32(item.NodePort)
}
}
} else {
// Not need to verify probePort is in correct range again.
for _, item := range serviceManifest.Spec.Ports {
//nolint:gosec
if item.Port == int32(port) {
//found the port
properties.Port = pointer.Int32(item.NodePort)
}
}
}
}

// Select request path
if strings.EqualFold(string(properties.Protocol), string(network.ProtocolHTTPS)) || strings.EqualFold(string(properties.Protocol), string(network.ProtocolHTTP)) {
// get request path ,only used with http/https probe
path, err := consts.GetHealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsRequestPath)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsRequestPath), err)
}
if path == nil {
if path, err = consts.GetAttributeValueInSvcAnnotation(serviceManifest.Annotations, consts.ServiceAnnotationLoadBalancerHealthProbeRequestPath); err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.ServiceAnnotationLoadBalancerHealthProbeRequestPath, err)
}
}
if path == nil {
path = pointer.String(consts.HealthProbeDefaultRequestPath)
}
properties.RequestPath = path
}
// get number of probes
var numOfProbeValidator = func(val *int32) error {
//minimum number of unhealthy responses is 2. ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
const (
MinimumNumOfProbe = 2
)
if *val < MinimumNumOfProbe {
return fmt.Errorf("the minimum value of %s is %d", consts.HealthProbeParamsNumOfProbe, MinimumNumOfProbe)
}
return nil
}
numberOfProbes, err := consts.GetInt32HealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsNumOfProbe, numOfProbeValidator)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsNumOfProbe), err)
}
if numberOfProbes == nil {
if numberOfProbes, err = consts.Getint32ValueFromK8sSvcAnnotation(serviceManifest.Annotations, consts.ServiceAnnotationLoadBalancerHealthProbeNumOfProbe, numOfProbeValidator); err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.ServiceAnnotationLoadBalancerHealthProbeNumOfProbe, err)
}
}

// if numberOfProbes is not set, set it to default instead ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
if numberOfProbes == nil {
numberOfProbes = pointer.Int32(consts.HealthProbeDefaultNumOfProbe)
}

// get probe interval in seconds
var probeIntervalValidator = func(val *int32) error {
//minimum probe interval in seconds is 5. ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
const (
MinimumProbeIntervalInSecond = 5
)
if *val < 5 {
return fmt.Errorf("the minimum value of %s is %d", consts.HealthProbeParamsProbeInterval, MinimumProbeIntervalInSecond)
}
return nil
}
probeInterval, err := consts.GetInt32HealthProbeConfigOfPortFromK8sSvcAnnotation(serviceManifest.Annotations, port.Port, consts.HealthProbeParamsProbeInterval, probeIntervalValidator)
if err != nil {
return nil, fmt.Errorf("failed to parse annotation %s:%w", consts.BuildHealthProbeAnnotationKeyForPort(port.Port, consts.HealthProbeParamsProbeInterval), err)
}
if probeInterval == nil {
if probeInterval, err = consts.Getint32ValueFromK8sSvcAnnotation(serviceManifest.Annotations, consts.ServiceAnnotationLoadBalancerHealthProbeInterval, probeIntervalValidator); err != nil {
return nil, fmt.Errorf("failed to parse annotation %s: %w", consts.ServiceAnnotationLoadBalancerHealthProbeInterval, err)
}
}
// if probeInterval is not set, set it to default instead ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
if probeInterval == nil {
probeInterval = pointer.Int32(consts.HealthProbeDefaultProbeInterval)
}

// total probe should be less than 120 seconds ref: https://docs.microsoft.com/en-us/rest/api/load-balancer/load-balancers/create-or-update#probe
if (*probeInterval)*(*numberOfProbes) >= 120 {
return nil, fmt.Errorf("total probe should be less than 120, please adjust interval and number of probe accordingly")
}
properties.IntervalInSeconds = probeInterval
properties.ProbeThreshold = numberOfProbes
probe := &network.Probe{
Name: &lbrule,
ProbePropertiesFormat: properties,
}
return probe, nil
}

// buildLBRules
// for following sku: basic loadbalancer vs standard load balancer
// for following scenario: internal vs external
Expand All @@ -2344,15 +2141,19 @@ func (az *Cloud) getExpectedLBRules(
if servicehelpers.NeedsHealthCheck(service) && !(consts.IsPLSEnabled(service.Annotations) && consts.IsPLSProxyProtocolEnabled(service.Annotations)) {
podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service)
lbRuleName := az.getLoadBalancerRuleName(service, v1.ProtocolTCP, podPresencePort)
probeInterval, numberOfProbes, err := az.getHealthProbeConfigProbeIntervalAndNumOfProbe(service, podPresencePort)
if err != nil {
return nil, nil, err
}

nodeEndpointHealthprobe = &network.Probe{
Name: &lbRuleName,
ProbePropertiesFormat: &network.ProbePropertiesFormat{
RequestPath: pointer.String(podPresencePath),
Protocol: network.ProbeProtocolHTTP,
Port: pointer.Int32(podPresencePort),
IntervalInSeconds: pointer.Int32(consts.HealthProbeDefaultProbeInterval),
ProbeThreshold: pointer.Int32(consts.HealthProbeDefaultNumOfProbe),
IntervalInSeconds: probeInterval,
ProbeThreshold: numberOfProbes,
},
}
expectedProbes = append(expectedProbes, *nodeEndpointHealthprobe)
Expand Down Expand Up @@ -3333,20 +3134,6 @@ func (az *Cloud) safeDeletePublicIP(service *v1.Service, pipResourceGroup string
return nil
}

func findProbe(probes []network.Probe, probe network.Probe) bool {
for _, existingProbe := range probes {
if strings.EqualFold(pointer.StringDeref(existingProbe.Name, ""), pointer.StringDeref(probe.Name, "")) &&
pointer.Int32Deref(existingProbe.Port, 0) == pointer.Int32Deref(probe.Port, 0) &&
strings.EqualFold(string(existingProbe.Protocol), string(probe.Protocol)) &&
strings.EqualFold(pointer.StringDeref(existingProbe.RequestPath, ""), pointer.StringDeref(probe.RequestPath, "")) &&
pointer.Int32Deref(existingProbe.IntervalInSeconds, 0) == pointer.Int32Deref(probe.IntervalInSeconds, 0) &&
pointer.Int32Deref(existingProbe.ProbeThreshold, 0) == pointer.Int32Deref(probe.ProbeThreshold, 0) {
return true
}
}
return false
}

func findRule(rules []network.LoadBalancingRule, rule network.LoadBalancingRule, wantLB bool) bool {
for _, existingRule := range rules {
if strings.EqualFold(pointer.StringDeref(existingRule.Name, ""), pointer.StringDeref(rule.Name, "")) &&
Expand Down

0 comments on commit 224eeca

Please sign in to comment.