Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support local service in multiple standard load balancer mode #4450

Merged
merged 2 commits into from
Aug 19, 2023
Merged

feat: support local service in multiple standard load balancer mode #4450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9f9d85c
feat: support local service in multiple standard load balancer mode
nilo19 Aug 3, 2023
06108f4
fix: change endpoint slice cache key to add namespace
nilo19 Aug 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 11 additions & 4 deletions pkg/consts/consts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,11 @@ const (

// IP family variables
const (
IPVersionIPv6 bool = true
IPVersionIPv4 bool = false
IPVersionIPv6 bool = true
IPVersionIPv4 bool = false
IPVersionIPv4String string = "IPv4"
IPVersionIPv6String string = "IPv6"
IPVersionDualStackString string = "DualStack"
)

// LB variables for dual-stack
Expand Down Expand Up @@ -428,8 +431,8 @@ const (
RouteNameFmt = "%s____%s"
RouteNameSeparator = "____"

// routeUpdateInterval defines the route reconciling interval.
RouteUpdateInterval = 30 * time.Second
// DefaultRouteUpdateIntervalInSeconds defines the route reconciling interval.
DefaultRouteUpdateIntervalInSeconds = 30
)

// cloud provider config secret
Expand Down Expand Up @@ -541,4 +544,8 @@ type LoadBalancerBackendPoolUpdateOperation string
const (
LoadBalancerBackendPoolUpdateOperationAdd LoadBalancerBackendPoolUpdateOperation = "add"
LoadBalancerBackendPoolUpdateOperationRemove LoadBalancerBackendPoolUpdateOperation = "remove"

DefaultLoadBalancerBackendPoolUpdateIntervalInSeconds = 30

ServiceNameLabel = "kubernetes.io/service-name"
)
48 changes: 35 additions & 13 deletions pkg/provider/azure.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,11 @@ type Config struct {

// DisableAPICallCache disables the cache for Azure API calls. It is for ARG support and not all resources will be disabled.
DisableAPICallCache bool `json:"disableAPICallCache,omitempty" yaml:"disableAPICallCache,omitempty"`

// RouteUpdateIntervalInSeconds is the interval for updating routes. Default is 30 seconds.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks making those intervals configurable.

RouteUpdateIntervalInSeconds int `json:"routeUpdateIntervalInSeconds,omitempty" yaml:"routeUpdateIntervalInSeconds,omitempty"`
// LoadBalancerBackendPoolUpdateIntervalInSeconds is the interval for updating load balancer backend pool of local services. Default is 30 seconds.
LoadBalancerBackendPoolUpdateIntervalInSeconds int `json:"loadBalancerBackendPoolUpdateIntervalInSeconds,omitempty" yaml:"loadBalancerBackendPoolUpdateIntervalInSeconds,omitempty"`
}

// MultipleStandardLoadBalancerConfiguration stores the properties regarding multiple standard load balancers.
Expand Down Expand Up @@ -402,10 +407,11 @@ type Cloud struct {
regionZonesMap map[string][]string
refreshZonesLock sync.RWMutex

KubeClient clientset.Interface
eventBroadcaster record.EventBroadcaster
eventRecorder record.EventRecorder
routeUpdater batchProcessor
KubeClient clientset.Interface
eventBroadcaster record.EventBroadcaster
eventRecorder record.EventRecorder
routeUpdater batchProcessor
backendPoolUpdater batchProcessor

vmCache azcache.Resource
lbCache azcache.Resource
Expand All @@ -430,10 +436,11 @@ type Cloud struct {
// runs only once every time the cloud provide restarts.
multipleStandardLoadBalancerConfigurationsSynced bool
// nodesWithCorrectLoadBalancerByPrimaryVMSet marks nodes that are matched with load balancers by primary vmSet.
nodesWithCorrectLoadBalancerByPrimaryVMSet sync.Map

nodesWithCorrectLoadBalancerByPrimaryVMSet sync.Map
multipleStandardLoadBalancersActiveServicesLock sync.Mutex
multipleStandardLoadBalancersActiveNodesLock sync.Mutex
localServiceNameToServiceInfoMap sync.Map
endpointSlicesCache sync.Map
}

// NewCloud returns a Cloud with initialized clients
Expand Down Expand Up @@ -609,12 +616,6 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *Config,
}
}

if az.useMultipleStandardLoadBalancers() {
if err := az.checkEnableMultipleStandardLoadBalancers(); err != nil {
return err
}
}

env, err := ratelimitconfig.ParseAzureEnvironment(config.Cloud, config.ResourceManagerEndpoint, config.IdentitySystem)
if err != nil {
return err
Expand Down Expand Up @@ -698,6 +699,12 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *Config,
az.LoadBalancerBackendPool = newBackendPoolTypeNodeIP(az)
}

if az.useMultipleStandardLoadBalancers() {
if err := az.checkEnableMultipleStandardLoadBalancers(); err != nil {
return err
}
}

err = az.initCaches()
if err != nil {
return err
Expand All @@ -710,9 +717,18 @@ func (az *Cloud) InitializeCloudFromConfig(ctx context.Context, config *Config,
// updating routes and syncing zones only in CCM
if callFromCCM {
// start delayed route updater.
az.routeUpdater = newDelayedRouteUpdater(az, consts.RouteUpdateInterval)
if az.RouteUpdateIntervalInSeconds == 0 {
az.RouteUpdateIntervalInSeconds = consts.DefaultRouteUpdateIntervalInSeconds
}
az.routeUpdater = newDelayedRouteUpdater(az, time.Duration(az.RouteUpdateIntervalInSeconds)*time.Second)
go az.routeUpdater.run(ctx)

// start backend pool updater.
if az.useMultipleStandardLoadBalancers() {
az.backendPoolUpdater = newLoadBalancerBackendPoolUpdater(az, time.Duration(az.LoadBalancerBackendPoolUpdateIntervalInSeconds)*time.Second)
go az.backendPoolUpdater.run(ctx)
}

// Azure Stack does not support zone at the moment
// https://docs.microsoft.com/en-us/azure-stack/user/azure-stack-network-differences?view=azs-2102
if !az.isStackCloud() {
Expand Down Expand Up @@ -761,6 +777,10 @@ func (az *Cloud) checkEnableMultipleStandardLoadBalancers() error {
primaryVMSets.Insert(multiSLBConfig.PrimaryVMSet)
}

if az.LoadBalancerBackendPoolUpdateIntervalInSeconds == 0 {
az.LoadBalancerBackendPoolUpdateIntervalInSeconds = consts.DefaultLoadBalancerBackendPoolUpdateIntervalInSeconds
}

return nil
}

Expand Down Expand Up @@ -1180,6 +1200,8 @@ func (az *Cloud) SetInformers(informerFactory informers.SharedInformerFactory) {
az.nodeInformerSynced = nodeInformer.HasSynced

az.serviceLister = informerFactory.Core().V1().Services().Lister()

az.setUpEndpointSlicesInformer(informerFactory)
}

// updateNodeCaches updates local cache for node's zones and external resource groups.
Expand Down