Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Start informers after leader election #115754

Merged
merged 1 commit into from
May 15, 2023
Merged

Start informers after leader election #115754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
27 changes: 18 additions & 9 deletions cmd/kube-scheduler/app/options/options_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,8 @@ profiles:
TypeMeta: metav1.TypeMeta{
APIVersion: v1.SchemeGroupVersion.String(),
},
Parallelism: 16,
Parallelism: 16,
DelayCacheUntilActive: false,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
Expand Down Expand Up @@ -469,7 +470,8 @@ profiles:
TypeMeta: metav1.TypeMeta{
APIVersion: v1beta3.SchemeGroupVersion.String(),
},
Parallelism: 16,
Parallelism: 16,
DelayCacheUntilActive: false,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
Expand Down Expand Up @@ -571,7 +573,8 @@ profiles:
TypeMeta: metav1.TypeMeta{
APIVersion: v1.SchemeGroupVersion.String(),
},
Parallelism: 16,
Parallelism: 16,
DelayCacheUntilActive: false,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
Expand Down Expand Up @@ -641,7 +644,8 @@ profiles:
TypeMeta: metav1.TypeMeta{
APIVersion: v1.SchemeGroupVersion.String(),
},
Parallelism: 16,
Parallelism: 16,
DelayCacheUntilActive: false,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
Expand Down Expand Up @@ -685,7 +689,8 @@ profiles:
TypeMeta: metav1.TypeMeta{
APIVersion: v1.SchemeGroupVersion.String(),
},
Parallelism: 16,
Parallelism: 16,
DelayCacheUntilActive: false,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
Expand Down Expand Up @@ -804,7 +809,8 @@ profiles:
TypeMeta: metav1.TypeMeta{
APIVersion: v1beta3.SchemeGroupVersion.String(),
},
Parallelism: 16,
Parallelism: 16,
DelayCacheUntilActive: false,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
Expand Down Expand Up @@ -926,7 +932,8 @@ profiles:
TypeMeta: metav1.TypeMeta{
APIVersion: v1.SchemeGroupVersion.String(),
},
Parallelism: 16,
Parallelism: 16,
DelayCacheUntilActive: false,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
Expand Down Expand Up @@ -1040,7 +1047,8 @@ profiles:
TypeMeta: metav1.TypeMeta{
APIVersion: v1beta3.SchemeGroupVersion.String(),
},
Parallelism: 16,
Parallelism: 16,
DelayCacheUntilActive: false,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
Expand Down Expand Up @@ -1179,7 +1187,8 @@ profiles:
TypeMeta: metav1.TypeMeta{
APIVersion: v1.SchemeGroupVersion.String(),
},
Parallelism: 16,
Parallelism: 16,
DelayCacheUntilActive: false,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
Expand Down
32 changes: 21 additions & 11 deletions cmd/kube-scheduler/app/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,25 +194,35 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched *
}
}

// Start all informers.
cc.InformerFactory.Start(ctx.Done())
// DynInformerFactory can be nil in tests.
if cc.DynInformerFactory != nil {
cc.DynInformerFactory.Start(ctx.Done())
}
startInformersAndWaitForSync := func(ctx context.Context) {
// Start all informers.
cc.InformerFactory.Start(ctx.Done())
// DynInformerFactory can be nil in tests.
if cc.DynInformerFactory != nil {
cc.DynInformerFactory.Start(ctx.Done())
}

// Wait for all caches to sync before scheduling.
cc.InformerFactory.WaitForCacheSync(ctx.Done())
// DynInformerFactory can be nil in tests.
if cc.DynInformerFactory != nil {
cc.DynInformerFactory.WaitForCacheSync(ctx.Done())
// Wait for all caches to sync before scheduling.
cc.InformerFactory.WaitForCacheSync(ctx.Done())
// DynInformerFactory can be nil in tests.
if cc.DynInformerFactory != nil {
cc.DynInformerFactory.WaitForCacheSync(ctx.Done())
}
}
if !cc.ComponentConfig.DelayCacheUntilActive || cc.LeaderElection == nil {
startInformersAndWaitForSync(ctx)
}

// If leader election is enabled, runCommand via LeaderElector until done and exit.
if cc.LeaderElection != nil {
cc.LeaderElection.Callbacks = leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
close(waitingForLeader)
if cc.ComponentConfig.DelayCacheUntilActive {
logger.Info("Starting informers and waiting for sync...")
startInformersAndWaitForSync(ctx)
liggitt marked this conversation as resolved.
Show resolved Hide resolved
logger.Info("Sync completed")
}
sched.Run(ctx)
},
OnStoppedLeading: func() {
Expand Down
7 changes: 7 additions & 0 deletions pkg/generated/openapi/zz_generated.openapi.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 8 additions & 3 deletions pkg/scheduler/apis/config/scheme/scheme_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -999,7 +999,8 @@ profiles:
name: "v1beta3 in-tree and out-of-tree plugins from internal",
version: v1beta3.SchemeGroupVersion,
obj: &config.KubeSchedulerConfiguration{
Parallelism: 8,
Parallelism: 8,
DelayCacheUntilActive: true,
Profiles: []config.KubeSchedulerProfile{
{
PluginConfig: []config.PluginConfig{
Expand Down Expand Up @@ -1219,7 +1220,8 @@ profiles:
name: "v1 in-tree and out-of-tree plugins from internal",
version: v1.SchemeGroupVersion,
obj: &config.KubeSchedulerConfiguration{
Parallelism: 8,
Parallelism: 8,
DelayCacheUntilActive: true,
Profiles: []config.KubeSchedulerProfile{
{
PluginConfig: []config.PluginConfig{
Expand Down Expand Up @@ -1265,6 +1267,7 @@ clientConnection:
contentType: ""
kubeconfig: ""
qps: 0
delayCacheUntilActive: true
enableContentionProfiling: false
enableProfiling: false
kind: KubeSchedulerConfiguration
Expand Down Expand Up @@ -1315,7 +1318,8 @@ profiles:
name: "v1 ignorePreferredTermsOfExistingPods is enabled",
version: v1.SchemeGroupVersion,
obj: &config.KubeSchedulerConfiguration{
Parallelism: 8,
Parallelism: 8,
DelayCacheUntilActive: true,
Profiles: []config.KubeSchedulerProfile{
{
PluginConfig: []config.PluginConfig{
Expand All @@ -1337,6 +1341,7 @@ clientConnection:
contentType: ""
kubeconfig: ""
qps: 0
delayCacheUntilActive: true
enableContentionProfiling: false
enableProfiling: false
kind: KubeSchedulerConfiguration
Expand Down
6 changes: 6 additions & 0 deletions pkg/scheduler/apis/config/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,12 @@ type KubeSchedulerConfiguration struct {
// Extenders are the list of scheduler extenders, each holding the values of how to communicate
// with the extender. These extenders are shared by all scheduler profiles.
Extenders []Extender

// DelayCacheUntilActive specifies when to start caching. If this is true and leader election is enabled,
// the scheduler will wait to fill informer caches until it is the leader. Doing so will have slower
// failover with the benefit of lower memory overhead while waiting to become leader.
// Defaults to false.
DelayCacheUntilActive bool
}

// KubeSchedulerProfile is a scheduling profile.
Expand Down
38 changes: 38 additions & 0 deletions pkg/scheduler/apis/config/v1/defaults_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,44 @@ func TestSchedulerDefaults(t *testing.T) {
},
},
},
{
name: "set non default delayCacheUntilActive",
config: &configv1.KubeSchedulerConfiguration{
DelayCacheUntilActive: true,
},
expected: &configv1.KubeSchedulerConfiguration{
Parallelism: pointer.Int32(16),
DelayCacheUntilActive: true,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: &enable,
EnableContentionProfiling: &enable,
},
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: pointer.Bool(true),
LeaseDuration: metav1.Duration{Duration: 15 * time.Second},
RenewDeadline: metav1.Duration{Duration: 10 * time.Second},
RetryPeriod: metav1.Duration{Duration: 2 * time.Second},
ResourceLock: "leases",
ResourceNamespace: "kube-system",
ResourceName: "kube-scheduler",
},
ClientConnection: componentbaseconfig.ClientConnectionConfiguration{
QPS: 50,
Burst: 100,
ContentType: "application/vnd.kubernetes.protobuf",
},
PercentageOfNodesToScore: pointer.Int32(config.DefaultPercentageOfNodesToScore),
PodInitialBackoffSeconds: pointer.Int64(1),
PodMaxBackoffSeconds: pointer.Int64(10),
Profiles: []configv1.KubeSchedulerProfile{
{
Plugins: getDefaultPlugins(),
PluginConfig: pluginConfigs,
SchedulerName: pointer.String("default-scheduler"),
},
},
},
},
{
name: "set non default global percentageOfNodesToScore",
config: &configv1.KubeSchedulerConfiguration{
Expand Down
2 changes: 2 additions & 0 deletions pkg/scheduler/apis/config/v1/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions staging/src/k8s.io/kube-scheduler/config/v1/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,12 @@ type KubeSchedulerConfiguration struct {
// with the extender. These extenders are shared by all scheduler profiles.
// +listType=set
Extenders []Extender `json:"extenders,omitempty"`

// DelayCacheUntilActive specifies when to start caching. If this is true and leader election is enabled,
// the scheduler will wait to fill informer caches until it is the leader. Doing so will have slower
// failover with the benefit of lower memory overhead while waiting to become leader.
// Defaults to false.
DelayCacheUntilActive bool `json:"delayCacheUntilActive,omitempty"`
}

// DecodeNestedObjects decodes plugin args for known types.
Expand Down