From 34b4a5bbee78c8bec93208c5d889b81c798bcd78 Mon Sep 17 00:00:00 2001 From: "Bobby (Babak) Salamat" Date: Thu, 15 Nov 2018 11:02:14 -0800 Subject: [PATCH] Add watchdog for leader election logic getting wedged --- cmd/kube-scheduler/app/options/options.go | 2 ++ cmd/kube-scheduler/app/server.go | 14 ++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 0f4c7d0467810..15c79acd7c5ce 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -286,6 +286,8 @@ func makeLeaderElectionConfig(config kubeschedulerconfig.KubeSchedulerLeaderElec LeaseDuration: config.LeaseDuration.Duration, RenewDeadline: config.RenewDeadline.Duration, RetryPeriod: config.RetryPeriod.Duration, + WatchDog: leaderelection.NewLeaderHealthzAdaptor(time.Second * 20), + Name: "kube-scheduler", }, nil } diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go index 9f43680ddb688..615f9e8f18f76 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go @@ -200,10 +200,16 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error cc.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: cc.EventClient.Events("")}) } + // Setup healthz checks. + var checks []healthz.HealthzChecker + if cc.ComponentConfig.LeaderElection.LeaderElect { + checks = append(checks, cc.LeaderElection.WatchDog) + } + // Start up the healthz server. if cc.InsecureServing != nil { separateMetrics := cc.InsecureMetricsServing != nil - handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, separateMetrics), nil, nil) + handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, separateMetrics, checks...), nil, nil) if err := cc.InsecureServing.Serve(handler, 0, stopCh); err != nil { return fmt.Errorf("failed to start healthz server: %v", err) } @@ -215,7 +221,7 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error } } if cc.SecureServing != nil { - handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, false), cc.Authentication.Authenticator, cc.Authorization.Authorizer) + handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, false, checks...), cc.Authentication.Authenticator, cc.Authorization.Authorizer) if err := cc.SecureServing.Serve(handler, 0, stopCh); err != nil { // fail early for secure handlers, removing the old error loop from above return fmt.Errorf("failed to start healthz server: %v", err) @@ -313,9 +319,9 @@ func newMetricsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration) h // newHealthzServer creates a healthz server from the config, and will also // embed the metrics handler if the healthz and metrics address configurations // are the same. -func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, separateMetrics bool) http.Handler { +func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, separateMetrics bool, checks ...healthz.HealthzChecker) http.Handler { pathRecorderMux := mux.NewPathRecorderMux("kube-scheduler") - healthz.InstallHandler(pathRecorderMux) + healthz.InstallHandler(pathRecorderMux, checks...) if !separateMetrics { installMetricHandler(pathRecorderMux) }