Refactor scheduler config API

Refactor the kube-scheduler configuration API, command setup, and server
setup according to the guidelines established in kubernetes#32215 and using the
kube-proxy refactor (kubernetes#34727) as a model of a well factored component
adhering to said guidelines.

* Config API: clarify meaning and use of algorithm source by replacing
modality derived from bools and string emptiness checks with an explicit
AlgorithmSource type hierarchy.
* Config API: consolidate client connection config with common structs.
* Config API: split and simplify healthz/metrics server configuration.
* Config API: clarify leader election configuration.
* Config API: improve defaulting.
* CLI: deprecate all flags except `--config`.
* CLI: port all flags to new config API.
* CLI: refactor to match kube-proxy Cobra command style.
* Server: refactor away configurator.go to clarify application wiring.
* Server: refactor to more clearly separate wiring/setup from running.

Fixes kubernetes#52428.

cmd/hyperkube/kube-scheduler.go

@@ -17,24 +17,38 @@ limitations under the License.
 package main
 
 import (
+	"flag"
+
+	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
-	"k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options"
 )
 
 // NewScheduler creates a new hyperkube Server object that includes the
 // description and flags.
 func NewScheduler() *Server {
-	s := options.NewSchedulerServer()
+	healthz.DefaultHealthz()
+
+	command := app.NewSchedulerCommand()
 
 	hks := Server{
 		name:            "scheduler",
 		AlternativeName: "kube-scheduler",
 		SimpleUsage:     "scheduler",
-		Long:            "Implements a Kubernetes scheduler.  This will assign pods to kubelets based on capacity and constraints.",
-		Run: func(_ *Server, _ []string, stopCh <-chan struct{}) error {
-			return app.Run(s)
-		},
+		Long:            command.Long,
 	}
-	s.AddFlags(hks.Flags())
+
+	serverFlags := hks.Flags()
+	serverFlags.AddFlagSet(command.Flags())
+
+	// FIXME this is here because hyperkube does its own flag parsing, and we need
+	// the command to know about the go flag set. Remove this once hyperkube is
+	// refactored to use cobra throughout.
+	command.Flags().AddGoFlagSet(flag.CommandLine)
+
+	hks.Run = func(_ *Server, args []string, stopCh <-chan struct{}) error {
+		command.SetArgs(args)
+		return command.Execute()
+	}
+
 	return &hks
 }

cmd/kubeadm/app/preflight/checks.go

@@ -52,7 +52,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/initsystem"
 	versionutil "k8s.io/kubernetes/pkg/util/version"
 	kubeadmversion "k8s.io/kubernetes/pkg/version"
-	schoptions "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options"
+	schedulerapp "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
 	"k8s.io/kubernetes/test/e2e_node/system"
 )
 
@@ -409,9 +409,9 @@ func (eac ExtraArgsCheck) Check() (warnings, errors []error) {
 		warnings = append(warnings, argsCheck("kube-controller-manager", eac.ControllerManagerExtraArgs, flags)...)
 	}
 	if len(eac.SchedulerExtraArgs) > 0 {
+		command := schedulerapp.NewSchedulerCommand()
 		flags := pflag.NewFlagSet("", pflag.ContinueOnError)
-		s := schoptions.NewSchedulerServer()
-		s.AddFlags(flags)
+		flags.AddFlagSet(command.Flags())
 		warnings = append(warnings, argsCheck("kube-scheduler", eac.SchedulerExtraArgs, flags)...)
 	}
 	return warnings, nil

pkg/apis/componentconfig/types.go

@@ -20,58 +20,107 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+// ClientConnectionConfiguration contains details for constructing a client.
+type ClientConnectionConfiguration struct {
+	// kubeConfigFile is the path to a kubeconfig file.
+	KubeConfigFile string
+	// acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the
+	// default value of 'application/json'. This field will control all connections to the server used by a particular
+	// client.
+	AcceptContentTypes string
+	// contentType is the content type used when sending data to the server from this client.
+	ContentType string
+	// cps controls the number of queries per second allowed for this connection.
+	QPS float32
+	// burst allows extra queries to accumulate when a client is exceeding its rate.
+	Burst int
+}
+
+// SchedulerPolicyConfigMapKey defines the key of the element in the
+// scheduler's policy ConfigMap that contains scheduler's policy config.
+const SchedulerPolicyConfigMapKey string = "policy.cfg"
+
+// SchedulerPolicySource configures a means to obtain a scheduler Policy. One
+// source field must be specified, and source fields are mutually exclusive.
+type SchedulerPolicySource struct {
+	// File is a file policy source.
+	File *SchedulerPolicyFileSource
+	// ConfigMap is a config map policy source.
+	ConfigMap *SchedulerPolicyConfigMapSource
+}
+
+// SchedulerPolicyFileSource is a policy serialized to disk and accessed via
+// path.
+type SchedulerPolicyFileSource struct {
+	// Path is the location of a serialized policy.
+	Path string
+}
+
+// SchedulerPolicyConfigMapSource is a policy serialized into a config map value
+// under the SchedulerPolicyConfigMapKey key.
+type SchedulerPolicyConfigMapSource struct {
+	// Namespace is the namespace of the policy config map.
+	Namespace string
+	// Name is the name of hte policy config map.
+	Name string
+}
+
+// SchedulerAlgorithmSource is the source of a scheduler algorithm. One source
+// field must be specified, and source fields are mutually exclusive.
+type SchedulerAlgorithmSource struct {
+	// Policy is a policy based algorithm source.
+	Policy *SchedulerPolicySource
+	// Provider is the name of a scheduling algorithm provider to use.
+	Provider *string
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 type KubeSchedulerConfiguration struct {
 	metav1.TypeMeta
 
-	// port is the port that the scheduler's http service runs on.
-	Port int32
-	// address is the IP address to serve on.
-	Address string
-	// algorithmProvider is the scheduling algorithm provider to use.
-	AlgorithmProvider string
-	// policyConfigFile is the filepath to the scheduler policy configuration.
-	PolicyConfigFile string
-	// enableProfiling enables profiling via web interface.
-	EnableProfiling bool
-	// enableContentionProfiling enables lock contention profiling, if enableProfiling is true.
-	EnableContentionProfiling bool
-	// contentType is contentType of requests sent to apiserver.
-	ContentType string
-	// kubeAPIQPS is the QPS to use while talking with kubernetes apiserver.
-	KubeAPIQPS float32
-	// kubeAPIBurst is the QPS burst to use while talking with kubernetes apiserver.
-	KubeAPIBurst int32
 	// schedulerName is name of the scheduler, used to select which pods
 	// will be processed by this scheduler, based on pod's "spec.SchedulerName".
 	SchedulerName string
+	// AlgorithmSource specifies the scheduler algorithm source.
+	AlgorithmSource SchedulerAlgorithmSource
 	// RequiredDuringScheduling affinity is not symmetric, but there is an implicit PreferredDuringScheduling affinity rule
 	// corresponding to every RequiredDuringScheduling affinity rule.
 	// HardPodAffinitySymmetricWeight represents the weight of implicit PreferredDuringScheduling affinity rule, in the range 0-100.
 	HardPodAffinitySymmetricWeight int
+
+	// LeaderElection defines the configuration of leader election client.
+	LeaderElection KubeSchedulerLeaderElectionConfiguration
+
+	// ClientConnection specifies the kubeconfig file and client connection
+	// settings for the proxy server to use when communicating with the apiserver.
+	ClientConnection ClientConnectionConfiguration
+	// HealthzBindAddress is the IP address and port for the health check server to serve on,
+	// defaulting to 0.0.0.0:10251
+	HealthzBindAddress string
+	// MetricsBindAddress is the IP address and port for the metrics server to
+	// serve on, defaulting to 0.0.0.0:10251.
+	MetricsBindAddress string
+	// EnableProfiling enables profiling via web interface on /debug/pprof
+	// handler. Profiling handlers will be handled by metrics server.
+	EnableProfiling bool
+	// EnableContentionProfiling enables lock contention profiling, if
+	// EnableProfiling is true.
+	EnableContentionProfiling bool
+
 	// Indicate the "all topologies" set for empty topologyKey when it's used for PreferredDuringScheduling pod anti-affinity.
 	// DEPRECATED: This is no longer used.
 	FailureDomains string
-	// leaderElection defines the configuration of leader election client.
-	LeaderElection LeaderElectionConfiguration
+}
+
+// KubeSchedulerLeaderElectionConfiguration expands LeaderElectionConfiguration
+// to include scheduler specific configuration.
+type KubeSchedulerLeaderElectionConfiguration struct {
+	LeaderElectionConfiguration
 	// LockObjectNamespace defines the namespace of the lock object
 	LockObjectNamespace string
 	// LockObjectName defines the lock object name
 	LockObjectName string
-	// PolicyConfigMapName is the name of the ConfigMap object that specifies
-	// the scheduler's policy config. If UseLegacyPolicyConfig is true, scheduler
-	// uses PolicyConfigFile. If UseLegacyPolicyConfig is false and
-	// PolicyConfigMapName is not empty, the ConfigMap object with this name must
-	// exist in PolicyConfigMapNamespace before scheduler initialization.
-	PolicyConfigMapName string
-	// PolicyConfigMapNamespace is the namespace where the above policy config map
-	// is located. If none is provided default system namespace ("kube-system")
-	// will be used.
-	PolicyConfigMapNamespace string
-	// UseLegacyPolicyConfig tells the scheduler to ignore Policy ConfigMap and
-	// to use PolicyConfigFile if available.
-	UseLegacyPolicyConfig bool
 }
 
 // LeaderElectionConfiguration defines the configuration of leader election

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"net"
+	"strconv"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,41 +33,63 @@ func addDefaultingFuncs(scheme *kruntime.Scheme) error {
 }
 
 func SetDefaults_KubeSchedulerConfiguration(obj *KubeSchedulerConfiguration) {
-	if obj.Port == 0 {
-		obj.Port = ports.SchedulerPort
+	if len(obj.SchedulerName) == 0 {
+		obj.SchedulerName = api.DefaultSchedulerName
 	}
-	if obj.Address == "" {
-		obj.Address = "0.0.0.0"
+
+	if obj.HardPodAffinitySymmetricWeight == 0 {
+		obj.HardPodAffinitySymmetricWeight = api.DefaultHardPodAffinitySymmetricWeight
 	}
-	if obj.AlgorithmProvider == "" {
-		obj.AlgorithmProvider = "DefaultProvider"
+
+	if obj.AlgorithmSource.Policy == nil &&
+		(obj.AlgorithmSource.Provider == nil || len(*obj.AlgorithmSource.Provider) == 0) {
+		val := SchedulerDefaultProviderName
+		obj.AlgorithmSource.Provider = &val
 	}
-	if obj.ContentType == "" {
-		obj.ContentType = "application/vnd.kubernetes.protobuf"
+
+	if policy := obj.AlgorithmSource.Policy; policy != nil {
+		if policy.ConfigMap != nil && len(policy.ConfigMap.Namespace) == 0 {
+			obj.AlgorithmSource.Policy.ConfigMap.Namespace = api.NamespaceSystem
+		}
 	}
-	if obj.KubeAPIQPS == 0 {
-		obj.KubeAPIQPS = 50.0
+
+	if host, port, err := net.SplitHostPort(obj.HealthzBindAddress); err == nil {
+		if len(host) == 0 {
+			host = "0.0.0.0"
+		}
+		obj.HealthzBindAddress = net.JoinHostPort(host, port)
+	} else {
+		obj.HealthzBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.SchedulerPort))
 	}
-	if obj.KubeAPIBurst == 0 {
-		obj.KubeAPIBurst = 100
+
+	if host, port, err := net.SplitHostPort(obj.MetricsBindAddress); err == nil {
+		if len(host) == 0 {
+			host = "0.0.0.0"
+		}
+		obj.MetricsBindAddress = net.JoinHostPort(host, port)
+	} else {
+		obj.MetricsBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.SchedulerPort))
 	}
-	if obj.SchedulerName == "" {
-		obj.SchedulerName = api.DefaultSchedulerName
+
+	if len(obj.ClientConnection.ContentType) == 0 {
+		obj.ClientConnection.ContentType = "application/vnd.kubernetes.protobuf"
 	}
-	if obj.HardPodAffinitySymmetricWeight == 0 {
-		obj.HardPodAffinitySymmetricWeight = api.DefaultHardPodAffinitySymmetricWeight
+	if obj.ClientConnection.QPS == 0.0 {
+		obj.ClientConnection.QPS = 50.0
 	}
-	if obj.FailureDomains == "" {
-		obj.FailureDomains = kubeletapis.DefaultFailureDomains
+	if obj.ClientConnection.Burst == 0 {
+		obj.ClientConnection.Burst = 100
 	}
-	if obj.LockObjectNamespace == "" {
-		obj.LockObjectNamespace = SchedulerDefaultLockObjectNamespace
+
+	if len(obj.LeaderElection.LockObjectNamespace) == 0 {
+		obj.LeaderElection.LockObjectNamespace = SchedulerDefaultLockObjectNamespace
 	}
-	if obj.LockObjectName == "" {
-		obj.LockObjectName = SchedulerDefaultLockObjectName
+	if len(obj.LeaderElection.LockObjectName) == 0 {
+		obj.LeaderElection.LockObjectName = SchedulerDefaultLockObjectName
 	}
-	if obj.PolicyConfigMapNamespace == "" {
-		obj.PolicyConfigMapNamespace = api.NamespaceSystem
+
+	if len(obj.FailureDomains) == 0 {
+		obj.FailureDomains = kubeletapis.DefaultFailureDomains
 	}
 }