Skip to content

Commit

Permalink
Refactor scheduler config API
Browse files Browse the repository at this point in the history
Refactor the kube-scheduler configuration API, command setup, and server
setup according to the guidelines established in kubernetes#32215 and using the
kube-proxy refactor (kubernetes#34727) as a model of a well factored component
adhering to said guidelines.

* Config API: clarify meaning and use of algorithm source by replacing
modality derived from bools and string emptiness checks with an explicit
AlgorithmSource type hierarchy.
* Config API: consolidate client connection config with common structs.
* Config API: split and simplify healthz/metrics server configuration.
* Config API: clarify leader election configuration.
* Config API: improve defaulting.
* CLI: deprecate all flags except `--config`.
* CLI: port all flags to new config API.
* CLI: refactor to match kube-proxy Cobra command style.
* Server: refactor away configurator.go to clarify application wiring.
* Server: refactor to more clearly separate wiring/setup from running.

Fixes kubernetes#52428.
  • Loading branch information
ironcladlou committed Nov 7, 2017
1 parent 25ca287 commit efb2bb7
Show file tree
Hide file tree
Showing 14 changed files with 1,072 additions and 730 deletions.
28 changes: 21 additions & 7 deletions cmd/hyperkube/kube-scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,38 @@ limitations under the License.
package main

import (
"flag"

"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
"k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options"
)

// NewScheduler creates a new hyperkube Server object that includes the
// description and flags.
func NewScheduler() *Server {
s := options.NewSchedulerServer()
healthz.DefaultHealthz()

command := app.NewSchedulerCommand()

hks := Server{
name: "scheduler",
AlternativeName: "kube-scheduler",
SimpleUsage: "scheduler",
Long: "Implements a Kubernetes scheduler. This will assign pods to kubelets based on capacity and constraints.",
Run: func(_ *Server, _ []string, stopCh <-chan struct{}) error {
return app.Run(s)
},
Long: command.Long,
}
s.AddFlags(hks.Flags())

serverFlags := hks.Flags()
serverFlags.AddFlagSet(command.Flags())

// FIXME this is here because hyperkube does its own flag parsing, and we need
// the command to know about the go flag set. Remove this once hyperkube is
// refactored to use cobra throughout.
command.Flags().AddGoFlagSet(flag.CommandLine)

hks.Run = func(_ *Server, args []string, stopCh <-chan struct{}) error {
command.SetArgs(args)
return command.Execute()
}

return &hks
}
6 changes: 3 additions & 3 deletions cmd/kubeadm/app/preflight/checks.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ import (
"k8s.io/kubernetes/pkg/util/initsystem"
versionutil "k8s.io/kubernetes/pkg/util/version"
kubeadmversion "k8s.io/kubernetes/pkg/version"
schoptions "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options"
schedulerapp "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
"k8s.io/kubernetes/test/e2e_node/system"
)

Expand Down Expand Up @@ -409,9 +409,9 @@ func (eac ExtraArgsCheck) Check() (warnings, errors []error) {
warnings = append(warnings, argsCheck("kube-controller-manager", eac.ControllerManagerExtraArgs, flags)...)
}
if len(eac.SchedulerExtraArgs) > 0 {
command := schedulerapp.NewSchedulerCommand()
flags := pflag.NewFlagSet("", pflag.ContinueOnError)
s := schoptions.NewSchedulerServer()
s.AddFlags(flags)
flags.AddFlagSet(command.Flags())
warnings = append(warnings, argsCheck("kube-scheduler", eac.SchedulerExtraArgs, flags)...)
}
return warnings, nil
Expand Down
115 changes: 82 additions & 33 deletions pkg/apis/componentconfig/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,58 +20,107 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// ClientConnectionConfiguration contains details for constructing a client.
type ClientConnectionConfiguration struct {
// kubeConfigFile is the path to a kubeconfig file.
KubeConfigFile string
// acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the
// default value of 'application/json'. This field will control all connections to the server used by a particular
// client.
AcceptContentTypes string
// contentType is the content type used when sending data to the server from this client.
ContentType string
// cps controls the number of queries per second allowed for this connection.
QPS float32
// burst allows extra queries to accumulate when a client is exceeding its rate.
Burst int
}

// SchedulerPolicyConfigMapKey defines the key of the element in the
// scheduler's policy ConfigMap that contains scheduler's policy config.
const SchedulerPolicyConfigMapKey string = "policy.cfg"

// SchedulerPolicySource configures a means to obtain a scheduler Policy. One
// source field must be specified, and source fields are mutually exclusive.
type SchedulerPolicySource struct {
// File is a file policy source.
File *SchedulerPolicyFileSource
// ConfigMap is a config map policy source.
ConfigMap *SchedulerPolicyConfigMapSource
}

// SchedulerPolicyFileSource is a policy serialized to disk and accessed via
// path.
type SchedulerPolicyFileSource struct {
// Path is the location of a serialized policy.
Path string
}

// SchedulerPolicyConfigMapSource is a policy serialized into a config map value
// under the SchedulerPolicyConfigMapKey key.
type SchedulerPolicyConfigMapSource struct {
// Namespace is the namespace of the policy config map.
Namespace string
// Name is the name of hte policy config map.
Name string
}

// SchedulerAlgorithmSource is the source of a scheduler algorithm. One source
// field must be specified, and source fields are mutually exclusive.
type SchedulerAlgorithmSource struct {
// Policy is a policy based algorithm source.
Policy *SchedulerPolicySource
// Provider is the name of a scheduling algorithm provider to use.
Provider *string
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type KubeSchedulerConfiguration struct {
metav1.TypeMeta

// port is the port that the scheduler's http service runs on.
Port int32
// address is the IP address to serve on.
Address string
// algorithmProvider is the scheduling algorithm provider to use.
AlgorithmProvider string
// policyConfigFile is the filepath to the scheduler policy configuration.
PolicyConfigFile string
// enableProfiling enables profiling via web interface.
EnableProfiling bool
// enableContentionProfiling enables lock contention profiling, if enableProfiling is true.
EnableContentionProfiling bool
// contentType is contentType of requests sent to apiserver.
ContentType string
// kubeAPIQPS is the QPS to use while talking with kubernetes apiserver.
KubeAPIQPS float32
// kubeAPIBurst is the QPS burst to use while talking with kubernetes apiserver.
KubeAPIBurst int32
// schedulerName is name of the scheduler, used to select which pods
// will be processed by this scheduler, based on pod's "spec.SchedulerName".
SchedulerName string
// AlgorithmSource specifies the scheduler algorithm source.
AlgorithmSource SchedulerAlgorithmSource
// RequiredDuringScheduling affinity is not symmetric, but there is an implicit PreferredDuringScheduling affinity rule
// corresponding to every RequiredDuringScheduling affinity rule.
// HardPodAffinitySymmetricWeight represents the weight of implicit PreferredDuringScheduling affinity rule, in the range 0-100.
HardPodAffinitySymmetricWeight int

// LeaderElection defines the configuration of leader election client.
LeaderElection KubeSchedulerLeaderElectionConfiguration

// ClientConnection specifies the kubeconfig file and client connection
// settings for the proxy server to use when communicating with the apiserver.
ClientConnection ClientConnectionConfiguration
// HealthzBindAddress is the IP address and port for the health check server to serve on,
// defaulting to 0.0.0.0:10251
HealthzBindAddress string
// MetricsBindAddress is the IP address and port for the metrics server to
// serve on, defaulting to 0.0.0.0:10251.
MetricsBindAddress string
// EnableProfiling enables profiling via web interface on /debug/pprof
// handler. Profiling handlers will be handled by metrics server.
EnableProfiling bool
// EnableContentionProfiling enables lock contention profiling, if
// EnableProfiling is true.
EnableContentionProfiling bool

// Indicate the "all topologies" set for empty topologyKey when it's used for PreferredDuringScheduling pod anti-affinity.
// DEPRECATED: This is no longer used.
FailureDomains string
// leaderElection defines the configuration of leader election client.
LeaderElection LeaderElectionConfiguration
}

// KubeSchedulerLeaderElectionConfiguration expands LeaderElectionConfiguration
// to include scheduler specific configuration.
type KubeSchedulerLeaderElectionConfiguration struct {
LeaderElectionConfiguration
// LockObjectNamespace defines the namespace of the lock object
LockObjectNamespace string
// LockObjectName defines the lock object name
LockObjectName string
// PolicyConfigMapName is the name of the ConfigMap object that specifies
// the scheduler's policy config. If UseLegacyPolicyConfig is true, scheduler
// uses PolicyConfigFile. If UseLegacyPolicyConfig is false and
// PolicyConfigMapName is not empty, the ConfigMap object with this name must
// exist in PolicyConfigMapNamespace before scheduler initialization.
PolicyConfigMapName string
// PolicyConfigMapNamespace is the namespace where the above policy config map
// is located. If none is provided default system namespace ("kube-system")
// will be used.
PolicyConfigMapNamespace string
// UseLegacyPolicyConfig tells the scheduler to ignore Policy ConfigMap and
// to use PolicyConfigFile if available.
UseLegacyPolicyConfig bool
}

// LeaderElectionConfiguration defines the configuration of leader election
Expand Down
72 changes: 48 additions & 24 deletions pkg/apis/componentconfig/v1alpha1/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ limitations under the License.
package v1alpha1

import (
"net"
"strconv"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -31,41 +33,63 @@ func addDefaultingFuncs(scheme *kruntime.Scheme) error {
}

func SetDefaults_KubeSchedulerConfiguration(obj *KubeSchedulerConfiguration) {
if obj.Port == 0 {
obj.Port = ports.SchedulerPort
if len(obj.SchedulerName) == 0 {
obj.SchedulerName = api.DefaultSchedulerName
}
if obj.Address == "" {
obj.Address = "0.0.0.0"

if obj.HardPodAffinitySymmetricWeight == 0 {
obj.HardPodAffinitySymmetricWeight = api.DefaultHardPodAffinitySymmetricWeight
}
if obj.AlgorithmProvider == "" {
obj.AlgorithmProvider = "DefaultProvider"

if obj.AlgorithmSource.Policy == nil &&
(obj.AlgorithmSource.Provider == nil || len(*obj.AlgorithmSource.Provider) == 0) {
val := SchedulerDefaultProviderName
obj.AlgorithmSource.Provider = &val
}
if obj.ContentType == "" {
obj.ContentType = "application/vnd.kubernetes.protobuf"

if policy := obj.AlgorithmSource.Policy; policy != nil {
if policy.ConfigMap != nil && len(policy.ConfigMap.Namespace) == 0 {
obj.AlgorithmSource.Policy.ConfigMap.Namespace = api.NamespaceSystem
}
}
if obj.KubeAPIQPS == 0 {
obj.KubeAPIQPS = 50.0

if host, port, err := net.SplitHostPort(obj.HealthzBindAddress); err == nil {
if len(host) == 0 {
host = "0.0.0.0"
}
obj.HealthzBindAddress = net.JoinHostPort(host, port)
} else {
obj.HealthzBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.SchedulerPort))
}
if obj.KubeAPIBurst == 0 {
obj.KubeAPIBurst = 100

if host, port, err := net.SplitHostPort(obj.MetricsBindAddress); err == nil {
if len(host) == 0 {
host = "0.0.0.0"
}
obj.MetricsBindAddress = net.JoinHostPort(host, port)
} else {
obj.MetricsBindAddress = net.JoinHostPort("0.0.0.0", strconv.Itoa(ports.SchedulerPort))
}
if obj.SchedulerName == "" {
obj.SchedulerName = api.DefaultSchedulerName

if len(obj.ClientConnection.ContentType) == 0 {
obj.ClientConnection.ContentType = "application/vnd.kubernetes.protobuf"
}
if obj.HardPodAffinitySymmetricWeight == 0 {
obj.HardPodAffinitySymmetricWeight = api.DefaultHardPodAffinitySymmetricWeight
if obj.ClientConnection.QPS == 0.0 {
obj.ClientConnection.QPS = 50.0
}
if obj.FailureDomains == "" {
obj.FailureDomains = kubeletapis.DefaultFailureDomains
if obj.ClientConnection.Burst == 0 {
obj.ClientConnection.Burst = 100
}
if obj.LockObjectNamespace == "" {
obj.LockObjectNamespace = SchedulerDefaultLockObjectNamespace

if len(obj.LeaderElection.LockObjectNamespace) == 0 {
obj.LeaderElection.LockObjectNamespace = SchedulerDefaultLockObjectNamespace
}
if obj.LockObjectName == "" {
obj.LockObjectName = SchedulerDefaultLockObjectName
if len(obj.LeaderElection.LockObjectName) == 0 {
obj.LeaderElection.LockObjectName = SchedulerDefaultLockObjectName
}
if obj.PolicyConfigMapNamespace == "" {
obj.PolicyConfigMapNamespace = api.NamespaceSystem

if len(obj.FailureDomains) == 0 {
obj.FailureDomains = kubeletapis.DefaultFailureDomains
}
}

Expand Down
Loading

0 comments on commit efb2bb7

Please sign in to comment.