Support additional kube-scheduler config parameters via config file

hack/.packages

@@ -82,6 +82,7 @@ k8s.io/kops/pkg/client/simple/api
 k8s.io/kops/pkg/client/simple/vfsclientset
 k8s.io/kops/pkg/cloudinstances
 k8s.io/kops/pkg/commands
+k8s.io/kops/pkg/configbuilder
 k8s.io/kops/pkg/diff
 k8s.io/kops/pkg/dns
 k8s.io/kops/pkg/drain

k8s/crds/kops.k8s.io_clusters.yaml

@@ -1609,6 +1609,11 @@ spec:
             kubeScheduler:
               description: KubeSchedulerConfig is the configuration for the kube-scheduler
               properties:
+                burst:
+                  description: Burst sets the maximum qps to send to apiserver after
+                    the burst quota is exhausted
+                  format: int32
+                  type: integer
                 featureGates:
                   additionalProperties:
                     type: string
@@ -1677,6 +1682,10 @@ spec:
                     and the cloud provider as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/'
                   format: int32
                   type: integer
+                qps:
+                  description: Qps sets the maximum qps to send to apiserver after
+                    the burst quota is exhausted
+                  type: string
                 usePolicyConfigMap:
                   description: UsePolicyConfigMap enable setting the scheduler policy
                     from a configmap

nodeup/pkg/model/BUILD.bazel

@@ -46,6 +46,7 @@ go_library(
         "//pkg/apis/kops/util:go_default_library",
         "//pkg/apis/nodeup:go_default_library",
         "//pkg/assets:go_default_library",
+        "//pkg/configbuilder:go_default_library",
         "//pkg/dns:go_default_library",
         "//pkg/flagbuilder:go_default_library",
         "//pkg/k8scodecs:go_default_library",
@@ -88,6 +89,7 @@ go_test(
         "docker_test.go",
         "kube_apiserver_test.go",
         "kube_proxy_test.go",
+        "kube_scheduler_test.go",
         "kubelet_test.go",
         "protokube_test.go",
     ],
@@ -97,6 +99,7 @@ go_test(
         "//nodeup/pkg/distros:go_default_library",
         "//pkg/apis/kops:go_default_library",
         "//pkg/apis/nodeup:go_default_library",
+        "//pkg/configbuilder:go_default_library",
         "//pkg/flagbuilder:go_default_library",
         "//pkg/testutils:go_default_library",
         "//upup/pkg/fi:go_default_library",

nodeup/pkg/model/kube_scheduler.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"strconv"
 
+	"k8s.io/kops/pkg/configbuilder"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubemanifest"
@@ -34,21 +35,37 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+// ClientConnectionConfig is used by kube-scheduler to talk to the api server
+type ClientConnectionConfig struct {
+	Burst      int32    `yaml:"burst,omitempty"`
+	Kubeconfig string   `yaml:"kubeconfig"`
+	QPS        *float64 `yaml:"qps,omitempty"`
+}
+
+// SchedulerConfig is used to generate the config file
+type SchedulerConfig struct {
+	APIVersion       string                 `yaml:"apiVersion"`
+	Kind             string                 `yaml:"kind"`
+	ClientConnection ClientConnectionConfig `yaml:"clientConnection,omitempty"`
+}
+
 // KubeSchedulerBuilder install kube-scheduler
 type KubeSchedulerBuilder struct {
 	*NodeupModelContext
 }
 
 var _ fi.ModelBuilder = &KubeSchedulerBuilder{}
 
+const defaultKubeConfig = "/var/lib/kube-scheduler/kubeconfig"
+
 // Build is responsible for building the manifest for the kube-scheduler
 func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 	if !b.IsMaster {
 		return nil
 	}
-
+	useConfigFile := b.IsKubernetesGTE("1.11")
 	{
-		pod, err := b.buildPod()
+		pod, err := b.buildPod(useConfigFile)
 		if err != nil {
 			return fmt.Errorf("error building kube-scheduler pod: %v", err)
 		}
@@ -78,6 +95,19 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 			Mode:     s("0400"),
 		})
 	}
+	if useConfigFile {
+		config, err := configbuilder.BuildConfigYaml(b.Cluster.Spec.KubeScheduler, NewSchedulerConfig())
+		if err != nil {
+			return err
+		}
+
+		c.AddTask(&nodetasks.File{
+			Path:     "/var/lib/kube-scheduler/config.yaml",
+			Contents: fi.NewBytesResource(config),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0400"),
+		})
+	}
 
 	{
 		c.AddTask(&nodetasks.File{
@@ -92,16 +122,30 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 	return nil
 }
 
+// NewSchedulerConfig initializes a new kube-scheduler config file
+func NewSchedulerConfig() *SchedulerConfig {
+	schedConfig := new(SchedulerConfig)
+	schedConfig.APIVersion = "kubescheduler.config.k8s.io/v1alpha1"
+	schedConfig.Kind = "KubeSchedulerConfiguration"
+	schedConfig.ClientConnection = ClientConnectionConfig{}
+	schedConfig.ClientConnection.Kubeconfig = defaultKubeConfig
+	return schedConfig
+}
+
 // buildPod is responsible for constructing the pod specification
-func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) {
+func (b *KubeSchedulerBuilder) buildPod(useConfigFile bool) (*v1.Pod, error) {
 	c := b.Cluster.Spec.KubeScheduler
 
 	flags, err := flagbuilder.BuildFlagsList(c)
 	if err != nil {
 		return nil, fmt.Errorf("error building kube-scheduler flags: %v", err)
 	}
-	// Add kubeconfig flag
-	flags = append(flags, "--kubeconfig="+"/var/lib/kube-scheduler/kubeconfig")
+	if useConfigFile {
+		flags = append(flags, "--config="+"/var/lib/kube-scheduler/config.yaml")
+	} else {
+		// Add kubeconfig flag
+		flags = append(flags, "--kubeconfig="+defaultKubeConfig)
+	}
 
 	if c.UsePolicyConfigMap != nil {
 		flags = append(flags, "--policy-configmap=scheduler-policy", "--policy-configmap-namespace=kube-system")

nodeup/pkg/model/kube_scheduler_test.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"bytes"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/configbuilder"
+)
+
+func TestParseDefault(t *testing.T) {
+	expect := []byte(
+		`apiVersion: kubescheduler.config.k8s.io/v1alpha1
+kind: KubeSchedulerConfiguration
+clientConnection:
+  kubeconfig: /var/lib/kube-scheduler/kubeconfig
+`)
+
+	s := &kops.KubeSchedulerConfig{}
+
+	yaml, err := configbuilder.BuildConfigYaml(s, NewSchedulerConfig())
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if !bytes.Equal(yaml, expect) {
+		t.Errorf("unexpected result: \n%s, expected: \n%s", yaml, expect)
+	}
+}
+
+func TestParse(t *testing.T) {
+	expect := []byte(
+		`apiVersion: kubescheduler.config.k8s.io/v1alpha1
+kind: KubeSchedulerConfiguration
+clientConnection:
+  burst: 100
+  kubeconfig: /var/lib/kube-scheduler/kubeconfig
+  qps: 3.1
+`)
+	qps, _ := resource.ParseQuantity("3.1")
+
+	s := &kops.KubeSchedulerConfig{Qps: &qps, Burst: 100}
+
+	yaml, err := configbuilder.BuildConfigYaml(s, NewSchedulerConfig())
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if !bytes.Equal(yaml, expect) {
+		t.Errorf("unexpected result: \n%s, expected: \n%s", yaml, expect)
+	}
+}

pkg/apis/kops/componentconfig.go

@@ -609,6 +609,10 @@ type KubeSchedulerConfig struct {
 	// which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider
 	// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
 	MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
+	// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"`
+	// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"`
 }
 
 // LeaderElectionConfiguration defines the configuration of leader election

pkg/apis/kops/v1alpha1/componentconfig.go

@@ -609,6 +609,10 @@ type KubeSchedulerConfig struct {
 	// which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider
 	// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
 	MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
+	// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"`
+	// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"`
 }
 
 // LeaderElectionConfiguration defines the configuration of leader election

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -610,6 +610,10 @@ type KubeSchedulerConfig struct {
 	// which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider
 	// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
 	MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
+	// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Qps *resource.Quantity `json:"qps,omitempty"`
+	// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Burst int32 `json:"burst,omitempty"`
 }
 
 // LeaderElectionConfiguration defines the configuration of leader election

pkg/configbuilder/BUILD.bazel

@@ -0,0 +1,21 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["buildconfigfile.go"],
+    importpath = "k8s.io/kops/pkg/configbuilder",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/kops:go_default_library",
+        "//util/pkg/reflectutils:go_default_library",
+        "//vendor/gopkg.in/yaml.v2:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["buildconfigfile_test.go"],
+    embed = [":go_default_library"],
+)