feat: support kube scheduler config

Support kube-scheduler config.

Fixes: #7905
Partially fixes: #7911

Signed-off-by: Noel Georgi <git@frezbo.dev>

api/resource/definitions/k8s/k8s.proto

@@ -220,6 +220,7 @@ message SchedulerConfigSpec {
   repeated ExtraVolume extra_volumes = 4;
   map<string, string> environment_variables = 5;
   Resources resources = 6;
+  google.protobuf.Struct config = 7;
 }
 
 // SecretsStatusSpec describes status of rendered secrets.

go.mod

@@ -22,6 +22,7 @@ require (
 	k8s.io/client-go v0.29.0-alpha.3
 	k8s.io/component-base v0.29.0-alpha.3
 	k8s.io/cri-api v0.29.0-alpha.3
+	k8s.io/kube-scheduler v0.29.0-alpha.3
 	k8s.io/kubectl v0.29.0-alpha.3
 	k8s.io/kubelet v0.29.0-alpha.3
 )
@@ -111,7 +112,7 @@ require (
 	github.com/siderolabs/go-debug v0.2.3
 	github.com/siderolabs/go-kmsg v0.1.3
 	github.com/siderolabs/go-kubeconfig v0.1.0
-	github.com/siderolabs/go-kubernetes v0.2.7
+	github.com/siderolabs/go-kubernetes v0.2.8
 	github.com/siderolabs/go-loadbalancer v0.3.2
 	github.com/siderolabs/go-pcidb v0.2.0
 	github.com/siderolabs/go-pointer v1.0.0

go.sum

@@ -663,8 +663,8 @@ github.com/siderolabs/go-kmsg v0.1.3 h1:rYxuDCN52Y6XdTNnEe52f0NZa4F6GHDEAQoVMHME
 github.com/siderolabs/go-kmsg v0.1.3/go.mod h1:MrxoGwR6WNC7knaMlIKbQM3DFKqJ/flTQH9OtW7M3c8=
 github.com/siderolabs/go-kubeconfig v0.1.0 h1:t/2oMWkLSdWHXglKPMz8ySXnx6ZjHckeGY79NaDcBTo=
 github.com/siderolabs/go-kubeconfig v0.1.0/go.mod h1:eM3mO02Td6wYDvdi9zTbMrj1Q4WqEFN8XQ6pNjCUWkI=
-github.com/siderolabs/go-kubernetes v0.2.7 h1:IHRAiGRDiEj0O5RlwUdt3oB/lm+zf1PnHpgqizghnLs=
-github.com/siderolabs/go-kubernetes v0.2.7/go.mod h1:JrhhdnHeeuEi+4nGCSrnkTjEy9oNQ6YJ/2/0Tzy+PQU=
+github.com/siderolabs/go-kubernetes v0.2.8 h1:ks+xA0sZdYhdRTkSZlngaku+BWngbZxijidh2e+XX5Q=
+github.com/siderolabs/go-kubernetes v0.2.8/go.mod h1:JrhhdnHeeuEi+4nGCSrnkTjEy9oNQ6YJ/2/0Tzy+PQU=
 github.com/siderolabs/go-loadbalancer v0.3.2 h1:R2jKq8ifWOARxJ5blXwOOkiWCA5/46stGxUR8+qV8GE=
 github.com/siderolabs/go-loadbalancer v0.3.2/go.mod h1:sKP/xSN4R+1fifcqIjnk1FtM5sSW20d+pi+0FV6CpVo=
 github.com/siderolabs/go-pcidb v0.2.0 h1:ZCkF1cz6UjoEIHpP7+aeTI5BwmSxE627Jl1Wy2VZAwU=
@@ -1187,6 +1187,8 @@ k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
 k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
+k8s.io/kube-scheduler v0.29.0-alpha.3 h1:7nX3uMwD/k5FQSsiKlQ/bxnKh0/3i0HP6SZEqusyaF4=
+k8s.io/kube-scheduler v0.29.0-alpha.3/go.mod h1:hH7ItZdOFsP7V+eNqxIiU19YuOqXWYll71SoS4u0o68=
 k8s.io/kubectl v0.29.0-alpha.3 h1:Q9TkGB5HHJNjX9+sdzNeRNDashB7EF1aDOkwqcs0SlI=
 k8s.io/kubectl v0.29.0-alpha.3/go.mod h1:tG9srWZBHIZHifk1ERUAmRyXLrR4hGeXzuAVU2Sz48I=
 k8s.io/kubelet v0.29.0-alpha.3 h1:wiVt06zqSypK5WNWIuVs1LIyEOTuI0P4rTaRi/AdZSE=

hack/release.toml

@@ -30,6 +30,13 @@ Talos now supports specifying the kubelet credential provider configuration in t
 It can be set under `machine.kubelet.credentialProviderConfig` and kubelet will be automatically configured to with the correct flags.
 The credential binaries are expected to be present under `/usr/local/lib/kubelet/credentialproviders`.
 Talos System Extensions can be used to install the credential binaries.
+"""
+
+    [notes.kube-scheduler]
+        title = "Kube-Scheduler Configuration"
+        description = """\
+Talos now supports specifying the kube-scheduler configuration in the Talos configuration file.
+It can be set under `cluster.scheduler.config` and kube-scheduler will be automatically configured to with the correct flags.
 """
 
     [notes.updates]

internal/app/machined/pkg/controllers/k8s/control_plane.go

@@ -195,6 +195,7 @@ func NewControlPlaneSchedulerController() *ControlPlaneSchedulerController {
 					ExtraVolumes:         convertVolumes(cfgProvider.Cluster().Scheduler().ExtraVolumes()),
 					EnvironmentVariables: cfgProvider.Cluster().Scheduler().Env(),
 					Resources:            convertResources(cfgProvider.Cluster().Scheduler().Resources()),
+					Config:               cfgProvider.Cluster().Scheduler().Config(),
 				}
 
 				return nil

internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go

@@ -619,7 +619,6 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
 		env = append(env, goGCEnv)
 	}
 
-	//nolint:dupl
 	return k8s.ControllerManagerID, r.Modify(ctx, k8s.NewStaticPod(k8s.NamespaceName, k8s.ControllerManagerID), func(r resource.Resource) error {
 		return k8sadapter.StaticPod(r.(*k8s.StaticPod)).SetPod(&v1.Pod{
 			TypeMeta: metav1.TypeMeta{
@@ -726,7 +725,7 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
 	}
 
 	builder := argsbuilder.Args{
-		"kubeconfig":                             filepath.Join(constants.KubernetesSchedulerSecretsDir, "kubeconfig"),
+		"config":                                 filepath.Join(constants.KubernetesSchedulerConfigDir, "scheduler-config.yaml"),
 		"authentication-tolerate-lookup-failure": "false",
 		"authentication-kubeconfig":              filepath.Join(constants.KubernetesSchedulerSecretsDir, "kubeconfig"),
 		"authorization-kubeconfig":               filepath.Join(constants.KubernetesSchedulerSecretsDir, "kubeconfig"),
@@ -740,6 +739,7 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
 		"kubeconfig":                argsbuilder.MergeDenied,
 		"authentication-kubeconfig": argsbuilder.MergeDenied,
 		"authorization-kubeconfig":  argsbuilder.MergeDenied,
+		"config":                    argsbuilder.MergeDenied,
 	}
 
 	if err := builder.Merge(cfg.ExtraArgs, argsbuilder.WithMergePolicies(mergePolicies)); err != nil {
@@ -758,7 +758,6 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
 		env = append(env, goGCEnv)
 	}
 
-	//nolint:dupl
 	return k8s.SchedulerID, r.Modify(ctx, k8s.NewStaticPod(k8s.NamespaceName, k8s.SchedulerID), func(r resource.Resource) error {
 		return k8sadapter.StaticPod(r.(*k8s.StaticPod)).SetPod(&v1.Pod{
 			TypeMeta: metav1.TypeMeta{
@@ -792,6 +791,11 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
 								MountPath: constants.KubernetesSchedulerSecretsDir,
 								ReadOnly:  true,
 							},
+							{
+								Name:      "config",
+								MountPath: constants.KubernetesSchedulerConfigDir,
+								ReadOnly:  true,
+							},
 						}, volumeMounts(cfg.ExtraVolumes)...),
 						StartupProbe: &v1.Probe{
 							ProbeHandler: v1.ProbeHandler{
@@ -845,6 +849,14 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
 							},
 						},
 					},
+					{
+						Name: "config",
+						VolumeSource: v1.VolumeSource{
+							HostPath: &v1.HostPathVolumeSource{
+								Path: constants.KubernetesSchedulerConfigDir,
+							},
+						},
+					},
 				}, volumes(cfg.ExtraVolumes)...),
 			},
 		})

internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go

@@ -15,11 +15,13 @@ import (
 	"github.com/cosi-project/runtime/pkg/controller"
 	"github.com/cosi-project/runtime/pkg/resource"
 	"github.com/cosi-project/runtime/pkg/state"
+	"github.com/siderolabs/go-kubernetes/kubernetes/compatibility"
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/runtime"
 	k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json"
 	apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
 	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
+	schedulerv1 "k8s.io/kube-scheduler/config/v1"
 
 	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
@@ -46,6 +48,11 @@ func (ctrl *RenderConfigsStaticPodController) Inputs() []controller.Input {
 			Type:      k8s.AuditPolicyConfigType,
 			Kind:      controller.InputWeak,
 		},
+		{
+			Namespace: k8s.ControlPlaneNamespaceName,
+			Type:      k8s.SchedulerConfigType,
+			Kind:      controller.InputWeak,
+		},
 	}
 }
 
@@ -92,6 +99,19 @@ func (ctrl *RenderConfigsStaticPodController) Run(ctx context.Context, r control
 
 		auditConfig := auditRes.(*k8s.AuditPolicyConfig).TypedSpec()
 
+		kubeSchedulerRes, err := r.Get(ctx, k8s.NewSchedulerConfig().Metadata())
+		if err != nil {
+			if state.IsNotFoundError(err) {
+				continue
+			}
+
+			return fmt.Errorf("error getting scheduler config resource: %w", err)
+		}
+
+		kubeSchedulerConfig := kubeSchedulerRes.(*k8s.SchedulerConfig).TypedSpec()
+
+		kubeSchedulerVersion := compatibility.VersionFromImageRef(kubeSchedulerConfig.Image)
+
 		type configFile struct {
 			filename string
 			f        func() (runtime.Object, error)
@@ -129,6 +149,18 @@ func (ctrl *RenderConfigsStaticPodController) Run(ctx context.Context, r control
 					},
 				},
 			},
+			{
+				name:      "kube-scheduler",
+				directory: constants.KubernetesSchedulerConfigDir,
+				uid:       constants.KubernetesSchedulerRunUser,
+				gid:       constants.KubernetesSchedulerRunGroup,
+				configs: []configFile{
+					{
+						filename: "scheduler-config.yaml",
+						f:        schedulerConfig(kubeSchedulerConfig, kubeSchedulerVersion),
+					},
+				},
+			},
 		} {
 			if err = os.MkdirAll(pod.directory, 0o755); err != nil {
 				return fmt.Errorf("error creating config directory for %q: %w", pod.name, err)
@@ -160,7 +192,7 @@ func (ctrl *RenderConfigsStaticPodController) Run(ctx context.Context, r control
 
 		if err = r.Modify(ctx, k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID), func(r resource.Resource) error {
 			r.(*k8s.ConfigStatus).TypedSpec().Ready = true
-			r.(*k8s.ConfigStatus).TypedSpec().Version = admissionRes.Metadata().Version().String() + auditRes.Metadata().Version().String()
+			r.(*k8s.ConfigStatus).TypedSpec().Version = admissionRes.Metadata().Version().String() + auditRes.Metadata().Version().String() + kubeSchedulerRes.Metadata().Version().String()
 
 			return nil
 		}); err != nil {
@@ -210,3 +242,19 @@ func auditPolicyConfig(spec *k8s.AuditPolicyConfigSpec) func() (runtime.Object,
 		return &cfg, nil
 	}
 }
+
+func schedulerConfig(spec *k8s.SchedulerConfigSpec, kubeSchedulerVersion compatibility.Version) func() (runtime.Object, error) {
+	return func() (runtime.Object, error) {
+		var cfg schedulerv1.KubeSchedulerConfiguration
+
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(spec.Config, &cfg, false); err != nil {
+			return nil, fmt.Errorf("error unmarshaling scheduler configuration: %w", err)
+		}
+
+		cfg.APIVersion = kubeSchedulerVersion.KubeSchedulerConfigurationAPIVersion()
+		cfg.Kind = "KubeSchedulerConfiguration"
+		cfg.ClientConnection.Kubeconfig = filepath.Join(constants.KubernetesSchedulerSecretsDir, "kubeconfig")
+
+		return &cfg, nil
+	}
+}