From 59c71f45157c0019deb5f9fc507572ba543eae25 Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Wed, 21 Jun 2023 17:22:06 +0300 Subject: [PATCH 1/2] [api/config] Drop dependency on sigs.k8s.io/controller-runtime/pkg/config/v1alpha1 --- apis/config/v1beta1/configuration_types.go | 101 ++++++++++++++++- apis/config/v1beta1/defaults_test.go | 85 +++++++-------- apis/config/v1beta1/zz_generated.deepcopy.go | 109 ++++++++++++++++++- 3 files changed, 248 insertions(+), 47 deletions(-) diff --git a/apis/config/v1beta1/configuration_types.go b/apis/config/v1beta1/configuration_types.go index 691b17f432..5b873ad96a 100644 --- a/apis/config/v1beta1/configuration_types.go +++ b/apis/config/v1beta1/configuration_types.go @@ -17,8 +17,10 @@ limitations under the License. package v1beta1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + configv1alpha1 "k8s.io/component-base/config/v1alpha1" ) //+kubebuilder:object:root=true @@ -31,8 +33,8 @@ type Configuration struct { // Defaults to kueue-system. Namespace *string `json:"namespace,omitempty"` - // ControllerManagerConfigurationSpec returns the configurations for controllers - cfg.ControllerManagerConfigurationSpec `json:",inline"` + // ControllerManager returns the configurations for controllers + ControllerManager `json:",inline"` // ManageJobsWithoutQueueName controls whether or not Kueue reconciles // batch/v1.Jobs that don't set the annotation kueue.x-k8s.io/queue-name. @@ -61,6 +63,99 @@ type Configuration struct { Integrations *Integrations `json:"integrations,omitempty"` } +type ControllerManager struct { + // Webhook contains the controllers webhook configuration + // +optional + Webhook ControllerWebhook `json:"webhook,omitempty"` + + // LeaderElection is the LeaderElection config to be used when configuring + // the manager.Manager leader election + // +optional + LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"` + + // Metrics contains thw controller metrics configuration + // +optional + Metrics ControllerMetrics `json:"metrics,omitempty"` + + // Health contains the controller health configuration + // +optional + Health ControllerHealth `json:"health,omitempty"` + + // Controller contains global configuration options for controllers + // registered within this manager. + // +optional + Controller *ControllerConfigurationSpec `json:"controller,omitempty"` +} + +// ControllerWebhook defines the webhook server for the controller. +type ControllerWebhook struct { + // Port is the port that the webhook server serves at. + // It is used to set webhook.Server.Port. + // +optional + Port *int `json:"port,omitempty"` + + // Host is the hostname that the webhook server binds to. + // It is used to set webhook.Server.Host. + // +optional + Host string `json:"host,omitempty"` + + // CertDir is the directory that contains the server key and certificate. + // if not set, webhook server would look up the server key and certificate in + // {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate + // must be named tls.key and tls.crt, respectively. + // +optional + CertDir string `json:"certDir,omitempty"` +} + +// ControllerMetrics defines the metrics configs. +type ControllerMetrics struct { + // BindAddress is the TCP address that the controller should bind to + // for serving prometheus metrics. + // It can be set to "0" to disable the metrics serving. + // +optional + BindAddress string `json:"bindAddress,omitempty"` +} + +// ControllerHealth defines the health configs. +type ControllerHealth struct { + // HealthProbeBindAddress is the TCP address that the controller should bind to + // for serving health probes + // It can be set to "0" or "" to disable serving the health probe. + // +optional + HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"` + + // ReadinessEndpointName, defaults to "readyz" + // +optional + ReadinessEndpointName string `json:"readinessEndpointName,omitempty"` + + // LivenessEndpointName, defaults to "healthz" + // +optional + LivenessEndpointName string `json:"livenessEndpointName,omitempty"` +} + +// ControllerConfigurationSpec defines the global configuration for +// controllers registered with the manager. +type ControllerConfigurationSpec struct { + // GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation + // allowed for that controller. + // + // When a controller is registered within this manager using the builder utilities, + // users have to specify the type the controller reconciles in the For(...) call. + // If the object's kind passed matches one of the keys in this map, the concurrency + // for that controller is set to the number specified. + // + // The key is expected to be consistent in form with GroupKind.String(), + // e.g. ReplicaSet in apps group (regardless of version) would be `ReplicaSet.apps`. + // + // +optional + GroupKindConcurrency map[string]int `json:"groupKindConcurrency,omitempty"` + + // CacheSyncTimeout refers to the time limit set to wait for syncing caches. + // Defaults to 2 minutes if not set. + // +optional + CacheSyncTimeout *time.Duration `json:"cacheSyncTimeout,omitempty"` +} + type WaitForPodsReady struct { // Enable when true, indicates that each admitted workload // blocks the admission of all other workloads from all queues until it is in the diff --git a/apis/config/v1beta1/defaults_test.go b/apis/config/v1beta1/defaults_test.go index 48174d47e5..c3bdcc2dd0 100644 --- a/apis/config/v1beta1/defaults_test.go +++ b/apis/config/v1beta1/defaults_test.go @@ -24,7 +24,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" componentconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" "k8s.io/utils/pointer" - ctrlconfigv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/kueue/pkg/controller/jobs/job" ) @@ -38,14 +37,14 @@ const ( ) func TestSetDefaults_Configuration(t *testing.T) { - defaultCtrlManagerConfigurationSpec := ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{ - Webhook: ctrlconfigv1alpha1.ControllerWebhook{ + defaultCtrlManagerConfigurationSpec := ControllerManager{ + Webhook: ControllerWebhook{ Port: pointer.Int(DefaultWebhookPort), }, - Metrics: ctrlconfigv1alpha1.ControllerMetrics{ + Metrics: ControllerMetrics{ BindAddress: DefaultMetricsBindAddress, }, - Health: ctrlconfigv1alpha1.ControllerHealth{ + Health: ControllerHealth{ HealthProbeBindAddress: DefaultHealthProbeBindAddress, }, } @@ -70,8 +69,8 @@ func TestSetDefaults_Configuration(t *testing.T) { }, }, want: &Configuration{ - Namespace: pointer.String(DefaultNamespace), - ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec, + Namespace: pointer.String(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, InternalCertManagement: &InternalCertManagement{ Enable: pointer.Bool(false), }, @@ -79,9 +78,9 @@ func TestSetDefaults_Configuration(t *testing.T) { Integrations: defaultIntegrations, }, }, - "defaulting ControllerManagerConfigurationSpec": { + "defaulting ControllerManager": { original: &Configuration{ - ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{ + ControllerManager: ControllerManager{ LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ LeaderElect: pointer.Bool(true), }, @@ -92,14 +91,14 @@ func TestSetDefaults_Configuration(t *testing.T) { }, want: &Configuration{ Namespace: pointer.String(DefaultNamespace), - ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{ - Webhook: ctrlconfigv1alpha1.ControllerWebhook{ + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ Port: pointer.Int(DefaultWebhookPort), }, - Metrics: ctrlconfigv1alpha1.ControllerMetrics{ + Metrics: ControllerMetrics{ BindAddress: DefaultMetricsBindAddress, }, - Health: ctrlconfigv1alpha1.ControllerHealth{ + Health: ControllerHealth{ HealthProbeBindAddress: DefaultHealthProbeBindAddress, }, LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ @@ -114,16 +113,16 @@ func TestSetDefaults_Configuration(t *testing.T) { Integrations: defaultIntegrations, }, }, - "should not default ControllerManagerConfigurationSpec": { + "should not default ControllerManager": { original: &Configuration{ - ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{ - Webhook: ctrlconfigv1alpha1.ControllerWebhook{ + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ Port: pointer.Int(overwriteWebhookPort), }, - Metrics: ctrlconfigv1alpha1.ControllerMetrics{ + Metrics: ControllerMetrics{ BindAddress: overwriteMetricBindAddress, }, - Health: ctrlconfigv1alpha1.ControllerHealth{ + Health: ControllerHealth{ HealthProbeBindAddress: overwriteHealthProbeBindAddress, }, LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ @@ -138,14 +137,14 @@ func TestSetDefaults_Configuration(t *testing.T) { }, want: &Configuration{ Namespace: pointer.String(DefaultNamespace), - ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{ - Webhook: ctrlconfigv1alpha1.ControllerWebhook{ + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ Port: pointer.Int(overwriteWebhookPort), }, - Metrics: ctrlconfigv1alpha1.ControllerMetrics{ + Metrics: ControllerMetrics{ BindAddress: overwriteMetricBindAddress, }, - Health: ctrlconfigv1alpha1.ControllerHealth{ + Health: ControllerHealth{ HealthProbeBindAddress: overwriteHealthProbeBindAddress, }, LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ @@ -162,7 +161,7 @@ func TestSetDefaults_Configuration(t *testing.T) { }, "should not set LeaderElectionID": { original: &Configuration{ - ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{ + ControllerManager: ControllerManager{ LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ LeaderElect: pointer.Bool(false), }, @@ -173,14 +172,14 @@ func TestSetDefaults_Configuration(t *testing.T) { }, want: &Configuration{ Namespace: pointer.String(DefaultNamespace), - ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{ - Webhook: ctrlconfigv1alpha1.ControllerWebhook{ + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ Port: pointer.Int(DefaultWebhookPort), }, - Metrics: ctrlconfigv1alpha1.ControllerMetrics{ + Metrics: ControllerMetrics{ BindAddress: DefaultMetricsBindAddress, }, - Health: ctrlconfigv1alpha1.ControllerHealth{ + Health: ControllerHealth{ HealthProbeBindAddress: DefaultHealthProbeBindAddress, }, LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ @@ -199,8 +198,8 @@ func TestSetDefaults_Configuration(t *testing.T) { Namespace: pointer.String(overwriteNamespace), }, want: &Configuration{ - Namespace: pointer.String(overwriteNamespace), - ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec, + Namespace: pointer.String(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, InternalCertManagement: &InternalCertManagement{ Enable: pointer.Bool(true), WebhookServiceName: pointer.String(DefaultWebhookServiceName), @@ -218,8 +217,8 @@ func TestSetDefaults_Configuration(t *testing.T) { }, }, want: &Configuration{ - Namespace: pointer.String(overwriteNamespace), - ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec, + Namespace: pointer.String(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, InternalCertManagement: &InternalCertManagement{ Enable: pointer.Bool(false), }, @@ -239,8 +238,8 @@ func TestSetDefaults_Configuration(t *testing.T) { }, }, want: &Configuration{ - Namespace: pointer.String(overwriteNamespace), - ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec, + Namespace: pointer.String(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, InternalCertManagement: &InternalCertManagement{ Enable: pointer.Bool(false), }, @@ -260,8 +259,8 @@ func TestSetDefaults_Configuration(t *testing.T) { ClientConnection: &ClientConnection{}, }, want: &Configuration{ - Namespace: pointer.String(overwriteNamespace), - ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec, + Namespace: pointer.String(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, InternalCertManagement: &InternalCertManagement{ Enable: pointer.Bool(false), }, @@ -284,8 +283,8 @@ func TestSetDefaults_Configuration(t *testing.T) { BlockAdmission: pointer.Bool(true), Timeout: &podsReadyTimeoutTimeout, }, - Namespace: pointer.String(DefaultNamespace), - ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec, + Namespace: pointer.String(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, InternalCertManagement: &InternalCertManagement{ Enable: pointer.Bool(false), }, @@ -308,8 +307,8 @@ func TestSetDefaults_Configuration(t *testing.T) { BlockAdmission: pointer.Bool(false), Timeout: &podsReadyTimeoutTimeout, }, - Namespace: pointer.String(DefaultNamespace), - ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec, + Namespace: pointer.String(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, InternalCertManagement: &InternalCertManagement{ Enable: pointer.Bool(false), }, @@ -333,8 +332,8 @@ func TestSetDefaults_Configuration(t *testing.T) { BlockAdmission: pointer.Bool(true), Timeout: &podsReadyTimeoutOverwrite, }, - Namespace: pointer.String(DefaultNamespace), - ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec, + Namespace: pointer.String(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, InternalCertManagement: &InternalCertManagement{ Enable: pointer.Bool(false), }, @@ -352,8 +351,8 @@ func TestSetDefaults_Configuration(t *testing.T) { }, }, want: &Configuration{ - Namespace: pointer.String(DefaultNamespace), - ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec, + Namespace: pointer.String(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, InternalCertManagement: &InternalCertManagement{ Enable: pointer.Bool(false), }, diff --git a/apis/config/v1beta1/zz_generated.deepcopy.go b/apis/config/v1beta1/zz_generated.deepcopy.go index 85cf765bd7..12bc37a5a6 100644 --- a/apis/config/v1beta1/zz_generated.deepcopy.go +++ b/apis/config/v1beta1/zz_generated.deepcopy.go @@ -24,6 +24,8 @@ package v1beta1 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/component-base/config/v1alpha1" + timex "time" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -60,7 +62,7 @@ func (in *Configuration) DeepCopyInto(out *Configuration) { *out = new(string) **out = **in } - in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) + in.ControllerManager.DeepCopyInto(&out.ControllerManager) if in.InternalCertManagement != nil { in, out := &in.InternalCertManagement, &out.InternalCertManagement *out = new(InternalCertManagement) @@ -101,6 +103,111 @@ func (in *Configuration) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerConfigurationSpec) DeepCopyInto(out *ControllerConfigurationSpec) { + *out = *in + if in.GroupKindConcurrency != nil { + in, out := &in.GroupKindConcurrency, &out.GroupKindConcurrency + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.CacheSyncTimeout != nil { + in, out := &in.CacheSyncTimeout, &out.CacheSyncTimeout + *out = new(timex.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerConfigurationSpec. +func (in *ControllerConfigurationSpec) DeepCopy() *ControllerConfigurationSpec { + if in == nil { + return nil + } + out := new(ControllerConfigurationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerHealth) DeepCopyInto(out *ControllerHealth) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerHealth. +func (in *ControllerHealth) DeepCopy() *ControllerHealth { + if in == nil { + return nil + } + out := new(ControllerHealth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerManager) DeepCopyInto(out *ControllerManager) { + *out = *in + in.Webhook.DeepCopyInto(&out.Webhook) + if in.LeaderElection != nil { + in, out := &in.LeaderElection, &out.LeaderElection + *out = new(v1alpha1.LeaderElectionConfiguration) + (*in).DeepCopyInto(*out) + } + out.Metrics = in.Metrics + out.Health = in.Health + if in.Controller != nil { + in, out := &in.Controller, &out.Controller + *out = new(ControllerConfigurationSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManager. +func (in *ControllerManager) DeepCopy() *ControllerManager { + if in == nil { + return nil + } + out := new(ControllerManager) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerMetrics) DeepCopyInto(out *ControllerMetrics) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerMetrics. +func (in *ControllerMetrics) DeepCopy() *ControllerMetrics { + if in == nil { + return nil + } + out := new(ControllerMetrics) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerWebhook) DeepCopyInto(out *ControllerWebhook) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerWebhook. +func (in *ControllerWebhook) DeepCopy() *ControllerWebhook { + if in == nil { + return nil + } + out := new(ControllerWebhook) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Integrations) DeepCopyInto(out *Integrations) { *out = *in From 65431a41e72c859daa9b7e31cb5d4676f88ba866 Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Thu, 22 Jun 2023 15:02:35 +0300 Subject: [PATCH 2/2] [config] Switch to internal implementation --- main.go | 52 +--- main_test.go | 322 +------------------- pkg/config/config.go | 159 ++++++++++ pkg/config/config_test.go | 600 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 777 insertions(+), 356 deletions(-) create mode 100644 pkg/config/config.go create mode 100644 pkg/config/config_test.go diff --git a/main.go b/main.go index 0f0b62819e..b15fc66777 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,6 @@ limitations under the License. package main import ( - "bytes" "context" "flag" "fmt" @@ -33,7 +32,6 @@ import ( schedulingv1 "k8s.io/api/scheduling/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" @@ -43,9 +41,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/cache" + "sigs.k8s.io/kueue/pkg/config" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/controller/core/indexer" @@ -75,7 +74,7 @@ func init() { utilruntime.Must(schedulingv1.AddToScheme(scheme)) utilruntime.Must(kueue.AddToScheme(scheme)) - utilruntime.Must(config.AddToScheme(scheme)) + utilruntime.Must(configapi.AddToScheme(scheme)) utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) // Add any additional framework integration types. utilruntime.Must( @@ -173,7 +172,7 @@ func main() { } } -func setupIndexes(ctx context.Context, mgr ctrl.Manager, cfg *config.Configuration) { +func setupIndexes(ctx context.Context, mgr ctrl.Manager, cfg *configapi.Configuration) { err := indexer.Setup(ctx, mgr.GetFieldIndexer()) if err != nil { setupLog.Error(err, "Unable to setup core api indexes") @@ -192,7 +191,7 @@ func setupIndexes(ctx context.Context, mgr ctrl.Manager, cfg *config.Configurati } } -func setupControllers(ctx context.Context, mgr ctrl.Manager, cCache *cache.Cache, queues *queue.Manager, certsReady chan struct{}, cfg *config.Configuration) { +func setupControllers(ctx context.Context, mgr ctrl.Manager, cCache *cache.Cache, queues *queue.Manager, certsReady chan struct{}, cfg *configapi.Configuration) { // The controllers won't work until the webhooks are operating, and the webhook won't work until the // certs are all in place. setupLog.Info("Waiting for certificate generation to complete") @@ -259,7 +258,7 @@ func setupProbeEndpoints(mgr ctrl.Manager) { } } -func setupScheduler(mgr ctrl.Manager, cCache *cache.Cache, queues *queue.Manager, cfg *config.Configuration) { +func setupScheduler(mgr ctrl.Manager, cCache *cache.Cache, queues *queue.Manager, cfg *configapi.Configuration) { sched := scheduler.New( queues, cCache, @@ -272,43 +271,16 @@ func setupScheduler(mgr ctrl.Manager, cCache *cache.Cache, queues *queue.Manager } } -func blockForPodsReady(cfg *config.Configuration) bool { +func blockForPodsReady(cfg *configapi.Configuration) bool { return waitForPodsReady(cfg) && cfg.WaitForPodsReady.BlockAdmission != nil && *cfg.WaitForPodsReady.BlockAdmission } -func waitForPodsReady(cfg *config.Configuration) bool { +func waitForPodsReady(cfg *configapi.Configuration) bool { return cfg.WaitForPodsReady != nil && cfg.WaitForPodsReady.Enable } -func encodeConfig(cfg *config.Configuration) (string, error) { - codecs := serializer.NewCodecFactory(scheme) - const mediaType = runtime.ContentTypeYAML - info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) - if !ok { - return "", fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType) - } - - encoder := codecs.EncoderForVersion(info.Serializer, config.GroupVersion) - buf := new(bytes.Buffer) - if err := encoder.Encode(cfg, buf); err != nil { - return "", err - } - return buf.String(), nil -} - -func apply(configFile string) (ctrl.Options, config.Configuration, error) { - var err error - options := ctrl.Options{ - Scheme: scheme, - } - cfg := config.Configuration{} - - if configFile == "" { - scheme.Default(&cfg) - options, err = options.AndFrom(&cfg) - } else { - options, err = options.AndFrom(ctrl.ConfigFile().AtPath(configFile).OfKind(&cfg)) - } +func apply(configFile string) (ctrl.Options, configapi.Configuration, error) { + options, cfg, err := config.Load(scheme, configFile) if err != nil { return options, cfg, err } @@ -328,7 +300,7 @@ func apply(configFile string) (ctrl.Options, config.Configuration, error) { } } - cfgStr, err := encodeConfig(&cfg) + cfgStr, err := config.Encode(scheme, &cfg) if err != nil { return options, cfg, err } @@ -337,7 +309,7 @@ func apply(configFile string) (ctrl.Options, config.Configuration, error) { return options, cfg, nil } -func isFrameworkEnabled(cfg *config.Configuration, name string) bool { +func isFrameworkEnabled(cfg *configapi.Configuration, name string) bool { for _, framework := range cfg.Integrations.Frameworks { if framework == name { return true diff --git a/main_test.go b/main_test.go index bfc9b02792..2129fed4b9 100644 --- a/main_test.go +++ b/main_test.go @@ -21,19 +21,17 @@ import ( "os" "path/filepath" "testing" - "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" - ctrl "sigs.k8s.io/controller-runtime" config "sigs.k8s.io/kueue/apis/config/v1beta1" "sigs.k8s.io/kueue/pkg/controller/jobs/job" ) -func TestApply(t *testing.T) { +func TestValidateIntegrationsName(t *testing.T) { // temp dir tmpDir, err := os.MkdirTemp("", "temp") if err != nil { @@ -41,132 +39,6 @@ func TestApply(t *testing.T) { } defer os.RemoveAll(tmpDir) - namespaceOverWriteConfig := filepath.Join(tmpDir, "namespace-overwrite.yaml") - if err := os.WriteFile(namespaceOverWriteConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 -kind: Configuration -namespace: kueue-tenant-a -health: - healthProbeBindAddress: :8081 -metrics: - bindAddress: :8080 -leaderElection: - leaderElect: true - resourceName: c1f6bfd2.kueue.x-k8s.io -webhook: - port: 9443 -`), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - ctrlManagerConfigSpecOverWriteConfig := filepath.Join(tmpDir, "ctrl-manager-config-spec-overwrite.yaml") - if err := os.WriteFile(ctrlManagerConfigSpecOverWriteConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 -kind: Configuration -namespace: kueue-system -health: - healthProbeBindAddress: :38081 -metrics: - bindAddress: :38080 -leaderElection: - leaderElect: true - resourceName: test-id -webhook: - port: 9444 -`), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - certOverWriteConfig := filepath.Join(tmpDir, "cert-overwrite.yaml") - if err := os.WriteFile(certOverWriteConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 -kind: Configuration -namespace: kueue-system -health: - healthProbeBindAddress: :8081 -metrics: - bindAddress: :8080 -leaderElection: - leaderElect: true - resourceName: c1f6bfd2.kueue.x-k8s.io -webhook: - port: 9443 -internalCertManagement: - enable: true - webhookServiceName: kueue-tenant-a-webhook-service - webhookSecretName: kueue-tenant-a-webhook-server-cert -`), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - disableCertOverWriteConfig := filepath.Join(tmpDir, "disable-cert-overwrite.yaml") - if err := os.WriteFile(disableCertOverWriteConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 -kind: Configuration -namespace: kueue-system -health: - healthProbeBindAddress: :8081 -metrics: - bindAddress: :8080 -leaderElection: - leaderElect: true - resourceName: c1f6bfd2.kueue.x-k8s.io -webhook: - port: 9443 -internalCertManagement: - enable: false -`), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - leaderElectionDisabledConfig := filepath.Join(tmpDir, "leaderElection-disabled.yaml") - if err := os.WriteFile(leaderElectionDisabledConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 -kind: Configuration -namespace: kueue-system -health: - healthProbeBindAddress: :8081 -metrics: - bindAddress: :8080 -leaderElection: - leaderElect: false -webhook: - port: 9443 -`), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - waitForPodsReadyEnabledConfig := filepath.Join(tmpDir, "waitForPodsReady-enabled.yaml") - if err := os.WriteFile(waitForPodsReadyEnabledConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 -kind: Configuration -waitForPodsReady: - enable: true -`), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - clientConnectionConfig := filepath.Join(tmpDir, "clientConnection.yaml") - if err := os.WriteFile(clientConnectionConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 -kind: Configuration -namespace: kueue-system -health: - healthProbeBindAddress: :8081 -metrics: - bindAddress: :8080 -leaderElection: - leaderElect: true - resourceName: c1f6bfd2.kueue.x-k8s.io -webhook: - port: 9443 -clientConnection: - qps: 50 - burst: 100 -`), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - integrationsConfig := filepath.Join(tmpDir, "integrations.yaml") if err := os.WriteFile(integrationsConfig, []byte(` apiVersion: config.kueue.x-k8s.io/v1beta1 @@ -189,27 +61,14 @@ integrations: t.Fatal(err) } - defaultControlOptions := ctrl.Options{ - Port: config.DefaultWebhookPort, - HealthProbeBindAddress: config.DefaultHealthProbeBindAddress, - MetricsBindAddress: config.DefaultMetricsBindAddress, - LeaderElectionID: config.DefaultLeaderElectionID, - LeaderElection: true, - } - enableDefaultInternalCertManagement := &config.InternalCertManagement{ Enable: pointer.Bool(true), WebhookServiceName: pointer.String(config.DefaultWebhookServiceName), WebhookSecretName: pointer.String(config.DefaultWebhookSecretName), } - ctrlOptsCmpOpts := []cmp.Option{ - cmpopts.IgnoreUnexported(ctrl.Options{}), - cmpopts.IgnoreFields(ctrl.Options{}, "Scheme", "Logger"), - } - configCmpOpts := []cmp.Option{ - cmpopts.IgnoreFields(config.Configuration{}, "ControllerManagerConfigurationSpec"), + cmpopts.IgnoreFields(config.Configuration{}, "ControllerManager"), } defaultClientConnection := &config.ClientConnection{ @@ -217,176 +76,12 @@ integrations: Burst: pointer.Int32(config.DefaultClientConnectionBurst), } - defaultIntegrations := &config.Integrations{ - Frameworks: []string{job.FrameworkName}, - } - testcases := []struct { name string configFile string wantConfiguration config.Configuration - wantOptions ctrl.Options wantError error }{ - { - name: "default config", - configFile: "", - wantConfiguration: config.Configuration{ - Namespace: pointer.String(config.DefaultNamespace), - InternalCertManagement: enableDefaultInternalCertManagement, - ClientConnection: defaultClientConnection, - Integrations: defaultIntegrations, - }, - wantOptions: ctrl.Options{ - Port: config.DefaultWebhookPort, - HealthProbeBindAddress: config.DefaultHealthProbeBindAddress, - MetricsBindAddress: config.DefaultMetricsBindAddress, - LeaderElectionID: "", - LeaderElection: false, - }, - }, - { - name: "namespace overwrite config", - configFile: namespaceOverWriteConfig, - wantConfiguration: config.Configuration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: config.GroupVersion.String(), - Kind: "Configuration", - }, - Namespace: pointer.String("kueue-tenant-a"), - ManageJobsWithoutQueueName: false, - InternalCertManagement: enableDefaultInternalCertManagement, - ClientConnection: defaultClientConnection, - Integrations: defaultIntegrations, - }, - wantOptions: defaultControlOptions, - }, - { - name: "ControllerManagerConfigurationSpec overwrite config", - configFile: ctrlManagerConfigSpecOverWriteConfig, - wantConfiguration: config.Configuration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: config.GroupVersion.String(), - Kind: "Configuration", - }, - Namespace: pointer.String(config.DefaultNamespace), - ManageJobsWithoutQueueName: false, - InternalCertManagement: enableDefaultInternalCertManagement, - ClientConnection: defaultClientConnection, - Integrations: defaultIntegrations, - }, - wantOptions: ctrl.Options{ - HealthProbeBindAddress: ":38081", - MetricsBindAddress: ":38080", - Port: 9444, - LeaderElection: true, - LeaderElectionID: "test-id", - }, - }, - { - name: "cert options overwrite config", - configFile: certOverWriteConfig, - wantConfiguration: config.Configuration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: config.GroupVersion.String(), - Kind: "Configuration", - }, - Namespace: pointer.String(config.DefaultNamespace), - ManageJobsWithoutQueueName: false, - InternalCertManagement: &config.InternalCertManagement{ - Enable: pointer.Bool(true), - WebhookServiceName: pointer.String("kueue-tenant-a-webhook-service"), - WebhookSecretName: pointer.String("kueue-tenant-a-webhook-server-cert"), - }, - ClientConnection: defaultClientConnection, - Integrations: defaultIntegrations, - }, - wantOptions: defaultControlOptions, - }, - { - name: "disable cert overwrite config", - configFile: disableCertOverWriteConfig, - wantConfiguration: config.Configuration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: config.GroupVersion.String(), - Kind: "Configuration", - }, - Namespace: pointer.String(config.DefaultNamespace), - ManageJobsWithoutQueueName: false, - InternalCertManagement: &config.InternalCertManagement{ - Enable: pointer.Bool(false), - }, - ClientConnection: defaultClientConnection, - Integrations: defaultIntegrations, - }, - wantOptions: defaultControlOptions, - }, - { - name: "leaderElection disabled config", - configFile: leaderElectionDisabledConfig, - wantConfiguration: config.Configuration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: config.GroupVersion.String(), - Kind: "Configuration", - }, - Namespace: pointer.String("kueue-system"), - ManageJobsWithoutQueueName: false, - InternalCertManagement: enableDefaultInternalCertManagement, - ClientConnection: defaultClientConnection, - Integrations: defaultIntegrations, - }, - wantOptions: ctrl.Options{ - Port: config.DefaultWebhookPort, - HealthProbeBindAddress: config.DefaultHealthProbeBindAddress, - MetricsBindAddress: config.DefaultMetricsBindAddress, - LeaderElectionID: "", - LeaderElection: false, - }, - }, - { - name: "enable waitForPodsReady config", - configFile: waitForPodsReadyEnabledConfig, - wantConfiguration: config.Configuration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: config.GroupVersion.String(), - Kind: "Configuration", - }, - Namespace: pointer.String(config.DefaultNamespace), - ManageJobsWithoutQueueName: false, - InternalCertManagement: enableDefaultInternalCertManagement, - WaitForPodsReady: &config.WaitForPodsReady{ - Enable: true, - BlockAdmission: pointer.Bool(true), - Timeout: &metav1.Duration{Duration: 5 * time.Minute}, - }, - ClientConnection: defaultClientConnection, - Integrations: defaultIntegrations, - }, - wantOptions: ctrl.Options{ - Port: config.DefaultWebhookPort, - HealthProbeBindAddress: config.DefaultHealthProbeBindAddress, - MetricsBindAddress: config.DefaultMetricsBindAddress, - }, - }, - { - name: "clientConnection config", - configFile: clientConnectionConfig, - wantConfiguration: config.Configuration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: config.GroupVersion.String(), - Kind: "Configuration", - }, - Namespace: pointer.String(config.DefaultNamespace), - ManageJobsWithoutQueueName: false, - InternalCertManagement: enableDefaultInternalCertManagement, - ClientConnection: &config.ClientConnection{ - QPS: pointer.Float32(50), - Burst: pointer.Int32(100), - }, - Integrations: defaultIntegrations, - }, - wantOptions: defaultControlOptions, - }, { name: "integrations config", configFile: integrationsConfig, @@ -405,11 +100,6 @@ integrations: Frameworks: []string{job.FrameworkName}, }, }, - wantOptions: ctrl.Options{ - Port: config.DefaultWebhookPort, - HealthProbeBindAddress: config.DefaultHealthProbeBindAddress, - MetricsBindAddress: config.DefaultMetricsBindAddress, - }, }, { name: "bad integrations config", @@ -420,14 +110,14 @@ integrations: for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - options, cfg, err := apply(tc.configFile) + _, cfg, err := apply(tc.configFile) if tc.wantError == nil { + if err != nil { + t.Errorf("Unexpected error:%s", err) + } if diff := cmp.Diff(tc.wantConfiguration, cfg, configCmpOpts...); diff != "" { t.Errorf("Unexpected config (-want +got):\n%s", diff) } - if diff := cmp.Diff(tc.wantOptions, options, ctrlOptsCmpOpts...); diff != "" { - t.Errorf("Unexpected options (-want +got):\n%s", diff) - } } else { if diff := cmp.Diff(tc.wantError.Error(), err.Error()); diff != "" { t.Errorf("Unexpected error (-want +got):\n%s", diff) diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000000..61e0f0811d --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,159 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "bytes" + "fmt" + "os" + "reflect" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + ctrl "sigs.k8s.io/controller-runtime" + + configapi "sigs.k8s.io/kueue/apis/config/v1beta1" +) + +// fromFile provides an alternative to the deprecated ctrl.ConfigFile().AtPath(path).OfKind(&cfg) +func fromFile(path string, scheme *runtime.Scheme, cfg *configapi.Configuration) error { + content, err := os.ReadFile(path) + if err != nil { + return err + } + + codecs := serializer.NewCodecFactory(scheme) + + // Regardless of if the bytes are of any external version, + // it will be read successfully and converted into the internal version + return runtime.DecodeInto(codecs.UniversalDecoder(), content, cfg) +} + +// addTo provides an alternative to the deprecated o.AndFrom(&cfg) +func addTo(o *ctrl.Options, cfg *configapi.Configuration) { + addLeaderElectionTo(o, cfg) + if o.MetricsBindAddress == "" && cfg.Metrics.BindAddress != "" { + o.MetricsBindAddress = cfg.Metrics.BindAddress + } + + if o.HealthProbeBindAddress == "" && cfg.Health.HealthProbeBindAddress != "" { + o.HealthProbeBindAddress = cfg.Health.HealthProbeBindAddress + } + + if o.ReadinessEndpointName == "" && cfg.Health.ReadinessEndpointName != "" { + o.ReadinessEndpointName = cfg.Health.ReadinessEndpointName + } + + if o.LivenessEndpointName == "" && cfg.Health.LivenessEndpointName != "" { + o.LivenessEndpointName = cfg.Health.LivenessEndpointName + } + + if o.Port == 0 && cfg.Webhook.Port != nil { + o.Port = *cfg.Webhook.Port + } + + if o.Host == "" && cfg.Webhook.Host != "" { + o.Host = cfg.Webhook.Host + } + + if o.CertDir == "" && cfg.Webhook.CertDir != "" { + o.CertDir = cfg.Webhook.CertDir + } + + if cfg.Controller != nil { + if o.Controller.CacheSyncTimeout == nil && cfg.Controller.CacheSyncTimeout != nil { + o.Controller.CacheSyncTimeout = cfg.Controller.CacheSyncTimeout + } + + if len(o.Controller.GroupKindConcurrency) == 0 && len(cfg.Controller.GroupKindConcurrency) > 0 { + o.Controller.GroupKindConcurrency = cfg.Controller.GroupKindConcurrency + } + } +} + +func addLeaderElectionTo(o *ctrl.Options, cfg *configapi.Configuration) { + if cfg.LeaderElection == nil { + // The source does not have any configuration; noop + return + } + + if !o.LeaderElection && cfg.LeaderElection.LeaderElect != nil { + o.LeaderElection = *cfg.LeaderElection.LeaderElect + } + + if o.LeaderElectionResourceLock == "" && cfg.LeaderElection.ResourceLock != "" { + o.LeaderElectionResourceLock = cfg.LeaderElection.ResourceLock + } + + if o.LeaderElectionNamespace == "" && cfg.LeaderElection.ResourceNamespace != "" { + o.LeaderElectionNamespace = cfg.LeaderElection.ResourceNamespace + } + + if o.LeaderElectionID == "" && cfg.LeaderElection.ResourceName != "" { + o.LeaderElectionID = cfg.LeaderElection.ResourceName + } + + if o.LeaseDuration == nil && !reflect.DeepEqual(cfg.LeaderElection.LeaseDuration, metav1.Duration{}) { + o.LeaseDuration = &cfg.LeaderElection.LeaseDuration.Duration + } + + if o.RenewDeadline == nil && !reflect.DeepEqual(cfg.LeaderElection.RenewDeadline, metav1.Duration{}) { + o.RenewDeadline = &cfg.LeaderElection.RenewDeadline.Duration + } + + if o.RetryPeriod == nil && !reflect.DeepEqual(cfg.LeaderElection.RetryPeriod, metav1.Duration{}) { + o.RetryPeriod = &cfg.LeaderElection.RetryPeriod.Duration + } +} + +func Encode(scheme *runtime.Scheme, cfg *configapi.Configuration) (string, error) { + codecs := serializer.NewCodecFactory(scheme) + const mediaType = runtime.ContentTypeYAML + info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) + if !ok { + return "", fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType) + } + + encoder := codecs.EncoderForVersion(info.Serializer, configapi.GroupVersion) + buf := new(bytes.Buffer) + if err := encoder.Encode(cfg, buf); err != nil { + return "", err + } + return buf.String(), nil +} + +// Load returns a set of controller options and configuration from the given file, if the config file path is empty +// it used the default configapi values. +func Load(scheme *runtime.Scheme, configFile string) (ctrl.Options, configapi.Configuration, error) { + var err error + options := ctrl.Options{ + Scheme: scheme, + } + + cfg := configapi.Configuration{} + if configFile == "" { + scheme.Default(&cfg) + } else { + err := fromFile(configFile, scheme, &cfg) + if err != nil { + return options, cfg, err + } + } + addTo(&options, &cfg) + return options, cfg, err +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 0000000000..d853dac002 --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,600 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + runtimeconfig "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + + configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + "sigs.k8s.io/kueue/pkg/controller/jobs/job" +) + +func TestLoad(t *testing.T) { + test_scheme := runtime.NewScheme() + err := configapi.AddToScheme(test_scheme) + if err != nil { + t.Fatal(err) + } + + // temp dir + tmpDir, err := os.MkdirTemp("", "temp") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + namespaceOverWriteConfig := filepath.Join(tmpDir, "namespace-overwrite.yaml") + if err := os.WriteFile(namespaceOverWriteConfig, []byte(` +apiVersion: config.kueue.x-k8s.io/v1beta1 +kind: Configuration +namespace: kueue-tenant-a +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io +webhook: + port: 9443 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + ctrlManagerConfigSpecOverWriteConfig := filepath.Join(tmpDir, "ctrl-manager-config-spec-overwrite.yaml") + if err := os.WriteFile(ctrlManagerConfigSpecOverWriteConfig, []byte(` +apiVersion: config.kueue.x-k8s.io/v1beta1 +kind: Configuration +namespace: kueue-system +health: + healthProbeBindAddress: :38081 +metrics: + bindAddress: :38080 +leaderElection: + leaderElect: true + resourceName: test-id +webhook: + port: 9444 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + certOverWriteConfig := filepath.Join(tmpDir, "cert-overwrite.yaml") + if err := os.WriteFile(certOverWriteConfig, []byte(` +apiVersion: config.kueue.x-k8s.io/v1beta1 +kind: Configuration +namespace: kueue-system +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io +webhook: + port: 9443 +internalCertManagement: + enable: true + webhookServiceName: kueue-tenant-a-webhook-service + webhookSecretName: kueue-tenant-a-webhook-server-cert +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + disableCertOverWriteConfig := filepath.Join(tmpDir, "disable-cert-overwrite.yaml") + if err := os.WriteFile(disableCertOverWriteConfig, []byte(` +apiVersion: config.kueue.x-k8s.io/v1beta1 +kind: Configuration +namespace: kueue-system +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io +webhook: + port: 9443 +internalCertManagement: + enable: false +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + leaderElectionDisabledConfig := filepath.Join(tmpDir, "leaderElection-disabled.yaml") + if err := os.WriteFile(leaderElectionDisabledConfig, []byte(` +apiVersion: config.kueue.x-k8s.io/v1beta1 +kind: Configuration +namespace: kueue-system +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: false +webhook: + port: 9443 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + waitForPodsReadyEnabledConfig := filepath.Join(tmpDir, "waitForPodsReady-enabled.yaml") + if err := os.WriteFile(waitForPodsReadyEnabledConfig, []byte(` +apiVersion: config.kueue.x-k8s.io/v1beta1 +kind: Configuration +waitForPodsReady: + enable: true +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + clientConnectionConfig := filepath.Join(tmpDir, "clientConnection.yaml") + if err := os.WriteFile(clientConnectionConfig, []byte(` +apiVersion: config.kueue.x-k8s.io/v1beta1 +kind: Configuration +namespace: kueue-system +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io +webhook: + port: 9443 +clientConnection: + qps: 50 + burst: 100 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + fullControllerConfig := filepath.Join(tmpDir, "fullControllerConfig.yaml") + if err := os.WriteFile(fullControllerConfig, []byte(` +apiVersion: config.kueue.x-k8s.io/v1beta1 +kind: Configuration +namespace: kueue-system +health: + healthProbeBindAddress: :8081 + readinessEndpointName: ready + livenessEndpointName: live +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: c1f6bfd2.kueue.x-k8s.io + resourceNamespace: namespace + resourceLock: lock + leaseDuration: 100s + renewDeadline: 15s + retryPeriod: 30s +webhook: + port: 9443 + host: host + certDir: certDir +controller: + groupKindConcurrency: + workload: 5 + cacheSyncTimeout: 3 +clientConnection: + qps: 50 + burst: 100 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + integrationsConfig := filepath.Join(tmpDir, "integrations.yaml") + if err := os.WriteFile(integrationsConfig, []byte(` +apiVersion: config.kueue.x-k8s.io/v1beta1 +kind: Configuration +integrations: + frameworks: + - batch/job +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + defaultControlOptions := ctrl.Options{ + Port: configapi.DefaultWebhookPort, + HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, + MetricsBindAddress: configapi.DefaultMetricsBindAddress, + LeaderElectionID: configapi.DefaultLeaderElectionID, + LeaderElection: true, + } + + enableDefaultInternalCertManagement := &configapi.InternalCertManagement{ + Enable: pointer.Bool(true), + WebhookServiceName: pointer.String(configapi.DefaultWebhookServiceName), + WebhookSecretName: pointer.String(configapi.DefaultWebhookSecretName), + } + + ctrlOptsCmpOpts := []cmp.Option{ + cmpopts.IgnoreUnexported(ctrl.Options{}), + cmpopts.IgnoreFields(ctrl.Options{}, "Scheme", "Logger"), + } + + // Ignore the controller manager section since it's side effect is checked against + // the content of the resulting options + configCmpOpts := []cmp.Option{ + cmpopts.IgnoreFields(configapi.Configuration{}, "ControllerManager"), + } + + defaultClientConnection := &configapi.ClientConnection{ + QPS: pointer.Float32(configapi.DefaultClientConnectionQPS), + Burst: pointer.Int32(configapi.DefaultClientConnectionBurst), + } + + defaultIntegrations := &configapi.Integrations{ + Frameworks: []string{job.FrameworkName}, + } + + testcases := []struct { + name string + configFile string + wantConfiguration configapi.Configuration + wantOptions ctrl.Options + wantError error + }{ + { + name: "default config", + configFile: "", + wantConfiguration: configapi.Configuration{ + Namespace: pointer.String(configapi.DefaultNamespace), + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + }, + wantOptions: ctrl.Options{ + Port: configapi.DefaultWebhookPort, + HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, + MetricsBindAddress: configapi.DefaultMetricsBindAddress, + LeaderElectionID: "", + LeaderElection: false, + }, + }, + { + name: "bad path", + configFile: ".", + wantError: &fs.PathError{ + Op: "read", + Path: ".", + Err: errors.New("is a directory"), + }, + }, + { + name: "namespace overwrite config", + configFile: namespaceOverWriteConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: pointer.String("kueue-tenant-a"), + ManageJobsWithoutQueueName: false, + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + }, + wantOptions: defaultControlOptions, + }, + { + name: "ControllerManagerConfigurationSpec overwrite config", + configFile: ctrlManagerConfigSpecOverWriteConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: pointer.String(configapi.DefaultNamespace), + ManageJobsWithoutQueueName: false, + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + }, + wantOptions: ctrl.Options{ + HealthProbeBindAddress: ":38081", + MetricsBindAddress: ":38080", + Port: 9444, + LeaderElection: true, + LeaderElectionID: "test-id", + }, + }, + { + name: "cert options overwrite config", + configFile: certOverWriteConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: pointer.String(configapi.DefaultNamespace), + ManageJobsWithoutQueueName: false, + InternalCertManagement: &configapi.InternalCertManagement{ + Enable: pointer.Bool(true), + WebhookServiceName: pointer.String("kueue-tenant-a-webhook-service"), + WebhookSecretName: pointer.String("kueue-tenant-a-webhook-server-cert"), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + }, + wantOptions: defaultControlOptions, + }, + { + name: "disable cert overwrite config", + configFile: disableCertOverWriteConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: pointer.String(configapi.DefaultNamespace), + ManageJobsWithoutQueueName: false, + InternalCertManagement: &configapi.InternalCertManagement{ + Enable: pointer.Bool(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + }, + wantOptions: defaultControlOptions, + }, + { + name: "leaderElection disabled config", + configFile: leaderElectionDisabledConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: pointer.String("kueue-system"), + ManageJobsWithoutQueueName: false, + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + }, + wantOptions: ctrl.Options{ + Port: configapi.DefaultWebhookPort, + HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, + MetricsBindAddress: configapi.DefaultMetricsBindAddress, + LeaderElectionID: "", + LeaderElection: false, + }, + }, + { + name: "enable waitForPodsReady config", + configFile: waitForPodsReadyEnabledConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: pointer.String(configapi.DefaultNamespace), + ManageJobsWithoutQueueName: false, + InternalCertManagement: enableDefaultInternalCertManagement, + WaitForPodsReady: &configapi.WaitForPodsReady{ + Enable: true, + BlockAdmission: pointer.Bool(true), + Timeout: &metav1.Duration{Duration: 5 * time.Minute}, + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + }, + wantOptions: ctrl.Options{ + Port: configapi.DefaultWebhookPort, + HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, + MetricsBindAddress: configapi.DefaultMetricsBindAddress, + }, + }, + { + name: "clientConnection config", + configFile: clientConnectionConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: pointer.String(configapi.DefaultNamespace), + ManageJobsWithoutQueueName: false, + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: &configapi.ClientConnection{ + QPS: pointer.Float32(50), + Burst: pointer.Int32(100), + }, + Integrations: defaultIntegrations, + }, + wantOptions: defaultControlOptions, + }, + { + name: "fullController config", + configFile: fullControllerConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: pointer.String(configapi.DefaultNamespace), + ManageJobsWithoutQueueName: false, + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: &configapi.ClientConnection{ + QPS: pointer.Float32(50), + Burst: pointer.Int32(100), + }, + Integrations: defaultIntegrations, + }, + wantOptions: ctrl.Options{ + Port: configapi.DefaultWebhookPort, + HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, + ReadinessEndpointName: "ready", + LivenessEndpointName: "live", + MetricsBindAddress: configapi.DefaultMetricsBindAddress, + LeaderElection: true, + CertDir: "certDir", + Host: "host", + LeaderElectionID: configapi.DefaultLeaderElectionID, + LeaderElectionNamespace: "namespace", + LeaderElectionResourceLock: "lock", + LeaseDuration: pointer.Duration(time.Second * 100), + RenewDeadline: pointer.Duration(time.Second * 15), + RetryPeriod: pointer.Duration(time.Second * 30), + Controller: runtimeconfig.ControllerConfigurationSpec{ + GroupKindConcurrency: map[string]int{ + "workload": 5, + }, + CacheSyncTimeout: pointer.Duration(3), + }, + }, + }, + { + name: "integrations config", + configFile: integrationsConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: pointer.String(configapi.DefaultNamespace), + ManageJobsWithoutQueueName: false, + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: defaultClientConnection, + Integrations: &configapi.Integrations{ + // referencing job.FrameworkName ensures the link of job package + // therefore the batch/framework should be registered + Frameworks: []string{job.FrameworkName}, + }, + }, + wantOptions: ctrl.Options{ + Port: configapi.DefaultWebhookPort, + HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, + MetricsBindAddress: configapi.DefaultMetricsBindAddress, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + options, cfg, err := Load(test_scheme, tc.configFile) + if tc.wantError == nil { + if err != nil { + t.Errorf("Unexpected error:%s", err) + } + if diff := cmp.Diff(tc.wantConfiguration, cfg, configCmpOpts...); diff != "" { + t.Errorf("Unexpected config (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tc.wantOptions, options, ctrlOptsCmpOpts...); diff != "" { + t.Errorf("Unexpected options (-want +got):\n%s", diff) + } + } else { + if diff := cmp.Diff(tc.wantError.Error(), err.Error()); diff != "" { + t.Errorf("Unexpected error (-want +got):\n%s", diff) + } + } + }) + } +} + +func TestEncode(t *testing.T) { + test_scheme := runtime.NewScheme() + err := configapi.AddToScheme(test_scheme) + if err != nil { + t.Fatal(err) + } + + defaultConfig := &configapi.Configuration{} + test_scheme.Default(defaultConfig) + + testcases := []struct { + name string + scheme *runtime.Scheme + cfg *configapi.Configuration + wantResult map[string]any + }{ + + { + name: "empty", + scheme: test_scheme, + cfg: &configapi.Configuration{}, + wantResult: map[string]any{ + "apiVersion": "config.kueue.x-k8s.io/v1beta1", + "kind": "Configuration", + "manageJobsWithoutQueueName": false, + "health": map[string]any{}, + "metrics": map[string]any{}, + "webhook": map[string]any{}, + }, + }, + { + name: "default", + scheme: test_scheme, + cfg: defaultConfig, + wantResult: map[string]any{ + "apiVersion": "config.kueue.x-k8s.io/v1beta1", + "kind": "Configuration", + "namespace": configapi.DefaultNamespace, + "webhook": map[string]any{ + "port": int64(configapi.DefaultWebhookPort), + }, + "metrics": map[string]any{ + "bindAddress": configapi.DefaultMetricsBindAddress, + }, + "health": map[string]any{ + "healthProbeBindAddress": configapi.DefaultHealthProbeBindAddress, + }, + "internalCertManagement": map[string]any{ + "enable": true, + "webhookServiceName": configapi.DefaultWebhookServiceName, + "webhookSecretName": configapi.DefaultWebhookSecretName, + }, + "clientConnection": map[string]any{ + "burst": int64(configapi.DefaultClientConnectionBurst), + "qps": int64(configapi.DefaultClientConnectionQPS), + }, + "manageJobsWithoutQueueName": false, + "integrations": map[string]any{ + "frameworks": []any{"batch/job"}, + }, + }, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := Encode(tc.scheme, tc.cfg) + if err != nil { + t.Errorf("Unexpected error:%s", err) + } + gotMap := map[string]interface{}{} + err = yaml.Unmarshal([]byte(got), &gotMap) + if err != nil { + t.Errorf("Unable to unmarshal result:%s", err) + } + if diff := cmp.Diff(tc.wantResult, gotMap); diff != "" { + t.Errorf("Unexpected result (-want +got):\n%s", diff) + } + }) + } +}