Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Custom config loader #889

Merged
merged 2 commits into from
Jun 26, 2023
Merged

Custom config loader #889

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
59c71f4
[api/config] Drop dependency on sigs.k8s.io/controller-runtime/pkg/co…
trasc Jun 21, 2023
65431a4
[config] Switch to internal implementation
trasc Jun 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
101 changes: 98 additions & 3 deletions apis/config/v1beta1/configuration_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ limitations under the License.
package v1beta1

import (
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
configv1alpha1 "k8s.io/component-base/config/v1alpha1"
)

//+kubebuilder:object:root=true
Expand All @@ -31,8 +33,8 @@ type Configuration struct {
// Defaults to kueue-system.
Namespace *string `json:"namespace,omitempty"`

// ControllerManagerConfigurationSpec returns the configurations for controllers
cfg.ControllerManagerConfigurationSpec `json:",inline"`
// ControllerManager returns the configurations for controllers
ControllerManager `json:",inline"`

// ManageJobsWithoutQueueName controls whether or not Kueue reconciles
// batch/v1.Jobs that don't set the annotation kueue.x-k8s.io/queue-name.
Expand Down Expand Up @@ -61,6 +63,99 @@ type Configuration struct {
Integrations *Integrations `json:"integrations,omitempty"`
}

type ControllerManager struct {
trasc marked this conversation as resolved.
Show resolved Hide resolved
// Webhook contains the controllers webhook configuration
// +optional
Webhook ControllerWebhook `json:"webhook,omitempty"`

// LeaderElection is the LeaderElection config to be used when configuring
// the manager.Manager leader election
// +optional
LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"`

// Metrics contains thw controller metrics configuration
// +optional
Metrics ControllerMetrics `json:"metrics,omitempty"`

// Health contains the controller health configuration
// +optional
Health ControllerHealth `json:"health,omitempty"`

// Controller contains global configuration options for controllers
// registered within this manager.
// +optional
Controller *ControllerConfigurationSpec `json:"controller,omitempty"`
}

// ControllerWebhook defines the webhook server for the controller.
type ControllerWebhook struct {
// Port is the port that the webhook server serves at.
// It is used to set webhook.Server.Port.
// +optional
Port *int `json:"port,omitempty"`

// Host is the hostname that the webhook server binds to.
// It is used to set webhook.Server.Host.
// +optional
Host string `json:"host,omitempty"`

// CertDir is the directory that contains the server key and certificate.
// if not set, webhook server would look up the server key and certificate in
// {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate
// must be named tls.key and tls.crt, respectively.
// +optional
CertDir string `json:"certDir,omitempty"`
}

// ControllerMetrics defines the metrics configs.
type ControllerMetrics struct {
// BindAddress is the TCP address that the controller should bind to
// for serving prometheus metrics.
// It can be set to "0" to disable the metrics serving.
// +optional
BindAddress string `json:"bindAddress,omitempty"`
}

// ControllerHealth defines the health configs.
type ControllerHealth struct {
// HealthProbeBindAddress is the TCP address that the controller should bind to
// for serving health probes
// It can be set to "0" or "" to disable serving the health probe.
// +optional
HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"`

// ReadinessEndpointName, defaults to "readyz"
// +optional
ReadinessEndpointName string `json:"readinessEndpointName,omitempty"`

// LivenessEndpointName, defaults to "healthz"
// +optional
LivenessEndpointName string `json:"livenessEndpointName,omitempty"`
}

// ControllerConfigurationSpec defines the global configuration for
// controllers registered with the manager.
type ControllerConfigurationSpec struct {
// GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation
// allowed for that controller.
//
// When a controller is registered within this manager using the builder utilities,
// users have to specify the type the controller reconciles in the For(...) call.
// If the object's kind passed matches one of the keys in this map, the concurrency
// for that controller is set to the number specified.
//
// The key is expected to be consistent in form with GroupKind.String(),
// e.g. ReplicaSet in apps group (regardless of version) would be `ReplicaSet.apps`.
//
// +optional
GroupKindConcurrency map[string]int `json:"groupKindConcurrency,omitempty"`

// CacheSyncTimeout refers to the time limit set to wait for syncing caches.
// Defaults to 2 minutes if not set.
// +optional
CacheSyncTimeout *time.Duration `json:"cacheSyncTimeout,omitempty"`
}

type WaitForPodsReady struct {
// Enable when true, indicates that each admitted workload
// blocks the admission of all other workloads from all queues until it is in the
Expand Down
85 changes: 42 additions & 43 deletions apis/config/v1beta1/defaults_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
componentconfigv1alpha1 "k8s.io/component-base/config/v1alpha1"
"k8s.io/utils/pointer"
ctrlconfigv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"

"sigs.k8s.io/kueue/pkg/controller/jobs/job"
)
Expand All @@ -38,14 +37,14 @@ const (
)

func TestSetDefaults_Configuration(t *testing.T) {
defaultCtrlManagerConfigurationSpec := ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{
Webhook: ctrlconfigv1alpha1.ControllerWebhook{
defaultCtrlManagerConfigurationSpec := ControllerManager{
Webhook: ControllerWebhook{
Port: pointer.Int(DefaultWebhookPort),
},
Metrics: ctrlconfigv1alpha1.ControllerMetrics{
Metrics: ControllerMetrics{
BindAddress: DefaultMetricsBindAddress,
},
Health: ctrlconfigv1alpha1.ControllerHealth{
Health: ControllerHealth{
HealthProbeBindAddress: DefaultHealthProbeBindAddress,
},
}
Expand All @@ -70,18 +69,18 @@ func TestSetDefaults_Configuration(t *testing.T) {
},
},
want: &Configuration{
Namespace: pointer.String(DefaultNamespace),
ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
Namespace: pointer.String(DefaultNamespace),
ControllerManager: defaultCtrlManagerConfigurationSpec,
InternalCertManagement: &InternalCertManagement{
Enable: pointer.Bool(false),
},
ClientConnection: defaultClientConnection,
Integrations: defaultIntegrations,
},
},
"defaulting ControllerManagerConfigurationSpec": {
"defaulting ControllerManager": {
original: &Configuration{
ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{
ControllerManager: ControllerManager{
LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{
LeaderElect: pointer.Bool(true),
},
Expand All @@ -92,14 +91,14 @@ func TestSetDefaults_Configuration(t *testing.T) {
},
want: &Configuration{
Namespace: pointer.String(DefaultNamespace),
ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{
Webhook: ctrlconfigv1alpha1.ControllerWebhook{
ControllerManager: ControllerManager{
Webhook: ControllerWebhook{
Port: pointer.Int(DefaultWebhookPort),
},
Metrics: ctrlconfigv1alpha1.ControllerMetrics{
Metrics: ControllerMetrics{
BindAddress: DefaultMetricsBindAddress,
},
Health: ctrlconfigv1alpha1.ControllerHealth{
Health: ControllerHealth{
HealthProbeBindAddress: DefaultHealthProbeBindAddress,
},
LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{
Expand All @@ -114,16 +113,16 @@ func TestSetDefaults_Configuration(t *testing.T) {
Integrations: defaultIntegrations,
},
},
"should not default ControllerManagerConfigurationSpec": {
"should not default ControllerManager": {
original: &Configuration{
ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{
Webhook: ctrlconfigv1alpha1.ControllerWebhook{
ControllerManager: ControllerManager{
Webhook: ControllerWebhook{
Port: pointer.Int(overwriteWebhookPort),
},
Metrics: ctrlconfigv1alpha1.ControllerMetrics{
Metrics: ControllerMetrics{
BindAddress: overwriteMetricBindAddress,
},
Health: ctrlconfigv1alpha1.ControllerHealth{
Health: ControllerHealth{
HealthProbeBindAddress: overwriteHealthProbeBindAddress,
},
LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{
Expand All @@ -138,14 +137,14 @@ func TestSetDefaults_Configuration(t *testing.T) {
},
want: &Configuration{
Namespace: pointer.String(DefaultNamespace),
ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{
Webhook: ctrlconfigv1alpha1.ControllerWebhook{
ControllerManager: ControllerManager{
Webhook: ControllerWebhook{
Port: pointer.Int(overwriteWebhookPort),
},
Metrics: ctrlconfigv1alpha1.ControllerMetrics{
Metrics: ControllerMetrics{
BindAddress: overwriteMetricBindAddress,
},
Health: ctrlconfigv1alpha1.ControllerHealth{
Health: ControllerHealth{
HealthProbeBindAddress: overwriteHealthProbeBindAddress,
},
LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{
Expand All @@ -162,7 +161,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
},
"should not set LeaderElectionID": {
original: &Configuration{
ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{
ControllerManager: ControllerManager{
LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{
LeaderElect: pointer.Bool(false),
},
Expand All @@ -173,14 +172,14 @@ func TestSetDefaults_Configuration(t *testing.T) {
},
want: &Configuration{
Namespace: pointer.String(DefaultNamespace),
ControllerManagerConfigurationSpec: ctrlconfigv1alpha1.ControllerManagerConfigurationSpec{
Webhook: ctrlconfigv1alpha1.ControllerWebhook{
ControllerManager: ControllerManager{
Webhook: ControllerWebhook{
Port: pointer.Int(DefaultWebhookPort),
},
Metrics: ctrlconfigv1alpha1.ControllerMetrics{
Metrics: ControllerMetrics{
BindAddress: DefaultMetricsBindAddress,
},
Health: ctrlconfigv1alpha1.ControllerHealth{
Health: ControllerHealth{
HealthProbeBindAddress: DefaultHealthProbeBindAddress,
},
LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{
Expand All @@ -199,8 +198,8 @@ func TestSetDefaults_Configuration(t *testing.T) {
Namespace: pointer.String(overwriteNamespace),
},
want: &Configuration{
Namespace: pointer.String(overwriteNamespace),
ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
Namespace: pointer.String(overwriteNamespace),
ControllerManager: defaultCtrlManagerConfigurationSpec,
InternalCertManagement: &InternalCertManagement{
Enable: pointer.Bool(true),
WebhookServiceName: pointer.String(DefaultWebhookServiceName),
Expand All @@ -218,8 +217,8 @@ func TestSetDefaults_Configuration(t *testing.T) {
},
},
want: &Configuration{
Namespace: pointer.String(overwriteNamespace),
ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
Namespace: pointer.String(overwriteNamespace),
ControllerManager: defaultCtrlManagerConfigurationSpec,
InternalCertManagement: &InternalCertManagement{
Enable: pointer.Bool(false),
},
Expand All @@ -239,8 +238,8 @@ func TestSetDefaults_Configuration(t *testing.T) {
},
},
want: &Configuration{
Namespace: pointer.String(overwriteNamespace),
ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
Namespace: pointer.String(overwriteNamespace),
ControllerManager: defaultCtrlManagerConfigurationSpec,
InternalCertManagement: &InternalCertManagement{
Enable: pointer.Bool(false),
},
Expand All @@ -260,8 +259,8 @@ func TestSetDefaults_Configuration(t *testing.T) {
ClientConnection: &ClientConnection{},
},
want: &Configuration{
Namespace: pointer.String(overwriteNamespace),
ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
Namespace: pointer.String(overwriteNamespace),
ControllerManager: defaultCtrlManagerConfigurationSpec,
InternalCertManagement: &InternalCertManagement{
Enable: pointer.Bool(false),
},
Expand All @@ -284,8 +283,8 @@ func TestSetDefaults_Configuration(t *testing.T) {
BlockAdmission: pointer.Bool(true),
Timeout: &podsReadyTimeoutTimeout,
},
Namespace: pointer.String(DefaultNamespace),
ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
Namespace: pointer.String(DefaultNamespace),
ControllerManager: defaultCtrlManagerConfigurationSpec,
InternalCertManagement: &InternalCertManagement{
Enable: pointer.Bool(false),
},
Expand All @@ -308,8 +307,8 @@ func TestSetDefaults_Configuration(t *testing.T) {
BlockAdmission: pointer.Bool(false),
Timeout: &podsReadyTimeoutTimeout,
},
Namespace: pointer.String(DefaultNamespace),
ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
Namespace: pointer.String(DefaultNamespace),
ControllerManager: defaultCtrlManagerConfigurationSpec,
InternalCertManagement: &InternalCertManagement{
Enable: pointer.Bool(false),
},
Expand All @@ -333,8 +332,8 @@ func TestSetDefaults_Configuration(t *testing.T) {
BlockAdmission: pointer.Bool(true),
Timeout: &podsReadyTimeoutOverwrite,
},
Namespace: pointer.String(DefaultNamespace),
ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
Namespace: pointer.String(DefaultNamespace),
ControllerManager: defaultCtrlManagerConfigurationSpec,
InternalCertManagement: &InternalCertManagement{
Enable: pointer.Bool(false),
},
Expand All @@ -352,8 +351,8 @@ func TestSetDefaults_Configuration(t *testing.T) {
},
},
want: &Configuration{
Namespace: pointer.String(DefaultNamespace),
ControllerManagerConfigurationSpec: defaultCtrlManagerConfigurationSpec,
Namespace: pointer.String(DefaultNamespace),
ControllerManager: defaultCtrlManagerConfigurationSpec,
InternalCertManagement: &InternalCertManagement{
Enable: pointer.Bool(false),
},
Expand Down