Use protobuf in gardener components (gardener#3467)

* Don't default content type in config APIs

* Make content type optional in charts

* Configure clients to use protobuf wherever possible

* Enable protobuf also for seedmanagement

* Correct typos

* Improve gardenlet config defaults tests

charts/gardener/controlplane/charts/runtime/templates/admission-controller/configmap-componentconfig.yaml

@@ -16,8 +16,12 @@ data:
     apiVersion: admissioncontroller.config.gardener.cloud/v1alpha1
     kind: AdmissionControllerConfiguration
     gardenClientConnection:
-      acceptContentTypes: {{ required ".Values.global.admission.config.gardenClientConnection.acceptContentTypes is required" .Values.global.admission.config.gardenClientConnection.acceptContentTypes }}
-      contentType: {{ required ".Values.global.admission.config.gardenClientConnection.contentType is required" .Values.global.admission.config.gardenClientConnection.contentType }}
+      {{- with .Values.global.admission.config.gardenClientConnection.acceptContentTypes }}
+      acceptContentTypes: {{ . | quote }}
+      {{- end }}
+      {{- with .Values.global.admission.config.gardenClientConnection.contentType }}
+      contentType: {{ . | quote }}
+      {{- end }}
       qps: {{ required ".Values.global.admission.config.gardenClientConnection.qps is required" .Values.global.admission.config.gardenClientConnection.qps }}
       burst: {{ required ".Values.global.admission.config.gardenClientConnection.burst is required" .Values.global.admission.config.gardenClientConnection.burst }}
       {{- if .Values.global.admission.config.gardenClientConnection.kubeconfig }}

charts/gardener/controlplane/charts/runtime/templates/controller-manager/configmap-componentconfig.yaml

@@ -16,8 +16,12 @@ data:
     apiVersion: controllermanager.config.gardener.cloud/v1alpha1
     kind: ControllerManagerConfiguration
     gardenClientConnection:
-      acceptContentTypes: {{ required ".Values.global.controller.config.gardenClientConnection.acceptContentTypes is required" .Values.global.controller.config.gardenClientConnection.acceptContentTypes }}
-      contentType: {{ required ".Values.global.controller.config.gardenClientConnection.contentType is required" .Values.global.controller.config.gardenClientConnection.contentType }}
+      {{- with .Values.global.controller.config.gardenClientConnection.acceptContentTypes }}
+      acceptContentTypes: {{ . | quote }}
+      {{- end }}
+      {{- with .Values.global.controller.config.gardenClientConnection.contentType }}
+      contentType: {{ . | quote }}
+      {{- end }}
       qps: {{ required ".Values.global.controller.config.gardenClientConnection.qps is required" .Values.global.controller.config.gardenClientConnection.qps }}
       burst: {{ required ".Values.global.controller.config.gardenClientConnection.burst is required" .Values.global.controller.config.gardenClientConnection.burst }}
       {{- if .Values.global.controller.config.gardenClientConnection.kubeconfig }}

charts/gardener/controlplane/charts/runtime/templates/scheduler/configmap-componentconfig.yaml

@@ -16,8 +16,12 @@ data:
     apiVersion: scheduler.config.gardener.cloud/v1alpha1
     kind: SchedulerConfiguration
     clientConnection:
-      acceptContentTypes: {{ required ".Values.global.scheduler.config.clientConnection.acceptContentTypes is required" .Values.global.scheduler.config.clientConnection.acceptContentTypes }}
-      contentType: {{ required ".Values.global.scheduler.config.clientConnection.contentType is required" .Values.global.scheduler.config.clientConnection.contentType }}
+      {{- with .Values.global.scheduler.config.clientConnection.acceptContentTypes }}
+      acceptContentTypes: {{ . | quote }}
+      {{- end }}
+      {{- with .Values.global.scheduler.config.clientConnection.contentType }}
+      contentType: {{ . | quote }}
+      {{- end }}
       qps: {{ required ".Values.global.scheduler.config.clientConnection.qps is required" .Values.global.scheduler.config.clientConnection.qps }}
       burst: {{ required ".Values.global.scheduler.config.clientConnection.burst is required" .Values.global.scheduler.config.clientConnection.burst }}
       {{- if .Values.global.scheduler.config.clientConnection.kubeconfig }}

charts/gardener/controlplane/values.yaml

@@ -272,8 +272,8 @@ global:
     vpa: false
     config:
       gardenClientConnection:
-        acceptContentTypes: application/json
-        contentType: application/json
+       # acceptContentTypes: application/json
+       # contentType: application/json
         qps: 100
         burst: 130
       server:
@@ -328,8 +328,8 @@ global:
     vpa: false
     config:
       gardenClientConnection:
-        acceptContentTypes: application/json
-        contentType: application/json
+      # acceptContentTypes: application/json
+      # contentType: application/json
         qps: 100
         burst: 130
       controllers:
@@ -421,8 +421,8 @@ global:
     vpa: false
     config:
       clientConnection:
-        acceptContentTypes: application/json
-        contentType: application/json
+      # acceptContentTypes: application/json
+      # contentType: application/json
         qps: 100
         burst: 130
       leaderElection:

charts/gardener/gardenlet/charts/runtime/templates/configmap-componentconfig.yaml

@@ -16,8 +16,12 @@ data:
     apiVersion: gardenlet.config.gardener.cloud/v1alpha1
     kind: GardenletConfiguration
     gardenClientConnection:
-      acceptContentTypes: {{ required ".Values.global.gardenlet.config.gardenClientConnection.acceptContentTypes is required" .Values.global.gardenlet.config.gardenClientConnection.acceptContentTypes }}
-      contentType: {{ required ".Values.global.gardenlet.config.gardenClientConnection.contentType is required" .Values.global.gardenlet.config.gardenClientConnection.contentType }}
+      {{- with .Values.global.gardenlet.config.gardenClientConnection.acceptContentTypes }}
+      acceptContentTypes: {{ . | quote }}
+      {{- end }}
+      {{- with .Values.global.gardenlet.config.gardenClientConnection.contentType }}
+      contentType: {{ . | quote }}
+      {{- end }}
       qps: {{ required ".Values.global.gardenlet.config.gardenClientConnection.qps is required" .Values.global.gardenlet.config.gardenClientConnection.qps }}
       burst: {{ required ".Values.global.gardenlet.config.gardenClientConnection.burst is required" .Values.global.gardenlet.config.gardenClientConnection.burst }}
       {{- if .Values.global.gardenlet.config.gardenClientConnection.gardenClusterAddress }}
@@ -40,16 +44,24 @@ data:
       kubeconfig: /etc/gardenlet/kubeconfig-garden/kubeconfig
       {{- end }}
     seedClientConnection:
-      acceptContentTypes: {{ required ".Values.global.gardenlet.config.seedClientConnection.acceptContentTypes is required" .Values.global.gardenlet.config.seedClientConnection.acceptContentTypes }}
-      contentType: {{ required ".Values.global.gardenlet.config.seedClientConnection.contentType is required" .Values.global.gardenlet.config.seedClientConnection.contentType }}
+      {{- with .Values.global.gardenlet.config.seedClientConnection.acceptContentTypes }}
+      acceptContentTypes: {{ . | quote }}
+      {{- end }}
+      {{- with .Values.global.gardenlet.config.seedClientConnection.contentType }}
+      contentType: {{ . | quote }}
+      {{- end }}
       qps: {{ required ".Values.global.gardenlet.config.seedClientConnection.qps is required" .Values.global.gardenlet.config.seedClientConnection.qps }}
       burst: {{ required ".Values.global.gardenlet.config.seedClientConnection.burst is required" .Values.global.gardenlet.config.seedClientConnection.burst }}
       {{- if .Values.global.gardenlet.config.seedClientConnection.kubeconfig }}
       kubeconfig: /etc/gardenlet/kubeconfig-seed/kubeconfig
       {{- end }}
     shootClientConnection:
-      acceptContentTypes: {{ required ".Values.global.gardenlet.config.shootClientConnection.acceptContentTypes is required" .Values.global.gardenlet.config.shootClientConnection.acceptContentTypes }}
-      contentType: {{ required ".Values.global.gardenlet.config.shootClientConnection.contentType is required" .Values.global.gardenlet.config.shootClientConnection.contentType }}
+      {{- with .Values.global.gardenlet.config.shootClientConnection.acceptContentTypes }}
+      acceptContentTypes: {{ . | quote }}
+      {{- end }}
+      {{- with .Values.global.gardenlet.config.shootClientConnection.contentType }}
+      contentType: {{ . | quote }}
+      {{- end }}
       qps: {{ required ".Values.global.gardenlet.config.shootClientConnection.qps is required" .Values.global.gardenlet.config.shootClientConnection.qps }}
       burst: {{ required ".Values.global.gardenlet.config.shootClientConnection.burst is required" .Values.global.gardenlet.config.shootClientConnection.burst }}
     controllers:

charts/gardener/gardenlet/values.yaml

@@ -28,8 +28,8 @@ global:
   #  Please find documentation in docs/deployment/image_vector.md
     config:
       gardenClientConnection:
-        acceptContentTypes: application/json
-        contentType: application/json
+      # acceptContentTypes: application/json
+      # contentType: application/json
         qps: 100
         burst: 130
       # gardenClusterAddress: https://some-external-ip-address-to-garden-cluster
@@ -52,15 +52,15 @@ global:
       #   `bootstrapKubeconfig` and `kubeconfigSecret` then it will try to create a CertificateSigningRequest
       #   and to procure a client certificate.
       seedClientConnection:
-        acceptContentTypes: application/json
-        contentType: application/json
+      # acceptContentTypes: application/json
+      # contentType: application/json
         qps: 100
         burst: 130
       # kubeconfig: |
       #   Specify a kubeconfig for the seed cluster here if you don't want to use the Gardenlet's service account.
       shootClientConnection:
-        acceptContentTypes: application/json
-        contentType: application/json
+      # acceptContentTypes: application/json
+      # contentType: application/json
         qps: 25
         burst: 50
       controllers:

cmd/gardener-apiserver/app/gardener_apiserver.go

@@ -164,19 +164,24 @@ func (o *Options) config(kubeAPIServerConfig *rest.Config, kubeClient *kubernete
 
 	// Initialize admission plugins
 	o.Recommended.ExtraAdmissionInitializers = func(c *genericapiserver.RecommendedConfig) ([]admission.PluginInitializer, error) {
+		protobufLoopbackConfig := *gardenerAPIServerConfig.LoopbackClientConfig
+		if protobufLoopbackConfig.ContentType == "" {
+			protobufLoopbackConfig.ContentType = runtime.ContentTypeProtobuf
+		}
+
 		// core client
-		coreClient, err := gardencoreclientset.NewForConfig(gardenerAPIServerConfig.LoopbackClientConfig)
+		coreClient, err := gardencoreclientset.NewForConfig(&protobufLoopbackConfig)
 		if err != nil {
 			return nil, err
 		}
-		o.CoreInformerFactory = gardencoreinformers.NewSharedInformerFactory(coreClient, gardenerAPIServerConfig.LoopbackClientConfig.Timeout)
+		o.CoreInformerFactory = gardencoreinformers.NewSharedInformerFactory(coreClient, protobufLoopbackConfig.Timeout)
 
 		// versioned core client
-		versionedCoreClient, err := gardenversionedcoreclientset.NewForConfig(gardenerAPIServerConfig.LoopbackClientConfig)
+		versionedCoreClient, err := gardenversionedcoreclientset.NewForConfig(&protobufLoopbackConfig)
 		if err != nil {
 			return nil, err
 		}
-		o.ExternalCoreInformerFactory = gardenexternalcoreinformers.NewSharedInformerFactory(versionedCoreClient, gardenerAPIServerConfig.LoopbackClientConfig.Timeout)
+		o.ExternalCoreInformerFactory = gardenexternalcoreinformers.NewSharedInformerFactory(versionedCoreClient, protobufLoopbackConfig.Timeout)
 
 		// seedmanagement client
 		seedManagementClient, err := seedmanagementclientset.NewForConfig(gardenerAPIServerConfig.LoopbackClientConfig)
@@ -186,11 +191,11 @@ func (o *Options) config(kubeAPIServerConfig *rest.Config, kubeClient *kubernete
 		o.SeedManagementInformerFactory = seedmanagementinformer.NewSharedInformerFactory(seedManagementClient, gardenerAPIServerConfig.LoopbackClientConfig.Timeout)
 
 		// settings client
-		settingsClient, err := settingsclientset.NewForConfig(gardenerAPIServerConfig.LoopbackClientConfig)
+		settingsClient, err := settingsclientset.NewForConfig(&protobufLoopbackConfig)
 		if err != nil {
 			return nil, err
 		}
-		o.SettingsInformerFactory = settingsinformer.NewSharedInformerFactory(settingsClient, gardenerAPIServerConfig.LoopbackClientConfig.Timeout)
+		o.SettingsInformerFactory = settingsinformer.NewSharedInformerFactory(settingsClient, protobufLoopbackConfig.Timeout)
 
 		// dynamic client
 		dynamicClient, err := dynamic.NewForConfig(kubeAPIServerConfig)
@@ -238,8 +243,13 @@ func (o Options) run(stopCh <-chan struct{}) error {
 		return err
 	}
 
+	protobufConfig := *kubeAPIServerConfig
+	if protobufConfig.ContentType == "" {
+		protobufConfig.ContentType = runtime.ContentTypeProtobuf
+	}
+
 	// kube client
-	kubeClient, err := kubernetes.NewForConfig(kubeAPIServerConfig)
+	kubeClient, err := kubernetes.NewForConfig(&protobufConfig)
 	if err != nil {
 		return err
 	}

example/20-componentconfig-gardener-admission-controller.yaml

@@ -2,8 +2,6 @@
 apiVersion: admissioncontroller.config.gardener.cloud/v1alpha1
 kind: AdmissionControllerConfiguration
 gardenClientConnection:
-  acceptContentTypes: application/json
-  contentType: application/json
   qps: 100
   burst: 130
 server:

example/20-componentconfig-gardener-controller-manager.yaml

@@ -2,8 +2,6 @@
 apiVersion: controllermanager.config.gardener.cloud/v1alpha1
 kind: ControllerManagerConfiguration
 gardenClientConnection:
-  acceptContentTypes: application/json
-  contentType: application/json
   qps: 100
   burst: 130
 controllers:

example/20-componentconfig-gardener-scheduler.yaml

@@ -2,8 +2,6 @@
 apiVersion: scheduler.config.gardener.cloud/v1alpha1
 kind: SchedulerConfiguration
 clientConnection:
-  acceptContentTypes: application/json
-  contentType: application/json
   qps: 100
   burst: 130
 leaderElection:

example/20-componentconfig-gardenlet.yaml

@@ -2,18 +2,12 @@
 apiVersion: gardenlet.config.gardener.cloud/v1alpha1
 kind: GardenletConfiguration
 gardenClientConnection:
-  acceptContentTypes: application/json
-  contentType: application/json
   qps: 100
   burst: 130
 seedClientConnection:
-  acceptContentTypes: application/json
-  contentType: application/json
   qps: 100
   burst: 130
 shootClientConnection:
-  acceptContentTypes: application/json
-  contentType: application/json
   qps: 25
   burst: 50
 controllers:

pkg/admissioncontroller/apis/config/v1alpha1/defaults.go

@@ -17,6 +17,7 @@ package v1alpha1
 import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1"
 )
 
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
@@ -46,3 +47,13 @@ func SetDefaults_AdmissionControllerConfiguration(obj *AdmissionControllerConfig
 		}
 	}
 }
+
+// SetDefaults_ClientConnectionConfiguration sets defaults for the garden client connection.
+func SetDefaults_ClientConnectionConfiguration(obj *componentbaseconfigv1alpha1.ClientConnectionConfiguration) {
+	if obj.QPS == 0.0 {
+		obj.QPS = 50.0
+	}
+	if obj.Burst == 0 {
+		obj.Burst = 100
+	}
+}

pkg/admissioncontroller/apis/config/v1alpha1/defaults_test.go

@@ -15,15 +15,16 @@
 package v1alpha1_test
 
 import (
-	. "github.com/gardener/gardener/pkg/admissioncontroller/apis/config/v1alpha1"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	rbacv1 "k8s.io/api/rbac/v1"
+	componentbaseconfigv1alpha1 "k8s.io/component-base/config/v1alpha1"
+
+	. "github.com/gardener/gardener/pkg/admissioncontroller/apis/config/v1alpha1"
 )
 
 var _ = Describe("Defaults", func() {
-	Describe("#SetDefaults_AdmissionControllerConfiguration", func() {
+	Describe("AdmissionControllerConfiguration", func() {
 		var obj *AdmissionControllerConfiguration
 
 		Context("Empty configuration", func() {
@@ -32,7 +33,7 @@ var _ = Describe("Defaults", func() {
 			})
 
 			It("should correctly default the admission controller configuration", func() {
-				SetDefaults_AdmissionControllerConfiguration(obj)
+				SetObjectDefaults_AdmissionControllerConfiguration(obj)
 
 				Expect(obj.LogLevel).To(Equal("info"))
 				Expect(obj.Server.HTTPS.BindAddress).To(Equal("0.0.0.0"))
@@ -56,12 +57,31 @@ var _ = Describe("Defaults", func() {
 				}
 			})
 			It("should correctly default the resource admission configuration if given", func() {
-				SetDefaults_AdmissionControllerConfiguration(obj)
+				SetObjectDefaults_AdmissionControllerConfiguration(obj)
 
 				Expect(obj.Server.ResourceAdmissionConfiguration.UnrestrictedSubjects[0].APIGroup).To(Equal(rbacv1.GroupName))
 				Expect(obj.Server.ResourceAdmissionConfiguration.UnrestrictedSubjects[1].APIGroup).To(Equal(rbacv1.GroupName))
 				Expect(obj.Server.ResourceAdmissionConfiguration.UnrestrictedSubjects[2].APIGroup).To(Equal(""))
 			})
 		})
+
+		Describe("GardenClientConnection", func() {
+			It("should not default ContentType and AcceptContentTypes", func() {
+				SetObjectDefaults_AdmissionControllerConfiguration(obj)
+
+				// ContentType fields will be defaulted by client constructors / controller-runtime based on whether a
+				// given APIGroup supports protobuf or not. defaults must not touch these, otherwise the integelligent
+				// logic will be overwritten
+				Expect(obj.GardenClientConnection.ContentType).To(BeEmpty())
+				Expect(obj.GardenClientConnection.AcceptContentTypes).To(BeEmpty())
+			})
+			It("should correctly default GardenClientConnection", func() {
+				SetObjectDefaults_AdmissionControllerConfiguration(obj)
+				Expect(obj.GardenClientConnection).To(Equal(componentbaseconfigv1alpha1.ClientConnectionConfiguration{
+					QPS:   50.0,
+					Burst: 100,
+				}))
+			})
+		})
 	})
 })