diff --git a/docs/README.md b/docs/README.md index f337d15bd9f..c0c2414a755 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,7 @@ * Components * [Gardener API server](concepts/apiserver.md) - * [In-Tree admission plugins](concepts/apiserver_admission_plugins.md) + * [In-Tree admission plugins](concepts/apiserver-admission-plugins.md) * [Gardener Controller Manager](concepts/controller-manager.md) * [Gardener Scheduler](concepts/scheduler.md) * [Gardener Admission Controller](concepts/admission-controller.md) diff --git a/docs/concepts/apiserver_admission_plugins.md b/docs/concepts/apiserver-admission-plugins.md similarity index 100% rename from docs/concepts/apiserver_admission_plugins.md rename to docs/concepts/apiserver-admission-plugins.md diff --git a/docs/concepts/etcd.md b/docs/concepts/etcd.md index 446b2fc15df..e51c71620de 100644 --- a/docs/concepts/etcd.md +++ b/docs/concepts/etcd.md @@ -35,14 +35,6 @@ When a [`gardenlet`](gardenlet.md) reconciles a `Shoot` resource or a [`gardener `etcd-druid` needs to manage the lifecycle of the desired etcd instance (today `main` or `events`). Likewise, when the `Shoot` or `Garden` is deleted, `gardenlet` or `gardener-operator` deletes the `Etcd` resources and [etcd Druid](https://github.com/gardener/etcd-druid/) takes care of cleaning up all related objects, e.g. the backing `StatefulSet`s. -## Autoscaling - -Gardenlet maintains [`HVPA`](https://github.com/gardener/hvpa-controller/blob/master/config/samples/autoscaling_v1alpha1_hvpa.yaml) objects for etcd `StatefulSet`s if the corresponding [feature gate](../deployment/feature_gates.md) is enabled. -This enables a vertical scaling for etcd. -Downscaling is handled more pessimistically to prevent many subsequent etcd restarts. -Thus, for `production` and `infrastructure` shoot clusters (or all garden clusters), downscaling is deactivated for the main etcd. -For all other shoot clusters, lower advertised requests/limits are only applied during a shoot's maintenance time window. - ## Backup If `Seed`s specify backups for etcd ([example](../../example/50-seed.yaml)), then Gardener and the respective [provider extensions](../extensions/overview.md) are responsible for creating a bucket on the cloud provider's side (modelled through a [BackupBucket resource](../extensions/backupbucket.md)). diff --git a/docs/deployment/feature_gates.md b/docs/deployment/feature_gates.md index 9f3ca1d98a3..f32bfcf6995 100644 --- a/docs/deployment/feature_gates.md +++ b/docs/deployment/feature_gates.md @@ -31,6 +31,7 @@ The following tables are a summary of the feature gates that you can set on diff | UseNamespacedCloudProfile | `false` | `Alpha` | `1.92` | | | ShootManagedIssuer | `false` | `Alpha` | `1.93` | | | VPAForETCD | `false` | `Alpha` | `1.94` | | +| VPAAndHPAForAPIServer | `false` | `Alpha` | `1.95` | | ## Feature Gates for Graduated or Deprecated Features @@ -190,15 +191,16 @@ A *General Availability* (GA) feature is also referred to as a *stable* feature. ## List of Feature Gates -| Feature | Relevant Components | Description | -|------------------------------------|-----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| HVPA | `gardenlet`, `gardener-operator` | Enables simultaneous horizontal and vertical scaling in garden or seed clusters. | -| HVPAForShootedSeed | `gardenlet` | Enables simultaneous horizontal and vertical scaling in managed seed (aka "shooted seed") clusters. | -| DefaultSeccompProfile | `gardenlet`, `gardener-operator` | Enables the defaulting of the seccomp profile for Gardener managed workload in the garden or seed to `RuntimeDefault`. | -| CoreDNSQueryRewriting | `gardenlet` | Enables automatic DNS query rewriting in shoot cluster's CoreDNS to shortcut name resolution of fully qualified in-cluster and out-of-cluster names, which follow a user-defined pattern. Details can be found in [DNS Search Path Optimization](../usage/dns-search-path-optimization.md). | -| IPv6SingleStack | `gardener-apiserver`, `gardenlet` | Allows creating seed and shoot clusters with [IPv6 single-stack networking](../usage/ipv6.md) enabled in their spec ([GEP-21](../proposals/21-ipv6-singlestack-local.md)). If enabled in gardenlet, the default behavior is unchanged, but setting `ipFamilies=[IPv6]` in the `seedConfig` is allowed. Only if the `ipFamilies` setting is changed, gardenlet behaves differently. | -| MutableShootSpecNetworkingNodes | `gardener-apiserver` | Allows updating the field `spec.networking.nodes`. The validity of the values has to be checked in the provider extensions. Only enable this feature gate when your system runs provider extensions which have implemented the validation. | -| ShootForceDeletion | `gardener-apiserver` | Allows forceful deletion of Shoots by annotating them with the `confirmation.gardener.cloud/force-deletion` annotation. | -| UseNamespacedCloudProfile | `gardener-apiserver` | Enables usage of `NamespacedCloudProfile`s in `Shoot`s. | -| ShootManagedIssuer | `gardenlet` | Enables the shoot managed issuer functionality described in GEP 24. | -| VPAForETCD | `gardenlet`, `gardener-operator` | Enables VPA for `etcd-main` and `etcd-events`, regardless of HVPA enablement. | +| Feature | Relevant Components | Description | +|---------------------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| HVPA | `gardenlet`, `gardener-operator` | Enables simultaneous horizontal and vertical scaling in garden or seed clusters. | +| HVPAForShootedSeed | `gardenlet` | Enables simultaneous horizontal and vertical scaling in managed seed (aka "shooted seed") clusters. | +| DefaultSeccompProfile | `gardenlet`, `gardener-operator` | Enables the defaulting of the seccomp profile for Gardener managed workload in the garden or seed to `RuntimeDefault`. | +| CoreDNSQueryRewriting | `gardenlet` | Enables automatic DNS query rewriting in shoot cluster's CoreDNS to shortcut name resolution of fully qualified in-cluster and out-of-cluster names, which follow a user-defined pattern. Details can be found in [DNS Search Path Optimization](../usage/dns-search-path-optimization.md). | +| IPv6SingleStack | `gardener-apiserver`, `gardenlet` | Allows creating seed and shoot clusters with [IPv6 single-stack networking](../usage/ipv6.md) enabled in their spec ([GEP-21](../proposals/21-ipv6-singlestack-local.md)). If enabled in gardenlet, the default behavior is unchanged, but setting `ipFamilies=[IPv6]` in the `seedConfig` is allowed. Only if the `ipFamilies` setting is changed, gardenlet behaves differently. | +| MutableShootSpecNetworkingNodes | `gardener-apiserver` | Allows updating the field `spec.networking.nodes`. The validity of the values has to be checked in the provider extensions. Only enable this feature gate when your system runs provider extensions which have implemented the validation. | +| ShootForceDeletion | `gardener-apiserver` | Allows forceful deletion of Shoots by annotating them with the `confirmation.gardener.cloud/force-deletion` annotation. | +| UseNamespacedCloudProfile | `gardener-apiserver` | Enables usage of `NamespacedCloudProfile`s in `Shoot`s. | +| ShootManagedIssuer | `gardenlet` | Enables the shoot managed issuer functionality described in GEP 24. | +| VPAForETCD | `gardenlet`, `gardener-operator` | Enables VPA for `etcd-main` and `etcd-events`, regardless of HVPA enablement. | +| VPAAndHPAForAPIServer | `gardenlet` | Enables an autoscaling mechanism for shoot kube-apiserver where it is scaled simultaneously by VPA and HPA on the same metric (CPU and memory usage). The pod-trashing cycle between VPA and HPA scaling on the same metric is avoided by configuring the HPA to scale on average usage (not on average utilization) and by picking the target average utilization values in sync with VPA's allowed maximums. The feature gate takes precedence over the `HVPA` feature gate when they are both enabled. | diff --git a/docs/development/autoscaling-specifics-for-components.md b/docs/development/autoscaling-specifics-for-components.md new file mode 100644 index 00000000000..e76fafd5836 --- /dev/null +++ b/docs/development/autoscaling-specifics-for-components.md @@ -0,0 +1,83 @@ +--- +title: Autoscaling Specifics for Components +--- + +## Overview + +This document describes the used autoscaling mechanism for several components. + +## Garden or Shoot Cluster etcd + +By default, if none of the autoscaling modes is requested the `etcd` is deployed with static resources, without autoscaling. + +However, there are two supported autoscaling modes for the Garden or Shoot cluster etcd. + +- `HVPA` + + In `HVPA` mode, the etcd is scaled by the [hvpa-controller](https://github.com/gardener/hvpa-controller). The gardenlet/gardener-operator is creating an `HVPA` resource for the etcd (`main` or `events`). + The `HVPA` enables a vertical scaling for etcd. + + The `HVPA` mode is the used autoscaling mode when the `HVPA` feature gate is enabled (and the `VPAForETCD` feature gate is disabled). + +- `VPA` + + In `VPA` mode, the etcd is scaled by a native `VPA` resource. + + The `VPA` mode is the used autoscaling mode when the `VPAForETCD` feature gate is enabled (takes precedence over the `HVPA` feature gate). + +For both of the autoscaling modes downscaling is handled more pessimistically to prevent many subsequent etcd restarts. Thus, for `production` and `infrastructure` Shoot clusters (or all Garden clusters), downscaling is deactivated for the main etcd. For all other Shoot clusters, lower advertised requests/limits are only applied during the Shoot's maintenance time window. + +## Shoot Kubernetes API Server + +There are three supported autoscaling modes for the Shoot Kubernetes API server. + +- `Baseline` + + In `Baseline` mode, the Shoot Kubernetes API server is scaled by active HPA and VPA in passive, recommend-only mode. + + The API server resource requests are computed based on the Shoot's minimum Nodes count: + | Range | Resource Requests | + |-------------|-------------------| + | [0, 2] | `800m`, `800Mi` | + | (2, 10] | `1000m`, `1100Mi` | + | (10, 50] | `1200m`, `1600Mi` | + | (50, 100] | `2500m`, `5200Mi` | + | (100, inf.) | `3000m`, `5200Mi` | + + The `Baseline` mode is the used autoscaling mode when the `HVPA` and `VPAAndHPAForAPIServer` feature gates are not enabled. + +- `HVPA` + + In `HVPA` mode, the Shoot Kubernetes API server is scaled by the [hvpa-controller](https://github.com/gardener/hvpa-controller). The gardenlet is creating an `HVPA` resource for the API server. The `HVPA` resource is backed by HPA and VPA both in recommend-only mode. The hvpa-controller is responsible for enabling simultaneous horizontal and vertical scaling by incorporating the recommendations from the HPA and VPA. + + The initial API server resource requests are `500m` and `1Gi`. + HVPA's HPA is scaling only on CPU (average utilization 80%). HVPA's VPA max allowed values are `8` CPU and `25G`. + + The `HVPA` mode is the used autoscaling mode when the `HVPA` feature gate is enabled (and the `VPAAndHPAForAPIServer` feature gate is disabled). + +- `VPAAndHPA` + + In `VPAAndHPA` mode, the Shoot Kubernetes API server is scaled simultaneously by VPA and HPA on the same metric (CPU and memory usage). The pod-trashing cycle between VPA and HPA scaling on the same metric is avoided by configuring the HPA to scale on average usage (not on average utilization) and by picking the target average utilization values in sync with VPA's allowed maximums. This makes possible VPA to first scale vertically on CPU/memory usage. Once all Pods' average CPU/memory usage is close to exceed the VPA's allowed maximum CPU/memory (the HPA's target average utilization, 1/7 less than VPA's allowed maximums), HPA is scaling horizontally (by adding a new replica). + + The `VPAAndHPA` mode is introduced to address disadvantages with HVPA: additional component; modifies the deployment triggering unnecessary rollouts; vertical scaling only at max replicas; stuck vertical resource requests when scaling in again; etc. + + The initial API server resource requests are `250m` and `500Mi`. + VPA's max allowed values are `7` CPU and `28G`. HPA's average target usage values are `6` CPU and `24G`. + + The `VPAAndHPA` mode is the used autoscaling mode when the `VPAAndHPAForAPIServer` feature gate is enabled (takes precedence over the `HVPA` feature gate). + +The API server's replica count in all scaling modes varies between 2 and 3. The min replicas count of 2 is imposed by the [High Availability of Shoot Control Plane Components](../development/high-availability.md#control-plane-components). + +The gardenlet sets the initial API server resource requests only when the Deployment is not found. When the Deployment exists, it is not overwriting the kube-apiserver container resources. + +## Disabling Scale Down for Components in the Shoot Control Plane + +Some Shoot clusters' control plane components can be overloaded and can have very high resource usage. The existing autoscaling solution could be imperfect to cover these cases. Scale down actions for such overloaded components could be disruptive. + +To prevent such disruptive scale-down actions it is possible to disable scale down of the etcd, Kubernetes API server and Kubernetes controller manager in the Shoot control plane by annotating the Shoot with `alpha.control-plane.scaling.shoot.gardener.cloud/scale-down-disabled=true`. + +There are the following specifics for when disabling scale-down for the Kubernetes API server component: +- In `Baseline` and `HVPA` modes the HPA's min and max replicas count are set to 4. +- In `VPAAndHPA` mode if the HPA resource exists and HPA's `spec.minReplicas` is not nil then the min replicas count is `max(spec.minReplicas, status.desiredReplicas)`. When scale-down is disabled, this allows operators to specify a custom value for HPA `spec.minReplicas` and this value not to be reverted by gardenlet. I.e, HPA _does_ scale down to min replicas but not below min replicas. HPA's max replicas count is 4. + +> Note: The `alpha.control-plane.scaling.shoot.gardener.cloud/scale-down-disabled` annotation is alpha and can be removed anytime without further notice. Only use it if you know what you do. diff --git a/pkg/component/apiserver/types.go b/pkg/component/apiserver/types.go index fbdea3d6132..6e324a71615 100644 --- a/pkg/component/apiserver/types.go +++ b/pkg/component/apiserver/types.go @@ -76,12 +76,27 @@ type AuditWebhook struct { Version *string } +// AutoscalingMode represents the different autoscaling modes for an API Server. +type AutoscalingMode int8 + +const ( + // AutoscalingModeBaseline differs substantially between kube-apiserver and gardener-apiserver. + // For kube-apiserver, it is active HPA and VPA in passive, recommend-only mode. + // For gardener-apiserver, it is VPA only. + AutoscalingModeBaseline = 0 + iota + // AutoscalingModeHVPA uses Gardener's custom HVPA autoscaler. + AutoscalingModeHVPA + // AutoscalingModeVPAAndHPA uses VPA on CPU utilization and HPA on CPU usage. + AutoscalingModeVPAAndHPA +) + // AutoscalingConfig contains information for configuring autoscaling settings for the API server. type AutoscalingConfig struct { + // Mode is the strategy for scaling the API server. + // Defaults to AutoscalingModeBaseline. + Mode AutoscalingMode // APIServerResources are the resource requirements for the API server container. APIServerResources corev1.ResourceRequirements - // HVPAEnabled states whether an HVPA object shall be deployed. If false, HPA and VPA will be used. - HVPAEnabled bool // Replicas is the number of pod replicas for the API server. Replicas *int32 // MinReplicas are the minimum Replicas for horizontal autoscaling. @@ -91,9 +106,9 @@ type AutoscalingConfig struct { // UseMemoryMetricForHvpaHPA states whether the memory metric shall be used when the HPA is configured in an HVPA // resource. UseMemoryMetricForHvpaHPA bool - // ScaleDownDisabledForHvpa states whether scale-down shall be disabled when HPA or VPA are configured in an HVPA - // resource. - ScaleDownDisabledForHvpa bool + // ScaleDownDisabled states whether scale-down shall be disabled. + // Only HVPA and VPAAndHPA autoscaling modes support disabling scale-down. + ScaleDownDisabled bool } // ETCDEncryptionConfig contains configuration for the encryption of resources in etcd. diff --git a/pkg/component/gardener/apiserver/apiserver_test.go b/pkg/component/gardener/apiserver/apiserver_test.go index d01417cd9ef..47e0f1ee0d4 100644 --- a/pkg/component/gardener/apiserver/apiserver_test.go +++ b/pkg/component/gardener/apiserver/apiserver_test.go @@ -1466,9 +1466,9 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook Expect(managedResourceSecretVirtual.Labels["resources.gardener.cloud/garbage-collectable-reference"]).To(Equal("true")) }) - Context("when HVPA is disabled", func() { + Context("when autoscaling mode is baseline", func() { BeforeEach(func() { - values.Values.Autoscaling.HVPAEnabled = false + values.Values.Autoscaling.Mode = apiserver.AutoscalingModeBaseline deployer = New(fakeClient, namespace, fakeSecretManager, values) }) @@ -1484,9 +1484,9 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook }) }) - Context("when HVPA is enabled", func() { + Context("when autoscaling mode is HVPA", func() { BeforeEach(func() { - values.Values.Autoscaling.HVPAEnabled = true + values.Values.Autoscaling.Mode = apiserver.AutoscalingModeHVPA deployer = New(fakeClient, namespace, fakeSecretManager, values) }) diff --git a/pkg/component/gardener/apiserver/hvpa.go b/pkg/component/gardener/apiserver/hvpa.go index e8a03052d19..9ce3c100ca6 100644 --- a/pkg/component/gardener/apiserver/hvpa.go +++ b/pkg/component/gardener/apiserver/hvpa.go @@ -15,11 +15,12 @@ import ( "k8s.io/utils/ptr" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/component/apiserver" "github.com/gardener/gardener/pkg/utils" ) func (g *gardenerAPIServer) hvpa() *hvpav1alpha1.Hvpa { - if !g.values.Autoscaling.HVPAEnabled { + if g.values.Autoscaling.Mode != apiserver.AutoscalingModeHVPA { return nil } diff --git a/pkg/component/gardener/apiserver/vpa.go b/pkg/component/gardener/apiserver/vpa.go index 8c2c4bb6f57..a959effb223 100644 --- a/pkg/component/gardener/apiserver/vpa.go +++ b/pkg/component/gardener/apiserver/vpa.go @@ -11,10 +11,12 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + + "github.com/gardener/gardener/pkg/component/apiserver" ) func (g *gardenerAPIServer) verticalPodAutoscaler() *vpaautoscalingv1.VerticalPodAutoscaler { - if g.values.Autoscaling.HVPAEnabled { + if g.values.Autoscaling.Mode != apiserver.AutoscalingModeBaseline { return nil } diff --git a/pkg/component/kubernetes/apiserver/apiserver_test.go b/pkg/component/kubernetes/apiserver/apiserver_test.go index 4580c04617c..40ec9db8eb9 100644 --- a/pkg/component/kubernetes/apiserver/apiserver_test.go +++ b/pkg/component/kubernetes/apiserver/apiserver_test.go @@ -222,71 +222,138 @@ var _ = Describe("KubeAPIServer", func() { Expect(c.Get(ctx, client.ObjectKeyFromObject(horizontalPodAutoscaler), horizontalPodAutoscaler)).To(BeNotFoundError()) }, - Entry("HVPA is enabled", apiserver.AutoscalingConfig{HVPAEnabled: true}), - Entry("replicas is nil", apiserver.AutoscalingConfig{HVPAEnabled: false, Replicas: nil}), - Entry("replicas is 0", apiserver.AutoscalingConfig{HVPAEnabled: false, Replicas: ptr.To[int32](0)}), + Entry("autoscaling mode is HVPA", apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeHVPA}), + Entry("replicas is nil", apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeBaseline, Replicas: nil}), + Entry("replicas is 0", apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeBaseline, Replicas: ptr.To[int32](0)}), ) - BeforeEach(func() { - autoscalingConfig = apiserver.AutoscalingConfig{ - HVPAEnabled: false, - Replicas: ptr.To[int32](2), - MinReplicas: 4, - MaxReplicas: 6, - } + DescribeTable("should successfully deploy the HPA resource", + func(autoscalingConfig apiserver.AutoscalingConfig, metrics []autoscalingv2.MetricSpec, behavior *autoscalingv2.HorizontalPodAutoscalerBehavior) { + kapi = New(kubernetesInterface, namespace, sm, Values{ + Values: apiserver.Values{ + Autoscaling: autoscalingConfig, + RuntimeVersion: runtimeVersion, + }, + Version: version}, + ) - runtimeVersion = semver.MustParse("1.25.0") - }) + Expect(c.Get(ctx, client.ObjectKeyFromObject(horizontalPodAutoscaler), horizontalPodAutoscaler)).To(BeNotFoundError()) + Expect(kapi.Deploy(ctx)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKeyFromObject(horizontalPodAutoscaler), horizontalPodAutoscaler)).To(Succeed()) + Expect(horizontalPodAutoscaler).To(DeepEqual(&autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: horizontalPodAutoscaler.Name, + Namespace: horizontalPodAutoscaler.Namespace, + Labels: map[string]string{ + "high-availability-config.resources.gardener.cloud/type": "server", + }, + ResourceVersion: "1", + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: &autoscalingConfig.MinReplicas, + MaxReplicas: autoscalingConfig.MaxReplicas, + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "kube-apiserver", + }, + Metrics: metrics, + Behavior: behavior, + }, + })) + }, - It("should successfully deploy the HPA resource", func() { - Expect(c.Get(ctx, client.ObjectKeyFromObject(horizontalPodAutoscaler), horizontalPodAutoscaler)).To(BeNotFoundError()) - Expect(kapi.Deploy(ctx)).To(Succeed()) - Expect(c.Get(ctx, client.ObjectKeyFromObject(horizontalPodAutoscaler), horizontalPodAutoscaler)).To(Succeed()) - Expect(horizontalPodAutoscaler).To(DeepEqual(&autoscalingv2.HorizontalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{ - Name: horizontalPodAutoscaler.Name, - Namespace: horizontalPodAutoscaler.Namespace, - ResourceVersion: "1", + Entry("autoscaling mode is baseline", + apiserver.AutoscalingConfig{ + Mode: apiserver.AutoscalingModeBaseline, + Replicas: ptr.To[int32](2), + MinReplicas: 4, + MaxReplicas: 6, }, - Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ - MinReplicas: &autoscalingConfig.MinReplicas, - MaxReplicas: autoscalingConfig.MaxReplicas, - ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "kube-apiserver", - }, - Metrics: []autoscalingv2.MetricSpec{ - { - Type: "Resource", - Resource: &autoscalingv2.ResourceMetricSource{ - Name: "cpu", - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.UtilizationMetricType, - AverageUtilization: ptr.To[int32](80), - }, + []autoscalingv2.MetricSpec{ + { + Type: "Resource", + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "cpu", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](80), }, }, - { - Type: "Resource", - Resource: &autoscalingv2.ResourceMetricSource{ - Name: "memory", - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.UtilizationMetricType, - AverageUtilization: ptr.To[int32](80), - }, + }, + { + Type: "Resource", + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "memory", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: ptr.To[int32](80), }, }, }, }, - })) - }) + nil, + ), + + Entry("autoscaling mode is VPAAndHPA", + apiserver.AutoscalingConfig{ + Mode: apiserver.AutoscalingModeVPAAndHPA, + Replicas: ptr.To[int32](2), + MinReplicas: 4, + MaxReplicas: 6, + }, + []autoscalingv2.MetricSpec{ + { + Type: "Resource", + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "cpu", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: ptr.To(resource.MustParse("6")), + }, + }, + }, + { + Type: "Resource", + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "memory", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: ptr.To(resource.MustParse("24G")), + }, + }, + }, + }, + &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: ptr.To[int32](60), + Policies: []autoscalingv2.HPAScalingPolicy{ + { + Type: autoscalingv2.PercentScalingPolicy, + Value: 100, + PeriodSeconds: 60, + }, + }, + }, + ScaleDown: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: ptr.To[int32](1800), + Policies: []autoscalingv2.HPAScalingPolicy{ + { + Type: autoscalingv2.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 300, + }, + }, + }, + }, + ), + ) }) Describe("VerticalPodAutoscaler", func() { - Context("HVPAEnabled = true", func() { + Context("autoscaling mode is HVPA", func() { BeforeEach(func() { - autoscalingConfig = apiserver.AutoscalingConfig{HVPAEnabled: true} + autoscalingConfig = apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeHVPA} }) It("should delete the VPA resource", func() { @@ -297,9 +364,149 @@ var _ = Describe("KubeAPIServer", func() { }) }) - Context("HVPAEnabled = false", func() { + DescribeTable("should successfully deploy the VPA resource", + func(autoscalingConfig apiserver.AutoscalingConfig, haVPN bool, annotations, labels map[string]string, vpaUpdateMode *vpaautoscalingv1.UpdateMode, containerPolicies []vpaautoscalingv1.ContainerResourcePolicy) { + kapi = New(kubernetesInterface, namespace, sm, Values{ + Values: apiserver.Values{ + Autoscaling: autoscalingConfig, + RuntimeVersion: runtimeVersion, + }, + Version: version, + VPN: VPNConfig{ + HighAvailabilityEnabled: haVPN, + HighAvailabilityNumberOfSeedServers: 2, + }, + }) + + Expect(c.Get(ctx, client.ObjectKeyFromObject(verticalPodAutoscaler), verticalPodAutoscaler)).To(BeNotFoundError()) + Expect(kapi.Deploy(ctx)).To(Succeed()) + Expect(c.Get(ctx, client.ObjectKeyFromObject(verticalPodAutoscaler), verticalPodAutoscaler)).To(Succeed()) + Expect(verticalPodAutoscaler).To(DeepEqual(&vpaautoscalingv1.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: verticalPodAutoscaler.Name, + Namespace: verticalPodAutoscaler.Namespace, + Annotations: annotations, + Labels: labels, + ResourceVersion: "1", + }, + Spec: vpaautoscalingv1.VerticalPodAutoscalerSpec{ + TargetRef: &autoscalingv1.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "kube-apiserver", + }, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: vpaUpdateMode, + }, + ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ + ContainerPolicies: containerPolicies, + }, + }, + })) + }, + + Entry("autoscaling mode is baseline", + apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeBaseline}, + false, + nil, + nil, + ptr.To(vpaautoscalingv1.UpdateModeOff), + []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: vpaautoscalingv1.DefaultContainerResourcePolicy, + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + }, + }, + ), + Entry("autoscaling mode is VPAAndHPA", + apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeVPAAndHPA}, + false, + nil, + nil, + ptr.To(vpaautoscalingv1.UpdateModeAuto), + []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: "kube-apiserver", + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + MinAllowed: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("200M"), + }, + MaxAllowed: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("7"), + corev1.ResourceMemory: resource.MustParse("28G"), + }, + }, + }, + ), + Entry("autoscaling mode is VPAAndHPA and HA VPN is enabled", + apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeVPAAndHPA}, + true, + nil, + nil, + ptr.To(vpaautoscalingv1.UpdateModeAuto), + []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: "kube-apiserver", + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + MinAllowed: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("200M"), + }, + MaxAllowed: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("7"), + corev1.ResourceMemory: resource.MustParse("28G"), + }, + }, + { + ContainerName: "vpn-client-0", + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + MinAllowed: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("20Mi"), + }, + }, + { + ContainerName: "vpn-client-1", + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + MinAllowed: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("20Mi"), + }, + }, + { + ContainerName: "vpn-path-controller", + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + MinAllowed: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("20Mi"), + }, + }, + }, + ), + Entry("autoscaling mode is VPAAndHPA and scale-down is disabled", + apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeVPAAndHPA, ScaleDownDisabled: true}, + false, + map[string]string{"eviction-requirements.autoscaling.gardener.cloud/downscale-restriction": "never"}, + map[string]string{"autoscaling.gardener.cloud/eviction-requirements": "managed-by-controller"}, + ptr.To(vpaautoscalingv1.UpdateModeAuto), + []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: "kube-apiserver", + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + MinAllowed: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("200M"), + }, + MaxAllowed: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("7"), + corev1.ResourceMemory: resource.MustParse("28G"), + }, + }, + }, + ), + ) + + Context("autoscaling mode is baseline", func() { BeforeEach(func() { - autoscalingConfig = apiserver.AutoscalingConfig{HVPAEnabled: false} + autoscalingConfig = apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeBaseline} }) It("should successfully deploy the VPA resource", func() { @@ -350,9 +557,10 @@ var _ = Describe("KubeAPIServer", func() { Expect(c.Get(ctx, client.ObjectKeyFromObject(hvpa), hvpa)).To(BeNotFoundError()) }, - Entry("HVPA disabled", apiserver.AutoscalingConfig{HVPAEnabled: false}), - Entry("HVPA enabled but replicas nil", apiserver.AutoscalingConfig{HVPAEnabled: true}), - Entry("HVPA enabled but replicas zero", apiserver.AutoscalingConfig{HVPAEnabled: true, Replicas: ptr.To[int32](0)}), + Entry("autoscaling mode is baseline", apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeBaseline}), + Entry("autoscaling mode is VPAAndHPA", apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeVPAAndHPA}), + Entry("autoscaling mode is HVPA but replicas nil", apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeHVPA}), + Entry("autoscaling mode is HVPA but replicas zero", apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeHVPA, Replicas: ptr.To[int32](0)}), ) var ( @@ -516,7 +724,7 @@ var _ = Describe("KubeAPIServer", func() { Entry("default behaviour", apiserver.AutoscalingConfig{ - HVPAEnabled: true, + Mode: apiserver.AutoscalingModeHVPA, Replicas: ptr.To[int32](2), MinReplicas: 5, MaxReplicas: 5, @@ -529,7 +737,7 @@ var _ = Describe("KubeAPIServer", func() { ), Entry("UseMemoryMetricForHvpaHPA is true", apiserver.AutoscalingConfig{ - HVPAEnabled: true, + Mode: apiserver.AutoscalingModeHVPA, Replicas: ptr.To[int32](2), UseMemoryMetricForHvpaHPA: true, MinReplicas: 5, @@ -558,11 +766,11 @@ var _ = Describe("KubeAPIServer", func() { ), Entry("scale down is disabled", apiserver.AutoscalingConfig{ - HVPAEnabled: true, - Replicas: ptr.To[int32](2), - MinReplicas: 5, - MaxReplicas: 5, - ScaleDownDisabledForHvpa: true, + Mode: apiserver.AutoscalingModeHVPA, + Replicas: ptr.To[int32](2), + MinReplicas: 5, + MaxReplicas: 5, + ScaleDownDisabled: true, }, SNIConfig{}, "Off", @@ -572,7 +780,7 @@ var _ = Describe("KubeAPIServer", func() { ), Entry("max replicas > min replicas", apiserver.AutoscalingConfig{ - HVPAEnabled: true, + Mode: apiserver.AutoscalingModeHVPA, Replicas: ptr.To[int32](2), MinReplicas: 3, MaxReplicas: 5, diff --git a/pkg/component/kubernetes/apiserver/horizontalpodautoscaler.go b/pkg/component/kubernetes/apiserver/horizontalpodautoscaler.go index 963f7577744..e318bd8e566 100644 --- a/pkg/component/kubernetes/apiserver/horizontalpodautoscaler.go +++ b/pkg/component/kubernetes/apiserver/horizontalpodautoscaler.go @@ -10,10 +10,13 @@ import ( appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/component/apiserver" "github.com/gardener/gardener/pkg/controllerutils" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) @@ -33,13 +36,22 @@ func (k *kubeAPIServer) emptyHorizontalPodAutoscaler() *autoscalingv2.Horizontal } func (k *kubeAPIServer) reconcileHorizontalPodAutoscaler(ctx context.Context, hpa *autoscalingv2.HorizontalPodAutoscaler, deployment *appsv1.Deployment) error { - if k.values.Autoscaling.HVPAEnabled || + if k.values.Autoscaling.Mode == apiserver.AutoscalingModeHVPA || k.values.Autoscaling.Replicas == nil || *k.values.Autoscaling.Replicas == 0 { return kubernetesutils.DeleteObject(ctx, k.client.Client(), hpa) } + if k.values.Autoscaling.Mode == apiserver.AutoscalingModeVPAAndHPA { + return k.reconcileHorizontalPodAutoscalerInVPAAndHPAMode(ctx, hpa, deployment) + } + + return k.reconcileHorizontalPodAutoscalerInBaselineMode(ctx, hpa, deployment) +} + +func (k *kubeAPIServer) reconcileHorizontalPodAutoscalerInBaselineMode(ctx context.Context, hpa *autoscalingv2.HorizontalPodAutoscaler, deployment *appsv1.Deployment) error { _, err := controllerutils.GetAndCreateOrMergePatch(ctx, k.client.Client(), hpa, func() error { + metav1.SetMetaDataLabel(&hpa.ObjectMeta, resourcesv1alpha1.HighAvailabilityConfigType, resourcesv1alpha1.HighAvailabilityConfigTypeServer) hpa.Spec = autoscalingv2.HorizontalPodAutoscalerSpec{ MinReplicas: &k.values.Autoscaling.MinReplicas, MaxReplicas: k.values.Autoscaling.MaxReplicas, @@ -74,6 +86,83 @@ func (k *kubeAPIServer) reconcileHorizontalPodAutoscaler(ctx context.Context, hp return nil }) + return err +} + +func (k *kubeAPIServer) reconcileHorizontalPodAutoscalerInVPAAndHPAMode(ctx context.Context, hpa *autoscalingv2.HorizontalPodAutoscaler, deployment *appsv1.Deployment) error { + // The chosen value is 6 CPU: 1 CPU less (ratio 1/7) than the VPA's maxAllowed 7 CPU in VPAAndHPA mode to have a headroom for the horizontal scaling. + hpaTargetAverageValueCPU := resource.MustParse("6") + // The chosen value is 24G: 4G less (ratio 1/7) than the VPA's maxAllowed 28G in VPAAndHPA mode to have a headroom for the horizontal scaling. + hpaTargetAverageValueMemory := resource.MustParse("24G") + + _, err := controllerutils.GetAndCreateOrMergePatch(ctx, k.client.Client(), hpa, func() error { + minReplicas := k.values.Autoscaling.MinReplicas + if k.values.Autoscaling.ScaleDownDisabled && hpa.Spec.MinReplicas != nil { + // If scale-down is disabled and the HPA resource exists and HPA's spec.minReplicas is not nil, + // then minReplicas is max(spec.minReplicas, status.desiredReplicas). + // When scale-down is disabled, this allows operators to specify a custom value for HPA spec.minReplicas + // and this value not to be reverted by gardenlet. + minReplicas = max(*hpa.Spec.MinReplicas, hpa.Status.DesiredReplicas) + } + + metav1.SetMetaDataLabel(&hpa.ObjectMeta, resourcesv1alpha1.HighAvailabilityConfigType, resourcesv1alpha1.HighAvailabilityConfigTypeServer) + hpa.Spec = autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: &minReplicas, + MaxReplicas: k.values.Autoscaling.MaxReplicas, + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + Name: deployment.Name, + }, + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: &hpaTargetAverageValueCPU, + }, + }, + }, + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: &hpaTargetAverageValueMemory, + }, + }, + }, + }, + Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: ptr.To[int32](60), + Policies: []autoscalingv2.HPAScalingPolicy{ + // Allow to upscale 100% of the current number of pods every 1 minute to see whether any upscale recommendation will still hold true after the cluster has settled + { + Type: autoscalingv2.PercentScalingPolicy, + Value: 100, + PeriodSeconds: 60, + }, + }, + }, + ScaleDown: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: ptr.To[int32](1800), + Policies: []autoscalingv2.HPAScalingPolicy{ + // Allow to downscale one pod every 5 minutes to see whether any downscale recommendation will still hold true after the cluster has settled (conservatively) + { + Type: autoscalingv2.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 300, + }, + }, + }, + }, + } + return nil + }) return err } diff --git a/pkg/component/kubernetes/apiserver/hvpa.go b/pkg/component/kubernetes/apiserver/hvpa.go index 81c255f7a49..33695e5eb8e 100644 --- a/pkg/component/kubernetes/apiserver/hvpa.go +++ b/pkg/component/kubernetes/apiserver/hvpa.go @@ -19,6 +19,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/component/apiserver" "github.com/gardener/gardener/pkg/controllerutils" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) @@ -28,7 +29,7 @@ func (k *kubeAPIServer) emptyHVPA() *hvpav1alpha1.Hvpa { } func (k *kubeAPIServer) reconcileHVPA(ctx context.Context, hvpa *hvpav1alpha1.Hvpa, deployment *appsv1.Deployment) error { - if !k.values.Autoscaling.HVPAEnabled || + if k.values.Autoscaling.Mode != apiserver.AutoscalingModeHVPA || k.values.Autoscaling.Replicas == nil || *k.values.Autoscaling.Replicas == 0 { return kubernetesutils.DeleteObject(ctx, k.client.Client(), hvpa) @@ -39,7 +40,6 @@ func (k *kubeAPIServer) reconcileHVPA(ctx context.Context, hvpa *hvpav1alpha1.Hv vpaLabels = map[string]string{v1beta1constants.LabelRole: v1beta1constants.LabelAPIServer + "-vpa"} updateModeAuto = hvpav1alpha1.UpdateModeAuto scaleDownUpdateMode = updateModeAuto - controlledValues = vpaautoscalingv1.ContainerControlledValuesRequestsOnly hpaMetrics = []autoscalingv2beta1.MetricSpec{ { Type: autoscalingv2beta1.ResourceMetricSourceType, @@ -49,18 +49,12 @@ func (k *kubeAPIServer) reconcileHVPA(ctx context.Context, hvpa *hvpav1alpha1.Hv }, }, } - vpaContainerResourcePolicies = []vpaautoscalingv1.ContainerResourcePolicy{ - { - ContainerName: ContainerNameKubeAPIServer, - MinAllowed: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("200M"), - }, - MaxAllowed: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), - corev1.ResourceMemory: resource.MustParse("25G"), - }, - ControlledValues: &controlledValues, - }, + kubeAPIServerMinAllowed = corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("200M"), + } + kubeAPIServerMaxAllowed = corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("8"), + corev1.ResourceMemory: resource.MustParse("25G"), } weightBasedScalingIntervals = []hvpav1alpha1.WeightBasedScalingInterval{ { @@ -81,7 +75,7 @@ func (k *kubeAPIServer) reconcileHVPA(ctx context.Context, hvpa *hvpav1alpha1.Hv }) } - if k.values.Autoscaling.ScaleDownDisabledForHvpa { + if k.values.Autoscaling.ScaleDownDisabled { scaleDownUpdateMode = hvpav1alpha1.UpdateModeOff } @@ -93,25 +87,6 @@ func (k *kubeAPIServer) reconcileHVPA(ctx context.Context, hvpa *hvpav1alpha1.Hv }) } - if k.values.VPN.HighAvailabilityEnabled { - for i := 0; i < k.values.VPN.HighAvailabilityNumberOfSeedServers; i++ { - vpaContainerResourcePolicies = append(vpaContainerResourcePolicies, vpaautoscalingv1.ContainerResourcePolicy{ - ContainerName: fmt.Sprintf("%s-%d", containerNameVPNSeedClient, i), - MinAllowed: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("20Mi"), - }, - ControlledValues: &controlledValues, - }) - } - vpaContainerResourcePolicies = append(vpaContainerResourcePolicies, vpaautoscalingv1.ContainerResourcePolicy{ - ContainerName: containerNameVPNPathController, - MinAllowed: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("20Mi"), - }, - ControlledValues: &controlledValues, - }) - } - _, err := controllerutils.GetAndCreateOrMergePatch(ctx, k.client.Client(), hvpa, func() error { metav1.SetMetaDataLabel(&hvpa.ObjectMeta, resourcesv1alpha1.HighAvailabilityConfigType, resourcesv1alpha1.HighAvailabilityConfigTypeServer) hvpa.Spec.Replicas = ptr.To[int32](1) @@ -190,7 +165,7 @@ func (k *kubeAPIServer) reconcileHVPA(ctx context.Context, hvpa *hvpav1alpha1.Hv }, Spec: hvpav1alpha1.VpaTemplateSpec{ ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ - ContainerPolicies: vpaContainerResourcePolicies, + ContainerPolicies: k.computeVerticalPodAutoscalerContainerResourcePolicies(kubeAPIServerMinAllowed, kubeAPIServerMaxAllowed), }, }, }, @@ -205,3 +180,37 @@ func (k *kubeAPIServer) reconcileHVPA(ctx context.Context, hvpa *hvpav1alpha1.Hv }) return err } + +func (k *kubeAPIServer) computeVerticalPodAutoscalerContainerResourcePolicies(kubeAPIServerMinAllowed, kubeAPIServerMaxAllowed corev1.ResourceList) []vpaautoscalingv1.ContainerResourcePolicy { + var ( + vpaContainerResourcePolicies = []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: ContainerNameKubeAPIServer, + MinAllowed: kubeAPIServerMinAllowed, + MaxAllowed: kubeAPIServerMaxAllowed, + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + }, + } + ) + + if k.values.VPN.HighAvailabilityEnabled { + for i := 0; i < k.values.VPN.HighAvailabilityNumberOfSeedServers; i++ { + vpaContainerResourcePolicies = append(vpaContainerResourcePolicies, vpaautoscalingv1.ContainerResourcePolicy{ + ContainerName: fmt.Sprintf("%s-%d", containerNameVPNSeedClient, i), + MinAllowed: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("20Mi"), + }, + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + }) + } + vpaContainerResourcePolicies = append(vpaContainerResourcePolicies, vpaautoscalingv1.ContainerResourcePolicy{ + ContainerName: containerNameVPNPathController, + MinAllowed: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("20Mi"), + }, + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + }) + } + + return vpaContainerResourcePolicies +} diff --git a/pkg/component/kubernetes/apiserver/verticalpodautoscaler.go b/pkg/component/kubernetes/apiserver/verticalpodautoscaler.go index bf5bd9936f6..5532846f505 100644 --- a/pkg/component/kubernetes/apiserver/verticalpodautoscaler.go +++ b/pkg/component/kubernetes/apiserver/verticalpodautoscaler.go @@ -9,10 +9,14 @@ import ( appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/utils/ptr" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/component/apiserver" "github.com/gardener/gardener/pkg/controllerutils" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) @@ -22,13 +26,17 @@ func (k *kubeAPIServer) emptyVerticalPodAutoscaler() *vpaautoscalingv1.VerticalP } func (k *kubeAPIServer) reconcileVerticalPodAutoscaler(ctx context.Context, verticalPodAutoscaler *vpaautoscalingv1.VerticalPodAutoscaler, deployment *appsv1.Deployment) error { - if k.values.Autoscaling.HVPAEnabled { + switch k.values.Autoscaling.Mode { + case apiserver.AutoscalingModeHVPA: return kubernetesutils.DeleteObject(ctx, k.client.Client(), verticalPodAutoscaler) + case apiserver.AutoscalingModeVPAAndHPA: + return k.reconcileVerticalPodAutoscalerInVPAAndHPAMode(ctx, verticalPodAutoscaler, deployment) + default: + return k.reconcileVerticalPodAutoscalerInBaselineMode(ctx, verticalPodAutoscaler, deployment) } +} - vpaUpdateMode := vpaautoscalingv1.UpdateModeOff - controlledValues := vpaautoscalingv1.ContainerControlledValuesRequestsOnly - +func (k *kubeAPIServer) reconcileVerticalPodAutoscalerInBaselineMode(ctx context.Context, verticalPodAutoscaler *vpaautoscalingv1.VerticalPodAutoscaler, deployment *appsv1.Deployment) error { _, err := controllerutils.GetAndCreateOrMergePatch(ctx, k.client.Client(), verticalPodAutoscaler, func() error { verticalPodAutoscaler.Spec = vpaautoscalingv1.VerticalPodAutoscalerSpec{ TargetRef: &autoscalingv1.CrossVersionObjectReference{ @@ -37,12 +45,12 @@ func (k *kubeAPIServer) reconcileVerticalPodAutoscaler(ctx context.Context, vert Name: deployment.Name, }, UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ - UpdateMode: &vpaUpdateMode, + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeOff), }, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{{ ContainerName: vpaautoscalingv1.DefaultContainerResourcePolicy, - ControlledValues: &controlledValues, + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), }}, }, } @@ -50,3 +58,43 @@ func (k *kubeAPIServer) reconcileVerticalPodAutoscaler(ctx context.Context, vert }) return err } + +func (k *kubeAPIServer) reconcileVerticalPodAutoscalerInVPAAndHPAMode(ctx context.Context, verticalPodAutoscaler *vpaautoscalingv1.VerticalPodAutoscaler, deployment *appsv1.Deployment) error { + kubeAPIServerMinAllowed := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("200M"), + } + kubeAPIServerMaxAllowed := corev1.ResourceList{ + // The CPU and memory are aligned to the machine ration of 1:4. + corev1.ResourceCPU: resource.MustParse("7"), + corev1.ResourceMemory: resource.MustParse("28G"), + } + + _, err := controllerutils.GetAndCreateOrMergePatch(ctx, k.client.Client(), verticalPodAutoscaler, func() error { + verticalPodAutoscaler.Spec = vpaautoscalingv1.VerticalPodAutoscalerSpec{ + TargetRef: &autoscalingv1.CrossVersionObjectReference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + Name: deployment.Name, + }, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto), + }, + ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ + ContainerPolicies: k.computeVerticalPodAutoscalerContainerResourcePolicies(kubeAPIServerMinAllowed, kubeAPIServerMaxAllowed), + }, + } + + if k.values.Autoscaling.ScaleDownDisabled { + metav1.SetMetaDataLabel(&verticalPodAutoscaler.ObjectMeta, v1beta1constants.LabelVPAEvictionRequirementsController, v1beta1constants.EvictionRequirementManagedByController) + metav1.SetMetaDataAnnotation(&verticalPodAutoscaler.ObjectMeta, v1beta1constants.AnnotationVPAEvictionRequirementDownscaleRestriction, v1beta1constants.EvictionRequirementNever) + } else { + delete(verticalPodAutoscaler.GetLabels(), v1beta1constants.LabelVPAEvictionRequirementsController) + delete(verticalPodAutoscaler.GetAnnotations(), v1beta1constants.AnnotationVPAEvictionRequirementDownscaleRestriction) + } + + return nil + }) + + return err +} diff --git a/pkg/component/shared/kubeapiserver.go b/pkg/component/shared/kubeapiserver.go index 04495a1e1ab..881e55ff963 100644 --- a/pkg/component/shared/kubeapiserver.go +++ b/pkg/component/shared/kubeapiserver.go @@ -220,7 +220,15 @@ func DeployKubeAPIServer( kubeAPIServer.SetAutoscalingReplicas(computeKubeAPIServerReplicas(values.Autoscaling, deployment, wantScaleDown)) - if deployment != nil && values.Autoscaling.HVPAEnabled { + // For safety reasons, when the Deployment exists we don't overwrite the kube-apiserver container resources + // although it is not required in all cases. Few cases that require it: + // - When autoscaling mode is HVPA. hvpa-controller updates the resources in Deployment spec directly. + // Overwriting the resources in the Deployment spec would revert hvpa-controller's recommendations. + // - When scale-down is disabled, operators might want to overwrite the kube-apiserver container resource requests. + // - When transitioning from HVPA to VPAAndHPA autoscaling mode, we need to preserve the kube-apiserver container resources + // to do not cause an unwanted Deployment rollout by overwriting the container resources set by the HVPA with the initial + // resources in VPAAndHPA mode. + if deployment != nil { for _, container := range deployment.Spec.Template.Spec.Containers { if container.Name == kubeapiserver.ContainerNameKubeAPIServer { // Only set requests to allow limits to be removed diff --git a/pkg/component/shared/kubeapiserver_test.go b/pkg/component/shared/kubeapiserver_test.go index f87384724af..ed5c0b176c5 100644 --- a/pkg/component/shared/kubeapiserver_test.go +++ b/pkg/component/shared/kubeapiserver_test.go @@ -975,12 +975,12 @@ exemptions: Expect(DeployKubeAPIServer(ctx, runtimeClient, namespace, kubeAPIServer, serviceAccountConfig, serverCertificateConfig, sniConfig, externalHostname, externalServer, &nodeNetworkCIDR, nil, nil, etcdEncryptionKeyRotationPhase, wantScaleDown)).To(Succeed()) }, - Entry("nothing is set because deployment is not found", + Entry("nothing is set when deployment is not found", nil, apiserver.AutoscalingConfig{}, nil, ), - Entry("nothing is set because HVPA is disabled", + Entry("set the existing requirements when the deployment is found and autoscaling mode is HVPA", func() { Expect(runtimeClient.Create(ctx, &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -999,10 +999,32 @@ exemptions: }, })).To(Succeed()) }, - apiserver.AutoscalingConfig{HVPAEnabled: false}, - nil, + apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeHVPA}, + &apiServerResources, + ), + Entry("set the existing requirements when the deployment is found and scale-down is disabled", + func() { + Expect(runtimeClient.Create(ctx, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver", + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "kube-apiserver", + Resources: apiServerResources, + }}, + }, + }, + }, + })).To(Succeed()) + }, + apiserver.AutoscalingConfig{ScaleDownDisabled: true}, + &apiServerResources, ), - Entry("set the existing requirements because deployment found and HVPA enabled", + Entry("set the existing requirements when the deployment is found and autoscaling mode is VPAAndHPA", func() { Expect(runtimeClient.Create(ctx, &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -1021,7 +1043,7 @@ exemptions: }, })).To(Succeed()) }, - apiserver.AutoscalingConfig{HVPAEnabled: true}, + apiserver.AutoscalingConfig{Mode: apiserver.AutoscalingModeVPAAndHPA}, &apiServerResources, ), ) diff --git a/pkg/features/features.go b/pkg/features/features.go index dbeca8279a3..e95015ed028 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -71,6 +71,15 @@ const ( // owner: @dimityrmirchev // alpha: v1.93.0 ShootManagedIssuer featuregate.Feature = "ShootManagedIssuer" + + // VPAAndHPAForAPIServer enables an autoscaling mechanism for shoot kube-apiserver where it is scaled simultaneously + // by VPA and HPA on the same metric (CPU and memory usage). The pod-trashing cycle between VPA and HPA scaling on + // the same metric is avoided by configuring the HPA to scale on average usage (not on average utilization) and + // by picking the target average utilization values in sync with VPA's allowed maximums. + // The feature gate takes precedence over the `HVPA` feature gate when they are both enabled. + // owner: @ialidzhikov + // alpha: v1.95.0 + VPAAndHPAForAPIServer = "VPAAndHPAForAPIServer" ) // DefaultFeatureGate is the central feature gate map used by all gardener components. @@ -108,6 +117,7 @@ var AllFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ ShootManagedIssuer: {Default: false, PreRelease: featuregate.Alpha}, ShootForceDeletion: {Default: true, PreRelease: featuregate.Beta}, UseNamespacedCloudProfile: {Default: false, PreRelease: featuregate.Alpha}, + VPAAndHPAForAPIServer: {Default: false, PreRelease: featuregate.Alpha}, } // GetFeatures returns a feature gate map with the respective specifications. Non-existing feature gates are ignored. diff --git a/pkg/gardenlet/features/features.go b/pkg/gardenlet/features/features.go index e852d8988a5..fe2b96d8aad 100644 --- a/pkg/gardenlet/features/features.go +++ b/pkg/gardenlet/features/features.go @@ -26,5 +26,6 @@ func GetFeatures() []featuregate.Feature { features.CoreDNSQueryRewriting, features.IPv6SingleStack, features.ShootManagedIssuer, + features.VPAAndHPAForAPIServer, } } diff --git a/pkg/gardenlet/operation/botanist/kubeapiserver.go b/pkg/gardenlet/operation/botanist/kubeapiserver.go index 9e167ebe2cf..bec1a6f5177 100644 --- a/pkg/gardenlet/operation/botanist/kubeapiserver.go +++ b/pkg/gardenlet/operation/botanist/kubeapiserver.go @@ -82,23 +82,18 @@ func (b *Botanist) DefaultKubeAPIServer(ctx context.Context) (kubeapiserver.Inte func (b *Botanist) computeKubeAPIServerAutoscalingConfig() apiserver.AutoscalingConfig { var ( - hvpaEnabled = features.DefaultFeatureGate.Enabled(features.HVPA) useMemoryMetricForHvpaHPA = false - scaleDownDisabledForHvpa = false + scaleDownDisabled = false defaultReplicas *int32 - minReplicas int32 = 1 - maxReplicas int32 = 3 - apiServerResources corev1.ResourceRequirements + // kube-apiserver is a control plane component of type "server". + // The HA webhook sets at least 2 replicas to components of type "server" (w/o HA or with w/ HA). + // Ref https://github.com/gardener/gardener/blob/master/docs/development/high-availability.md#control-plane-components. + // That's why minReplicas is set to 2. + minReplicas int32 = 2 + maxReplicas int32 = 3 + apiServerResources corev1.ResourceRequirements ) - if b.ManagedSeed != nil { - hvpaEnabled = features.DefaultFeatureGate.Enabled(features.HVPAForShootedSeed) - } - - if b.Shoot.Purpose == gardencorev1beta1.ShootPurposeProduction { - minReplicas = 2 - } - if v1beta1helper.IsHAControlPlaneConfigured(b.Shoot.GetInfo()) { minReplicas = 3 } @@ -106,23 +101,29 @@ func (b *Botanist) computeKubeAPIServerAutoscalingConfig() apiserver.Autoscaling if metav1.HasAnnotation(b.Shoot.GetInfo().ObjectMeta, v1beta1constants.ShootAlphaControlPlaneScaleDownDisabled) { minReplicas = 4 maxReplicas = 4 - scaleDownDisabledForHvpa = true + scaleDownDisabled = true } + autoscalingMode := b.autoscalingMode() nodeCount := b.Shoot.GetMinNodeCount() - if hvpaEnabled { - nodeCount = b.Shoot.GetMaxNodeCount() - } - if hvpaEnabled { + switch autoscalingMode { + case apiserver.AutoscalingModeHVPA: apiServerResources = corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("1Gi"), }, } - } else { - apiServerResources = resourcesRequirementsForKubeAPIServer(nodeCount) + case apiserver.AutoscalingModeVPAAndHPA: + apiServerResources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + } + default: + apiServerResources = resourcesRequirementsForKubeAPIServerInBaselineMode(nodeCount) } if b.ManagedSeed != nil { @@ -132,7 +133,7 @@ func (b *Botanist) computeKubeAPIServerAutoscalingConfig() apiserver.Autoscaling minReplicas = *b.ManagedSeedAPIServer.Autoscaler.MinReplicas maxReplicas = b.ManagedSeedAPIServer.Autoscaler.MaxReplicas - if !hvpaEnabled { + if autoscalingMode == apiserver.AutoscalingModeBaseline { defaultReplicas = b.ManagedSeedAPIServer.Replicas apiServerResources = corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -145,17 +146,34 @@ func (b *Botanist) computeKubeAPIServerAutoscalingConfig() apiserver.Autoscaling } return apiserver.AutoscalingConfig{ + Mode: autoscalingMode, APIServerResources: apiServerResources, - HVPAEnabled: hvpaEnabled, Replicas: defaultReplicas, MinReplicas: minReplicas, MaxReplicas: maxReplicas, UseMemoryMetricForHvpaHPA: useMemoryMetricForHvpaHPA, - ScaleDownDisabledForHvpa: scaleDownDisabledForHvpa, + ScaleDownDisabled: scaleDownDisabled, + } +} + +func (b *Botanist) autoscalingMode() apiserver.AutoscalingMode { + // The VPAAndHPAForAPIServer feature gate takes precedence over the HVPA feature gate. + if features.DefaultFeatureGate.Enabled(features.VPAAndHPAForAPIServer) { + return apiserver.AutoscalingModeVPAAndHPA + } + + hvpaEnabled := features.DefaultFeatureGate.Enabled(features.HVPA) + if b.ManagedSeed != nil { + hvpaEnabled = features.DefaultFeatureGate.Enabled(features.HVPAForShootedSeed) + } + + if hvpaEnabled { + return apiserver.AutoscalingModeHVPA } + return apiserver.AutoscalingModeBaseline } -func resourcesRequirementsForKubeAPIServer(nodeCount int32) corev1.ResourceRequirements { +func resourcesRequirementsForKubeAPIServerInBaselineMode(nodeCount int32) corev1.ResourceRequirements { var cpuRequest, memoryRequest string switch { diff --git a/pkg/gardenlet/operation/botanist/kubeapiserver_test.go b/pkg/gardenlet/operation/botanist/kubeapiserver_test.go index a10e013bdf0..a5e1af4e997 100644 --- a/pkg/gardenlet/operation/botanist/kubeapiserver_test.go +++ b/pkg/gardenlet/operation/botanist/kubeapiserver_test.go @@ -182,35 +182,59 @@ var _ = Describe("KubeAPIServer", func() { Expect(kubeAPIServer.GetValues().Autoscaling).To(Equal(expectedConfig)) }, - Entry("default behaviour, HVPA is disabled", + Entry("default behaviour, HVPA is disabled, VPAAndHPAForAPIServer is disabled", nil, - map[featuregate.Feature]bool{features.HVPA: false}, + map[featuregate.Feature]bool{ + features.HVPA: false, + features.VPAAndHPAForAPIServer: false, + }, apiserver.AutoscalingConfig{ - APIServerResources: resourcesRequirementsForKubeAPIServer(4), - HVPAEnabled: false, - MinReplicas: 1, + Mode: apiserver.AutoscalingModeBaseline, + APIServerResources: resourcesRequirementsForKubeAPIServerInBaselineMode(4), + MinReplicas: 2, MaxReplicas: 3, UseMemoryMetricForHvpaHPA: false, - ScaleDownDisabledForHvpa: false, + ScaleDownDisabled: false, }, ), - Entry("default behaviour, HVPA is enabled", + Entry("default behaviour, HVPA is enabled, VPAAndHPAForAPIServer is disabled", nil, map[featuregate.Feature]bool{ - features.HVPA: true, + features.HVPA: true, + features.VPAAndHPAForAPIServer: false, }, apiserver.AutoscalingConfig{ + Mode: apiserver.AutoscalingModeHVPA, APIServerResources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("1Gi"), }, }, - HVPAEnabled: true, - MinReplicas: 1, + MinReplicas: 2, + MaxReplicas: 3, + UseMemoryMetricForHvpaHPA: false, + ScaleDownDisabled: false, + }, + ), + Entry("default behaviour, HVPA is enabled, VPAAndHPAForAPIServer is enabled", + nil, + map[featuregate.Feature]bool{ + features.HVPA: true, + features.VPAAndHPAForAPIServer: true, + }, + apiserver.AutoscalingConfig{ + Mode: apiserver.AutoscalingModeVPAAndHPA, + APIServerResources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + MinReplicas: 2, MaxReplicas: 3, UseMemoryMetricForHvpaHPA: false, - ScaleDownDisabledForHvpa: false, + ScaleDownDisabled: false, }, ), Entry("shoot purpose production", @@ -219,12 +243,12 @@ var _ = Describe("KubeAPIServer", func() { }, nil, apiserver.AutoscalingConfig{ - APIServerResources: resourcesRequirementsForKubeAPIServer(4), - HVPAEnabled: false, + Mode: apiserver.AutoscalingModeBaseline, + APIServerResources: resourcesRequirementsForKubeAPIServerInBaselineMode(4), MinReplicas: 2, MaxReplicas: 3, UseMemoryMetricForHvpaHPA: false, - ScaleDownDisabledForHvpa: false, + ScaleDownDisabled: false, }, ), Entry("shoot disables scale down", @@ -233,12 +257,12 @@ var _ = Describe("KubeAPIServer", func() { }, nil, apiserver.AutoscalingConfig{ - APIServerResources: resourcesRequirementsForKubeAPIServer(4), - HVPAEnabled: false, + Mode: apiserver.AutoscalingModeBaseline, + APIServerResources: resourcesRequirementsForKubeAPIServerInBaselineMode(4), MinReplicas: 4, MaxReplicas: 4, UseMemoryMetricForHvpaHPA: false, - ScaleDownDisabledForHvpa: true, + ScaleDownDisabled: true, }, ), Entry("shoot is a managed seed and HVPAForShootedSeed is disabled", @@ -247,12 +271,12 @@ var _ = Describe("KubeAPIServer", func() { }, map[featuregate.Feature]bool{features.HVPAForShootedSeed: false}, apiserver.AutoscalingConfig{ - APIServerResources: resourcesRequirementsForKubeAPIServer(4), - HVPAEnabled: false, - MinReplicas: 1, + Mode: apiserver.AutoscalingModeBaseline, + APIServerResources: resourcesRequirementsForKubeAPIServerInBaselineMode(4), + MinReplicas: 2, MaxReplicas: 3, UseMemoryMetricForHvpaHPA: true, - ScaleDownDisabledForHvpa: false, + ScaleDownDisabled: false, }, ), Entry("shoot is a managed seed w/ APIServer settings and HVPAForShootedSeed is enabled", @@ -270,17 +294,17 @@ var _ = Describe("KubeAPIServer", func() { features.HVPAForShootedSeed: true, }, apiserver.AutoscalingConfig{ + Mode: apiserver.AutoscalingModeHVPA, APIServerResources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("1Gi"), }, }, - HVPAEnabled: true, MinReplicas: 16, MaxReplicas: 32, UseMemoryMetricForHvpaHPA: true, - ScaleDownDisabledForHvpa: false, + ScaleDownDisabled: false, }, ), Entry("shoot is a managed seed w/ APIServer settings and HVPAForShootedSeed is disabled", @@ -296,18 +320,18 @@ var _ = Describe("KubeAPIServer", func() { }, map[featuregate.Feature]bool{features.HVPAForShootedSeed: false}, apiserver.AutoscalingConfig{ + Mode: apiserver.AutoscalingModeBaseline, APIServerResources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1750m"), corev1.ResourceMemory: resource.MustParse("2Gi"), }, }, - HVPAEnabled: false, MinReplicas: 16, MaxReplicas: 32, Replicas: ptr.To[int32](24), UseMemoryMetricForHvpaHPA: true, - ScaleDownDisabledForHvpa: false, + ScaleDownDisabled: false, }, ), Entry("shoot is a managed seed w/ APIServer settings and HVPAForShootedSeed is disabled", @@ -325,18 +349,18 @@ var _ = Describe("KubeAPIServer", func() { features.HVPAForShootedSeed: false, }, apiserver.AutoscalingConfig{ + Mode: apiserver.AutoscalingModeBaseline, APIServerResources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1750m"), corev1.ResourceMemory: resource.MustParse("2Gi"), }, }, - HVPAEnabled: false, MinReplicas: 16, MaxReplicas: 32, Replicas: ptr.To[int32](24), UseMemoryMetricForHvpaHPA: true, - ScaleDownDisabledForHvpa: false, + ScaleDownDisabled: false, }, ), Entry("shoot enables HA control planes", @@ -349,21 +373,21 @@ var _ = Describe("KubeAPIServer", func() { }, nil, apiserver.AutoscalingConfig{ - APIServerResources: resourcesRequirementsForKubeAPIServer(4), - HVPAEnabled: false, + Mode: apiserver.AutoscalingModeBaseline, + APIServerResources: resourcesRequirementsForKubeAPIServerInBaselineMode(4), MinReplicas: 3, MaxReplicas: 3, UseMemoryMetricForHvpaHPA: false, - ScaleDownDisabledForHvpa: false, + ScaleDownDisabled: false, }, ), ) }) }) - DescribeTable("#resourcesRequirementsForKubeAPIServer", + DescribeTable("#resourcesRequirementsForKubeAPIServerInBaselineMode", func(nodes int, expectedCPURequest, expectedMemoryRequest string) { - Expect(resourcesRequirementsForKubeAPIServer(int32(nodes))).To(Equal( + Expect(resourcesRequirementsForKubeAPIServerInBaselineMode(int32(nodes))).To(Equal( corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(expectedCPURequest), diff --git a/pkg/operator/controller/garden/garden/components.go b/pkg/operator/controller/garden/garden/components.go index a75e3dd9932..21d6b49fdcf 100644 --- a/pkg/operator/controller/garden/garden/components.go +++ b/pkg/operator/controller/garden/garden/components.go @@ -628,18 +628,25 @@ func defaultAPIServerAutoscalingConfig(garden *operatorv1alpha1.Garden) apiserve minReplicas = 3 } + var autoscalingMode apiserver.AutoscalingMode + if hvpaEnabled() { + autoscalingMode = apiserver.AutoscalingModeHVPA + } else { + autoscalingMode = apiserver.AutoscalingModeBaseline + } + return apiserver.AutoscalingConfig{ + Mode: autoscalingMode, APIServerResources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("600m"), corev1.ResourceMemory: resource.MustParse("512Mi"), }, }, - HVPAEnabled: hvpaEnabled(), MinReplicas: minReplicas, MaxReplicas: 6, UseMemoryMetricForHvpaHPA: true, - ScaleDownDisabledForHvpa: false, + ScaleDownDisabled: false, } }