diff --git a/pkg/component/gardener/apiserver/apiserver.go b/pkg/component/gardener/apiserver/apiserver.go index 7dc84ca7324..ce0a05cdccc 100644 --- a/pkg/component/gardener/apiserver/apiserver.go +++ b/pkg/component/gardener/apiserver/apiserver.go @@ -145,6 +145,7 @@ func (g *gardenerAPIServer) Deploy(ctx context.Context) error { runtimeResources, err := runtimeRegistry.AddAllAndSerialize( g.podDisruptionBudget(), g.serviceRuntime(), + g.horizontalPodAutoscaler(), g.verticalPodAutoscaler(), g.hvpa(), g.deployment(secretCAETCD, secretETCDClient, secretGenericTokenKubeconfig, secretServer, secretAdmissionKubeconfigs, secretETCDEncryptionConfiguration, secretAuditWebhookKubeconfig, secretVirtualGardenAccess, configMapAuditPolicy, configMapAdmissionConfigs), diff --git a/pkg/component/gardener/apiserver/apiserver_test.go b/pkg/component/gardener/apiserver/apiserver_test.go index 87062a531d2..56aaccf0c37 100644 --- a/pkg/component/gardener/apiserver/apiserver_test.go +++ b/pkg/component/gardener/apiserver/apiserver_test.go @@ -15,6 +15,7 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" autoscalingv2beta1 "k8s.io/api/autoscaling/v2beta1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" @@ -79,7 +80,9 @@ var _ = Describe("GardenerAPIServer", func() { podDisruptionBudgetFor func(bool) *policyv1.PodDisruptionBudget serviceRuntimeFor func(bool) *corev1.Service - vpa *vpaautoscalingv1.VerticalPodAutoscaler + vpaInBaselineMode *vpaautoscalingv1.VerticalPodAutoscaler + vpaInHPAAndVPAMode *vpaautoscalingv1.VerticalPodAutoscaler + hpaOnHPAAndVPAMode *autoscalingv2.HorizontalPodAutoscaler hvpa *hvpav1alpha1.Hvpa deployment *appsv1.Deployment apiServiceFor = func(group, version string) *apiregistrationv1.APIService { @@ -236,8 +239,7 @@ var _ = Describe("GardenerAPIServer", func() { return svc } - vpaUpdateMode := vpaautoscalingv1.UpdateModeAuto - vpa = &vpaautoscalingv1.VerticalPodAutoscaler{ + vpaInBaselineMode = &vpaautoscalingv1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "gardener-apiserver-vpa", Namespace: namespace, @@ -253,12 +255,12 @@ var _ = Describe("GardenerAPIServer", func() { Name: "gardener-apiserver", }, UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ - UpdateMode: &vpaUpdateMode, + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto), }, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ { - ContainerName: "*", + ContainerName: "gardener-apiserver", MinAllowed: corev1.ResourceList{ corev1.ResourceMemory: resource.MustParse("256Mi"), }, @@ -267,6 +269,105 @@ var _ = Describe("GardenerAPIServer", func() { }, }, } + vpaInHPAAndVPAMode = &vpaautoscalingv1.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-apiserver-vpa", + Namespace: namespace, + Labels: map[string]string{ + "app": "gardener", + "role": "apiserver", + }, + }, + Spec: vpaautoscalingv1.VerticalPodAutoscalerSpec{ + TargetRef: &autoscalingv1.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "gardener-apiserver", + }, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto), + }, + ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ + ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: "gardener-apiserver", + MinAllowed: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("400M"), + }, + MaxAllowed: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("7"), + corev1.ResourceMemory: resource.MustParse("28G"), + }, + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + }, + }, + }, + }, + } + hpaOnHPAAndVPAMode = &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener-apiserver", + Namespace: namespace, + Labels: map[string]string{ + "app": "gardener", + "role": "apiserver", + "high-availability-config.resources.gardener.cloud/type": "server", + }, + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](2), + MaxReplicas: 4, + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + Name: "gardener-apiserver", + }, + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: ptr.To(resource.MustParse("6")), + }, + }, + }, + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: ptr.To(resource.MustParse("24G")), + }, + }, + }, + }, + Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: ptr.To[int32](60), + Policies: []autoscalingv2.HPAScalingPolicy{ + { + Type: autoscalingv2.PercentScalingPolicy, + Value: 100, + PeriodSeconds: 60, + }, + }, + }, + ScaleDown: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: ptr.To[int32](1800), + Policies: []autoscalingv2.HPAScalingPolicy{ + { + Type: autoscalingv2.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 300, + }, + }, + }, + }, + }, + } hvpa = &hvpav1alpha1.Hvpa{ ObjectMeta: metav1.ObjectMeta{ Name: "gardener-apiserver-hvpa", @@ -1476,7 +1577,7 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook It("should successfully deploy all resources", func() { expectedRuntimeObjects = append( expectedRuntimeObjects, - vpa, + vpaInBaselineMode, podDisruptionBudgetFor(true), serviceRuntimeFor(true), ) @@ -1503,6 +1604,25 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook }) }) + Context("when autoscaling mode is VPAAndHPA", func() { + BeforeEach(func() { + values.Values.Autoscaling.Mode = apiserver.AutoscalingModeVPAAndHPA + deployer = New(fakeClient, namespace, fakeSecretManager, values) + }) + + It("should successfully deploy all resources", func() { + expectedRuntimeObjects = append( + expectedRuntimeObjects, + vpaInHPAAndVPAMode, + hpaOnHPAAndVPAMode, + podDisruptionBudgetFor(true), + serviceRuntimeFor(true), + ) + + Expect(managedResourceRuntime).To(consistOf(expectedRuntimeObjects...)) + }) + }) + Context("when kubernetes version is < 1.26", func() { BeforeEach(func() { values.RuntimeVersion = semver.MustParse("1.25.0") @@ -1512,7 +1632,7 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook It("should successfully deploy all resources", func() { expectedRuntimeObjects = append( expectedRuntimeObjects, - vpa, + vpaInBaselineMode, podDisruptionBudgetFor(false), serviceRuntimeFor(false), ) @@ -1530,7 +1650,7 @@ kubeConfigFile: /etc/kubernetes/admission-kubeconfigs/validatingadmissionwebhook It("should successfully deploy all resources", func() { expectedRuntimeObjects = append( expectedRuntimeObjects, - vpa, + vpaInBaselineMode, podDisruptionBudgetFor(true), serviceRuntimeFor(false), ) diff --git a/pkg/component/gardener/apiserver/hpa.go b/pkg/component/gardener/apiserver/hpa.go new file mode 100644 index 00000000000..8989533338f --- /dev/null +++ b/pkg/component/gardener/apiserver/hpa.go @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package apiserver + +import ( + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/component/apiserver" + "github.com/gardener/gardener/pkg/utils" +) + +func (g *gardenerAPIServer) horizontalPodAutoscaler() *autoscalingv2.HorizontalPodAutoscaler { + if g.values.Autoscaling.Mode != apiserver.AutoscalingModeVPAAndHPA { + return nil + } + + return g.horizontalPodAutoscalerInVPAAndHPAMode() +} + +func (g *gardenerAPIServer) horizontalPodAutoscalerInVPAAndHPAMode() *autoscalingv2.HorizontalPodAutoscaler { + // The chosen value is 6 CPU: 1 CPU less than the VPA's maxAllowed 7 CPU in VPAAndHPA mode to have a headroom for the horizontal scaling. + hpaTargetAverageValueCPU := resource.MustParse("6") + // The chosen value is 24G: 4G less than the VPA's maxAllowed 28G in VPAAndHPA mode to have a headroom for the horizontal scaling. + hpaTargetAverageValueMemory := resource.MustParse("24G") + + return &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: DeploymentName, + Namespace: g.namespace, + Labels: utils.MergeStringMaps(GetLabels(), map[string]string{resourcesv1alpha1.HighAvailabilityConfigType: resourcesv1alpha1.HighAvailabilityConfigTypeServer}), + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](2), + MaxReplicas: 4, + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + Name: DeploymentName, + }, + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: &hpaTargetAverageValueCPU, + }, + }, + }, + { + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.AverageValueMetricType, + AverageValue: &hpaTargetAverageValueMemory, + }, + }, + }, + }, + Behavior: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: ptr.To[int32](60), + Policies: []autoscalingv2.HPAScalingPolicy{ + // Allow to upscale 100% of the current number of pods every 1 minute to see whether any upscale recommendation will still hold true after the cluster has settled + { + Type: autoscalingv2.PercentScalingPolicy, + Value: 100, + PeriodSeconds: 60, + }, + }, + }, + ScaleDown: &autoscalingv2.HPAScalingRules{ + StabilizationWindowSeconds: ptr.To[int32](1800), + Policies: []autoscalingv2.HPAScalingPolicy{ + // Allow to downscale one pod every 5 minutes to see whether any downscale recommendation will still hold true after the cluster has settled (conservatively) + { + Type: autoscalingv2.PodsScalingPolicy, + Value: 1, + PeriodSeconds: 300, + }, + }, + }, + }, + }, + } +} diff --git a/pkg/component/gardener/apiserver/vpa.go b/pkg/component/gardener/apiserver/vpa.go index a959effb223..2b541e9299d 100644 --- a/pkg/component/gardener/apiserver/vpa.go +++ b/pkg/component/gardener/apiserver/vpa.go @@ -11,16 +11,23 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/utils/ptr" "github.com/gardener/gardener/pkg/component/apiserver" ) func (g *gardenerAPIServer) verticalPodAutoscaler() *vpaautoscalingv1.VerticalPodAutoscaler { - if g.values.Autoscaling.Mode != apiserver.AutoscalingModeBaseline { + switch g.values.Autoscaling.Mode { + case apiserver.AutoscalingModeHVPA: return nil + case apiserver.AutoscalingModeVPAAndHPA: + return g.verticalPodAutoscalerInVPAAndHPAMode() + default: + return g.verticalPodAutoscalerInBaselineMode() } +} - vpaUpdateMode := vpaautoscalingv1.UpdateModeAuto +func (g *gardenerAPIServer) verticalPodAutoscalerInBaselineMode() *vpaautoscalingv1.VerticalPodAutoscaler { return &vpaautoscalingv1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: DeploymentName + "-vpa", @@ -34,12 +41,12 @@ func (g *gardenerAPIServer) verticalPodAutoscaler() *vpaautoscalingv1.VerticalPo Name: DeploymentName, }, UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ - UpdateMode: &vpaUpdateMode, + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto), }, ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ { - ContainerName: vpaautoscalingv1.DefaultContainerResourcePolicy, + ContainerName: containerName, MinAllowed: corev1.ResourceList{ corev1.ResourceMemory: resource.MustParse("256Mi"), }, @@ -49,3 +56,39 @@ func (g *gardenerAPIServer) verticalPodAutoscaler() *vpaautoscalingv1.VerticalPo }, } } + +func (g *gardenerAPIServer) verticalPodAutoscalerInVPAAndHPAMode() *vpaautoscalingv1.VerticalPodAutoscaler { + return &vpaautoscalingv1.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: DeploymentName + "-vpa", + Namespace: g.namespace, + Labels: GetLabels(), + }, + Spec: vpaautoscalingv1.VerticalPodAutoscalerSpec{ + TargetRef: &autoscalingv1.CrossVersionObjectReference{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + Name: DeploymentName, + }, + UpdatePolicy: &vpaautoscalingv1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpaautoscalingv1.UpdateModeAuto), + }, + ResourcePolicy: &vpaautoscalingv1.PodResourcePolicy{ + ContainerPolicies: []vpaautoscalingv1.ContainerResourcePolicy{ + { + ContainerName: containerName, + MinAllowed: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("400M"), + }, + MaxAllowed: corev1.ResourceList{ + // The CPU and memory are aligned to the machine ration of 1:4. + corev1.ResourceCPU: resource.MustParse("7"), + corev1.ResourceMemory: resource.MustParse("28G"), + }, + ControlledValues: ptr.To(vpaautoscalingv1.ContainerControlledValuesRequestsOnly), + }, + }, + }, + }, + } +}