From 51812d49edc4713280145428cc315f0512c9e65c Mon Sep 17 00:00:00 2001 From: Niclas Schad Date: Wed, 24 May 2023 10:46:51 +0200 Subject: [PATCH 1/6] Increase client-side rate limits of seed resource-manager Signed-off-by: Niclas Schad --- pkg/component/gardener/resourcemanager/resource_manager.go | 2 ++ pkg/component/gardener/resourcemanager/resource_manager_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pkg/component/gardener/resourcemanager/resource_manager.go b/pkg/component/gardener/resourcemanager/resource_manager.go index 49e72411d4e..21191b1b6c5 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager.go +++ b/pkg/component/gardener/resourcemanager/resource_manager.go @@ -618,6 +618,8 @@ func (r *resourceManager) ensureConfigMap(ctx context.Context, configMap *corev1 } if r.values.ResponsibilityMode == ForSource || r.values.ResponsibilityMode == ForSourceAndTarget { + config.SourceClientConnection.QPS = 300 + config.SourceClientConnection.Burst = 500 config.Webhooks.CRDDeletionProtection.Enabled = true config.Webhooks.ExtensionValidation.Enabled = true } diff --git a/pkg/component/gardener/resourcemanager/resource_manager_test.go b/pkg/component/gardener/resourcemanager/resource_manager_test.go index c02fb426dfa..5e4ca94c903 100644 --- a/pkg/component/gardener/resourcemanager/resource_manager_test.go +++ b/pkg/component/gardener/resourcemanager/resource_manager_test.go @@ -465,6 +465,8 @@ var _ = Describe("ResourceManager", func() { }, IngressControllerSelector: ingressControllerSelector, } + config.SourceClientConnection.QPS = 300 + config.SourceClientConnection.Burst = 500 config.Webhooks.CRDDeletionProtection.Enabled = true config.Webhooks.ExtensionValidation.Enabled = true config.Webhooks.SeccompProfile.Enabled = true From 383ce7f459f702ac40450ee93c884add39e96d0d Mon Sep 17 00:00:00 2001 From: Marcel Boehm Date: Thu, 25 Apr 2024 15:48:07 +0200 Subject: [PATCH 2/6] Enable AlwaysUpdate for resource manager targeting runtime cluster --- pkg/component/shared/resourcemanager.go | 1 + pkg/component/shared/resourcemanager_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/component/shared/resourcemanager.go b/pkg/component/shared/resourcemanager.go index 88e07529251..df729e4fc3f 100644 --- a/pkg/component/shared/resourcemanager.go +++ b/pkg/component/shared/resourcemanager.go @@ -55,6 +55,7 @@ func NewRuntimeGardenerResourceManager( defaultValues := resourcemanager.Values{ ConcurrentSyncs: ptr.To(20), + AlwaysUpdate: ptr.To(true), HealthSyncPeriod: &metav1.Duration{Duration: time.Minute}, Image: image.String(), MaxConcurrentNetworkPolicyWorkers: ptr.To(20), diff --git a/pkg/component/shared/resourcemanager_test.go b/pkg/component/shared/resourcemanager_test.go index 50a1f9f9973..fd8e40173d7 100644 --- a/pkg/component/shared/resourcemanager_test.go +++ b/pkg/component/shared/resourcemanager_test.go @@ -58,6 +58,7 @@ var _ = Describe("ResourceManager", func() { }) Expect(err).NotTo(HaveOccurred()) Expect(resourceManager.GetValues()).To(Equal(resourcemanager.Values{ + AlwaysUpdate: ptr.To(true), ClusterIdentity: ptr.To("foo"), ConcurrentSyncs: ptr.To(21), HealthSyncPeriod: &metav1.Duration{Duration: time.Minute}, From af0834a7dbab8ee292771fdd0bd223ba953f1c91 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 27 Jun 2024 14:39:00 +0200 Subject: [PATCH 3/6] adapt gardener-node-agent to images built using ko The changes are similar to those made by pkg/provider-local/webhook/nodeagentosc/mutator.go --- .../nodeinit/nodeinit_test.go | 2 +- .../nodeinit/templates/scripts/init.tpl.sh | 2 +- .../original/components/nodeagent/component.go | 18 ++++-------------- .../components/nodeagent/component_test.go | 2 +- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go b/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go index 41dbad947fb..e87eac766fc 100644 --- a/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go +++ b/pkg/component/extensions/operatingsystemconfig/nodeinit/nodeinit_test.go @@ -130,7 +130,7 @@ ctr images mount "` + image + `" "$tmp_dir" echo "> Copy gardener-node-agent binary to host (/opt/bin) and make it executable" mkdir -p "/opt/bin" -cp -f "$tmp_dir/gardener-node-agent" "/opt/bin" +cp -f "$tmp_dir/ko-app/gardener-node-agent" "/opt/bin" chmod +x "/opt/bin/gardener-node-agent" echo "> Bootstrap gardener-node-agent" diff --git a/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh b/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh index c9ff98d44f0..8395ac8a32c 100644 --- a/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh +++ b/pkg/component/extensions/operatingsystemconfig/nodeinit/templates/scripts/init.tpl.sh @@ -17,7 +17,7 @@ ctr images mount "{{ .image }}" "$tmp_dir" echo "> Copy gardener-node-agent binary to host ({{ .binaryDirectory }}) and make it executable" mkdir -p "{{ .binaryDirectory }}" -cp -f "$tmp_dir/gardener-node-agent" "{{ .binaryDirectory }}" +cp -f "$tmp_dir/ko-app/gardener-node-agent" "{{ .binaryDirectory }}" chmod +x "{{ .binaryDirectory }}/gardener-node-agent" echo "> Bootstrap gardener-node-agent" diff --git a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component.go b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component.go index 8436e0e7a7f..21352e888d4 100644 --- a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component.go +++ b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component.go @@ -6,7 +6,6 @@ package nodeagent import ( "fmt" - "strings" "time" "github.com/Masterminds/semver/v3" @@ -79,7 +78,10 @@ func (component) Config(ctx components.Context) ([]extensionsv1alpha1.Unit, []ex Path: PathBinary, Permissions: ptr.To[uint32](0755), Content: extensionsv1alpha1.FileContent{ - ImageRef: fileContentImageRef(ctx.Images[imagevector.ContainerImageNameGardenerNodeAgent].String()), + ImageRef: &extensionsv1alpha1.FileContentImageRef{ + Image: ctx.Images[imagevector.ContainerImageNameGardenerNodeAgent].String(), + FilePathInImage: "/ko-app/gardener-node-agent", + }, }, }) @@ -160,15 +162,3 @@ func Files(config *nodeagentconfigv1alpha1.NodeAgentConfiguration) ([]extensions Content: extensionsv1alpha1.FileContent{Inline: &extensionsv1alpha1.FileContentInline{Encoding: "b64", Data: utils.EncodeBase64(configRaw)}}, }}, nil } - -func fileContentImageRef(image string) *extensionsv1alpha1.FileContentImageRef { - var prefix string - if strings.HasPrefix(image, "garden.local.gardener.cloud:5001") { - prefix = "/ko-app" - } - - return &extensionsv1alpha1.FileContentImageRef{ - Image: image, - FilePathInImage: prefix + "/gardener-node-agent", - } -} diff --git a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component_test.go b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component_test.go index b9de2d24b5c..dcae7338a18 100644 --- a/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component_test.go +++ b/pkg/component/extensions/operatingsystemconfig/original/components/nodeagent/component_test.go @@ -82,7 +82,7 @@ WantedBy=multi-user.target`), Content: extensionsv1alpha1.FileContent{ ImageRef: &extensionsv1alpha1.FileContentImageRef{ Image: "gardener-node-agent:v1", - FilePathInImage: "/gardener-node-agent", + FilePathInImage: "/ko-app/gardener-node-agent", }, }, }))) From 9dc76bec270dffefa689c586655a93f1c9900e7e Mon Sep 17 00:00:00 2001 From: Tim Ebert Date: Tue, 25 Jun 2024 10:15:12 +0200 Subject: [PATCH 4/6] add custom configuration for `Garden` --- pkg/component/gardener/apiserver/apiserver_test.go | 1 + pkg/component/gardener/apiserver/deployment.go | 2 ++ pkg/component/gardener/controllermanager/configmaps.go | 2 ++ .../gardener/controllermanager/controller_manager_test.go | 4 +++- 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/component/gardener/apiserver/apiserver_test.go b/pkg/component/gardener/apiserver/apiserver_test.go index 445357df500..cc3257dccff 100644 --- a/pkg/component/gardener/apiserver/apiserver_test.go +++ b/pkg/component/gardener/apiserver/apiserver_test.go @@ -336,6 +336,7 @@ var _ = Describe("GardenerAPIServer", func() { "--log-level=" + logLevel, "--log-format=" + logFormat, "--secure-port=8443", + "--shoot-admin-kubeconfig-max-expiration=4320h", "--goaway-chance=0.001500", "--workload-identity-token-issuer=" + workloadIdentityIssuer, "--workload-identity-signing-key-file=/etc/gardener-apiserver/workload-identity/signing/key.pem", diff --git a/pkg/component/gardener/apiserver/deployment.go b/pkg/component/gardener/apiserver/deployment.go index a7eb02ab87a..ffa904a1df7 100644 --- a/pkg/component/gardener/apiserver/deployment.go +++ b/pkg/component/gardener/apiserver/deployment.go @@ -53,6 +53,8 @@ func (g *gardenerAPIServer) deployment( "--log-level=" + g.values.LogLevel, "--log-format=" + g.values.LogFormat, fmt.Sprintf("--secure-port=%d", port), + // TODO: replace this hardcoded configuration with proper fields in the Garden API + "--shoot-admin-kubeconfig-max-expiration=4320h", // 6 months } if g.values.GoAwayChance != nil { diff --git a/pkg/component/gardener/controllermanager/configmaps.go b/pkg/component/gardener/controllermanager/configmaps.go index 76a2b1d5fdc..af1e8d17808 100644 --- a/pkg/component/gardener/controllermanager/configmaps.go +++ b/pkg/component/gardener/controllermanager/configmaps.go @@ -62,6 +62,8 @@ func (g *gardenerControllerManager) configMapControllerManagerConfig() (*corev1. Project: &controllermanagerconfigv1alpha1.ProjectControllerConfiguration{ ConcurrentSyncs: ptr.To(20), Quotas: g.values.Quotas, + // TODO: replace this hardcoded configuration with proper fields in the Garden API + StaleExpirationTimeDays: ptr.To(6000), }, SecretBinding: &controllermanagerconfigv1alpha1.SecretBindingControllerConfiguration{ ConcurrentSyncs: ptr.To(20), diff --git a/pkg/component/gardener/controllermanager/controller_manager_test.go b/pkg/component/gardener/controllermanager/controller_manager_test.go index 41267ebdedc..afd30b8ae05 100644 --- a/pkg/component/gardener/controllermanager/controller_manager_test.go +++ b/pkg/component/gardener/controllermanager/controller_manager_test.go @@ -315,7 +315,7 @@ var _ = Describe("GardenerControllerManager", func() { managedResourceSecretRuntime.Name = managedResourceRuntime.Spec.SecretRefs[0].Name Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecretRuntime), managedResourceSecretRuntime)).To(Succeed()) cm := configMap(namespace, values) - Expect(cm.Name).To(Equal("gardener-controller-manager-config-960e3f19")) + Expect(cm.Name).To(Equal("gardener-controller-manager-config-625036ea")) expectedRuntimeObjects = []client.Object{ cm, podDisruptionBudget, @@ -692,6 +692,8 @@ func configMap(namespace string, testValues Values) *corev1.ConfigMap { Project: &controllermanagerconfigv1alpha1.ProjectControllerConfiguration{ ConcurrentSyncs: ptr.To(20), Quotas: testValues.Quotas, + // TODO: replace this hardcoded configuration with proper fields in the Garden API + StaleExpirationTimeDays: ptr.To(6000), }, SecretBinding: &controllermanagerconfigv1alpha1.SecretBindingControllerConfiguration{ ConcurrentSyncs: ptr.To(20), From 0e8573f62a85c3237880f83efefd767430f7955a Mon Sep 17 00:00:00 2001 From: Duciwuci Date: Tue, 15 Oct 2024 11:33:26 +0200 Subject: [PATCH 5/6] set insecure_skip_verify to true for cadvisor and kubelet job Since Prometheus Targets are all down due to gardener#9716 in Gardener v1.96, we need to set insecure_skip_verify to true on cadvisor & kubelet scrape config. Since this is a hotfix, it can be dropped whenever we have a long term solution. --- .../prometheus/cache/assets/scrapeconfigs/cadvisor.yaml | 2 +- .../prometheus/cache/assets/scrapeconfigs/kubelet.yaml | 2 +- .../monitoring/prometheus/cache/scrapeconfigs_test.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/cadvisor.yaml b/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/cadvisor.yaml index 3ed2c00c822..72b51e431d8 100644 --- a/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/cadvisor.yaml +++ b/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/cadvisor.yaml @@ -6,7 +6,7 @@ metrics_path: /metrics/cadvisor tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: {{.IsManagedSeed}} + insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: diff --git a/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/kubelet.yaml b/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/kubelet.yaml index 8a901530729..b984c74240c 100644 --- a/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/kubelet.yaml +++ b/pkg/component/observability/monitoring/prometheus/cache/assets/scrapeconfigs/kubelet.yaml @@ -4,7 +4,7 @@ scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: {{.IsManagedSeed}} + insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: diff --git a/pkg/component/observability/monitoring/prometheus/cache/scrapeconfigs_test.go b/pkg/component/observability/monitoring/prometheus/cache/scrapeconfigs_test.go index eab3607c0f7..64e469a4881 100644 --- a/pkg/component/observability/monitoring/prometheus/cache/scrapeconfigs_test.go +++ b/pkg/component/observability/monitoring/prometheus/cache/scrapeconfigs_test.go @@ -122,7 +122,7 @@ metrics_path: /metrics/cadvisor tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: false + insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: @@ -183,7 +183,7 @@ scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: false + insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: From e0c85f8d6ef4dc086f2bee5ff16f551a1eb46c99 Mon Sep 17 00:00:00 2001 From: Duciwuci Date: Fri, 21 Mar 2025 13:33:42 +0100 Subject: [PATCH 6/6] Remove Extension Validation for Provider in ETCD-Backup With Gardener v1.114+ the operator extensions are a required resource for the upgrade. In our use case, we need to deactivate this validation since this is a breaking change. We need to do a backup-s3 migration. Drop this commit, after the migration of the extension was performed. --- .../extension/required/runtime/add_test.go | 1 - pkg/utils/gardener/garden.go | 7 ++++--- pkg/utils/gardener/garden_test.go | 5 +---- .../extension/required/runtime/runtime_test.go | 13 ++----------- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/pkg/operator/controller/extension/required/runtime/add_test.go b/pkg/operator/controller/extension/required/runtime/add_test.go index 4dab1a65832..be89a579d6e 100644 --- a/pkg/operator/controller/extension/required/runtime/add_test.go +++ b/pkg/operator/controller/extension/required/runtime/add_test.go @@ -122,7 +122,6 @@ var _ = Describe("Add", func() { It("should return the expected extensions", func() { Expect(mapperFunc(ctx, garden)).To(ConsistOf( - Equal(reconcile.Request{NamespacedName: types.NamespacedName{Name: infraExtension.Name}}), Equal(reconcile.Request{NamespacedName: types.NamespacedName{Name: dnsExtension.Name}}), )) }) diff --git a/pkg/utils/gardener/garden.go b/pkg/utils/gardener/garden.go index 279911570ba..e94c1035ec5 100644 --- a/pkg/utils/gardener/garden.go +++ b/pkg/utils/gardener/garden.go @@ -378,9 +378,10 @@ func IsServedByKubeAPIServer(resource string) bool { func ComputeRequiredExtensionsForGarden(garden *operatorv1alpha1.Garden) sets.Set[string] { requiredExtensions := sets.New[string]() - if helper.GetETCDMainBackup(garden) != nil { - requiredExtensions.Insert(ExtensionsID(extensionsv1alpha1.BackupBucketResource, garden.Spec.VirtualCluster.ETCD.Main.Backup.Provider)) - } + // TODO: reapply validation when STACKITSKE-830 is resolved + // if helper.GetETCDMainBackup(garden) != nil { + // requiredExtensions.Insert(ExtensionsID(extensionsv1alpha1.BackupBucketResource, garden.Spec.VirtualCluster.ETCD.Main.Backup.Provider)) + // } for _, provider := range helper.GetDNSProviders(garden) { requiredExtensions.Insert(ExtensionsID(extensionsv1alpha1.DNSRecordResource, provider.Type)) diff --git a/pkg/utils/gardener/garden_test.go b/pkg/utils/gardener/garden_test.go index 5a7a10a0b7b..2c9e26b5b93 100644 --- a/pkg/utils/gardener/garden_test.go +++ b/pkg/utils/gardener/garden_test.go @@ -396,9 +396,7 @@ var _ = Describe("Garden", func() { }, } - Expect(ComputeRequiredExtensionsForGarden(garden).UnsortedList()).To(ConsistOf( - "BackupBucket/local-infrastructure", - )) + Expect(ComputeRequiredExtensionsForGarden(garden).UnsortedList()).To(BeEmpty()) }) It("should return required DNSRecord extension types", func() { @@ -446,7 +444,6 @@ var _ = Describe("Garden", func() { } Expect(ComputeRequiredExtensionsForGarden(garden).UnsortedList()).To(ConsistOf( - "BackupBucket/local-infrastructure", "DNSRecord/local-dns", "Extension/local-extension-1", "Extension/local-extension-2", diff --git a/test/integration/operator/extension/required/runtime/runtime_test.go b/test/integration/operator/extension/required/runtime/runtime_test.go index 9c961ab8a24..5a1e7d0525c 100644 --- a/test/integration/operator/extension/required/runtime/runtime_test.go +++ b/test/integration/operator/extension/required/runtime/runtime_test.go @@ -189,10 +189,10 @@ var _ = Describe("Extension Required Runtime controller tests", Ordered, func() It("should report extensions as required after garden was created", func() { Expect(testClient.Create(ctx, garden)).To(Succeed()) - for _, ext := range []client.Object{providerExtension, dnsExtension} { + for _, ext := range []client.Object{dnsExtension} { Eventually(func(g Gomega) []gardencorev1beta1.Condition { g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(ext), ext)).To(Succeed()) - return providerExtension.Status.Conditions + return dnsExtension.Status.Conditions }).Should(ContainCondition( OfType(operatorv1alpha1.ExtensionRequiredRuntime), WithStatus(gardencorev1beta1.ConditionTrue), @@ -300,15 +300,6 @@ var _ = Describe("Extension Required Runtime controller tests", Ordered, func() WithStatus(gardencorev1beta1.ConditionFalse), WithReason("ExtensionNotRequired"), )) - - Consistently(func(g Gomega) []gardencorev1beta1.Condition { - g.Expect(testClient.Get(ctx, client.ObjectKeyFromObject(providerExtension), providerExtension)).To(Succeed()) - return providerExtension.Status.Conditions - }).Should(ContainCondition( - OfType(operatorv1alpha1.ExtensionRequiredRuntime), - WithStatus(gardencorev1beta1.ConditionTrue), - WithReason("ExtensionRequired"), - )) }) It("should report provider extension as not required during garden deletion after backupbucket is gone", func() {