-
Notifications
You must be signed in to change notification settings - Fork 450
/
gardenlet_kubeconfig.go
131 lines (109 loc) · 5.56 KB
/
gardenlet_kubeconfig.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rotation
import (
"context"
"fmt"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest"
clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
"github.com/gardener/gardener/pkg/utils"
"github.com/gardener/gardener/pkg/utils/kubernetes/health"
. "github.com/gardener/gardener/pkg/utils/test"
)
const (
gardenletDeploymentName = "gardenlet"
gardenletDeploymentNamespace = "garden"
)
// GardenletKubeconfigRotationVerifier verifies if a gardenlet kubeconfig rotation was successful
type GardenletKubeconfigRotationVerifier struct {
GardenReader client.Reader
SeedReader client.Reader
Seed *gardencorev1beta1.Seed
GardenletKubeconfigSecretName string
GardenletKubeconfigSecretNamespace string
timeBeforeRotation time.Time
oldGardenletName string
}
// Before saves the status before the rotation
func (v *GardenletKubeconfigRotationVerifier) Before(ctx context.Context) {
v.timeBeforeRotation = time.Now().UTC()
Eventually(func(_ Gomega) {
Expect(v.GardenReader.Get(ctx, client.ObjectKeyFromObject(v.Seed), v.Seed)).To(Succeed())
v.oldGardenletName = v.Seed.Status.Gardener.Name
}).Should(Succeed())
}
// After verifies the state after the rotation
func (v *GardenletKubeconfigRotationVerifier) After(parentCtx context.Context, expectPodRestart bool) {
ctx, cancel := context.WithTimeout(parentCtx, 5*time.Minute)
defer cancel()
if expectPodRestart {
By("Verify that new gardenlet pod has taken over responsibility for seed")
CEventually(ctx, func() error {
if err := v.GardenReader.Get(ctx, client.ObjectKeyFromObject(v.Seed), v.Seed); err != nil {
return err
}
if v.Seed.Status.Gardener.Name != v.oldGardenletName {
return nil
}
return fmt.Errorf("new gardenlet pod has not yet taken over responsibility for seed: %s", client.ObjectKeyFromObject(v.Seed))
}).WithPolling(5 * time.Second).Should(Succeed())
}
By("Verify that gardenlet's kubeconfig secret has actually been renewed")
CEventually(ctx, func() error {
secret := &corev1.Secret{}
if err := v.SeedReader.Get(ctx, client.ObjectKey{Name: v.GardenletKubeconfigSecretName, Namespace: v.GardenletKubeconfigSecretNamespace}, secret); err != nil {
return err
}
kubeconfig := &clientcmdv1.Config{}
if _, _, err := clientcmdlatest.Codec.Decode(secret.Data["kubeconfig"], nil, kubeconfig); err != nil {
return err
}
clientCertificate, err := utils.DecodeCertificate(kubeconfig.AuthInfos[0].AuthInfo.ClientCertificateData)
if err != nil {
return err
}
newClientCertificateIssuedAt := clientCertificate.NotBefore.UTC()
// The kube-controller-manager always backdates the issued certificate by 5m, see https://github.com/kubernetes/kubernetes/blob/252935368ab67f38cb252df0a961a6dcb81d20eb/pkg/controller/certificates/signer/signer.go#L197.
// Consequently, we add these 5m so that we can assert whether the certificate was actually issued after the
// time we recorded before the rotation was triggered.
newClientCertificateIssuedAt = newClientCertificateIssuedAt.Add(5 * time.Minute)
// The newClientCertificateIssuedAt time does not contain any nanoseconds, however the v.timeBeforeRotation
// does. This was leading to failing tests in case the new client certificate was issued at the very same second
// like the v.timeBeforeRotation, e.g. v.timeBeforeRotation = 2022-09-02 20:12:24.058418988 +0000 UTC,
// newClientCertificateIssuedAt = 2022-09-02 20:12:24 +0000 UTC. Hence, let's round the times down to the second
// to avoid such discrepancies. See https://github.com/gardener/gardener/issues/6618 for more details.
newClientCertificateIssuedAt = newClientCertificateIssuedAt.Truncate(time.Second)
timeBeforeRotation := v.timeBeforeRotation.Truncate(time.Second)
if newClientCertificateIssuedAt.Equal(timeBeforeRotation) || newClientCertificateIssuedAt.After(timeBeforeRotation) {
return nil
}
return fmt.Errorf("kubeconfig secret has not yet been renewed, timeBeforeRotation: %s, newClientCertificateIssuedAt: %s", v.timeBeforeRotation, newClientCertificateIssuedAt)
}).WithPolling(5 * time.Second).Should(Succeed())
By("Verify that gardenlet's deployment is updated and healthy after kubeconfig secret was renewed")
gardenletDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: gardenletDeploymentName, Namespace: gardenletDeploymentNamespace}}
isUpdated := health.IsDeploymentUpdated(v.SeedReader, gardenletDeployment)
CEventually(ctx, func(g Gomega) {
updated, err := isUpdated(ctx)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(updated).To(BeTrue())
}).WithPolling(5 * time.Second).Should(Succeed())
}