/
etcd.go
151 lines (129 loc) · 6.6 KB
/
etcd.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package secretsrotation
import (
"context"
"fmt"
"github.com/go-logr/logr"
"golang.org/x/time/rate"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
"sigs.k8s.io/controller-runtime/pkg/client"
v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
"github.com/gardener/gardener/pkg/utils"
"github.com/gardener/gardener/pkg/utils/flow"
kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager"
)
// RewriteSecretsAddLabel patches all secrets in all namespaces in the target clusters and adds a label whose value is
// the name of the current ETCD encryption key secret. This function is useful for the ETCD encryption key secret
// rotation which requires all secrets to be rewritten to ETCD so that they become encrypted with the new key.
// After it's done, it snapshots ETCD so that we can restore backups in case we lose the cluster before the next
// incremental snapshot is taken.
func RewriteSecretsAddLabel(ctx context.Context, log logr.Logger, c client.Client, secretsManager secretsmanager.Interface) error {
etcdEncryptionKeySecret, found := secretsManager.Get(v1beta1constants.SecretNameETCDEncryptionKey, secretsmanager.Current)
if !found {
return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameETCDEncryptionKey)
}
return rewriteSecrets(
ctx,
log,
c,
utils.MustNewRequirement(labelKeyRotationKeyName, selection.NotEquals, etcdEncryptionKeySecret.Name),
func(objectMeta *metav1.ObjectMeta) {
metav1.SetMetaDataLabel(objectMeta, labelKeyRotationKeyName, etcdEncryptionKeySecret.Name)
},
)
}
// SnapshotETCDAfterRewritingSecrets performs a full snapshot on ETCD after the secrets got rewritten as part of the
// ETCD encryption secret rotation. It adds an annotation to the kube-apiserver deployment after it's done so that it
// does not take another snapshot again after it succeeded once.
func SnapshotETCDAfterRewritingSecrets(ctx context.Context, runtimeClient client.Client, snapshotEtcd func(ctx context.Context) error, kubeAPIServerNamespace, namePrefix string) error {
// Check if we have to snapshot ETCD now that we have rewritten all secrets.
meta := &metav1.PartialObjectMetadata{}
meta.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("Deployment"))
if err := runtimeClient.Get(ctx, kubernetesutils.Key(kubeAPIServerNamespace, namePrefix+v1beta1constants.DeploymentNameKubeAPIServer), meta); err != nil {
return err
}
if metav1.HasAnnotation(meta.ObjectMeta, AnnotationKeyEtcdSnapshotted) {
return nil
}
if err := snapshotEtcd(ctx); err != nil {
return err
}
// If we have hit this point then we have snapshotted ETCD successfully. Now we can mark this step as "completed"
// (via an annotation) so that we do not trigger a snapshot again in a future reconciliation in case the current one
// fails after this step.
return PatchKubeAPIServerDeploymentMeta(ctx, runtimeClient, kubeAPIServerNamespace, namePrefix, func(meta *metav1.PartialObjectMetadata) {
metav1.SetMetaDataAnnotation(&meta.ObjectMeta, AnnotationKeyEtcdSnapshotted, "true")
})
}
// RewriteSecretsRemoveLabel patches all secrets in all namespaces in the target clusters and removes the label whose
// value is the name of the current ETCD encryption key secret. This function is useful for the ETCD encryption key
// secret rotation which requires all secrets to be rewritten to ETCD so that they become encrypted with the new key.
func RewriteSecretsRemoveLabel(ctx context.Context, log logr.Logger, runtimeClient, targetClient client.Client, kubeAPIServerNamespace, namePrefix string) error {
if err := rewriteSecrets(
ctx,
log,
targetClient,
utils.MustNewRequirement(labelKeyRotationKeyName, selection.Exists),
func(objectMeta *metav1.ObjectMeta) {
delete(objectMeta.Labels, labelKeyRotationKeyName)
},
); err != nil {
return err
}
return PatchKubeAPIServerDeploymentMeta(ctx, runtimeClient, kubeAPIServerNamespace, namePrefix, func(meta *metav1.PartialObjectMetadata) {
delete(meta.Annotations, AnnotationKeyEtcdSnapshotted)
})
}
func rewriteSecrets(ctx context.Context, log logr.Logger, c client.Client, requirement labels.Requirement, mutateObjectMeta func(*metav1.ObjectMeta)) error {
secretList := &metav1.PartialObjectMetadataList{}
secretList.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("SecretList"))
if err := c.List(ctx, secretList, client.MatchingLabelsSelector{Selector: labels.NewSelector().Add(requirement)}); err != nil {
return err
}
log.Info("Secrets requiring to be rewritten after ETCD encryption key rotation", "number", len(secretList.Items))
var (
limiter = rate.NewLimiter(rate.Limit(rotationQPS), rotationQPS)
taskFns []flow.TaskFn
)
for _, obj := range secretList.Items {
secret := obj
taskFns = append(taskFns, func(ctx context.Context) error {
patch := client.StrategicMergeFrom(secret.DeepCopy())
mutateObjectMeta(&secret.ObjectMeta)
// Wait until we are allowed by the limiter to not overload the kube-apiserver with too many requests.
if err := limiter.Wait(ctx); err != nil {
return err
}
return c.Patch(ctx, &secret, patch)
})
}
return flow.Parallel(taskFns...)(ctx)
}
// PatchKubeAPIServerDeploymentMeta patches metadata of a Kubernetes API-Server deployment
func PatchKubeAPIServerDeploymentMeta(ctx context.Context, c client.Client, namespace, namePrefix string, mutate func(deployment *metav1.PartialObjectMetadata)) error {
meta := &metav1.PartialObjectMetadata{}
meta.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("Deployment"))
if err := c.Get(ctx, kubernetesutils.Key(namespace, namePrefix+v1beta1constants.DeploymentNameKubeAPIServer), meta); err != nil {
return err
}
patch := client.MergeFrom(meta.DeepCopy())
mutate(meta)
return c.Patch(ctx, meta, patch)
}