/
shootmigrationtest.go
490 lines (432 loc) · 18.7 KB
/
shootmigrationtest.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
// Copyright 2020 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package framework
import (
"context"
"errors"
"fmt"
"os"
"reflect"
"slices"
"strconv"
"strings"
"github.com/onsi/ginkgo/v2"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1"
"github.com/gardener/gardener/pkg/client/kubernetes"
secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager"
"github.com/gardener/gardener/test/utils/access"
)
// ShootMigrationTest represents a shoot migration test.
// It can be used to test the migration of shoots between various seeds.
type ShootMigrationTest struct {
GardenerFramework *GardenerFramework
Config *ShootMigrationConfig
TargetSeedClient kubernetes.Interface
SourceSeedClient kubernetes.Interface
ShootClient kubernetes.Interface
TargetSeed *gardencorev1beta1.Seed
SourceSeed *gardencorev1beta1.Seed
ComparisonElementsBeforeMigration ShootComparisonElements
ComparisonElementsAfterMigration ShootComparisonElements
Shoot gardencorev1beta1.Shoot
SeedShootNamespace string
MigrationTime metav1.Time
}
// ShootMigrationConfig is the configuration for a shoot migration test that will be filled with user provided data
type ShootMigrationConfig struct {
TargetSeedName string
SourceSeedName string
ShootName string
ShootNamespace string
AddTestRunTaint string
SkipNodeCheck bool
SkipMachinesCheck bool
SkipShootClientCreation bool
SkipProtectedToleration bool
}
// ShootComparisonElements contains details about Machines and Nodes that will be compared during the tests
type ShootComparisonElements struct {
MachineNames []string
MachineNodes []string
NodeNames []string
SecretsMap map[string]corev1.Secret
}
// NewShootMigrationTest creates a new simple shoot migration test
func NewShootMigrationTest(ctx context.Context, f *GardenerFramework, cfg *ShootMigrationConfig) (*ShootMigrationTest, error) {
t := &ShootMigrationTest{
GardenerFramework: f,
Config: cfg,
}
return t, t.initializeShootMigrationTest(ctx)
}
func (t *ShootMigrationTest) initializeShootMigrationTest(ctx context.Context) error {
if err := t.initShootAndClient(ctx); err != nil {
return err
}
t.SeedShootNamespace = ComputeTechnicalID(t.GardenerFramework.ProjectNamespace, &t.Shoot)
if err := t.initSeedsAndClients(ctx); err != nil {
return err
}
return t.populateBeforeMigrationComparisonElements(ctx)
}
func (t *ShootMigrationTest) initShootAndClient(ctx context.Context) (err error) {
shoot := &gardencorev1beta1.Shoot{ObjectMeta: metav1.ObjectMeta{Name: t.Config.ShootName, Namespace: t.Config.ShootNamespace}}
if err = t.GardenerFramework.GetShoot(ctx, shoot); err != nil {
return err
}
if !shoot.Status.IsHibernated && !t.Config.SkipShootClientCreation {
t.ShootClient, err = access.CreateShootClientFromAdminKubeconfig(ctx, t.GardenerFramework.GardenClient, shoot)
if err != nil {
return err
}
}
t.Shoot = *shoot
return
}
func (t *ShootMigrationTest) initSeedsAndClients(ctx context.Context) error {
t.Config.SourceSeedName = *t.Shoot.Spec.SeedName
seed, seedClient, err := t.GardenerFramework.GetSeed(ctx, t.Config.TargetSeedName)
if err != nil {
return err
}
t.TargetSeedClient = seedClient
t.TargetSeed = seed
seed, seedClient, err = t.GardenerFramework.GetSeed(ctx, t.Config.SourceSeedName)
if err != nil {
return err
}
t.SourceSeedClient = seedClient
t.SourceSeed = seed
return nil
}
// MigrateShoot triggers shoot migration by changing the value of "shoot.Spec.SeedName" to the value of "ShootMigrationConfig.TargetSeedName"
func (t *ShootMigrationTest) MigrateShoot(ctx context.Context) error {
// Dump gardener state if delete shoot is in exit handler
if os.Getenv("TM_PHASE") == "Exit" {
if shootFramework, err := t.GardenerFramework.NewShootFramework(ctx, &t.Shoot); err == nil {
shootFramework.DumpState(ctx)
} else {
t.GardenerFramework.DumpState(ctx)
}
}
t.MigrationTime = metav1.Now()
return t.GardenerFramework.MigrateShoot(ctx, &t.Shoot, t.TargetSeed, func(shoot *gardencorev1beta1.Shoot) error {
if !t.Config.SkipProtectedToleration {
shoot.Spec.Tolerations = appendToleration(shoot.Spec.Tolerations, gardencorev1beta1.SeedTaintProtected, nil)
}
if applyTestRunTaint, err := strconv.ParseBool(t.Config.AddTestRunTaint); applyTestRunTaint && err == nil {
shoot.Spec.Tolerations = appendToleration(shoot.Spec.Tolerations, SeedTaintTestRun, ptr.To(GetTestRunID()))
}
return nil
})
}
func appendToleration(tolerations []gardencorev1beta1.Toleration, key string, value *string) []gardencorev1beta1.Toleration {
toleration := gardencorev1beta1.Toleration{
Key: key,
Value: value,
}
if tolerations == nil {
tolerations = make([]gardencorev1beta1.Toleration, 0)
} else {
for _, t := range tolerations {
if t.Key == key {
t.Value = value
return tolerations
}
}
}
return append(tolerations, toleration)
}
// VerifyMigration checks that the shoot components are migrated properly
func (t ShootMigrationTest) VerifyMigration(ctx context.Context) error {
if err := t.populateAfterMigrationComparisonElements(ctx); err != nil {
return err
}
ginkgo.By("Compare all Machines, Nodes and persisted Secrets after the migration")
if err := t.compareElementsAfterMigration(); err != nil {
return err
}
ginkgo.By("Check for orphaned resources")
return t.checkForOrphanedNonNamespacedResources(ctx)
}
// GetNodeNames uses the shootClient to fetch all Node names from the Shoot
func (t *ShootMigrationTest) GetNodeNames(ctx context.Context, shootClient kubernetes.Interface) (nodeNames []string, err error) {
if t.Shoot.Status.IsHibernated {
return make([]string, 0), nil // Initialize to empty slice in order pass 0 elements DeepEqual check
}
nodeList := corev1.NodeList{}
t.GardenerFramework.Logger.Info("Listing nodes")
if err := shootClient.Client().List(ctx, &nodeList); err != nil {
return nil, err
}
nodeNames = make([]string, len(nodeList.Items))
for i, node := range nodeList.Items {
t.GardenerFramework.Logger.Info("Found node", "index", i, "nodeName", node.Name)
nodeNames[i] = node.Name
}
slices.Sort(nodeNames)
return
}
// GetMachineDetails uses the seedClient to fetch all Machine names and the names of their corresponding Nodes
func (t *ShootMigrationTest) GetMachineDetails(ctx context.Context, seedClient kubernetes.Interface) (machineNames, machineNodes []string, err error) {
log := t.GardenerFramework.Logger.WithValues("namespace", t.SeedShootNamespace)
machineList := unstructured.UnstructuredList{}
machineList.SetAPIVersion("machine.sapcloud.io/v1alpha1")
machineList.SetKind("Machine")
log.Info("Listing machines")
if err := seedClient.Client().List(ctx, &machineList, client.InNamespace(t.SeedShootNamespace)); err != nil {
return nil, nil, err
}
log.Info("Found machines", "count", len(machineList.Items))
machineNames = make([]string, len(machineList.Items))
machineNodes = make([]string, len(machineList.Items))
for i, machine := range machineList.Items {
log.Info("Found machine", "index", i, "machineName", machine.GetName(), "nodeName", machine.GetLabels()["node"])
machineNames[i] = machine.GetName()
machineNodes[i] = machine.GetLabels()["node"]
}
slices.Sort(machineNames)
slices.Sort(machineNodes)
return
}
// GetPersistedSecrets uses the seedClient to fetch the data of all Secrets that have the `persist` label key set to true
// from the Shoot's control plane namespace
func (t *ShootMigrationTest) GetPersistedSecrets(ctx context.Context, seedClient kubernetes.Interface) (map[string]corev1.Secret, error) {
secretList := &corev1.SecretList{}
if err := seedClient.Client().List(
ctx,
secretList,
client.InNamespace(t.SeedShootNamespace),
client.MatchingLabels(map[string]string{secretsmanager.LabelKeyPersist: secretsmanager.LabelValueTrue}),
); err != nil {
return nil, err
}
secretsMap := make(map[string]corev1.Secret, len(secretList.Items))
for _, secret := range secretList.Items {
secretsMap[secret.Name] = secret
}
return secretsMap, nil
}
// PopulateBeforeMigrationComparisonElements fills the ShootMigrationTest.ComparisonElementsBeforeMigration with the necessary Machine details and Node names
func (t *ShootMigrationTest) populateBeforeMigrationComparisonElements(ctx context.Context) (err error) {
if !t.Config.SkipMachinesCheck {
t.ComparisonElementsBeforeMigration.MachineNames, t.ComparisonElementsBeforeMigration.MachineNodes, err = t.GetMachineDetails(ctx, t.SourceSeedClient)
if err != nil {
return
}
}
if !t.Config.SkipNodeCheck {
t.ComparisonElementsBeforeMigration.NodeNames, err = t.GetNodeNames(ctx, t.ShootClient)
if err != nil {
return
}
}
t.ComparisonElementsBeforeMigration.SecretsMap, err = t.GetPersistedSecrets(ctx, t.SourceSeedClient)
return
}
// PopulateAfterMigrationComparisonElements fills the ShootMigrationTest.ComparisonElementsAfterMigration with the necessary Machine details and Node names
func (t *ShootMigrationTest) populateAfterMigrationComparisonElements(ctx context.Context) (err error) {
if !t.Config.SkipMachinesCheck {
t.ComparisonElementsAfterMigration.MachineNames, t.ComparisonElementsAfterMigration.MachineNodes, err = t.GetMachineDetails(ctx, t.TargetSeedClient)
if err != nil {
return
}
}
if !t.Config.SkipNodeCheck {
t.ComparisonElementsAfterMigration.NodeNames, err = t.GetNodeNames(ctx, t.ShootClient)
if err != nil {
return
}
}
t.ComparisonElementsAfterMigration.SecretsMap, err = t.GetPersistedSecrets(ctx, t.TargetSeedClient)
return
}
// CompareElementsAfterMigration compares the Machine details, Node names and Pod statuses before and after migration and returns error if there are differences.
func (t *ShootMigrationTest) compareElementsAfterMigration() error {
if !t.Config.SkipMachinesCheck {
if !reflect.DeepEqual(t.ComparisonElementsBeforeMigration.MachineNames, t.ComparisonElementsAfterMigration.MachineNames) {
return fmt.Errorf("initial Machines %s, do not match after-migrate Machines %s", t.ComparisonElementsBeforeMigration.MachineNames, t.ComparisonElementsAfterMigration.MachineNames)
}
if !reflect.DeepEqual(t.ComparisonElementsBeforeMigration.MachineNodes, t.ComparisonElementsAfterMigration.MachineNodes) {
return fmt.Errorf("initial Machine Nodes (label) %s, do not match after-migrate Machine Nodes (label) %s", t.ComparisonElementsBeforeMigration.MachineNodes, t.ComparisonElementsAfterMigration.MachineNodes)
}
}
if t.Config.SkipNodeCheck {
if !reflect.DeepEqual(t.ComparisonElementsBeforeMigration.NodeNames, t.ComparisonElementsAfterMigration.NodeNames) {
return fmt.Errorf("initial Nodes %s, do not match after-migrate Nodes %s", t.ComparisonElementsBeforeMigration.NodeNames, t.ComparisonElementsAfterMigration.NodeNames)
}
if !reflect.DeepEqual(t.ComparisonElementsAfterMigration.MachineNodes, t.ComparisonElementsAfterMigration.NodeNames) {
return fmt.Errorf("machine Nodes (label) %s, do not match after-migrate Nodes %s", t.ComparisonElementsAfterMigration.MachineNodes, t.ComparisonElementsAfterMigration.NodeNames)
}
}
var errorMsg string
for name, secret := range t.ComparisonElementsBeforeMigration.SecretsMap {
if !reflect.DeepEqual(secret.Data, t.ComparisonElementsAfterMigration.SecretsMap[name].Data) {
errorMsg += fmt.Sprintf("Secret %s/%s did not have it's data persisted.\n", secret.Namespace, secret.Name)
}
if !reflect.DeepEqual(secret.Labels, t.ComparisonElementsAfterMigration.SecretsMap[name].Labels) {
errorMsg += fmt.Sprintf("Secret %s/%s did not have it's labels persisted: labels before migration: %v, labels after migration: %v\n",
secret.Namespace,
secret.Name,
secret.Labels,
t.ComparisonElementsAfterMigration.SecretsMap[name].Labels,
)
}
}
if len(errorMsg) > 0 {
return fmt.Errorf("control plane secrets did not have their data or labels persisted during control plane migration:\n %s", errorMsg)
}
return nil
}
// CheckObjectsTimestamp checks the timestamp of all objects that the resource-manager creates in the Shoot cluster.
// The timestamp should not be after ShootMigrationTest.MigrationTime.
func (t *ShootMigrationTest) CheckObjectsTimestamp(ctx context.Context, mrExcludeList, resourcesWithGeneratedName []string) error {
mrList := &resourcesv1alpha1.ManagedResourceList{}
if err := t.TargetSeedClient.Client().List(
ctx,
mrList,
client.InNamespace(t.SeedShootNamespace),
); err != nil {
return err
}
for _, mr := range mrList.Items {
if mr.Spec.Class == nil || *mr.Spec.Class != "seed" {
if !slices.Contains(mrExcludeList, mr.GetName()) {
log := t.GardenerFramework.Logger.WithValues("managedResource", client.ObjectKeyFromObject(&mr))
log.Info("Found ManagedResource")
for _, r := range mr.Status.Resources {
if len(r.Name) > 9 && slices.Contains(resourcesWithGeneratedName, r.Name[:len(r.Name)-9]) {
continue
}
obj := &unstructured.Unstructured{}
obj.SetAPIVersion(r.APIVersion)
obj.SetKind(r.Kind)
if err := t.ShootClient.Client().Get(ctx, client.ObjectKey{Namespace: r.Namespace, Name: r.Name}, obj); err != nil {
return err
}
// Ignore immutable objects because if their data changes, they will be recreated
if isImmutable, ok := obj.Object["immutable"]; ok && isImmutable == true {
continue
}
creationTimestamp := obj.GetCreationTimestamp()
objectLog := log.WithValues("objectKind", obj.GetKind(), "objectNamespace", obj.GetNamespace(), "objectName", obj.GetName(), "creationTimestamp", creationTimestamp)
objectLog.Info("Found object")
if t.MigrationTime.Before(&creationTimestamp) {
objectLog.Info("Object is created after shoot migration", "migrationTime", t.MigrationTime)
return fmt.Errorf("object: %s %s/%s Created At: %s is created after the Shoot migration %s", obj.GetKind(), obj.GetNamespace(), obj.GetName(), creationTimestamp, t.MigrationTime)
}
}
}
}
}
return nil
}
// CheckForOrphanedNonNamespacedResources checks if there are orphaned resources left on the target seed after the shoot migration.
// The function checks for Cluster, DNSOwner, BackupEntry, ClusterRoleBinding, ClusterRole and PersistentVolume
func (t *ShootMigrationTest) checkForOrphanedNonNamespacedResources(ctx context.Context) error {
seedClientScheme := t.SourceSeedClient.Client().Scheme()
if err := extensionsv1alpha1.AddToScheme(seedClientScheme); err != nil {
return err
}
leakedObjects := []string{}
for _, obj := range []client.ObjectList{
&extensionsv1alpha1.ClusterList{},
&extensionsv1alpha1.BackupEntryList{},
&rbacv1.ClusterRoleBindingList{},
&rbacv1.ClusterRoleList{},
} {
if err := t.SourceSeedClient.Client().List(ctx, obj, client.InNamespace(corev1.NamespaceAll)); err != nil {
return err
}
if err := meta.EachListItem(obj, func(object runtime.Object) error {
if strings.Contains(object.(client.Object).GetName(), t.SeedShootNamespace) {
leakedObjects = append(leakedObjects, fmt.Sprintf("%T %s", object, object.(client.Object).GetName()))
}
return nil
}); err != nil {
return err
}
}
pvList := &corev1.PersistentVolumeList{}
if err := t.SourceSeedClient.Client().List(ctx, pvList, client.InNamespace(corev1.NamespaceAll)); err != nil {
return err
}
if err := meta.EachListItem(pvList, func(obj runtime.Object) error {
pv := obj.(*corev1.PersistentVolume)
if strings.Contains(pv.Spec.ClaimRef.Namespace, t.SeedShootNamespace) {
leakedObjects = append(leakedObjects, fmt.Sprintf("PersistentVolume/%s", pv.GetName()))
}
return nil
}); err != nil {
return err
}
if len(leakedObjects) > 0 {
return fmt.Errorf("the following object(s) still exists in the source seed %v", leakedObjects)
}
return nil
}
// CreateSecretAndServiceAccount creates test secret and service account
func (t ShootMigrationTest) CreateSecretAndServiceAccount(ctx context.Context) error {
testSecret, testServiceAccount := constructTestSecretAndServiceAccount()
if err := t.ShootClient.Client().Create(ctx, testSecret); err != nil {
return err
}
return t.ShootClient.Client().Create(ctx, testServiceAccount)
}
// CheckSecretAndServiceAccount checks the test secret and service account exists in the shoot.
func (t ShootMigrationTest) CheckSecretAndServiceAccount(ctx context.Context) error {
testSecret, testServiceAccount := constructTestSecretAndServiceAccount()
if err := t.ShootClient.Client().Get(ctx, client.ObjectKeyFromObject(testSecret), testSecret); err != nil {
return err
}
return t.ShootClient.Client().Get(ctx, client.ObjectKeyFromObject(testServiceAccount), testServiceAccount)
}
// CleanUpSecretAndServiceAccount cleans up the test secret and service account
func (t ShootMigrationTest) CleanUpSecretAndServiceAccount(ctx context.Context) error {
testSecret, testServiceAccount := constructTestSecretAndServiceAccount()
return errors.Join(
t.ShootClient.Client().Delete(ctx, testSecret),
t.ShootClient.Client().Delete(ctx, testServiceAccount),
)
}
func constructTestSecretAndServiceAccount() (*corev1.Secret, *corev1.ServiceAccount) {
const (
secretName = "test-shoot-migration-secret"
secretNamespace = metav1.NamespaceDefault
serviceAccountName = "test-service-account"
serviceAccountNamespace = metav1.NamespaceDefault
)
testSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: secretNamespace,
},
}
testServiceAccount := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
Namespace: serviceAccountNamespace,
}}
return testSecret, testServiceAccount
}