-
Notifications
You must be signed in to change notification settings - Fork 462
/
upgrade.go
189 lines (174 loc) · 8.26 KB
/
upgrade.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors
//
// SPDX-License-Identifier: Apache-2.0
package highavailability
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/onsi/gomega/gstruct"
gomegatypes "github.com/onsi/gomega/types"
istionetworkingv1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper"
"github.com/gardener/gardener/pkg/client/kubernetes"
kubeapiserverconstants "github.com/gardener/gardener/pkg/component/kubernetes/apiserver/constants"
gardenerutils "github.com/gardener/gardener/pkg/utils/gardener"
. "github.com/gardener/gardener/pkg/utils/test"
"github.com/gardener/gardener/test/framework"
)
// UpgradeAndVerify runs the HA control-plane upgrade tests for an existing shoot cluster.
func UpgradeAndVerify(ctx context.Context, f *framework.ShootFramework, failureToleranceType gardencorev1beta1.FailureToleranceType) {
verifyEnvoyFilterInIstioNamespace(ctx, f.SeedClient, f.ShootSeedNamespace(), false)
By("Update Shoot control plane to HA with failure tolerance type " + string(failureToleranceType))
Expect(f.UpdateShoot(ctx, func(shoot *gardencorev1beta1.Shoot) error {
shoot.Spec.ControlPlane = &gardencorev1beta1.ControlPlane{
HighAvailability: &gardencorev1beta1.HighAvailability{
FailureTolerance: gardencorev1beta1.FailureTolerance{
Type: failureToleranceType,
},
},
}
return nil
})).To(Succeed())
By("Verify Shoot's control plane components")
verifyTopologySpreadConstraint(ctx, f.SeedClient, f.Shoot, f.ShootSeedNamespace())
verifyEtcdAffinity(ctx, f.SeedClient, f.Shoot, f.ShootSeedNamespace())
verifyEnvoyFilterInIstioNamespace(ctx, f.SeedClient, f.ShootSeedNamespace(), true)
}
func verifyTopologySpreadConstraint(ctx context.Context, seedClient kubernetes.Interface, shoot *gardencorev1beta1.Shoot, namespace string) {
for _, name := range []string{
v1beta1constants.DeploymentNameGardenerResourceManager,
} {
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
}
Expect(seedClient.Client().Get(ctx, client.ObjectKeyFromObject(deployment), deployment)).To(Succeed(), "trying to get deployment obj: "+deployment.Name+", but not succeeded.")
Expect(deployment.Spec.Template.Spec.TopologySpreadConstraints).To(getTSCMatcherForFailureToleranceType(shoot.Spec.ControlPlane.HighAvailability.FailureTolerance.Type), "for component "+deployment.Name)
}
}
func getTSCMatcherForFailureToleranceType(failureToleranceType gardencorev1beta1.FailureToleranceType) gomegatypes.GomegaMatcher {
var (
nodeSpread = MatchFields(IgnoreExtras, Fields{
"MaxSkew": Equal(int32(1)),
"TopologyKey": Equal(corev1.LabelHostname),
"WhenUnsatisfiable": Equal(corev1.DoNotSchedule),
})
zoneSpread = MatchFields(IgnoreExtras, Fields{
"MaxSkew": Equal(int32(1)),
"TopologyKey": Equal(corev1.LabelTopologyZone),
"WhenUnsatisfiable": Equal(corev1.DoNotSchedule),
})
)
switch failureToleranceType {
case gardencorev1beta1.FailureToleranceTypeNode:
return ConsistOf(nodeSpread)
case gardencorev1beta1.FailureToleranceTypeZone:
return ConsistOf(nodeSpread, zoneSpread)
default:
return BeNil()
}
}
func verifyEtcdAffinity(ctx context.Context, seedClient kubernetes.Interface, shoot *gardencorev1beta1.Shoot, namespace string) {
for _, componentName := range []string{
v1beta1constants.ETCDRoleEvents,
v1beta1constants.ETCDRoleMain,
} {
numberOfZones := 1
if v1beta1helper.IsMultiZonalShootControlPlane(shoot) {
numberOfZones = 3
}
sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{
Name: "etcd-" + componentName,
Namespace: namespace,
}}
Expect(seedClient.Client().Get(ctx, client.ObjectKeyFromObject(sts), sts)).To(Succeed(), "get StatefulSet "+sts.Name)
Expect(sts.Spec.Template.Spec.Affinity).NotTo(BeNil(), "for component "+sts.Name)
Expect(sts.Spec.Template.Spec.Affinity.NodeAffinity).NotTo(BeNil(), "for component "+sts.Name)
Expect(sts.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).NotTo(BeNil(), "for component "+sts.Name)
Expect(sts.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms).To(HaveLen(1), "for component "+sts.Name)
Expect(sts.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions).To(HaveLen(1), "for component "+sts.Name)
Expect(sts.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0]).To(MatchFields(IgnoreExtras, Fields{
"Key": Equal(corev1.LabelTopologyZone),
"Operator": Equal(corev1.NodeSelectorOpIn),
"Values": HaveLen(numberOfZones),
}), "for component "+sts.Name)
Expect(sts.Spec.Template.Spec.Affinity.PodAntiAffinity).To(BeNil(), "for component "+sts.Name)
}
}
// DeployZeroDownTimeValidatorJob deploys a Job into the cluster which ensures
// zero downtime by continuously checking the kube-apiserver's health.
// This job fails once a health check fails. Its associated pod results in error status.
func DeployZeroDownTimeValidatorJob(ctx context.Context, c client.Client, testName, namespace, token string) (*batchv1.Job, error) {
job := batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: "zero-down-time-validator-" + testName,
Namespace: namespace,
},
Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Labels: map[string]string{
v1beta1constants.LabelNetworkPolicyToDNS: v1beta1constants.LabelNetworkPolicyAllowed,
gardenerutils.NetworkPolicyLabel(v1beta1constants.DeploymentNameKubeAPIServer, kubeapiserverconstants.Port): v1beta1constants.LabelNetworkPolicyAllowed,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "validator",
Image: "alpine/curl",
Command: []string{"/bin/sh", "-ec",
// To avoid flakiness, consider downtime when curl fails consecutively back-to-back three times.
"failed=0; threshold=3; " +
"while [ $failed -lt $threshold ]; do " +
"if curl -m 2 -k https://kube-apiserver/healthz -H 'Authorization: " + token + "' -s -f -o /dev/null ; then " +
"echo $(date +'%Y-%m-%dT%H:%M:%S.%3N%z') INFO: kube-apiserver is healthy.; failed=0; " +
"else failed=$((failed+1)); " +
"echo $(date +'%Y-%m-%dT%H:%M:%S.%3N%z') ERROR: kube-apiserver is unhealthy and retrying.; " +
"fi; " +
"sleep 10; " +
"done; " +
"echo $(date +'%Y-%m-%dT%H:%M:%S.%3N%z') ERROR: kube-apiserver is still unhealthy after $failed attempts. Considered as downtime.; " +
"exit 1; "},
},
},
RestartPolicy: corev1.RestartPolicyNever,
},
},
BackoffLimit: ptr.To[int32](0),
},
}
return &job, c.Create(ctx, &job)
}
func verifyEnvoyFilterInIstioNamespace(ctx context.Context, seedClient kubernetes.Interface, shootName string, checkLabels bool) {
var filteredList []*istionetworkingv1alpha3.EnvoyFilter
CEventually(ctx, func(g Gomega) {
envoyFilters := &istionetworkingv1alpha3.EnvoyFilterList{}
g.Expect(seedClient.Client().List(ctx, envoyFilters)).To(Succeed(), "trying to list envoy filters, but did not succeed.")
filteredList = []*istionetworkingv1alpha3.EnvoyFilter{}
for _, filter := range envoyFilters.Items {
if filter.Name != shootName {
continue
}
// Old Gardener releases do not manage the envoy filter and hence we may end up with one managed and one unmanaged filter
if checkLabels && filter.Labels["resources.gardener.cloud/managed-by"] != "gardener" {
continue
}
filteredList = append(filteredList, filter)
}
g.Expect(filteredList).To(HaveLen(1))
}).Should(Succeed())
Expect(filteredList[0].Namespace).To(HavePrefix("istio-ingress"))
}