-
Notifications
You must be signed in to change notification settings - Fork 2.7k
/
resources.go
414 lines (356 loc) · 14 KB
/
resources.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package multus
import (
"context"
"fmt"
"time"
core "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
)
type podNetworkInfo struct {
// node the pod is running on
nodeName string
// multus public addr attached (if any)
publicAddr string
// multus cluster addr attached (if any)
clusterAddr string
}
var manualCleanupInstructions = fmt.Sprintf(
"manually delete owner configmap %q, and wait for all multus-validation-test resources to be deleted", ownerConfigMapName)
var previousTestSuggestion = "there could be a past test preventing this one from proceeding; " + manualCleanupInstructions
var unableToProvideAddressSuggestions = []string{
"multus may be unable to provide addresses for pods",
"check networking events on the pod and multus logs",
"macvlan: NIC or switch hardware/software may block the association of some number of additional MAC addresses on an interface",
"macvlan: interfaces and network switching must enable promiscuous mode to allow receiving packets for unknown (Multus) MACs",
"macvlan/ipvlan: switch hardware/software may block an interface from receiving packets to an unknown (Multus) IP",
}
// create a validation test config object that stores the configuration of the running validation
// test. this object serves as the owner of all associated test objects. when this object is
// deleted, all validation test objects should also be deleted, effectively cleaning up all
// components of this test.
func (vt *ValidationTest) createOwningConfigMap(ctx context.Context) ([]meta.OwnerReference, error) {
c := core.ConfigMap{
ObjectMeta: meta.ObjectMeta{
Name: ownerConfigMapName,
},
}
configObject, err := vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Create(ctx, &c, meta.CreateOptions{})
if err != nil {
return nil, fmt.Errorf("failed to create validation test config object [%+v]: %w", c, err)
}
// for cleanup, we want to make sure all children are deleted
BlockOwnerDeletion := true
refToConfigObject := meta.OwnerReference{
APIVersion: "v1",
Kind: "ConfigMap",
Name: configObject.GetName(),
UID: configObject.GetUID(),
BlockOwnerDeletion: &BlockOwnerDeletion,
}
return []meta.OwnerReference{refToConfigObject}, nil
}
func (vt *ValidationTest) startWebServer(ctx context.Context, owners []meta.OwnerReference) error {
placement, err := vt.BestNodePlacementForServer()
if err != nil {
return fmt.Errorf("failed to place web server pod: %w", err)
}
// infer good placement for web server pod from the node type with the most OSDs
pod, err := vt.generateWebServerPod(placement)
if err != nil {
return fmt.Errorf("failed to generate web server pod: %w", err)
}
pod.SetOwnerReferences(owners) // set owner refs so cleanup is easier
configMap, err := vt.generateWebServerConfigMap()
if err != nil {
return fmt.Errorf("failed to generate web server config: %w", err)
}
configMap.SetOwnerReferences(owners) // set owner refs so cleanup is easier
// create configmap before pod so pod doesn't crashloopbackoff on first creation
_, err = vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Create(ctx, configMap, meta.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create web server config: %w", err)
}
_, err = vt.Clientset.CoreV1().Pods(vt.Namespace).Create(ctx, pod, meta.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create web server pod: %w", err)
}
return nil
}
func (vt *ValidationTest) getWebServerInfo(
ctx context.Context,
desiredPublicNet, desiredClusterNet *types.NamespacedName,
) (podNetworkInfo, []string, error) {
podInfo := podNetworkInfo{}
pod, err := vt.Clientset.CoreV1().Pods(vt.Namespace).Get(ctx, webServerPodName(), meta.GetOptions{})
if err != nil {
return podInfo, []string{}, fmt.Errorf("unexpected error when getting web server pod: %w", err)
}
var publicAddr, clusterAddr string
publicAddr, clusterAddr, networkSuggestions, err := getNetworksFromPod(pod, desiredPublicNet, desiredClusterNet)
if err != nil {
return podInfo, networkSuggestions, fmt.Errorf("no web server network info: %w", err)
}
if !podIsReady(*pod) {
return podInfo, []string{}, fmt.Errorf("web server pod is not ready yet")
}
podInfo.nodeName = pod.Spec.NodeName
podInfo.publicAddr = publicAddr
podInfo.clusterAddr = clusterAddr
return podInfo, []string{}, nil // no suggestions if successful
}
func (vt *ValidationTest) startImagePullers(ctx context.Context, owners []meta.OwnerReference) error {
for typeName, nodeType := range vt.NodeTypes {
ds, err := vt.generateImagePullDaemonSet(typeName, nodeType.Placement)
if err != nil {
return fmt.Errorf("failed to generate image pull daemonset: %w", err)
}
ds.SetOwnerReferences(owners) // set owner so cleanup is easier
_, err = vt.Clientset.AppsV1().DaemonSets(vt.Namespace).Create(ctx, ds, meta.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create image pull daemonset: %w", err)
}
}
return nil
}
func (vt *ValidationTest) deleteImagePullers(ctx context.Context) error {
noGracePeriod := int64(0)
delOpts := meta.DeleteOptions{
GracePeriodSeconds: &noGracePeriod,
}
listOpts := meta.ListOptions{
LabelSelector: imagePullAppLabel(),
}
err := vt.Clientset.AppsV1().DaemonSets(vt.Namespace).DeleteCollection(ctx, delOpts, listOpts)
if err != nil {
if kerrors.IsNotFound(err) {
return nil // already deleted
}
return fmt.Errorf("failed to delete image pullers: %w", err)
}
return nil
}
func (vt *ValidationTest) startClients(
ctx context.Context,
owners []meta.OwnerReference,
serverPublicAddr, serverClusterAddr string,
nodeType string,
) (int, error) {
numDaemonsetsCreated := 0
nodeConfig := vt.NodeTypes[nodeType]
// start clients that simulate OSDs (connected to both public and cluster nets)
osdsPerNode := nodeConfig.OSDsPerNode
vt.Logger.Infof("starting %d %s validation clients for node type %q", osdsPerNode, ClientTypeOSD, nodeType)
for i := 0; i < osdsPerNode; i++ {
attachToClusterNet := true
ds, err := vt.generateClientDaemonSet(true, attachToClusterNet, serverPublicAddr, serverClusterAddr, nodeType, ClientTypeOSD, i, nodeConfig.Placement)
if err != nil {
return numDaemonsetsCreated, fmt.Errorf("failed to generate client daemonset for node type %q, client type %q, client #%d: %w", nodeType, ClientTypeOSD, i, err)
}
ds.SetOwnerReferences(owners) // set owner refs so cleanup is easier
_, err = vt.Clientset.AppsV1().DaemonSets(vt.Namespace).Create(ctx, ds, meta.CreateOptions{})
if err != nil {
return numDaemonsetsCreated, fmt.Errorf("failed to create client daemonset for node type %q, client type %q, client #%d: %w", nodeType, ClientTypeOSD, i, err)
}
numDaemonsetsCreated++
}
// start clients that simulate non-OSD daemons (connected only to public net)
if serverPublicAddr == "" {
return numDaemonsetsCreated, nil // no public net; thus, no public-net-only clients to run
}
otherPerNode := nodeConfig.OtherDaemonsPerNode
vt.Logger.Infof("starting %d %s (non-OSD) validation clients for node type %q", otherPerNode, ClientTypeNonOSD, nodeType)
for i := 0; i < otherPerNode; i++ {
attachToClusterNet := false
ds, err := vt.generateClientDaemonSet(true, attachToClusterNet, serverPublicAddr, serverClusterAddr, nodeType, ClientTypeNonOSD, i, nodeConfig.Placement)
if err != nil {
return numDaemonsetsCreated, fmt.Errorf("failed to generate client daemonset for node type %q, client type %q, client #%d: %w", nodeType, ClientTypeNonOSD, i, err)
}
ds.SetOwnerReferences(owners) // set owner refs so cleanup is easier
_, err = vt.Clientset.AppsV1().DaemonSets(vt.Namespace).Create(ctx, ds, meta.CreateOptions{})
if err != nil {
return numDaemonsetsCreated, fmt.Errorf("failed to create client daemonset for node type %q, client type %q, client #%d: %w", nodeType, ClientTypeNonOSD, i, err)
}
numDaemonsetsCreated++
}
return numDaemonsetsCreated, nil
}
type perNodeTypeCount map[string]int
func (a *perNodeTypeCount) Increment(nodeType string) {
current, ok := (*a)[nodeType]
if !ok {
current = 0
}
(*a)[nodeType] = current + 1
}
func (a *perNodeTypeCount) Total() int {
t := 0
for _, c := range *a {
t += c
}
return t
}
func (a *perNodeTypeCount) Equal(b *perNodeTypeCount) bool {
if len(*a) != len(*b) {
return false
}
for nodeType, numA := range *a {
numB, ok := (*b)[nodeType]
if !ok {
return false
}
if numA != numB {
return false
}
}
return true
}
func (vt *ValidationTest) getImagePullPodCountPerNodeType(
ctx context.Context,
) (perNodeTypeCount, error) {
emptyCount := perNodeTypeCount{}
listOpts := meta.ListOptions{
LabelSelector: imagePullAppLabel(),
}
dsets, err := vt.Clientset.AppsV1().DaemonSets(vt.Namespace).List(ctx, listOpts)
if err != nil {
return emptyCount, fmt.Errorf("unexpected error listing daemonsets: %w", err)
}
expectedNumDaemonsets := len(vt.NodeTypes)
if len(dsets.Items) != expectedNumDaemonsets {
return emptyCount, fmt.Errorf("got %d daemonsets when %d should exist", len(dsets.Items), expectedNumDaemonsets)
}
numsScheduled := perNodeTypeCount{}
for i, d := range dsets.Items {
nodeType := getNodeType(&dsets.Items[i].ObjectMeta)
numScheduled := d.Status.CurrentNumberScheduled
if numScheduled == 0 {
return emptyCount, fmt.Errorf("image pull daemonset for node type %q expects zero scheduled pods", nodeType)
}
numsScheduled[nodeType] = int(numScheduled)
}
return numsScheduled, nil
}
func (vt *ValidationTest) ensureOneImagePullPodPerNode(ctx context.Context) error {
listOpts := meta.ListOptions{
LabelSelector: imagePullAppLabel(),
}
pods, err := vt.Clientset.CoreV1().Pods(vt.Namespace).List(ctx, listOpts)
if err != nil {
return fmt.Errorf("failed to list pods: %w", err)
}
nodesFound := map[string]string{}
for _, p := range pods.Items {
nodeName := p.Spec.NodeName
nodeType := p.GetLabels()["nodeType"]
if otherNodeType, ok := nodesFound[nodeName]; ok {
return fmt.Errorf("node types must not overlap: node type %q has overlap with node type %q", nodeType, otherNodeType)
}
nodesFound[nodeName] = nodeType
}
return nil
}
func (vt *ValidationTest) getNumRunningPods(
ctx context.Context,
podSelectorLabel string,
) (int, error) {
listOpts := meta.ListOptions{
LabelSelector: podSelectorLabel,
}
pods, err := vt.Clientset.CoreV1().Pods(vt.Namespace).List(ctx, listOpts)
if err != nil {
return 0, fmt.Errorf("failed to list pods: %w", err)
}
numRunning := 0
for _, p := range pods.Items {
if podIsRunning(p) {
numRunning++
}
}
return numRunning, nil
}
func (vt *ValidationTest) numClientsReady(ctx context.Context, expectedNumPods int) (int, error) {
pods, err := vt.getClientPods(ctx, expectedNumPods)
if err != nil {
return 0, fmt.Errorf("unexpected error getting client pods: %w", err)
}
numReady := 0
for _, p := range pods.Items {
if podIsReady(p) {
numReady++
}
}
return numReady, nil
}
func (vt *ValidationTest) getClientPods(ctx context.Context, expectedNumPods int) (*core.PodList, error) {
listOpts := meta.ListOptions{
LabelSelector: clientAppLabel(),
}
pods, err := vt.Clientset.CoreV1().Pods(vt.Namespace).List(ctx, listOpts)
if err != nil {
return nil, fmt.Errorf("failed to list client pods: %w", err)
}
if len(pods.Items) != expectedNumPods {
return nil, fmt.Errorf("the number of pods listed [%d] does not match the number expected [%d]", len(pods.Items), expectedNumPods)
}
return pods, err
}
func (vt *ValidationTest) cleanUpTestResources() (string, error) {
// need a clean, non-canceled context in case the test is canceled by ctrl-c
ctx := context.Background()
// delete the config object in the foreground so we wait until all validation test resources are
// gone before stopping, and do it now because there's no need to wait for just a test
var gracePeriodZero int64 = 0
deleteForeground := meta.DeletePropagationForeground
delOpts := meta.DeleteOptions{
PropagationPolicy: &deleteForeground,
GracePeriodSeconds: &gracePeriodZero,
}
err := vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Delete(ctx, ownerConfigMapName, delOpts)
if err != nil {
if !kerrors.IsNotFound(err) {
return manualCleanupInstructions, fmt.Errorf("failed to clean up multus validation test resources: %w", err)
}
return "", nil
}
// clients take a long time to terminate, and the 0 grace period set on the configmap doesn't
// propagate to dependents. make a best-effort attempt to delete client pods with 0 grace period
listOpts := meta.ListOptions{
LabelSelector: clientAppLabel(),
}
// ignore errors for a best-effort attempt, they will delete eventually
_ = vt.Clientset.CoreV1().Pods(vt.Namespace).DeleteCollection(ctx, delOpts, listOpts)
// wait for resources to be cleaned up
ctx, cancel := context.WithTimeout(ctx, vt.ResourceTimeout)
defer cancel()
lastSuggestion := ""
err = wait.PollUntilContextCancel(ctx, 2*time.Second, true, func(ctx context.Context) (done bool, err error) {
_, getErr := vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Get(ctx, ownerConfigMapName, meta.GetOptions{})
if getErr != nil {
if kerrors.IsNotFound(getErr) {
return true, nil
}
lastSuggestion = fmt.Sprintf("unexpected error when cleaning up multus validation test resources; attempting to continue: %v", err)
}
return false, nil
})
if err != nil {
return lastSuggestion + "; " + manualCleanupInstructions,
fmt.Errorf("failed waiting for multus validation test resources to be deleted: %w", err)
}
return "", nil
}