/
vsphere_zone_support.go
529 lines (458 loc) · 29.9 KB
/
vsphere_zone_support.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package vsphere
import (
"context"
"fmt"
"strings"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
volumeevents "k8s.io/kubernetes/pkg/controller/volume/events"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
"k8s.io/kubernetes/test/e2e/storage/utils"
admissionapi "k8s.io/pod-security-admission/api"
)
/*
Test to verify multi-zone support for dynamic volume provisioning in kubernetes.
The test environment is illustrated below:
datacenter-1
--->cluster-vsan-1 (zone-a) ____________________ _________________
--->host-1 : master | | | |
--->host-2 : node1 ___________________ | | | |
--->host-3 (zone-c): node2 | || vsanDatastore | | |
| localDatastore || | | |
|___________________||____________________| | sharedVmfs-0 |
--->cluster-vsan-2 (zone-b) ____________________ | |
--->host-4 : node3 | | | |
--->host-5 : node4 | vsanDatastore (1) | | |
--->host-6 | | | |
|____________________| |_________________|
--->cluster-3 (zone-c) ___________________
--->host-7 : node5 | |
| localDatastore (1)|
|___________________|
datacenter-2
--->cluster-1 (zone-d) ___________________
--->host-8 : node6 | |
| localDatastore |
|___________________|
Testbed description :
1. cluster-vsan-1 is tagged with zone-a. So, vsanDatastore inherits zone-a since all the hosts under zone-a have vsanDatastore mounted on them.
2. cluster-vsan-2 is tagged with zone-b. So, vsanDatastore (1) inherits zone-b since all the hosts under zone-b have vsanDatastore (1) mounted on them.
3. sharedVmfs-0 inherits both zone-a and zone-b since all the hosts in both zone-a and zone-b have this datastore mounted on them.
4. cluster-3 is tagged with zone-c. cluster-3 only contains host-7.
5. host-3 under cluster-vsan-1 is tagged with zone-c.
6. Since there are no shared datastores between host-7 under cluster-3 and host-3 under cluster-vsan-1, no datastores in the environment inherit zone-c.
7. host-8 under datacenter-2 and cluster-1 is tagged with zone-d. So, localDatastore attached to host-8 inherits zone-d.
8. The six worker nodes are distributed among the hosts as shown in the above illustration.
9. Two storage policies are created on VC. One is a VSAN storage policy named as compatpolicy with hostFailuresToTolerate capability set to 1.
Testsuite description :
1. Tests to verify that zone labels are set correctly on a dynamically created PV.
2. Tests to verify dynamic pv creation fails if availability zones are not specified or if there are no shared datastores under the specified zones.
3. Tests to verify dynamic pv creation using availability zones works in combination with other storage class parameters such as storage policy,
datastore and VSAN capabilities.
4. Tests to verify dynamic pv creation using availability zones fails in combination with other storage class parameters such as storage policy,
datastore and VSAN capabilities specifications when any of the former mentioned parameters are incompatible with the rest.
5. Tests to verify dynamic pv creation using availability zones work across different datacenters in the same VC.
*/
var _ = utils.SIGDescribe("Zone Support [Feature:vsphere]", func() {
f := framework.NewDefaultFramework("zone-support")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
var (
client clientset.Interface
namespace string
scParameters map[string]string
zones []string
vsanDatastore1 string
vsanDatastore2 string
localDatastore string
compatPolicy string
nonCompatPolicy string
zoneA string
zoneB string
zoneC string
zoneD string
invalidZone string
)
ginkgo.BeforeEach(func(ctx context.Context) {
e2eskipper.SkipUnlessProviderIs("vsphere")
Bootstrap(f)
client = f.ClientSet
e2eskipper.SkipUnlessMultizone(ctx, client)
namespace = f.Namespace.Name
vsanDatastore1 = GetAndExpectStringEnvVar(VCPZoneVsanDatastore1)
vsanDatastore2 = GetAndExpectStringEnvVar(VCPZoneVsanDatastore2)
localDatastore = GetAndExpectStringEnvVar(VCPZoneLocalDatastore)
compatPolicy = GetAndExpectStringEnvVar(VCPZoneCompatPolicyName)
nonCompatPolicy = GetAndExpectStringEnvVar(VCPZoneNonCompatPolicyName)
zoneA = GetAndExpectStringEnvVar(VCPZoneA)
zoneB = GetAndExpectStringEnvVar(VCPZoneB)
zoneC = GetAndExpectStringEnvVar(VCPZoneC)
zoneD = GetAndExpectStringEnvVar(VCPZoneD)
invalidZone = GetAndExpectStringEnvVar(VCPInvalidZone)
scParameters = make(map[string]string)
zones = make([]string, 0)
_, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet)
framework.ExpectNoError(err)
})
ginkgo.It("Verify dynamically created pv with allowed zones specified in storage class, shows the right zone information on its labels", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with the following zones : %s", zoneA))
zones = append(zones, zoneA)
verifyPVZoneLabels(ctx, client, f.Timeouts, namespace, nil, zones)
})
ginkgo.It("Verify dynamically created pv with multiple zones specified in the storage class, shows both the zones on its labels", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with the following zones : %s, %s", zoneA, zoneB))
zones = append(zones, zoneA)
zones = append(zones, zoneB)
verifyPVZoneLabels(ctx, client, f.Timeouts, namespace, nil, zones)
})
ginkgo.It("Verify PVC creation with invalid zone specified in storage class fails", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with unknown zone : %s", invalidZone))
zones = append(zones, invalidZone)
err := verifyPVCCreationFails(ctx, client, namespace, nil, zones, "")
framework.ExpectError(err)
errorMsg := "Failed to find a shared datastore matching zone [" + invalidZone + "]"
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV, based on allowed zones specified in storage class ", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zones :%s", zoneA))
zones = append(zones, zoneA)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, nil, zones, "")
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV, based on multiple zones specified in storage class ", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zones :%s, %s", zoneA, zoneB))
zones = append(zones, zoneA)
zones = append(zones, zoneB)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, nil, zones, "")
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV, based on the allowed zones and datastore specified in storage class", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s and datastore :%s", zoneA, vsanDatastore1))
scParameters[Datastore] = vsanDatastore1
zones = append(zones, zoneA)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, scParameters, zones, "")
})
ginkgo.It("Verify PVC creation with incompatible datastore and zone combination specified in storage class fails", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s and datastore :%s", zoneC, vsanDatastore1))
scParameters[Datastore] = vsanDatastore1
zones = append(zones, zoneC)
err := verifyPVCCreationFails(ctx, client, namespace, scParameters, zones, "")
errorMsg := "No matching datastores found in the kubernetes cluster for zone " + zoneC
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV, based on the allowed zones and storage policy specified in storage class", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s and storage policy :%s", zoneA, compatPolicy))
scParameters[SpbmStoragePolicy] = compatPolicy
zones = append(zones, zoneA)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, scParameters, zones, "")
})
ginkgo.It("Verify a pod is created on a non-Workspace zone and attached to a dynamically created PV, based on the allowed zones and storage policy specified in storage class", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s and storage policy :%s", zoneB, compatPolicy))
scParameters[SpbmStoragePolicy] = compatPolicy
zones = append(zones, zoneB)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, scParameters, zones, "")
})
ginkgo.It("Verify PVC creation with incompatible storagePolicy and zone combination specified in storage class fails", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s and storage policy :%s", zoneA, nonCompatPolicy))
scParameters[SpbmStoragePolicy] = nonCompatPolicy
zones = append(zones, zoneA)
err := verifyPVCCreationFails(ctx, client, namespace, scParameters, zones, "")
errorMsg := "No compatible datastores found that satisfy the storage policy requirements"
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV, based on the allowed zones, datastore and storage policy specified in storage class", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s datastore :%s and storagePolicy :%s", zoneA, vsanDatastore1, compatPolicy))
scParameters[SpbmStoragePolicy] = compatPolicy
scParameters[Datastore] = vsanDatastore1
zones = append(zones, zoneA)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, scParameters, zones, "")
})
ginkgo.It("Verify PVC creation with incompatible storage policy along with compatible zone and datastore combination specified in storage class fails", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s datastore :%s and storagePolicy :%s", zoneA, vsanDatastore1, nonCompatPolicy))
scParameters[SpbmStoragePolicy] = nonCompatPolicy
scParameters[Datastore] = vsanDatastore1
zones = append(zones, zoneA)
err := verifyPVCCreationFails(ctx, client, namespace, scParameters, zones, "")
errorMsg := "User specified datastore is not compatible with the storagePolicy: \\\"" + nonCompatPolicy + "\\\"."
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify PVC creation with incompatible zone along with compatible storagePolicy and datastore combination specified in storage class fails", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s datastore :%s and storagePolicy :%s", zoneC, vsanDatastore2, compatPolicy))
scParameters[SpbmStoragePolicy] = compatPolicy
scParameters[Datastore] = vsanDatastore2
zones = append(zones, zoneC)
err := verifyPVCCreationFails(ctx, client, namespace, scParameters, zones, "")
errorMsg := "No matching datastores found in the kubernetes cluster for zone " + zoneC
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify PVC creation fails if no zones are specified in the storage class (No shared datastores exist among all the nodes)", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with no zones"))
err := verifyPVCCreationFails(ctx, client, namespace, nil, nil, "")
errorMsg := "No shared datastores found in the Kubernetes cluster"
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify PVC creation fails if only datastore is specified in the storage class (No shared datastores exist among all the nodes)", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with datastore :%s", vsanDatastore1))
scParameters[Datastore] = vsanDatastore1
err := verifyPVCCreationFails(ctx, client, namespace, scParameters, nil, "")
errorMsg := "No shared datastores found in the Kubernetes cluster"
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify PVC creation fails if only storage policy is specified in the storage class (No shared datastores exist among all the nodes)", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with storage policy :%s", compatPolicy))
scParameters[SpbmStoragePolicy] = compatPolicy
err := verifyPVCCreationFails(ctx, client, namespace, scParameters, nil, "")
errorMsg := "No shared datastores found in the Kubernetes cluster"
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify PVC creation with compatible policy and datastore without any zones specified in the storage class fails (No shared datastores exist among all the nodes)", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with storage policy :%s and datastore :%s", compatPolicy, vsanDatastore1))
scParameters[SpbmStoragePolicy] = compatPolicy
scParameters[Datastore] = vsanDatastore1
err := verifyPVCCreationFails(ctx, client, namespace, scParameters, nil, "")
errorMsg := "No shared datastores found in the Kubernetes cluster"
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify PVC creation fails if the availability zone specified in the storage class have no shared datastores under it.", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s", zoneC))
zones = append(zones, zoneC)
err := verifyPVCCreationFails(ctx, client, namespace, nil, zones, "")
errorMsg := "No matching datastores found in the kubernetes cluster for zone " + zoneC
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV, based on multiple zones specified in the storage class. (No shared datastores exist among both zones)", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with the following zones :%s and %s", zoneA, zoneC))
zones = append(zones, zoneA)
zones = append(zones, zoneC)
err := verifyPVCCreationFails(ctx, client, namespace, nil, zones, "")
errorMsg := "No matching datastores found in the kubernetes cluster for zone " + zoneC
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify PVC creation with an invalid VSAN capability along with a compatible zone combination specified in storage class fails", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with %s :%s and zone :%s", PolicyHostFailuresToTolerate, HostFailuresToTolerateCapabilityInvalidVal, zoneA))
scParameters[PolicyHostFailuresToTolerate] = HostFailuresToTolerateCapabilityInvalidVal
zones = append(zones, zoneA)
err := verifyPVCCreationFails(ctx, client, namespace, scParameters, zones, "")
errorMsg := "Invalid value for " + PolicyHostFailuresToTolerate + "."
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV, based on a VSAN capability, datastore and compatible zone specified in storage class", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with %s :%s, %s :%s, datastore :%s and zone :%s", PolicyObjectSpaceReservation, ObjectSpaceReservationCapabilityVal, PolicyIopsLimit, IopsLimitCapabilityVal, vsanDatastore1, zoneA))
scParameters[PolicyObjectSpaceReservation] = ObjectSpaceReservationCapabilityVal
scParameters[PolicyIopsLimit] = IopsLimitCapabilityVal
scParameters[Datastore] = vsanDatastore1
zones = append(zones, zoneA)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, scParameters, zones, "")
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV, based on the allowed zones specified in storage class when the datastore under the zone is present in another datacenter", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s", zoneD))
zones = append(zones, zoneD)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, scParameters, zones, "")
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV, based on the allowed zones and datastore specified in storage class when there are multiple datastores with the same name under different zones across datacenters", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with zone :%s and datastore name :%s", zoneD, localDatastore))
scParameters[Datastore] = localDatastore
zones = append(zones, zoneD)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, scParameters, zones, "")
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV with storage policy specified in storage class in waitForFirstConsumer binding mode", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with waitForFirstConsumer mode and storage policy :%s", compatPolicy))
scParameters[SpbmStoragePolicy] = compatPolicy
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, scParameters, nil, storagev1.VolumeBindingWaitForFirstConsumer)
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV with storage policy specified in storage class in waitForFirstConsumer binding mode with allowedTopologies", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with waitForFirstConsumer mode, storage policy :%s and zone :%s", compatPolicy, zoneA))
scParameters[SpbmStoragePolicy] = compatPolicy
zones = append(zones, zoneA)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, scParameters, zones, storagev1.VolumeBindingWaitForFirstConsumer)
})
ginkgo.It("Verify a pod is created and attached to a dynamically created PV with storage policy specified in storage class in waitForFirstConsumer binding mode with multiple allowedTopologies", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with waitForFirstConsumer mode and zones : %s, %s", zoneA, zoneB))
zones = append(zones, zoneA)
zones = append(zones, zoneB)
verifyPVCAndPodCreationSucceeds(ctx, client, f.Timeouts, namespace, nil, zones, storagev1.VolumeBindingWaitForFirstConsumer)
})
ginkgo.It("Verify a PVC creation fails when multiple zones are specified in the storage class without shared datastores among the zones in waitForFirstConsumer binding mode", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with waitForFirstConsumer mode and following zones :%s and %s", zoneA, zoneC))
zones = append(zones, zoneA)
zones = append(zones, zoneC)
err := verifyPodAndPvcCreationFailureOnWaitForFirstConsumerMode(ctx, client, namespace, nil, zones)
framework.ExpectError(err)
errorMsg := "No matching datastores found in the kubernetes cluster for zone " + zoneC
if !strings.Contains(err.Error(), errorMsg) {
framework.ExpectNoError(err, errorMsg)
}
})
ginkgo.It("Verify a pod fails to get scheduled when conflicting volume topology (allowedTopologies) and pod scheduling constraints(nodeSelector) are specified", func(ctx context.Context) {
ginkgo.By(fmt.Sprintf("Creating storage class with waitForFirstConsumerMode, storage policy :%s and zone :%s", compatPolicy, zoneA))
scParameters[SpbmStoragePolicy] = compatPolicy
// allowedTopologies set as zoneA
zones = append(zones, zoneA)
nodeSelectorMap := map[string]string{
// nodeSelector set as zoneB
v1.LabelTopologyZone: zoneB,
}
verifyPodSchedulingFails(ctx, client, namespace, nodeSelectorMap, scParameters, zones, storagev1.VolumeBindingWaitForFirstConsumer)
})
})
func verifyPVCAndPodCreationSucceeds(ctx context.Context, client clientset.Interface, timeouts *framework.TimeoutContext, namespace string, scParameters map[string]string, zones []string, volumeBindingMode storagev1.VolumeBindingMode) {
storageclass, err := client.StorageV1().StorageClasses().Create(ctx, getVSphereStorageClassSpec("zone-sc", scParameters, zones, volumeBindingMode), metav1.CreateOptions{})
framework.ExpectNoError(err, fmt.Sprintf("Failed to create storage class with err: %v", err))
ginkgo.DeferCleanup(client.StorageV1().StorageClasses().Delete, storageclass.Name, metav1.DeleteOptions{})
ginkgo.By("Creating PVC using the Storage Class")
pvclaim, err := e2epv.CreatePVC(ctx, client, namespace, getVSphereClaimSpecWithStorageClass(namespace, "2Gi", storageclass))
framework.ExpectNoError(err)
ginkgo.DeferCleanup(e2epv.DeletePersistentVolumeClaim, client, pvclaim.Name, namespace)
var pvclaims []*v1.PersistentVolumeClaim
pvclaims = append(pvclaims, pvclaim)
var persistentvolumes []*v1.PersistentVolume
// If WaitForFirstConsumer mode, verify pvc binding status after pod creation. For immediate mode, do now.
if volumeBindingMode != storagev1.VolumeBindingWaitForFirstConsumer {
persistentvolumes = waitForPVClaimBoundPhase(ctx, client, pvclaims, timeouts.ClaimProvision)
}
ginkgo.By("Creating pod to attach PV to the node")
pod, err := e2epod.CreatePod(ctx, client, namespace, nil, pvclaims, false, "")
framework.ExpectNoError(err)
if volumeBindingMode == storagev1.VolumeBindingWaitForFirstConsumer {
persistentvolumes = waitForPVClaimBoundPhase(ctx, client, pvclaims, timeouts.ClaimProvision)
}
if zones != nil {
ginkgo.By("Verify persistent volume was created on the right zone")
verifyVolumeCreationOnRightZone(ctx, persistentvolumes, pod.Spec.NodeName, zones)
}
ginkgo.By("Verify the volume is accessible and available in the pod")
verifyVSphereVolumesAccessible(ctx, client, pod, persistentvolumes)
ginkgo.By("Deleting pod")
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, client, pod))
ginkgo.By("Waiting for volumes to be detached from the node")
framework.ExpectNoError(waitForVSphereDiskToDetach(ctx, persistentvolumes[0].Spec.VsphereVolume.VolumePath, pod.Spec.NodeName))
}
func verifyPodAndPvcCreationFailureOnWaitForFirstConsumerMode(ctx context.Context, client clientset.Interface, namespace string, scParameters map[string]string, zones []string) error {
storageclass, err := client.StorageV1().StorageClasses().Create(ctx, getVSphereStorageClassSpec("zone-sc", scParameters, zones, storagev1.VolumeBindingWaitForFirstConsumer), metav1.CreateOptions{})
framework.ExpectNoError(err, fmt.Sprintf("Failed to create storage class with err: %v", err))
ginkgo.DeferCleanup(client.StorageV1().StorageClasses().Delete, storageclass.Name, metav1.DeleteOptions{})
ginkgo.By("Creating PVC using the Storage Class")
pvclaim, err := e2epv.CreatePVC(ctx, client, namespace, getVSphereClaimSpecWithStorageClass(namespace, "2Gi", storageclass))
framework.ExpectNoError(err)
ginkgo.DeferCleanup(e2epv.DeletePersistentVolumeClaim, client, pvclaim.Name, namespace)
var pvclaims []*v1.PersistentVolumeClaim
pvclaims = append(pvclaims, pvclaim)
ginkgo.By("Creating a pod")
pod := e2epod.MakePod(namespace, nil, pvclaims, false, "")
pod, err = client.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
framework.ExpectNoError(err)
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, client, pod)
ginkgo.By("Waiting for claim to be in bound phase")
err = e2epv.WaitForPersistentVolumeClaimPhase(ctx, v1.ClaimBound, client, pvclaim.Namespace, pvclaim.Name, framework.Poll, 2*time.Minute)
framework.ExpectError(err)
eventList, err := client.CoreV1().Events(pvclaim.Namespace).List(ctx, metav1.ListOptions{})
framework.ExpectNoError(err)
// Look for PVC ProvisioningFailed event and return the message.
for _, event := range eventList.Items {
if event.Source.Component == "persistentvolume-controller" && event.Reason == volumeevents.ProvisioningFailed {
return fmt.Errorf("Failure message: %s", event.Message)
}
}
return nil
}
func waitForPVClaimBoundPhase(ctx context.Context, client clientset.Interface, pvclaims []*v1.PersistentVolumeClaim, timeout time.Duration) []*v1.PersistentVolume {
ginkgo.By("Waiting for claim to be in bound phase")
persistentvolumes, err := e2epv.WaitForPVClaimBoundPhase(ctx, client, pvclaims, timeout)
framework.ExpectNoError(err)
return persistentvolumes
}
func verifyPodSchedulingFails(ctx context.Context, client clientset.Interface, namespace string, nodeSelector map[string]string, scParameters map[string]string, zones []string, volumeBindingMode storagev1.VolumeBindingMode) {
storageclass, err := client.StorageV1().StorageClasses().Create(ctx, getVSphereStorageClassSpec("zone-sc", scParameters, zones, volumeBindingMode), metav1.CreateOptions{})
framework.ExpectNoError(err, fmt.Sprintf("Failed to create storage class with err: %v", err))
ginkgo.DeferCleanup(client.StorageV1().StorageClasses().Delete, storageclass.Name, metav1.DeleteOptions{})
ginkgo.By("Creating PVC using the Storage Class")
pvclaim, err := e2epv.CreatePVC(ctx, client, namespace, getVSphereClaimSpecWithStorageClass(namespace, "2Gi", storageclass))
framework.ExpectNoError(err)
ginkgo.DeferCleanup(e2epv.DeletePersistentVolumeClaim, client, pvclaim.Name, namespace)
var pvclaims []*v1.PersistentVolumeClaim
pvclaims = append(pvclaims, pvclaim)
ginkgo.By("Creating a pod")
pod, err := e2epod.CreateUnschedulablePod(ctx, client, namespace, nodeSelector, pvclaims, false, "")
framework.ExpectNoError(err)
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, client, pod)
}
func verifyPVCCreationFails(ctx context.Context, client clientset.Interface, namespace string, scParameters map[string]string, zones []string, volumeBindingMode storagev1.VolumeBindingMode) error {
storageclass, err := client.StorageV1().StorageClasses().Create(ctx, getVSphereStorageClassSpec("zone-sc", scParameters, zones, volumeBindingMode), metav1.CreateOptions{})
framework.ExpectNoError(err, fmt.Sprintf("Failed to create storage class with err: %v", err))
ginkgo.DeferCleanup(client.StorageV1().StorageClasses().Delete, storageclass.Name, metav1.DeleteOptions{})
ginkgo.By("Creating PVC using the Storage Class")
pvclaim, err := e2epv.CreatePVC(ctx, client, namespace, getVSphereClaimSpecWithStorageClass(namespace, "2Gi", storageclass))
framework.ExpectNoError(err)
ginkgo.DeferCleanup(e2epv.DeletePersistentVolumeClaim, client, pvclaim.Name, namespace)
ginkgo.By("Waiting for claim to be in bound phase")
err = e2epv.WaitForPersistentVolumeClaimPhase(ctx, v1.ClaimBound, client, pvclaim.Namespace, pvclaim.Name, framework.Poll, 2*time.Minute)
framework.ExpectError(err)
eventList, err := client.CoreV1().Events(pvclaim.Namespace).List(ctx, metav1.ListOptions{})
framework.ExpectNoError(err)
framework.Logf("Failure message : %+q", eventList.Items[0].Message)
return fmt.Errorf("Failure message: %+q", eventList.Items[0].Message)
}
func verifyPVZoneLabels(ctx context.Context, client clientset.Interface, timeouts *framework.TimeoutContext, namespace string, scParameters map[string]string, zones []string) {
storageclass, err := client.StorageV1().StorageClasses().Create(ctx, getVSphereStorageClassSpec("zone-sc", nil, zones, ""), metav1.CreateOptions{})
framework.ExpectNoError(err, fmt.Sprintf("Failed to create storage class with err: %v", err))
ginkgo.DeferCleanup(client.StorageV1().StorageClasses().Delete, storageclass.Name, metav1.DeleteOptions{})
ginkgo.By("Creating PVC using the storage class")
pvclaim, err := e2epv.CreatePVC(ctx, client, namespace, getVSphereClaimSpecWithStorageClass(namespace, "2Gi", storageclass))
framework.ExpectNoError(err)
ginkgo.DeferCleanup(e2epv.DeletePersistentVolumeClaim, client, pvclaim.Name, namespace)
var pvclaims []*v1.PersistentVolumeClaim
pvclaims = append(pvclaims, pvclaim)
ginkgo.By("Waiting for claim to be in bound phase")
persistentvolumes, err := e2epv.WaitForPVClaimBoundPhase(ctx, client, pvclaims, timeouts.ClaimProvision)
framework.ExpectNoError(err)
ginkgo.By("Verify zone information is present in the volume labels")
for _, pv := range persistentvolumes {
// Multiple zones are separated with "__"
pvZoneLabels := strings.Split(pv.ObjectMeta.Labels[v1.LabelTopologyZone], "__")
for _, zone := range zones {
gomega.Expect(pvZoneLabels).Should(gomega.ContainElement(zone), "Incorrect or missing zone labels in pv.")
}
}
}