-
Notifications
You must be signed in to change notification settings - Fork 38.6k
/
readwriteoncepod.go
217 lines (181 loc) · 9.01 KB
/
readwriteoncepod.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testsuites
import (
"context"
"github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
errors "k8s.io/apimachinery/pkg/util/errors"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/test/e2e/framework"
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
admissionapi "k8s.io/pod-security-admission/api"
)
type readWriteOncePodTestSuite struct {
tsInfo storageframework.TestSuiteInfo
}
var _ storageframework.TestSuite = &readWriteOncePodTestSuite{}
type readWriteOncePodTest struct {
config *storageframework.PerTestConfig
cs clientset.Interface
volume *storageframework.VolumeResource
pods []*v1.Pod
migrationCheck *migrationOpCheck
}
func InitCustomReadWriteOncePodTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
return &readWriteOncePodTestSuite{
tsInfo: storageframework.TestSuiteInfo{
Name: "read-write-once-pod",
TestPatterns: patterns,
FeatureTag: "[Feature:ReadWriteOncePod]",
},
}
}
// InitReadWriteOncePodTestSuite returns a test suite for the ReadWriteOncePod PersistentVolume access mode feature.
func InitReadWriteOncePodTestSuite() storageframework.TestSuite {
// Only covers one test pattern since ReadWriteOncePod enforcement is
// handled through Kubernetes and does not differ across volume types.
patterns := []storageframework.TestPattern{storageframework.DefaultFsDynamicPV}
return InitCustomReadWriteOncePodTestSuite(patterns)
}
func (t *readWriteOncePodTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
return t.tsInfo
}
func (t *readWriteOncePodTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
driverInfo := driver.GetDriverInfo()
if !driverInfo.Capabilities[storageframework.CapReadWriteOncePod] {
e2eskipper.Skipf("Driver %q doesn't support ReadWriteOncePod - skipping", driverInfo.Name)
}
}
func (t *readWriteOncePodTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
var (
driverInfo = driver.GetDriverInfo()
l readWriteOncePodTest
)
// Beware that it also registers an AfterEach which renders f unusable. Any code using
// f must run inside an It or Context callback.
f := framework.NewFrameworkWithCustomTimeouts("read-write-once-pod", storageframework.GetDriverTimeouts(driver))
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
init := func() {
l = readWriteOncePodTest{}
l.config = driver.PrepareTest(f)
l.cs = f.ClientSet
l.pods = []*v1.Pod{}
l.migrationCheck = newMigrationOpCheck(f.ClientSet, f.ClientConfig(), driverInfo.InTreePluginName)
}
cleanup := func() {
var errs []error
for _, pod := range l.pods {
framework.Logf("Deleting pod %v", pod.Name)
err := e2epod.DeletePodWithWait(l.cs, pod)
errs = append(errs, err)
}
framework.Logf("Deleting volume %s", l.volume.Pvc.GetName())
err := l.volume.CleanupResource()
errs = append(errs, err)
framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource")
l.migrationCheck.validateMigrationVolumeOpCounts()
}
ginkgo.BeforeEach(func() {
init()
ginkgo.DeferCleanup(cleanup)
})
ginkgo.It("should block a second pod from using an in-use ReadWriteOncePod volume", func() {
// Create the ReadWriteOncePod PVC.
accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}
l.volume = storageframework.CreateVolumeResourceWithAccessModes(driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes)
podConfig := e2epod.Config{
NS: f.Namespace.Name,
PVCs: []*v1.PersistentVolumeClaim{l.volume.Pvc},
SeLinuxLabel: e2epv.SELinuxLabel,
}
// Create the first pod, which will take ownership of the ReadWriteOncePod PVC.
pod1, err := e2epod.MakeSecPod(&podConfig)
framework.ExpectNoError(err, "failed to create spec for pod1")
_, err = l.cs.CoreV1().Pods(pod1.Namespace).Create(context.TODO(), pod1, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create pod1")
err = e2epod.WaitTimeoutForPodRunningInNamespace(l.cs, pod1.Name, pod1.Namespace, f.Timeouts.PodStart)
framework.ExpectNoError(err, "failed to wait for pod1 running status")
l.pods = append(l.pods, pod1)
// Create the second pod, which will fail scheduling because the ReadWriteOncePod PVC is already in use.
pod2, err := e2epod.MakeSecPod(&podConfig)
framework.ExpectNoError(err, "failed to create spec for pod2")
_, err = l.cs.CoreV1().Pods(pod2.Namespace).Create(context.TODO(), pod2, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create pod2")
err = e2epod.WaitForPodNameUnschedulableInNamespace(l.cs, pod2.Name, pod2.Namespace)
framework.ExpectNoError(err, "failed to wait for pod2 unschedulable status")
l.pods = append(l.pods, pod2)
// Delete the first pod and observe the second pod can now start.
err = e2epod.DeletePodWithWait(l.cs, pod1)
framework.ExpectNoError(err, "failed to delete pod1")
err = e2epod.WaitTimeoutForPodRunningInNamespace(l.cs, pod2.Name, pod2.Namespace, f.Timeouts.PodStart)
framework.ExpectNoError(err, "failed to wait for pod2 running status")
})
ginkgo.It("should block a second pod from using an in-use ReadWriteOncePod volume on the same node", func() {
// Create the ReadWriteOncePod PVC.
accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}
l.volume = storageframework.CreateVolumeResourceWithAccessModes(driver, l.config, pattern, t.GetTestSuiteInfo().SupportedSizeRange, accessModes)
podConfig := e2epod.Config{
NS: f.Namespace.Name,
PVCs: []*v1.PersistentVolumeClaim{l.volume.Pvc},
SeLinuxLabel: e2epv.SELinuxLabel,
}
// Create the first pod, which will take ownership of the ReadWriteOncePod PVC.
pod1, err := e2epod.MakeSecPod(&podConfig)
framework.ExpectNoError(err, "failed to create spec for pod1")
_, err = l.cs.CoreV1().Pods(pod1.Namespace).Create(context.TODO(), pod1, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create pod1")
err = e2epod.WaitTimeoutForPodRunningInNamespace(l.cs, pod1.Name, pod1.Namespace, f.Timeouts.PodStart)
framework.ExpectNoError(err, "failed to wait for pod1 running status")
l.pods = append(l.pods, pod1)
// Get the node name for the first pod now that it's running.
pod1, err = l.cs.CoreV1().Pods(pod1.Namespace).Get(context.TODO(), pod1.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to get pod1")
nodeName := pod1.Spec.NodeName
// Create the second pod on the same node as the first pod.
pod2, err := e2epod.MakeSecPod(&podConfig)
framework.ExpectNoError(err, "failed to create spec for pod2")
// Set the node name to that of the first pod.
// Node name is set to bypass scheduling, which would enforce the access mode otherwise.
pod2.Spec.NodeName = nodeName
_, err = l.cs.CoreV1().Pods(pod2.Namespace).Create(context.TODO(), pod2, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create pod2")
l.pods = append(l.pods, pod2)
// Wait for the FailedMount event to be generated for the second pod.
eventSelector := fields.Set{
"involvedObject.kind": "Pod",
"involvedObject.name": pod2.Name,
"involvedObject.namespace": pod2.Namespace,
"reason": events.FailedMountVolume,
}.AsSelector().String()
msg := "volume uses the ReadWriteOncePod access mode and is already in use by another pod"
err = e2eevents.WaitTimeoutForEvent(l.cs, pod2.Namespace, eventSelector, msg, f.Timeouts.PodStart)
framework.ExpectNoError(err, "failed to wait for FailedMount event for pod2")
// Wait for the second pod to fail because it is stuck at container creating.
reason := "ContainerCreating"
err = e2epod.WaitForPodContainerToFail(l.cs, pod2.Namespace, pod2.Name, 0, reason, f.Timeouts.PodStart)
framework.ExpectNoError(err, "failed to wait for pod2 container to fail")
// Delete the first pod and observe the second pod can now start.
err = e2epod.DeletePodWithWait(l.cs, pod1)
framework.ExpectNoError(err, "failed to delete pod1")
err = e2epod.WaitTimeoutForPodRunningInNamespace(l.cs, pod2.Name, pod2.Namespace, f.Timeouts.PodStart)
framework.ExpectNoError(err, "failed to wait for pod2 running status")
})
}