forked from kubernetes/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 1
/
csi_selinux_mount.go
240 lines (220 loc) · 10.8 KB
/
csi_selinux_mount.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package csi_mock
import (
"context"
"fmt"
"sync/atomic"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/test/e2e/framework"
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
"k8s.io/kubernetes/test/e2e/storage/utils"
admissionapi "k8s.io/pod-security-admission/api"
)
var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() {
f := framework.NewDefaultFramework("csi-mock-volumes-selinux")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
m := newMockDriverSetup(f)
ginkgo.Context("SELinuxMount [LinuxOnly][Feature:SELinux][Feature:SELinuxMountReadWriteOncePod]", func() {
// Make sure all options are set so system specific defaults are not used.
seLinuxOpts1 := v1.SELinuxOptions{
User: "system_u",
Role: "object_r",
Type: "container_file_t",
Level: "s0:c0,c1",
}
seLinuxMountOption1 := "context=\"system_u:object_r:container_file_t:s0:c0,c1\""
seLinuxOpts2 := v1.SELinuxOptions{
User: "system_u",
Role: "object_r",
Type: "container_file_t",
Level: "s0:c98,c99",
}
seLinuxMountOption2 := "context=\"system_u:object_r:container_file_t:s0:c98,c99\""
tests := []struct {
name string
csiDriverSELinuxEnabled bool
firstPodSELinuxOpts *v1.SELinuxOptions
startSecondPod bool
secondPodSELinuxOpts *v1.SELinuxOptions
mountOptions []string
volumeMode v1.PersistentVolumeAccessMode
expectedFirstMountOptions []string
expectedSecondMountOptions []string
expectedUnstage bool
}{
// Start just a single pod and check its volume is mounted correctly
{
name: "should pass SELinux mount option for RWOP volume and Pod with SELinux context set",
csiDriverSELinuxEnabled: true,
firstPodSELinuxOpts: &seLinuxOpts1,
volumeMode: v1.ReadWriteOncePod,
expectedFirstMountOptions: []string{seLinuxMountOption1},
},
{
name: "should add SELinux mount option to existing mount options",
csiDriverSELinuxEnabled: true,
firstPodSELinuxOpts: &seLinuxOpts1,
mountOptions: []string{"noexec", "noatime"},
volumeMode: v1.ReadWriteOncePod,
expectedFirstMountOptions: []string{"noexec", "noatime", seLinuxMountOption1},
},
{
name: "should not pass SELinux mount option for RWO volume",
csiDriverSELinuxEnabled: true,
firstPodSELinuxOpts: &seLinuxOpts1,
volumeMode: v1.ReadWriteOnce,
expectedFirstMountOptions: nil,
},
{
name: "should not pass SELinux mount option for Pod without SELinux context",
csiDriverSELinuxEnabled: true,
firstPodSELinuxOpts: nil,
volumeMode: v1.ReadWriteOncePod,
expectedFirstMountOptions: nil,
},
{
name: "should not pass SELinux mount option for CSI driver that does not support SELinux mount",
csiDriverSELinuxEnabled: false,
firstPodSELinuxOpts: &seLinuxOpts1,
volumeMode: v1.ReadWriteOncePod,
expectedFirstMountOptions: nil,
},
// Start two pods in a sequence and check their volume is / is not unmounted in between
{
name: "should not unstage volume when starting a second pod with the same SELinux context",
csiDriverSELinuxEnabled: true,
firstPodSELinuxOpts: &seLinuxOpts1,
startSecondPod: true,
secondPodSELinuxOpts: &seLinuxOpts1,
volumeMode: v1.ReadWriteOncePod,
expectedFirstMountOptions: []string{seLinuxMountOption1},
expectedSecondMountOptions: []string{seLinuxMountOption1},
expectedUnstage: false,
},
{
name: "should unstage volume when starting a second pod with different SELinux context",
csiDriverSELinuxEnabled: true,
firstPodSELinuxOpts: &seLinuxOpts1,
startSecondPod: true,
secondPodSELinuxOpts: &seLinuxOpts2,
volumeMode: v1.ReadWriteOncePod,
expectedFirstMountOptions: []string{seLinuxMountOption1},
expectedSecondMountOptions: []string{seLinuxMountOption2},
expectedUnstage: true,
},
}
for _, t := range tests {
t := t
ginkgo.It(t.name, func(ctx context.Context) {
if framework.NodeOSDistroIs("windows") {
e2eskipper.Skipf("SELinuxMount is only applied on linux nodes -- skipping")
}
var nodeStageMountOpts, nodePublishMountOpts []string
var unstageCalls, stageCalls, unpublishCalls, publishCalls atomic.Int32
m.init(ctx, testParameters{
disableAttach: true,
registerDriver: true,
enableSELinuxMount: &t.csiDriverSELinuxEnabled,
hooks: createSELinuxMountPreHook(&nodeStageMountOpts, &nodePublishMountOpts, &stageCalls, &unstageCalls, &publishCalls, &unpublishCalls),
})
defer m.cleanup(ctx)
// Act
ginkgo.By("Starting the initial pod")
accessModes := []v1.PersistentVolumeAccessMode{t.volumeMode}
_, claim, pod := m.createPodWithSELinux(ctx, accessModes, t.mountOptions, t.firstPodSELinuxOpts)
err := e2epod.WaitForPodNameRunningInNamespace(ctx, m.cs, pod.Name, pod.Namespace)
framework.ExpectNoError(err, "starting the initial pod")
// Assert
ginkgo.By("Checking the initial pod mount options")
framework.ExpectEqual(nodeStageMountOpts, t.expectedFirstMountOptions, "NodeStage MountFlags for the initial pod")
framework.ExpectEqual(nodePublishMountOpts, t.expectedFirstMountOptions, "NodePublish MountFlags for the initial pod")
ginkgo.By("Checking the CSI driver calls for the initial pod")
gomega.Expect(unstageCalls.Load()).To(gomega.BeNumerically("==", 0), "NodeUnstage call count for the initial pod")
gomega.Expect(unpublishCalls.Load()).To(gomega.BeNumerically("==", 0), "NodeUnpublish call count for the initial pod")
gomega.Expect(stageCalls.Load()).To(gomega.BeNumerically(">", 0), "NodeStage for the initial pod")
gomega.Expect(publishCalls.Load()).To(gomega.BeNumerically(">", 0), "NodePublish for the initial pod")
if !t.startSecondPod {
return
}
// Arrange 2nd part of the test
ginkgo.By("Starting the second pod to check if a volume used by the initial pod is / is not unmounted based on SELinux context")
// Skip scheduler, it would block scheduling the second pod with ReadWriteOncePod PV.
pod, err = m.cs.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, fmt.Sprintf("getting the initial pod"))
nodeSelection := e2epod.NodeSelection{Name: pod.Spec.NodeName}
pod2, err := startPausePodWithSELinuxOptions(f.ClientSet, claim, nodeSelection, f.Namespace.Name, t.secondPodSELinuxOpts)
framework.ExpectNoError(err, "creating second pod with SELinux context %s", t.secondPodSELinuxOpts)
m.pods = append(m.pods, pod2)
// Delete the initial pod only after kubelet processes the second pod and adds its volumes to
// DesiredStateOfWorld.
// In this state, any volume UnPublish / UnStage must be done because of SELinux contexts and not
// because of random races because volumes of the second pod are not in DesiredStateOfWorld yet.
ginkgo.By("Waiting for the second pod to fail to start because of ReadWriteOncePod.")
eventSelector := fields.Set{
"involvedObject.kind": "Pod",
"involvedObject.name": pod2.Name,
"involvedObject.namespace": pod2.Namespace,
"reason": events.FailedMountVolume,
}.AsSelector().String()
var msg string
if t.expectedUnstage {
// This message is emitted before kubelet checks for ReadWriteOncePod
msg = "conflicting SELinux labels of volume"
} else {
msg = "volume uses the ReadWriteOncePod access mode and is already in use by another pod"
}
err = e2eevents.WaitTimeoutForEvent(ctx, m.cs, pod2.Namespace, eventSelector, msg, f.Timeouts.PodStart)
framework.ExpectNoError(err, "waiting for event %q in the second test pod", msg)
// count fresh CSI driver calls between the first and the second pod
nodeStageMountOpts = nil
nodePublishMountOpts = nil
unstageCalls.Store(0)
unpublishCalls.Store(0)
stageCalls.Store(0)
publishCalls.Store(0)
// Act 2nd part of the test
ginkgo.By("Deleting the initial pod")
err = e2epod.DeletePodWithWait(ctx, m.cs, pod)
framework.ExpectNoError(err, "deleting the initial pod")
// Assert 2nd part of the test
ginkgo.By("Waiting for the second pod to start")
err = e2epod.WaitForPodNameRunningInNamespace(ctx, m.cs, pod2.Name, pod2.Namespace)
framework.ExpectNoError(err, "starting the second pod")
ginkgo.By("Checking CSI driver calls for the second pod")
if t.expectedUnstage {
// Volume should be fully unstaged between the first and the second pod
gomega.Expect(unstageCalls.Load()).To(gomega.BeNumerically(">", 0), "NodeUnstage calls after the first pod is deleted")
gomega.Expect(stageCalls.Load()).To(gomega.BeNumerically(">", 0), "NodeStage calls for the second pod")
// The second pod got the right mount option
framework.ExpectEqual(nodeStageMountOpts, t.expectedSecondMountOptions, "NodeStage MountFlags for the second pod")
} else {
// Volume should not be fully unstaged between the first and the second pod
gomega.Expect(unstageCalls.Load()).To(gomega.BeNumerically("==", 0), "NodeUnstage calls after the first pod is deleted")
gomega.Expect(stageCalls.Load()).To(gomega.BeNumerically("==", 0), "NodeStage calls for the second pod")
}
// In both cases, Unublish and Publish is called, with the right mount opts
gomega.Expect(unpublishCalls.Load()).To(gomega.BeNumerically(">", 0), "NodeUnpublish calls after the first pod is deleted")
gomega.Expect(publishCalls.Load()).To(gomega.BeNumerically(">", 0), "NodePublish calls for the second pod")
framework.ExpectEqual(nodePublishMountOpts, t.expectedSecondMountOptions, "NodePublish MountFlags for the second pod")
})
}
})
})