Skip to content

Commit

Permalink
Fix subpath disruptive tests
Browse files Browse the repository at this point in the history
Add volumePath parameter to all disruptive checks, so subpath tests can use
"/test-volume" and disruptive tests can use "/mnt/volume1" for their
respective Pods.
  • Loading branch information
jsafrane committed Nov 11, 2022
1 parent 77b0279 commit 3fdd4ca
Show file tree
Hide file tree
Showing 8 changed files with 43 additions and 40 deletions.
13 changes: 10 additions & 3 deletions test/e2e/framework/pod/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ import (
imageutils "k8s.io/kubernetes/test/utils/image"
)

const (
VolumeMountPathTemplate = "/mnt/volume%d"
VolumeMountPath1 = "/mnt/volume1"
)

// Config is a struct containing all arguments for creating a pod.
// SELinux testing requires to pass HostIPC and HostPID as boolean arguments.
type Config struct {
Expand Down Expand Up @@ -222,10 +227,11 @@ func setVolumes(podSpec *v1.PodSpec, pvcs []*v1.PersistentVolumeClaim, inlineVol
volumeIndex := 0
for _, pvclaim := range pvcs {
volumename := fmt.Sprintf("volume%v", volumeIndex+1)
volumeMountPath := fmt.Sprintf(VolumeMountPathTemplate, volumeIndex+1)
if pvclaim.Spec.VolumeMode != nil && *pvclaim.Spec.VolumeMode == v1.PersistentVolumeBlock {
volumeDevices = append(volumeDevices, v1.VolumeDevice{Name: volumename, DevicePath: "/mnt/" + volumename})
volumeDevices = append(volumeDevices, v1.VolumeDevice{Name: volumename, DevicePath: volumeMountPath})
} else {
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumename, MountPath: "/mnt/" + volumename})
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumename, MountPath: volumeMountPath})
}
volumes[volumeIndex] = v1.Volume{
Name: volumename,
Expand All @@ -240,8 +246,9 @@ func setVolumes(podSpec *v1.PodSpec, pvcs []*v1.PersistentVolumeClaim, inlineVol
}
for _, src := range inlineVolumeSources {
volumename := fmt.Sprintf("volume%v", volumeIndex+1)
volumeMountPath := fmt.Sprintf(VolumeMountPathTemplate, volumeIndex+1)
// In-line volumes can be only filesystem, not block.
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumename, MountPath: "/mnt/" + volumename})
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumename, MountPath: volumeMountPath})
volumes[volumeIndex] = v1.Volume{Name: volumename, VolumeSource: *src}
volumeIndex++
}
Expand Down
1 change: 0 additions & 1 deletion test/e2e/framework/volume/fixtures.go
Original file line number Diff line number Diff line change
Expand Up @@ -659,7 +659,6 @@ func PodExec(f *framework.Framework, pod *v1.Pod, shExec string) (string, string
func VerifyExecInPodSucceed(f *framework.Framework, pod *v1.Pod, shExec string) {
stdout, stderr, err := PodExec(f, pod, shExec)
if err != nil {

if exiterr, ok := err.(uexec.CodeExitError); ok {
exitCode := exiterr.ExitStatus()
framework.ExpectNoError(err,
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/storage/generic_persistent_volume-disruptive.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ var _ = utils.SIGDescribe("GenericPersistentVolume[Disruptive]", func() {
func(t disruptiveTest) {
ginkgo.It(t.testItStmt, func() {
ginkgo.By("Executing Spec")
t.runTest(c, f, clientPod)
t.runTest(c, f, clientPod, e2epod.VolumeMountPath1)
})
}(test)
}
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/storage/nfs_persistent_volume-disruptive.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import (
admissionapi "k8s.io/pod-security-admission/api"
)

type testBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod)
type testBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, volumePath string)
type disruptiveTest struct {
testItStmt string
runTest testBody
Expand Down Expand Up @@ -272,7 +272,7 @@ var _ = utils.SIGDescribe("NFSPersistentVolumes[Disruptive][Flaky]", func() {
func(t disruptiveTest) {
ginkgo.It(t.testItStmt, func() {
ginkgo.By("Executing Spec")
t.runTest(c, f, clientPod)
t.runTest(c, f, clientPod, e2epod.VolumeMountPath1)
})
}(test)
}
Expand Down
14 changes: 7 additions & 7 deletions test/e2e/storage/testsuites/disruptive.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func (s *disruptiveTestSuite) DefineTests(driver storageframework.TestDriver, pa
framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource")
}

type singlePodTestBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod)
type singlePodTestBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, mountPath string)
type singlePodTest struct {
testItStmt string
runTestFile singlePodTestBody
Expand Down Expand Up @@ -185,10 +185,10 @@ func (s *disruptiveTestSuite) DefineTests(driver storageframework.TestDriver, pa
framework.ExpectNoError(err, "While creating pods for kubelet restart test")

if pattern.VolMode == v1.PersistentVolumeBlock && t.runTestBlock != nil {
t.runTestBlock(l.cs, l.config.Framework, l.pod)
t.runTestBlock(l.cs, l.config.Framework, l.pod, e2epod.VolumeMountPath1)
}
if pattern.VolMode == v1.PersistentVolumeFilesystem && t.runTestFile != nil {
t.runTestFile(l.cs, l.config.Framework, l.pod)
t.runTestFile(l.cs, l.config.Framework, l.pod, e2epod.VolumeMountPath1)
}
})
}
Expand All @@ -204,27 +204,27 @@ func (s *disruptiveTestSuite) DefineTests(driver storageframework.TestDriver, pa
{
testItStmt: "Should test that pv used in a pod that is deleted while the kubelet is down is usable by a new pod when kubelet returns [Feature:SELinuxMountReadWriteOncePod].",
runTestFile: func(c clientset.Interface, f *framework.Framework, pod1, pod2 *v1.Pod) {
storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, pod1, false, false, pod2)
storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, pod1, false, false, pod2, e2epod.VolumeMountPath1)
},
},
{
testItStmt: "Should test that pv used in a pod that is force deleted while the kubelet is down is usable by a new pod when kubelet returns [Feature:SELinuxMountReadWriteOncePod].",
runTestFile: func(c clientset.Interface, f *framework.Framework, pod1, pod2 *v1.Pod) {
storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, pod1, true, false, pod2)
storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, pod1, true, false, pod2, e2epod.VolumeMountPath1)
},
},
{
testItStmt: "Should test that pv used in a pod that is deleted while the kubelet is down is usable by a new pod with a different SELinux context when kubelet returns [Feature:SELinuxMountReadWriteOncePod].",
changeSELinuxContexts: true,
runTestFile: func(c clientset.Interface, f *framework.Framework, pod1, pod2 *v1.Pod) {
storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, pod1, false, false, pod2)
storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, pod1, false, false, pod2, e2epod.VolumeMountPath1)
},
},
{
testItStmt: "Should test that pv used in a pod that is force deleted while the kubelet is down is usable by a new pod with a different SELinux context when kubelet returns [Feature:SELinuxMountReadWriteOncePod].",
changeSELinuxContexts: true,
runTestFile: func(c clientset.Interface, f *framework.Framework, pod1, pod2 *v1.Pod) {
storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, pod1, true, false, pod2)
storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, pod1, true, false, pod2, e2epod.VolumeMountPath1)
},
},
}
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/storage/testsuites/subpath.go
Original file line number Diff line number Diff line change
Expand Up @@ -1002,7 +1002,7 @@ func testSubpathReconstruction(f *framework.Framework, hostExec storageutils.Hos
}
framework.ExpectNotEqual(podNode, nil, "pod node should exist in schedulable nodes")

storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true, nil)
storageutils.TestVolumeUnmountsFromDeletedPodWithForceOption(f.ClientSet, f, pod, forceDelete, true, nil, volumePath)

if podNode != nil {
mountPoints := globalMountPointsByNode[podNode.Name]
Expand Down
43 changes: 20 additions & 23 deletions test/e2e/storage/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,46 +95,44 @@ func getKubeletMainPid(nodeIP string, sudoPresent bool, systemctlPresent bool) s
}

// TestKubeletRestartsAndRestoresMount tests that a volume mounted to a pod remains mounted after a kubelet restarts
func TestKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
path := "/mnt/volume1"
func TestKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, volumePath string) {
byteLen := 64
seed := time.Now().UTC().UnixNano()

ginkgo.By("Writing to the volume.")
CheckWriteToPath(f, clientPod, v1.PersistentVolumeFilesystem, false, path, byteLen, seed)
CheckWriteToPath(f, clientPod, v1.PersistentVolumeFilesystem, false, volumePath, byteLen, seed)

ginkgo.By("Restarting kubelet")
KubeletCommand(KRestart, c, clientPod)

ginkgo.By("Testing that written file is accessible.")
CheckReadFromPath(f, clientPod, v1.PersistentVolumeFilesystem, false, path, byteLen, seed)
CheckReadFromPath(f, clientPod, v1.PersistentVolumeFilesystem, false, volumePath, byteLen, seed)

framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, path)
framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, volumePath)
}

// TestKubeletRestartsAndRestoresMap tests that a volume mapped to a pod remains mapped after a kubelet restarts
func TestKubeletRestartsAndRestoresMap(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
path := "/mnt/volume1"
func TestKubeletRestartsAndRestoresMap(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, volumePath string) {
byteLen := 64
seed := time.Now().UTC().UnixNano()

ginkgo.By("Writing to the volume.")
CheckWriteToPath(f, clientPod, v1.PersistentVolumeBlock, false, path, byteLen, seed)
CheckWriteToPath(f, clientPod, v1.PersistentVolumeBlock, false, volumePath, byteLen, seed)

ginkgo.By("Restarting kubelet")
KubeletCommand(KRestart, c, clientPod)

ginkgo.By("Testing that written pv is accessible.")
CheckReadFromPath(f, clientPod, v1.PersistentVolumeBlock, false, path, byteLen, seed)
CheckReadFromPath(f, clientPod, v1.PersistentVolumeBlock, false, volumePath, byteLen, seed)

framework.Logf("Volume map detected on pod %s and written data %s is readable post-restart.", clientPod.Name, path)
framework.Logf("Volume map detected on pod %s and written data %s is readable post-restart.", clientPod.Name, volumePath)
}

// TestVolumeUnmountsFromDeletedPodWithForceOption tests that a volume unmounts if the client pod was deleted while the kubelet was down.
// forceDelete is true indicating whether the pod is forcefully deleted.
// checkSubpath is true indicating whether the subpath should be checked.
// If secondPod is set, it is started when kubelet is down to check that the volume is usable while the old pod is being deleted and the new pod is starting.
func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool, secondPod *v1.Pod) {
func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, checkSubpath bool, secondPod *v1.Pod, volumePath string) {
nodeIP, err := getHostAddress(c, clientPod)
framework.ExpectNoError(err)
nodeIP = nodeIP + ":22"
Expand All @@ -154,10 +152,9 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f
}

ginkgo.By("Writing to the volume.")
path := "/mnt/volume1"
byteLen := 64
seed := time.Now().UTC().UnixNano()
CheckWriteToPath(f, clientPod, v1.PersistentVolumeFilesystem, false, path, byteLen, seed)
CheckWriteToPath(f, clientPod, v1.PersistentVolumeFilesystem, false, volumePath, byteLen, seed)

// This command is to make sure kubelet is started after test finishes no matter it fails or not.
defer func() {
Expand Down Expand Up @@ -209,7 +206,7 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f
framework.ExpectEqual(result.Code, 0, fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))

ginkgo.By("Testing that written file is accessible in the second pod.")
CheckReadFromPath(f, secondPod, v1.PersistentVolumeFilesystem, false, path, byteLen, seed)
CheckReadFromPath(f, secondPod, v1.PersistentVolumeFilesystem, false, volumePath, byteLen, seed)
err = c.CoreV1().Pods(secondPod.Namespace).Delete(context.TODO(), secondPod.Name, metav1.DeleteOptions{})
framework.ExpectNoError(err, "when deleting the second pod")
err = e2epod.WaitForPodNotFoundInNamespace(f.ClientSet, secondPod.Name, f.Namespace.Name, f.Timeouts.PodDelete)
Expand All @@ -235,18 +232,18 @@ func TestVolumeUnmountsFromDeletedPodWithForceOption(c clientset.Interface, f *f
}

// TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down.
func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false, nil)
func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, volumePath string) {
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, false, false, nil, volumePath)
}

// TestVolumeUnmountsFromForceDeletedPod tests that a volume unmounts if the client pod was forcefully deleted while the kubelet was down.
func TestVolumeUnmountsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false, nil)
func TestVolumeUnmountsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, volumePath string) {
TestVolumeUnmountsFromDeletedPodWithForceOption(c, f, clientPod, true, false, nil, volumePath)
}

// TestVolumeUnmapsFromDeletedPodWithForceOption tests that a volume unmaps if the client pod was deleted while the kubelet was down.
// forceDelete is true indicating whether the pod is forcefully deleted.
func TestVolumeUnmapsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool) {
func TestVolumeUnmapsFromDeletedPodWithForceOption(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, forceDelete bool, devicePath string) {
nodeIP, err := getHostAddress(c, clientPod)
framework.ExpectNoError(err, "Failed to get nodeIP.")
nodeIP = nodeIP + ":22"
Expand Down Expand Up @@ -317,13 +314,13 @@ func TestVolumeUnmapsFromDeletedPodWithForceOption(c clientset.Interface, f *fra
}

// TestVolumeUnmapsFromDeletedPod tests that a volume unmaps if the client pod was deleted while the kubelet was down.
func TestVolumeUnmapsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
TestVolumeUnmapsFromDeletedPodWithForceOption(c, f, clientPod, false)
func TestVolumeUnmapsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, devicePath string) {
TestVolumeUnmapsFromDeletedPodWithForceOption(c, f, clientPod, false, devicePath)
}

// TestVolumeUnmapsFromForceDeletedPod tests that a volume unmaps if the client pod was forcefully deleted while the kubelet was down.
func TestVolumeUnmapsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod) {
TestVolumeUnmapsFromDeletedPodWithForceOption(c, f, clientPod, true)
func TestVolumeUnmapsFromForceDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, devicePath string) {
TestVolumeUnmapsFromDeletedPodWithForceOption(c, f, clientPod, true, devicePath)
}

// RunInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/storage/vsphere/persistent_volumes-vsphere.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ var _ = utils.SIGDescribe("PersistentVolumes:vsphere [Feature:vsphere]", func()
*/
ginkgo.It("should test that a file written to the vsphere volume mount before kubelet restart can be read after restart [Disruptive]", func() {
e2eskipper.SkipUnlessSSHKeyPresent()
utils.TestKubeletRestartsAndRestoresMount(c, f, clientPod)
utils.TestKubeletRestartsAndRestoresMount(c, f, clientPod, e2epod.VolumeMountPath1)
})

/*
Expand All @@ -175,7 +175,7 @@ var _ = utils.SIGDescribe("PersistentVolumes:vsphere [Feature:vsphere]", func()
*/
ginkgo.It("should test that a vsphere volume mounted to a pod that is deleted while the kubelet is down unmounts when the kubelet returns [Disruptive]", func() {
e2eskipper.SkipUnlessSSHKeyPresent()
utils.TestVolumeUnmountsFromDeletedPod(c, f, clientPod)
utils.TestVolumeUnmountsFromDeletedPod(c, f, clientPod, e2epod.VolumeMountPath1)
})

/*
Expand Down

0 comments on commit 3fdd4ca

Please sign in to comment.