Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Isolate recycler behavior in PV E2E #40177

Merged
merged 1 commit into from
Feb 15, 2017
Merged

Isolate recycler behavior in PV E2E #40177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
72 changes: 61 additions & 11 deletions test/e2e/persistent_volumes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package e2e

import (
"fmt"
"time"

. "github.com/onsi/ginkgo"
Expand All @@ -32,7 +33,7 @@ import (
// phase. Note: the PV is deleted in the AfterEach, not here.
func completeTest(f *framework.Framework, c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) {

// 1. verify that the PV and PVC have binded correctly
// 1. verify that the PV and PVC have bound correctly
By("Validating the PV-PVC binding")
waitOnPVandPVC(c, ns, pv, pvc)

Expand All @@ -41,18 +42,17 @@ func completeTest(f *framework.Framework, c clientset.Interface, ns string, pv *
By("Checking pod has write access to PersistentVolume")
createWaitAndDeletePod(f, c, ns, pvc.Name)

// 3. delete the PVC, wait for PV to become "Available"
By("Deleting the PVC to invoke the recycler")
deletePVCandValidatePV(c, ns, pvc, pv, v1.VolumeAvailable)
// 3. delete the PVC, wait for PV to become "Released"
By("Deleting the PVC to invoke the reclaim policy.")
deletePVCandValidatePV(c, ns, pvc, pv, v1.VolumeReleased)
}

// Validate pairs of PVs and PVCs, create and verify writer pod, delete PVC and validate
// PV. Ensure each step succeeds.
// Note: the PV is deleted in the AfterEach, not here.
// Note: this func is serialized, we wait for each pod to be deleted before creating the
// next pod. Adding concurrency is a TODO item.
// Note: this func is called recursively when there are more claims than pvs.
func completeMultiTest(f *framework.Framework, c clientset.Interface, ns string, pvols pvmap, claims pvcmap) {
func completeMultiTest(f *framework.Framework, c clientset.Interface, ns string, pvols pvmap, claims pvcmap, expectPhase v1.PersistentVolumePhase) {

// 1. verify each PV permits write access to a client pod
By("Checking pod has write access to PersistentVolumes")
Expand All @@ -69,9 +69,9 @@ func completeMultiTest(f *framework.Framework, c clientset.Interface, ns string,
createWaitAndDeletePod(f, c, pvcKey.Namespace, pvcKey.Name)
}

// 2. delete each PVC, wait for its bound PV to become "Available"
// 2. delete each PVC, wait for its bound PV to reach `expectedPhase`
By("Deleting PVCs to invoke recycler")
deletePVCandValidatePVGroup(c, ns, pvols, claims)
deletePVCandValidatePVGroup(c, ns, pvols, claims, expectPhase)
}

// Creates a PV, PVC, and ClientPod that will run until killed by test or clean up.
Expand Down Expand Up @@ -218,7 +218,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
numPVs, numPVCs := 2, 4
pvols, claims = createPVsPVCs(numPVs, numPVCs, c, ns, pvConfig)
waitAndVerifyBinds(c, ns, pvols, claims, true)
completeMultiTest(f, c, ns, pvols, claims)
completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased)
})

// Create 3 PVs and 3 PVCs.
Expand All @@ -227,7 +227,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
numPVs, numPVCs := 3, 3
pvols, claims = createPVsPVCs(numPVs, numPVCs, c, ns, pvConfig)
waitAndVerifyBinds(c, ns, pvols, claims, true)
completeMultiTest(f, c, ns, pvols, claims)
completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased)
})

// Create 4 PVs and 2 PVCs.
Expand All @@ -236,7 +236,57 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Serial]", func() {
numPVs, numPVCs := 4, 2
pvols, claims = createPVsPVCs(numPVs, numPVCs, c, ns, pvConfig)
waitAndVerifyBinds(c, ns, pvols, claims, true)
completeMultiTest(f, c, ns, pvols, claims)
completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased)
})
})

// This Context isolates and tests the "Recycle" reclaim behavior. On deprecation of the
// Recycler, this entire context can be removed without affecting the test suite or leaving behind
// dead code.
Context("when invoking the Recycle reclaim policy", func() {
var pv *v1.PersistentVolume
var pvc *v1.PersistentVolumeClaim

BeforeEach(func() {
pvConfig.reclaimPolicy = v1.PersistentVolumeReclaimRecycle
pv, pvc = createPVPVC(c, pvConfig, ns, false)
waitOnPVandPVC(c, ns, pv, pvc)
})

AfterEach(func() {
framework.Logf("AfterEach: Cleaning up test resources.")
pvPvcCleanup(c, ns, pv, pvc)
})

// This It() tests a scenario where a PV is written to by a Pod, recycled, then the volume checked
// for files. If files are found, the checking Pod fails, failing the test. Otherwise, the pod
// (and test) succeed.
It("should test that a PV becomes Available and is clean after the PVC is deleted. [Volume][Serial][Flaky]", func() {
By("Writing to the volume.")
pod := makeWritePod(ns, pvc.Name)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

L653-657: can you call createClientPod here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no, but we can write a couple new functions to mirror the pattern of makeWritePod wrapping makePod. Alternatively, we could use createClientPod then podExec to make a file on the volume

pod, err := c.Core().Pods(ns).Create(pod)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPodSuccessInNamespace(c, pod.Name, ns)
Expect(err).NotTo(HaveOccurred())

deletePVCandValidatePV(c, ns, pvc, pv, v1.VolumeAvailable)

By("Re-mounting the volume.")
pvc = makePersistentVolumeClaim(ns)
pvc = createPVC(c, ns, pvc)
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, 2*time.Second, 60*time.Second)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need to handle err != nil here.

Expect(err).NotTo(HaveOccurred())

// If a file is detected in /mnt, fail the pod and do not restart it.
By("Verifying the mount has been cleaned.")
mount := pod.Spec.Containers[0].VolumeMounts[0].MountPath
pod = makePod(ns, pvc.Name, fmt.Sprintf("[ $(ls -A %s | wc -l) -eq 0 ] && exit 0 || exit 1", mount))

pod, err = c.Core().Pods(ns).Create(pod)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForPodSuccessInNamespace(c, pod.Name, ns)
Expect(err).NotTo(HaveOccurred())
framework.Logf("Pod exited without failure; the volume has been recycled.")
})
})
})
Expand Down
61 changes: 27 additions & 34 deletions test/e2e/pvutil.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,10 @@ type pvcmap map[types.NamespacedName]pvcval
// },
// }
type persistentVolumeConfig struct {
pvSource v1.PersistentVolumeSource
prebind *v1.PersistentVolumeClaim
namePrefix string
pvSource v1.PersistentVolumeSource
prebind *v1.PersistentVolumeClaim
reclaimPolicy v1.PersistentVolumeReclaimPolicy
namePrefix string
}

// Delete the nfs-server pod. Only done once per KubeDescription().
Expand Down Expand Up @@ -117,48 +118,48 @@ func deletePersistentVolumeClaim(c clientset.Interface, pvcName string, ns strin
}
}

// Delete the PVC and wait for the PV to become Available again. Validate that the PV
// has recycled (assumption here about reclaimPolicy). Caller tells this func which
// Delete the PVC and wait for the PV to enter its expected phase. Validate that the PV
// has been reclaimed (assumption here about reclaimPolicy). Caller tells this func which
// phase value to expect for the pv bound to the to-be-deleted claim.
func deletePVCandValidatePV(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume, expctPVPhase v1.PersistentVolumePhase) {
func deletePVCandValidatePV(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume, expectPVPhase v1.PersistentVolumePhase) {

pvname := pvc.Spec.VolumeName
framework.Logf("Deleting PVC %v to trigger recycling of PV %v", pvc.Name, pvname)
framework.Logf("Deleting PVC %v to trigger reclamation of PV %v", pvc.Name, pvname)
deletePersistentVolumeClaim(c, pvc.Name, ns)

// Check that the PVC is really deleted.
pvc, err := c.Core().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
Expect(apierrs.IsNotFound(err)).To(BeTrue())

// Wait for the PV's phase to return to Available
framework.Logf("Waiting for recycling process to complete.")
err = framework.WaitForPersistentVolumePhase(expctPVPhase, c, pv.Name, 1*time.Second, 300*time.Second)
// Wait for the PV's phase to return to be `expectPVPhase`
framework.Logf("Waiting for reclaim process to complete.")
err = framework.WaitForPersistentVolumePhase(expectPVPhase, c, pv.Name, 1*time.Second, 300*time.Second)
Expect(err).NotTo(HaveOccurred())

// examine the pv's ClaimRef and UID and compare to expected values
pv, err = c.Core().PersistentVolumes().Get(pv.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
cr := pv.Spec.ClaimRef
if expctPVPhase == v1.VolumeAvailable {
if expectPVPhase == v1.VolumeAvailable {
if cr != nil { // may be ok if cr != nil
Expect(len(cr.UID)).To(BeZero())
Expect(cr.UID).To(BeEmpty())
}
} else if expctPVPhase == v1.VolumeBound {
} else if expectPVPhase == v1.VolumeBound {
Expect(cr).NotTo(BeNil())
Expect(len(cr.UID)).NotTo(BeZero())
Expect(cr.UID).NotTo(BeEmpty())
}

framework.Logf("PV %v now in %q phase", pv.Name, expctPVPhase)
framework.Logf("PV %v now in %q phase", pv.Name, expectPVPhase)
}

// Wraps deletePVCandValidatePV() by calling the function in a loop over the PV map. Only
// bound PVs are deleted. Validates that the claim was deleted and the PV is Available.
// bound PVs are deleted. Validates that the claim was deleted and the PV is in the relevant Phase (Released, Available,
// Bound).
// Note: if there are more claims than pvs then some of the remaining claims will bind to
// the just-made-available pvs.
func deletePVCandValidatePVGroup(c clientset.Interface, ns string, pvols pvmap, claims pvcmap) {
func deletePVCandValidatePVGroup(c clientset.Interface, ns string, pvols pvmap, claims pvcmap, expectPVPhase v1.PersistentVolumePhase) {

var boundPVs, deletedPVCs int
var expctPVPhase v1.PersistentVolumePhase

for pvName := range pvols {
pv, err := c.Core().PersistentVolumes().Get(pvName, metav1.GetOptions{})
Expand All @@ -174,17 +175,7 @@ func deletePVCandValidatePVGroup(c clientset.Interface, ns string, pvols pvmap,
Expect(found).To(BeTrue())
pvc, err := c.Core().PersistentVolumeClaims(ns).Get(cr.Name, metav1.GetOptions{})
Expect(apierrs.IsNotFound(err)).To(BeFalse())

// what Phase do we expect the PV that was bound to the claim to
// be in after that claim is deleted?
expctPVPhase = v1.VolumeAvailable
if len(claims) > len(pvols) {
// there are excess pvcs so expect the previously bound
// PV to become bound again
expctPVPhase = v1.VolumeBound
}

deletePVCandValidatePV(c, ns, pvc, pv, expctPVPhase)
deletePVCandValidatePV(c, ns, pvc, pv, expectPVPhase)
delete(claims, pvcKey)
deletedPVCs++
}
Expand Down Expand Up @@ -453,24 +444,26 @@ func makePvcKey(ns, name string) types.NamespacedName {

// Returns a PV definition based on the nfs server IP. If the PVC is not nil
// then the PV is defined with a ClaimRef which includes the PVC's namespace.
// If the PVC is nil then the PV is not defined with a ClaimRef.
// If the PVC is nil then the PV is not defined with a ClaimRef. If no reclaimPolicy
// is assigned, assumes "Retain".
// Note: the passed-in claim does not have a name until it is created
// (instantiated) and thus the PV's ClaimRef cannot be completely filled-in in
// this func. Therefore, the ClaimRef's name is added later in
// createPVCPV.

func makePersistentVolume(pvConfig persistentVolumeConfig) *v1.PersistentVolume {
// Specs are expected to match this test's PersistentVolumeClaim

var claimRef *v1.ObjectReference

// If the reclaimPolicy is not provided, assume Retain
if pvConfig.reclaimPolicy == "" {
pvConfig.reclaimPolicy = v1.PersistentVolumeReclaimRetain
}
if pvConfig.prebind != nil {
claimRef = &v1.ObjectReference{
Name: pvConfig.prebind.Name,
Namespace: pvConfig.prebind.Namespace,
}
}

return &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
GenerateName: pvConfig.namePrefix,
Expand All @@ -479,7 +472,7 @@ func makePersistentVolume(pvConfig persistentVolumeConfig) *v1.PersistentVolume
},
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRecycle,
PersistentVolumeReclaimPolicy: pvConfig.reclaimPolicy,
Capacity: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
},
Expand Down