Permalink
Browse files

Merge pull request #68581 from davidz627/fix/pdTestLeak

Fixed GCE PD tests to cleanup pods and disk properly after testing
  • Loading branch information...
k8s-ci-robot committed Sep 13, 2018
2 parents efe960c + e38aff2 commit 7eab6a49736cc7b01869a15f9f05dc5b49efb9fc
Showing with 24 additions and 42 deletions.
  1. +11 −8 test/e2e/framework/pv_util.go
  2. +7 −24 test/e2e/framework/volume_util.go
  3. +6 −10 test/e2e/storage/volumes.go
@@ -502,22 +502,25 @@ func testPodSuccessOrFail(c clientset.Interface, ns string, pod *v1.Pod) error {
// Deletes the passed-in pod and waits for the pod to be terminated. Resilient to the pod
// not existing.
func DeletePodWithWait(f *Framework, c clientset.Interface, pod *v1.Pod) error {
return DeletePodWithWaitByName(f, c, pod.GetName(), pod.GetNamespace())
}
// Deletes the named and namespaced pod and waits for the pod to be terminated. Resilient to the pod
// not existing.
func DeletePodWithWaitByName(f *Framework, c clientset.Interface, podName, podNamespace string) error {
const maxWait = 5 * time.Minute
if pod == nil {
return nil
}
Logf("Deleting pod %q in namespace %q", pod.Name, pod.Namespace)
err := c.CoreV1().Pods(pod.Namespace).Delete(pod.Name, nil)
Logf("Deleting pod %q in namespace %q", podName, podNamespace)
err := c.CoreV1().Pods(podNamespace).Delete(podName, nil)
if err != nil {
if apierrs.IsNotFound(err) {
return nil // assume pod was already deleted
}
return fmt.Errorf("pod Delete API error: %v", err)
}
Logf("Wait up to %v for pod %q to be fully deleted", maxWait, pod.Name)
err = f.WaitForPodNotFound(pod.Name, maxWait)
Logf("Wait up to %v for pod %q to be fully deleted", maxWait, podName)
err = f.WaitForPodNotFound(podName, maxWait)
if err != nil {
return fmt.Errorf("pod %q was not deleted: %v", pod.Name, err)
return fmt.Errorf("pod %q was not deleted: %v", podName, err)
}
return nil
}
@@ -50,7 +50,6 @@ import (
clientset "k8s.io/client-go/kubernetes"
imageutils "k8s.io/kubernetes/test/utils/image"
"github.com/golang/glog"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
@@ -335,30 +334,13 @@ func VolumeTestCleanup(f *Framework, config VolumeTestConfig) {
defer GinkgoRecover()
client := f.ClientSet
podClient := client.CoreV1().Pods(config.Namespace)
err := podClient.Delete(config.Prefix+"-client", nil)
if err != nil {
// Log the error before failing test: if the test has already failed,
// framework.ExpectNoError() won't print anything to logs!
glog.Warningf("Failed to delete client pod: %v", err)
ExpectNoError(err, "Failed to delete client pod: %v", err)
}
err := DeletePodWithWaitByName(f, client, config.Prefix+"-client", config.Namespace)
Expect(err).To(BeNil(), "Failed waiting for pod %v to be not running in namespace %v", config.Prefix+"-client", config.Namespace)
if config.ServerImage != "" {
if err := f.WaitForPodTerminated(config.Prefix+"-client", ""); !apierrs.IsNotFound(err) {
ExpectNoError(err, "Failed to wait client pod terminated: %v", err)
}
// See issue #24100.
// Prevent umount errors by making sure making sure the client pod exits cleanly *before* the volume server pod exits.
By("sleeping a bit so client can stop and unmount")
time.Sleep(20 * time.Second)
err = podClient.Delete(config.Prefix+"-server", nil)
if err != nil {
glog.Warningf("Failed to delete server pod: %v", err)
ExpectNoError(err, "Failed to delete server pod: %v", err)
}
err := DeletePodWithWaitByName(f, client, config.Prefix+"-server", config.Namespace)
Expect(err).To(BeNil(), "Failed waiting for pod %v to be not running in namespace %v", config.Prefix+"-server", config.Namespace)
}
}
@@ -448,7 +430,7 @@ func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGro
// Insert index.html with given content into given volume. It does so by
// starting and auxiliary pod which writes the file there.
// The volume must be writable.
func InjectHtml(client clientset.Interface, config VolumeTestConfig, volume v1.VolumeSource, content string) {
func InjectHtml(f *Framework, client clientset.Interface, config VolumeTestConfig, volume v1.VolumeSource, content string) {
By(fmt.Sprint("starting ", config.Prefix, " injector"))
podClient := client.CoreV1().Pods(config.Namespace)
@@ -495,7 +477,8 @@ func InjectHtml(client clientset.Interface, config VolumeTestConfig, volume v1.V
}
defer func() {
podClient.Delete(config.Prefix+"-injector", nil)
err := DeletePodWithWaitByName(f, client, injectPod.GetName(), config.Namespace)
Expect(err).To(BeNil(), "Failed deleting pod %v in namespace %v", injectPod.GetName(), config.Namespace)
}()
injectPod, err := podClient.Create(injectPod)
@@ -51,7 +51,6 @@ import (
. "github.com/onsi/gomega"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
clientset "k8s.io/client-go/kubernetes"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
"k8s.io/kubernetes/test/e2e/framework"
@@ -384,7 +383,7 @@ var _ = utils.SIGDescribe("Volumes", func() {
},
}
framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
framework.InjectHtml(f, cs, config, tests[0].Volume, tests[0].ExpectedContent)
fsGroup := int64(1234)
framework.TestVolumeClient(cs, config, &fsGroup, tests)
@@ -537,7 +536,7 @@ var _ = utils.SIGDescribe("Volumes", func() {
},
}
framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
framework.InjectHtml(f, cs, config, tests[0].Volume, tests[0].ExpectedContent)
fsGroup := int64(1234)
framework.TestVolumeClient(cs, config, &fsGroup, tests)
@@ -586,7 +585,7 @@ var _ = utils.SIGDescribe("Volumes", func() {
},
}
framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
framework.InjectHtml(f, cs, config, tests[0].Volume, tests[0].ExpectedContent)
fsGroup := int64(1234)
framework.TestVolumeClient(cs, config, &fsGroup, tests)
@@ -599,11 +598,8 @@ func testGCEPD(f *framework.Framework, config framework.VolumeTestConfig, cs cli
volumeName, err := framework.CreatePDWithRetry()
Expect(err).NotTo(HaveOccurred())
defer func() {
// - Get NodeName from the pod spec to which the volume is mounted.
// - Force detach and delete.
pod, err := f.PodClient().Get(config.Prefix+"-client", metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred(), "Failed getting pod %q.", config.Prefix+"-client")
detachAndDeletePDs(volumeName, []types.NodeName{types.NodeName(pod.Spec.NodeName)})
err := framework.DeletePDWithRetry(volumeName)
Expect(err).NotTo(HaveOccurred(), "Failed deleting PD.", volumeName)
}()
defer func() {
@@ -627,7 +623,7 @@ func testGCEPD(f *framework.Framework, config framework.VolumeTestConfig, cs cli
},
}
framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
framework.InjectHtml(f, cs, config, tests[0].Volume, tests[0].ExpectedContent)
fsGroup := int64(1234)
framework.TestVolumeClient(cs, config, &fsGroup, tests)

0 comments on commit 7eab6a4

Please sign in to comment.