Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Speed up block volume e2e test by checking events #81196

Merged
merged 3 commits into from
Aug 21, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions test/e2e/common/container_probe.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,8 +259,8 @@ var _ = framework.KubeDescribe("Probing container", func() {
"involvedObject.namespace": f.Namespace.Name,
"reason": events.ContainerProbeWarning,
}.AsSelector().String()
framework.ExpectNoError(e2epod.WaitTimeoutForPodEvent(
f.ClientSet, pod.Name, f.Namespace.Name, expectedEvent, "0.0.0.0", framework.PodEventTimeout))
framework.ExpectNoError(WaitTimeoutForEvent(
f.ClientSet, f.Namespace.Name, expectedEvent, "0.0.0.0", framework.PodEventTimeout))
})
})

Expand Down
24 changes: 24 additions & 0 deletions test/e2e/common/events.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package common

import (
"fmt"
"strings"
"sync"
"time"

Expand All @@ -27,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/kubernetes/test/e2e/framework"

Expand Down Expand Up @@ -149,3 +151,25 @@ func ObserveEventAfterAction(f *framework.Framework, eventPredicate func(*v1.Eve
})
return err == nil, err
}

// WaitTimeoutForEvent waits the given timeout duration for an event to occur.
func WaitTimeoutForEvent(c clientset.Interface, namespace, eventSelector, msg string, timeout time.Duration) error {
interval := 2 * time.Second
return wait.PollImmediate(interval, timeout, eventOccurred(c, namespace, eventSelector, msg))
}

func eventOccurred(c clientset.Interface, namespace, eventSelector, msg string) wait.ConditionFunc {
options := metav1.ListOptions{FieldSelector: eventSelector}
return func() (bool, error) {
events, err := c.CoreV1().Events(namespace).List(options)
if err != nil {
return false, fmt.Errorf("got error while getting events: %v", err)
}
for _, event := range events.Items {
if strings.Contains(event.Message, msg) {
return true, nil
}
}
return false, nil
}
}
4 changes: 2 additions & 2 deletions test/e2e/common/runtimeclass.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,6 @@ func expectSandboxFailureEvent(f *framework.Framework, pod *v1.Pod, msg string)
"involvedObject.namespace": f.Namespace.Name,
"reason": events.FailedCreatePodSandBox,
}.AsSelector().String()
framework.ExpectNoError(e2epod.WaitTimeoutForPodEvent(
f.ClientSet, pod.Name, f.Namespace.Name, eventSelector, msg, framework.PodEventTimeout))
framework.ExpectNoError(WaitTimeoutForEvent(
f.ClientSet, f.Namespace.Name, eventSelector, msg, framework.PodEventTimeout))
}
22 changes: 0 additions & 22 deletions test/e2e/framework/pod/wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"bytes"
"errors"
"fmt"
"strings"
"sync"
"text/tabwriter"
"time"
Expand Down Expand Up @@ -348,27 +347,6 @@ func WaitForPodRunningInNamespace(c clientset.Interface, pod *v1.Pod) error {
return WaitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace, podStartTimeout)
}

// WaitTimeoutForPodEvent waits the given timeout duration for a pod event to occur.
func WaitTimeoutForPodEvent(c clientset.Interface, podName, namespace, eventSelector, msg string, timeout time.Duration) error {
return wait.PollImmediate(poll, timeout, eventOccurred(c, podName, namespace, eventSelector, msg))
}

func eventOccurred(c clientset.Interface, podName, namespace, eventSelector, msg string) wait.ConditionFunc {
options := metav1.ListOptions{FieldSelector: eventSelector}
return func() (bool, error) {
events, err := c.CoreV1().Events(namespace).List(options)
if err != nil {
return false, fmt.Errorf("got error while getting pod events: %s", err)
}
for _, event := range events.Items {
if strings.Contains(event.Message, msg) {
return true, nil
}
}
return false, nil
}
}

// WaitTimeoutForPodNoLongerRunningInNamespace waits the given timeout duration for the specified pod to stop.
func WaitTimeoutForPodNoLongerRunningInNamespace(c clientset.Interface, podName, namespace string, timeout time.Duration) error {
return wait.PollImmediate(poll, timeout, podCompleted(c, podName, namespace))
Expand Down
2 changes: 2 additions & 0 deletions test/e2e/storage/testsuites/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ go_library(
importpath = "k8s.io/kubernetes/test/e2e/storage/testsuites",
visibility = ["//visibility:public"],
deps = [
"//pkg/controller/volume/events:go_default_library",
"//pkg/kubelet/events:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/storage/v1:go_default_library",
Expand All @@ -38,6 +39,7 @@ go_library(
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/csi-translation-lib:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/log:go_default_library",
"//test/e2e/framework/metrics:go_default_library",
Expand Down
72 changes: 55 additions & 17 deletions test/e2e/storage/testsuites/volumemode.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
clientset "k8s.io/client-go/kubernetes"
volevents "k8s.io/kubernetes/pkg/controller/volume/events"
"k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
Expand Down Expand Up @@ -188,27 +190,47 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern

ginkgo.By("Creating sc")
l.sc, err = l.cs.StorageV1().StorageClasses().Create(l.sc)
framework.ExpectNoError(err)
framework.ExpectNoError(err, "Failed to create sc")

ginkgo.By("Creating pv and pvc")
l.pv, err = l.cs.CoreV1().PersistentVolumes().Create(l.pv)
framework.ExpectNoError(err)
framework.ExpectNoError(err, "Failed to create pv")

// Prebind pv
l.pvc.Spec.VolumeName = l.pv.Name
l.pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(l.pvc)
framework.ExpectNoError(err)
framework.ExpectNoError(err, "Failed to create pvc")

framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc))
framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc), "Failed to bind pv and pvc")

ginkgo.By("Creating pod")
pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
nil, false, "", false, false, framework.SELinuxLabel,
nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
pod := framework.MakeSecPod(l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc}, nil, false, "", false, false, framework.SELinuxLabel, nil)
// Setting node
pod.Spec.NodeName = l.config.ClientNodeName
pod, err = l.cs.CoreV1().Pods(l.ns.Name).Create(pod)
framework.ExpectNoError(err, "Failed to create pod")
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod), "Failed to delete pod")
}()
framework.ExpectError(err)

eventSelector := fields.Set{
"involvedObject.kind": "Pod",
"involvedObject.name": pod.Name,
"involvedObject.namespace": l.ns.Name,
"reason": events.FailedMountVolume,
}.AsSelector().String()
msg := "Unable to attach or mount volumes"

err = common.WaitTimeoutForEvent(l.cs, l.ns.Name, eventSelector, msg, framework.PodStartTimeout)
// Events are unreliable, don't depend on the event. It's used only to speed up the test.
if err != nil {
e2elog.Logf("Warning: did not get event about FailedMountVolume")
}

// Check the pod is still not running
p, err := l.cs.CoreV1().Pods(l.ns.Name).Get(pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "could not re-read the pod after event (or timeout)")
framework.ExpectEqual(p.Status.Phase, v1.PodPending, "Pod phase isn't pending")
})
}

Expand All @@ -222,14 +244,30 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern

ginkgo.By("Creating sc")
l.sc, err = l.cs.StorageV1().StorageClasses().Create(l.sc)
framework.ExpectNoError(err)
framework.ExpectNoError(err, "Failed to create sc")

ginkgo.By("Creating pv and pvc")
l.pvc, err = l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Create(l.pvc)
framework.ExpectNoError(err)
framework.ExpectNoError(err, "Failed to create pvc")

eventSelector := fields.Set{
"involvedObject.kind": "PersistentVolumeClaim",
"involvedObject.name": l.pvc.Name,
"involvedObject.namespace": l.ns.Name,
"reason": volevents.ProvisioningFailed,
}.AsSelector().String()
msg := "does not support block volume provisioning"

err = common.WaitTimeoutForEvent(l.cs, l.ns.Name, eventSelector, msg, framework.ClaimProvisionTimeout)
// Events are unreliable, don't depend on the event. It's used only to speed up the test.
if err != nil {
e2elog.Logf("Warning: did not get event about provisioing failed")
}

err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, l.cs, l.pvc.Namespace, l.pvc.Name, framework.Poll, framework.ClaimProvisionTimeout)
framework.ExpectError(err)
// Check the pvc is still pending
pvc, err := l.cs.CoreV1().PersistentVolumeClaims(l.ns.Name).Get(l.pvc.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "Failed to re-read the pvc after event (or timeout)")
framework.ExpectEqual(pvc.Status.Phase, v1.ClaimPending, "PVC phase isn't pending")
})
}
default:
Expand All @@ -250,9 +288,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern

// Run the pod
pod, err = l.cs.CoreV1().Pods(l.ns.Name).Create(pod)
framework.ExpectNoError(err)
framework.ExpectNoError(err, "Failed to create pod")
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod), "Failed to delete pod")
}()

ginkgo.By("Waiting for the pod to fail")
Expand All @@ -270,7 +308,7 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
} else {
msg = "has volumeMode Filesystem, but is specified in volumeDevices"
}
err = e2epod.WaitTimeoutForPodEvent(l.cs, pod.Name, l.ns.Name, eventSelector, msg, framework.PodStartTimeout)
err = common.WaitTimeoutForEvent(l.cs, l.ns.Name, eventSelector, msg, framework.PodStartTimeout)
// Events are unreliable, don't depend on them. They're used only to speed up the test.
if err != nil {
e2elog.Logf("Warning: did not get event about mismatched volume use")
Expand All @@ -279,7 +317,7 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
// Check the pod is still not running
p, err := l.cs.CoreV1().Pods(l.ns.Name).Get(pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "could not re-read the pod after event (or timeout)")
framework.ExpectEqual(p.Status.Phase, v1.PodPending)
framework.ExpectEqual(p.Status.Phase, v1.PodPending, "Pod phase isn't pending")
})

ginkgo.It("should not mount / map unused volumes in a pod", func() {
Expand Down