Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix problems of not-starting image pullers #43758

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/e2e/e2e.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
// #41007. To avoid those pods preventing the whole test runs (and just
// wasting the whole run), we allow for some not-ready pods (with the
// number equal to the number of allowed not-ready nodes).
if err := framework.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods), int32(framework.TestContext.AllowedNotReadyNodes), podStartupTimeout, framework.ImagePullerLabels, true); err != nil {
if err := framework.WaitForPodsRunningReady(c, metav1.NamespaceSystem, int32(framework.TestContext.MinStartupPods), int32(framework.TestContext.AllowedNotReadyNodes), podStartupTimeout, framework.ImagePullerLabels); err != nil {
framework.DumpAllNamespaceInfo(c, metav1.NamespaceSystem)
framework.LogFailedContainers(c, metav1.NamespaceSystem, framework.Logf)
runKubernetesServiceTestContainer(c, metav1.NamespaceDefault)
Expand Down
24 changes: 5 additions & 19 deletions test/e2e/framework/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -485,29 +485,19 @@ func WaitForPodsSuccess(c clientset.Interface, ns string, successPodLabels map[s
// ready. It has separate behavior from other 'wait for' pods functions in
// that it requests the list of pods on every iteration. This is useful, for
// example, in cluster startup, because the number of pods increases while
// waiting.
// If ignoreLabels is not empty, pods matching this selector are ignored and
// this function waits for minPods to enter Running/Ready and for all pods
// matching ignoreLabels to enter Success phase. Otherwise an error is returned
// even if there are minPods pods, some of which are in Running/Ready
// and some in Success. This is to allow the client to decide if "Success"
// means "Ready" or not.
// If skipSucceeded is true, any pods that are Succeeded are not counted.
func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedNotReadyPods int32, timeout time.Duration, ignoreLabels map[string]string, skipSucceeded bool) error {
// waiting. All pods that are in SUCCESS state are not counted.
//
// If ignoreLabels is not empty, pods matching this selector are ignored.
func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedNotReadyPods int32, timeout time.Duration, ignoreLabels map[string]string) error {
ignoreSelector := labels.SelectorFromSet(ignoreLabels)
start := time.Now()
Logf("Waiting up to %v for all pods (need at least %d) in namespace '%s' to be running and ready",
timeout, minPods, ns)
wg := sync.WaitGroup{}
wg.Add(1)
var waitForSuccessError error
var ignoreNotReady bool
badPods := []v1.Pod{}
desiredPods := 0
go func() {
waitForSuccessError = WaitForPodsSuccess(c, ns, ignoreLabels, timeout)
wg.Done()
}()

if wait.PollImmediate(Poll, timeout, func() (bool, error) {
// We get the new list of pods, replication controllers, and
Expand Down Expand Up @@ -554,7 +544,7 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN
switch {
case res && err == nil:
nOk++
case pod.Status.Phase == v1.PodSucceeded && skipSucceeded:
case pod.Status.Phase == v1.PodSucceeded:
continue
case pod.Status.Phase == v1.PodSucceeded:
Logf("The status of Pod %s is Succeeded which is unexpected", pod.ObjectMeta.Name)
Expand Down Expand Up @@ -590,10 +580,6 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods, allowedN
}
Logf("Number of not-ready pods is allowed.")
}
wg.Wait()
if waitForSuccessError != nil {
return waitForSuccessError
}
return nil
}

Expand Down
2 changes: 1 addition & 1 deletion test/e2e/mesos.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ var _ = framework.KubeDescribe("Mesos", func() {
nodelist := framework.GetReadySchedulableNodesOrDie(client)
const ns = "static-pods"
numpods := int32(len(nodelist.Items))
framework.ExpectNoError(framework.WaitForPodsRunningReady(client, ns, numpods, 0, wait.ForeverTestTimeout, map[string]string{}, false),
framework.ExpectNoError(framework.WaitForPodsRunningReady(client, ns, numpods, 0, wait.ForeverTestTimeout, map[string]string{}),
fmt.Sprintf("number of static pods in namespace %s is %d", ns, numpods))
})

Expand Down
2 changes: 1 addition & 1 deletion test/e2e/resize_nodes.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() {
// Many e2e tests assume that the cluster is fully healthy before they start. Wait until
// the cluster is restored to health.
By("waiting for system pods to successfully restart")
err := framework.WaitForPodsRunningReady(c, metav1.NamespaceSystem, systemPodsNo, 0, framework.PodReadyBeforeTimeout, ignoreLabels, true)
err := framework.WaitForPodsRunningReady(c, metav1.NamespaceSystem, systemPodsNo, 0, framework.PodReadyBeforeTimeout, ignoreLabels)
Expect(err).NotTo(HaveOccurred())
By("waiting for image prepulling pods to complete")
framework.WaitForPodsSuccess(c, metav1.NamespaceSystem, framework.ImagePullerLabels, imagePrePullingTimeout)
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/scheduling/predicates.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() {
}
}

err = framework.WaitForPodsRunningReady(cs, metav1.NamespaceSystem, int32(systemPodsNo), 0, framework.PodReadyBeforeTimeout, ignoreLabels, true)
err = framework.WaitForPodsRunningReady(cs, metav1.NamespaceSystem, int32(systemPodsNo), 0, framework.PodReadyBeforeTimeout, ignoreLabels)
Expect(err).NotTo(HaveOccurred())

for _, node := range nodeList.Items {
Expand Down