Check whether all nodes are healthy in e2e framework

hack/jenkins/e2e.sh

@@ -69,12 +69,12 @@ fi
 GCE_DEFAULT_SKIP_TEST_REGEX="Skipped|Density|Reboot|Restart|Example"
 # The following tests are known to be flaky, and are thus run only in their own
 # -flaky- build variants.
-GCE_FLAKY_TEST_REGEX="Elasticsearch|Proxy.*logs|Shell.*services|MaxPods.*"
+GCE_FLAKY_TEST_REGEX="Elasticsearch|Shell.*services|MaxPods.*"
 # Tests which are not able to be run in parallel.
 GCE_PARALLEL_SKIP_TEST_REGEX="${GCE_DEFAULT_SKIP_TEST_REGEX}|Etcd|NetworkingNew|Nodes\sNetwork|Nodes\sResize|MaxPods"
 # Tests which are known to be flaky when run in parallel.
 # TODO: figure out why GCE_FLAKY_TEST_REGEX is not a perfect subset of this list.
-GCE_PARALLEL_FLAKY_TEST_REGEX="Addon|Elasticsearch|Hostdir.*MOD|Networking.*intra|PD|Proxy.*logs|ServiceAccounts|Service\sendpoints\slatency|Services.*change\sthe\stype|Services.*functioning\sexternal\sload\sbalancer|Services.*identically\snamed|Services.*release.*load\sbalancer|Shell|multiport\sendpoints"
+GCE_PARALLEL_FLAKY_TEST_REGEX="Addon|Elasticsearch|Hostdir.*MOD|Networking.*intra|PD|ServiceAccounts|Service\sendpoints\slatency|Services.*change\sthe\stype|Services.*functioning\sexternal\sload\sbalancer|Services.*identically\snamed|Services.*release.*load\sbalancer|Shell|multiport\sendpoints"
 
 # Define environment variables based on the Jenkins project name.
 case ${JOB_NAME} in

test/e2e/framework.go

@@ -85,6 +85,11 @@ func (f *Framework) afterEach() {
 		// you may or may not see the killing/deletion/cleanup events.
 	}
 
+	// Check whether all nodes are ready after the test.
+	if err := allNodesReady(f.Client); err != nil {
+		Failf("All nodes should be ready after test, %v", err)
+	}
+
 	By(fmt.Sprintf("Destroying namespace %q for this suite.", f.Namespace.Name))
 	if err := f.Client.Namespaces().Delete(f.Namespace.Name); err != nil {
 		Failf("Couldn't delete ns %q: %s", f.Namespace.Name, err)

test/e2e/resize_nodes.go

@@ -412,6 +412,10 @@ var _ = Describe("Nodes", func() {
 	})
 
 	AfterEach(func() {
+		By("checking whether all nodes are healthy")
+		if err := allNodesReady(c); err != nil {
+			Failf("Not all nodes are ready: %v", err)
+		}
 		By(fmt.Sprintf("destroying namespace for this suite %s", ns))
 		if err := c.Namespaces().Delete(ns); err != nil {
 			Failf("Couldn't delete namespace '%s', %v", ns, err)

test/e2e/util.go

@@ -1461,6 +1461,22 @@ func waitForNodeToBe(c *client.Client, name string, wantReady bool, timeout time
 	return false
 }
 
+// checks whether all registered nodes are ready
+func allNodesReady(c *client.Client) error {
+	nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
+	Expect(err).NotTo(HaveOccurred())
+	var notReady []api.Node
+	for _, node := range nodes.Items {
+		if isNodeReadySetAsExpected(&node, false) {
+			notReady = append(notReady, node)
+		}
+	}
+	if len(notReady) > 0 {
+		return fmt.Errorf("Not ready nodes: %v", notReady)
+	}
+	return nil
+}
+
 // Filters nodes in NodeList in place, removing nodes that do not
 // satisfy the given condition
 // TODO: consider merging with pkg/client/cache.NodeLister