Add debug info of failedPods

- improve documentation
- use Should overall to be more coherent with gomega

test/README.md

@@ -2,11 +2,20 @@
 
 The test can be run locally or in CI.
 
+# Summary:
+
+- [Requirements](#requirements)
+- [Howto](#howto)
+- [Testing Guidelines](#testing-guidelines)
+- [Specification/API](#specification)
+
+
 # Requirements:
 
 - the infrastructure is already deployed. ( Terraform is not necessary a requirement, only the ips)
+  See mandatory variable for the number of instances required.
 
-# HOW TO RUN:
+# Howto 
 
 You can run the `e2e-tests` in 2 ways:
 
@@ -15,7 +24,6 @@ You can run the `e2e-tests` in 2 ways:
 `CONTROLPLANE=10.86.2.23 MASTER00=10.86.1.103 WORKER00=10.86.2.130 make test-e2e` 
 
 2) without IP but with a terraform state,  specify the platform you have deployed the infrastructure.
-
 `IP_FROM_TF_STATE=TRUE PLATFORM=openstack make test-e2e`
 
 This method will read the tfstate file and read the IPs of host and pass them to `ginkgo`
@@ -27,10 +35,26 @@ Boths methods are convenients: 1) method is usefull when we don't have the terra
 `~/go/src/github.com/SUSE/skuba> GINKGO_BIN_PATH="$PWD/ginkgo" IP_FROM_TF_STATE=TRUE PLATFORM=openstack make test-e2e`
 In the following example we assume you have builded ginkgo from vendor.
 
-# Env. Variable:
+# Testing Guidelines:
+
+## Assertions:
+
+- Use `Should`  for assertion`gomega.Expect(err).Should(gomega.BeNil()` over `to`.
 
-## Guidelines:
+- Use `Expect` over `Ω`
 
+## Misc:
+
+### Loops/Timeouts:
+
+Don't create a customized loop for checking each minute that a condition is satisfied. Instead use `eventually`.
+For examples, look in the test codebases,  we already use `eventually` in some tests.
+
+Upstream doc: https://onsi.github.io/gomega/#eventually
+
+# Specification
+
+## create new ENV variables
 Syntax: use `_` underscores for separating words and use UPPERCASE for naming.
 
 Adding NEW variables:
@@ -43,7 +67,8 @@ All needed variable should have been already implemented, only the NODE ips vari
 
 When you create a var specify also the behaviour and the usage: mandatory/optional and default value if any.
 
-### Currenlty supported:
+
+## Variables  supported:
 
 ### Cluster:
 
@@ -70,7 +95,7 @@ As showed, in future we will have `WORKER01`, `MASTER01`, `MASTER02` etc.
 SEE 2) example in HOW TO RUN
 
 
-# Internal Development:
+# Internal info
 
 The Env. variables are set by `ci/tasks/e2e-tests.py`.
 

test/core-features/00_cluster_init.go

@@ -43,55 +43,57 @@ var _ = ginkgo.Describe("Create Skuba Cluster", func() {
 		gomega.Eventually(session.Out).Should(gbytes.Say(".*configuration files written to"))
 
 		//TODO: this assertion should be Eventually and wait for Exit(0)
-		gomega.Expect(session).Should(gexec.Exit(), "configuration was not created")
-		gomega.Expect(err).To(gomega.BeNil(), "configuration was not created")
+		gomega.Expect(err).Should(gomega.BeNil(), "configuration was not created")
 
 		_, err = os.Stat(skuba.ClusterName())
-		gomega.Expect(os.IsNotExist(err)).NotTo(gomega.BeTrue())
+		gomega.Expect(os.IsNotExist(err)).ShouldNot(gomega.BeTrue())
 
 		ginkgo.By("add master00 to the cluster")
 		session, err = skuba.Bootstrap(master00IP, master00Name)
 
 		// hack: we wait in this print until the command to finish. (if removed the following cmd fails because command hasn't finished)
 		fmt.Println(session.Wait().Out.Contents())
 		gomega.Expect(session).Should(gexec.Exit(0), "skuba adding master00 failed")
-		gomega.Expect(err).To(gomega.BeNil(), "skuba adding master00 failed")
+		gomega.Expect(err).Should(gomega.BeNil(), "skuba adding master00 failed")
 
 		ginkgo.By("verify master00 with skuba status")
 		session, err = skuba.Status()
 
 		gomega.Eventually(session.Out).Should(gbytes.Say(".*" + master00Name))
 		gomega.Expect(session).Should(gexec.Exit(0), "skuba status verify master00 failed")
-		gomega.Expect(err).To(gomega.BeNil(), "skuba status verify master00 failed")
+		gomega.Expect(err).Should(gomega.BeNil(), "skuba status verify master00 failed")
 
 		ginkgo.By("add a worker00 to the cluster")
 		session, err = skuba.JoinNode(worker00IP, worker00Name, "worker")
 
 		// hack: we wait in this print until the command to finish. (if removed the following cmd fails because command hasn't finished)
 		fmt.Println(session.Wait().Out.Contents())
 		gomega.Expect(session).Should(gexec.Exit(0), "skuba adding worker00 failed")
-		gomega.Expect(err).To(gomega.BeNil(), "skuba adding worker00 failed")
+		gomega.Expect(err).Should(gomega.BeNil(), "skuba adding worker00 failed")
 
 		ginkgo.By("verify worker00 with skuba status")
 		session, err = skuba.Status()
 
 		gomega.Eventually(session.Out).Should(gbytes.Say(".*" + worker00Name))
 		gomega.Expect(session).Should(gexec.Exit(0), "skuba status verify worker00 failed")
-		gomega.Expect(err).To(gomega.BeNil(), "skuba status verify worker00 failed")
+		gomega.Expect(err).Should(gomega.BeNil(), "skuba status verify worker00 failed")
 
 		ginkgo.By("verify all system pods are running")
 		client, err := skuba.GetClient()
 		if err != nil {
 			panic(err)
 		}
-
+		// retrieve debug info of failedPods for printing in case of failure
+		var failedPods []v1.Pod
 		gomega.Eventually(func() []v1.Pod {
 			podList, err := client.CoreV1().Pods("kube-system").List(metav1.ListOptions{FieldSelector: "status.phase!=Running"})
 			if err != nil {
 				panic(err)
 			}
+			failedPods = podList.Items
 			return podList.Items
-		}, 500*time.Second, 3*time.Second).ShouldNot(gomega.HaveLen(0), "Some system pods are not in 'running' state")
+			// the timeout 700 must be generous enough because the startup time of different pods can vary a lot, causing flakiness if small timeout
+		}, 700*time.Second, 3*time.Second).ShouldNot(gomega.HaveLen(0), "Some system pods are not in 'running' state: %#v ", failedPods)
 	})
 
 })