e2e: Test that all operators are stable

This should catch flakes or failure retries
wking · Feb 15, 2019 · 6300a87 · 6300a87
1 parent bb5f71c
commit 6300a87
Showing 1 changed file with 71 additions and 62 deletions.
diff --git a/test/extended/operators/operators.go b/test/extended/operators/operators.go
@@ -80,80 +80,89 @@ var _ = g.Describe("[Feature:Platform][Smoke] Managed cluster should", func() {
 
 		// gate on all clusteroperators being ready
 		available := make(map[string]struct{})
-		for _, group := range []string{"config.openshift.io"} {
-			g.By(fmt.Sprintf("waiting for all cluster operators in %s to be available", group))
-			coc := dc.Resource(schema.GroupVersionResource{Group: group, Resource: "clusteroperators", Version: "v1"})
+		g.By(fmt.Sprintf("waiting for all cluster operators to be stable at the same time"))
+		coc := dc.Resource(schema.GroupVersionResource{Group: "config.openshift.io", Resource: "clusteroperators", Version: "v1"})
+		lastErr = nil
+		var lastCOs []objx.Map
+		wait.PollImmediate(time.Second, operatorWait, func() (bool, error) {
+			obj, err := coc.List(metav1.ListOptions{})
+			if err != nil {
+				lastErr = err
+				e2e.Logf("Unable to check for cluster operators: %v", err)
+				return false, nil
+			}
+			cv := objx.Map(obj.UnstructuredContent())
 			lastErr = nil
-			var lastCOs []objx.Map
-			wait.PollImmediate(time.Second, operatorWait, func() (bool, error) {
-				obj, err := coc.List(metav1.ListOptions{})
-				if err != nil {
-					lastErr = err
-					e2e.Logf("Unable to check for cluster operators: %v", err)
-					return false, nil
-				}
-				cv := objx.Map(obj.UnstructuredContent())
-				lastErr = nil
-				items := objects(cv.Get("items"))
-				lastCOs = items
-
-				// TODO: make this an error condition once we know at least one cluster operator status is reported
-				if len(items) == 0 {
-					e2e.Logf("No cluster operators registered in %s", group)
-					return true, nil
-				}
+			items := objects(cv.Get("items"))
+			lastCOs = items
 
-				var unavailable []objx.Map
-				var unavailableNames []string
-				for _, co := range items {
-					if condition(co, "Available").Get("status").String() != "True" {
-						ns := co.Get("metadata.namespace").String()
-						name := co.Get("metadata.name").String()
-						unavailableNames = append(unavailableNames, fmt.Sprintf("%s/%s", ns, name))
-						unavailable = append(unavailable, co)
-						break
-					}
-				}
-				if len(unavailable) > 0 {
-					e2e.Logf("Operators in group %s still unavailable: %s", group, strings.Join(unavailableNames, ", "))
-					return false, nil
-				}
-				return true, nil
-			})
+			if len(items) == 0 {
+				return false, nil
+			}
 
-			o.Expect(lastErr).NotTo(o.HaveOccurred())
-			var unavailable []string
-			buf := &bytes.Buffer{}
-			w := tabwriter.NewWriter(buf, 0, 4, 1, ' ', 0)
-			fmt.Fprintf(w, "NAMESPACE\tNAME\tPROGRESSING\tAVAILABLE\tVERSION\tMESSAGE\n")
-			for _, co := range lastCOs {
-				ns := co.Get("metadata.namespace").String()
-				name := co.Get("metadata.name").String()
+			var unavailable []objx.Map
+			var unavailableNames []string
+			for _, co := range items {
 				if condition(co, "Available").Get("status").String() != "True" {
-					unavailable = append(unavailable, fmt.Sprintf("%s/%s", ns, name))
-				} else {
-					available[fmt.Sprintf("%s/%s", ns, name)] = struct{}{}
+					ns := co.Get("metadata.namespace").String()
+					name := co.Get("metadata.name").String()
+					unavailableNames = append(unavailableNames, fmt.Sprintf("%s/%s", ns, name))
+					unavailable = append(unavailable, co)
+					break
+				}
+				if condition(co, "Progressing").Get("status").String() != "False" {
+					ns := co.Get("metadata.namespace").String()
+					name := co.Get("metadata.name").String()
+					unavailableNames = append(unavailableNames, fmt.Sprintf("%s/%s", ns, name))
+					unavailable = append(unavailable, co)
+					break
+				}
+				if condition(co, "Failing").Get("status").String() != "False" {
+					ns := co.Get("metadata.namespace").String()
+					name := co.Get("metadata.name").String()
+					unavailableNames = append(unavailableNames, fmt.Sprintf("%s/%s", ns, name))
+					unavailable = append(unavailable, co)
+					break
 				}
-				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
-					ns,
-					name,
-					condition(co, "Progressing").Get("status").String(),
-					condition(co, "Available").Get("status").String(),
-					co.Get("status.version").String(),
-					condition(co, "Failing").Get("message").String(),
-				)
 			}
-			w.Flush()
-			e2e.Logf("ClusterOperators:\n%s", buf.String())
-			// TODO: make this an e2e.Failf()
 			if len(unavailable) > 0 {
-				e2e.Logf("Some cluster operators never became available %s", strings.Join(unavailable, ", "))
+				e2e.Logf("Operators still doing work: %s", strings.Join(unavailableNames, ", "))
+				return false, nil
+			}
+			return true, nil
+		})
+
+		o.Expect(lastErr).NotTo(o.HaveOccurred())
+		var unavailable []string
+		buf := &bytes.Buffer{}
+		w := tabwriter.NewWriter(buf, 0, 4, 1, ' ', 0)
+		fmt.Fprintf(w, "NAMESPACE\tNAME\tPROGRESSING\tAVAILABLE\tVERSION\tMESSAGE\n")
+		for _, co := range lastCOs {
+			ns := co.Get("metadata.namespace").String()
+			name := co.Get("metadata.name").String()
+			if condition(co, "Available").Get("status").String() != "True" {
+				unavailable = append(unavailable, fmt.Sprintf("%s/%s", ns, name))
+			} else {
+				available[fmt.Sprintf("%s/%s", ns, name)] = struct{}{}
 			}
+			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
+				ns,
+				name,
+				condition(co, "Progressing").Get("status").String(),
+				condition(co, "Available").Get("status").String(),
+				co.Get("status.version").String(),
+				condition(co, "Failing").Get("message").String(),
+			)
 		}
+		w.Flush()
+		e2e.Logf("ClusterOperators:\n%s", buf.String())
 
+		if len(unavailable) > 0 {
+			e2e.Failf("Some cluster operators never became available %s", strings.Join(unavailable, ", "))
+		}
 		// Check at least one core operator is available
 		if len(available) == 0 {
-			e2e.Failf("None of the required core operators are available")
+			e2e.Failf("There must be at least one cluster operator")
 		}
 	})
 })