Support listing objects in test harness

docs/testing.md

@@ -17,6 +17,7 @@ KUDO uses a declarative integration testing harness for testing itself and Opera
    * [Test case directory structure](#test-case-directory-structure)
    * [Test steps](#test-steps)
      * [Test assertions](#test-assertions)
+       * [Listing objects](#listing-objects)
        * [Advanced test assertions](#advanced-test-assertions)
 * [Further Reading](#further-reading)
 
@@ -148,6 +149,22 @@ status:
 
 This watches an `Instance` called `zk` to have its status set to `COMPLETE` and it expects a `StatefulSet` to also be created called `zk-zk` and it waits for all `Pods` in the `StatefulSet` to be ready.
 
+##### Listing objects
+
+If the object `name` is omitted from the object metadata, it is possible to list objects and verify that one of them matches the desired state. This can be useful, for example, to check the `Pods` created by a `Deployment`.
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: nginx
+status:
+  phase: Running
+```
+
+This would verify that a pod with the `app=nginx` label is running.
+
 ##### Advanced test assertions
 
 The test harness recognizes special `TestAssert` objects defined in the assert file. If present, they override default settings of the test assert.

go.mod

@@ -6,6 +6,7 @@ require (
 	cloud.google.com/go v0.38.0 // indirect
 	github.com/Masterminds/goutils v1.1.0 // indirect
 	github.com/Masterminds/semver v1.4.2
+	github.com/davecgh/go-spew v1.1.1
 	github.com/dustinkirkland/golang-petname v0.0.0-20170921220637-d3c2ba80e75e
 	github.com/emicklei/go-restful v2.9.0+incompatible // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect

keps/0008-operator-testing.md

@@ -260,7 +260,7 @@ The test harness will wait for the `Pod` with name `test` in the namespace gener
 
 ##### Resources that have non-deterministic names
 
-Because some resource types have non-deterministic names (for example, the `Pods` created for a `Deployment`), if a resource has no name and only labels then the harness will list the resources of that type and wait for a resource of that type to match the state defined.
+Because some resource types have non-deterministic names (for example, the `Pods` created for a `Deployment`), if a resource has no name then the harness will list the resources of that type and wait for a resource of that type to match the state defined.
 
 For example, given an assertion file containing:
 

pkg/test/case_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -163,6 +164,61 @@ func TestLoadTestSteps(t *testing.T) {
 				},
 			},
 		},
+		{
+			"test_data/list-pods",
+			[]Step{
+				{
+					Name:  "deployment",
+					Index: 0,
+					Apply: []runtime.Object{
+						testutils.WithSpec(testutils.NewResource("apps/v1", "Deployment", "nginx-deployment", ""), map[string]interface{}{
+							"selector": map[string]interface{}{
+								"matchLabels": map[string]interface{}{
+									"app": "nginx",
+								},
+							},
+							"template": map[string]interface{}{
+								"metadata": map[string]interface{}{
+									"labels": map[string]interface{}{
+										"app": "nginx",
+									},
+								},
+								"spec": map[string]interface{}{
+									"containers": []map[string]interface{}{
+										{
+											"name":  "nginx",
+											"image": "nginx:1.7.9",
+										},
+									},
+								},
+							},
+						}),
+					},
+					Asserts: []runtime.Object{
+						&unstructured.Unstructured{
+							Object: map[string]interface{}{
+								"apiVersion": "v1",
+								"kind":       "Pod",
+								"metadata": map[string]interface{}{
+									"labels": map[string]interface{}{
+										"app": "nginx",
+									},
+								},
+								"spec": map[string]interface{}{
+									"containers": []interface{}{
+										map[string]interface{}{
+											"image": "nginx:1.7.9",
+											"name":  "nginx",
+										},
+									},
+								},
+							},
+						},
+					},
+					Errors: []runtime.Object{},
+				},
+			},
+		},
 	} {
 		t.Run(tt.path, func(t *testing.T) {
 			test := &Case{Dir: tt.path}
@@ -177,6 +233,10 @@ func TestLoadTestSteps(t *testing.T) {
 
 			assert.Equal(t, len(tt.testSteps), len(testStepsVal))
 			for index := range tt.testSteps {
+				assert.Equal(t, tt.testSteps[index].Apply, testStepsVal[index].Apply)
+				assert.Equal(t, tt.testSteps[index].Asserts, testStepsVal[index].Asserts)
+				assert.Equal(t, tt.testSteps[index].Errors, testStepsVal[index].Errors)
+				assert.Equal(t, tt.testSteps[index].Step, testStepsVal[index].Step)
 				assert.Equal(t, tt.testSteps[index], testStepsVal[index])
 			}
 		})
@@ -211,6 +271,15 @@ func TestCollectTestStepFiles(t *testing.T) {
 				},
 			},
 		},
+		{
+			"test_data/list-pods",
+			map[int64][]string{
+				int64(0): {
+					"test_data/list-pods/00-assert.yaml",
+					"test_data/list-pods/00-deployment.yaml",
+				},
+			},
+		},
 	} {
 		t.Run(tt.path, func(t *testing.T) {
 			test := &Case{Dir: tt.path}

pkg/test/step.go

@@ -92,13 +92,34 @@ func (s *Step) CheckResource(expected runtime.Object, namespace string) []error
 
 	gvk := expected.GetObjectKind().GroupVersionKind()
 
-	actual := &unstructured.Unstructured{}
-	actual.SetGroupVersionKind(gvk)
+	actuals := []*unstructured.Unstructured{}
 
-	err = s.Client.Get(context.TODO(), client.ObjectKey{
-		Namespace: namespace,
-		Name:      name,
-	}, actual)
+	if name != "" {
+		actual := &unstructured.Unstructured{}
+		actual.SetGroupVersionKind(gvk)
+
+		err = s.Client.Get(context.TODO(), client.ObjectKey{
+			Namespace: namespace,
+			Name:      name,
+		}, actual)
+
+		actuals = append(actuals, actual)
+	} else {
+		actual := &unstructured.UnstructuredList{}
+		actual.SetGroupVersionKind(gvk)
+
+		listOptions := []client.ListOptionFunc{}
+
+		if namespace != "" {
+			listOptions = append(listOptions, client.InNamespace(namespace))
+		}
+
+		err = s.Client.List(context.TODO(), actual, listOptions...)
+
+		for _, item := range actual.Items {
+			actuals = append(actuals, &item)
+		}
+	}
 	if err != nil {
 		return append(testErrors, err)
 	}
@@ -108,15 +129,25 @@ func (s *Step) CheckResource(expected runtime.Object, namespace string) []error
 		return append(testErrors, err)
 	}
 
-	if err := testutils.IsSubset(expectedObj, actual.UnstructuredContent()); err != nil {
-		diff, diffErr := testutils.PrettyDiff(expected, actual)
-		if diffErr == nil {
-			testErrors = append(testErrors, errors.New(diff))
-		} else {
-			testErrors = append(testErrors, diffErr)
+	for _, actual := range actuals {
+		tmpTestErrors := []error{}
+
+		if err := testutils.IsSubset(expectedObj, actual.UnstructuredContent()); err != nil {
+			diff, diffErr := testutils.PrettyDiff(expected, actual)
+			if diffErr == nil {
+				tmpTestErrors = append(tmpTestErrors, errors.New(diff))
+			} else {
+				tmpTestErrors = append(tmpTestErrors, diffErr)
+			}
+
+			tmpTestErrors = append(tmpTestErrors, fmt.Errorf("resource %s: %s", testutils.ResourceID(expected), err))
+		}
+
+		if len(tmpTestErrors) == 0 {
+			return tmpTestErrors
 		}
 
-		testErrors = append(testErrors, fmt.Errorf("resource %s: %s", testutils.ResourceID(expected), err))
+		testErrors = append(testErrors, tmpTestErrors...)
 	}
 
 	return testErrors