Add documentation for KUDO integration tests and move into test/integ…

…ration. (#490)

* Add documentation for KUDO integration tests and move into test/integration.

* Add test/ to Dockerfile.

* Document --test since it is already merged.

* Improve comments around go test CLI arguments

* Move test data for test harness into pkg/test/test_data/.

CONTRIBUTING.md

@@ -41,46 +41,7 @@ After updating CRD manifests, use `make deploy` to apply the new CRDs to your cl
 
 #### Testing
 
-You can run the unit tests:
-
-```
-make test
-```
-
-Or the integration tests:
-
-```
-make integration-test
-```
-
-See [the testing documentation](https://kudo.dev/docs/testing) for more details.
-
-##### Debugging KUDO integration tests
-
-To debug the integration tests, run them locally:
-
-```
-go run ./cmd/kubectl-kudo test
-```
-
-By default, the test harness starts a local etcd and kube-apiserver for use in the tests. When debugging, it can be helpful to run your tests against a live cluster to make access easier:
-
-```
-go run ./cmd/kubectl-kudo test --start-control-plane=false
-```
-
-It also defaults to deleting any resources created during tests. You can configure the test harness to skip cleanup:
-
-```
-go run ./cmd/kubectl-kudo test --start-control-plane=false --skip-delete
-```
-
-### Build and run tests using Docker
-If you don't want to install kubebuilder and other dependencies of KUDO locally, you can build KUDO and run the tests inside a Docker container.
-
-To run tests inside a Docker container, you can just execute:
-
-`./test/run_tests.sh`
+See the [contributor's testing guide](https://github.com/kudobuilder/kudo/blob/master/test/README.md).
 
 ## Community, Discussion, and Support
 

kudo-test.yaml

@@ -1,8 +1,8 @@
 apiVersion: kudo.k8s.io/v1alpha1
 kind: TestSuite
 crdDir: ./config/crds/
-manifestsDir: ./config/samples/test-operator/
+manifestsDir: ./test/manifests/
 testDirs:
-- ./pkg/test/test_data/
+- ./test/integration
 startKUDO: true
 startControlPlane: true

pkg/kudoctl/cmd/test.go

@@ -23,13 +23,13 @@ var (
             kubectl kudo test --config test.yaml
 
       Run tests against an existing Kubernetes cluster:
-            kubectl kudo test ./pkg/test/test_data/
+            kubectl kudo test ./test/integration/
 
       Run tests against an existing Kubernetes cluster, and install KUDO, manifests, and CRDs for the tests:
-            kubectl kudo test --crd-dir ./config/crds/ --manifests-dir ./config/samples/test-operator/ ./pkg/test/test_data/
+            kubectl kudo test --crd-dir ./config/crds/ --manifests-dir ./test/manifests/ ./test/integration/
 
       Run a Kubernetes control plane and KUDO and install manifests and CRDs for the running tests:
-            kubectl kudo test --start-control-plane --start-kudo --crd-dir ./config/crds/ --manifests-dir ./config/samples/test-operator/ ./pkg/test/test_data/
+            kubectl kudo test --start-control-plane --start-kudo --crd-dir ./config/crds/ --manifests-dir ./test/manifests/ ./test/integration/
 `
 )
 

pkg/test/utils/testing.go

@@ -15,10 +15,12 @@ import (
 // If testToRun is set to a non-empty string, it is passed as a `-run` argument to the go test harness.
 func RunTests(testName string, testToRun string, testFunc func(*testing.T)) {
 	// Set the verbose test flag to true since we are not using the regular go test CLI.
-	flag.Lookup("test.v").Value.Set("true")
+	flag.Set("test.v", "true")
 
+	// Set the -run flag on the Go test harness.
+	// See the go test documentation: https://golang.org/pkg/cmd/go/internal/test/
 	if testToRun != "" {
-		flag.Lookup("test.run").Value.Set(fmt.Sprintf("//%s", testToRun))
+		flag.Set("test.run", fmt.Sprintf("//%s", testToRun))
 	}
 
 	os.Exit(testing.MainStart(&testDeps{}, []testing.InternalTest{

test/Dockerfile

@@ -7,6 +7,7 @@ RUN make download
 COPY config/ config/
 COPY pkg/ pkg/
 COPY cmd/ cmd/
+COPY test/ test/
 COPY kudo-test.yaml kudo-test.yaml
 
 ENTRYPOINT make integration-test

test/README.md

@@ -0,0 +1,75 @@
+# Setup
+
+To setup the tests locally, you need either:
+
+* Docker
+
+Or:
+
+* Go
+* kubebuilder
+
+## Downloading kubebuilder
+
+To setup kubebuilder, fetch the latest release from [Github](https://github.com/kubernetes-sigs/kubebuilder/releases) and extract `etcd` and `kube-apiserver` into `/usr/local/kubebuilder/bin/`.
+
+# Docker only
+
+If you don't want to install kubebuilder and other dependencies of KUDO locally, you can build KUDO and run the tests inside a Docker container.
+
+To run tests inside a Docker container, you can just execute:
+
+`./test/run_tests.sh`
+
+
+# Without Docker
+
+## Running unit tests
+
+Unit tests are written for KUDO using the standard Go testing library. You can run the unit tests:
+
+```
+make test
+```
+
+## Running integration tests
+
+Or run all tests:
+
+```
+make integration-test
+```
+
+## Declarative tests
+
+Most tests written for KUDO use the [declarative test harness](https://kudo.dev/docs/testing) with the controller-runtime's envtest (which starts `etcd` and `kube-apiserver` locally). This means that tests can be written for and run against KUDO without requiring a Kubernetes cluster (or even Docker).
+
+The test suite is configured by `kudo-test.yaml` and the tests live in `test/integration/`. Prior to running the tests, all KUDO CRDs and manifests in `test/manifests/` are installed into the test cluster.
+
+The test harness also starts KUDO, so it is recommended to use `go run` to run the test suite as this will include the latest built changes from your KUDO checkout.
+
+### CLI examples
+
+Run all integration tests:
+
+```
+go run ./cmd/kubectl-kudo test
+```
+
+Run a specific integration test (e.g., the `patch` test from `test/integration/patch`):
+
+```
+go run ./cmd/kubectl-kudo test --test patch
+```
+
+Run tests against a live cluster:
+
+```
+go run ./cmd/kubectl-kudo test --start-control-plane=false
+```
+
+Run tests against a live cluster and do not delete resources after running:
+
+```
+go run ./cmd/kubectl-kudo test --start-control-plane=false --skip-delete
+```

test/integration/list-pods/00-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: nginx
+spec:
+  containers:
+  - name: nginx
+    image: nginx:1.7.9

test/integration/list-pods/00-deployment.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9