Integrate KUDO test harness into kudoctl. (#366)

* DCOS-54808 - Integrate KUDO test harness into kudoctl.

* Improve kudoctl test description

Makefile

@@ -28,6 +28,7 @@ test:
 # Run integration tests
 integration-test:
 	go test -tags integration ./pkg/... ./cmd/... -v -mod=readonly -coverprofile cover-integration.out
+	go run ./cmd/kubectl-kudo test
 
 .PHONY: test-clean
 # Clean test reports

keps/0008-framework-testing.md

@@ -130,6 +130,33 @@ Once all of the test steps in a test case have run, a report for the test case i
 
 The test harness can also be run in a unit testing mode, where it spins up a dummy Kubernetes API server and runs any test cases marked as unit tests.
 
+### Test suite configuration
+
+The test suite is configured either using command line arguments to `kubectl kudo test` or a provided configuration file.
+
+The configuration format is a YAML file containing the following struct:
+
+```
+type TestSuite struct {
+	TypeMeta
+	ObjectMeta
+
+	// Path to CRDs to install before running tests.
+	CRDDir            string
+	// Path to manifests to install before running tests.
+	ManifestsDir      string
+	// Directories containing test cases to run.
+	TestDirs          []string
+	// Whether or not to start a local etcd and kubernetes API server for the tests.
+	StartControlPlane bool
+	// Whether or not to start the KUDO controller for the tests.
+	StartKUDO         bool
+}
+
+```
+
+A configuration file can be provided to `kubectl kudo test` using the `--config` argument.
+
 ### Test suite directory structure
 
 Test suites are defined in an easy to understand directory structure:

kudo-test.yaml

@@ -0,0 +1,8 @@
+apiVersion: kudo.k8s.io/v1alpha1
+kind: TestSuite
+crdDir: ./config/crds/
+manifestsDir: ./config/samples/test-framework/
+testDirs:
+- ./pkg/test/test_data/
+startKUDO: true
+startControlPlane: true

pkg/apis/kudo/v1alpha1/test_types.go

@@ -7,6 +7,27 @@ import (
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
+// TestSuite configures which tests should be loaded.
+type TestSuite struct {
+	// The type meta object, should always be a GVK of kudo.k8s.io/v1alpha1/TestSuite.
+	metav1.TypeMeta `json:",inline"`
+	// Set labels or the test suite name.
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	// Path to CRDs to install before running tests.
+	CRDDir string `json:"crdDir"`
+	// Path to manifests to install before running tests.
+	ManifestsDir string `json:"manifestsDir"`
+	// Directories containing test cases to run.
+	TestDirs []string `json:"testDirs"`
+	// Whether or not to start a local etcd and kubernetes API server for the tests.
+	StartControlPlane bool `json:"startControlPlane"`
+	// Whether or not to start the KUDO controller for the tests.
+	StartKUDO bool `json:"startKUDO"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
 // TestStep settings to apply to a test step.
 type TestStep struct {
 	// The type meta object, should always be a GVK of kudo.k8s.io/v1alpha1/TestStep.

pkg/kudoctl/cmd/root.go

@@ -25,6 +25,9 @@ and serves as an API aggregation layer.
 	# View all plan history of a specific package
 	kubectl kudo plan history [flags]
 
+	# Run integration tests against a Kubernetes cluster or mocked control plane.
+	kubectl kudo test
+
 	# Get instances
 	kubectl kudo get instances [flags]
 
@@ -41,6 +44,7 @@ and serves as an API aggregation layer.
 	cmd.AddCommand(newInstallCmd())
 	cmd.AddCommand(newGetCmd())
 	cmd.AddCommand(newPlanCmd())
+	cmd.AddCommand(newTestCmd())
 	cmd.AddCommand(newVersionCmd())
 
 	return cmd

pkg/kudoctl/cmd/test.go

@@ -0,0 +1,114 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"reflect"
+	"testing"
+
+	kudo "github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
+	"github.com/kudobuilder/kudo/pkg/test"
+	testutils "github.com/kudobuilder/kudo/pkg/test/utils"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	testExample = `
+      Run tests configured by kudo-test.yaml:
+            kubectl kudo test
+
+      Load a specific test configuration:
+            kubectl kudo test --config test.yaml
+
+      Run tests against an existing Kubernetes cluster:
+            kubectl kudo test ./pkg/test/test_data/
+
+      Run tests against an existing Kubernetes cluster, and install KUDO, manifests, and CRDs for the tests:
+            kubectl kudo test --crd-dir ./config/crds/ --manifests-dir ./config/samples/test-framework/ ./pkg/test/test_data/
+
+      Run a Kubernetes control plane and KUDO and install manifests and CRDs for the running tests:
+            kubectl kudo test --start-control-plane --start-kudo --crd-dir ./config/crds/ --manifests-dir ./config/samples/test-framework/ ./pkg/test/test_data/
+`
+)
+
+// newTestCmd creates the test command for the CLI
+func newTestCmd() *cobra.Command {
+	configPath := ""
+	options := kudo.TestSuite{}
+	defaults := kudo.TestSuite{
+		TestDirs: []string{},
+	}
+
+	testCmd := &cobra.Command{
+		Use:   "test [flags]... [test directories]...",
+		Short: "-> Test KUDO and Frameworks.",
+		Long: `Runs integration tests against a Kubernetes cluster.
+
+The test framework supports connecting to an existing Kubernetes cluster or it can start a Kubernetes API server during the test run.
+
+It can also start up KUDO and apply manifests before running the tests.
+
+If no arguments are provided, the test harness will attempt to load the test configuration from kudo-test.yaml.
+
+For more detailed documentation, visit: https://kudo.dev/docs/testing`,
+		Example: testExample,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			options.TestDirs = args
+
+			if configPath == "" && reflect.DeepEqual(options, defaults) {
+				if _, err := os.Stat("kudo-test.yaml"); err == nil {
+					configPath = "kudo-test.yaml"
+				} else {
+					return fmt.Errorf("kudo-test.yaml not found, provide either --config or arguments indicating the tests to load")
+				}
+			}
+
+			if configPath != "" && !reflect.DeepEqual(options, defaults) {
+				return fmt.Errorf("provide either --config or other arguments, but not both")
+			}
+
+			if configPath != "" {
+				objects, err := testutils.LoadYAML(configPath)
+				if err != nil {
+					return err
+				}
+
+				for _, obj := range objects {
+					kind := obj.GetObjectKind().GroupVersionKind().Kind
+
+					if kind == "TestSuite" {
+						options = *obj.(*kudo.TestSuite)
+					} else {
+						log.Println(fmt.Errorf("unknown object type: %s", kind))
+					}
+				}
+			}
+
+			if len(options.TestDirs) == 0 {
+				return fmt.Errorf("no test directories provided, please provide either --config or test directories on the command line")
+			}
+
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			testutils.RunTests("kudo", func(t *testing.T) {
+				harness := test.Harness{
+					TestSuite: options,
+					T:         t,
+				}
+
+				harness.Run()
+			})
+		},
+	}
+
+	testCmd.Flags().StringVar(&configPath, "config", "", "Path to file to load test settings from (must not be set with any other arguments).")
+	testCmd.Flags().StringVar(&options.CRDDir, "crd-dir", "", "Directory to load CustomResourceDefinitions from prior to running the tests.")
+	testCmd.Flags().StringVar(&options.ManifestsDir, "manifests-dir", "", "A directory containing manifests to apply before running the tests.")
+	testCmd.Flags().BoolVar(&options.StartControlPlane, "start-control-plane", false, "Start a local Kubernetes control plane for the tests (requires etcd and kube-apiserver binaries, implies --start-kudo).")
+	testCmd.Flags().BoolVar(&options.StartKUDO, "start-kudo", false, "Start KUDO during the test run.")
+
+	return testCmd
+}

pkg/test/harness.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/kudobuilder/kudo/pkg/apis"
+	kudo "github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
 	"github.com/kudobuilder/kudo/pkg/controller"
 	testutils "github.com/kudobuilder/kudo/pkg/test/utils"
 	"github.com/kudobuilder/kudo/pkg/webhook"
@@ -22,12 +23,8 @@ import (
 
 // Harness loads and runs tests based on the configuration provided.
 type Harness struct {
-	T                 *testing.T
-	CRDDir            string
-	ManifestsDir      string
-	TestDirs          []string
-	StartControlPlane bool
-	StartKUDO         bool
+	TestSuite kudo.TestSuite
+	T         *testing.T
 
 	managerStopCh chan struct{}
 	config        *rest.Config
@@ -87,7 +84,7 @@ func (h *Harness) Config() (*rest.Config, error) {
 
 	var err error
 
-	if h.StartControlPlane {
+	if h.TestSuite.StartControlPlane {
 		h.config, err = h.RunTestEnv()
 	} else {
 		h.config, err = config.GetConfig()
@@ -103,17 +100,17 @@ func (h *Harness) RunKUDO(crdPath string) error {
 		return err
 	}
 
-	if h.CRDDir != "" {
+	if h.TestSuite.CRDDir != "" {
 		crds, err := envtest.InstallCRDs(config, envtest.CRDInstallOptions{
-			Paths:              []string{h.CRDDir},
+			Paths:              []string{h.TestSuite.CRDDir},
 			ErrorIfPathMissing: true,
 		})
 		if err != nil {
 			return err
 		}
 
 		err = envtest.WaitForCRDs(config, crds, envtest.CRDInstallOptions{
-			Paths:              []string{h.CRDDir},
+			Paths:              []string{h.TestSuite.CRDDir},
 			ErrorIfPathMissing: true,
 		})
 		if err != nil {
@@ -189,7 +186,7 @@ func (h *Harness) RunTests() {
 
 	tests := []*Case{}
 
-	for _, testDir := range h.TestDirs {
+	for _, testDir := range h.TestSuite.TestDirs {
 		tempTests, err := h.LoadTests(testDir)
 		if err != nil {
 			h.T.Fatal(err)
@@ -218,7 +215,7 @@ func (h *Harness) Run() {
 
 	defer h.Stop()
 
-	if h.StartKUDO || h.StartControlPlane {
+	if h.TestSuite.StartKUDO || h.TestSuite.StartControlPlane {
 		if err := h.RunKUDO("../../config/crds/"); err != nil {
 			h.T.Fatal(err)
 		}
@@ -234,7 +231,7 @@ func (h *Harness) Run() {
 		h.T.Fatal(err)
 	}
 
-	if err := testutils.InstallManifests(context.TODO(), cl, dClient, h.ManifestsDir); err != nil {
+	if err := testutils.InstallManifests(context.TODO(), cl, dClient, h.TestSuite.ManifestsDir); err != nil {
 		h.T.Fatal(err)
 	}
 

pkg/test/harness_integration_test.go

@@ -4,15 +4,19 @@ package test
 
 import (
 	"testing"
+
+	kudo "github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
 )
 
 func TestRunWithKudo(t *testing.T) {
 	harness := Harness{
-		T:                 t,
-		CRDDir:            "../../config/crds/",
-		ManifestsDir:      "../../config/samples/test-framework/",
-		TestDirs:          []string{"./test_data/"},
-		StartControlPlane: true,
+		T: t,
+		TestSuite: kudo.TestSuite{
+			CRDDir:            "../../config/crds/",
+			ManifestsDir:      "../../config/samples/test-framework/",
+			TestDirs:          []string{"./test_data/"},
+			StartControlPlane: true,
+		},
 	}
 
 	harness.Run()