added background commands

Signed-off-by: Ken Sipe <kensipe@gmail.com>

pkg/apis/testharness/v1beta1/test_types.go

@@ -104,6 +104,8 @@ type Command struct {
 	Namespaced bool `json:"namespaced"`
 	// If set, failures will be ignored.
 	IgnoreFailure bool `json:"ignoreFailure"`
+	// If set, the command is run in the background.
+	Background bool `json:"background"`
 }
 
 // DefaultKINDContext defines the default kind context to use.

pkg/test/harness.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"sync"
 	"testing"
@@ -43,6 +44,8 @@ type Harness struct {
 	kubeConfigPath string
 	clientLock     sync.Mutex
 	configLock     sync.Mutex
+	stopping       bool
+	bgProcesses    []*exec.Cmd
 }
 
 // LoadTests loads all of the tests in a given directory.
@@ -60,7 +63,7 @@ func (h *Harness) LoadTests(dir string) ([]*Case, error) {
 	tests := []*Case{}
 
 	timeout := h.GetTimeout()
-	h.T.Logf("Going to run test suite with timeout of %d seconds for each step", timeout)
+	h.T.Logf("going to run test suite with timeout of %d seconds for each step", timeout)
 
 	for _, file := range files {
 		if !file.IsDir() {
@@ -207,13 +210,13 @@ func (h *Harness) Config() (*rest.Config, error) {
 	var err error
 
 	if h.TestSuite.StartControlPlane {
-		h.T.Log("Running tests with a mocked control plane (kube-apiserver and etcd).")
+		h.T.Log("running tests with a mocked control plane (kube-apiserver and etcd).")
 		h.config, err = h.RunTestEnv()
 	} else if h.TestSuite.StartKIND {
-		h.T.Log("Running tests with KIND.")
+		h.T.Log("running tests with KIND.")
 		h.config, err = h.RunKIND()
 	} else {
-		h.T.Log("Running tests using configured kubeconfig.")
+		h.T.Log("running tests using configured kubeconfig.")
 		h.config, err = config.GetConfig()
 	}
 
@@ -283,7 +286,9 @@ func (h *Harness) DockerClient() (testutils.DockerClient, error) {
 // RunTests should be called from within a Go test (t) and launches all of the KUDO integration
 // tests at dir.
 func (h *Harness) RunTests() {
-	h.T.Log("Running tests")
+	// cleanup after running tests
+	defer h.Stop()
+	h.T.Log("running tests")
 	tests := []*Case{}
 
 	for _, testDir := range h.TestSuite.TestDirs {
@@ -324,53 +329,54 @@ func (h *Harness) Run() {
 // It can be used to start env which can than be modified prior to running tests, otherwise use Run().
 func (h *Harness) Setup() {
 	rand.Seed(time.Now().UTC().UnixNano())
-	h.T.Log("Starting Setup")
-	defer h.Stop()
+	h.T.Log("starting setup")
 
 	cl, err := h.Client(false)
 	if err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	dClient, err := h.DiscoveryClient()
 	if err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	// Install CRDs
 	crdKind := testutils.NewResource("apiextensions.k8s.io/v1beta1", "CustomResourceDefinition", "", "")
 	crds, err := testutils.InstallManifests(context.TODO(), cl, dClient, h.TestSuite.CRDDir, crdKind)
 	if err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	if err := testutils.WaitForCRDs(dClient, crds); err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	// Create a new client to bust the client's CRD cache.
 	cl, err = h.Client(true)
 	if err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	// Install required manifests.
 	for _, manifestDir := range h.TestSuite.ManifestDirs {
 		if _, err := testutils.InstallManifests(context.TODO(), cl, dClient, manifestDir); err != nil {
-			h.T.Fatal(err)
+			h.fatal(err)
 		}
 	}
-
-	if err := testutils.RunCommands(h.GetLogger(), "default", "", h.TestSuite.Commands, ""); err != nil {
-		h.T.Fatal(err)
+	bgs, errs := testutils.RunCommands(h.GetLogger(), "default", "", h.TestSuite.Commands, "")
+	// assign any background processes first for cleanup if err is fatal
+	h.bgProcesses = append(h.bgProcesses, bgs...)
+	if errs != nil {
+		h.fatal(err)
 	}
 
-	if err := testutils.RunKubectlCommands(h.GetLogger(), "default", h.TestSuite.Kubectl, ""); err != nil {
-		h.T.Fatal(err)
+	if errs := testutils.RunKubectlCommands(h.GetLogger(), "default", h.TestSuite.Kubectl, ""); errs != nil {
+		h.fatal(errs)
 	}
 }
 
-// Stop the test environment and KUDO, clean up the harness.
+// Stop the test environment and clean up the harness.
 func (h *Harness) Stop() {
 	if h.managerStopCh != nil {
 		close(h.managerStopCh)
@@ -397,6 +403,15 @@ func (h *Harness) Stop() {
 		return
 	}
 
+	if h.bgProcesses != nil {
+		for _, process := range h.bgProcesses {
+			err := process.Process.Kill()
+			if err != nil {
+				h.T.Log("background process kill error", err)
+			}
+		}
+	}
+
 	if h.env != nil {
 		h.T.Log("tearing down mock control plane")
 		if err := h.env.Stop(); err != nil {
@@ -420,6 +435,17 @@ func (h *Harness) Stop() {
 	}
 }
 
+// wraps Test.Fatal in order to clean up harness
+func (h *Harness) fatal(args ...interface{}) {
+	// clean up on fatal in setup
+	if !h.stopping {
+		// stopping prevents reentry into h.Stop
+		h.stopping = true
+		h.Stop()
+	}
+	h.T.Fatal(args...)
+}
+
 func (h *Harness) explicitPath() string {
 	return filepath.Join(h.kubeConfigPath, "kubeconfig")
 }

pkg/test/step.go

@@ -373,7 +373,13 @@ func (s *Step) Run(namespace string) []error {
 	testErrors := []error{}
 
 	if s.Step != nil {
-		if errors := testutils.RunCommands(s.Logger, namespace, "", s.Step.Commands, s.Dir); errors != nil {
+		for _, command := range s.Step.Commands {
+			if command.Background {
+				s.Logger.Log("background commands are not allowed for steps and will be run in foreground")
+				command.Background = false
+			}
+		}
+		if _, errors := testutils.RunCommands(s.Logger, namespace, "", s.Step.Commands, s.Dir); errors != nil {
 			testErrors = append(testErrors, errors...)
 		}
 

pkg/test/utils/kubernetes.go

@@ -933,15 +933,16 @@ func GetArgs(ctx context.Context, command string, cmd harness.Command, namespace
 
 // RunCommand runs a command with args.
 // args gets split on spaces (respecting quoted strings).
-func RunCommand(ctx context.Context, namespace string, command string, cmd harness.Command, cwd string, stdout io.Writer, stderr io.Writer) error {
+// if the command is run in the background a reference to the process is returned for later cleanup
+func RunCommand(ctx context.Context, namespace string, command string, cmd harness.Command, cwd string, stdout io.Writer, stderr io.Writer) (*exec.Cmd, error) {
 	actualDir, err := os.Getwd()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	builtCmd, err := GetArgs(ctx, command, cmd, namespace)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	builtCmd.Dir = cwd
@@ -952,23 +953,34 @@ func RunCommand(ctx context.Context, namespace string, command string, cmd harne
 		fmt.Sprintf("PATH=%s/bin/:%s", actualDir, os.Getenv("PATH")),
 	}
 
-	err = builtCmd.Run()
+	err = builtCmd.Start()
 	if err != nil {
 		if _, ok := err.(*exec.ExitError); ok && cmd.IgnoreFailure {
-			return nil
+			return nil, nil
 		}
+		return nil, err
 	}
 
-	return err
+	if cmd.Background {
+		return builtCmd, nil
+	}
+
+	err = builtCmd.Wait()
+	if _, ok := err.(*exec.ExitError); ok && cmd.IgnoreFailure {
+		return nil, nil
+	}
+	return nil, err
 }
 
 // RunCommands runs a set of commands, returning any errors.
 // If `command` is set, then `command` will be the command that is invoked (if a command specifies it already, it will not be prepended again).
-func RunCommands(logger Logger, namespace string, command string, commands []harness.Command, workdir string) []error {
+// commands running in the background are returned
+func RunCommands(logger Logger, namespace string, command string, commands []harness.Command, workdir string) ([]*exec.Cmd, []error) {
 	errs := []error{}
+	bgs := []*exec.Cmd{}
 
 	if commands == nil {
-		return nil
+		return nil, nil
 	}
 
 	for _, cmd := range commands {
@@ -977,20 +989,21 @@ func RunCommands(logger Logger, namespace string, command string, commands []har
 
 		logger.Logf("Running command: %s %s", command, cmd)
 
-		err := RunCommand(context.TODO(), namespace, command, cmd, workdir, stdout, stderr)
+		bg, err := RunCommand(context.TODO(), namespace, command, cmd, workdir, stdout, stderr)
 		if err != nil {
 			errs = append(errs, err)
+			bgs = append(bgs, bg)
 		}
 
 		logger.Log(stderr.String())
 		logger.Log(stdout.String())
 	}
 
 	if len(errs) == 0 {
-		return nil
+		return nil, nil
 	}
 
-	return errs
+	return bgs, errs
 }
 
 // RunKubectlCommands runs a set of kubectl commands, returning any errors.
@@ -1003,8 +1016,9 @@ func RunKubectlCommands(logger Logger, namespace string, commands []string, work
 			Namespaced: true,
 		})
 	}
-
-	return RunCommands(logger, namespace, "kubectl", apiCommands, workdir)
+	// ignore background processes as kubectl commands are not allowed to have them
+	_, errs := RunCommands(logger, namespace, "kubectl", apiCommands, workdir)
+	return errs
 }
 
 // Kubeconfig converts a rest.Config into a YAML kubeconfig and writes it to w