Support Background Processes

pkg/apis/testharness/v1beta1/test_types.go

@@ -33,8 +33,6 @@ type TestSuite struct {
 	KINDNodeCache bool `json:"kindNodeCache"`
 	// Containers to load to each KIND node prior to running the tests.
 	KINDContainers []string `json:"kindContainers"`
-	// Whether or not to start the KUDO controller for the tests.
-	StartKUDO bool `json:"startKUDO"`
 	// If set, do not delete the resources after running the tests (implies SkipClusterDelete).
 	SkipDelete bool `json:"skipDelete"`
 	// If set, do not delete the mocked control plane or kind cluster.
@@ -104,8 +102,10 @@ type Command struct {
 	Command string `json:"command"`
 	// If set, the `--namespace` flag will be appended to the command with the namespace to use.
 	Namespaced bool `json:"namespaced"`
-	// If set, failures will be ignored.
+	// If set, exit failures (`exec.ExitError`) will be ignored. `exec.Error` are NOT ignored.
 	IgnoreFailure bool `json:"ignoreFailure"`
+	// If set, the command is run in the background.
+	Background bool `json:"background"`
 }
 
 // DefaultKINDContext defines the default kind context to use.

pkg/test/harness.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"sync"
 	"testing"
@@ -43,6 +44,8 @@ type Harness struct {
 	kubeConfigPath string
 	clientLock     sync.Mutex
 	configLock     sync.Mutex
+	stopping       bool
+	bgProcesses    []*exec.Cmd
 }
 
 // LoadTests loads all of the tests in a given directory.
@@ -60,7 +63,7 @@ func (h *Harness) LoadTests(dir string) ([]*Case, error) {
 	tests := []*Case{}
 
 	timeout := h.GetTimeout()
-	h.T.Logf("Going to run test suite with timeout of %d seconds for each step", timeout)
+	h.T.Logf("going to run test suite with timeout of %d seconds for each step", timeout)
 
 	for _, file := range files {
 		if !file.IsDir() {
@@ -207,13 +210,13 @@ func (h *Harness) Config() (*rest.Config, error) {
 	var err error
 
 	if h.TestSuite.StartControlPlane {
-		h.T.Log("Running tests with a mocked control plane (kube-apiserver and etcd).")
+		h.T.Log("running tests with a mocked control plane (kube-apiserver and etcd).")
 		h.config, err = h.RunTestEnv()
 	} else if h.TestSuite.StartKIND {
-		h.T.Log("Running tests with KIND.")
+		h.T.Log("running tests with KIND.")
 		h.config, err = h.RunKIND()
 	} else {
-		h.T.Log("Running tests using configured kubeconfig.")
+		h.T.Log("running tests using configured kubeconfig.")
 		h.config, err = config.GetConfig()
 	}
 
@@ -283,6 +286,9 @@ func (h *Harness) DockerClient() (testutils.DockerClient, error) {
 // RunTests should be called from within a Go test (t) and launches all of the KUDO integration
 // tests at dir.
 func (h *Harness) RunTests() {
+	// cleanup after running tests
+	defer h.Stop()
+	h.T.Log("running tests")
 	tests := []*Case{}
 
 	for _, testDir := range h.TestSuite.TestDirs {
@@ -313,59 +319,64 @@ func (h *Harness) RunTests() {
 	})
 }
 
-// Run the test harness - start KUDO and the control plane and install the operators, if necessary
-// and then run the tests.
+// Run the test harness - start KUDO and the control plane and then run the tests.
 func (h *Harness) Run() {
-	rand.Seed(time.Now().UTC().UnixNano())
+	h.Setup()
+	h.RunTests()
+}
 
-	defer h.Stop()
+// Setup spins up the test env based on configuration
+// It can be used to start env which can than be modified prior to running tests, otherwise use Run().
+func (h *Harness) Setup() {
+	rand.Seed(time.Now().UTC().UnixNano())
+	h.T.Log("starting setup")
 
 	cl, err := h.Client(false)
 	if err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	dClient, err := h.DiscoveryClient()
 	if err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	// Install CRDs
 	crdKind := testutils.NewResource("apiextensions.k8s.io/v1beta1", "CustomResourceDefinition", "", "")
 	crds, err := testutils.InstallManifests(context.TODO(), cl, dClient, h.TestSuite.CRDDir, crdKind)
 	if err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	if err := testutils.WaitForCRDs(dClient, crds); err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	// Create a new client to bust the client's CRD cache.
 	cl, err = h.Client(true)
 	if err != nil {
-		h.T.Fatal(err)
+		h.fatal(err)
 	}
 
 	// Install required manifests.
 	for _, manifestDir := range h.TestSuite.ManifestDirs {
 		if _, err := testutils.InstallManifests(context.TODO(), cl, dClient, manifestDir); err != nil {
-			h.T.Fatal(err)
+			h.fatal(err)
 		}
 	}
-
-	if err := testutils.RunCommands(h.GetLogger(), "default", "", h.TestSuite.Commands, ""); err != nil {
-		h.T.Fatal(err)
+	bgs, errs := testutils.RunCommands(h.GetLogger(), "default", "", h.TestSuite.Commands, "")
+	// assign any background processes first for cleanup in case of any errors
+	h.bgProcesses = append(h.bgProcesses, bgs...)
+	if errs != nil {
+		h.fatal(errs)
 	}
 
-	if err := testutils.RunKubectlCommands(h.GetLogger(), "default", h.TestSuite.Kubectl, ""); err != nil {
-		h.T.Fatal(err)
+	if errs := testutils.RunKubectlCommands(h.GetLogger(), "default", h.TestSuite.Kubectl, ""); errs != nil {
+		h.fatal(errs)
 	}
-
-	h.RunTests()
 }
 
-// Stop the test environment and KUDO, clean up the harness.
+// Stop the test environment and clean up the harness.
 func (h *Harness) Stop() {
 	if h.managerStopCh != nil {
 		close(h.managerStopCh)
@@ -392,6 +403,15 @@ func (h *Harness) Stop() {
 		return
 	}
 
+	if h.bgProcesses != nil {
+		for _, p := range h.bgProcesses {
+			err := p.Process.Kill()
+			if err != nil {
+				h.T.Log("background process kill error", err)
+			}
+		}
+	}
+
 	if h.env != nil {
 		h.T.Log("tearing down mock control plane")
 		if err := h.env.Stop(); err != nil {
@@ -415,6 +435,18 @@ func (h *Harness) Stop() {
 	}
 }
 
+// wraps Test.Fatal in order to clean up harness
+// fatal should NOT be used with a go routine, it is not thread safe
+func (h *Harness) fatal(args ...interface{}) {
+	// clean up on fatal in setup
+	if !h.stopping {
+		// stopping prevents reentry into h.Stop
+		h.stopping = true
+		h.Stop()
+	}
+	h.T.Fatal(args...)
+}
+
 func (h *Harness) explicitPath() string {
 	return filepath.Join(h.kubeConfigPath, "kubeconfig")
 }

pkg/test/step.go

@@ -373,7 +373,13 @@ func (s *Step) Run(namespace string) []error {
 	testErrors := []error{}
 
 	if s.Step != nil {
-		if errors := testutils.RunCommands(s.Logger, namespace, "", s.Step.Commands, s.Dir); errors != nil {
+		for _, command := range s.Step.Commands {
+			if command.Background {
+				s.Logger.Log("background commands are not allowed for steps and will be run in foreground")
+				command.Background = false
+			}
+		}
+		if _, errors := testutils.RunCommands(s.Logger, namespace, "", s.Step.Commands, s.Dir); errors != nil {
 			testErrors = append(testErrors, errors...)
 		}
 

pkg/test/utils/kubernetes.go

@@ -933,42 +933,58 @@ func GetArgs(ctx context.Context, command string, cmd harness.Command, namespace
 
 // RunCommand runs a command with args.
 // args gets split on spaces (respecting quoted strings).
-func RunCommand(ctx context.Context, namespace string, command string, cmd harness.Command, cwd string, stdout io.Writer, stderr io.Writer) error {
+// if the command is run in the background a reference to the process is returned for later cleanup
+func RunCommand(ctx context.Context, namespace string, command string, cmd harness.Command, cwd string, stdout io.Writer, stderr io.Writer) (*exec.Cmd, error) {
 	actualDir, err := os.Getwd()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	builtCmd, err := GetArgs(ctx, command, cmd, namespace)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	builtCmd.Dir = cwd
-	builtCmd.Stdout = stdout
-	builtCmd.Stderr = stderr
+	if !cmd.Background {
+		builtCmd.Stdout = stdout
+		builtCmd.Stderr = stderr
+	}
 	builtCmd.Env = []string{
 		fmt.Sprintf("KUBECONFIG=%s/kubeconfig", actualDir),
 		fmt.Sprintf("PATH=%s/bin/:%s", actualDir, os.Getenv("PATH")),
 	}
 
-	err = builtCmd.Run()
+	// process started and exited with error
+	var exerr *exec.ExitError
+	err = builtCmd.Start()
 	if err != nil {
-		if _, ok := err.(*exec.ExitError); ok && cmd.IgnoreFailure {
-			return nil
+		if errors.As(err, &exerr) && cmd.IgnoreFailure {
+			return nil, nil
 		}
+		return nil, err
 	}
 
-	return err
+	if cmd.Background {
+		return builtCmd, nil
+	}
+
+	err = builtCmd.Wait()
+	if errors.As(err, &exerr) && cmd.IgnoreFailure {
+		return nil, nil
+	}
+	return nil, err
 }
 
 // RunCommands runs a set of commands, returning any errors.
 // If `command` is set, then `command` will be the command that is invoked (if a command specifies it already, it will not be prepended again).
-func RunCommands(logger Logger, namespace string, command string, commands []harness.Command, workdir string) []error {
+// commands running in the background are returned
+func RunCommands(logger Logger, namespace string, command string, commands []harness.Command, workdir string) ([]*exec.Cmd, []error) {
 	errs := []error{}
+	bgs := []*exec.Cmd{}
 
 	if commands == nil {
-		return nil
+		return nil, nil
 	}
 
 	for _, cmd := range commands {
@@ -977,20 +993,23 @@ func RunCommands(logger Logger, namespace string, command string, commands []har
 
 		logger.Logf("Running command: %s %s", command, cmd)
 
-		err := RunCommand(context.TODO(), namespace, command, cmd, workdir, stdout, stderr)
+		bg, err := RunCommand(context.TODO(), namespace, command, cmd, workdir, stdout, stderr)
 		if err != nil {
 			errs = append(errs, err)
+			if bg != nil {
+				bgs = append(bgs, bg)
+			}
 		}
 
 		logger.Log(stderr.String())
 		logger.Log(stdout.String())
 	}
 
 	if len(errs) == 0 {
-		return nil
+		return nil, nil
 	}
 
-	return errs
+	return bgs, errs
 }
 
 // RunKubectlCommands runs a set of kubectl commands, returning any errors.
@@ -1003,8 +1022,9 @@ func RunKubectlCommands(logger Logger, namespace string, commands []string, work
 			Namespaced: true,
 		})
 	}
-
-	return RunCommands(logger, namespace, "kubectl", apiCommands, workdir)
+	// ignore background processes as kubectl commands are not allowed to have them
+	_, errs := RunCommands(logger, namespace, "kubectl", apiCommands, workdir)
+	return errs
 }
 
 // Kubeconfig converts a rest.Config into a YAML kubeconfig and writes it to w

pkg/test/utils/kubernetes_integration_test.go

@@ -3,6 +3,7 @@
 package utils
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"log"
@@ -13,6 +14,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"k8s.io/apimachinery/pkg/watch"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1"
 )
 
 var testenv TestEnvironment
@@ -107,3 +110,56 @@ func TestClientWatch(t *testing.T) {
 
 	events.Stop()
 }
+
+func TestRunCommand(t *testing.T) {
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	hcmd := harness.Command{
+		Command: "echo 'hello'",
+	}
+
+	// assert foreground cmd returns nil
+	cmd, err := RunCommand(context.TODO(), "", "", hcmd, "", stdout, stderr)
+	assert.NoError(t, err)
+	assert.Nil(t, cmd)
+	// foreground processes should have stdout
+	assert.NotEmpty(t, stdout)
+
+	hcmd.Background = true
+	stdout = &bytes.Buffer{}
+
+	// assert background cmd returns process
+	cmd, err = RunCommand(context.TODO(), "", "", hcmd, "", stdout, stderr)
+	assert.NoError(t, err)
+	assert.NotNil(t, cmd)
+	// no stdout for background processes
+	assert.Empty(t, stdout)
+}
+
+func TestRunCommandIgnoreErrors(t *testing.T) {
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	hcmd := harness.Command{
+		Command:       "sleep -u",
+		IgnoreFailure: true,
+	}
+
+	// assert foreground cmd returns nil
+	cmd, err := RunCommand(context.TODO(), "", "", hcmd, "", stdout, stderr)
+	assert.NoError(t, err)
+	assert.Nil(t, cmd)
+
+	hcmd.IgnoreFailure = false
+	cmd, err = RunCommand(context.TODO(), "", "", hcmd, "", stdout, stderr)
+	assert.Error(t, err)
+	assert.Nil(t, cmd)
+
+	// bad commands have errors regardless of ignore setting
+	hcmd = harness.Command{
+		Command:       "bad-command",
+		IgnoreFailure: true,
+	}
+	cmd, err = RunCommand(context.TODO(), "", "", hcmd, "", stdout, stderr)
+	assert.Error(t, err)
+	assert.Nil(t, cmd)
+}