Add a small test which runs a basic plugin (#903)

This change also modifies our Makefile and CI scripts so that the tests
can be run on a separate kind cluster.

Signed-off-by: Bridget McErlean <bmcerlean@vmware.com>

.travis.yml

@@ -8,7 +8,8 @@ install: true
 script:
 - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo $TRAVIS_PULL_REQUEST_BRANCH; fi)
 - echo "TRAVIS_BRANCH=$TRAVIS_BRANCH, PR=$TRAVIS_PULL_REQUEST, BRANCH=$BRANCH"
-- VERBOSE=true make test stress int
+- VERBOSE=true make test stress
+- ./scripts/run_integration_tests.sh
 - ./travis-ci.sh
 before_install:
 - curl -L https://github.com/kubernetes-sigs/kind/releases/download/v0.4.0/kind-linux-amd64

Makefile

@@ -27,6 +27,7 @@ DOCKER ?= docker
 LINUX_ARCH := amd64 arm64
 DOCKERFILE :=
 PLATFORMS := $(subst $(SPACE),$(COMMA),$(foreach arch,$(LINUX_ARCH),linux/$(arch)))
+KIND_CLUSTER = kind
 
 # Not used for pushing images, just for local building on other GOOS. Defaults to
 # grabbing from the local go env but can be set manually to avoid that requirement.
@@ -68,8 +69,8 @@ VET = go vet $(TEST_PKGS)
 GOLINT_FLAGS ?= -set_exit_status
 LINT = golint $(GOLINT_FLAGS) $(TEST_PKGS)
 
-WORKDIR ?= /sonobuoy
-DOCKER_BUILD ?= $(DOCKER) run --rm -v $(DIR):$(BUILDMNT) -w $(BUILDMNT) $(BUILD_IMAGE) /bin/sh -c
+DOCKER_FLAGS =
+DOCKER_BUILD ?= $(DOCKER) run --rm -v $(DIR):$(BUILDMNT) $(DOCKER_FLAGS) -w $(BUILDMNT) $(BUILD_IMAGE) /bin/sh -c
 GO_BUILD ?= CGO_ENABLED=0 $(GO_SYSTEM_FLAGS) go build -o $(BINARY) $(VERBOSE_FLAG) -ldflags="-s -w -X $(GOTARGET)/pkg/buildinfo.Version=$(GIT_VERSION) -X $(GOTARGET)/pkg/buildinfo.GitSHA=$(GIT_REF_LONG)" $(GOTARGET)
 
 .PHONY: all container push clean test local-test local generate plugins int
@@ -88,6 +89,7 @@ stress: sonobuoy
 	$(DOCKER_BUILD) 'CGO_ENABLED=0 $(STRESS_TEST)'
 
 # Integration tests
+int: DOCKER_FLAGS=-v $(KUBECONFIG):/root/.kube/kubeconfig --env KUBECONFIG=/root/.kube/kubeconfig --network host
 int: sonobuoy
 	$(DOCKER_BUILD) 'CGO_ENABLED=0 $(INT_TEST)'
 
@@ -183,8 +185,8 @@ clean:
 		$(MAKE) clean_image TARGET=$(TARGET)-$$arch; \
 	done
 
-deploy_kind:
-	kind load docker-image $(REGISTRY)/$(TARGET):$(IMAGE_VERSION) || true
+deploy_kind: container
+	kind load docker-image --name $(KIND_CLUSTER) $(REGISTRY)/$(TARGET):$(IMAGE_VERSION) || true
 
 native:
 	$(GO_BUILD)

scripts/run_integration_tests.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+SCRIPTS_DIR="$( cd "$( dirname "$0" )" >/dev/null 2>&1 && pwd )"
+DIR=$(cd $SCRIPTS_DIR; cd ..; pwd)
+
+cluster="integration"
+testImage="sonobuoy/testimage:v0.1"
+
+if ! kind get clusters | grep -q "^$cluster$"; then
+    kind create cluster --name $cluster
+    # Although the cluster has been created, not all the pods in kube-system are created/available
+    sleep 20
+fi
+
+# Build and load the test plugin image
+make -C $DIR/test/integration/testImage
+kind load docker-image --name $cluster $testImage
+
+# Build and load the sonobuoy image and run integration tests
+make -C $DIR KIND_CLUSTER=$cluster deploy_kind
+KUBECONFIG="$(kind get kubeconfig-path --name="$cluster")" VERBOSE=true make -C $DIR int

test/integration/sonobuoy_integration_test.go

@@ -4,11 +4,13 @@ package integration
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
 	"strings"
 	"testing"
+	"time"
 )
 
 const (
@@ -32,17 +34,66 @@ func findSonobuoyCLI() (string, error) {
 	return sonobuoyPath, nil
 }
 
-// TestSonobuoyVersion checks that all fields in the output from `version` are non-empty
-func TestSonobuoyVersion(t *testing.T) {
+// runSonobuoyCommandWithContext runs the Sonobuoy CLI with the given context and arguments.
+// It returns any encountered error and the stdout and stderr from the command execution.
+func runSonobuoyCommandWithContext(ctx context.Context, t *testing.T, args string) (error, bytes.Buffer, bytes.Buffer) {
 	var stdout, stderr bytes.Buffer
 
-	command := exec.Command(sonobuoy, "version")
+	command := exec.CommandContext(ctx, sonobuoy, strings.Fields(args)...)
 	command.Stdout = &stdout
 	command.Stderr = &stderr
 
 	t.Logf("Running %q\n", command.String())
-	if err := command.Run(); err != nil {
-		t.Errorf("Sonobuoy exited with an error: %v", err)
+
+	return command.Run(), stdout, stderr
+}
+
+// runSonobuoyCommand runs the Sonobuoy CLI with the given arguments and a background context.
+// It returns any encountered error and the stdout and stderr from the command execution.
+func runSonobuoyCommand(t *testing.T, args string) (error, bytes.Buffer, bytes.Buffer) {
+	return runSonobuoyCommandWithContext(context.Background(), t, args)
+}
+
+// cleanup runs sonobuoy delete for the given namespace. If no namespace is provided, it will
+// omit the namespace argument and use the default.
+func cleanup(t *testing.T, namespace string) {
+	args := "delete"
+	if namespace != "" {
+		args += " -n " + namespace
+	}
+
+	err, stdout, stderr := runSonobuoyCommand(t, args)
+
+	if err != nil {
+		t.Logf("Error encountered during cleanup: %q\n", err)
+		t.Log(stdout.String())
+		t.Log(stderr.String())
+	}
+}
+
+// TestSimpleRun runs a simple plugin to check that it runs successfully
+func TestSimpleRun(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	ns := "sonobuoy-" + strings.ToLower(t.Name())
+	defer cleanup(t, ns)
+
+	args := fmt.Sprintf("run --image-pull-policy IfNotPresent --wait -p testImage/yaml/job-junit-passing-singlefile.yaml -n %v", ns)
+	err, _, stderr := runSonobuoyCommandWithContext(ctx, t, args)
+
+	if err != nil {
+		t.Errorf("Sonobuoy exited with an error: %q\n", err)
+		t.Log(stderr.String())
+	}
+}
+
+// TestSonobuoyVersion checks that all fields in the output from `version` are non-empty
+func TestSonobuoyVersion(t *testing.T) {
+	err, stdout, stderr := runSonobuoyCommand(t, "version")
+
+	if err != nil {
+		t.Errorf("Sonobuoy exited with an error: %q\n", err)
 		t.Log(stderr.String())
 		t.FailNow()
 	}

travis-ci.sh

@@ -40,7 +40,7 @@ if [ $e2eCode -ne 0 ]; then
     mkdir results; tar xzf $outFile -C results
 
     echo "Full contents of tarball:"
-    find results 
+    find results
 
     echo "Printing data on the following files:"
     find results/plugins -type f