Testing Infra AND Circle CI Integration (#29)

Signed-off-by: Ken Sipe <kensipe@gmail.com>

.circleci/config.yml

@@ -0,0 +1,51 @@
+# Golang CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-go/ for more details
+version: 2.1
+jobs:
+  integration-test:
+    machine: true
+    steps:
+      - checkout
+      - run: echo 'export INTEGRATION_OUTPUT_JUNIT="true"' >> $BASH_ENV
+      - run: ./test/run_tests.sh integration-test
+      - store_test_results:
+          path: reports/
+
+  e2e-test:
+    machine: true
+    steps:
+      - checkout
+      - run: echo 'export INTEGRATION_OUTPUT_JUNIT="true"' >> $BASH_ENV
+      - run: ./test/run_tests.sh e2e-test
+      - store_test_results:
+          path: reports/
+      - store_artifacts:
+          path: kind-logs.tar.bz2
+
+  lint:
+    docker:
+      - image: kudobuilder/golang:1.13
+    working_directory: /go/src/github.com/kudobuilder/kuttl
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - go-mod-v1-{{ checksum "go.sum" }}
+      - run: make lint
+      - save_cache:
+          key: go-mod-v1-{{ checksum "go.sum" }}
+          paths:
+            - "/go/pkg/mod"
+            - "/go/bin"
+workflows:
+  version: 2
+  lint:
+    jobs:
+      - lint
+  integration-test:
+    jobs:
+      - integration-test
+  e2e-test:
+    jobs:
+      - e2e-test

.gitignore

@@ -5,7 +5,7 @@ hack/code-gen
 hack/controller-gen
 test/.git-credentials
 reports/
-config/manager_image_patch.yaml-e
+
 
 ### mac
 .DS_Store

.golangci.yml

@@ -27,5 +27,5 @@ linters-settings:
   dupl:
     threshold: 400
   goimports:
-    # Don't use 'github.com/kudobuilder/kudo', it'll result in unreliable output!
+    # Don't use 'github.com/kudobuilder/kuttl', it'll result in unreliable output!
     local-prefixes: github.com/kudobuilder

Makefile

@@ -31,6 +31,10 @@ endif
 integration-test:
 	./hack/run-integration-tests.sh
 
+# Run e2e tests
+.PHONY: e2e-test
+e2e-test: cli
+	./hack/run-e2e-tests.sh
 
 .PHONY: lint
 lint:
@@ -43,42 +47,11 @@ endif
 download:
 	go mod download
 
-.PHONY: prebuild
-prebuild: generate lint
-
-.PHONY: generate
-# Generate code
-generate:
-ifeq (, $(shell which controller-gen))
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@$$(go list -f '{{.Version}}' -m sigs.k8s.io/controller-tools)
-endif
-	controller-gen crd paths=./pkg/apis/... output:crd:dir=config/crds output:stdout
-	./hack/update_codegen.sh
-
-.PHONY: generate-clean
-generate-clean:
-	rm -rf hack/code-gen
-
-.PHONY: clean
-# Clean all
-clean: test-clean
-
-.PHONY: imports
-# used to update imports on project.  NOT a linter.
-imports:
-ifeq (, $(shell which golangci-lint))
-	./hack/install-golangcilint.sh
-endif
-	golangci-lint run --disable-all -E goimports --fix
-
-.PHONY: cli-fast
-# Build CLI but don't lint or run code generation first.
-cli-fast:
-	go build -ldflags "${LDFLAGS}" -o bin/${CLI} ./cmd/kubectl-kuttl
 
 .PHONY: cli
 # Build CLI
-cli: prebuild cli-fast
+cli:
+	go build -ldflags "${LDFLAGS}" -o bin/${CLI} ./cmd/kubectl-kuttl
 
 .PHONY: cli-clean
 # Clean CLI build
@@ -89,13 +62,14 @@ cli-clean:
 cli-install:
 	go install -ldflags "${LDFLAGS}" ./cmd/kubectl-kuttl
 
-
 .PHONY: todo
 # Show to-do items per file.
 todo:
 	@grep \
 		--exclude-dir=hack \
 		--exclude=Makefile \
+		--exclude-dir=.git \
+		--exclude-dir=bin \
 		--text \
 		--color \
-		-nRo -E ' TODO:.*|SkipNow' .
+		-nRo -E " *[^\.]TODO.*|SkipNow" .

cmd/kubectl-kuttl/main.go

@@ -1,4 +1,4 @@
-// Copyright 2020 KUDO Authors
+// Copyright 2020 KUTTL Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

hack/run-e2e-tests.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+INTEGRATION_OUTPUT_JUNIT=${INTEGRATION_OUTPUT_JUNIT:-false}
+VERSION=${VERSION:-test}
+
+if [ "$INTEGRATION_OUTPUT_JUNIT" == true ]
+then
+    echo "Running E2E tests with junit output"
+    mkdir -p reports/
+    go get github.com/jstemmer/go-junit-report
+
+    ./bin/kubectl-kuttl test pkg/test/test_data/ 2>&1 \
+        | tee /dev/fd/2 \
+        | go-junit-report -set-exit-code \
+        > reports/kuttl_e2e_test_report.xml
+
+else
+    echo "Running E2E tests without junit output"
+
+    ./bin/kubectl-kuttl test pkg/test/test_data/
+fi

pkg/test/harness.go

@@ -283,7 +283,7 @@ func (h *Harness) DockerClient() (testutils.DockerClient, error) {
 	return h.docker, err
 }
 
-// RunTests should be called from within a Go test (t) and launches all of the KUDO integration
+// RunTests should be called from within a Go test (t) and launches all of the KUTTL integration
 // tests at dir.
 func (h *Harness) RunTests() {
 	// cleanup after running tests
@@ -319,7 +319,7 @@ func (h *Harness) RunTests() {
 	})
 }
 
-// Run the test harness - start KUDO and the control plane and then run the tests.
+// Run the test harness - start the control plane and then run the tests.
 func (h *Harness) Run() {
 	h.Setup()
 	h.RunTests()

pkg/test/step.go

@@ -360,7 +360,7 @@ func (s *Step) Check(namespace string) []error {
 	return testErrors
 }
 
-// Run runs a KUDO test step:
+// Run runs a KUTTL test step:
 // 1. Apply all desired objects to Kubernetes.
 // 2. Wait for all of the states defined in the test step's asserts to be true.'
 func (s *Step) Run(namespace string) []error {