Separate E2E and operator tests (#1540)

This runs operator tests as a separate test in parallel with the other tests. It makes operator tests independent from E2E test results and their failures distinguishable.

Signed-off-by: Jan Schlicht <jan@d2iq.com>
Signed-off-by: Ken Sipe <kensipe@gmail.com>

Co-authored-by: Ken Sipe <kensipe@gmail.com>

.circleci/config.yml

@@ -23,6 +23,17 @@ jobs:
       - store_artifacts:
           path: kind-logs.tar.bz2
 
+  operator-test:
+    machine: true
+    steps:
+      - checkout
+      - run: echo 'export INTEGRATION_OUTPUT_JUNIT="true"' >> $BASH_ENV
+      - run: ./test/run_tests.sh operator-test
+      - store_test_results:
+          path: reports/
+      - store_artifacts:
+          path: kind-logs.tar.bz2
+
   lint:
     docker:
       - image: kudobuilder/golang:1.14
@@ -50,3 +61,6 @@ workflows:
   e2e-test:
     jobs:
       - e2e-test
+  operator-test:
+    jobs:
+      - operator-test

Makefile

@@ -41,6 +41,10 @@ e2e-test: cli-fast manager-fast
 integration-test: cli-fast manager-fast
 	./hack/run-integration-tests.sh
 
+.PHONY: operator-test
+operator-test: cli-fast manager-fast
+	./hack/run-operator-tests.sh
+
 .PHONY: test-clean
 # Clean test reports
 test-clean:

hack/run-e2e-tests.sh

@@ -24,27 +24,8 @@ then
         | tee /dev/fd/2 \
         | go-junit-report -set-exit-code \
         > reports/kudo_e2e_test_report.xml
-
-    # Operators tests
-    rm -rf operators
-    git clone https://github.com/kudobuilder/operators
-    mkdir operators/bin/
-    cp ./bin/kubectl-kudo operators/bin/
-    sed "s/%version%/$KUDO_VERSION/" operators/kudo-test.yaml.tmpl > operators/kudo-test.yaml
-    cd operators && ./bin/kubectl-kudo test --artifacts-dir /tmp/kudo-e2e-test 2>&1 \
-        | tee /dev/fd/2 \
-        | go-junit-report -set-exit-code \
-        > ../reports/kudo_operators_test_report.xml
 else
     echo "Running E2E tests without junit output"
 
     ./bin/kubectl-kudo test --config kudo-e2e-test.yaml
-
-    # Operators tests
-    rm -rf operators
-    git clone https://github.com/kudobuilder/operators
-    mkdir operators/bin/
-    cp ./bin/kubectl-kudo operators/bin/
-    sed "s/%version%/$KUDO_VERSION/" operators/kudo-test.yaml.tmpl > operators/kudo-test.yaml
-    cd operators && ./bin/kubectl-kudo test --artifacts-dir /tmp/kudo-e2e-test
 fi

hack/run-operator-tests.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+# Operator tests have the same setup but different concerns from E2E tests. 
+# The motivation is to get earlier feedback when these tests fail. In the past, the operator tests were only run if the E2E tests succeeded. 
+# This made it hard for larger PRs to distinguish between changes that fix operators and changes that fix E2E tests.
+
+INTEGRATION_OUTPUT_JUNIT=${INTEGRATION_OUTPUT_JUNIT:-false}
+# Exposing `KUDO_VERSION` allows for KUDO_VERSION to be overriden. The default value of test is used as a tag for the Docker image that is build and injected to the kind cluster. 
+# This is to avoid clashes with existing Docker images as test is only used in this context.
+KUDO_VERSION=${KUDO_VERSION:-test}
+
+docker build . \
+    --build-arg ldflags_arg="" \
+    -t "kudobuilder/controller:$KUDO_VERSION"
+
+rm -rf operators
+git clone https://github.com/kudobuilder/operators
+mkdir operators/bin/
+cp ./bin/kubectl-kudo operators/bin/
+sed "s/%version%/$KUDO_VERSION/" operators/kudo-test.yaml.tmpl > operators/kudo-test.yaml
+
+if [ "$INTEGRATION_OUTPUT_JUNIT" == true ]
+then
+    echo "Running operator tests with junit output"
+    mkdir -p reports/
+    go get github.com/jstemmer/go-junit-report
+
+    cd operators && ./bin/kubectl-kudo test --artifacts-dir /tmp/kudo-e2e-test 2>&1 \
+        | tee /dev/fd/2 \
+        | go-junit-report -set-exit-code \
+        > ../reports/kudo_operators_test_report.xml
+else
+    echo "Running operator tests without junit output"
+
+    cd operators && ./bin/kubectl-kudo test --artifacts-dir /tmp/kudo-e2e-test
+fi

test/Dockerfile

@@ -19,6 +19,7 @@ COPY pkg/ pkg/
 COPY cmd/ cmd/
 COPY hack/run-integration-tests.sh hack/run-integration-tests.sh
 COPY hack/run-e2e-tests.sh hack/run-e2e-tests.sh
+COPY hack/run-operator-tests.sh hack/run-operator-tests.sh
 COPY test/ test/
 COPY kudo-test.yaml kudo-test.yaml
 COPY kudo-e2e-test.yaml.tmpl kudo-e2e-test.yaml.tmpl

test/run_tests.sh

@@ -12,7 +12,7 @@ INTEGRATION_OUTPUT_JUNIT=${INTEGRATION_OUTPUT_JUNIT:-false}
 
 function archive_logs() {
     # Archive test harness artifacts
-    if [ "$TARGET" == "e2e-test" ]; then
+    if [ "$TARGET" == "e2e-test" ] || [ "$TARGET" == "operator-test" ]; then
         tar -cjvf kind-logs.tar.bz2 kind-logs/
     fi
 }