Decouple building binaries from images (#1078)

As a precursor to supporting Windows builds, I'm separating out
more parts of our build in CircleCI.

Major changes:
 - unit tests in parallel with other build steps
 - separate building the binaries, building the images, and running
integration tests
 - using workspaces to persist binaries/images across jobs

Total time to run the workflow decreased by about 3m (30%) since
the unit tests were no longer on the critical path.

Signed-off-by: John Schnake <jschnake@vmware.com>

.circleci/config.yml

@@ -32,7 +32,51 @@ jobs:
       - checkout
       - run: ./scripts/ci/publish.sh
 
-  build_and_test:
+  build_clients:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          name: "Build Sonobuoy binaries"
+          command: make build/linux/amd64/sonobuoy build/linux/arm64/sonobuoy
+      - persist_to_workspace:
+          root: ~/project
+          paths: 
+            - build/linux/amd64/*
+            - build/linux/arm64/*
+            - build/linux/windows/amd64/*
+
+  build_containers:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /home/circleci/project
+      - run:
+          name: "Build Sonobuoy container images"
+          command: |
+            make containers
+      - run:
+          name: Save as tar files to persist to next step
+          command: |
+            docker save -o sonobuoyImages.tar.gz sonobuoy/sonobuoy
+      - persist_to_workspace:
+          root: /home/circleci/project
+          paths:
+            - sonobuoyImages.tar.gz
+
+  unit_tests:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          name: "Run unit and stress tests"
+          command: VERBOSE=true make test stress
+
+  integration_tests:
     machine:
       enabled: true
     steps:
@@ -51,16 +95,16 @@ jobs:
           command: |
             kind create cluster --config kind-config.yaml
             echo 'export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"' >> $BASH_ENV
+      - attach_workspace:
+          at: /home/circleci/project
       - run:
-          name: "Build Sonobuoy container image and add to kind cluster"
-          command: make container deploy_kind
-      - run:
-          name: "Run unit and stress tests"
-          command: VERBOSE=true make test stress
+          name: Load images into docker from workspace
+          command: |
+            docker load -i /home/circleci/project/sonobuoyImages.tar.gz
       - run:
           name: "Run Sonobuoy integration tests"
           command: |
-            ./scripts/run_integration_tests.sh
+            SONOBUOY_CLI=../../build/linux/amd64/sonobuoy ./scripts/run_integration_tests.sh
       - store_artifacts:
           path: /tmp/artifacts
       - run:
@@ -74,15 +118,26 @@ workflows:
     jobs:
       - check_go_mod
       - check_readme_sync
-      - build_and_test:
+      - build_clients
+      - build_containers:
+          requires:
+            - build_clients
+      - unit_tests
+      - integration_tests:
+          requires:
+            - build_clients
+            - build_containers
           filters:
             tags:
               only: /^v.*/
       - publish_images:
           requires:
             - check_go_mod
             - check_readme_sync
-            - build_and_test
+            - unit_tests
+            - build_clients
+            - build_containers
+            - integration_tests
           filters:
             branches:
               only: master

Makefile

@@ -77,25 +77,25 @@ GO_BUILD ?= CGO_ENABLED=0 $(GO_SYSTEM_FLAGS) go build -o $(BINARY) $(VERBOSE_FLA
 K8S_PATH ?= $(GOPATH)/src/github.com/kubernetes/kubernetes
 KIND_K8S_TAG ?= $(shell cd $(K8S_PATH) && git describe)
 
-.PHONY: all container push clean test local-test local generate plugins int
+.PHONY: all build_container containers build_sonobuoy push push_images push_manifest clean clean_image test local-test local int lint stress vet pre native deploy_kind kind_images push_kind_images check-kind-env
 
-all: container
+all: containers
 
 local-test:
 	$(TEST)
 
 # Unit tests
-test: sonobuoy vet
+test:
 	$(DOCKER_BUILD) 'CGO_ENABLED=0 $(TEST)'
 
 # Stress tests
-stress: sonobuoy
+stress:
 	$(DOCKER_BUILD) 'CGO_ENABLED=0 $(STRESS_TEST)'
 
 # Integration tests
-int: DOCKER_FLAGS=-v $(KUBECONFIG):/root/.kube/kubeconfig -v /tmp/artifacts:/tmp/artifacts --env ARTIFACTS_DIR=/tmp/artifacts --env KUBECONFIG=/root/.kube/kubeconfig --network host
+int: DOCKER_FLAGS=-v $(KUBECONFIG):/root/.kube/kubeconfig -v /tmp/artifacts:/tmp/artifacts --env ARTIFACTS_DIR=/tmp/artifacts --env KUBECONFIG=/root/.kube/kubeconfig --network host --env SONOBUOY_CLI=$(SONOBUOY_CLI)
 int: TESTARGS= $(VERBOSE_FLAG) -timeout 3m
-int: sonobuoy
+int:
 	$(DOCKER_BUILD) 'CGO_ENABLED=0 $(INT_TEST)'
 
 lint:
@@ -119,7 +119,7 @@ build_container:
        -f $(DOCKERFILE) \
 		.
 
-container: sonobuoy
+containers: build/linux/arm64/sonobuoy build/linux/amd64/sonobuoy
 	for arch in $(LINUX_ARCH); do \
 		if [ $$arch = amd64 ]; then \
 			sed -e 's|BASEIMAGE|$(AMD_IMAGE)|g' \
@@ -140,14 +140,18 @@ container: sonobuoy
 build_sonobuoy:
 	$(DOCKER_BUILD) '$(GO_BUILD)'
 
-sonobuoy:
-	for arch in $(LINUX_ARCH); do \
-		mkdir -p build/linux/$$arch; \
-		echo Building: linux/$$arch; \
-		$(MAKE) build_sonobuoy GO_SYSTEM_FLAGS="GOOS=linux GOARCH=$$arch" BINARY="build/linux/$$arch/sonobuoy"; \
-	done
-	@echo Building: host
-	$(MAKE) build_sonobuoy
+build/linux/arm64/sonobuoy:
+	echo Building: linux/arm64
+	mkdir -p build/linux/arm64
+	$(MAKE) build_sonobuoy GO_SYSTEM_FLAGS="GOOS=linux GOARCH=arm64" BINARY=$@
+
+build/linux/amd64/sonobuoy:
+	echo Building: linux/amd64
+	mkdir -p build/linux/amd64
+	$(MAKE) build_sonobuoy GO_SYSTEM_FLAGS="GOOS=linux GOARCH=amd64" BINARY=$@
+
+native:
+	$(GO_BUILD)
 
 push_images:
 	$(DOCKER) push $(REGISTRY)/$(TARGET):$(IMAGE_BRANCH)
@@ -164,7 +168,7 @@ push_images:
 push_manifest:
 	./manifest-tool push from-args --platforms $(PLATFORMS) --template $(REGISTRY)/$(TARGET)-ARCH:$(VERSION) --target $(REGISTRY)/$(TARGET):$(VERSION)
 
-push: pre container
+push: pre containers
 	for arch in $(LINUX_ARCH); do \
 		$(MAKE) push_images TARGET="sonobuoy-$$arch"; \
 	done
@@ -191,7 +195,7 @@ clean:
 		$(MAKE) clean_image TARGET=$(TARGET)-$$arch; \
 	done
 
-deploy_kind: container
+deploy_kind: containers
 	kind load docker-image --name $(KIND_CLUSTER) $(REGISTRY)/$(TARGET):$(IMAGE_VERSION) || true
 
 # kind_images will build the kind-node image. Generally building the base image is not necessary
@@ -212,6 +216,3 @@ ifndef KIND_K8S_TAG
 	$(error KIND_K8S_TAG is undefined)
 endif
 	echo --kube-root=$(K8S_PATH) tagging as --image $(REGISTRY)/kind-node:$(KIND_K8S_TAG)
-
-native:
-	$(GO_BUILD)

scripts/ci/publish.sh

@@ -10,7 +10,7 @@ fi
 
 function image_push() {
     echo ${DOCKERHUB_TOKEN} | docker login --username sonobuoybot --password-stdin
-    make container push
+    make containers push
 }
 
 if [ ! -z "$CIRCLE_TAG" ]; then

test/integration/sonobuoy_integration_test.go

@@ -271,6 +271,7 @@ func TestMain(m *testing.M) {
 		fmt.Printf("Skipping integration tests: failed to find sonobuoy CLI: %v\n", err)
 		os.Exit(1)
 	}
+	fmt.Printf("Using Sonobuoy CLI at %q\n", sonobuoy)
 
 	result := m.Run()
 	os.Exit(result)