splunk · mikedickey · Jan 30, 2020 · Jan 23, 2020 · Jan 23, 2020 · Jan 23, 2020
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,16 +1,168 @@
-version: 2
-jobs:
-  compile-binary:
+# See https://circleci.com/blog/using-circleci-workflows-to-replicate-docker-hub-automated-builds/
+version: 2.1
+workflows:
+  version: 2
+  build-and-push:
+    jobs:
+      - build-image
+      - unit-tests
+      - sdk-tests:
+          requires:
+            - build-image
+      - push-latest:
+          requires:
+            - build-image
+            - unit-tests
+            - sdk-tests
+          filters:
+            branches:
+              only:
+                - master
+      - push-edge:
+          requires:
+            - build-image
+            - unit-tests
+            - sdk-tests
+          filters:
+            branches:
+              only:
+                - develop
+      - push-release:
+          requires:
+            - build-image
+            - unit-tests
+            - sdk-tests
+          filters:
+            tags:
+              only: /^\d+\.\d+\.\d+/
+            branches:
+              ignore: /.*/
+executors:
+  golang-builder:
+    environment:
+      IMAGE_NAME: splunk/splunk-operator
+      IMAGE_FILENAME: splunk-operator
+    working_directory: /opt/app-root/src/splunk-operator
+    docker:
+      - image: splunk/splunk-operator-builder:develop
+  docker-publisher:
+    environment:
+      IMAGE_NAME: splunk/splunk-operator
+      IMAGE_FILENAME: splunk-operator
     docker:
-      - image: splunk/splunk-operator-builder:master
+      - image: circleci/buildpack-deps:buster
+commands:
+  save_image:
+    description: "Save image"
+    steps:
+      - run:
+          name: "Save image"
+          command: |
+            mkdir -p /tmp/images
+            docker image save -o /tmp/images/${IMAGE_FILENAME}-${CIRCLE_SHA1}.tar ${IMAGE_NAME}:${CIRCLE_SHA1}
+  load_image:
+    description: "Load image"
+    steps:
+      - run:
+          name: "Load image"
+          command: docker load -i /tmp/images/${IMAGE_FILENAME}-${CIRCLE_SHA1}.tar
+  push_image:
+    description: "Load, tag and push an image"
+    parameters:
+      tag:
+        type: string
+    steps:
+      - load_image
+      - run: 
+          name: Tag image
+          command: |
+            docker tag ${IMAGE_NAME}:${CIRCLE_SHA1} ${IMAGE_NAME}:<< parameters.tag >>
+      - run:
+          name: Push latest image to DockerHub
+          command: |
+            echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+            docker push ${IMAGE_NAME}:<< parameters.tag >>
+jobs:
+  build-image:
+    executor: golang-builder
     steps:
       - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run: 
+          name: Pull base image updates
+          command: docker pull registry.access.redhat.com/ubi8/ubi-minimal:latest
+      - run: 
+          name: Build splunk/splunk-operator image
+          command: operator-sdk build --verbose ${IMAGE_NAME}:${CIRCLE_SHA1}
+      - save_image
+      - persist_to_workspace:
+          name: Persist images to workspace
+          root: /tmp
+          paths:
+            - images
+      - store_artifacts:
+          name: Save images as artifacts
+          path: /tmp/images
+  unit-tests:
+    executor: golang-builder
+    steps:
+      - checkout
+      - run: 
+          name: Check source formatting
+          command: X=`make fmt` && if [[ "x$X" != "x" ]]; then echo $X && false; fi
+      - run: 
+          name: Lint source code
+          command: X=`make lint` && if [[ "x$X" != "x" ]]; then echo $X && false; fi
       - run:
-          name: Build splunk-operator binary
-          command: go build -v -o ./build/_output/bin/splunk-operator ./cmd/manager
-workflows:
-  version: 2
-  build:
-    jobs:
-      - compile-binary
-
+          name: Run package tests
+          command: make test
+      - run:
+          name: Upload coverage.out
+          command: goveralls -coverprofile=coverage.out -service=circle-ci -repotoken $COVERALLS_TOKEN
+      - store_artifacts:
+          name: Save coverage.out as artifact
+          path: coverage.out
+  sdk-tests:
+    executor: golang-builder
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - attach_workspace:
+          name: Restore workspace
+          at: /tmp
+      - load_image
+      - run: 
+          name: Placeholder for e2e tests
+          command: docker inspect ${IMAGE_NAME}:${CIRCLE_SHA1}
+  push-latest:
+    executor: docker-publisher
+    steps:
+      - setup_remote_docker:
+          docker_layer_caching: false
+      - attach_workspace:
+          name: Restore workspace
+          at: /tmp
+      - push_image:
+          tag: "latest"
+  push-edge:
+    executor: docker-publisher
+    steps:
+      - setup_remote_docker:
+          docker_layer_caching: false
+      - attach_workspace:
+          name: Restore workspace
+          at: /tmp
+      - push_image:
+          tag: "edge"
+  push-release:
+    executor: docker-publisher
+    steps:
+      - setup_remote_docker:
+          docker_layer_caching: false
+      - attach_workspace:
+          name: Restore workspace
+          at: /tmp
+      - push_image:
+          tag: "${CIRCLE_TAG}"
diff --git a/Makefile b/Makefile
@@ -1,29 +1,34 @@
 # Makefile for Splunk Operator
 
-.PHONY: all image package local clean run fmt lint
+.PHONY: all builder builder-image image package local clean run fmt lint
 
-all: image
+all: builder builder-image
 
-builder-image:
+builder: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
 	@echo Creating container image to build splunk-operator
 	@docker build -f ./build/Dockerfile.builder -t splunk/splunk-operator-builder .
 
-builder: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
-	@echo Using container to build splunk-operator
+builder-image:
+	@echo Using builder container to build splunk-operator
 	@mkdir -p ./build/_output/bin
-	@docker run -v ${PWD}:/opt/app-root/src/splunk-operator -u root -it splunk/splunk-operator-builder bash -c "cd /opt/app-root/src/splunk-operator && go build -v -o ./build/_output/bin/splunk-operator ./cmd/manager"
+	@docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/opt/app-root/src/splunk-operator -w /opt/app-root/src/splunk-operator -u root -it splunk/splunk-operator-builder bash -c "operator-sdk build --verbose splunk/splunk-operator"
 
 image: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
 	@echo Building splunk-operator image
-	@operator-sdk build splunk/splunk-operator
-
-package: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
-	@build/package.sh
+	@operator-sdk build --verbose splunk/splunk-operator
 
 local: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
+	@echo Building splunk-operator-local binary only
 	@mkdir -p ./build/_output/bin
 	@go build -v -o ./build/_output/bin/splunk-operator-local ./cmd/manager
 
+test:
+	@echo Running unit tests for splunk-operator
+	@go test -v -covermode=count -coverprofile=coverage.out --timeout=300s github.com/splunk/splunk-operator/pkg/splunk/resources github.com/splunk/splunk-operator/pkg/splunk/spark github.com/splunk/splunk-operator/pkg/splunk/enterprise github.com/splunk/splunk-operator/pkg/splunk/deploy
+
+package: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
+	@build/package.sh
+
 clean:
 	@rm -rf ./build/_output
 	@docker rmi splunk/splunk-operator || true

diff --git a/README.md b/README.md
@@ -4,6 +4,7 @@
 [![GoDoc](https://godoc.org/github.com/splunk/splunk-operator?status.svg)](https://godoc.org/github.com/splunk/splunk-operator)
 [![Go Report Card](https://goreportcard.com/badge/github.com/splunk/splunk-operator)](https://goreportcard.com/report/github.com/splunk/splunk-operator)
 [![CircleCI](https://circleci.com/gh/splunk/splunk-operator/tree/develop.svg?style=shield)](https://circleci.com/gh/splunk/splunk-operator/tree/develop)
+[![Coverage Status](https://coveralls.io/repos/github/splunk/splunk-operator/badge.svg?branch=develop)](https://coveralls.io/github/splunk/splunk-operator?branch=develop)
 
 The Splunk Operator for Kubernetes (SOK) makes it easy for Splunk
 Administrators to deploy and operate Enterprise deployments in a Kubernetes
@@ -60,10 +61,13 @@ You can build the operator by just running `make`.
 
 Other make targets include (more info below):
 
-* `make all`: builds the `splunk/splunk-operator` docker image (same as `make image`)
-* `make image`: builds the `splunk/splunk-operator` docker image
-* `make package`: generates tarball of the `splunk/splunk-operator` docker image and installation YAML file
+* `make all`: builds `splunk/splunk-operator` using the `splunk/splunk-operator-builder` image (same as `make builder builder-image`)
+* `make builder`: builds the `splunk/splunk-operator-builder` docker image
+* `make builder-image`: builds `splunk/splunk-operator` using the `splunk/splunk-operator-builder` image
+* `make image`: builds the `splunk/splunk-operator` docker image without using `splunk/splunk-operator-builder`
 * `make local`: builds the splunk-operator-local binary for test and debugging purposes
+* `make test`: Runs unit tests with Coveralls code coverage output to coverage.out
+* `make package`: generates tarball of the `splunk/splunk-operator` docker image and installation YAML file
 * `make clean`: removes the binary build output and `splunk/splunk-operator` container image
 * `make run`: runs the splunk operator locally, monitoring the Kubernetes cluster configured in your current `kubectl` context
 * `make fmt`: runs `go fmt` on all `*.go` source files in this project

diff --git a/build/Dockerfile.builder b/build/Dockerfile.builder
@@ -1,12 +1,31 @@
 # Note that go-toolset on UBI8 provides a FIPS-compatible compiler: https://developers.redhat.com/blog/2019/06/24/go-and-fips-140-2-on-red-hat-enterprise-linux/
-FROM registry.access.redhat.com/ubi8/go-toolset
+# From https://access.redhat.com/containers/?tab=overview#/registry.access.redhat.com/ubi8/go-toolset
+FROM registry.access.redhat.com/ubi8/go-toolset:1.12.8-18
 
 USER root
 
-RUN dnf install -y git openssh tar gzip ca-certificates && dnf clean all
+ENV OPERATOR_SDK_VERSION 0.10.0
+ENV OPERATOR_SDK_URL https://github.com/operator-framework/operator-sdk/releases/download/v${OPERATOR_SDK_VERSION}/operator-sdk-v${OPERATOR_SDK_VERSION}-x86_64-linux-gnu
+ENV DOCKER_CLI_URL https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-cli-19.03.5-3.el7.x86_64.rpm
+
+RUN dnf install -y git openssh tar gzip ca-certificates \
+    && dnf clean all \
+    && wget -O /usr/local/bin/operator-sdk ${OPERATOR_SDK_URL} \
+    && chmod a+x /usr/local/bin/operator-sdk \
+    && wget -O /tmp/docker-cli.rpm ${DOCKER_CLI_URL} \
+    && rpm -ivh /tmp/docker-cli.rpm \
+    && rm /tmp/docker-cli.rpm
 
 USER default
 
-COPY --chown=default:root go.mod go.sum ${HOME}/
+COPY --chown=default:root go.mod go.sum ${HOME}/initcache/
+
+ENV GOBIN /opt/app-root/bin
 
-RUN go mod download && rm -rf go/pkg/mod/cache/vcs
+RUN mkdir -p ${GOBIN} \
+    && go get -u golang.org/x/lint/golint \
+    && go get -u golang.org/x/tools/cmd/cover \
+    && go get -u github.com/mattn/goveralls \
+    && cd ${HOME}/initcache \
+    && go mod download \
+    && rm -rf ${HOME}/go/pkg/mod/cache/vcs
diff --git a/go.mod b/go.mod
@@ -1,7 +1,6 @@
 module github.com/splunk/splunk-operator
 
 require (
-	github.com/NYTimes/gziphandler v1.0.1 // indirect
 	github.com/operator-framework/operator-sdk v0.11.1-0.20191014155558-888dde512025
 	k8s.io/api v0.0.0-20190612125737-db0771252981
 	k8s.io/apimachinery v0.0.0-20190612125636-6a5db36e93ad

diff --git a/pkg/splunk/deploy/deployment.go b/pkg/splunk/deploy/deployment.go
@@ -19,18 +19,16 @@ import (
 	"log"
 
 	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/splunk/splunk-operator/pkg/apis/enterprise/v1alpha1"
 	"github.com/splunk/splunk-operator/pkg/splunk/spark"
 )
 
 // ApplySparkDeployment creates or updates a Kubernetes Deployment for a given type of Spark instance.
-func ApplySparkDeployment(cr *v1alpha1.SplunkEnterprise, client client.Client, instanceType spark.InstanceType, replicas int, envVariables []corev1.EnvVar, ports []corev1.ContainerPort) error {
+func ApplySparkDeployment(cr *v1alpha1.SplunkEnterprise, client ControllerClient, instanceType spark.InstanceType, replicas int) error {
 
-	deployment, err := spark.GetSparkDeployment(cr, instanceType, replicas, envVariables, ports)
+	deployment, err := spark.GetSparkDeployment(cr, instanceType, replicas)
 	if err != nil {
 		return err
 	}
@@ -39,7 +37,7 @@ func ApplySparkDeployment(cr *v1alpha1.SplunkEnterprise, client client.Client, i
 }
 
 // ApplyDeployment creates or updates a Kubernetes Deployment
-func ApplyDeployment(client client.Client, deployment *appsv1.Deployment) error {
+func ApplyDeployment(client ControllerClient, deployment *appsv1.Deployment) error {
 
 	var current appsv1.Deployment
 	namespacedName := types.NamespacedName{