CSPL-194: SplunkEnterprise CRD Updates

Makefile

@@ -13,6 +13,10 @@ builder-image:
 	@mkdir -p ./build/_output/bin
 	@docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/opt/app-root/src/splunk-operator -w /opt/app-root/src/splunk-operator -u root -it splunk/splunk-operator-builder bash -c "operator-sdk build --verbose splunk/splunk-operator"
 
+builder-test:
+	@echo Running unit tests for splunk-operator inside of builder container
+	@docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/opt/app-root/src/splunk-operator -w /opt/app-root/src/splunk-operator -u root -it splunk/splunk-operator-builder bash -c "go test -v -covermode=count -coverprofile=coverage.out --timeout=300s github.com/splunk/splunk-operator/pkg/splunk/resources github.com/splunk/splunk-operator/pkg/splunk/spark github.com/splunk/splunk-operator/pkg/splunk/enterprise github.com/splunk/splunk-operator/pkg/splunk/deploy"
+
 image: deploy/all-in-one-scoped.yaml deploy/all-in-one-cluster.yaml
 	@echo Building splunk-operator image
 	@operator-sdk build --verbose splunk/splunk-operator
@@ -34,18 +38,18 @@ clean:
 	@docker rmi splunk/splunk-operator || true
 
 run:
-	@OPERATOR_NAME=splunk-operator operator-sdk up local
+	@OPERATOR_NAME=splunk-operator operator-sdk run --local
 
 fmt:
 	@gofmt -l -w `find ./ -name "*.go"`
 
 lint:
 	@golint ./...
 
-deploy/all-in-one-scoped.yaml: deploy/crds/enterprise_v1alpha1_splunkenterprise_crd.yaml deploy/rbac.yaml deploy/operator.yaml
+deploy/all-in-one-scoped.yaml: deploy/crds/enterprise.splunk.com_splunkenterprises_crd.yaml deploy/rbac.yaml deploy/operator.yaml
 	@echo Rebuilding deploy/all-in-one-scoped.yaml
-	@cat deploy/crds/enterprise_v1alpha1_splunkenterprise_crd.yaml deploy/rbac.yaml deploy/operator.yaml > deploy/all-in-one-scoped.yaml
+	@cat deploy/crds/enterprise.splunk.com_splunkenterprises_crd.yaml deploy/rbac.yaml deploy/operator.yaml > deploy/all-in-one-scoped.yaml
 
-deploy/all-in-one-cluster.yaml: deploy/crds/enterprise_v1alpha1_splunkenterprise_crd.yaml deploy/rbac.yaml deploy/cluster_operator.yaml
+deploy/all-in-one-cluster.yaml: deploy/crds/enterprise.splunk.com_splunkenterprises_crd.yaml deploy/rbac.yaml deploy/cluster_operator.yaml
 	@echo Rebuilding deploy/all-in-one-cluster.yaml
-	@cat deploy/crds/enterprise_v1alpha1_splunkenterprise_crd.yaml deploy/rbac.yaml deploy/cluster_operator.yaml > deploy/all-in-one-cluster.yaml
+	@cat deploy/crds/enterprise.splunk.com_splunkenterprises_crd.yaml deploy/rbac.yaml deploy/cluster_operator.yaml > deploy/all-in-one-cluster.yaml

README.md

@@ -26,16 +26,17 @@ You must have [Docker Engine](https://docs.docker.com/install/) installed to
 build the Splunk Operator.
 
 This project now uses [Go modules](https://blog.golang.org/using-go-modules),
-which requires [golang](https://golang.org/doc/install) 1.12 or later.
+which requires [golang](https://golang.org/doc/install) 1.13 or later.
 You must `export GO111MODULE=on` if cloning these repositories into your
 `$GOPATH` (not recommended).
 
 The [Kubernetes Operator SDK](https://github.com/operator-framework/operator-sdk)
 must also be installed to build this project.
 
 ```
-git clone -b v0.10.0 https://github.com/operator-framework/operator-sdk
+git clone -b v0.15.1 https://github.com/operator-framework/operator-sdk
 cd operator-sdk
+make tidy
 make install
 ```
 
@@ -64,6 +65,7 @@ Other make targets include (more info below):
 * `make all`: builds `splunk/splunk-operator` using the `splunk/splunk-operator-builder` image (same as `make builder builder-image`)
 * `make builder`: builds the `splunk/splunk-operator-builder` docker image
 * `make builder-image`: builds `splunk/splunk-operator` using the `splunk/splunk-operator-builder` image
+* `make builder-test`: Runs unit tests using the `splunk/splunk-operator-builder` image
 * `make image`: builds the `splunk/splunk-operator` docker image without using `splunk/splunk-operator-builder`
 * `make local`: builds the splunk-operator-local binary for test and debugging purposes
 * `make test`: Runs unit tests with Coveralls code coverage output to coverage.out

build/Dockerfile

@@ -7,7 +7,7 @@ ENV OPERATOR=/usr/local/bin/splunk-operator \
 LABEL name="splunk" \
       maintainer="support@splunk.com" \
       vendor="splunk" \
-      version="0.0.4" \
+      version="0.1.0" \
       release="1" \
       summary="Simplify the Deployment & Management of Splunk Products on Kubernetes" \
       description="The Splunk Operator for Kubernetes (SOK) makes it easy for Splunk Administrators to deploy and operate Enterprise deployments in a Kubernetes infrastructure. Packaged as a container, it uses the operator pattern to manage Splunk-specific custom resources, following best practices to manage all the underlying Kubernetes objects for you."

build/Dockerfile.builder

@@ -1,14 +1,36 @@
 # Note that go-toolset on UBI8 provides a FIPS-compatible compiler: https://developers.redhat.com/blog/2019/06/24/go-and-fips-140-2-on-red-hat-enterprise-linux/
+# Unfortunately, the latest release uses golang 1.12, which is no longer compatible with the operator-sdk
 # From https://access.redhat.com/containers/?tab=overview#/registry.access.redhat.com/ubi8/go-toolset
-FROM registry.access.redhat.com/ubi8/go-toolset:1.12.8-18
+#FROM registry.access.redhat.com/ubi8/go-toolset:1.12.8-18
+
+##################################################################
+## BEGIN: REPLACE ME WHEN go-toolset IS UPDATED TO GOLANG 1.13  ##
+## THIS IS A TEMPORARY HACK TO MIMIC go-toolset FOR GOLANG 1.13 ##
+##################################################################
+FROM registry.access.redhat.com/ubi8/ubi-minimal
+ENV HOME /opt/app-root/src
+ENV PATH ${PATH}:/usr/local/go/bin
+WORKDIR ${HOME}
+RUN ln -s /usr/bin/microdnf /usr/bin/dnf \
+    && dnf install -y wget tar gzip ca-certificates shadow-utils \
+    && dnf clean all \
+    && wget -O /tmp/go.tar.gz https://dl.google.com/go/go1.13.6.linux-amd64.tar.gz \
+    && tar -C /usr/local -xzf /tmp/go.tar.gz \
+    && rm -f /tmp/go.tar.gz \
+    && mkdir -p /opt/app-root \
+    && useradd -r -m -u 1000 -G root -d ${HOME} default \
+    && chown -R default.root /opt/app-root
+###############################################################
+## END: REPLACE ME WHEN go-toolset IS UPDATED TO GOLANG 1.13 ##
+###############################################################
 
 USER root
 
-ENV OPERATOR_SDK_VERSION 0.10.0
+ENV OPERATOR_SDK_VERSION 0.15.1
 ENV OPERATOR_SDK_URL https://github.com/operator-framework/operator-sdk/releases/download/v${OPERATOR_SDK_VERSION}/operator-sdk-v${OPERATOR_SDK_VERSION}-x86_64-linux-gnu
 ENV DOCKER_CLI_URL https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-cli-19.03.5-3.el7.x86_64.rpm
 
-RUN dnf install -y git openssh tar gzip ca-certificates \
+RUN dnf install -y git make gcc findutils openssh tar gzip ca-certificates \
     && dnf clean all \
     && wget -O /usr/local/bin/operator-sdk ${OPERATOR_SDK_URL} \
     && chmod a+x /usr/local/bin/operator-sdk \
@@ -21,6 +43,7 @@ USER default
 COPY --chown=default:root go.mod go.sum ${HOME}/initcache/
 
 ENV GOBIN /opt/app-root/bin
+ENV PATH ${PATH}:${GOBIN}
 
 RUN mkdir -p ${GOBIN} \
     && go get -u golang.org/x/lint/golint \

build/bin/entrypoint

@@ -5,7 +5,7 @@
 
 if ! whoami &>/dev/null; then
   if [ -w /etc/passwd ]; then
-    echo "${USER_NAME:-app-operator}:x:$(id -u):$(id -g):${USER_NAME:-app-operator} user:${HOME}:/sbin/nologin" >> /etc/passwd
+    echo "${USER_NAME:-splunk-operator}:x:$(id -u):$(id -g):${USER_NAME:-splunk-operator} user:${HOME}:/sbin/nologin" >> /etc/passwd
   fi
 fi
 

cmd/manager/main.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2019 Splunk Inc. All rights reserved.
+// Copyright (c) 2018-2020 Splunk Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -15,65 +15,192 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"flag"
-	"log"
+	"fmt"
+	"os"
 	"runtime"
 
+	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
+
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
+	kubemetrics "github.com/operator-framework/operator-sdk/pkg/kube-metrics"
+	"github.com/operator-framework/operator-sdk/pkg/leader"
+	"github.com/operator-framework/operator-sdk/pkg/log/zap"
+	"github.com/operator-framework/operator-sdk/pkg/metrics"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"github.com/spf13/pflag"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	"github.com/splunk/splunk-operator/pkg/apis"
 	"github.com/splunk/splunk-operator/pkg/controller"
 	"github.com/splunk/splunk-operator/version"
 )
 
+// Change below variables to serve metrics on different host or port.
+var (
+	metricsHost               = "0.0.0.0"
+	metricsPort         int32 = 8383
+	operatorMetricsPort int32 = 8686
+)
+var log = logf.Log.WithName("cmd")
+
 func printVersion() {
-	log.Printf("Splunk Operator for Kubernetes Version: %s", version.Version)
-	log.Printf("Go Version: %s", runtime.Version())
-	log.Printf("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)
-	log.Printf("operator-sdk Version: %v", sdkVersion.Version)
+	log.Info("Splunk Operator for Kubernetes",
+		"Version", version.Version,
+		"SDK.Version", sdkVersion.Version,
+		"Go.Version", runtime.Version(),
+		"Go.OS", runtime.GOOS,
+		"Go.Arch", runtime.GOARCH)
 }
 
 func main() {
+	// Add the zap logger flag set to the CLI. The flag set must
+	// be added before calling pflag.Parse().
+	pflag.CommandLine.AddFlagSet(zap.FlagSet())
+
+	// Add flags registered by imported packages (e.g. glog and
+	// controller-runtime)
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+
+	pflag.Parse()
+
+	// Use a zap logr.Logger implementation. If none of the zap
+	// flags are configured (or if the zap flag set is not being
+	// used), this defaults to a production zap logger.
+	//
+	// The logger instantiated here can be changed to any logger
+	// implementing the logr.Logger interface. This logger will
+	// be propagated through the whole operator, generating
+	// uniform and structured logs.
+	logf.SetLogger(zap.Logger())
+
 	printVersion()
-	flag.Parse()
 
 	namespace, err := k8sutil.GetWatchNamespace()
 	if err != nil {
-		log.Fatalf("failed to get watch namespace: %v", err)
+		log.Error(err, "Failed to get watch namespace")
+		os.Exit(1)
 	}
-	log.Printf("Watching namespace: %s", namespace)
 
 	// Get a config to talk to the apiserver
 	cfg, err := config.GetConfig()
 	if err != nil {
-		log.Fatal(err)
+		log.Error(err, "")
+		os.Exit(1)
+	}
+
+	ctx := context.TODO()
+	// Become the leader before proceeding
+	err = leader.Become(ctx, "splunk-operator-lock")
+	if err != nil {
+		log.Error(err, "")
+		os.Exit(1)
 	}
 
+	log.Info("Creating new manager", "namespace", namespace)
+
 	// Create a new Cmd to provide shared dependencies and start components
-	mgr, err := manager.New(cfg, manager.Options{Namespace: namespace})
+	mgr, err := manager.New(cfg, manager.Options{
+		Namespace:          namespace,
+		MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort),
+	})
 	if err != nil {
-		log.Fatal(err)
+		log.Error(err, "")
+		os.Exit(1)
 	}
 
-	log.Print("Registering Components.")
+	log.Info("Registering Components.")
 
 	// Setup Scheme for all resources
 	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
-		log.Fatal(err)
+		log.Error(err, "")
+		os.Exit(1)
 	}
 
 	// Setup all Controllers
 	if err := controller.AddToManager(mgr); err != nil {
-		log.Fatal(err)
+		log.Error(err, "")
+		os.Exit(1)
 	}
 
-	log.Print("Starting the Cmd.")
+	// Add the Metrics Service
+	addMetrics(ctx, cfg, namespace)
+
+	log.Info("Starting the Manager.")
 
 	// Start the Cmd
-	log.Fatal(mgr.Start(signals.SetupSignalHandler()))
+	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
+		log.Error(err, "Manager exited non-zero")
+		os.Exit(1)
+	}
+}
+
+// addMetrics will create the Services and Service Monitors to allow the operator export the metrics by using
+// the Prometheus operator
+func addMetrics(ctx context.Context, cfg *rest.Config, namespace string) {
+	if err := serveCRMetrics(cfg); err != nil {
+		if errors.Is(err, k8sutil.ErrRunLocal) {
+			log.Info("Skipping CR metrics server creation; not running in a cluster.")
+			return
+		}
+		log.Info("Could not generate and serve custom resource metrics", "error", err.Error())
+	}
+
+	// Add to the below struct any other metrics ports you want to expose.
+	servicePorts := []v1.ServicePort{
+		{Port: metricsPort, Name: metrics.OperatorPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: metricsPort}},
+		{Port: operatorMetricsPort, Name: metrics.CRPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: operatorMetricsPort}},
+	}
+
+	// Create Service object to expose the metrics port(s).
+	service, err := metrics.CreateMetricsService(ctx, cfg, servicePorts)
+	if err != nil {
+		log.Info("Could not create metrics Service", "error", err.Error())
+	}
+
+	// CreateServiceMonitors will automatically create the prometheus-operator ServiceMonitor resources
+	// necessary to configure Prometheus to scrape metrics from this operator.
+	services := []*v1.Service{service}
+	_, err = metrics.CreateServiceMonitors(cfg, namespace, services)
+	if err != nil {
+		log.Info("Could not create ServiceMonitor object", "error", err.Error())
+		// If this operator is deployed to a cluster without the prometheus-operator running, it will return
+		// ErrServiceMonitorNotPresent, which can be used to safely skip ServiceMonitor creation.
+		if err == metrics.ErrServiceMonitorNotPresent {
+			log.Info("Install prometheus-operator in your cluster to create ServiceMonitor objects", "error", err.Error())
+		}
+	}
+}
+
+// serveCRMetrics gets the Operator/CustomResource GVKs and generates metrics based on those types.
+// It serves those metrics on "http://metricsHost:operatorMetricsPort".
+func serveCRMetrics(cfg *rest.Config) error {
+	// Below function returns filtered operator/CustomResource specific GVKs.
+	// For more control override the below GVK list with your own custom logic.
+	filteredGVK, err := k8sutil.GetGVKsFromAddToScheme(apis.AddToScheme)
+	if err != nil {
+		return err
+	}
+	// Get the namespace the operator is currently deployed in.
+	operatorNs, err := k8sutil.GetOperatorNamespace()
+	if err != nil {
+		return err
+	}
+	// To generate metrics in other namespaces, add the values below.
+	ns := []string{operatorNs}
+	// Generate and serve custom resource specific metrics.
+	err = kubemetrics.GenerateAndServeCRMetrics(cfg, ns, filteredGVK, metricsHost, operatorMetricsPort)
+	if err != nil {
+		return err
+	}
+	return nil
 }