Upgrade to operator sdk v0.10.0

Gopkg.toml

@@ -1,66 +1,53 @@
 # Force dep to vendor the code generators, which aren't imported just used at dev time.
 required = [
-  "k8s.io/code-generator/cmd/defaulter-gen",
-  "k8s.io/code-generator/cmd/deepcopy-gen",
-  "k8s.io/code-generator/cmd/conversion-gen",
-  "k8s.io/code-generator/cmd/client-gen",
-  "k8s.io/code-generator/cmd/lister-gen",
-  "k8s.io/code-generator/cmd/informer-gen",
-  "k8s.io/kube-openapi/cmd/openapi-gen",
-  "k8s.io/gengo/args",
   "sigs.k8s.io/controller-tools/pkg/crd/generator",
 ]
 
-[[override]]
-  name = "k8s.io/code-generator"
-  # revision for tag "kubernetes-1.13.1"
-  revision = "c2090bec4d9b1fb25de3812f868accc2bc9ecbae"
-
-[[override]]
-  name = "k8s.io/kube-openapi"
-  revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803"
-
 [[override]]
   name = "github.com/go-openapi/spec"
   branch = "master"
 
 [[override]]
   name = "sigs.k8s.io/controller-tools"
-  version = "=v0.1.8"
+  revision = "9d55346c2bde73fb3326ac22eac2e5210a730207"
 
 [[override]]
   name = "k8s.io/api"
-  # revision for tag "kubernetes-1.13.1"
-  revision = "05914d821849570fba9eacfb29466f2d8d3cd229"
+  # revision for tag "kubernetes-1.13.4"
+  revision = "5cb15d34447165a97c76ed5a60e4e99c8a01ecfe"
 
 [[override]]
   name = "k8s.io/apiextensions-apiserver"
-  # revision for tag "kubernetes-1.13.1"
-  revision = "0fe22c71c47604641d9aa352c785b7912c200562"
+  # revision for tag "kubernetes-1.13.4"
+  revision = "d002e88f6236312f0289d9d1deab106751718ff0"
 
 [[override]]
   name = "k8s.io/apimachinery"
-  # revision for tag "kubernetes-1.13.1"
-  revision = "2b1284ed4c93a43499e781493253e2ac5959c4fd"
+  # revision for tag "kubernetes-1.13.4"
+  revision = "86fb29eff6288413d76bd8506874fddd9fccdff0"
 
 [[override]]
   name = "k8s.io/client-go"
-  # revision for tag "kubernetes-1.13.1"
-  revision = "8d9ed539ba3134352c586810e749e58df4e94e4f"
+  # revision for tag "kubernetes-1.13.4"
+  revision = "b40b2a5939e43f7ffe0028ad67586b7ce50bb675"
 
 [[override]]
   name = "github.com/coreos/prometheus-operator"
   version = "=v0.29.0"
 
+[[override]]
+  name = "k8s.io/kube-state-metrics"
+  version = "v1.6.0"
+
 [[override]]
   name = "sigs.k8s.io/controller-runtime"
-  version = "=v0.1.10"
+  version = "=v0.1.12"
 
 [[constraint]]
   name = "github.com/operator-framework/operator-sdk"
   # The version rule is used for a specific release and the master branch for in between releases.
-  branch = "master" #osdk_branch_annotation
-  # version = "=v0.7.0" #osdk_version_annotation
+  # branch = "master" #osdk_branch_annotation
+  version = "=v0.10.0" #osdk_version_annotation
 
 [[constraint]]
   name = "github.com/enmasseproject/enmasse"
@@ -69,15 +56,4 @@ required = [
 [prune]
   go-tests = true
   non-go = true
-
-  [[prune.project]]
-    name = "k8s.io/code-generator"
-    non-go = false
-
-  [[prune.project]]
-    name = "k8s.io/gengo"
-    non-go = false
-
-  [[prune.project]]
-    name = "github.com/enmasseproject/enmasse"
-    unused-packages = true
+  unused-packages = true

Jenkinsfile

@@ -58,6 +58,20 @@ pipeline {
             }
         }
 
+        stage("Install Operator SDK") {
+            steps {
+                // qe-pipeline-library step
+                installOperatorSdk version: "v0.10.0"
+            }
+            post {
+                failure {
+                    echo "====++++'Install Operator SDK' execution failed++++===="
+                    echo "Please check if the version of operator-sdk you provided exists"
+                    echo "https://github.com/operator-framework/operator-sdk/releases"
+                }
+            }
+        }
+
         stage("Create an OpenShift project") {
             steps {
                 // qe-pipeline-library step

cmd/manager/main.go

@@ -7,24 +7,33 @@ import (
 	"os"
 	"runtime"
 
-	enmassev1beta "github.com/enmasseproject/enmasse/pkg/apis/enmasse/v1beta1"
-	messaginguserv1beta "github.com/enmasseproject/enmasse/pkg/apis/user/v1beta1"
-	routev1 "github.com/openshift/api/route/v1"
-
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 
 	"github.com/aerogear/unifiedpush-operator/pkg/apis"
 	"github.com/aerogear/unifiedpush-operator/pkg/controller"
 
 	openshiftappsv1 "github.com/openshift/api/apps/v1"
 	imagev1 "github.com/openshift/api/image/v1"
+	routev1 "github.com/openshift/api/route/v1"
+
+	enmassev1beta "github.com/enmasseproject/enmasse/pkg/apis/enmasse/v1beta1"
+	messaginguserv1beta "github.com/enmasseproject/enmasse/pkg/apis/user/v1beta1"
+
+	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+	monclientv1 "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
+	kubemetrics "github.com/operator-framework/operator-sdk/pkg/kube-metrics"
 	"github.com/operator-framework/operator-sdk/pkg/leader"
 	"github.com/operator-framework/operator-sdk/pkg/log/zap"
 	"github.com/operator-framework/operator-sdk/pkg/metrics"
+	"github.com/operator-framework/operator-sdk/pkg/restmapper"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
 	"github.com/spf13/pflag"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
@@ -33,8 +42,9 @@ import (
 
 // Change below variables to serve metrics on different host or port.
 var (
-	metricsHost       = "0.0.0.0"
-	metricsPort int32 = 8383
+	metricsHost               = "0.0.0.0"
+	metricsPort         int32 = 8383
+	operatorMetricsPort int32 = 8686
 )
 var log = logf.Log.WithName("cmd")
 
@@ -92,6 +102,7 @@ func main() {
 	// Create a new Cmd to provide shared dependencies and start components
 	mgr, err := manager.New(cfg, manager.Options{
 		Namespace:          namespace,
+		MapperProvider:     restmapper.NewDynamicRESTMapper,
 		MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort),
 	})
 	if err != nil {
@@ -141,10 +152,40 @@ func main() {
 		os.Exit(1)
 	}
 
-	// Create Service object to expose the metrics port.
-	_, err = metrics.ExposeMetricsPort(ctx, metricsPort)
+	operatorNamespace, err := k8sutil.GetOperatorNamespace()
 	if err != nil {
-		log.Info(err.Error())
+		log.Error(err, "")
+	}
+
+	if err = serveCRMetrics(cfg); err != nil {
+		log.Info("Could not generate and serve custom resource metrics", "error", err.Error())
+	}
+
+	// Add to the below struct any other metrics ports you want to expose.
+	servicePorts := []v1.ServicePort{
+		{Port: metricsPort, Name: metrics.OperatorPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: metricsPort}},
+		{Port: operatorMetricsPort, Name: metrics.CRPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: operatorMetricsPort}},
+	}
+
+	// Create Service object to expose the metrics port(s).
+	service, err := metrics.CreateMetricsService(ctx, cfg, servicePorts)
+	if err != nil {
+		log.Info("Could not create metrics Service", "error", err.Error())
+	}
+
+	err = addMonitoringKeyLabelToService(cfg, operatorNamespace, service)
+	if err != nil {
+		log.Error(err, "Could not add monitoring-key label to operator metrics Service")
+	}
+
+	err = createServiceMonitor(cfg, operatorNamespace, service)
+	if err != nil {
+		log.Info("Could not create ServiceMonitor object", "error", err.Error())
+		// If this operator is deployed to a cluster without the prometheus-operator running, it will return
+		// ErrServiceMonitorNotPresent, which can be used to safely skip ServiceMonitor creation.
+		if err == metrics.ErrServiceMonitorNotPresent {
+			log.Info("Install prometheus-operator in you cluster to create ServiceMonitor objects", "error", err.Error())
+		}
 	}
 
 	log.Info("Starting the Cmd.")
@@ -155,3 +196,67 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func addMonitoringKeyLabelToService(cfg *rest.Config, ns string, service *v1.Service) error {
+	kclient, err := client.New(cfg, client.Options{})
+	if err != nil {
+		return err
+	}
+
+	updatedLabels := map[string]string{"monitoring-key": "middleware"}
+	for k, v := range service.ObjectMeta.Labels {
+		updatedLabels[k] = v
+	}
+	service.ObjectMeta.Labels = updatedLabels
+
+	err = kclient.Update(context.TODO(), service)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// createServiceMonitor is a temporary fix until the version in the
+// operator-sdk is fixed to have the correct Path set on the Endpoints
+func createServiceMonitor(config *rest.Config, ns string, service *v1.Service) error {
+	mclient := monclientv1.NewForConfigOrDie(config)
+
+	sm := metrics.GenerateServiceMonitor(service)
+	eps := []monitoringv1.Endpoint{}
+	for _, ep := range sm.Spec.Endpoints {
+		eps = append(eps, monitoringv1.Endpoint{Port: ep.Port, Path: "/metrics"})
+	}
+	sm.Spec.Endpoints = eps
+
+	_, err := mclient.ServiceMonitors(ns).Create(sm)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// serveCRMetrics gets the Operator/CustomResource GVKs and generates metrics based on those types.
+// It serves those metrics on "http://metricsHost:operatorMetricsPort".
+func serveCRMetrics(cfg *rest.Config) error {
+	// Below function returns filtered operator/CustomResource specific GVKs.
+	// For more control override the below GVK list with your own custom logic.
+	filteredGVK, err := k8sutil.GetGVKsFromAddToScheme(apis.AddToScheme)
+	if err != nil {
+		return err
+	}
+	// Get the namespace the operator is currently deployed in.
+	operatorNs, err := k8sutil.GetOperatorNamespace()
+	if err != nil {
+		return err
+	}
+	// To generate metrics in other namespaces, add the values below.
+	ns := []string{operatorNs}
+	// Generate and serve custom resource specific metrics.
+	err = kubemetrics.GenerateAndServeCRMetrics(cfg, ns, filteredGVK, metricsHost, operatorMetricsPort)
+	if err != nil {
+		return err
+	}
+	return nil
+}

deploy/role.yaml

@@ -25,6 +25,7 @@ rules:
   - ""
   resources:
   - services
+  - services/finalizers
   - persistentvolumeclaims
   - events
   - configmaps
@@ -123,3 +124,17 @@ rules:
   - replicasets
   verbs:
   - get
+
+# Needed to be able to create ServiceMonitor for operator & operands
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+    - servicemonitors
+  verbs:
+    - get
+    - list
+    - watch
+    - create
+    - update
+    - patch
+    - delete