[Feature] Ensure ServiceMonitor labels

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
+- Add Labels and Annotations to ServiceMonitor
 
 ## [1.0.4](https://github.com/arangodb/kube-arangodb/tree/1.0.4) (2020-07-28)
 - Add Encryption Key rotation feature for ArangoDB EE 3.7+

chart/kube-arangodb/templates/deployment-operator/role.yaml

@@ -30,7 +30,7 @@ rules:
       verbs: ["get", "list", "watch"]
     - apiGroups: ["monitoring.coreos.com"]
       resources: ["servicemonitors"]
-      verbs: ["get", "create", "delete", "update"]
+      verbs: ["get", "create", "delete", "update", "list", "watch", "patch"]
 
 {{- end }}
 {{- end }}

dashboard/assets.go

@@ -1,3 +1,23 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
 package dashboard
 
 import (

main.go

@@ -279,6 +279,11 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
 		return operator.Config{}, operator.Dependencies{}, maskAny(err)
 	}
 
+	kubeMonCli, err := k8sutil.NewKubeMonitoringV1Client()
+	if err != nil {
+		return operator.Config{}, operator.Dependencies{}, maskAny(err)
+	}
+
 	image, serviceAccount, err := getMyPodInfo(kubecli, namespace, name)
 	if err != nil {
 		return operator.Config{}, operator.Dependencies{}, maskAny(fmt.Errorf("Failed to get my pod's service account: %s", err))
@@ -320,6 +325,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
 		LogService:                 logService,
 		KubeCli:                    kubecli,
 		KubeExtCli:                 kubeExtCli,
+		KubeMonitoringCli:          kubeMonCli,
 		CRCli:                      crCli,
 		EventRecorder:              eventRecorder,
 		LivenessProbe:              &livenessProbe,

pkg/deployment/context_impl.go

@@ -31,6 +31,8 @@ import (
 	"strconv"
 	"time"
 
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/features"
 
 	"github.com/arangodb/go-driver/http"
@@ -78,6 +80,10 @@ func (d *Deployment) GetKubeCli() kubernetes.Interface {
 	return d.deps.KubeCli
 }
 
+func (d *Deployment) GetMonitoringV1Cli() monitoringClient.MonitoringV1Interface {
+	return d.deps.KubeMonitoringCli
+}
+
 // GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image)
 func (d *Deployment) GetLifecycleImage() string {
 	return d.config.LifecycleImage

pkg/deployment/deployment.go

@@ -29,6 +29,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	"github.com/arangodb/kube-arangodb/pkg/util/arangod/conn"
 
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
@@ -65,11 +67,12 @@ type Config struct {
 
 // Dependencies holds dependent services for a Deployment
 type Dependencies struct {
-	Log           zerolog.Logger
-	KubeCli       kubernetes.Interface
-	KubeExtCli    apiextensionsclient.Interface
-	DatabaseCRCli versioned.Interface
-	EventRecorder record.EventRecorder
+	Log               zerolog.Logger
+	KubeCli           kubernetes.Interface
+	KubeExtCli        apiextensionsclient.Interface
+	KubeMonitoringCli monitoringClient.MonitoringV1Interface
+	DatabaseCRCli     versioned.Interface
+	EventRecorder     record.EventRecorder
 }
 
 // deploymentEventType strongly typed type of event
@@ -237,7 +240,7 @@ func (d *Deployment) run() {
 	for {
 		select {
 		case <-d.stopCh:
-			cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace())
+			cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetNamespace())
 			if err != nil {
 				log.Error().Err(err).Msg("Unable to get resources")
 			}

pkg/deployment/deployment_inspector.go

@@ -65,7 +65,7 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval
 	deploymentName := d.apiObject.GetName()
 	defer metrics.SetDuration(inspectDeploymentDurationGauges.WithLabelValues(deploymentName), start)
 
-	cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace())
+	cachedStatus, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetNamespace())
 	if err != nil {
 		log.Error().Err(err).Msg("Unable to get resources")
 		return minInspectionInterval // Retry ASAP

pkg/deployment/deployment_run_test.go

@@ -62,7 +62,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) {
 
 		errs := 0
 		for {
-			cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace())
+			cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetNamespace())
 			require.NoError(t, err)
 			err = d.resources.EnsureSecrets(log.Logger, cache)
 			if err == nil {
@@ -104,7 +104,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) {
 		}
 
 		// Act
-		cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace())
+		cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetNamespace())
 		require.NoError(t, err)
 		err = d.resources.EnsurePods(cache)
 

pkg/deployment/deployment_suite_test.go

@@ -29,6 +29,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	monitoringFakeClient "github.com/coreos/prometheus-operator/pkg/client/versioned/fake"
+
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/arangod/conn"
@@ -419,17 +421,19 @@ func createTestDeployment(config Config, arangoDeployment *api.ArangoDeployment)
 
 	eventRecorder := recordfake.NewFakeRecorder(10)
 	kubernetesClientSet := fake.NewSimpleClientset()
+	monitoringClientSet := monitoringFakeClient.NewSimpleClientset()
 
 	arangoDeployment.ObjectMeta = metav1.ObjectMeta{
 		Name:      testDeploymentName,
 		Namespace: testNamespace,
 	}
 
 	deps := Dependencies{
-		Log:           zerolog.New(ioutil.Discard),
-		KubeCli:       kubernetesClientSet,
-		DatabaseCRCli: arangofake.NewSimpleClientset(&api.ArangoDeployment{}),
-		EventRecorder: eventRecorder,
+		Log:               zerolog.New(ioutil.Discard),
+		KubeCli:           kubernetesClientSet,
+		KubeMonitoringCli: monitoringClientSet.MonitoringV1(),
+		DatabaseCRCli:     arangofake.NewSimpleClientset(&api.ArangoDeployment{}),
+		EventRecorder:     eventRecorder,
 	}
 
 	d := &Deployment{

pkg/deployment/reconcile/plan_builder_test.go

@@ -28,6 +28,8 @@ import (
 	"io/ioutil"
 	"testing"
 
+	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+
 	"github.com/pkg/errors"
 
 	policy "k8s.io/api/policy/v1beta1"
@@ -548,6 +550,7 @@ func TestCreatePlan(t *testing.T) {
 		PVCS            map[string]*core.PersistentVolumeClaim
 		ServiceAccounts map[string]*core.ServiceAccount
 		PDBS            map[string]*policy.PodDisruptionBudget
+		ServiceMonitors map[string]*monitoring.ServiceMonitor
 	}{
 		{
 			Name: "Can not create plan for single deployment",
@@ -807,7 +810,7 @@ func TestCreatePlan(t *testing.T) {
 			if testCase.Helper != nil {
 				testCase.Helper(testCase.context.ArangoDeployment)
 			}
-			err, _ := r.CreatePlan(ctx, inspector.NewInspectorFromData(testCase.Pods, testCase.Secrets, testCase.PVCS, testCase.Services, testCase.ServiceAccounts, testCase.PDBS))
+			err, _ := r.CreatePlan(ctx, inspector.NewInspectorFromData(testCase.Pods, testCase.Secrets, testCase.PVCS, testCase.Services, testCase.ServiceAccounts, testCase.PDBS, testCase.ServiceMonitors))
 
 			// Assert
 			if testCase.ExpectedEvent != nil {

pkg/deployment/resources/annotations.go

@@ -25,6 +25,9 @@ package resources
 import (
 	"time"
 
+	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
+	monitoringTypedClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
 
 	"github.com/arangodb/kube-arangodb/pkg/apis/deployment"
@@ -41,6 +44,7 @@ import (
 
 func (r *Resources) EnsureAnnotations(cachedStatus inspector.Inspector) error {
 	kubecli := r.context.GetKubeCli()
+	monitoringcli := r.context.GetMonitoringV1Cli()
 
 	log.Info().Msgf("Ensuring annotations")
 
@@ -99,6 +103,15 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspector.Inspector) error {
 		return err
 	}
 
+	if err := ensureServiceMonitorsAnnotations(monitoringcli.ServiceMonitors(r.context.GetNamespace()),
+		cachedStatus,
+		deployment.ArangoDeploymentResourceKind,
+		r.context.GetAPIObject().GetName(),
+		r.context.GetAPIObject().GetNamespace(),
+		r.context.GetSpec().Annotations); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -287,6 +300,43 @@ func setPvcAnnotations(client typedCore.PersistentVolumeClaimInterface, persiste
 	})
 }
 
+func ensureServiceMonitorsAnnotations(client monitoringTypedClient.ServiceMonitorInterface, cachedStatus inspector.Inspector, kind, name, namespace string, annotations map[string]string) error {
+	if err := cachedStatus.IterateServiceMonitors(func(serviceMonitor *monitoring.ServiceMonitor) error {
+		if !k8sutil.CompareAnnotations(serviceMonitor.GetAnnotations(), annotations) {
+			log.Info().Msgf("Replacing annotations for ServiceMonitors %s", serviceMonitor.Name)
+			if err := setServiceMonitorAnnotations(client, serviceMonitor, annotations); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}, func(serviceMonitor *monitoring.ServiceMonitor) bool {
+		return k8sutil.IsChildResource(kind, name, namespace, serviceMonitor)
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func setServiceMonitorAnnotations(client monitoringTypedClient.ServiceMonitorInterface, serviceMonitor *monitoring.ServiceMonitor, annotations map[string]string) error {
+	return utils.Retry(5, 200*time.Millisecond, func() error {
+		currentServiceMonitor, err := client.Get(serviceMonitor.Name, meta.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceMonitor.Annotations = k8sutil.MergeAnnotations(k8sutil.GetSecuredAnnotations(currentServiceMonitor.Annotations), annotations)
+
+		_, err = client.Update(currentServiceMonitor)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 func getObjectGroup(obj meta.Object) api.ServerGroup {
 	l := obj.GetLabels()
 	if len(l) == 0 {

pkg/deployment/resources/context.go

@@ -25,6 +25,8 @@ package resources
 import (
 	"context"
 
+	monitoringClient "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
 
 	driver "github.com/arangodb/go-driver"
@@ -61,6 +63,8 @@ type Context interface {
 	UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error
 	// GetKubeCli returns the kubernetes client
 	GetKubeCli() kubernetes.Interface
+	// GetMonitoringV1Cli returns monitoring client
+	GetMonitoringV1Cli() monitoringClient.MonitoringV1Interface
 	// GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image)
 	GetLifecycleImage() string
 	// GetOperatorUUIDImage returns the image name containing the uuid helper (== name of operator image)