Fix ES servicemonitor for user-workload-monitoring

internal/elasticsearch/common.go

@@ -47,6 +47,14 @@ var defaultResources = map[string]v1.ResourceRequirements{
 	},
 }
 
+func serviceMonitorServiceAccountName(dplName string) string {
+	return fmt.Sprintf("%s-metrics", dplName)
+}
+
+func serviceCABundleName(dplName string) string {
+	return fmt.Sprintf("%s-ca-bundle", dplName)
+}
+
 func getESImage() string {
 	return utils.LookupEnvWithDefault("RELATED_IMAGE_ELASTICSEARCH", constants.ElasticsearchDefaultImage)
 }
@@ -241,7 +249,7 @@ func newProxyContainer(imageName, clusterName, namespace string, logConfig LogCo
 			"--upstream-ca=/etc/proxy/elasticsearch/admin-ca",
 			"--cache-expiry=60s",
 			`--auth-backend-role=admin_reader={"namespace": "default", "verb": "get", "resource": "pods/log"}`,
-			`--auth-backend-role=prometheus={"verb": "get", "resource": "/metrics"}`,
+			fmt.Sprintf(`--auth-backend-role=prometheus={"namespace":"%s", "verb": "get", "resource": "metrics", "resourceAPIGroup": "elasticsearch.openshift.io"}`, namespace),
 			`--auth-backend-role=jaeger={"verb": "get", "resource": "/jaeger", "resourceAPIGroup": "elasticsearch.jaegertracing.io"}`,
 			`--auth-backend-role=elasticsearch-operator={"namespace": "*", "verb": "*", "resource": "*", "resourceAPIGroup": "logging.openshift.io"}`,
 			fmt.Sprintf("--auth-backend-role=index-management={\"namespace\":\"%s\", \"verb\": \"*\", \"resource\": \"indices\", \"resourceAPIGroup\": \"elasticsearch.openshift.io\"}", namespace),

internal/elasticsearch/configmaps.go

@@ -88,6 +88,19 @@ func (er *ElasticsearchRequest) CreateOrUpdateConfigMaps() error {
 		}
 	}
 
+	cm = configmap.New(serviceCABundleName(dpl.Name), dpl.Namespace, dpl.Labels, nil)
+	cm.Annotations = map[string]string{
+		"service.beta.openshift.io/inject-cabundle": "true",
+	}
+
+	_, err = configmap.CreateOrUpdate(context.TODO(), er.client, cm, configmap.AnnotationsEqual, configmap.MutateAnnotationsOnly)
+	if err != nil {
+		return kverrors.Wrap(err, "failed to create or update elasticsearch ca-bundle configmap",
+			"cluster", er.cluster.Name,
+			"namespace", er.cluster.Namespace,
+		)
+	}
+
 	return nil
 }
 

internal/elasticsearch/rbac.go

@@ -4,11 +4,11 @@ import (
 	"context"
 
 	"github.com/ViaQ/logerr/v2/kverrors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	v1 "github.com/openshift/elasticsearch-operator/apis/logging/v1"
 	"github.com/openshift/elasticsearch-operator/internal/manifests/rbac"
 	rbacv1 "k8s.io/api/rbac/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 func (er *ElasticsearchRequest) CreateOrUpdateRBAC() error {
@@ -112,15 +112,15 @@ func (er *ElasticsearchRequest) CreateOrUpdateRBAC() error {
 
 	// metrics RBAC
 	metricsRole := rbac.NewRole(
-		"elasticsearch-metrics",
+		serviceMonitorServiceAccountName(dpl.Name),
 		dpl.Namespace,
 		rbac.NewPolicyRules(
 			rbac.NewPolicyRule(
-				[]string{""},
-				[]string{"pods", "services", "endpoints"},
-				[]string{},
-				[]string{"list", "watch"},
-				[]string{},
+				[]string{"elasticsearch.openshift.io"},
+				[]string{"metrics"},
+				nil,
+				[]string{"get"},
+				nil,
 			),
 		),
 	)
@@ -129,7 +129,6 @@ func (er *ElasticsearchRequest) CreateOrUpdateRBAC() error {
 	if err != nil {
 		return kverrors.Wrap(err, "failed to create or update elasticsearch metrics role",
 			"cluster", dpl.Name,
-			"namespace", dpl.Namespace,
 		)
 	}
 
@@ -142,15 +141,15 @@ func (er *ElasticsearchRequest) CreateOrUpdateRBAC() error {
 
 	subject := rbac.NewSubject(
 		"ServiceAccount",
-		"prometheus-k8s",
-		"openshift-monitoring",
+		serviceMonitorServiceAccountName(dpl.Name),
+		dpl.Namespace,
 	)
 	subject.APIGroup = ""
 
 	metricsRoleBinding := rbac.NewRoleBinding(
-		"elasticsearch-metrics",
+		serviceMonitorServiceAccountName(dpl.Name),
 		dpl.Namespace,
-		"elasticsearch-metrics",
+		serviceMonitorServiceAccountName(dpl.Name),
 		rbac.NewSubjects(subject),
 	)
 

internal/elasticsearch/reconciler.go

@@ -154,7 +154,7 @@ func Reconcile(log logr.Logger, requestCluster *elasticsearchv1.Elasticsearch, r
 	}
 
 	// Ensure existence of servicesaccount
-	if err := elasticsearchRequest.CreateOrUpdateServiceAccount(); err != nil {
+	if err := elasticsearchRequest.CreateOrUpdateServiceAccounts(); err != nil {
 		return kverrors.Wrap(err, "Failed to reconcile ServiceAccount for Elasticsearch cluster")
 	}
 

internal/elasticsearch/service_monitor.go

@@ -5,20 +5,30 @@ import (
 	"fmt"
 
 	"github.com/ViaQ/logerr/v2/kverrors"
+	"github.com/openshift/elasticsearch-operator/internal/manifests/secret"
+	"github.com/openshift/elasticsearch-operator/internal/manifests/serviceaccount"
 	"github.com/openshift/elasticsearch-operator/internal/manifests/servicemonitor"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
-	prometheusCAFile = "/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt"
+	prometheusCAFile = "service-ca.crt"
 )
 
 // CreateOrUpdateServiceMonitors ensures the existence of ServiceMonitors for Elasticsearch cluster
 func (er *ElasticsearchRequest) CreateOrUpdateServiceMonitors() error {
 	dpl := er.cluster
+
+	tokenName, err := er.getServiceAccountTokenSecretName()
+	if err != nil {
+		return kverrors.Wrap(err, "failed to fetch elasticsearch-metrics token secret name")
+	}
+
 	serviceMonitorName := fmt.Sprintf("monitor-%s-%s", dpl.Name, "cluster")
 
 	labelsWithDefault := appendDefaultLabel(dpl.Name, dpl.Labels)
@@ -28,25 +38,40 @@ func (er *ElasticsearchRequest) CreateOrUpdateServiceMonitors() error {
 
 	tlsConfig := monitoringv1.TLSConfig{
 		SafeTLSConfig: monitoringv1.SafeTLSConfig{
+			CA: monitoringv1.SecretOrConfigMap{
+				ConfigMap: &corev1.ConfigMapKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: serviceCABundleName(dpl.Name),
+					},
+					Key: prometheusCAFile,
+				},
+			},
 			// ServerName can be e.g. elasticsearch-metrics.openshift-logging.svc
 			ServerName: fmt.Sprintf("%s-%s.%s.svc", dpl.Name, "metrics", dpl.Namespace),
 		},
-		CAFile: prometheusCAFile,
 	}
+
+	tokenSecret := corev1.SecretKeySelector{
+		LocalObjectReference: corev1.LocalObjectReference{
+			Name: tokenName,
+		},
+		Key: "token",
+	}
+
 	endpoints := []monitoringv1.Endpoint{
 		{
-			Port:            dpl.Name,
-			Path:            "/metrics",
-			Scheme:          "https",
-			BearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token",
-			TLSConfig:       &tlsConfig,
+			Port:              dpl.Name,
+			Path:              "/metrics",
+			Scheme:            "https",
+			TLSConfig:         &tlsConfig,
+			BearerTokenSecret: tokenSecret,
 		},
 		{
-			Port:            dpl.Name,
-			Path:            "/_prometheus/metrics",
-			Scheme:          "https",
-			BearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token",
-			TLSConfig:       &tlsConfig,
+			Port:              dpl.Name,
+			Path:              "/_prometheus/metrics",
+			Scheme:            "https",
+			TLSConfig:         &tlsConfig,
+			BearerTokenSecret: tokenSecret,
 		},
 	}
 
@@ -63,7 +88,7 @@ func (er *ElasticsearchRequest) CreateOrUpdateServiceMonitors() error {
 
 	dpl.AddOwnerRefTo(monitor)
 
-	err := servicemonitor.CreateOrUpdate(context.TODO(), er.client, monitor, servicemonitor.Equal, servicemonitor.Mutate)
+	err = servicemonitor.CreateOrUpdate(context.TODO(), er.client, monitor, servicemonitor.Equal, servicemonitor.Mutate)
 	if err != nil {
 		return kverrors.Wrap(err, "failed to create or update elasticsearch servicemonitor",
 			"cluster", er.cluster.Name,
@@ -73,3 +98,47 @@ func (er *ElasticsearchRequest) CreateOrUpdateServiceMonitors() error {
 
 	return nil
 }
+
+func (er *ElasticsearchRequest) getServiceAccountTokenSecretName() (string, error) {
+	key := client.ObjectKey{
+		Name:      serviceMonitorServiceAccountName(er.cluster.Name),
+		Namespace: er.cluster.Namespace,
+	}
+
+	sa, err := serviceaccount.Get(context.TODO(), er.client, key)
+	if err != nil {
+		return "", kverrors.Wrap(err, "failed to get elasticsearch-metrics serviceacount",
+			"cluster", er.cluster.Name,
+			"namespace", er.cluster.Namespace,
+			"key", key,
+		)
+	}
+
+	var tokenSecret string
+	for _, oref := range sa.Secrets {
+		key := client.ObjectKey{Name: oref.Name, Namespace: er.cluster.Namespace}
+		secret, err := secret.Get(context.TODO(), er.client, key)
+		if err != nil {
+			return "", kverrors.Wrap(err, "failed to get elasticsearch-metrics serviceacount secret",
+				"cluster", er.cluster.Name,
+				"namespace", er.cluster.Namespace,
+				"key", key,
+			)
+		}
+
+		if secret.Type == corev1.SecretTypeServiceAccountToken {
+			tokenSecret = secret.Name
+			break
+		}
+	}
+
+	if tokenSecret == "" {
+		return "", kverrors.New(
+			"failed to fetch elasticearch-metrics service account token secret name",
+			"cluster", er.cluster.Name,
+			"namespace", er.cluster.Namespace,
+		)
+	}
+
+	return tokenSecret, nil
+}