Merge pull request #1970 from rexagod/OCPBUGS-11958

OCPBUGS-11958: Add the trusted CA bundle in UWM Prometheus pods

assets/prometheus-user-workload/trusted-ca-bundle.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+data: {}
+kind: ConfigMap
+metadata:
+  labels:
+    config.openshift.io/inject-trusted-cabundle: "true"
+  name: prometheus-user-workload-trusted-ca-bundle
+  namespace: openshift-user-workload-monitoring

jsonnet/components/prometheus-user-workload.libsonnet

@@ -20,6 +20,8 @@ function(params)
     serviceMonitorCoreDNS:: {},
     secretEtcdCerts:: {},
 
+    trustedCaBundle: generateCertInjection.trustedCNOCaBundleCM(cfg.namespace, 'prometheus-user-workload-trusted-ca-bundle'),
+
     grpcTlsSecret: {
       apiVersion: 'v1',
       kind: 'Secret',

pkg/manifests/manifests.go

@@ -34,7 +34,6 @@ import (
 	consolev1 "github.com/openshift/api/console/v1"
 	routev1 "github.com/openshift/api/route/v1"
 	securityv1 "github.com/openshift/api/security/v1"
-	"github.com/openshift/cluster-monitoring-operator/pkg/promqlgen"
 	"github.com/openshift/library-go/pkg/crypto"
 	"github.com/pkg/errors"
 	monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -50,6 +49,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/yaml"
 	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
 	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+
+	"github.com/openshift/cluster-monitoring-operator/pkg/promqlgen"
 )
 
 const (
@@ -157,6 +158,7 @@ var (
 	PrometheusK8sTelemetry                        = "prometheus-k8s/telemetry-secret.yaml"
 
 	PrometheusUserWorkloadServingCertsCABundle        = "prometheus-user-workload/serving-certs-ca-bundle.yaml"
+	PrometheusUserWorkloadTrustedCABundle             = "prometheus-user-workload/trusted-ca-bundle.yaml"
 	PrometheusUserWorkloadServiceAccount              = "prometheus-user-workload/service-account.yaml"
 	PrometheusUserWorkloadClusterRole                 = "prometheus-user-workload/cluster-role.yaml"
 	PrometheusUserWorkloadClusterRoleBinding          = "prometheus-user-workload/cluster-role-binding.yaml"
@@ -1219,6 +1221,10 @@ func (f *Factory) PrometheusK8sTrustedCABundle() (*v1.ConfigMap, error) {
 	return f.NewConfigMap(f.assets.MustNewAssetReader(PrometheusK8sTrustedCABundle))
 }
 
+func (f *Factory) PrometheusUserWorkloadTrustedCABundle() (*v1.ConfigMap, error) {
+	return f.NewConfigMap(f.assets.MustNewAssetReader(PrometheusUserWorkloadTrustedCABundle))
+}
+
 func (f *Factory) NewPrometheusK8s() (*monv1.Prometheus, error) {
 	return f.NewPrometheus(f.assets.MustNewAssetReader(PrometheusK8s))
 }
@@ -1629,7 +1635,7 @@ func (f *Factory) PrometheusUserWorkloadAdditionalAlertManagerConfigsSecret() (*
 	}, nil
 }
 
-func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus, error) {
+func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret, trustedCABundleCM *v1.ConfigMap) (*monv1.Prometheus, error) {
 	p, err := f.NewPrometheus(f.assets.MustNewAssetReader(PrometheusUserWorkload))
 	if err != nil {
 		return nil, err
@@ -1705,6 +1711,25 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus,
 		return nil, err
 	}
 
+	var trustedCABundleVolumeName string
+	if trustedCABundleCM != nil {
+		trustedCABundleVolumeName = "prometheus-user-workload-trusted-ca-bundle"
+		volume := trustedCABundleVolume(trustedCABundleCM.Name, trustedCABundleVolumeName)
+		volume.VolumeSource.ConfigMap.Items = append(volume.VolumeSource.ConfigMap.Items, v1.KeyToPath{
+			Key:  TrustedCABundleKey,
+			Path: "tls-ca-bundle.pem",
+		})
+		p.Spec.Volumes = append(p.Spec.Volumes, volume)
+	}
+	p.Spec.Volumes = append(p.Spec.Volumes, v1.Volume{
+		Name: "secret-grpc-tls",
+		VolumeSource: v1.VolumeSource{
+			Secret: &v1.SecretVolumeSource{
+				SecretName: grpcTLS.GetName(),
+			},
+		},
+	})
+
 	for i, container := range p.Spec.Containers {
 		switch container.Name {
 		case "prometheus":
@@ -1717,6 +1742,13 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus,
 				PeriodSeconds:    15,
 				FailureThreshold: 240,
 			}
+			// Support CA bundles for Prometheus UWM.
+			if trustedCABundleVolumeName != "" {
+				p.Spec.Containers[i].VolumeMounts = append(
+					p.Spec.Containers[i].VolumeMounts,
+					trustedCABundleVolumeMount(trustedCABundleVolumeName),
+				)
+			}
 		case "kube-rbac-proxy-metrics", "kube-rbac-proxy-federate", "kube-rbac-proxy-thanos":
 			p.Spec.Containers[i].Image = f.config.Images.KubeRbacProxy
 			p.Spec.Containers[i].Args = f.setTLSSecurityConfiguration(container.Args, KubeRbacProxyTLSCipherSuitesFlag, KubeRbacProxyMinTLSVersionFlag)
@@ -1729,15 +1761,6 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus,
 		setupAlerting(p, platformAlertmanagerService, f.namespace)
 	}
 
-	p.Spec.Volumes = append(p.Spec.Volumes, v1.Volume{
-		Name: "secret-grpc-tls",
-		VolumeSource: v1.VolumeSource{
-			Secret: &v1.SecretVolumeSource{
-				SecretName: grpcTLS.GetName(),
-			},
-		},
-	})
-
 	alertManagerConfigs := f.config.AdditionalAlertmanagerConfigsForPrometheusUserWorkload()
 	if len(alertManagerConfigs) > 0 {
 		p.Spec.AdditionalAlertManagerConfigs = &v1.SecretKeySelector{

pkg/manifests/manifests_test.go

@@ -506,7 +506,10 @@ func TestUnconfiguredManifests(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = f.PrometheusUserWorkload(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}})
+	_, err = f.PrometheusUserWorkload(
+		&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+		&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -3725,6 +3728,7 @@ func TestNonHighlyAvailableInfrastructure(t *testing.T) {
 			getSpec: func(f *Factory) (spec, error) {
 				p, err := f.PrometheusUserWorkload(
 					&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
+					&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}},
 				)
 				if err != nil {
 					return spec{}, err

pkg/tasks/helpers.go

@@ -18,11 +18,12 @@ import (
 	"context"
 	"time"
 
-	"github.com/openshift/cluster-monitoring-operator/pkg/client"
-	"github.com/openshift/cluster-monitoring-operator/pkg/manifests"
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/openshift/cluster-monitoring-operator/pkg/client"
+	"github.com/openshift/cluster-monitoring-operator/pkg/manifests"
 )
 
 type caBundleSyncer struct {

pkg/tasks/prometheus_user_workload.go

@@ -17,10 +17,11 @@ package tasks
 import (
 	"context"
 
-	"github.com/openshift/cluster-monitoring-operator/pkg/client"
-	"github.com/openshift/cluster-monitoring-operator/pkg/manifests"
 	"github.com/pkg/errors"
 	"k8s.io/klog/v2"
+
+	"github.com/openshift/cluster-monitoring-operator/pkg/client"
+	"github.com/openshift/cluster-monitoring-operator/pkg/manifests"
 )
 
 type PrometheusUserWorkloadTask struct {
@@ -209,6 +210,22 @@ func (t *PrometheusUserWorkloadTask) create(ctx context.Context) error {
 		return errors.Wrap(err, "creating or updating UserWorkload Prometheus RBAC federate endpoint Secret failed")
 	}
 
+	trustedCA, err := t.factory.PrometheusUserWorkloadTrustedCABundle()
+	if err != nil {
+		return errors.Wrap(err, "initializing UserWorkload CA bundle ConfigMap failed")
+	}
+
+	cbs := &caBundleSyncer{
+		client:  t.client,
+		factory: t.factory,
+		prefix:  "prometheus-user-workload",
+	}
+
+	trustedCA, err = cbs.syncTrustedCABundle(ctx, trustedCA)
+	if err != nil {
+		return errors.Wrap(err, "syncing UserWorkload trusted CA bundle ConfigMap failed")
+	}
+
 	secret, err := t.factory.PrometheusUserWorkloadAdditionalAlertManagerConfigsSecret()
 	if err != nil {
 		return errors.Wrap(err, "initializing UserWorkload Prometheus additionalAlertmanagerConfigs secret failed")
@@ -232,7 +249,7 @@ func (t *PrometheusUserWorkloadTask) create(ctx context.Context) error {
 	}
 
 	klog.V(4).Info("initializing UserWorkload Prometheus object")
-	p, err := t.factory.PrometheusUserWorkload(s)
+	p, err := t.factory.PrometheusUserWorkload(s, trustedCA)
 	if err != nil {
 		return errors.Wrap(err, "initializing UserWorkload Prometheus object failed")
 	}
@@ -351,7 +368,22 @@ func (t *PrometheusUserWorkloadTask) destroy(ctx context.Context) error {
 		}
 	}
 
-	p, err := t.factory.PrometheusUserWorkload(s)
+	trustedCA, err := t.factory.PrometheusUserWorkloadTrustedCABundle()
+	if err != nil {
+		return errors.Wrap(err, "initializing UserWorkload CA bundle ConfigMap failed")
+	}
+
+	err = t.client.DeleteConfigMap(ctx, trustedCA)
+	if err != nil {
+		return errors.Wrap(err, "deleting UserWorkload trusted CA Bundle ConfigMap failed")
+	}
+
+	err = t.client.DeleteHashedConfigMap(ctx, trustedCA.GetNamespace(), "prometheus-user-workload", "")
+	if err != nil {
+		return errors.Wrap(err, "deleting UserWorkload trusted CA Bundle ConfigMap failed")
+	}
+
+	p, err := t.factory.PrometheusUserWorkload(s, nil)
 	if err != nil {
 		return errors.Wrap(err, "initializing UserWorkload Prometheus object failed")
 	}