From e9a8c72bae2ee837cda998de97928e4f6031310a Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava Date: Tue, 23 May 2023 01:17:00 +0530 Subject: [PATCH 1/3] Add the trusted CA bundle in UWM Prometheus pods Add the trusted CA bundle in UWM Prometheus pods, so users can secure the remote-write endpoint, in response to [OCPBUGS-11958](https://issues.redhat.com/browse/OCPBUGS-11958). Signed-off-by: Pranshu Srivastava --- .../trusted-ca-bundle.yaml | 8 +++ .../prometheus-user-workload.libsonnet | 2 + pkg/manifests/manifests.go | 31 ++++++++++- pkg/manifests/manifests_test.go | 6 ++- pkg/tasks/prometheus_user_workload.go | 51 +++++++++++++++++-- 5 files changed, 91 insertions(+), 7 deletions(-) create mode 100644 assets/prometheus-user-workload/trusted-ca-bundle.yaml diff --git a/assets/prometheus-user-workload/trusted-ca-bundle.yaml b/assets/prometheus-user-workload/trusted-ca-bundle.yaml new file mode 100644 index 0000000000..d9a97c6d9c --- /dev/null +++ b/assets/prometheus-user-workload/trusted-ca-bundle.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + labels: + config.openshift.io/inject-trusted-cabundle: "true" + name: prometheus-user-workload-trusted-ca-bundle + namespace: openshift-user-workload-monitoring diff --git a/jsonnet/components/prometheus-user-workload.libsonnet b/jsonnet/components/prometheus-user-workload.libsonnet index 8e3c4a93a1..350222a799 100644 --- a/jsonnet/components/prometheus-user-workload.libsonnet +++ b/jsonnet/components/prometheus-user-workload.libsonnet @@ -20,6 +20,8 @@ function(params) serviceMonitorCoreDNS:: {}, secretEtcdCerts:: {}, + trustedCaBundle: generateCertInjection.trustedCNOCaBundleCM(cfg.namespace, 'prometheus-user-workload-trusted-ca-bundle'), + grpcTlsSecret: { apiVersion: 'v1', kind: 'Secret', diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go index e84358c61a..58609fee47 100644 --- a/pkg/manifests/manifests.go +++ b/pkg/manifests/manifests.go @@ -33,7 +33,6 @@ import ( configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" securityv1 "github.com/openshift/api/security/v1" - "github.com/openshift/cluster-monitoring-operator/pkg/promqlgen" "github.com/openshift/library-go/pkg/crypto" "github.com/pkg/errors" monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -47,6 +46,8 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + + "github.com/openshift/cluster-monitoring-operator/pkg/promqlgen" ) const ( @@ -154,6 +155,7 @@ var ( PrometheusK8sTelemetry = "prometheus-k8s/telemetry-secret.yaml" PrometheusUserWorkloadServingCertsCABundle = "prometheus-user-workload/serving-certs-ca-bundle.yaml" + PrometheusUserWorkloadTrustedCABundle = "prometheus-user-workload/trusted-ca-bundle.yaml" PrometheusUserWorkloadServiceAccount = "prometheus-user-workload/service-account.yaml" PrometheusUserWorkloadClusterRole = "prometheus-user-workload/cluster-role.yaml" PrometheusUserWorkloadClusterRoleBinding = "prometheus-user-workload/cluster-role-binding.yaml" @@ -1201,6 +1203,10 @@ func (f *Factory) PrometheusK8sTrustedCABundle() (*v1.ConfigMap, error) { return f.NewConfigMap(f.assets.MustNewAssetReader(PrometheusK8sTrustedCABundle)) } +func (f *Factory) PrometheusUserWorkloadTrustedCABundle() (*v1.ConfigMap, error) { + return f.NewConfigMap(f.assets.MustNewAssetReader(PrometheusUserWorkloadTrustedCABundle)) +} + func (f *Factory) NewPrometheusK8s() (*monv1.Prometheus, error) { return f.NewPrometheus(f.assets.MustNewAssetReader(PrometheusK8s)) } @@ -1611,7 +1617,7 @@ func (f *Factory) PrometheusUserWorkloadAdditionalAlertManagerConfigsSecret() (* }, nil } -func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus, error) { +func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret, trustedCABundleCM *v1.ConfigMap) (*monv1.Prometheus, error) { p, err := f.NewPrometheus(f.assets.MustNewAssetReader(PrometheusUserWorkload)) if err != nil { return nil, err @@ -1731,6 +1737,27 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret) (*monv1.Prometheus, p.Spec.Secrets = append(p.Spec.Secrets, getAdditionalAlertmanagerSecrets(alertManagerConfigs)...) } + if trustedCABundleCM != nil { + volumeName := "prometheus-user-workload-trusted-ca-bundle" + volume := trustedCABundleVolume(trustedCABundleCM.Name, volumeName) + volume.VolumeSource.ConfigMap.Items = append(volume.VolumeSource.ConfigMap.Items, v1.KeyToPath{ + Key: TrustedCABundleKey, + Path: "tls-ca-bundle.pem", + }) + p.Spec.Volumes = append(p.Spec.Volumes, volume) + + // we only need the trusted CA bundle in: + // 1. Prometheus, because users might want to configure external remote write. + for i, container := range p.Spec.Containers { + if container.Name == "prometheus" { + p.Spec.Containers[i].VolumeMounts = append( + p.Spec.Containers[i].VolumeMounts, + trustedCABundleVolumeMount(volumeName), + ) + } + } + } + return p, nil } diff --git a/pkg/manifests/manifests_test.go b/pkg/manifests/manifests_test.go index 98e96d44a4..1fbe9eef2a 100644 --- a/pkg/manifests/manifests_test.go +++ b/pkg/manifests/manifests_test.go @@ -506,7 +506,10 @@ func TestUnconfiguredManifests(t *testing.T) { t.Fatal(err) } - _, err = f.PrometheusUserWorkload(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) + _, err = f.PrometheusUserWorkload( + &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + ) if err != nil { t.Fatal(err) } @@ -3700,6 +3703,7 @@ func TestNonHighlyAvailableInfrastructure(t *testing.T) { getSpec: func(f *Factory) (spec, error) { p, err := f.PrometheusUserWorkload( &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, + &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}, ) if err != nil { return spec{}, err diff --git a/pkg/tasks/prometheus_user_workload.go b/pkg/tasks/prometheus_user_workload.go index b8eb5e2c83..59ad387403 100644 --- a/pkg/tasks/prometheus_user_workload.go +++ b/pkg/tasks/prometheus_user_workload.go @@ -17,10 +17,11 @@ package tasks import ( "context" - "github.com/openshift/cluster-monitoring-operator/pkg/client" - "github.com/openshift/cluster-monitoring-operator/pkg/manifests" "github.com/pkg/errors" "k8s.io/klog/v2" + + "github.com/openshift/cluster-monitoring-operator/pkg/client" + "github.com/openshift/cluster-monitoring-operator/pkg/manifests" ) type PrometheusUserWorkloadTask struct { @@ -209,6 +210,22 @@ func (t *PrometheusUserWorkloadTask) create(ctx context.Context) error { return errors.Wrap(err, "creating or updating UserWorkload Prometheus RBAC federate endpoint Secret failed") } + trustedCA, err := t.factory.PrometheusUserWorkloadTrustedCABundle() + if err != nil { + return errors.Wrap(err, "initializing UserWorkload CA bundle ConfigMap failed") + } + + cbs := &caBundleSyncer{ + client: t.client, + factory: t.factory, + prefix: "prometheus-user-workload", + } + + trustedCA, err = cbs.syncTrustedCABundle(ctx, trustedCA) + if err != nil { + return errors.Wrap(err, "syncing UserWorkload trusted CA bundle ConfigMap failed") + } + secret, err := t.factory.PrometheusUserWorkloadAdditionalAlertManagerConfigsSecret() if err != nil { return errors.Wrap(err, "initializing UserWorkload Prometheus additionalAlertmanagerConfigs secret failed") @@ -232,7 +249,7 @@ func (t *PrometheusUserWorkloadTask) create(ctx context.Context) error { } klog.V(4).Info("initializing UserWorkload Prometheus object") - p, err := t.factory.PrometheusUserWorkload(s) + p, err := t.factory.PrometheusUserWorkload(s, trustedCA) if err != nil { return errors.Wrap(err, "initializing UserWorkload Prometheus object failed") } @@ -347,7 +364,33 @@ func (t *PrometheusUserWorkloadTask) destroy(ctx context.Context) error { } } - p, err := t.factory.PrometheusUserWorkload(s) + trustedCA, err := t.factory.PrometheusUserWorkloadTrustedCABundle() + if err != nil { + return errors.Wrap(err, "initializing UserWorkload CA bundle ConfigMap failed") + } + + cbs := &caBundleSyncer{ + client: t.client, + factory: t.factory, + prefix: "prometheus-user-workload", + } + + hashedTrustedCA, err := cbs.syncTrustedCABundle(ctx, trustedCA) + if err != nil { + return errors.Wrap(err, "syncing UserWorkload trusted CA bundle ConfigMap failed") + } + + err = t.client.DeleteConfigMap(ctx, trustedCA) + if err != nil { + return errors.Wrap(err, "deleting UserWorkload trusted CA Bundle ConfigMap failed") + } + + err = t.client.DeleteConfigMap(ctx, hashedTrustedCA) + if err != nil { + return errors.Wrap(err, "deleting UserWorkload trusted CA Bundle ConfigMap failed") + } + + p, err := t.factory.PrometheusUserWorkload(s, nil) if err != nil { return errors.Wrap(err, "initializing UserWorkload Prometheus object failed") } From ce9696698ef4ed554c33ef8493c0e73dcdb9605e Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava Date: Mon, 29 May 2023 21:52:55 +0530 Subject: [PATCH 2/3] fixup! Add the trusted CA bundle in UWM Prometheus pods --- pkg/manifests/manifests.go | 56 +++++++++++++-------------- pkg/tasks/helpers.go | 7 ++-- pkg/tasks/prometheus_user_workload.go | 2 +- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go index 58609fee47..da7509c0b6 100644 --- a/pkg/manifests/manifests.go +++ b/pkg/manifests/manifests.go @@ -1693,6 +1693,25 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret, trustedCABundleCM * return nil, err } + var trustedCABundleVolumeName string + if trustedCABundleCM != nil { + trustedCABundleVolumeName = "prometheus-user-workload-trusted-ca-bundle" + volume := trustedCABundleVolume(trustedCABundleCM.Name, trustedCABundleVolumeName) + volume.VolumeSource.ConfigMap.Items = append(volume.VolumeSource.ConfigMap.Items, v1.KeyToPath{ + Key: TrustedCABundleKey, + Path: "tls-ca-bundle.pem", + }) + p.Spec.Volumes = append(p.Spec.Volumes, volume) + } + p.Spec.Volumes = append(p.Spec.Volumes, v1.Volume{ + Name: "secret-grpc-tls", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: grpcTLS.GetName(), + }, + }, + }) + for i, container := range p.Spec.Containers { switch container.Name { case "prometheus": @@ -1705,6 +1724,13 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret, trustedCABundleCM * PeriodSeconds: 15, FailureThreshold: 240, } + // Support CA bundles for Prometheus UWM. + if trustedCABundleVolumeName != "" { + p.Spec.Containers[i].VolumeMounts = append( + p.Spec.Containers[i].VolumeMounts, + trustedCABundleVolumeMount(trustedCABundleVolumeName), + ) + } case "kube-rbac-proxy-metrics", "kube-rbac-proxy-federate", "kube-rbac-proxy-thanos": p.Spec.Containers[i].Image = f.config.Images.KubeRbacProxy p.Spec.Containers[i].Args = f.setTLSSecurityConfiguration(container.Args, KubeRbacProxyTLSCipherSuitesFlag, KubeRbacProxyMinTLSVersionFlag) @@ -1717,15 +1743,6 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret, trustedCABundleCM * setupAlerting(p, platformAlertmanagerService, f.namespace) } - p.Spec.Volumes = append(p.Spec.Volumes, v1.Volume{ - Name: "secret-grpc-tls", - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: grpcTLS.GetName(), - }, - }, - }) - alertManagerConfigs := f.config.AdditionalAlertmanagerConfigsForPrometheusUserWorkload() if len(alertManagerConfigs) > 0 { p.Spec.AdditionalAlertManagerConfigs = &v1.SecretKeySelector{ @@ -1737,27 +1754,6 @@ func (f *Factory) PrometheusUserWorkload(grpcTLS *v1.Secret, trustedCABundleCM * p.Spec.Secrets = append(p.Spec.Secrets, getAdditionalAlertmanagerSecrets(alertManagerConfigs)...) } - if trustedCABundleCM != nil { - volumeName := "prometheus-user-workload-trusted-ca-bundle" - volume := trustedCABundleVolume(trustedCABundleCM.Name, volumeName) - volume.VolumeSource.ConfigMap.Items = append(volume.VolumeSource.ConfigMap.Items, v1.KeyToPath{ - Key: TrustedCABundleKey, - Path: "tls-ca-bundle.pem", - }) - p.Spec.Volumes = append(p.Spec.Volumes, volume) - - // we only need the trusted CA bundle in: - // 1. Prometheus, because users might want to configure external remote write. - for i, container := range p.Spec.Containers { - if container.Name == "prometheus" { - p.Spec.Containers[i].VolumeMounts = append( - p.Spec.Containers[i].VolumeMounts, - trustedCABundleVolumeMount(volumeName), - ) - } - } - } - return p, nil } diff --git a/pkg/tasks/helpers.go b/pkg/tasks/helpers.go index daaf73349f..3113540487 100644 --- a/pkg/tasks/helpers.go +++ b/pkg/tasks/helpers.go @@ -18,11 +18,12 @@ import ( "context" "time" - "github.com/openshift/cluster-monitoring-operator/pkg/client" - "github.com/openshift/cluster-monitoring-operator/pkg/manifests" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openshift/cluster-monitoring-operator/pkg/client" + "github.com/openshift/cluster-monitoring-operator/pkg/manifests" ) type caBundleSyncer struct { @@ -83,7 +84,7 @@ func (cbs *caBundleSyncer) syncTrustedCABundle(ctx context.Context, trustedCA *v ctx, trustedCA.GetNamespace(), cbs.prefix, - string(hashedCM.Labels["monitoring.openshift.io/hash"]), + hashedCM.Labels["monitoring.openshift.io/hash"], ) return hashedCM, errors.Wrap(err, "deleting old trusted CA bundle configmaps failed") } diff --git a/pkg/tasks/prometheus_user_workload.go b/pkg/tasks/prometheus_user_workload.go index 59ad387403..073543b1e7 100644 --- a/pkg/tasks/prometheus_user_workload.go +++ b/pkg/tasks/prometheus_user_workload.go @@ -385,7 +385,7 @@ func (t *PrometheusUserWorkloadTask) destroy(ctx context.Context) error { return errors.Wrap(err, "deleting UserWorkload trusted CA Bundle ConfigMap failed") } - err = t.client.DeleteConfigMap(ctx, hashedTrustedCA) + err = t.client.DeleteHashedConfigMap(ctx, hashedTrustedCA.GetNamespace(), "prometheus-user-workload", hashedTrustedCA.Labels["monitoring.openshift.io/hash"]) if err != nil { return errors.Wrap(err, "deleting UserWorkload trusted CA Bundle ConfigMap failed") } From cc650ea4f9b89750614f697984e5680669f15bb5 Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava Date: Mon, 19 Jun 2023 16:38:01 +0530 Subject: [PATCH 3/3] fixup! fixup! Add the trusted CA bundle in UWM Prometheus pods --- pkg/tasks/prometheus_user_workload.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/pkg/tasks/prometheus_user_workload.go b/pkg/tasks/prometheus_user_workload.go index 073543b1e7..7af1f77fbe 100644 --- a/pkg/tasks/prometheus_user_workload.go +++ b/pkg/tasks/prometheus_user_workload.go @@ -369,23 +369,12 @@ func (t *PrometheusUserWorkloadTask) destroy(ctx context.Context) error { return errors.Wrap(err, "initializing UserWorkload CA bundle ConfigMap failed") } - cbs := &caBundleSyncer{ - client: t.client, - factory: t.factory, - prefix: "prometheus-user-workload", - } - - hashedTrustedCA, err := cbs.syncTrustedCABundle(ctx, trustedCA) - if err != nil { - return errors.Wrap(err, "syncing UserWorkload trusted CA bundle ConfigMap failed") - } - err = t.client.DeleteConfigMap(ctx, trustedCA) if err != nil { return errors.Wrap(err, "deleting UserWorkload trusted CA Bundle ConfigMap failed") } - err = t.client.DeleteHashedConfigMap(ctx, hashedTrustedCA.GetNamespace(), "prometheus-user-workload", hashedTrustedCA.Labels["monitoring.openshift.io/hash"]) + err = t.client.DeleteHashedConfigMap(ctx, trustedCA.GetNamespace(), "prometheus-user-workload", "") if err != nil { return errors.Wrap(err, "deleting UserWorkload trusted CA Bundle ConfigMap failed") }