From ba85332b18b2d097cc63429e671446f500bdefe3 Mon Sep 17 00:00:00 2001 From: Mario Fernandez Date: Mon, 27 Feb 2023 10:53:49 +0100 Subject: [PATCH] Add federate to rbac proxy Signed-off-by: Mario Fernandez --- assets/prometheus-k8s/cluster-role.yaml | 4 +++ .../kube-rbac-proxy-secret.yaml | 5 ++++ assets/prometheus-k8s/prometheus.yaml | 2 +- jsonnet/components/prometheus.libsonnet | 26 ++++++++++++++-- jsonnet/utils/generate-secret.libsonnet | 4 +-- ...0_cluster-monitoring-operator_02-role.yaml | 4 +++ test/e2e/telemeter_test.go | 30 +++++++++++++++++++ 7 files changed, 70 insertions(+), 5 deletions(-) diff --git a/assets/prometheus-k8s/cluster-role.yaml b/assets/prometheus-k8s/cluster-role.yaml index 1d2e56b36b..369989f278 100644 --- a/assets/prometheus-k8s/cluster-role.yaml +++ b/assets/prometheus-k8s/cluster-role.yaml @@ -45,3 +45,7 @@ rules: - securitycontextconstraints verbs: - use +- nonResourceURLs: + - /federate + verbs: + - get diff --git a/assets/prometheus-k8s/kube-rbac-proxy-secret.yaml b/assets/prometheus-k8s/kube-rbac-proxy-secret.yaml index d7af59587c..2fa47fe636 100644 --- a/assets/prometheus-k8s/kube-rbac-proxy-secret.yaml +++ b/assets/prometheus-k8s/kube-rbac-proxy-secret.yaml @@ -15,4 +15,9 @@ stringData: "user": "name": "system:serviceaccount:openshift-monitoring:prometheus-k8s" "verb": "get" + - "path": "/federate" + "resourceRequest": false + "user": + "name": "system:serviceaccount:openshift-monitoring:telemeter-client" + "verb": "get" type: Opaque diff --git a/assets/prometheus-k8s/prometheus.yaml b/assets/prometheus-k8s/prometheus.yaml index 2832573161..edc61ba969 100644 --- a/assets/prometheus-k8s/prometheus.yaml +++ b/assets/prometheus-k8s/prometheus.yaml @@ -82,7 +82,7 @@ spec: - args: - --secure-listen-address=0.0.0.0:9092 - --upstream=http://127.0.0.1:9090 - - --allow-paths=/metrics + - --allow-paths=/metrics,/federate - --config-file=/etc/kube-rbac-proxy/config.yaml - --tls-cert-file=/etc/tls/private/tls.crt - --tls-private-key-file=/etc/tls/private/tls.key diff --git a/jsonnet/components/prometheus.libsonnet b/jsonnet/components/prometheus.libsonnet index 399342698b..40dd5dc40c 100644 --- a/jsonnet/components/prometheus.libsonnet +++ b/jsonnet/components/prometheus.libsonnet @@ -167,6 +167,10 @@ function(params) resourceNames: ['nonroot'], verbs: ['use'], }, + { + nonResourceURLs: ['/federate'], + verbs: ['get'], + }, ], }, @@ -203,7 +207,25 @@ function(params) data: {}, }, - kubeRbacProxySecret: generateSecret.staticAuthSecret(cfg.namespace, cfg.commonLabels, 'kube-rbac-proxy'), + kubeRbacProxySecret: generateSecret.staticAuthSecret( + cfg.namespace, + cfg.commonLabels, + 'kube-rbac-proxy', + { + authorization+: { + static+: [ + { + user: { + name: 'system:serviceaccount:openshift-monitoring:telemeter-client', + }, + verb: 'get', + path: '/federate', + resourceRequest: false, + }, + ], + }, + }, + ), // Secret holding the token to authenticate against the Telemetry server when using native remote-write. telemetrySecret: { @@ -397,7 +419,7 @@ function(params) args: [ '--secure-listen-address=0.0.0.0:9092', '--upstream=http://127.0.0.1:9090', - '--allow-paths=/metrics', + '--allow-paths=/metrics,/federate', '--config-file=/etc/kube-rbac-proxy/config.yaml', '--tls-cert-file=/etc/tls/private/tls.crt', '--tls-private-key-file=/etc/tls/private/tls.key', diff --git a/jsonnet/utils/generate-secret.libsonnet b/jsonnet/utils/generate-secret.libsonnet index acf1a0dced..23da43a19c 100644 --- a/jsonnet/utils/generate-secret.libsonnet +++ b/jsonnet/utils/generate-secret.libsonnet @@ -1,5 +1,5 @@ { - staticAuthSecret(cfgNamespace, cfgCommonLabels, cfgName):: { + staticAuthSecret(cfgNamespace, cfgCommonLabels, cfgName, additionalConfig={}):: { apiVersion: 'v1', kind: 'Secret', metadata: { @@ -23,7 +23,7 @@ }, ], }, - },), + } + additionalConfig), }, }, } diff --git a/manifests/0000_50_cluster-monitoring-operator_02-role.yaml b/manifests/0000_50_cluster-monitoring-operator_02-role.yaml index 6a25115386..2b6330ad18 100644 --- a/manifests/0000_50_cluster-monitoring-operator_02-role.yaml +++ b/manifests/0000_50_cluster-monitoring-operator_02-role.yaml @@ -449,6 +449,10 @@ rules: - securitycontextconstraints verbs: - use +- nonResourceURLs: + - /federate + verbs: + - get - apiGroups: - "" resources: diff --git a/test/e2e/telemeter_test.go b/test/e2e/telemeter_test.go index ba0122b8a0..0f7b0fab13 100644 --- a/test/e2e/telemeter_test.go +++ b/test/e2e/telemeter_test.go @@ -17,6 +17,7 @@ package e2e import ( "context" "errors" + "fmt" "testing" "time" @@ -98,3 +99,32 @@ func TestTelemeterRemoteWrite(t *testing.T) { }, ) } + +// TestTelemeterClient verifies that the telemeter client can collect metrics from the monitoring stack and forward them to the telemeter server. +func TestTelemeterClient(t *testing.T) { + { + f.PrometheusK8sClient.WaitForQueryReturn( + t, + 5*time.Minute, + `metricsclient_request_send{client="federate_to",job="telemeter-client",status_code="200"}`, + func(v float64) error { + if v == 0 { + return fmt.Errorf("expecting metricsclient request send more than 0 but got none") + } + return nil + }, + ) + + f.PrometheusK8sClient.WaitForQueryReturn( + t, + 5*time.Minute, + `federate_samples{job="telemeter-client"}`, + func(v float64) error { + if v < 10 { + return fmt.Errorf("expecting federate samples from telemeter client more than 10 but got %f", v) + } + return nil + }, + ) + } +}