Replace the oauth-proxy before thanos-querier with kube-rbac-proxy

openshift · Nov 30, 2023 · af6deae · af6deae
1 parent 958ddf0
commit af6deae
Show file tree

Hide file tree

Showing 16 changed files with 485 additions and 169 deletions.
diff --git a/assets/cluster-monitoring-operator/cluster-monitoring-api-role.yaml b/assets/cluster-monitoring-operator/cluster-monitoring-api-role.yaml
@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: cluster-monitoring-operator
+    app.kubernetes.io/part-of: openshift-monitoring
+  name: cluster-monitoring-metrics-api
+  namespace: openshift-monitoring
+rules:
+- apiGroups:
+  - monitoring.coreos.com
+  resourceNames:
+  - k8s
+  resources:
+  - prometheuses/api
+  verbs:
+  - get
+  - create
+  - update
diff --git a/assets/cluster-monitoring-operator/cluster-role-view.yaml b/assets/cluster-monitoring-operator/cluster-role-view.yaml
@@ -12,3 +12,13 @@ rules:
   - namespaces
   verbs:
   - get
+- apiGroups:
+  - monitoring.coreos.com
+  resourceNames:
+  - k8s
+  resources:
+  - prometheuses/api
+  verbs:
+  - get
+  - create
+  - update
diff --git a/assets/prometheus-k8s/cluster-role.yaml b/assets/prometheus-k8s/cluster-role.yaml
@@ -46,3 +46,13 @@ rules:
   - securitycontextconstraints
   verbs:
   - use
+- apiGroups:
+  - monitoring.coreos.com
+  resourceNames:
+  - k8s
+  resources:
+  - prometheuses/api
+  verbs:
+  - get
+  - create
+  - update
diff --git a/assets/thanos-querier/deployment.yaml b/assets/thanos-querier/deployment.yaml
@@ -90,29 +90,14 @@ spec:
         - mountPath: /etc/tls/grpc
           name: secret-grpc-tls
       - args:
-        - -provider=openshift
-        - -https-address=:9091
-        - -http-address=
-        - -email-domain=*
-        - -upstream=http://localhost:9090
-        - -openshift-service-account=thanos-querier
-        - '-openshift-sar={"resource": "namespaces", "verb": "get"}'
-        - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get"}}'
-        - -tls-cert=/etc/tls/private/tls.crt
-        - -tls-key=/etc/tls/private/tls.key
-        - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
-        - -cookie-secret-file=/etc/proxy/secrets/session_secret
-        - -openshift-ca=/etc/pki/tls/cert.pem
-        - -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        - -bypass-auth-for=^/-/(healthy|ready)$
-        env:
-        - name: HTTP_PROXY
-          value: ""
-        - name: HTTPS_PROXY
-          value: ""
-        - name: NO_PROXY
-          value: ""
-        image: quay.io/openshift/oauth-proxy:latest
+        - --secure-listen-address=0.0.0.0:9091
+        - --upstream=http://127.0.0.1:9090
+        - --config-file=/etc/kube-rbac-proxy/config.yaml
+        - --tls-cert-file=/etc/tls/private/tls.crt
+        - --tls-private-key-file=/etc/tls/private/tls.key
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+        - --ignore-paths=/-/healthy,/-/ready
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         livenessProbe:
           failureThreshold: 4
           httpGet:
@@ -121,7 +106,7 @@ spec:
             scheme: HTTPS
           initialDelaySeconds: 5
           periodSeconds: 30
-        name: oauth-proxy
+        name: kube-rbac-proxy-web
         ports:
         - containerPort: 9091
           name: web
@@ -136,7 +121,7 @@ spec:
         resources:
           requests:
             cpu: 1m
-            memory: 20Mi
+            memory: 15Mi
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
@@ -146,8 +131,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/tls/private
           name: secret-thanos-querier-tls
-        - mountPath: /etc/proxy/secrets
-          name: secret-thanos-querier-oauth-cookie
+        - mountPath: /etc/kube-rbac-proxy
+          name: secret-thanos-querier-kube-rbac-proxy-web
       - args:
         - --secure-listen-address=0.0.0.0:9092
         - --upstream=http://127.0.0.1:9095
@@ -267,12 +252,12 @@ spec:
       - name: secret-thanos-querier-tls
         secret:
           secretName: thanos-querier-tls
-      - name: secret-thanos-querier-oauth-cookie
-        secret:
-          secretName: thanos-querier-oauth-cookie
       - name: secret-thanos-querier-kube-rbac-proxy
         secret:
           secretName: thanos-querier-kube-rbac-proxy
+      - name: secret-thanos-querier-kube-rbac-proxy-web
+        secret:
+          secretName: thanos-querier-kube-rbac-proxy-web
       - name: secret-thanos-querier-kube-rbac-proxy-rules
         secret:
           secretName: thanos-querier-kube-rbac-proxy-rules

diff --git a/assets/thanos-querier/kube-rbac-proxy-web-secret.yaml b/assets/thanos-querier/kube-rbac-proxy-web-secret.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: cluster-monitoring-operator
+    app.kubernetes.io/name: thanos-query
+    app.kubernetes.io/part-of: openshift-monitoring
+  name: thanos-querier-kube-rbac-proxy-web
+  namespace: openshift-monitoring
+stringData:
+  config.yaml: |-
+    "authorization":
+      "resourceAttributes":
+        "apiGroup": "monitoring.coreos.com"
+        "name": "k8s"
+        "namespace": "openshift-monitoring"
+        "resource": "prometheuses"
+        "subresource": "api"
+type: Opaque
diff --git a/assets/thanos-querier/oauth-cookie-secret.yaml b/assets/thanos-querier/oauth-cookie-secret.yaml
diff --git a/jsonnet/components/cluster-monitoring-operator.libsonnet b/jsonnet/components/cluster-monitoring-operator.libsonnet
@@ -337,17 +337,46 @@ function(params) {
 
   // This cluster role enables access to the Observe page in the admin console
   // and the different API services.
+  // In previous version, anyone with a "get" access on "namespace" resource
+  // can access the web endpoint. But KubeRBACProxy takes "get" verb as HTTP GET
+  // method, while the console access the web endpoint using HTTP POST method.
+  // A dedicated resource will be used to implementing this security setting.
   clusterRoleView: {
     apiVersion: 'rbac.authorization.k8s.io/v1',
     kind: 'ClusterRole',
     metadata: {
       name: 'cluster-monitoring-view',
     },
-    rules: [{
-      apiGroups: [''],
-      resources: ['namespaces'],
-      verbs: ['get'],
-    }],
+    rules: [
+      {
+        apiGroups: [''],
+        resources: ['namespaces'],
+        verbs: ['get'],
+      },
+      {
+        apiGroups: ['monitoring.coreos.com'],
+        resources: ['prometheuses/api'],
+        resourceNames: ['k8s'],
+        verbs: ['get', 'create', 'update'],
+      },
+    ],
+  },
+
+  clusterMonitoringApiRole: {
+    apiVersion: 'rbac.authorization.k8s.io/v1',
+    kind: 'Role',
+    metadata: {
+      name: 'cluster-monitoring-metrics-api',
+      namespace: cfg.namespace,
+    },
+    rules: [
+      {
+        apiGroups: ['monitoring.coreos.com'],
+        resources: ['prometheuses/api'],
+        resourceNames: ['k8s'],
+        verbs: ['get', 'create', 'update'],
+      },
+    ],
   },
 
   // This role enables read/write access to the platform Alertmanager API through OAuth proxy.

diff --git a/jsonnet/components/prometheus.libsonnet b/jsonnet/components/prometheus.libsonnet
@@ -135,8 +135,10 @@ function(params)
       data: {},
     },
 
-    // As Prometheus is protected by the oauth proxy it requires the
-    // ability to create TokenReview and SubjectAccessReview requests.
+    // As Prometheus and Thanos are protected by the kube-rbac-proxy,
+    // it requires the ability to create TokenReview and SubjectAccessReview requests.
+    // The subresource prometheuses/api is used by the Thanos querier and Prometheus to
+    // check a user's privilege to query the Prometheus API.
     // Additionally in order to authenticate with the Alertmanager it
     // requires `get` method on all `namespaces`, which is the
     // SubjectAccessReview required by the Alertmanager instances.
@@ -167,6 +169,16 @@ function(params)
           resourceNames: ['nonroot'],
           verbs: ['use'],
         },
+        {
+          // Access to the Prometheus / Thanos HTTP API through kube-rbac-proxy.
+          // openshift/origin test using the service account "prometheus-k8s" to execute prometheus API calls.
+          // This is required for the "prometheus-k8s" service account to be able to query the web port of
+          // the thanos-querier service web port.
+          apiGroups: ['monitoring.coreos.com'],
+          resources: ['prometheuses/api'],
+          resourceNames: ['k8s'],
+          verbs: ['get', 'create', 'update'],
+        },
       ],
     },