Merge pull request #2260 from simonpasquier/MON-3381

MON-3381: replace OAuth proxy for Alertmanager
openshift · Mar 7, 2024 · b60f0b6 · b60f0b6
2 parents 5342dee + df7c279
commit b60f0b6
Show file tree

Hide file tree

Showing 12 changed files with 162 additions and 176 deletions.
diff --git a/assets/alertmanager/alertmanager.yaml b/assets/alertmanager/alertmanager.yaml
@@ -26,29 +26,15 @@ spec:
   automountServiceAccountToken: true
   containers:
   - args:
-    - -provider=openshift
-    - -https-address=:9095
-    - -http-address=
-    - -email-domain=*
-    - -upstream=http://localhost:9093
-    - '-openshift-sar=[{"resource": "namespaces", "verb": "get"}, {"resource": "alertmanagers", "resourceAPIGroup": "monitoring.coreos.com", "namespace": "openshift-monitoring", "verb": "patch", "resourceName": "non-existant"}]'
-    - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get"}, "/": {"resource":"alertmanagers", "group": "monitoring.coreos.com", "namespace": "openshift-monitoring", "verb": "patch", "name": "non-existant"}}'
-    - -tls-cert=/etc/tls/private/tls.crt
-    - -tls-key=/etc/tls/private/tls.key
-    - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
-    - -cookie-secret-file=/etc/proxy/secrets/session_secret
-    - -openshift-service-account=alertmanager-main
-    - -openshift-ca=/etc/pki/tls/cert.pem
-    - -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    env:
-    - name: HTTP_PROXY
-      value: ""
-    - name: HTTPS_PROXY
-      value: ""
-    - name: NO_PROXY
-      value: ""
-    image: quay.io/openshift/oauth-proxy:latest
-    name: alertmanager-proxy
+    - --secure-listen-address=0.0.0.0:9095
+    - --upstream=http://127.0.0.1:9093
+    - --config-file=/etc/kube-rbac-proxy/config.yaml
+    - --tls-cert-file=/etc/tls/private/tls.crt
+    - --tls-private-key-file=/etc/tls/private/tls.key
+    - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+    - --ignore-paths=/-/healthy,/-/ready
+    image: quay.io/brancz/kube-rbac-proxy:v0.15.0
+    name: kube-rbac-proxy-web
     ports:
     - containerPort: 9095
       name: web
@@ -58,10 +44,12 @@ spec:
         memory: 20Mi
     terminationMessagePolicy: FallbackToLogsOnError
     volumeMounts:
+    - mountPath: /etc/kube-rbac-proxy
+      name: secret-alertmanager-kube-rbac-proxy-web
+      readOnly: true
     - mountPath: /etc/tls/private
       name: secret-alertmanager-main-tls
-    - mountPath: /etc/proxy/secrets
-      name: secret-alertmanager-main-proxy
+      readOnly: true
   - args:
     - --secure-listen-address=0.0.0.0:9092
     - --upstream=http://127.0.0.1:9096
@@ -92,7 +80,6 @@ spec:
     - --tls-private-key-file=/etc/tls/private/tls.key
     - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
     - --client-ca-file=/etc/tls/client/client-ca.crt
-    - --logtostderr=true
     - --allow-paths=/metrics
     image: quay.io/brancz/kube-rbac-proxy:v0.15.0
     name: kube-rbac-proxy-metric
@@ -147,9 +134,9 @@ spec:
       memory: 40Mi
   secrets:
   - alertmanager-main-tls
-  - alertmanager-main-proxy
   - alertmanager-kube-rbac-proxy
   - alertmanager-kube-rbac-proxy-metric
+  - alertmanager-kube-rbac-proxy-web
   securityContext:
     fsGroup: 65534
     runAsNonRoot: true

diff --git a/assets/alertmanager/kube-rbac-proxy-web-secret.yaml b/assets/alertmanager/kube-rbac-proxy-web-secret.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+data: {}
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: cluster-monitoring-operator
+    app.kubernetes.io/part-of: openshift-monitoring
+  name: alertmanager-kube-rbac-proxy-web
+  namespace: openshift-monitoring
+stringData:
+  config.yaml: |-
+    "authorization":
+      "resourceAttributes":
+        "apiGroup": "monitoring.coreos.com"
+        "name": "main"
+        "namespace": "openshift-monitoring"
+        "resource": "alertmanagers"
+        "subresource": "api"
+type: Opaque
diff --git a/assets/alertmanager/proxy-secret.yaml b/assets/alertmanager/proxy-secret.yaml
diff --git a/assets/alertmanager/service-account.yaml b/assets/alertmanager/service-account.yaml
@@ -2,8 +2,6 @@ apiVersion: v1
 automountServiceAccountToken: false
 kind: ServiceAccount
 metadata:
-  annotations:
-    serviceaccounts.openshift.io/oauth-redirectreference.alertmanager-main: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"alertmanager-main"}}'
   labels:
     app.kubernetes.io/component: alert-router
     app.kubernetes.io/instance: main

diff --git a/assets/cluster-monitoring-operator/monitoring-alertmanager-edit-role.yaml b/assets/cluster-monitoring-operator/monitoring-alertmanager-edit-role.yaml
@@ -15,3 +15,11 @@ rules:
   - alertmanagers
   verbs:
   - patch
+- apiGroups:
+  - monitoring.coreos.com
+  resourceNames:
+  - main
+  resources:
+  - alertmanagers/api
+  verbs:
+  - '*'
diff --git a/jsonnet/components/alertmanager.libsonnet b/jsonnet/components/alertmanager.libsonnet
@@ -43,16 +43,7 @@ function(params)
       },
     },
 
-    // The ServiceAccount needs this annotation, to signify the identity
-    // provider, that when a users it doing the oauth flow through the oauth
-    // proxy, that it should redirect to the alertmanager-main route on
-    // successful authentication.
     serviceAccount+: {
-      metadata+: {
-        annotations+: {
-          'serviceaccounts.openshift.io/oauth-redirectreference.alertmanager-main': '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"alertmanager-main"}}',
-        },
-      },
       // Alertmanager can mount the token into the pod since
       // https://github.com/prometheus-operator/prometheus-operator/pull/5474
       // and v0.66.0
@@ -67,8 +58,8 @@ function(params)
     // cluster-monitoring-operator, that when reconciling this service the
     // cluster IP needs to be retained.
     //
-    // The ports are overridden, as due to the port binding of the oauth proxy
-    // the serving port is 9094 instead of the 9093 default.
+    // The ports are overridden because the kube-rbac-proxy sidecar listens on
+    // serving port 9094 instead of the default port (9093).
 
     service+: {
       metadata+: {
@@ -111,21 +102,8 @@ function(params)
       },
     },
 
-    // The proxy secret is there to encrypt session created by the oauth proxy.
-    proxySecret: {
-      apiVersion: 'v1',
-      kind: 'Secret',
-      metadata: {
-        name: 'alertmanager-main-proxy',
-        namespace: cfg.namespace,
-        labels: { 'app.kubernetes.io/name': 'alertmanager-main' },
-      },
-      type: 'Opaque',
-      data: {},
-    },
-
-    // In order for the oauth proxy to perform a TokenReview and
-    // SubjectAccessReview for authN and authZ the alertmanager ServiceAccount
+    // In order for the kube-rbac-proxy sidecar to perform a TokenReview and
+    // SubjectAccessReview for authN and authZ, the alertmanager ServiceAccount
     // requires the `create` action on both of these.
 
     clusterRole: {
@@ -182,7 +160,7 @@ function(params)
       metadata: {
         name: 'alertmanager-kube-rbac-proxy',
         namespace: cfg.namespace,
-        labels: { 'app.kubernetes.io/name': 'alertmanager-main' },
+        labels: cfg.commonLabels { 'app.kubernetes.io/name': 'alertmanager-main' },
       },
       type: 'Opaque',
       stringData: {
@@ -209,6 +187,31 @@ function(params)
       },
     },
 
+    kubeRbacProxyWebSecret: {
+      apiVersion: 'v1',
+      kind: 'Secret',
+      metadata: {
+        name: 'alertmanager-kube-rbac-proxy-web',
+        namespace: 'openshift-monitoring',
+        labels: cfg.commonLabels,
+      },
+      type: 'Opaque',
+      data: {},
+      stringData: {
+        'config.yaml': std.manifestYamlDoc({
+          authorization: {
+            resourceAttributes: {
+              apiGroup: 'monitoring.coreos.com',
+              resource: 'alertmanagers',
+              subresource: 'api',
+              namespace: 'openshift-monitoring',
+              name: 'main',
+            },
+          },
+        }),
+      },
+    },
+
     // This changes the alertmanager to be scraped with TLS, authN and authZ,
     // which are not present in kube-prometheus.
     serviceMonitor+: {
@@ -228,8 +231,6 @@ function(params)
       },
     },
 
-    // These patches inject the oauth proxy as a sidecar and configures it with
-    // TLS.
     alertmanager+: {
       spec+: {
         securityContext: {
@@ -247,9 +248,9 @@ function(params)
         },
         secrets: [
           'alertmanager-main-tls',
-          'alertmanager-main-proxy',
           $.kubeRbacProxySecret.metadata.name,
           $.kubeRbacProxyMetricSecret.metadata.name,
+          $.kubeRbacProxyWebSecret.metadata.name,
         ],
         listenLocal: true,
         resources: {
@@ -261,58 +262,40 @@ function(params)
         automountServiceAccountToken: true,
         containers: [
           {
-            name: 'alertmanager-proxy',
-            image: 'quay.io/openshift/oauth-proxy:latest',  //FIXME(paulfantom)
+            name: 'kube-rbac-proxy-web',
+            image: cfg.kubeRbacProxyImage,
+            resources: {
+              requests: {
+                cpu: '1m',
+                memory: '20Mi',
+              },
+            },
             ports: [
               {
                 containerPort: 9095,
                 name: 'web',
               },
             ],
-            env: [
-              {
-                name: 'HTTP_PROXY',
-                value: '',
-              },
-              {
-                name: 'HTTPS_PROXY',
-                value: '',
-              },
-              {
-                name: 'NO_PROXY',
-                value: '',
-              },
-            ],
             args: [
-              '-provider=openshift',
-              '-https-address=:9095',
-              '-http-address=',
-              '-email-domain=*',
-              '-upstream=http://localhost:9093',
-              '-openshift-sar=[{"resource": "namespaces", "verb": "get"}, {"resource": "alertmanagers", "resourceAPIGroup": "monitoring.coreos.com", "namespace": "openshift-monitoring", "verb": "patch", "resourceName": "non-existant"}]',
-              '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get"}, "/": {"resource":"alertmanagers", "group": "monitoring.coreos.com", "namespace": "openshift-monitoring", "verb": "patch", "name": "non-existant"}}',
-              '-tls-cert=/etc/tls/private/tls.crt',
-              '-tls-key=/etc/tls/private/tls.key',
-              '-client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token',
-              '-cookie-secret-file=/etc/proxy/secrets/session_secret',
-              '-openshift-service-account=alertmanager-main',
-              '-openshift-ca=/etc/pki/tls/cert.pem',
-              '-openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
+              '--secure-listen-address=0.0.0.0:9095',
+              '--upstream=http://127.0.0.1:9093',
+              '--config-file=/etc/kube-rbac-proxy/config.yaml',
+              '--tls-cert-file=/etc/tls/private/tls.crt',
+              '--tls-private-key-file=/etc/tls/private/tls.key',
+              '--tls-cipher-suites=' + cfg.tlsCipherSuites,
+              // Liveness and readiness endpoints are always allowed.
+              '--ignore-paths=' + std.join(',', ['/-/healthy', '/-/ready']),
             ],
-            resources: {
-              requests: {
-                cpu: '1m',
-                memory: '20Mi',
-              },
-            },
             volumeMounts: [
               {
-                mountPath: '/etc/tls/private',
-                name: 'secret-alertmanager-main-tls',
+                mountPath: '/etc/kube-rbac-proxy',
+                name: 'secret-' + $.kubeRbacProxyWebSecret.metadata.name,
+                readOnly: true,
               },
               {
-                mountPath: '/etc/proxy/secrets',
-                name: 'secret-alertmanager-main-proxy',
+                mountPath: '/etc/tls/private',
+                name: 'secret-alertmanager-main-tls',
+                readOnly: true,
               },
             ],
           },
@@ -351,7 +334,8 @@ function(params)
             ],
           },
           {
-            // TODO: merge this metric proxy with tenancy proxy when the issue below is fixed:
+            // TODO: merge this metric proxy with the kube-rbac-proxy-web when
+            // the issue below is fixed:
             // https://github.com/brancz/kube-rbac-proxy/issues/146
             name: 'kube-rbac-proxy-metric',
             image: cfg.kubeRbacProxyImage,
@@ -375,7 +359,6 @@ function(params)
               '--tls-private-key-file=/etc/tls/private/tls.key',
               '--tls-cipher-suites=' + cfg.tlsCipherSuites,
               '--client-ca-file=/etc/tls/client/client-ca.crt',
-              '--logtostderr=true',
               '--allow-paths=/metrics',
             ],
             volumeMounts: [

diff --git a/jsonnet/components/cluster-monitoring-operator.libsonnet b/jsonnet/components/cluster-monitoring-operator.libsonnet
@@ -298,7 +298,7 @@ function(params) {
       },
       {
         apiGroups: ['monitoring.coreos.com'],
-        resourceNames: ['user-workload'],
+        resourceNames: ['user-workload', 'main'],
         resources: ['alertmanagers/api'],
         verbs: ['*'],
       },
@@ -379,20 +379,31 @@ function(params) {
     ],
   },
 
-  // This role enables read/write access to the platform Alertmanager API through OAuth proxy.
+  // This role enables read/write access to the platform Alertmanager API
+  // through kube-rbac-proxy.
   monitoringAlertmanagerEditRole: {
     apiVersion: 'rbac.authorization.k8s.io/v1',
     kind: 'Role',
     metadata: {
       name: 'monitoring-alertmanager-edit',
       namespace: cfg.namespace,
     },
-    rules: [{
-      apiGroups: ['monitoring.coreos.com'],
-      resources: ['alertmanagers'],
-      verbs: ['patch'],
-      resourceNames: ['non-existant'],
-    }],
+    rules: [
+      {
+        // this permission used to be required when Alertmanager was protected via OAuth proxy.
+        // TODO: remove it after OCP 4.16 is released.
+        apiGroups: ['monitoring.coreos.com'],
+        resources: ['alertmanagers'],
+        verbs: ['patch'],
+        resourceNames: ['non-existant'],
+      },
+      {
+        apiGroups: ['monitoring.coreos.com'],
+        resources: ['alertmanagers/api'],
+        resourceNames: ['main'],
+        verbs: ['*'],
+      },
+    ],
   },
 
   // This role provides read access to the user-workload Alertmanager API.

diff --git a/manifests/0000_50_cluster-monitoring-operator_02-namespaced-cluster-role.yaml b/manifests/0000_50_cluster-monitoring-operator_02-namespaced-cluster-role.yaml
@@ -67,6 +67,7 @@ rules:
   - monitoring.coreos.com
   resourceNames:
   - user-workload
+  - main
   resources:
   - alertmanagers/api
   verbs: