OCPBUGS-29247: ibm-vpc-block-csi-driver is missing sidecar metrics

README.md

@@ -43,6 +43,7 @@ export NODE_DRIVER_REGISTRAR_IMAGE=registry.k8s.io/sig-storage/csi-node-driver-r
 export LIVENESS_PROBE_IMAGE=registry.k8s.io/sig-storage/livenessprobe:v2.9.0
 export RESIZER_IMAGE=registry.k8s.io/sig-storage/csi-resizer:v1.7.0
 export SNAPSHOTTER_IMAGE=registry.k8s.io/sig-storage/csi-snapshotter:v6.2.1
+export KUBE_RBAC_PROXY_IMAGE=quay.io/openshift/origin-kube-rbac-proxy:latest
 
 # Run the operator via CLI
 ./ibm-vpc-block-csi-driver-operator start --kubeconfig $MY_KUBECONFIG --namespace openshift-cluster-csi-drivers

assets/controller.yaml

@@ -27,13 +27,14 @@ spec:
         target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}'
     spec:
       containers:
-        - args:
+        - name: csi-resizer
+          image: ${RESIZER_IMAGE}
+          imagePullPolicy: IfNotPresent
+          args:
+            - --http-endpoint=localhost:8204
             - --v=${LOG_LEVEL}
             - --csi-address=/csi/csi.sock
             - --timeout=900s
-          name: csi-resizer
-          image: ${RESIZER_IMAGE}
-          imagePullPolicy: IfNotPresent
           securityContext:
             privileged: false
             allowPrivilegeEscalation: false
@@ -44,50 +45,127 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
-        - args:
+        # kube-rbac-proxy for external-resizer container.
+        # Provides https proxy for http-based metrics.
+        - name: resizer-kube-rbac-proxy
+          image: ${KUBE_RBAC_PROXY_IMAGE}
+          imagePullPolicy: IfNotPresent
+          args:
+            - --secure-listen-address=0.0.0.0:9204
+            - --upstream=http://127.0.0.1:8204/
+            - --tls-cert-file=/etc/tls/private/tls.crt
+            - --tls-private-key-file=/etc/tls/private/tls.key
+            - --tls-cipher-suites=${TLS_CIPHER_SUITES}
+            - --tls-min-version=${TLS_MIN_VERSION}
+            - --logtostderr=true
+          ports:
+            - containerPort: 9204
+              name: resizer-m
+              protocol: TCP
+          resources:
+            requests:
+              memory: 20Mi
+              cpu: 10m
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - mountPath: /etc/tls/private
+              name: metrics-serving-cert
+        - name: csi-provisioner
+          image: ${PROVISIONER_IMAGE}
+          imagePullPolicy: IfNotPresent
+          args:
+            - --http-endpoint=localhost:8202
             - --v=${LOG_LEVEL}
             - --csi-address=$(ADDRESS)
             - --timeout=600s
             - --feature-gates=Topology=true
           env:
             - name: ADDRESS
               value: /csi/csi.sock
-          image: ${PROVISIONER_IMAGE}
-          imagePullPolicy: IfNotPresent
           securityContext:
             privileged: false
             allowPrivilegeEscalation: false
-          name: csi-provisioner
           resources:
             requests:
               cpu: 10m
               memory: 20Mi
           volumeMounts:
             - mountPath: /csi
               name: socket-dir
-        - args:
+        # kube-rbac-proxy for external-provisioner container.
+        # Provides https proxy for http-based metrics.
+        - name: provisioner-kube-rbac-proxy
+          image: ${KUBE_RBAC_PROXY_IMAGE}
+          imagePullPolicy: IfNotPresent
+          args:
+            - --secure-listen-address=0.0.0.0:9202
+            - --upstream=http://127.0.0.1:8202/
+            - --tls-cert-file=/etc/tls/private/tls.crt
+            - --tls-private-key-file=/etc/tls/private/tls.key
+            - --tls-cipher-suites=${TLS_CIPHER_SUITES}
+            - --tls-min-version=${TLS_MIN_VERSION}
+            - --logtostderr=true
+          ports:
+            - containerPort: 9202
+              name: provisioner-m
+              protocol: TCP
+          resources:
+            requests:
+              memory: 20Mi
+              cpu: 10m
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - mountPath: /etc/tls/private
+              name: metrics-serving-cert
+        - name: csi-attacher
+          image: ${ATTACHER_IMAGE}
+          imagePullPolicy: IfNotPresent
+          args:
+            - --http-endpoint=localhost:8203
             - --v=${LOG_LEVEL}
             - --csi-address=/csi/csi.sock
             - --timeout=900s
-          image: ${ATTACHER_IMAGE}
-          imagePullPolicy: IfNotPresent
           securityContext:
             privileged: false
             allowPrivilegeEscalation: false
-          name: csi-attacher
           resources:
             requests:
               cpu: 10m
               memory: 20Mi
           volumeMounts:
             - mountPath: /csi
               name: socket-dir
-        - args:
-            - --csi-address=/csi/csi.sock
-            - --v=${LOG_LEVEL}
+        # kube-rbac-proxy for external-attacher container.
+        # Provides https proxy for http-based metrics.
+        - name: attacher-kube-rbac-proxy
+          image: ${KUBE_RBAC_PROXY_IMAGE}
+          imagePullPolicy: IfNotPresent
+          args:
+            - --secure-listen-address=0.0.0.0:9203
+            - --upstream=http://127.0.0.1:8203/
+            - --tls-cert-file=/etc/tls/private/tls.crt
+            - --tls-private-key-file=/etc/tls/private/tls.key
+            - --tls-cipher-suites=${TLS_CIPHER_SUITES}
+            - --tls-min-version=${TLS_MIN_VERSION}
+            - --logtostderr=true
+          ports:
+            - containerPort: 9203
+              name: attacher-m
+              protocol: TCP
+          resources:
+            requests:
+              memory: 20Mi
+              cpu: 10m
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - mountPath: /etc/tls/private
+              name: metrics-serving-cert
+        - name: liveness-probe
           image: ${LIVENESS_PROBE_IMAGE}
           imagePullPolicy: IfNotPresent
-          name: liveness-probe
+          args:
+            - --csi-address=/csi/csi.sock
+            - --v=${LOG_LEVEL}
           securityContext:
             privileged: false
             allowPrivilegeEscalation: false
@@ -102,6 +180,7 @@ spec:
           image: ${SNAPSHOTTER_IMAGE}
           imagePullPolicy: IfNotPresent
           args:
+            - --http-endpoint=localhost:8205
             - --v=${LOG_LEVEL}
             - --csi-address=/csi/csi.sock
             - --timeout=900s
@@ -116,7 +195,36 @@ spec:
           volumeMounts:
             - mountPath: /csi
               name: socket-dir
-        - args:
+        # kube-rbac-proxy for external-snapshotter container.
+        # Provides https proxy for http-based metrics.
+        - name: snapshotter-kube-rbac-proxy
+          image: ${KUBE_RBAC_PROXY_IMAGE}
+          imagePullPolicy: IfNotPresent
+          args:
+            - --secure-listen-address=0.0.0.0:9205
+            - --upstream=http://127.0.0.1:8205/
+            - --tls-cert-file=/etc/tls/private/tls.crt
+            - --tls-private-key-file=/etc/tls/private/tls.key
+            - --tls-cipher-suites=${TLS_CIPHER_SUITES}
+            - --tls-min-version=${TLS_MIN_VERSION}
+            - --logtostderr=true
+          ports:
+            - containerPort: 9205
+              name: snapshotter-m
+              protocol: TCP
+          resources:
+            requests:
+              memory: 20Mi
+              cpu: 10m
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - mountPath: /etc/tls/private
+              name: metrics-serving-cert
+        - name: csi-driver
+          image: ${DRIVER_IMAGE}
+          imagePullPolicy: IfNotPresent
+          args:
+            - --metrics-address=localhost:8206
             - --v=${LOG_LEVEL}
             - --endpoint=$(CSI_ENDPOINT)
             - --lock_enabled=false
@@ -137,8 +245,6 @@ spec:
           envFrom:
             - configMapRef:
                 name: ibm-vpc-block-csi-configmap
-          image: ${DRIVER_IMAGE}
-          imagePullPolicy: IfNotPresent
           securityContext:
             privileged: false
             allowPrivilegeEscalation: false
@@ -150,7 +256,6 @@ spec:
             initialDelaySeconds: 10
             periodSeconds: 10
             timeoutSeconds: 3
-          name: csi-driver
           ports:
             - containerPort: 9808
               name: healthz
@@ -165,6 +270,31 @@ spec:
             - mountPath: /etc/storage_ibmc
               name: customer-auth
               readOnly: true
+        # kube-rbac-proxy for driver container.
+        # Provides https proxy for http-based metrics.
+        - name: driver-kube-rbac-proxy
+          image: ${KUBE_RBAC_PROXY_IMAGE}
+          imagePullPolicy: IfNotPresent
+          args:
+            - --secure-listen-address=0.0.0.0:9206
+            - --upstream=http://127.0.0.1:8206/
+            - --tls-cert-file=/etc/tls/private/tls.crt
+            - --tls-private-key-file=/etc/tls/private/tls.key
+            - --tls-cipher-suites=${TLS_CIPHER_SUITES}
+            - --tls-min-version=${TLS_MIN_VERSION}
+            - --logtostderr=true
+          ports:
+            - containerPort: 9206
+              name: driver-m
+              protocol: TCP
+          resources:
+            requests:
+              memory: 20Mi
+              cpu: 10m
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - mountPath: /etc/tls/private
+              name: metrics-serving-cert
       priorityClassName: system-cluster-critical
       serviceAccountName: ibm-vpc-block-controller-sa
       volumes:
@@ -173,3 +303,6 @@ spec:
         - name: customer-auth
           secret:
             secretName: storage-secret-store
+        - name: metrics-serving-cert
+          secret:
+            secretName: ibm-vpc-block-csi-driver-controller-metrics-serving-cert

assets/rbac/kube_rbac_proxy_binding.yaml

@@ -0,0 +1,13 @@
+# Allow kube-rbac-proxies to create tokenreviews to check Prometheus identity when scraping metrics.
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ibm-vpc-block-kube-rbac-proxy-binding
+subjects:
+  - kind: ServiceAccount
+    name: ibm-vpc-block-controller-sa
+    namespace: openshift-cluster-csi-drivers
+roleRef:
+  kind: ClusterRole
+  name:  ibm-vpc-block-kube-rbac-proxy-role
+  apiGroup: rbac.authorization.k8s.io

assets/rbac/kube_rbac_proxy_role.yaml

@@ -0,0 +1,13 @@
+# Allow kube-rbac-proxies to create tokenreviews to check Prometheus identity when scraping metrics.
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name:  ibm-vpc-block-kube-rbac-proxy-role
+rules:
+  - apiGroups:
+    - "authentication.k8s.io"
+    resources:
+    - "tokenreviews"
+    verbs:
+    - "create"
+

assets/rbac/prometheus_role.yaml

@@ -0,0 +1,17 @@
+# Role for accessing metrics exposed by the operator
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name:  ibm-vpc-block-csi-driver-prometheus
+  namespace: openshift-cluster-csi-drivers
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch

assets/rbac/prometheus_rolebinding.yaml

@@ -0,0 +1,14 @@
+# Grant cluster-monitoring access to the operator metrics service
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: ibm-vpc-block-csi-driver-prometheus
+  namespace: openshift-cluster-csi-drivers
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ibm-vpc-block-csi-driver-prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: openshift-monitoring

assets/service.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: ibm-vpc-block-csi-driver-controller-metrics-serving-cert
+  labels:
+    app: ibm-vpc-block-csi-driver-controller-metrics
+  name: ibm-vpc-block-csi-driver-controller-metrics
+  namespace: openshift-cluster-csi-drivers
+spec:
+  ports:
+    - name: provisioner-m
+      port: 9202
+      protocol: TCP
+      targetPort: provisioner-m
+    - name: attacher-m
+      port: 9203
+      protocol: TCP
+      targetPort: attacher-m
+    - name: resizer-m
+      port: 9204
+      protocol: TCP
+      targetPort: resizer-m
+    - name: snapshotter-m
+      port: 9205
+      protocol: TCP
+      targetPort: snapshotter-m
+    - name: driver-m
+      port: 9206
+      protocol: TCP
+      targetPort: driver-m
+  selector:
+    app: ibm-vpc-block-csi-driver
+  sessionAffinity: None
+  type: ClusterIP