Configure TLS for OVN metrics endpoints

openshift · Apr 9, 2020 · e5fa853 · e5fa853
1 parent 0b5ec32
commit e5fa853
Show file tree

Hide file tree

Showing 4 changed files with 152 additions and 18 deletions.
diff --git a/bindata/network/ovn-kubernetes/monitor.yaml b/bindata/network/ovn-kubernetes/monitor.yaml
@@ -1,35 +1,42 @@
-
+---
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   labels:
-    app: ovnkube-master
+    app: ovnkube-master-metrics
   annotations:
     networkoperator.openshift.io/ignore-errors: ""
-  name: monitor-ovn-master
+  name: monitor-ovn-master-metrics
   namespace: openshift-ovn-kubernetes
 spec:
   endpoints:
   - interval: 30s
     port: metrics
+    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: ovn-kubernetes-master-metrics.openshift-ovn-kubernetes.svc
   jobLabel: app
   namespaceSelector:
     matchNames:
     - openshift-ovn-kubernetes
   selector:
     matchLabels:
-      app: ovnkube-master
+      app: ovnkube-master-metrics
 ---
 apiVersion: v1
 kind: Service
 metadata:
   labels:
-    app: ovnkube-master
-  name: ovn-kubernetes-master
+    app: ovnkube-master-metrics
+  name: ovn-kubernetes-master-metrics
   namespace: openshift-ovn-kubernetes
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: ovn-master-metrics-cert
 spec:
   selector:
-    app: ovnkube-master
+    app: ovnkube-master-metrics
   clusterIP: None
   publishNotReadyAddresses: true
   ports:
@@ -45,33 +52,39 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   labels:
-    app: ovnkube-node
+    app: ovnkube-node-metrics
   annotations:
     networkoperator.openshift.io/ignore-errors: ""
-  name: monitor-ovn-node
+  name: monitor-ovn-node-metrics
   namespace: openshift-ovn-kubernetes
 spec:
   endpoints:
   - interval: 30s
     port: metrics
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: ovn-kubernetes-master-metrics.openshift-ovn-kubernetes.svc
   jobLabel: app
   namespaceSelector:
     matchNames:
     - openshift-ovn-kubernetes
   selector:
     matchLabels:
-      app: ovnkube-node
+      app: ovnkube-node-metrics
 ---
 apiVersion: v1
 kind: Service
 metadata:
   labels:
-    app: ovnkube-node
-  name: ovn-kubernetes-node
-  namespace: openshift-ovn-kubernetes
+    app: ovnkube-node-metrics
+  name: ovn-kubernetes-node-metrics
+  namespace: openshift-ovn-kubernees
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: ovn-node-metrics-cert
 spec:
   selector:
-    app: ovnkube-node
+    app: ovnkube-node-metrics
   clusterIP: None
   publishNotReadyAddresses: true
   ports:

diff --git a/bindata/network/ovn-kubernetes/ovnkube-master.yaml b/bindata/network/ovn-kubernetes/ovnkube-master.yaml
@@ -342,7 +342,7 @@ spec:
             --ovn-empty-lb-events \
             --loglevel "${OVN_KUBE_LOG_LEVEL}" \
             ${hybrid_overlay_flags} \
-            --metrics-bind-address "0.0.0.0:9102" \
+            --metrics-bind-address "0.0.0.0:29102" \
             --sb-address "{{.OVN_SB_ADDR_LIST}}" \
             --sb-client-privkey /ovn-cert/tls.key \
             --sb-client-cert /ovn-cert/tls.crt \
@@ -383,7 +383,7 @@ spec:
               fieldPath: spec.nodeName
         ports:
         - name: metrics-port
-          containerPort: 9102
+          containerPort: 29102
         terminationMessagePolicy: FallbackToLogsOnError
 
       nodeSelector:
@@ -423,3 +423,63 @@ spec:
         operator: "Exists"
       - key: "node.kubernetes.io/network-unavailable"
         operator: "Exists"
+---
+
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ovnkube-master-metrics
+  namespace: openshift-ovn-kubernetes
+  annotations:
+    kubernetes.io/description: |
+      RBAC Proxy to expose metrics over HTTPS
+    release.openshift.io/version: "{{.ReleaseVersion}}"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ovnkube-master-metrics
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: ovnkube-master-metrics
+        component: network
+        type: infra
+        openshift.io/component: network
+        kubernetes.io/os: "linux"
+    spec:
+      serviceAccountName: ovn-kubernetes-controller
+      hostNetwork: true
+      containers:
+      - name: kube-rbac-proxy
+        image: {{.KubeRBACProxyImage}}
+        args:
+        - --logtostderr
+        - --secure-listen-address=:9102
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+        - --upstream=http://127.0.0.1:29102/
+        - --tls-private-key-file=/etc/pki/tls/metrics-cert/tls.key
+        - --tls-cert-file=/etc/pki/tls/metrics-cert/tls.crt
+        ports:
+        - containerPort: 9102
+          name: https
+        resources:
+          requests:
+            cpu: 10m
+            memory: 20Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: ovn-master-metrics-cert
+          mountPath: /etc/pki/tls/metrics-cert
+          readOnly: True
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+        beta.kubernetes.io/os: "linux"
+      volumes:
+      - name: ovn-master-metrics-cert
+        secret:
+          secretName: ovn-master-metrics-cert
+      tolerations:
+      - operator: "Exists"
diff --git a/bindata/network/ovn-kubernetes/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/ovnkube-node.yaml
@@ -1,3 +1,4 @@
+---
 kind: DaemonSet
 apiVersion: apps/v1
 metadata:
@@ -132,7 +133,7 @@ spec:
             --config-file=/run/ovnkube-config/ovnkube.conf \
             --loglevel "${OVN_KUBE_LOG_LEVEL}" \
             ${hybrid_overlay_flags} \
-            --metrics-bind-address "0.0.0.0:9103"
+            --metrics-bind-address "0.0.0.0:29103"
         env:
         # for kubectl
         - name: KUBERNETES_SERVICE_PORT
@@ -147,7 +148,7 @@ spec:
               fieldPath: spec.nodeName
         ports:
         - name: metrics-port
-          containerPort: 9103
+          containerPort: 29103
         securityContext:
           privileged: true
         terminationMessagePolicy: FallbackToLogsOnError
@@ -253,3 +254,62 @@ spec:
           secretName: ovn-cert
       tolerations:
       - operator: "Exists"
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ovnkube-node-metrics
+  namespace: openshift-ovn-kubernetes
+  annotations:
+    kubernetes.io/description: |
+      RBAC proxy to expose metrics over HTTPS
+    release.openshift.io/version: "{{.ReleaseVersion}}"
+spec:
+  selector:
+    matchLabels:
+      app: ovnkube-node-metrics
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: ovnkube-node-metrics
+        component: network
+        type: infra
+        openshift.io/component: network
+        kubernetes.io/os: "linux"
+    spec:
+      serviceAccountName: ovn-kubernetes-node
+      hostNetwork: true
+      containers:
+      - name: kube-rbac-proxy
+        image: {{.KubeRBACProxyImage}}
+        args:
+        - --logtostderr
+        - --secure-listen-address=:9103
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+        - --upstream=http://127.0.0.1:29103/
+        - --tls-private-key-file=/etc/pki/tls/metrics-cert/tls.key
+        - --tls-cert-file=/etc/pki/tls/metrics-cert/tls.crt
+        ports:
+        - containerPort: 9103
+          name: https
+        resources:
+          requests:
+            cpu: 10m
+            memory: 20Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: ovn-node-metrics-cert
+          mountPath: /etc/pki/tls/metrics-cert
+          readOnly: True
+
+      nodeSelector:
+        beta.kubernetes.io/os: "linux"
+      volumes:
+      - name: ovn-node-metrics-cert
+        secret:
+          secretName: ovn-master-metrics-cert
+      tolerations:
+      - operator: "Exists"
diff --git a/pkg/network/ovn_kubernetes.go b/pkg/network/ovn_kubernetes.go
@@ -47,6 +47,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 	data := render.MakeRenderData()
 	data.Data["ReleaseVersion"] = os.Getenv("RELEASE_VERSION")
 	data.Data["OvnImage"] = os.Getenv("OVN_IMAGE")
+	data.Data["KubeRBACProxyImage"] = os.Getenv("KUBE_RBAC_PROXY_IMAGE")
 	data.Data["KUBERNETES_SERVICE_HOST"] = os.Getenv("KUBERNETES_SERVICE_HOST")
 	data.Data["KUBERNETES_SERVICE_PORT"] = os.Getenv("KUBERNETES_SERVICE_PORT")
 	data.Data["K8S_APISERVER"] = fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT"))