Deploy the network metrics daemon as part of the CNO.

bindata/network/network-metrics/000-rbac.yaml

@@ -0,0 +1,38 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-daemon-sa
+  namespace: openshift-multus
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: metrics-daemon-role
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "watch", "list"]
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources: ["network-attachment-definitions"]
+    verbs: ["get", "watch", "list"]
+  - apiGroups: ["authentication.k8s.io"]
+    resources: ["tokenreviews"]
+    verbs: ["create"]
+  - apiGroups: ["authorization.k8s.io"]
+    resources: ["subjectaccessreviews"]
+    verbs: ["create"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: metrics-daemon-sa-rolebinding
+subjects:
+  - kind: ServiceAccount
+    name: metrics-daemon-sa
+    apiGroup: ""
+    namespace: openshift-multus
+roleRef:
+  kind: ClusterRole
+  name: metrics-daemon-role
+  apiGroup: rbac.authorization.k8s.io

bindata/network/network-metrics/001-daemonset.yaml

@@ -0,0 +1,71 @@
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: network-metrics-daemon
+  namespace: openshift-multus
+  annotations:
+    kubernetes.io/description: |
+      This daemonset launches the network metrics daemon on each node
+    release.openshift.io/version: "{{.ReleaseVersion}}"
+    networkoperator.openshift.io/non-critical: ""
+spec:
+  selector:
+    matchLabels:
+      app: network-metrics-daemon
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: network-metrics-daemon
+        component: network
+        type: infra
+        openshift.io/component: network
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/worker: ""
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: network-metrics-daemon
+          image: {{.NetworkMetricsImage}}
+          command:
+            - /usr/bin/network-metrics
+          args: ["--node-name", "$(NODE_NAME)"]
+          resources:
+            requests:
+              cpu: 10m
+              memory: 100Mi
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+        - name: kube-rbac-proxy
+          image: {{.KubeRBACProxyImage}}
+          args:
+            - --logtostderr
+            - --secure-listen-address=:8443
+            - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+            - --upstream=http://127.0.0.1:9091/
+            - --tls-private-key-file=/etc/metrics/tls.key
+            - --tls-cert-file=/etc/metrics/tls.crt
+          ports:
+            - containerPort: 8443
+              name: https
+          resources:
+            requests:
+              cpu: 10m
+              memory: 20Mi
+          terminationMessagePolicy: FallbackToLogsOnError
+          volumeMounts:
+            - name: metrics-certs
+              mountPath: /etc/metrics
+              readOnly: True
+      volumes:
+        - name: metrics-certs
+          secret:
+            secretName: metrics-daemon-secret
+      serviceAccountName: metrics-daemon-sa

bindata/network/network-metrics/002-prometheus.yaml

@@ -0,0 +1,77 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    name: monitor-network
+  annotations:
+    networkoperator.openshift.io/ignore-errors: ""
+  name: monitor-network
+  namespace: openshift-multus
+spec:
+  endpoints:
+    - interval: 10s
+      port: metrics
+      honorLabels: true
+      bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      scheme: "https"
+      tlsConfig:
+        caFile: "/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt"
+        serverName: "network-metrics-service.openshift-multus.svc"
+  selector:
+    matchLabels:
+      service: network-metrics-service
+  namespaceSelector:
+    matchNames:
+      - openshift-multus
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    prometheus.io/scrape: "true"
+    service.alpha.openshift.io/serving-cert-secret-name: metrics-daemon-secret
+  labels:
+    service: network-metrics-service
+  name: network-metrics-service
+  namespace: openshift-multus
+spec:
+  selector:
+    app: network-metrics-daemon
+  clusterIP: None
+  ports:
+    - name: metrics
+      port: 8443
+      targetPort: https
+  type: ClusterIP
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s
+  namespace: openshift-multus
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s
+  namespace: openshift-multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: openshift-monitoring

manifests/0000_70_cluster-network-operator_03_deployment.yaml

@@ -49,6 +49,8 @@ spec:
           value: "quay.io/openshift/origin-kuryr-cni:4.3"
         - name: KURYR_CONTROLLER_IMAGE
           value: "quay.io/openshift/origin-kuryr-controller:4.3"
+        - name: NETWORK_METRICS_DAEMON_IMAGE
+          value: "quay.io/openshift/origin-network-metrics-daemon:4.5"
         - name: POD_NAME
           valueFrom:
             fieldRef:

manifests/image-references

@@ -50,3 +50,8 @@ spec:
     from:
       kind: DockerImage
       name: quay.io/openshift/origin-kuryr-controller:4.3
+  - name: network-metrics-daemon
+    from:
+      kind: DockerImage
+      name: quay.io/openshift/origin-network-metrics-daemon:4.5
+

pkg/network/multus.go

@@ -22,12 +22,10 @@ func RenderMultus(conf *operv1.NetworkSpec, manifestDir string) ([]*uns.Unstruct
 		return nil, nil
 	}
 
-	var err error
 	out := []*uns.Unstructured{}
-	objs := []*uns.Unstructured{}
 
 	// enabling Multus always renders the CRD since Multus uses it
-	objs, err = renderAdditionalNetworksCRD(manifestDir)
+	objs, err := renderAdditionalNetworksCRD(manifestDir)
 	if err != nil {
 		return nil, err
 	}
@@ -39,6 +37,13 @@ func RenderMultus(conf *operv1.NetworkSpec, manifestDir string) ([]*uns.Unstruct
 		return nil, err
 	}
 	out = append(out, objs...)
+
+	objs, err = renderNetworkMetricsDaemon(manifestDir)
+	if err != nil {
+		return nil, err
+	}
+	out = append(out, objs...)
+
 	return out, nil
 }
 
@@ -68,6 +73,25 @@ func renderMultusConfig(manifestDir, defaultNetworkType string, useDHCP bool) ([
 	return objs, nil
 }
 
+// renderNetworkMetricsDaemon returns the manifests of the Network Metrics Daemon
+func renderNetworkMetricsDaemon(manifestDir string) ([]*uns.Unstructured, error) {
+
+	objs := []*uns.Unstructured{}
+
+	// render the manifests on disk
+	data := render.MakeRenderData()
+	data.Data["ReleaseVersion"] = os.Getenv("RELEASE_VERSION")
+	data.Data["NetworkMetricsImage"] = os.Getenv("NETWORK_METRICS_DAEMON_IMAGE")
+	data.Data["KubeRBACProxyImage"] = os.Getenv("KUBE_RBAC_PROXY_IMAGE")
+
+	manifests, err := render.RenderDir(filepath.Join(manifestDir, "network/network-metrics"), &data)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to render multus admission controller manifests")
+	}
+	objs = append(objs, manifests...)
+	return objs, nil
+}
+
 // pluginCNIDir is the directory where plugins should install their CNI
 // configuration file. By default, it is where multus looks, unless multus
 // is disabled

pkg/network/multus_test.go

@@ -50,7 +50,7 @@ func TestRenderMultus(t *testing.T) {
 	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "openshift-multus", "multus")))
 
 	// It's important that the namespace is first
-	g.Expect(len(objs)).To(Equal(10))
+	g.Expect(len(objs)).To(Equal(18))
 	g.Expect(objs[0]).To(HaveKubernetesID("CustomResourceDefinition", "", "network-attachment-definitions.k8s.cni.cncf.io"))
 	g.Expect(objs).To(ContainElement(HaveKubernetesID("Namespace", "", "openshift-multus")))
 	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRole", "", "multus")))

pkg/network/network_metrics_test.go

@@ -0,0 +1,76 @@
+package network
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	operv1 "github.com/openshift/api/operator/v1"
+	"github.com/openshift/cluster-network-operator/pkg/apply"
+)
+
+var NetworkMetricsDaemonConfig = operv1.Network{
+	Spec: operv1.NetworkSpec{
+		ServiceNetwork: []string{"172.30.0.0/16"},
+		ClusterNetwork: []operv1.ClusterNetworkEntry{
+			{
+				CIDR:       "10.128.0.0/15",
+				HostPrefix: 23,
+			},
+		},
+		DefaultNetwork: operv1.DefaultNetworkDefinition{
+			Type: operv1.NetworkTypeOpenShiftSDN,
+			OpenShiftSDNConfig: &operv1.OpenShiftSDNConfig{
+				Mode: operv1.SDNModeNetworkPolicy,
+			},
+		},
+	},
+}
+
+// TestRenderNetworkMetricsDaemon has some simple rendering tests
+func TestRenderNetworkMetricsDaemon(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	crd := NetworkMetricsDaemonConfig.DeepCopy()
+	config := &crd.Spec
+	disabled := true
+	config.DisableMultiNetwork = &disabled
+	FillDefaults(config, nil)
+
+	// disable MultusAdmissionController
+	objs, err := RenderMultus(config, manifestDir)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(objs).NotTo(ContainElement(HaveKubernetesID("DaemonSet", "openshift-multus", "network-metrics-daemon")))
+
+	// enable MultusAdmissionController
+	enabled := false
+	config.DisableMultiNetwork = &enabled
+	objs, err = RenderMultus(config, manifestDir)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "openshift-multus", "network-metrics-daemon")))
+
+	// Check rendered object
+
+	g.Expect(len(objs)).To(Equal(18))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "openshift-multus", "network-metrics-daemon")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("Service", "openshift-multus", "network-metrics-service")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRole", "", "metrics-daemon-role")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRoleBinding", "", "metrics-daemon-sa-rolebinding")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ServiceMonitor", "openshift-multus", "monitor-network")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("Role", "openshift-multus", "prometheus-k8s")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("RoleBinding", "openshift-multus", "prometheus-k8s")))
+
+	// Make sure every obj is reasonable:
+	// - it is supported
+	// - it reconciles to itself (steady state)
+	for _, obj := range objs {
+		g.Expect(apply.IsObjectSupported(obj)).NotTo(HaveOccurred())
+		cur := obj.DeepCopy()
+		upd := obj.DeepCopy()
+
+		err = apply.MergeObjectForUpdate(cur, upd)
+		g.Expect(err).NotTo(HaveOccurred())
+
+		tweakMetaForCompare(cur)
+		g.Expect(cur).To(Equal(upd))
+	}
+}