Multus should request certificates per-node.

This introduces configurations for multus per-node certification. slightly modified in multus.yaml the whereabouts-reconcile serviceAccountName did not apply cleanly
openshift · Mar 7, 2024 · ea0908a · ea0908a
1 parent b02ffc0
commit ea0908a
Show file tree

Hide file tree

Showing 12 changed files with 219 additions and 19 deletions.
diff --git a/bindata/network/multus-networkpolicy/multus-networkpolicy.yaml b/bindata/network/multus-networkpolicy/multus-networkpolicy.yaml
@@ -53,7 +53,7 @@ spec:
         kubernetes.io/os: linux
       priorityClassName: "system-node-critical"
       restartPolicy: Always
-      serviceAccountName: multus
+      serviceAccountName: multus-ancillary-tools
       tolerations:
       - operator: Exists
       volumes:

diff --git a/bindata/network/multus/002-rbac.yaml b/bindata/network/multus/002-rbac.yaml
@@ -53,19 +53,105 @@ rules:
   - create
   - patch
   - update
+- apiGroups: ["certificates.k8s.io"]
+  resources:
+    - certificatesigningrequests
+  verbs:
+    - create
+    - get
+    - list
+    - watch
+
+---
+# This role is ancillary tools in the multus namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: multus-ancillary-tools
+rules:
+- apiGroups: ["apiextensions.k8s.io"]
+  resources:
+  - customresourcedefinitions
+  - customresourcedefinitions/status
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: ["k8s.cni.cncf.io"]
+  resources: ["*"]
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups: ["", "events.k8s.io"]
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
 
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: multus
   namespace: openshift-multus
-
 ---
+kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-transient
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-ancillary-tools
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: openshift-multus
+
+
+{{ if .NETWORK_NODE_IDENTITY_ENABLE }}
+---
 kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
+  name: multus-group
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
   name: multus
+subjects:
+  - kind: Group
+    name: system:multus
+    apiGroup: rbac.authorization.k8s.io
+{{ else }}
+# Network node identity is disabled, therefore, we bind to a service account for Multus itself
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-non-identity-enabled
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
@@ -74,16 +160,37 @@ subjects:
 - kind: ServiceAccount
   name: multus
   namespace: openshift-multus
+{{ end }}
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus-ancillary-tools
+  namespace: openshift-multus
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-ancillary-tools
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-ancillary-tools
+subjects:
+- kind: ServiceAccount
+  name: multus-ancillary-tools
+  namespace: openshift-multus
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: multus-cluster-reader
+  name: multus-cluster-readers
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: multus
+  name: multus-ancillary-tools
 subjects:
 - kind: Group
   name: system:cluster-readers
@@ -99,7 +206,7 @@ roleRef:
   name: whereabouts-cni
 subjects:
 - kind: ServiceAccount
-  name: multus
+  name: multus-ancillary-tools
   namespace: openshift-multus
 
 ---
@@ -114,7 +221,7 @@ roleRef:
   name: whereabouts-cni
 subjects:
 - kind: ServiceAccount
-  name: multus
+  name: multus-ancillary-tools
   namespace: openshift-multus
 
 ---

diff --git a/bindata/network/multus/multus.yaml b/bindata/network/multus/multus.yaml
@@ -121,6 +121,13 @@ data:
         "logToStderr": true,
         "logLevel": "verbose",
         "binDir": "{{ .CNIBinDir }}",
+{{ if .NETWORK_NODE_IDENTITY_ENABLE }}
+        "perNodeCertificate": {
+          "enabled": true,
+          "bootstrapKubeconfig": "/hostroot/var/lib/kubelet/kubeconfig",
+          "certDir": "/run/multus/certs"
+        },
+{{ end }}
         "cniConfigDir": "/host/etc/cni/net.d",
         "multusConfigFile": "auto",
         "multusAutoconfigDir": "/host/run/multus/cni/net.d",
@@ -166,10 +173,12 @@ spec:
       hostPID: true
       nodeSelector:
         kubernetes.io/os: linux
+{{ if not .NETWORK_NODE_IDENTITY_ENABLE }}
+      serviceAccountName: multus
+{{ end }}
       priorityClassName: "system-node-critical"
       tolerations:
       - operator: Exists
-      serviceAccountName: multus
       containers:
       - name: kube-multus
         image: {{.MultusImage}}
@@ -218,6 +227,8 @@ spec:
         - name: multus-daemon-config
           mountPath: /etc/cni/net.d/multus.d
           readOnly: true
+        - name: host-run-multus-certs
+          mountPath: /run/multus/certs
         env:
         - name: RHEL8_SOURCE_DIRECTORY
           value: "/usr/src/multus-cni/rhel8/bin/"
@@ -241,6 +252,10 @@ spec:
         - name: "NO_PROXY"
           value: "{{ .NO_PROXY}}"
 {{ end }}
+        - name: K8S_NODE
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
       terminationGracePeriodSeconds: 10
       volumes:
         - name: system-cni-dir
@@ -294,6 +309,9 @@ spec:
             items:
             - key: daemon-config.json
               path: daemon-config.json
+        - name: host-run-multus-certs
+          hostPath:
+            path: /run/multus_certs
 ---
 kind: DaemonSet
 apiVersion: apps/v1
@@ -328,7 +346,7 @@ spec:
       priorityClassName: "system-node-critical"
       tolerations:
       - operator: Exists
-      serviceAccountName: multus
+      serviceAccountName: multus-ancillary-tools
       initContainers:
       - name: egress-router-binary-copy
         image: {{.EgressRouterImage}}
@@ -620,7 +638,7 @@ spec:
       hostNetwork: true      
       nodeSelector:
         kubernetes.io/os: linux
-      serviceAccountName: multus
+      serviceAccountName: multus-ancillary-tools
       priorityClassName: "system-node-critical"
       tolerations:
       - operator: Exists

diff --git a/...ntity/common/node-identity-namespace.yaml → ...y/common/001-node-identity-namespace.yaml b/...ntity/common/node-identity-namespace.yaml → ...y/common/001-node-identity-namespace.yaml
diff --git a/...e-identity/common/node-identity-rbac.yaml → ...entity/common/002-node-identity-rbac.yaml b/...e-identity/common/node-identity-rbac.yaml → ...entity/common/002-node-identity-rbac.yaml
diff --git a/bindata/network/node-identity/managed/node-identity-configmap.yml b/bindata/network/node-identity/managed/node-identity-configmap.yml
@@ -0,0 +1,23 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovnkube-identity-cm
+  namespace: {{.HostedClusterNamespace}}
+  annotations:
+    kubernetes.io/description: |
+      This configmap contains the ovnkube-identity configuration files.
+    network.operator.openshift.io/cluster-name:  {{.ManagementClusterName}}
+data:
+  additional-cert-acceptance-cond.json: |
+    [{
+      "commonNamePrefix":"system:multus",
+      "organizations": ["system:multus"],
+      "groups": ["system:nodes", "system:multus", "system:authenticated"],
+      "userPrefixes": ["system:node", "system:multus"]
+    }]
+  additional-pod-admission-cond.json: |
+    [{
+      "commonNamePrefix":"system:multus",
+      "allowedPodAnnotations": ["k8s.v1.cni.cncf.io/network-status"]
+    }]
diff --git a/bindata/network/node-identity/managed/node-identity.yaml b/bindata/network/node-identity/managed/node-identity.yaml
@@ -119,6 +119,7 @@ spec:
                 --webhook-port={{.NetworkNodeIdentityPort}} \
                 --disable-approver \
                 --extra-allowed-user="system:serviceaccount:openshift-ovn-kubernetes:ovn-kubernetes-control-plane" \
+                --pod-admission-conditions="/var/run/ovnkube-identity-config/additional-pod-admission-cond.json" \
                 --loglevel="${LOGLEVEL}"
         env:
           - name: LOGLEVEL
@@ -141,6 +142,8 @@ spec:
             name: hosted-cluster-api-access
           - mountPath: /hosted-ca
             name: hosted-ca-cert
+          - mountPath: /var/run/ovnkube-identity-config
+            name: ovnkube-identity-cm
       - name: approver
         image: "{{.NetworkNodeIdentityImage}}"
         command:
@@ -167,6 +170,7 @@ spec:
             exec /usr/bin/ovnkube-identity \
                 --kubeconfig=/var/run/secrets/hosted_cluster/kubeconfig \
                 --lease-namespace=openshift-network-node-identity \
+                --csr-acceptance-conditions="/var/run/ovnkube-identity-config/additional-cert-acceptance-cond.json" \
                 --disable-webhook \
                 --loglevel="${LOGLEVEL}"
         env:
@@ -184,6 +188,8 @@ spec:
             name: hosted-cluster-api-access
           - mountPath: /hosted-ca
             name: hosted-ca-cert
+          - mountPath: /var/run/ovnkube-identity-config
+            name: ovnkube-identity-cm
       # token-minter creates a token with the default service account path
       # The token is read by the containers to authenticate against the hosted cluster api server
       - name: token-minter
@@ -230,6 +236,14 @@ spec:
           secret:
             defaultMode: 0640
             secretName: network-node-identity-secret
+        - name: ovnkube-identity-cm
+          configMap:
+            name: ovnkube-identity-cm
+            items:
+              - key: additional-cert-acceptance-cond.json
+                path: additional-cert-acceptance-cond.json
+              - key: additional-pod-admission-cond.json
+                path: additional-pod-admission-cond.json
       tolerations:
         - key: "hypershift.openshift.io/control-plane"
           operator: "Equal"

diff --git a/bindata/network/node-identity/self-hosted/node-identity-configmap.yml b/bindata/network/node-identity/self-hosted/node-identity-configmap.yml
@@ -0,0 +1,22 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovnkube-identity-cm
+  namespace: openshift-network-node-identity
+  annotations:
+    kubernetes.io/description: |
+      This configmap contains the ovnkube-identity configuration files.
+data:
+  additional-cert-acceptance-cond.json: |
+    [{
+      "commonNamePrefix":"system:multus",
+      "organizations": ["system:multus"],
+      "groups": ["system:nodes", "system:multus", "system:authenticated"],
+      "userPrefixes": ["system:node", "system:multus"]
+    }]
+  additional-pod-admission-cond.json: |
+    [{
+      "commonNamePrefix":"system:multus",
+      "allowedPodAnnotations": ["k8s.v1.cni.cncf.io/network-status"]
+    }]
diff --git a/bindata/network/node-identity/self-hosted/node-identity.yaml b/bindata/network/node-identity/self-hosted/node-identity.yaml
@@ -55,6 +55,7 @@ spec:
                 --disable-approver \
                 --extra-allowed-user="system:serviceaccount:openshift-ovn-kubernetes:ovn-kubernetes-control-plane" \
                 --wait-for-kubernetes-api={{.NetworkNodeIdentityTerminationDurationSeconds}}s \
+                --pod-admission-conditions="/var/run/ovnkube-identity-config/additional-pod-admission-cond.json" \
                 --loglevel="${LOGLEVEL}"
         env:
           - name: LOGLEVEL
@@ -73,6 +74,8 @@ spec:
             name: webhook-cert
           - mountPath: /env
             name: env-overrides
+          - mountPath: /var/run/ovnkube-identity-config
+            name: ovnkube-identity-cm
       - name: approver
         image: "{{.NetworkNodeIdentityImage}}"
         command:
@@ -89,6 +92,7 @@ spec:
             echo "I$(date "+%m%d %H:%M:%S.%N") - network-node-identity - start approver"
             exec /usr/bin/ovnkube-identity  --k8s-apiserver={{.K8S_APISERVER}} \
                 --disable-webhook \
+                --csr-acceptance-conditions="/var/run/ovnkube-identity-config/additional-cert-acceptance-cond.json" \
                 --loglevel="${LOGLEVEL}"
         env:
           - name: LOGLEVEL
@@ -101,6 +105,8 @@ spec:
         volumeMounts:
           - mountPath: /env
             name: env-overrides
+          - mountPath: /var/run/ovnkube-identity-config
+            name: ovnkube-identity-cm
       terminationGracePeriodSeconds: {{.NetworkNodeIdentityTerminationDurationSeconds}}
       nodeSelector:
         node-role.kubernetes.io/master: ""
@@ -116,6 +122,14 @@ spec:
         - hostPath:
             path: /var/log/kube-apiserver
           name: audit-dir
+        - name: ovnkube-identity-cm
+          configMap:
+            name: ovnkube-identity-cm
+            items:
+              - key: additional-cert-acceptance-cond.json
+                path: additional-cert-acceptance-cond.json
+              - key: additional-pod-admission-cond.json
+                path: additional-pod-admission-cond.json
       tolerations:
       - key: "node-role.kubernetes.io/master"
         operator: "Exists"