From 5ba3dc276f6cd4484425e4f9fcb8de9d5cbb1af7 Mon Sep 17 00:00:00 2001 From: Zenghui Shi Date: Tue, 7 Feb 2023 09:10:46 +0800 Subject: [PATCH 1/2] Revert "optimize rbac for ovnkube-master" This reverts commit 2601aa5aca8d3c7f81b2877ab5c025a2431881e6. --- assets/components/ovn/clusterrole.yaml | 75 ++++++++++++++++++- assets/components/ovn/clusterrolebinding.yaml | 14 ++++ assets/components/ovn/configmap.yaml | 2 +- assets/components/ovn/master/daemonset.yaml | 5 ++ assets/components/ovn/role.yaml | 22 +++--- assets/components/ovn/rolebinding.yaml | 19 ++++- pkg/components/networking.go | 4 +- 7 files changed, 123 insertions(+), 18 deletions(-) diff --git a/assets/components/ovn/clusterrole.yaml b/assets/components/ovn/clusterrole.yaml index 61d75bd702..9d43f0bb81 100644 --- a/assets/components/ovn/clusterrole.yaml +++ b/assets/components/ovn/clusterrole.yaml @@ -1,3 +1,77 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: openshift-ovn-kubernetes-node +rules: +- apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + - patch +- apiGroups: [""] + resources: + - namespaces + - endpoints + - services + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: ["", "events.k8s.io"] + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: ["k8s.ovn.org"] + resources: + - egressips + verbs: + - get + - list + - watch +- apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: ['authentication.k8s.io'] + resources: ['tokenreviews'] + verbs: ['create'] +- apiGroups: ['authorization.k8s.io'] + resources: ['subjectaccessreviews'] + verbs: ['create'] + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -34,7 +108,6 @@ rules: - patch - apiGroups: [""] resources: - - namespaces - services - endpoints verbs: diff --git a/assets/components/ovn/clusterrolebinding.yaml b/assets/components/ovn/clusterrolebinding.yaml index 397f571ef2..6909d3c666 100644 --- a/assets/components/ovn/clusterrolebinding.yaml +++ b/assets/components/ovn/clusterrolebinding.yaml @@ -1,3 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: openshift-ovn-kubernetes-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: openshift-ovn-kubernetes-node +subjects: +- kind: ServiceAccount + name: ovn-kubernetes-controller + namespace: openshift-ovn-kubernetes + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/assets/components/ovn/configmap.yaml b/assets/components/ovn/configmap.yaml index ec15485292..7279ce1fb8 100644 --- a/assets/components/ovn/configmap.yaml +++ b/assets/components/ovn/configmap.yaml @@ -17,7 +17,7 @@ data: [kubernetes] service-cidrs={{.ServiceCIDR}} ovn-config-namespace="openshift-ovn-kubernetes" - apiserver="https://127.0.0.1:6443" + kubeconfig={{.KubeconfigPath}} host-network-namespace="openshift-host-network" platform-type="BareMetal" diff --git a/assets/components/ovn/master/daemonset.yaml b/assets/components/ovn/master/daemonset.yaml index 1cc27495d1..601b3e7742 100644 --- a/assets/components/ovn/master/daemonset.yaml +++ b/assets/components/ovn/master/daemonset.yaml @@ -406,6 +406,8 @@ spec: name: run-ovn - mountPath: /run/ovnkube-config/ name: ovnkube-config + - mountPath: {{.KubeconfigDir}} + name: kubeconfig - mountPath: /etc/cni/net.d name: host-cni-netd - mountPath: /cni-bin-dir @@ -490,6 +492,9 @@ spec: hostPath: path: "/opt/cni/bin" + - name: kubeconfig + hostPath: + path: {{.KubeconfigDir}} - name: ovnkube-config configMap: name: ovnkube-config diff --git a/assets/components/ovn/role.yaml b/assets/components/ovn/role.yaml index e5af2e22b3..72d2555d25 100644 --- a/assets/components/ovn/role.yaml +++ b/assets/components/ovn/role.yaml @@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: openshift-ovn-kubernetes-controller + name: openshift-ovn-kubernetes-node namespace: openshift-ovn-kubernetes rules: - apiGroups: [""] @@ -10,9 +10,6 @@ rules: - configmaps verbs: - get - - create - - update - - patch - list - watch - apiGroups: [certificates.k8s.io] @@ -23,6 +20,14 @@ rules: - delete - update - list + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: openshift-ovn-kubernetes-sbdb + namespace: openshift-ovn-kubernetes +rules: - apiGroups: [""] resources: - endpoints @@ -30,12 +35,3 @@ rules: - create - update - patch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - update diff --git a/assets/components/ovn/rolebinding.yaml b/assets/components/ovn/rolebinding.yaml index 780dd77be9..655c16a08a 100644 --- a/assets/components/ovn/rolebinding.yaml +++ b/assets/components/ovn/rolebinding.yaml @@ -2,12 +2,27 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: openshift-ovn-kubernetes-controller + name: openshift-ovn-kubernetes-node namespace: openshift-ovn-kubernetes roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: openshift-ovn-kubernetes-controller + name: openshift-ovn-kubernetes-node +subjects: +- kind: ServiceAccount + name: ovn-kubernetes-controller + namespace: openshift-ovn-kubernetes + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: openshift-ovn-kubernetes-sbdb + namespace: openshift-ovn-kubernetes +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: openshift-ovn-kubernetes-sbdb subjects: - kind: ServiceAccount name: ovn-kubernetes-controller diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 275b5fc71e..7775c04421 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -75,7 +75,9 @@ func startCNIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { return err } extraParams := assets.RenderParams{ - "OVNConfig": ovnConfig, + "OVNConfig": ovnConfig, + "KubeconfigPath": kubeconfigPath, + "KubeconfigDir": filepath.Dir(kubeconfigPath), } if err := assets.ApplyConfigMaps(cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply configMap %v %v", cm, err) From 3b3595e23d2f0cb34b844fe73dca6486243196c9 Mon Sep 17 00:00:00 2001 From: Zenghui Shi Date: Tue, 7 Feb 2023 09:10:52 +0800 Subject: [PATCH 2/2] Revert "Combine ovnkube master and node to single pod" This reverts commit 819b894f263a90c054203f345b0b8e1a922ac115. --- assets/components/ovn/clusterrolebinding.yaml | 2 +- assets/components/ovn/master/daemonset.yaml | 84 +++++------- assets/components/ovn/node/daemonset.yaml | 129 ++++++++++++++++++ .../components/ovn/node/serviceaccount.yaml | 6 + assets/components/ovn/rolebinding.yaml | 2 +- pkg/components/networking.go | 4 +- 6 files changed, 173 insertions(+), 54 deletions(-) create mode 100644 assets/components/ovn/node/daemonset.yaml create mode 100644 assets/components/ovn/node/serviceaccount.yaml diff --git a/assets/components/ovn/clusterrolebinding.yaml b/assets/components/ovn/clusterrolebinding.yaml index 6909d3c666..576d363a25 100644 --- a/assets/components/ovn/clusterrolebinding.yaml +++ b/assets/components/ovn/clusterrolebinding.yaml @@ -9,7 +9,7 @@ roleRef: name: openshift-ovn-kubernetes-node subjects: - kind: ServiceAccount - name: ovn-kubernetes-controller + name: ovn-kubernetes-node namespace: openshift-ovn-kubernetes --- diff --git a/assets/components/ovn/master/daemonset.yaml b/assets/components/ovn/master/daemonset.yaml index 601b3e7742..28c9945577 100644 --- a/assets/components/ovn/master/daemonset.yaml +++ b/assets/components/ovn/master/daemonset.yaml @@ -40,55 +40,8 @@ spec: # /etc/openvswitch -> /var/lib/ovn/etc - ovsdb data # /var/lib/openvswitch -> /var/lib/ovn/data - ovsdb pki state # /run/openvswitch -> tmpfs - sockets + # /env -> configmap env-overrides - debug overrides containers: - # ovn-controller: programs the vswitch with flows from the sbdb - - name: ovn-controller - image: {{ .ReleaseImage.ovn_kubernetes_microshift }} - command: - - /bin/bash - - -c - - | - set -e - - exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ - --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \ - --syslog-method="null" \ - --log-file=/var/log/ovn/acl-audit-log.log \ - -vFACILITY:"local0" \ - -vconsole:"${OVN_LOG_LEVEL}" -vconsole:"acl_log:off" \ - -vPATTERN:console:"%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m" \ - -vsyslog:"acl_log:info" \ - -vfile:"acl_log:info" - securityContext: - privileged: true - env: - - name: OVN_LOG_LEVEL - value: info - - name: K8S_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /run/openvswitch - name: run-openvswitch - - mountPath: /run/ovn/ - name: run-ovn - - mountPath: /etc/openvswitch - name: etc-openvswitch-node - - mountPath: /etc/ovn/ - name: etc-openvswitch-node - - mountPath: /var/lib/openvswitch - name: var-lib-openvswitch - - mountPath: /var/log/ovn - name: node-log - - mountPath: /dev/log - name: log-socket - resources: - requests: - cpu: 10m - memory: 10Mi - terminationMessagePolicy: FallbackToLogsOnError - # ovn-northd: convert network objects in nbdb to flows in sbdb - name: northd image: {{ .ReleaseImage.ovn_kubernetes_microshift }} @@ -97,6 +50,11 @@ spec: - -c - | set -xem + if [[ -f /env/_master ]]; then + set -o allexport + source /env/_master + set +o allexport + fi quit() { echo "$(date -Iseconds) - stopping ovn-northd" @@ -129,6 +87,8 @@ spec: name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /env + name: env-overrides resources: requests: cpu: 10m @@ -143,6 +103,11 @@ spec: - -c - | set -xem + if [[ -f /env/_master ]]; then + set -o allexport + source /env/_master + set +o allexport + fi quit() { echo "$(date -Iseconds) - stopping nbdb" @@ -248,6 +213,8 @@ spec: name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /env + name: env-overrides resources: requests: cpu: 10m @@ -262,6 +229,11 @@ spec: - -c - | set -xem + if [[ -f /env/_master ]]; then + set -o allexport + source /env/_master + set +o allexport + fi quit() { echo "$(date -Iseconds) - stopping sbdb" @@ -333,6 +305,8 @@ spec: name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /env + name: env-overrides resources: requests: cpu: 10m @@ -347,6 +321,11 @@ spec: - -c - | set -xe + if [[ -f "/env/_master" ]]; then + set -o allexport + source "/env/_master" + set +o allexport + fi # K8S_NODE_IP triggers reconcilation of this daemon when node IP changes echo "$(date -Iseconds) - starting ovnkube-master, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}" @@ -408,6 +387,8 @@ spec: name: ovnkube-config - mountPath: {{.KubeconfigDir}} name: kubeconfig + - mountPath: /env + name: env-overrides - mountPath: /etc/cni/net.d name: host-cni-netd - mountPath: /cni-bin-dir @@ -460,9 +441,6 @@ spec: - name: run-ovn hostPath: path: /var/run/ovn - - name: var-lib-openvswitch - hostPath: - path: /var/lib/openvswitch/data # used for iptables wrapper scripts - name: host-slash @@ -498,5 +476,9 @@ spec: - name: ovnkube-config configMap: name: ovnkube-config + - name: env-overrides + configMap: + name: env-overrides + optional: true tolerations: - operator: "Exists" diff --git a/assets/components/ovn/node/daemonset.yaml b/assets/components/ovn/node/daemonset.yaml new file mode 100644 index 0000000000..7e7cf95137 --- /dev/null +++ b/assets/components/ovn/node/daemonset.yaml @@ -0,0 +1,129 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovnkube-node + namespace: openshift-ovn-kubernetes + annotations: + kubernetes.io/description: | + This daemonset launches the ovn-kubernetes per node networking components. +spec: + selector: + matchLabels: + app: ovnkube-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 10% + template: + metadata: + annotations: + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + labels: + app: ovnkube-node + component: network + type: infra + openshift.io/component: network + kubernetes.io/os: "linux" + spec: + serviceAccountName: ovn-kubernetes-node + hostNetwork: true + dnsPolicy: Default + hostPID: true + priorityClassName: "system-node-critical" + # volumes in all containers: + # (container) -> (host) + # /etc/openvswitch -> /etc/openvswitch - ovsdb system id + # /var/lib/openvswitch -> /var/lib/openvswitch/data - ovsdb data + # /run/openvswitch -> tmpfs - ovsdb sockets + # /env -> configmap env-overrides - debug overrides + containers: + # ovn-controller: programs the vswitch with flows from the sbdb + - name: ovn-controller + image: {{ .ReleaseImage.ovn_kubernetes_microshift }} + command: + - /bin/bash + - -c + - | + set -e + if [[ -f "/env/${K8S_NODE}" ]]; then + set -o allexport + source "/env/${K8S_NODE}" + set +o allexport + fi + + # K8S_NODE_IP triggers reconcilation of this daemon when node IP changes + echo "$(date -Iseconds) - starting ovn-controller, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}" + + exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ + --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \ + --syslog-method="null" \ + --log-file=/var/log/ovn/acl-audit-log.log \ + -vFACILITY:"local0" \ + -vconsole:"${OVN_LOG_LEVEL}" -vconsole:"acl_log:off" \ + -vPATTERN:console:"%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m" \ + -vsyslog:"acl_log:info" \ + -vfile:"acl_log:info" + securityContext: + privileged: true + env: + - name: OVN_LOG_LEVEL + value: info + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + volumeMounts: + - mountPath: /run/openvswitch + name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn + - mountPath: /etc/openvswitch + name: etc-openvswitch + - mountPath: /etc/ovn/ + name: etc-openvswitch + - mountPath: /var/lib/openvswitch + name: var-lib-openvswitch + - mountPath: /env + name: env-overrides + - mountPath: /var/log/ovn + name: node-log + - mountPath: /dev/log + name: log-socket + terminationMessagePolicy: FallbackToLogsOnError + resources: + requests: + cpu: 10m + memory: 10Mi + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: var-lib-openvswitch + hostPath: + path: /var/lib/openvswitch/data + - name: etc-openvswitch + hostPath: + path: /etc/openvswitch + - name: run-openvswitch + hostPath: + path: /var/run/openvswitch + - name: run-ovn + hostPath: + path: /var/run/ovn + # Used for placement of ACL audit logs + - name: node-log + hostPath: + path: /var/log/ovn + - name: log-socket + hostPath: + path: /dev/log + - name: env-overrides + configMap: + name: env-overrides + optional: true + tolerations: + - operator: "Exists" diff --git a/assets/components/ovn/node/serviceaccount.yaml b/assets/components/ovn/node/serviceaccount.yaml new file mode 100644 index 0000000000..c4e7d41681 --- /dev/null +++ b/assets/components/ovn/node/serviceaccount.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovn-kubernetes-node + namespace: openshift-ovn-kubernetes diff --git a/assets/components/ovn/rolebinding.yaml b/assets/components/ovn/rolebinding.yaml index 655c16a08a..6df872e63f 100644 --- a/assets/components/ovn/rolebinding.yaml +++ b/assets/components/ovn/rolebinding.yaml @@ -10,7 +10,7 @@ roleRef: name: openshift-ovn-kubernetes-node subjects: - kind: ServiceAccount - name: ovn-kubernetes-controller + name: ovn-kubernetes-node namespace: openshift-ovn-kubernetes --- diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 7775c04421..2f5fcc7e51 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -17,6 +17,7 @@ func startCNIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { "components/ovn/namespace.yaml", } sa = []string{ + "components/ovn/node/serviceaccount.yaml", "components/ovn/master/serviceaccount.yaml", } r = []string{ @@ -36,6 +37,7 @@ func startCNIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { } apps = []string{ "components/ovn/master/daemonset.yaml", + "components/ovn/node/daemonset.yaml", } ) @@ -77,7 +79,7 @@ func startCNIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { extraParams := assets.RenderParams{ "OVNConfig": ovnConfig, "KubeconfigPath": kubeconfigPath, - "KubeconfigDir": filepath.Dir(kubeconfigPath), + "KubeconfigDir": filepath.Join(microshiftDataDir, "/resources/kubeadmin"), } if err := assets.ApplyConfigMaps(cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply configMap %v %v", cm, err)