diff --git a/bindata/network/node-identity/common/node-identity-namespace.yaml b/bindata/network/node-identity/common/node-identity-namespace.yaml new file mode 100644 index 0000000000..d516031b16 --- /dev/null +++ b/bindata/network/node-identity/common/node-identity-namespace.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-network-node-identity + labels: + openshift.io/cluster-monitoring: "true" + openshift.io/run-level: "0" + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + annotations: + include.release.openshift.io/self-managed-high-availability: "true" + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/single-node-developer: "true" + openshift.io/node-selector: "" + openshift.io/description: "OpenShift network node identity namespace - a controller used to manage node identity components" + workload.openshift.io/allowed: "management" diff --git a/bindata/network/node-identity/common/node-identity-rbac.yaml b/bindata/network/node-identity/common/node-identity-rbac.yaml new file mode 100644 index 0000000000..415e6eaa54 --- /dev/null +++ b/bindata/network/node-identity/common/node-identity-rbac.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: network-node-identity + namespace: openshift-network-node-identity + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: network-node-identity +roleRef: + name: network-node-identity + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: network-node-identity + namespace: openshift-network-node-identity + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: network-node-identity +rules: + - apiGroups: [""] + resources: + - nodes + - pods + verbs: ["get", "list", "watch"] + - apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["get", "list", "watch"] + - apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests/approval + verbs: ["update"] + - apiGroups: [""] + resources: + - events + verbs: ["create", "patch", "update"] + - apiGroups: ["certificates.k8s.io"] + resources: + - signers + resourceNames: + - kubernetes.io/kube-apiserver-client + verbs: ["approve"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: network-node-identity-leases + namespace: openshift-network-node-identity +roleRef: + name: network-node-identity-leases + kind: Role + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: network-node-identity + namespace: openshift-network-node-identity + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: openshift-network-node-identity + name: network-node-identity-leases +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - update diff --git a/bindata/network/node-identity/managed/node-identity-service.yaml b/bindata/network/node-identity/managed/node-identity-service.yaml new file mode 100644 index 0000000000..df55731bfd --- /dev/null +++ b/bindata/network/node-identity/managed/node-identity-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: network-node-identity + namespace: {{.HostedClusterNamespace}} + labels: + app: network-node-identity + hypershift.openshift.io/allow-guest-webhooks: "true" + annotations: + network.operator.openshift.io/cluster-name: {{.ManagementClusterName}} + service.alpha.openshift.io/serving-cert-secret-name: network-node-identity-secret +spec: + ports: + - name: webhook + port: {{.NetworkNodeIdentityPort}} + targetPort: {{.NetworkNodeIdentityPort}} + selector: + app: network-node-identity diff --git a/bindata/network/node-identity/managed/node-identity-webhook.yaml b/bindata/network/node-identity/managed/node-identity-webhook.yaml new file mode 100644 index 0000000000..f00ca38bee --- /dev/null +++ b/bindata/network/node-identity/managed/node-identity-webhook.yaml @@ -0,0 +1,29 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: network-node-identity.openshift.io +webhooks: + - name: node.network-node-identity.openshift.io + clientConfig: + url: https://network-node-identity.{{.HostedClusterNamespace}}.svc:{{.NetworkNodeIdentityPort}}/node + caBundle: {{.NetworkNodeIdentityCABundle}} + admissionReviewVersions: ['v1'] + sideEffects: None + rules: + - operations: [ "UPDATE" ] + apiGroups: ["*"] + apiVersions: ["*"] + resources: ["nodes/status"] + scope: "*" + - name: pod.network-node-identity.openshift.io + clientConfig: + url: https://network-node-identity.{{.HostedClusterNamespace}}.svc:{{.NetworkNodeIdentityPort}}/pod + caBundle: {{.NetworkNodeIdentityCABundle}} + admissionReviewVersions: ['v1'] + sideEffects: None + rules: + - operations: [ "UPDATE" ] + apiGroups: ["*"] + apiVersions: ["*"] + resources: ["pods/status"] + scope: "*" diff --git a/bindata/network/node-identity/managed/node-identity.yaml b/bindata/network/node-identity/managed/node-identity.yaml new file mode 100644 index 0000000000..645cb369d2 --- /dev/null +++ b/bindata/network/node-identity/managed/node-identity.yaml @@ -0,0 +1,247 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: network-node-identity + namespace: {{.HostedClusterNamespace}} + annotations: + network.operator.openshift.io/cluster-name: {{.ManagementClusterName}} + kubernetes.io/description: | + This deployment launches the network-node-identity control plane components. + release.openshift.io/version: "{{.ReleaseVersion}}" + labels: + # used by PodAffinity to prefer co-locating pods that belong to the same hosted cluster. + hypershift.openshift.io/hosted-control-plane: {{.HostedClusterNamespace}} +spec: + replicas: {{.NetworkNodeIdentityReplicas}} +{{ if (gt .NetworkNodeIdentityReplicas 1)}} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 +{{ end }} + selector: + matchLabels: + app: network-node-identity + template: + metadata: + annotations: + hypershift.openshift.io/release-image: {{.ReleaseImage}} + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + labels: + app: network-node-identity + component: network + type: infra + openshift.io/component: network + hypershift.openshift.io/control-plane-component: network-node-identity + kubernetes.io/os: "linux" + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: hypershift.openshift.io/control-plane + operator: In + values: + - "true" + - weight: 100 + preference: + matchExpressions: + - key: hypershift.openshift.io/cluster + operator: In + values: + - {{.HostedClusterNamespace}} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: network-node-identity + topologyKey: topology.kubernetes.io/zone + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + hypershift.openshift.io/hosted-control-plane: {{.HostedClusterNamespace}} + topologyKey: kubernetes.io/hostname + priorityClassName: hypershift-api-critical + initContainers: + - name: hosted-cluster-kubecfg-setup + image: "{{.CLIImage}}" + command: + - /bin/bash + - -c + - | + kc=/var/run/secrets/hosted_cluster/kubeconfig + kubectl --kubeconfig $kc config set clusters.default.server {{ .K8S_LOCAL_APISERVER }} + kubectl --kubeconfig $kc config set clusters.default.certificate-authority /hosted-ca/ca.crt + kubectl --kubeconfig $kc config set users.admin.tokenFile /var/run/secrets/hosted_cluster/token + kubectl --kubeconfig $kc config set contexts.default.cluster default + kubectl --kubeconfig $kc config set contexts.default.user admin + kubectl --kubeconfig $kc config set contexts.default.namespace openshift-network-node-identity + kubectl --kubeconfig $kc config use-context default + volumeMounts: + - mountPath: /var/run/secrets/hosted_cluster + name: hosted-cluster-api-access + containers: + - name: webhook + image: "{{.NetworkNodeIdentityImage}}" + command: + - /bin/bash + - -c + - | + set -xe + if [[ -f "/env/_master" ]]; then + set -o allexport + source "/env/_master" + set +o allexport + fi + + retries=0 + while [ ! -f /var/run/secrets/hosted_cluster/token ]; do + (( retries += 1 )) + sleep 1 + if [[ "${retries}" -gt 30 ]]; then + echo "$(date -Iseconds) - Hosted cluster token not found" + exit 1 + fi + done + ho_enable= +{{- if .OVNHybridOverlayEnable }} + ho_enable="--enable-hybrid-overlay" +{{ end }} + echo "I$(date "+%m%d %H:%M:%S.%N") - network-node-identity - start webhook" + # extra-allowed-user: service account `ovn-kubernetes-control-plane` + # sets pod annotations in multi-homing layer3 network controller (cluster-manager) + exec /usr/bin/ovnkube-identity \ + --kubeconfig=/var/run/secrets/hosted_cluster/kubeconfig \ + --webhook-cert-dir=/etc/webhook-cert \ + --webhook-host="" \ + --webhook-port={{.NetworkNodeIdentityPort}} \ + ${ho_enable} \ + --enable-interconnect \ + --disable-approver \ + --extra-allowed-user="system:serviceaccount:openshift-ovn-kubernetes:ovn-kubernetes-control-plane" \ + --loglevel="${LOGLEVEL}" + env: + - name: LOGLEVEL + value: "5" + resources: + requests: + cpu: 10m + memory: 50Mi + terminationMessagePolicy: FallbackToLogsOnError + ports: + - name: webhook + containerPort: {{.NetworkNodeIdentityPort}} + protocol: TCP + volumeMounts: + - mountPath: /etc/webhook-cert/ + name: webhook-cert + - mountPath: /env + name: env-overrides + - mountPath: /var/run/secrets/hosted_cluster + name: hosted-cluster-api-access + - mountPath: /hosted-ca + name: hosted-ca-cert + - name: approver + image: "{{.NetworkNodeIdentityImage}}" + command: + - /bin/bash + - -c + - | + set -xe + if [[ -f "/env/_master" ]]; then + set -o allexport + source "/env/_master" + set +o allexport + fi + + retries=0 + while [ ! -f /var/run/secrets/hosted_cluster/token ]; do + (( retries += 1 )) + sleep 1 + if [[ "${retries}" -gt 30 ]]; then + echo "$(date -Iseconds) - Hosted cluster token not found" + exit 1 + fi + done + echo "I$(date "+%m%d %H:%M:%S.%N") - network-node-identity - start approver" + exec /usr/bin/ovnkube-identity \ + --kubeconfig=/var/run/secrets/hosted_cluster/kubeconfig \ + --lease-namespace=openshift-network-node-identity \ + --disable-webhook \ + --loglevel="${LOGLEVEL}" + env: + - name: LOGLEVEL + value: "5" + resources: + requests: + cpu: 10m + memory: 50Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /env + name: env-overrides + - mountPath: /var/run/secrets/hosted_cluster + name: hosted-cluster-api-access + - mountPath: /hosted-ca + name: hosted-ca-cert + # token-minter creates a token with the default service account path + # The token is read by the containers to authenticate against the hosted cluster api server + - name: token-minter + image: "{{.TokenMinterImage}}" + command: ["/usr/bin/control-plane-operator", "token-minter"] + args: + - --service-account-namespace=openshift-network-node-identity + - --service-account-name=network-node-identity + - --token-audience={{.TokenAudience}} + - --token-file=/var/run/secrets/hosted_cluster/token + - --kubeconfig=/etc/kubernetes/kubeconfig + resources: + requests: + cpu: 10m + memory: 30Mi + volumeMounts: + - mountPath: /etc/kubernetes + name: admin-kubeconfig + - mountPath: /var/run/secrets/hosted_cluster + name: hosted-cluster-api-access + {{ if .HCPNodeSelector }} + nodeSelector: + {{ range $key, $value := .HCPNodeSelector }} + "{{$key}}": "{{$value}}" + {{ end }} + {{ end }} + volumes: + - name: env-overrides + configMap: + name: env-overrides + optional: true + - name: admin-kubeconfig + secret: + secretName: service-network-admin-kubeconfig + - name: hosted-cluster-api-access + emptyDir: {} + - name: hosted-ca-cert + secret: + secretName: root-ca + items: + - key: ca.crt + path: ca.crt + - name: webhook-cert + secret: + defaultMode: 0640 + secretName: network-node-identity-secret + tolerations: + - key: "hypershift.openshift.io/control-plane" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "hypershift.openshift.io/cluster" + operator: "Equal" + value: {{.HostedClusterNamespace}} + effect: "NoSchedule" diff --git a/bindata/network/node-identity/self-hosted/node-identity-pki.yaml b/bindata/network/node-identity/self-hosted/node-identity-pki.yaml new file mode 100644 index 0000000000..0e335a1feb --- /dev/null +++ b/bindata/network/node-identity/self-hosted/node-identity-pki.yaml @@ -0,0 +1,10 @@ +# Request that the cluster network operator PKI controller +# creates a certificate and key for network-node-identity webhook. +apiVersion: network.operator.openshift.io/v1 +kind: OperatorPKI +metadata: + name: network-node-identity + namespace: openshift-network-node-identity +spec: + targetCert: + commonName: {{.NetworkNodeIdentityAddress}} diff --git a/bindata/network/node-identity/self-hosted/node-identity-webhook.yaml b/bindata/network/node-identity/self-hosted/node-identity-webhook.yaml new file mode 100644 index 0000000000..9a6bca6f3f --- /dev/null +++ b/bindata/network/node-identity/self-hosted/node-identity-webhook.yaml @@ -0,0 +1,29 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: network-node-identity.openshift.io +webhooks: + - name: node.network-node-identity.openshift.io + clientConfig: + url: https://{{.NetworkNodeIdentityAddress}}:{{.NetworkNodeIdentityPort}}/node + caBundle: {{.NetworkNodeIdentityCABundle}} + admissionReviewVersions: ['v1'] + sideEffects: None + rules: + - operations: [ "UPDATE" ] + apiGroups: ["*"] + apiVersions: ["*"] + resources: ["nodes/status"] + scope: "*" + - name: pod.network-node-identity.openshift.io + clientConfig: + url: https://{{.NetworkNodeIdentityAddress}}:{{.NetworkNodeIdentityPort}}/pod + caBundle: {{.NetworkNodeIdentityCABundle}} + admissionReviewVersions: ['v1'] + sideEffects: None + rules: + - operations: [ "UPDATE" ] + apiGroups: ["*"] + apiVersions: ["*"] + resources: ["pods/status"] + scope: "*" diff --git a/bindata/network/node-identity/self-hosted/node-identity.yaml b/bindata/network/node-identity/self-hosted/node-identity.yaml new file mode 100644 index 0000000000..b64e9d7bbe --- /dev/null +++ b/bindata/network/node-identity/self-hosted/node-identity.yaml @@ -0,0 +1,135 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: network-node-identity + namespace: openshift-network-node-identity + annotations: + kubernetes.io/description: | + This daemonset launches the network-node-identity networking components. + release.openshift.io/version: "{{.ReleaseVersion}}" +spec: + selector: + matchLabels: + app: network-node-identity + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 + template: + metadata: + annotations: + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + labels: + app: network-node-identity + component: network + type: infra + openshift.io/component: network + kubernetes.io/os: "linux" + spec: + serviceAccountName: network-node-identity + hostNetwork: true + dnsPolicy: Default + priorityClassName: "system-node-critical" + containers: + - name: webhook + image: "{{.NetworkNodeIdentityImage}}" + command: + - /bin/bash + - -c + - | + set -xe + if [[ -f "/env/_master" ]]; then + set -o allexport + source "/env/_master" + set +o allexport + fi + + ho_enable= +{{- if .OVNHybridOverlayEnable }} + ho_enable="--enable-hybrid-overlay" +{{ end }} + echo "I$(date "+%m%d %H:%M:%S.%N") - network-node-identity - start webhook" + # extra-allowed-user: service account `ovn-kubernetes-control-plane` + # sets pod annotations in multi-homing layer3 network controller (cluster-manager) + exec /usr/bin/ovnkube-identity --k8s-apiserver={{.K8S_APISERVER}} \ + --webhook-cert-dir="/etc/webhook-cert" \ + --webhook-host={{.NetworkNodeIdentityAddress}} \ + --webhook-port={{.NetworkNodeIdentityPort}} \ + ${ho_enable} \ + --enable-interconnect \ + --disable-approver \ + --extra-allowed-user="system:serviceaccount:openshift-ovn-kubernetes:ovn-kubernetes-control-plane" \ + --wait-for-kubernetes-api={{.NetworkNodeIdentityTerminationDurationSeconds}}s \ + --loglevel="${LOGLEVEL}" + env: + - name: LOGLEVEL + value: "2" + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 10m + memory: 50Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /etc/webhook-cert/ + name: webhook-cert + - mountPath: /var/log/kube-apiserver + name: audit-dir + - mountPath: /env + name: env-overrides + - name: approver + image: "{{.NetworkNodeIdentityImage}}" + command: + - /bin/bash + - -c + - | + set -xe + if [[ -f "/env/_master" ]]; then + set -o allexport + source "/env/_master" + set +o allexport + fi + + echo "I$(date "+%m%d %H:%M:%S.%N") - network-node-identity - start approver" + exec /usr/bin/ovnkube-identity --k8s-apiserver={{.K8S_APISERVER}} \ + --disable-webhook \ + --loglevel="${LOGLEVEL}" + env: + - name: LOGLEVEL + value: "4" + resources: + requests: + cpu: 10m + memory: 50Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /env + name: env-overrides + terminationGracePeriodSeconds: {{.NetworkNodeIdentityTerminationDurationSeconds}} + nodeSelector: + node-role.kubernetes.io/master: "" + beta.kubernetes.io/os: "linux" + volumes: + - name: webhook-cert + secret: + secretName: network-node-identity-cert + - name: env-overrides + configMap: + name: env-overrides + optional: true + - hostPath: + path: /var/log/kube-apiserver + name: audit-dir + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + - key: "node.kubernetes.io/network-unavailable" + operator: "Exists" diff --git a/bindata/network/ovn-kubernetes/common/002-rbac-node.yaml b/bindata/network/ovn-kubernetes/common/002-rbac-node.yaml index c195710433..d4e9cc06ac 100644 --- a/bindata/network/ovn-kubernetes/common/002-rbac-node.yaml +++ b/bindata/network/ovn-kubernetes/common/002-rbac-node.yaml @@ -35,16 +35,27 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + # the name change is required to ensure that both bindings exist during upgrade to avoid disruptions + name: openshift-ovn-kubernetes-nodes-identity-limited +{{ else }} name: openshift-ovn-kubernetes-node-limited +{{ end }} namespace: openshift-ovn-kubernetes roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: openshift-ovn-kubernetes-node-limited subjects: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} +- kind: Group + name: system:ovn-nodes + apiGroup: rbac.authorization.k8s.io +{{ else }} - kind: ServiceAccount name: ovn-kubernetes-node namespace: openshift-ovn-kubernetes +{{ end }} --- apiVersion: rbac.authorization.k8s.io/v1 @@ -110,12 +121,6 @@ rules: - get - list - watch -- apiGroups: ['authentication.k8s.io'] - resources: ['tokenreviews'] - verbs: ['create'] -- apiGroups: ['authorization.k8s.io'] - resources: ['subjectaccessreviews'] - verbs: ['create'] - apiGroups: [certificates.k8s.io] resources: ['certificatesigningrequests'] verbs: @@ -184,12 +189,52 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + # the name change is required to ensure that both bindings exist during upgrade to avoid disruptions + name: openshift-ovn-kubernetes-node-identity-limited +{{ else }} name: openshift-ovn-kubernetes-node-limited +{{ end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: openshift-ovn-kubernetes-node-limited subjects: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} +- kind: Group + name: system:ovn-nodes + apiGroup: rbac.authorization.k8s.io +{{ else }} +- kind: ServiceAccount + name: ovn-kubernetes-node + namespace: openshift-ovn-kubernetes +{{ end }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: openshift-ovn-kubernetes-kube-rbac-proxy +rules: + - apiGroups: ['authentication.k8s.io'] + resources: ['tokenreviews'] + verbs: ['create'] + - apiGroups: ['authorization.k8s.io'] + resources: ['subjectaccessreviews'] + verbs: ['create'] + +--- +# openshift-ovn-kubernetes-kube-rbac-proxy cluster role is bound to ovn-kubernetes-node service account even if NETWORK_NODE_IDENTITY_ENABLE is true. +# The kube-rbac-proxy-node container continues to use the service account instead of the per-node certificates. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: openshift-ovn-kubernetes-node-kube-rbac-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: openshift-ovn-kubernetes-kube-rbac-proxy +subjects: - kind: ServiceAccount name: ovn-kubernetes-node namespace: openshift-ovn-kubernetes diff --git a/bindata/network/ovn-kubernetes/common/ipsec-containerized.yaml b/bindata/network/ovn-kubernetes/common/ipsec-containerized.yaml index 1ace0c3547..f18a0ab519 100644 --- a/bindata/network/ovn-kubernetes/common/ipsec-containerized.yaml +++ b/bindata/network/ovn-kubernetes/common/ipsec-containerized.yaml @@ -47,6 +47,48 @@ spec: - | #!/bin/bash set -exuo pipefail +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + # When NETWORK_NODE_IDENTITY_ENABLE is true, use the per-node certificate to create a kubeconfig + # that will be used to talk to the API + + + # Wait for cert file + retries=0 + tries=20 + key_cert="/etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem" + while [ ! -f "${key_cert}" ]; do + (( retries += 1 )) + if [[ "${retries}" -gt ${tries} ]]; then + echo "$(date -Iseconds) - ERROR - ${key_cert} not found" + return 1 + fi + sleep 1 + done + + cat << EOF > /var/run/ovnkube-kubeconfig + apiVersion: v1 + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: {{.K8S_APISERVER}} + name: default-cluster + contexts: + - context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context + current-context: default-context + kind: Config + preferences: {} + users: + - name: default-auth + user: + client-certificate: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + client-key: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + EOF + export KUBECONFIG=/var/run/ovnkube-kubeconfig +{{ end }} if rpm --dbpath=/usr/share/rpm -q libreswan; then echo "host has libreswan and therefore ipsec will be configured by ipsec daemonset, this ovn ipsec container doesnt need to init anything" @@ -140,6 +182,10 @@ spec: securityContext: privileged: true volumeMounts: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /etc/ovn/ + name: etc-ovn +{{ end }} - mountPath: /var/run/openvswitch name: host-var-run-ovs - mountPath: /signer-ca @@ -271,6 +317,11 @@ spec: beta.kubernetes.io/os: "linux" terminationGracePeriodSeconds: 10 volumes: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - name: etc-ovn + hostPath: + path: /var/lib/ovn-ic/etc +{{ end }} - name: host-var-log-ovs hostPath: path: /var/log/openvswitch diff --git a/bindata/network/ovn-kubernetes/common/ipsec-host.yaml b/bindata/network/ovn-kubernetes/common/ipsec-host.yaml index ffb79e3a45..66ff6cb9ba 100644 --- a/bindata/network/ovn-kubernetes/common/ipsec-host.yaml +++ b/bindata/network/ovn-kubernetes/common/ipsec-host.yaml @@ -48,6 +48,48 @@ spec: - | #!/bin/bash set -exuo pipefail +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + # When NETWORK_NODE_IDENTITY_ENABLE is true, use the per-node certificate to create a kubeconfig + # that will be used to talk to the API + + + # Wait for cert file + retries=0 + tries=20 + key_cert="/etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem" + while [ ! -f "${key_cert}" ]; do + (( retries += 1 )) + if [[ "${retries}" -gt ${tries} ]]; then + echo "$(date -Iseconds) - ERROR - ${key_cert} not found" + return 1 + fi + sleep 1 + done + + cat << EOF > /var/run/ovnkube-kubeconfig + apiVersion: v1 + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: {{.K8S_APISERVER}} + name: default-cluster + contexts: + - context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context + current-context: default-context + kind: Config + preferences: {} + users: + - name: default-auth + user: + client-certificate: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + client-key: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + EOF + export KUBECONFIG=/var/run/ovnkube-kubeconfig +{{ end }} if ! rpm --dbpath=/usr/share/rpm -q libreswan; then echo "host doesnt have libreswan, therefore ipsec will be configured by ipsec-pre414 daemonset, this ovn ipsec container has nothing to init" @@ -143,6 +185,10 @@ spec: securityContext: privileged: true volumeMounts: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /etc/ovn/ + name: etc-ovn +{{ end }} - mountPath: /var/run name: host-var-run - mountPath: /signer-ca @@ -286,6 +332,11 @@ spec: beta.kubernetes.io/os: "linux" terminationGracePeriodSeconds: 10 volumes: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - name: etc-ovn + hostPath: + path: /var/lib/ovn-ic/etc +{{ end }} - hostPath: path: /var/log/openvswitch type: DirectoryOrCreate diff --git a/bindata/network/ovn-kubernetes/managed/multi-zone-interconnect/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/managed/multi-zone-interconnect/ovnkube-node.yaml index da77e97de9..91dbe6f51e 100644 --- a/bindata/network/ovn-kubernetes/managed/multi-zone-interconnect/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/managed/multi-zone-interconnect/ovnkube-node.yaml @@ -833,6 +833,15 @@ spec: ip_forwarding_flag="--disable-forwarding" fi + NETWORK_NODE_IDENTITY_ENABLE= + if [[ "{{.NETWORK_NODE_IDENTITY_ENABLE}}" == "true" ]]; then + NETWORK_NODE_IDENTITY_ENABLE=" + --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig + --cert-dir=/etc/ovn/ovnkube-node-certs + --cert-duration={{.NodeIdentityCertDuration}} + " + fi + exec /usr/bin/ovnkube --init-ovnkube-controller "${K8S_NODE}" --init-node "${K8S_NODE}" \ --config-file=/run/ovnkube-config/ovnkube.conf \ --ovn-empty-lb-events \ @@ -859,7 +868,8 @@ spec: --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}" \ ${gw_interface_flag} \ --enable-multi-external-gateway=true \ - ${ip_forwarding_flag} + ${ip_forwarding_flag} \ + ${NETWORK_NODE_IDENTITY_ENABLE} env: # for kubectl - name: KUBERNETES_SERVICE_PORT @@ -914,6 +924,11 @@ spec: privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /var/lib/kubelet + name: host-kubelet + readOnly: true +{{ end }} # for checking ovs-configuration service - mountPath: /etc/systemd/system name: systemd-units @@ -986,6 +1001,44 @@ spec: - -c - | set -xe +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + # Wait for cert file + retries=0 + tries=20 + key_cert="/etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem" + while [ ! -f "${key_cert}" ]; do + (( retries += 1 )) + if [[ "${retries}" -gt ${tries} ]]; then + echo "$(date -Iseconds) - ERROR - ${key_cert} not found" + return 1 + fi + sleep 1 + done + + cat << EOF > /var/run/ovnkube-kubeconfig + apiVersion: v1 + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: {{.K8S_APISERVER}} + name: default-cluster + contexts: + - context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context + current-context: default-context + kind: Config + preferences: {} + users: + - name: default-auth + user: + client-certificate: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + client-key: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + EOF + export KUBECONFIG=/var/run/ovnkube-kubeconfig +{{ end }} touch /var/run/ovn/add_iptables.sh chmod 0755 /var/run/ovn/add_iptables.sh @@ -1029,6 +1082,10 @@ spec: securityContext: privileged: true volumeMounts: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /etc/ovn/ + name: etc-openvswitch +{{ end }} # for the iptables wrapper - mountPath: /host name: host-slash @@ -1049,6 +1106,11 @@ spec: nodeSelector: beta.kubernetes.io/os: "linux" volumes: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - name: host-kubelet + hostPath: + path: /var/lib/kubelet +{{ end }} # for checking ovs-configuration service - name: systemd-units hostPath: diff --git a/bindata/network/ovn-kubernetes/managed/single-zone-interconnect/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/managed/single-zone-interconnect/ovnkube-node.yaml index cce7731991..2b960c41ce 100644 --- a/bindata/network/ovn-kubernetes/managed/single-zone-interconnect/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/managed/single-zone-interconnect/ovnkube-node.yaml @@ -517,6 +517,15 @@ spec: multi_network_policy_enabled_flag="--enable-multi-networkpolicy" fi + NETWORK_NODE_IDENTITY_ENABLE= + if [[ "{{.NETWORK_NODE_IDENTITY_ENABLE}}" == "true" ]]; then + NETWORK_NODE_IDENTITY_ENABLE=" + --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig + --cert-dir=/etc/ovn/ovnkube-node-certs + --cert-duration={{.NodeIdentityCertDuration}} + " + fi + exec /usr/bin/ovnkube --init-node "${K8S_NODE}" \ --enable-interconnect \ --zone global \ @@ -547,6 +556,7 @@ spec: ${multi_network_enabled_flag} \ ${multi_network_policy_enabled_flag} \ ${gw_interface_flag} \ + ${NETWORK_NODE_IDENTITY_ENABLE} \ --enable-multi-external-gateway=true env: # for kubectl @@ -614,6 +624,11 @@ spec: privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /var/lib/kubelet + name: host-kubelet + readOnly: true +{{ end }} # for checking ovs-configuration service - mountPath: /etc/systemd/system name: systemd-units @@ -686,6 +701,31 @@ spec: - -c - | set -xe +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + cat << EOF > /var/run/ovnkube-kubeconfig + apiVersion: v1 + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: {{.K8S_APISERVER}} + name: default-cluster + contexts: + - context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context + current-context: default-context + kind: Config + preferences: {} + users: + - name: default-auth + user: + client-certificate: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + client-key: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + EOF + export KUBECONFIG=/var/run/ovnkube-kubeconfig +{{ end }} touch /var/run/ovn/add_iptables.sh chmod 0755 /var/run/ovn/add_iptables.sh @@ -729,6 +769,10 @@ spec: securityContext: privileged: true volumeMounts: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /etc/ovn/ + name: etc-openvswitch +{{ end }} # for the iptables wrapper - mountPath: /host name: host-slash @@ -749,6 +793,11 @@ spec: nodeSelector: beta.kubernetes.io/os: "linux" volumes: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - name: host-kubelet + hostPath: + path: /var/lib/kubelet +{{ end }} # for checking ovs-configuration service - name: systemd-units hostPath: diff --git a/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-node.yaml index a84723b274..0a52215423 100644 --- a/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-node.yaml @@ -79,6 +79,40 @@ spec: dnsPolicy: Default hostPID: true priorityClassName: "system-node-critical" +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + initContainers: + - name: kubecfg-setup + image: "{{.OvnImage}}" + command: + - /bin/bash + - -c + - | + cat << EOF > /etc/ovn/kubeconfig + apiVersion: v1 + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: {{.K8S_APISERVER}} + name: default-cluster + contexts: + - context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context + current-context: default-context + kind: Config + preferences: {} + users: + - name: default-auth + user: + client-certificate: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + client-key: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + EOF + volumeMounts: + - mountPath: /etc/ovn/ + name: etc-openvswitch +{{ end }} # volumes in all containers: # (container) -> (host) # /etc/openvswitch -> /etc/openvswitch - ovsdb system id @@ -212,6 +246,7 @@ spec: - | #!/bin/bash set -euo pipefail + TLS_PK=/etc/pki/tls/metrics-cert/tls.key TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt # As the secret mount is optional we must wait for the files to be present. @@ -264,6 +299,7 @@ spec: - | #!/bin/bash set -euo pipefail + TLS_PK=/etc/pki/tls/metrics-cert/tls.key TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt # As the secret mount is optional we must wait for the files to be present. @@ -855,6 +891,15 @@ spec: ip_forwarding_flag="--disable-forwarding" fi + NETWORK_NODE_IDENTITY_ENABLE= + if [[ "{{.NETWORK_NODE_IDENTITY_ENABLE}}" == "true" ]]; then + NETWORK_NODE_IDENTITY_ENABLE=" + --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig + --cert-dir=/etc/ovn/ovnkube-node-certs + --cert-duration={{.NodeIdentityCertDuration}} + " + fi + exec /usr/bin/ovnkube --init-ovnkube-controller "${K8S_NODE}" --init-node "${K8S_NODE}" \ --config-file=/run/ovnkube-config/ovnkube.conf \ --ovn-empty-lb-events \ @@ -881,7 +926,8 @@ spec: --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}" \ ${gw_interface_flag} \ --enable-multi-external-gateway=true \ - ${ip_forwarding_flag} + ${ip_forwarding_flag} \ + ${NETWORK_NODE_IDENTITY_ENABLE} env: # for kubectl - name: KUBERNETES_SERVICE_PORT @@ -936,6 +982,11 @@ spec: privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /var/lib/kubelet + name: host-kubelet + readOnly: true +{{ end }} # for checking ovs-configuration service - mountPath: /etc/systemd/system name: systemd-units @@ -1008,6 +1059,21 @@ spec: - -c - | set -xe +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + # Wait for cert file + retries=0 + tries=20 + key_cert="/etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem" + while [ ! -f "${key_cert}" ]; do + (( retries += 1 )) + if [[ "${retries}" -gt ${tries} ]]; then + echo "$(date -Iseconds) - ERROR - ${key_cert} not found" + return 1 + fi + sleep 1 + done + export KUBECONFIG=/etc/ovn/kubeconfig +{{ end }} touch /var/run/ovn/add_iptables.sh chmod 0755 /var/run/ovn/add_iptables.sh @@ -1051,6 +1117,10 @@ spec: securityContext: privileged: true volumeMounts: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /etc/ovn/ + name: etc-openvswitch +{{ end }} # for the iptables wrapper - mountPath: /host name: host-slash @@ -1071,6 +1141,11 @@ spec: nodeSelector: beta.kubernetes.io/os: "linux" volumes: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - name: host-kubelet + hostPath: + path: /var/lib/kubelet +{{ end }} # for checking ovs-configuration service - name: systemd-units hostPath: diff --git a/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-node.yaml index 5049b59fe0..6bd0a64e8b 100644 --- a/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/self-hosted/single-zone-interconnect/ovnkube-node.yaml @@ -424,6 +424,15 @@ spec: multi_network_policy_enabled_flag="--enable-multi-networkpolicy" fi + NETWORK_NODE_IDENTITY_ENABLE= + if [[ "{{.NETWORK_NODE_IDENTITY_ENABLE}}" == "true" ]]; then + NETWORK_NODE_IDENTITY_ENABLE=" + --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig + --cert-dir=/etc/ovn/ovnkube-node-certs + --cert-duration={{.NodeIdentityCertDuration}} + " + fi + exec /usr/bin/ovnkube --init-node "${K8S_NODE}" \ --enable-interconnect \ --zone global \ @@ -454,6 +463,7 @@ spec: ${multi_network_enabled_flag} \ ${multi_network_policy_enabled_flag} \ ${gw_interface_flag} \ + ${NETWORK_NODE_IDENTITY_ENABLE} \ --enable-multi-external-gateway=true env: # for kubectl @@ -509,6 +519,11 @@ spec: privileged: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: + {{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /var/lib/kubelet + name: host-kubelet + readOnly: true + {{ end }} # for checking ovs-configuration service - mountPath: /etc/systemd/system name: systemd-units @@ -581,6 +596,31 @@ spec: - -c - | set -xe +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + cat << EOF > /var/run/ovnkube-kubeconfig + apiVersion: v1 + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: {{.K8S_APISERVER}} + name: default-cluster + contexts: + - context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context + current-context: default-context + kind: Config + preferences: {} + users: + - name: default-auth + user: + client-certificate: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + client-key: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem + EOF + export KUBECONFIG=/var/run/ovnkube-kubeconfig +{{ end }} touch /var/run/ovn/add_iptables.sh chmod 0755 /var/run/ovn/add_iptables.sh @@ -624,6 +664,10 @@ spec: securityContext: privileged: true volumeMounts: +{{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - mountPath: /etc/ovn/ + name: etc-openvswitch +{{ end }} # for the iptables wrapper - mountPath: /host name: host-slash @@ -644,6 +688,11 @@ spec: nodeSelector: beta.kubernetes.io/os: "linux" volumes: + {{ if .NETWORK_NODE_IDENTITY_ENABLE }} + - name: host-kubelet + hostPath: + path: /var/lib/kubelet + {{ end }} # for checking ovs-configuration service - name: systemd-units hostPath: diff --git a/pkg/bootstrap/types.go b/pkg/bootstrap/types.go index 9e6be67656..3d06459e13 100644 --- a/pkg/bootstrap/types.go +++ b/pkg/bootstrap/types.go @@ -108,6 +108,9 @@ type InfraStatus struct { // HostedControlPlane defines the hosted control plane, only used in HyperShift HostedControlPlane *hyperv1.HostedControlPlane + + // NetworkNodeIdentityEnabled define if the network node identity feature should be enabled + NetworkNodeIdentityEnabled bool } // APIServer is the hostname & port of a given APIServer. (This is the diff --git a/pkg/controller/configmap_ca_injector/controller.go b/pkg/controller/configmap_ca_injector/controller.go index 74cd29cf28..dc2e8b4370 100644 --- a/pkg/controller/configmap_ca_injector/controller.go +++ b/pkg/controller/configmap_ca_injector/controller.go @@ -128,7 +128,7 @@ func (r *ReconcileConfigMapInjector) Reconcile(ctx context.Context, request reco log.Println(err) return reconcile.Result{}, err } - _, trustedCAbundleData, err := validation.TrustBundleConfigMap(trustedCAbundleConfigMap) + _, trustedCAbundleData, err := validation.TrustBundleConfigMap(trustedCAbundleConfigMap, names.TRUSTED_CA_BUNDLE_CONFIGMAP_KEY) if err != nil { log.Println(err) diff --git a/pkg/controller/proxyconfig/validation.go b/pkg/controller/proxyconfig/validation.go index 45113bdea0..393b5badb5 100644 --- a/pkg/controller/proxyconfig/validation.go +++ b/pkg/controller/proxyconfig/validation.go @@ -5,13 +5,14 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "k8s.io/apimachinery/pkg/types" "net/http" "net/url" "os" "strings" "time" + "k8s.io/apimachinery/pkg/types" + configv1 "github.com/openshift/api/config/v1" "github.com/openshift/cluster-network-operator/pkg/names" "github.com/openshift/cluster-network-operator/pkg/util/validation" @@ -161,7 +162,7 @@ func (r *ReconcileProxyConfig) validateConfigMapRef(trustedCA string) (*corev1.C // of the key is one or more valid PEM encoded certificates, returning slices of // the validated certificates and certificate data. func (r *ReconcileProxyConfig) validateTrustBundle(cfgMap *corev1.ConfigMap) ([]*x509.Certificate, []byte, error) { - certBundle, bundleData, err := validation.TrustBundleConfigMap(cfgMap) + certBundle, bundleData, err := validation.TrustBundleConfigMap(cfgMap, names.TRUSTED_CA_BUNDLE_CONFIGMAP_KEY) if err != nil { return nil, nil, err } diff --git a/pkg/controller/statusmanager/status_manager.go b/pkg/controller/statusmanager/status_manager.go index 29abe4c781..cb58102f33 100644 --- a/pkg/controller/statusmanager/status_manager.go +++ b/pkg/controller/statusmanager/status_manager.go @@ -10,7 +10,6 @@ import ( "sync" "github.com/ghodss/yaml" - "k8s.io/klog/v2" configv1 "github.com/openshift/api/config/v1" operv1 "github.com/openshift/api/operator/v1" @@ -33,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" crclient "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/pkg/network/node_identity.go b/pkg/network/node_identity.go new file mode 100644 index 0000000000..732fb3a892 --- /dev/null +++ b/pkg/network/node_identity.go @@ -0,0 +1,209 @@ +package network + +import ( + "context" + "encoding/base64" + "fmt" + "net" + "os" + "path/filepath" + + operv1 "github.com/openshift/api/operator/v1" + "github.com/openshift/cluster-network-operator/pkg/bootstrap" + cnoclient "github.com/openshift/cluster-network-operator/pkg/client" + "github.com/openshift/cluster-network-operator/pkg/names" + "github.com/openshift/cluster-network-operator/pkg/platform" + "github.com/openshift/cluster-network-operator/pkg/render" + "github.com/openshift/cluster-network-operator/pkg/util/k8s" + "github.com/openshift/cluster-network-operator/pkg/util/validation" + hyperv1 "github.com/openshift/hypershift/api/v1beta1" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + utilnet "k8s.io/utils/net" +) + +const NetworkNodeIdentityWebhookPort = 9743 +const NetworkNodeIdentityNamespace = "openshift-network-node-identity" + +// isBootstrapComplete checks whether the bootstrap phase of openshift installation completed +func isBootstrapComplete(cli cnoclient.Client) (bool, error) { + clusterBootstrap := &corev1.ConfigMap{} + clusterBootstrapLookup := types.NamespacedName{Name: "bootstrap", Namespace: CLUSTER_CONFIG_NAMESPACE} + if err := cli.ClientFor("").CRClient().Get(context.TODO(), clusterBootstrapLookup, clusterBootstrap); err != nil { + if !apierrors.IsNotFound(err) { + return false, fmt.Errorf("unable to bootstrap OVN, unable to retrieve cluster config: %s", err) + } + } + status, ok := clusterBootstrap.Data["status"] + if ok { + return status == "complete", nil + } + klog.Warningf("no status found in bootstrap configmap") + return false, nil +} + +// renderNetworkNodeIdentity renders the network node identity component +func renderNetworkNodeIdentity(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.BootstrapResult, manifestDir string, client cnoclient.Client) ([]*uns.Unstructured, error) { + if !bootstrapResult.Infra.NetworkNodeIdentityEnabled { + klog.Infof("Network node identity is disabled") + return nil, nil + } + data := render.MakeRenderData() + data.Data["ReleaseVersion"] = os.Getenv("RELEASE_VERSION") + data.Data["OVNHybridOverlayEnable"] = false + if conf.DefaultNetwork.OVNKubernetesConfig != nil { + data.Data["OVNHybridOverlayEnable"] = conf.DefaultNetwork.OVNKubernetesConfig.HybridOverlayConfig != nil + } + data.Data["NetworkNodeIdentityPort"] = NetworkNodeIdentityWebhookPort + + manifestDirs := make([]string, 0, 2) + manifestDirs = append(manifestDirs, filepath.Join(manifestDir, "network/node-identity/common")) + + clusterBootstrapFinished := true + webhookCAConfigMap := &corev1.ConfigMap{} + webhookCAClient := client.Default() + webhookCALookup := types.NamespacedName{Name: "network-node-identity-ca", Namespace: NetworkNodeIdentityNamespace} + caKey := "ca-bundle.crt" + + webhookReady := false + // HyperShift specific + if hcpCfg := platform.NewHyperShiftConfig(); hcpCfg.Enabled { + webhookCAClient = client.ClientFor(names.ManagementClusterName) + webhookCALookup = types.NamespacedName{Name: "openshift-service-ca.crt", Namespace: hcpCfg.Namespace} + caKey = "service-ca.crt" + + data.Data["HostedClusterNamespace"] = hcpCfg.Namespace + data.Data["ManagementClusterName"] = names.ManagementClusterName + data.Data["NetworkNodeIdentityReplicas"] = 1 + if bootstrapResult.Infra.HostedControlPlane.Spec.ControllerAvailabilityPolicy == hyperv1.HighlyAvailable { + data.Data["NetworkNodeIdentityReplicas"] = 3 + } + data.Data["ReleaseImage"] = hcpCfg.ReleaseImage + data.Data["CLIImage"] = os.Getenv("CLI_IMAGE") + data.Data["TokenMinterImage"] = os.Getenv("TOKEN_MINTER_IMAGE") + data.Data["TokenAudience"] = os.Getenv("TOKEN_AUDIENCE") + data.Data["HCPNodeSelector"] = bootstrapResult.Infra.HostedControlPlane.Spec.NodeSelector + data.Data["NetworkNodeIdentityImage"] = hcpCfg.ControlPlaneImage // OVN_CONTROL_PLANE_IMAGE + localAPIServer := bootstrapResult.Infra.APIServers[bootstrap.APIServerDefaultLocal] + data.Data["K8S_LOCAL_APISERVER"] = "https://" + net.JoinHostPort(localAPIServer.Host, localAPIServer.Port) + + webhookDeployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, + } + nsn := types.NamespacedName{Namespace: hcpCfg.Namespace, Name: "network-node-identity"} + if err := client.Default().CRClient().Get(context.TODO(), nsn, webhookDeployment); err != nil { + if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to retrieve existing network-node-identity deployment: %w", err) + } else { + klog.Infof("network-node-identity deployment does not exist") + } + } else { + webhookReady = !deploymentProgressing(webhookDeployment) + } + + manifestDirs = append(manifestDirs, filepath.Join(manifestDir, "network/node-identity/managed")) + } else { + // self-hosted specific + data.Data["NetworkNodeIdentityImage"] = os.Getenv("OVN_IMAGE") + + // NetworkNodeIdentityTerminationDurationSeconds holds the allowed termination duration + // During node reboot, the webhook has to wait for the API server to terminate first to avoid disruptions + data.Data["NetworkNodeIdentityTerminationDurationSeconds"] = 200 + + apiServer := bootstrapResult.Infra.APIServers[bootstrap.APIServerDefault] + data.Data["K8S_APISERVER"] = "https://" + net.JoinHostPort(apiServer.Host, apiServer.Port) + + // NetworkNodeIdentityAddress is only used in self-hosted deployments where the webhook listens on loopback + // listening on localhost always picks the v4 address while dialing to localhost can choose either one + // https://github.com/golang/go/issues/9334 + // For that reason set the webhook address use the loopback address of the primary IP family + // Note: ServiceNetwork cannot be empty, so it is safe to use the first element + data.Data["NetworkNodeIdentityAddress"] = "127.0.0.1" + if utilnet.IsIPv6CIDRString(conf.ServiceNetwork[0]) { + data.Data["NetworkNodeIdentityAddress"] = "::1" + } + + var err error + clusterBootstrapFinished, err = isBootstrapComplete(client) + if err != nil { + return nil, err + } + + webhookDaemonSet := &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, + } + nsn := types.NamespacedName{Namespace: NetworkNodeIdentityNamespace, Name: "network-node-identity"} + if err := client.Default().CRClient().Get(context.TODO(), nsn, webhookDaemonSet); err != nil { + if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to retrieve existing network-node-identity daemonset: %w", err) + } else { + klog.Infof("network-node-identity daemonset does not exist") + } + } else { + webhookReady = !daemonSetProgressing(webhookDaemonSet, false) + } + + manifestDirs = append(manifestDirs, filepath.Join(manifestDir, "network/node-identity/self-hosted")) + } + + var webhookCA []byte + if err := webhookCAClient.CRClient().Get(context.TODO(), webhookCALookup, webhookCAConfigMap); err != nil { + // If the CA doesn't exist, the ValidatingWebhookConfiguration will not be rendered + if !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("unable to retrieve ovnkube-identity webhook CA config: %s", err) + } + } else { + _, webhookCA, err = validation.TrustBundleConfigMap(webhookCAConfigMap, caKey) + if err != nil { + return nil, err + } + } + data.Data["NetworkNodeIdentityCABundle"] = base64.URLEncoding.EncodeToString(webhookCA) + + manifests, err := render.RenderDirs(manifestDirs, &data) + if err != nil { + return nil, errors.Wrap(err, "failed to render network-node-identity manifests") + } + + applyWebhook := true + if !clusterBootstrapFinished { + applyWebhook = false + klog.Infof("network-node-identity webhook will not be applied, bootstrap is not complete") + } + if len(webhookCA) == 0 { + applyWebhook = false + klog.Infof("network-node-identity webhook will not be applied, CA bundle not found") + } + + // This is useful only when upgrading from a version that didn't enable the webhook + // because marking an existing webhook config with CreateWaitAnnotation won't remove it + if !webhookReady { + applyWebhook = false + klog.Infof("network-node-identity webhook will not be applied, the deployment/daemonset is not ready") + } + + if !applyWebhook { + klog.Infof("network-node-identity webhook will not be applied, if it already exists it won't be removed") + k8s.UpdateObjByGroupKindName(manifests, "admissionregistration.k8s.io", "ValidatingWebhookConfiguration", "", "network-node-identity.openshift.io", func(o *uns.Unstructured) { + anno := o.GetAnnotations() + if anno == nil { + anno = map[string]string{} + } + anno[names.CreateWaitAnnotation] = "true" + o.SetAnnotations(anno) + }) + } + return manifests, nil +} diff --git a/pkg/network/ovn_kubernetes.go b/pkg/network/ovn_kubernetes.go index dd513483d6..1a4b6b8bcd 100644 --- a/pkg/network/ovn_kubernetes.go +++ b/pkg/network/ovn_kubernetes.go @@ -68,6 +68,7 @@ const OVN_NODE_MODE_SMART_NIC = "smart-nic" const OVN_NODE_SELECTOR_DEFAULT_DPU_HOST = "network.operator.openshift.io/dpu-host" const OVN_NODE_SELECTOR_DEFAULT_DPU = "network.operator.openshift.io/dpu" const OVN_NODE_SELECTOR_DEFAULT_SMART_NIC = "network.operator.openshift.io/smart-nic" +const OVN_NODE_IDENTITY_CERT_DURATION = "24h" // gRPC healthcheck port. See: https://github.com/openshift/enhancements/pull/1209 const OVN_EGRESSIP_HEALTHCHECK_PORT = "9107" @@ -148,6 +149,8 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo data.Data["V4JoinSubnet"] = c.V4InternalSubnet data.Data["V6JoinSubnet"] = c.V6InternalSubnet data.Data["EnableUDPAggregation"] = !bootstrapResult.OVN.OVNKubernetesConfig.DisableUDPAggregation + data.Data["NETWORK_NODE_IDENTITY_ENABLE"] = bootstrapResult.Infra.NetworkNodeIdentityEnabled + data.Data["NodeIdentityCertDuration"] = OVN_NODE_IDENTITY_CERT_DURATION if conf.Migration != nil && conf.Migration.MTU != nil { if *conf.Migration.MTU.Network.From > *conf.Migration.MTU.Network.To { diff --git a/pkg/network/render.go b/pkg/network/render.go index 1d4de646b5..63214b052c 100644 --- a/pkg/network/render.go +++ b/pkg/network/render.go @@ -116,6 +116,13 @@ func Render(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.BootstrapResult } objs = append(objs, o...) + // render + o, err = renderNetworkNodeIdentity(conf, bootstrapResult, manifestDir, client) + if err != nil { + return nil, progressing, err + } + objs = append(objs, o...) + log.Printf("Render phase done, rendered %d objects", len(objs)) return objs, progressing, nil } diff --git a/pkg/platform/platform.go b/pkg/platform/platform.go index 425cbd9d92..80883b836b 100644 --- a/pkg/platform/platform.go +++ b/pkg/platform/platform.go @@ -14,6 +14,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" ) var cloudProviderConfig = types.NamespacedName{ @@ -21,6 +22,26 @@ var cloudProviderConfig = types.NamespacedName{ Name: "kube-cloud-config", } +// isNetworkNodeIdentityEnabled determines if network node identity should be enabled. +// It checks the `enabled` key in the network-node-identity/openshift-network-operator configmap. +// If the configmap doesn't exist, it returns true (the feature is enabled by default). +func isNetworkNodeIdentityEnabled(client cnoclient.Client) (bool, error) { + nodeIdentity := &corev1.ConfigMap{} + nodeIdentityLookup := types.NamespacedName{Name: "network-node-identity", Namespace: names.APPLIED_NAMESPACE} + if err := client.ClientFor("").CRClient().Get(context.TODO(), nodeIdentityLookup, nodeIdentity); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, fmt.Errorf("unable to bootstrap OVN, unable to retrieve cluster config: %s", err) + } + enabled, ok := nodeIdentity.Data["enabled"] + if ok { + return enabled == "true", nil + } + klog.Warningf("key `enabled` not found in the network-node-identity configmap, defaulting to enabled") + return true, nil +} + func InfraStatus(client cnoclient.Client) (*bootstrap.InfraStatus, error) { infraConfig := &configv1.Infrastructure{} if err := client.Default().CRClient().Get(context.TODO(), types.NamespacedName{Name: "cluster"}, infraConfig); err != nil { @@ -95,5 +116,12 @@ func InfraStatus(client cnoclient.Client) (*bootstrap.InfraStatus, error) { } res.HostedControlPlane = hcp } + + netIDEnabled, err := isNetworkNodeIdentityEnabled(client) + if err != nil { + return nil, fmt.Errorf("failed to determine if network node identity should be enabled: %w", err) + } + res.NetworkNodeIdentityEnabled = netIDEnabled + return res, nil } diff --git a/pkg/util/validation/trustbundle.go b/pkg/util/validation/trustbundle.go index cc1e60d7ed..1ca546f27e 100644 --- a/pkg/util/validation/trustbundle.go +++ b/pkg/util/validation/trustbundle.go @@ -16,14 +16,14 @@ const ( ) // TrustBundleConfigMap validates that ConfigMap contains a -// trust bundle named "ca-bundle.crt" and that "ca-bundle.crt" +// trust bundle named and it // contains one or more valid PEM encoded certificates, returning -// a byte slice of "ca-bundle.crt" contents upon success. -func TrustBundleConfigMap(cfgMap *corev1.ConfigMap) ([]*x509.Certificate, []byte, error) { - if _, ok := cfgMap.Data[names.TRUSTED_CA_BUNDLE_CONFIGMAP_KEY]; !ok { +// a byte slice of contents upon success. +func TrustBundleConfigMap(cfgMap *corev1.ConfigMap, caDataKey string) ([]*x509.Certificate, []byte, error) { + if _, ok := cfgMap.Data[caDataKey]; !ok { return nil, nil, fmt.Errorf("ConfigMap %q is missing %q", cfgMap.Name, names.TRUSTED_CA_BUNDLE_CONFIGMAP_KEY) } - trustBundleData := []byte(cfgMap.Data[names.TRUSTED_CA_BUNDLE_CONFIGMAP_KEY]) + trustBundleData := []byte(cfgMap.Data[caDataKey]) if len(trustBundleData) == 0 { return nil, nil, fmt.Errorf("data key %q is empty from ConfigMap %q", names.TRUSTED_CA_BUNDLE_CONFIGMAP_KEY, cfgMap.Name) }