From a433c807cfe2575e4d191c9d0eb30110519347be Mon Sep 17 00:00:00 2001 From: Thomas Ricci Date: Tue, 11 Jul 2023 13:51:46 -0700 Subject: [PATCH 1/5] adding cni and cns deamonset specs to target managed DNC scenario (Open AI) --- cns/mdnc_azure-cns.yaml | 192 ++++++++++++++++++ .../deployment/mdnc_azure-cni-manager.yaml | 57 ++++++ 2 files changed, 249 insertions(+) create mode 100644 cns/mdnc_azure-cns.yaml create mode 100644 tools/acncli/deployment/mdnc_azure-cni-manager.yaml diff --git a/cns/mdnc_azure-cns.yaml b/cns/mdnc_azure-cns.yaml new file mode 100644 index 0000000000..1f45157633 --- /dev/null +++ b/cns/mdnc_azure-cns.yaml @@ -0,0 +1,192 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-cns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: kube-system + name: nodeNetConfigEditor +rules: +- apiGroups: ["acn.azure.com"] + resources: ["nodenetworkconfigs"] + verbs: ["get", "list", "watch", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pod-reader-all-namespaces +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: nodeNetConfigEditorRoleBinding + namespace: kube-system +subjects: +- kind: ServiceAccount + name: azure-cns + namespace: kube-system +roleRef: + kind: Role + name: nodeNetConfigEditor + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pod-reader-all-namespaces-binding +subjects: +- kind: ServiceAccount + name: azure-cns + namespace: kube-system +roleRef: + kind: ClusterRole + name: pod-reader-all-namespaces + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-cns + namespace: kube-system + labels: + app: azure-cns +spec: + selector: + matchLabels: + k8s-app: azure-cns + template: + metadata: + labels: + k8s-app: azure-cns + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: beta.kubernetes.io/os + operator: In + values: + - linux + priorityClassName: system-node-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cns-container + image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.32 + imagePullPolicy: IfNotPresent + args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] + volumeMounts: + - name: log + mountPath: /var/log + - name: cns-state + mountPath: /var/lib/azure-network + - name: azure-endpoints + mountPath: /var/run/azure-cns/ + - name: cns-config + mountPath: /etc/azure-cns + - name: cni-bin + mountPath: /opt/cni/bin + - name: azure-vnet + mountPath: /var/run/azure-vnet + - name: legacy-cni-state + mountPath: /var/run/azure-vnet.json + ports: + - containerPort: 10090 + env: + - name: CNSIpAddress + value: "127.0.0.1" + - name: CNSPort + value: "10090" + - name: CNSLogTarget + value: "stdoutfile" + - name: CNS_CONFIGURATION_PATH + value: /etc/azure-cns/cns_config.json + - name: NODENAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + hostNetwork: true + volumes: + - name: azure-endpoints + hostPath: + path: /var/run/azure-cns/ + type: DirectoryOrCreate + - name: log + hostPath: + path: /var/log + type: Directory + - name: cns-state + hostPath: + path: /var/lib/azure-network + type: DirectoryOrCreate + - name: cni-bin + hostPath: + path: /opt/cni/bin + type: Directory + - name: azure-vnet + hostPath: + path: /var/run/azure-vnet + type: DirectoryOrCreate + - name: legacy-cni-state + hostPath: + path: /var/run/azure-vnet.json + type: FileOrCreate + - name: cns-config + configMap: + name: cns-config + serviceAccountName: azure-cns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cns-config + namespace: kube-system +data: + cns_config.json: | + { + "TelemetrySettings": { + "TelemetryBatchSizeBytes": 16384, + "TelemetryBatchIntervalInSecs": 15, + "RefreshIntervalInSecs": 15, + "DisableAll": false, + "HeartBeatIntervalInMins": 30, + "DebugMode": false, + "SnapshotIntervalInMins": 60 + }, + "ManagedSettings": { + "PrivateEndpoint": "", + "InfrastructureNetworkID": "", + "NodeID": "", + "NodeSyncIntervalInSeconds": 30 + }, + "ChannelMode": "CRD", + "InitializeFromCNI": true, + "ManageEndpointState": false, + "ProgramSNATIPTables" : false + } +# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. diff --git a/tools/acncli/deployment/mdnc_azure-cni-manager.yaml b/tools/acncli/deployment/mdnc_azure-cni-manager.yaml new file mode 100644 index 0000000000..bd00b34910 --- /dev/null +++ b/tools/acncli/deployment/mdnc_azure-cni-manager.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-cni-manager + namespace: kube-system +spec: + selector: + matchLabels: + acn: azure-cni-manager + template: + metadata: + labels: + acn: azure-cni-manager + spec: + nodeSelector: + "beta.kubernetes.io/os": linux + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + hostNetwork: true + containers: + - name: azure-cni-installer + image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.4.35 + imagePullPolicy: Always + env: + - name: AZURE_CNI_OS + value: linux + - name: AZURE_CNI_TENANCY + value: singletenancy + - name: AZURE_CNI_MODE + value: transparent + - name: AZURE_CNI_IPAM + value: azure-cns + - name: AZURE_CNI_EXEMPT + value: azure-vnet-telemetry,azure-vnet-telemetry.config + volumeMounts: + - name: cni-bin + mountPath: /opt/cni/bin + - name: cni-conflist + mountPath: /etc/cni/net.d + - name: cni-datapath-log + mountPath: /var/log/azure-vnet.log + restartPolicy: Always + volumes: + - name: cni-bin + hostPath: + path: /opt/cni/bin + type: Directory + - name: cni-datapath-log + hostPath: + path: /var/log/azure-vnet.log + type: File + - name: cni-conflist + hostPath: + path: /etc/cni/net.d + type: Directory From cce5afff1cafe98ff0a894afaa9996a4e4e933fe Mon Sep 17 00:00:00 2001 From: Thomas Ricci Date: Wed, 12 Jul 2023 14:28:21 -0700 Subject: [PATCH 2/5] initializing CNI v1.4.35 via dropgz --- cns/mdnc_azure-cns.yaml | 11 ++++ .../deployment/mdnc_azure-cni-manager.yaml | 57 ------------------- 2 files changed, 11 insertions(+), 57 deletions(-) delete mode 100644 tools/acncli/deployment/mdnc_azure-cni-manager.yaml diff --git a/cns/mdnc_azure-cns.yaml b/cns/mdnc_azure-cns.yaml index 1f45157633..a3c522dce8 100644 --- a/cns/mdnc_azure-cns.yaml +++ b/cns/mdnc_azure-cns.yaml @@ -94,6 +94,17 @@ spec: effect: NoExecute - operator: "Exists" effect: NoSchedule + initContainers: + - name: init-cni-dropgz + image: "acnpublic.azurecr.io/containernetworking/mdnc-cni-dropgz" + imagePullPolicy: IfNotPresent + command: ["/dropgz"] + args: ["deploy" , "azure-vnet", "-o", "/opt/cni/bin/azure-vnet", "azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-swift-overlay.conflist", "-o", "/etc/cni/net.d/15-azure-swift-overlay.conflist"] + volumeMounts: + - name: cni-bin + mountPath: /opt/cni/bin + - name: cni-conflist + mountPath: /etc/cni/net.d containers: - name: cns-container image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.32 diff --git a/tools/acncli/deployment/mdnc_azure-cni-manager.yaml b/tools/acncli/deployment/mdnc_azure-cni-manager.yaml deleted file mode 100644 index bd00b34910..0000000000 --- a/tools/acncli/deployment/mdnc_azure-cni-manager.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: azure-cni-manager - namespace: kube-system -spec: - selector: - matchLabels: - acn: azure-cni-manager - template: - metadata: - labels: - acn: azure-cni-manager - spec: - nodeSelector: - "beta.kubernetes.io/os": linux - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - hostNetwork: true - containers: - - name: azure-cni-installer - image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.4.35 - imagePullPolicy: Always - env: - - name: AZURE_CNI_OS - value: linux - - name: AZURE_CNI_TENANCY - value: singletenancy - - name: AZURE_CNI_MODE - value: transparent - - name: AZURE_CNI_IPAM - value: azure-cns - - name: AZURE_CNI_EXEMPT - value: azure-vnet-telemetry,azure-vnet-telemetry.config - volumeMounts: - - name: cni-bin - mountPath: /opt/cni/bin - - name: cni-conflist - mountPath: /etc/cni/net.d - - name: cni-datapath-log - mountPath: /var/log/azure-vnet.log - restartPolicy: Always - volumes: - - name: cni-bin - hostPath: - path: /opt/cni/bin - type: Directory - - name: cni-datapath-log - hostPath: - path: /var/log/azure-vnet.log - type: File - - name: cni-conflist - hostPath: - path: /etc/cni/net.d - type: Directory From 829a518f7114d4d25a30b3db8b30f6a29088efff Mon Sep 17 00:00:00 2001 From: Thomas Ricci Date: Wed, 12 Jul 2023 15:22:26 -0700 Subject: [PATCH 3/5] adding volume for cni init --- cns/mdnc_azure-cns.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cns/mdnc_azure-cns.yaml b/cns/mdnc_azure-cns.yaml index a3c522dce8..dc784f086e 100644 --- a/cns/mdnc_azure-cns.yaml +++ b/cns/mdnc_azure-cns.yaml @@ -167,6 +167,10 @@ spec: hostPath: path: /var/run/azure-vnet.json type: FileOrCreate + - name: cni-conflist + hostPath: + path: /etc/cni/net.d + type: Directory - name: cns-config configMap: name: cns-config From 1ad4203d2792544dd7ceff53571372594a8b0a57 Mon Sep 17 00:00:00 2001 From: Thomas Ricci Date: Thu, 13 Jul 2023 14:23:22 -0700 Subject: [PATCH 4/5] point to azure-swift.conflist --- cns/mdnc_azure-cns.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/mdnc_azure-cns.yaml b/cns/mdnc_azure-cns.yaml index dc784f086e..b8bff26e47 100644 --- a/cns/mdnc_azure-cns.yaml +++ b/cns/mdnc_azure-cns.yaml @@ -99,7 +99,7 @@ spec: image: "acnpublic.azurecr.io/containernetworking/mdnc-cni-dropgz" imagePullPolicy: IfNotPresent command: ["/dropgz"] - args: ["deploy" , "azure-vnet", "-o", "/opt/cni/bin/azure-vnet", "azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-swift-overlay.conflist", "-o", "/etc/cni/net.d/15-azure-swift-overlay.conflist"] + args: ["deploy" , "azure-vnet", "-o", "/opt/cni/bin/azure-vnet", "azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-swift.conflist", "-o", "/etc/cni/net.d/10-azure.conflist"] volumeMounts: - name: cni-bin mountPath: /opt/cni/bin From 82f6458cbcece57ee20df53a64204c5f9c73a9dd Mon Sep 17 00:00:00 2001 From: Thomas Ricci Date: Fri, 14 Jul 2023 09:23:15 -0700 Subject: [PATCH 5/5] PR feedback --- cns/mdnc_azure-cns.yaml => .pipelines/mdnc/azure-cns-cni.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cns/mdnc_azure-cns.yaml => .pipelines/mdnc/azure-cns-cni.yaml (100%) diff --git a/cns/mdnc_azure-cns.yaml b/.pipelines/mdnc/azure-cns-cni.yaml similarity index 100% rename from cns/mdnc_azure-cns.yaml rename to .pipelines/mdnc/azure-cns-cni.yaml