feat: support cloud-node-manager on Windows clusters (#3044)

Azure · Apr 15, 2020 · b1612db · b1612db
1 parent 101555e
commit b1612db
Show file tree

Hide file tree

Showing 23 changed files with 1,296 additions and 154 deletions.
diff --git a/parts/k8s/addons/1.16/kubernetesmasteraddons-cloud-node-manager.yaml b/parts/k8s/addons/1.16/kubernetesmasteraddons-cloud-node-manager.yaml
@@ -19,7 +19,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["watch","list","get","update"]
+  verbs: ["watch","list","get","update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -55,6 +55,8 @@ spec:
     metadata:
       labels:
         k8s-app: cloud-node-manager
+      annotations:
+        cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
     spec:
       priorityClassName: system-node-critical
       serviceAccountName: cloud-node-manager
@@ -76,7 +78,14 @@ spec:
       - name: cloud-node-manager
         image: {{ContainerImage "cloud-node-manager"}}
         imagePullPolicy: IfNotPresent
-        command: ["cloud-node-manager"]
+        command:
+        - cloud-node-manager
+        - --node-name=$(NODE_NAME)
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         resources:
           requests:
             cpu: 50m

diff --git a/parts/k8s/addons/1.17/kubernetesmasteraddons-cloud-node-manager.yaml b/parts/k8s/addons/1.17/kubernetesmasteraddons-cloud-node-manager.yaml
@@ -19,7 +19,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["watch","list","get","update"]
+  verbs: ["watch","list","get","update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -78,7 +78,14 @@ spec:
       - name: cloud-node-manager
         image: {{ContainerImage "cloud-node-manager"}}
         imagePullPolicy: IfNotPresent
-        command: ["cloud-node-manager"]
+        command:
+        - cloud-node-manager
+        - --node-name=$(NODE_NAME)
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         resources:
           requests:
             cpu: 50m

diff --git a/parts/k8s/addons/1.18/kubernetesmasteraddons-cloud-node-manager.yaml b/parts/k8s/addons/1.18/kubernetesmasteraddons-cloud-node-manager.yaml
@@ -19,7 +19,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["watch","list","get","update"]
+  verbs: ["watch","list","get","update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -62,7 +62,7 @@ spec:
       serviceAccountName: cloud-node-manager
       hostNetwork: true {{/* required to fetch correct hostname */}}
       nodeSelector:
-        beta.kubernetes.io/os: linux
+        kubernetes.io/os: linux
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -78,11 +78,75 @@ spec:
       - name: cloud-node-manager
         image: {{ContainerImage "cloud-node-manager"}}
         imagePullPolicy: IfNotPresent
-        command: ["cloud-node-manager"]
+        command:
+        - cloud-node-manager
+        - --node-name=$(NODE_NAME)
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         resources:
           requests:
             cpu: 50m
             memory: 50Mi
           limits:
             cpu: 2000m
             memory: 512Mi
+{{- if HasWindows}}
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cloud-node-manager-windows
+  namespace: kube-system
+  labels:
+    component: cloud-node-manager
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cloud-node-manager-windows
+  template:
+    metadata:
+      labels:
+        k8s-app: cloud-node-manager-windows
+      annotations:
+        cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: cloud-node-manager
+      nodeSelector:
+        kubernetes.io/os: windows
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: "true"
+        effect: NoSchedule
+      - operator: "Exists"
+        effect: NoExecute
+      - operator: "Exists"
+        effect: NoSchedule
+      containers:
+      - name: cloud-node-manager
+        image: {{ContainerImage "cloud-node-manager"}}
+        imagePullPolicy: IfNotPresent
+        command:
+        - /cloud-node-manager.exe
+        - --node-name=$(NODE_NAME)
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+          limits:
+            cpu: 2000m
+            memory: 512Mi
+{{end}}
diff --git a/parts/k8s/addons/1.19/kubernetesmasteraddons-cloud-node-manager.yaml b/parts/k8s/addons/1.19/kubernetesmasteraddons-cloud-node-manager.yaml
@@ -19,7 +19,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["watch","list","get","update"]
+  verbs: ["watch","list","get","update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -62,7 +62,7 @@ spec:
       serviceAccountName: cloud-node-manager
       hostNetwork: true {{/* required to fetch correct hostname */}}
       nodeSelector:
-        beta.kubernetes.io/os: linux
+        kubernetes.io/os: linux
       tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
@@ -78,11 +78,75 @@ spec:
       - name: cloud-node-manager
         image: {{ContainerImage "cloud-node-manager"}}
         imagePullPolicy: IfNotPresent
-        command: ["cloud-node-manager"]
+        command:
+        - cloud-node-manager
+        - --node-name=$(NODE_NAME)
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         resources:
           requests:
             cpu: 50m
             memory: 50Mi
           limits:
             cpu: 2000m
             memory: 512Mi
+{{- if HasWindows}}
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cloud-node-manager-windows
+  namespace: kube-system
+  labels:
+    component: cloud-node-manager
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cloud-node-manager-windows
+  template:
+    metadata:
+      labels:
+        k8s-app: cloud-node-manager-windows
+      annotations:
+        cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: cloud-node-manager
+      nodeSelector:
+        kubernetes.io/os: windows
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: "true"
+        effect: NoSchedule
+      - operator: "Exists"
+        effect: NoExecute
+      - operator: "Exists"
+        effect: NoSchedule
+      containers:
+      - name: cloud-node-manager
+        image: {{ContainerImage "cloud-node-manager"}}
+        imagePullPolicy: IfNotPresent
+        command:
+        - /cloud-node-manager.exe
+        - --node-name=$(NODE_NAME)
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+          limits:
+            cpu: 2000m
+            memory: 512Mi
+{{end}}
diff --git a/pkg/api/addons.go b/pkg/api/addons.go
@@ -336,7 +336,7 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 
 	defaultCloudNodeManagerAddonsConfig := KubernetesAddon{
 		Name:    common.CloudNodeManagerAddonName,
-		Enabled: to.BoolPtr(common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.16.0") && to.Bool(o.KubernetesConfig.UseCloudControllerManager)),
+		Enabled: to.BoolPtr(cs.Properties.ShouldEnableAzureCloudAddon(common.CloudNodeManagerAddonName)),
 		Containers: []KubernetesContainerSpec{
 			{
 				Name:  common.CloudNodeManagerAddonName,
@@ -518,7 +518,7 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 
 	defaultAzureDiskCSIDriverAddonsConfig := KubernetesAddon{
 		Name:    common.AzureDiskCSIDriverAddonName,
-		Enabled: to.BoolPtr(DefaultAzureDiskCSIDriverAddonEnabled && to.Bool(o.KubernetesConfig.UseCloudControllerManager)),
+		Enabled: to.BoolPtr(DefaultAzureDiskCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureDiskCSIDriverAddonName)),
 		Containers: []KubernetesContainerSpec{
 			{
 				Name:           common.CSIProvisionerContainerName,
@@ -589,7 +589,7 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 
 	defaultAzureFileCSIDriverAddonsConfig := KubernetesAddon{
 		Name:    common.AzureFileCSIDriverAddonName,
-		Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && to.Bool(o.KubernetesConfig.UseCloudControllerManager)),
+		Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName)),
 		Containers: []KubernetesContainerSpec{
 			{
 				Name:           common.CSIProvisionerContainerName,
@@ -865,8 +865,10 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 	}
 
 	// Ensure cloud-node-manager and CSI components are enabled on appropriate upgrades
-	if isUpgrade && to.Bool(o.KubernetesConfig.UseCloudControllerManager) &&
-		common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.16.0") {
+	if isUpgrade &&
+		cs.Properties.ShouldEnableAzureCloudAddon(common.AzureDiskCSIDriverAddonName) &&
+		cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName) &&
+		cs.Properties.ShouldEnableAzureCloudAddon(common.CloudNodeManagerAddonName) {
 		componentry := map[string]KubernetesAddon{
 			common.AzureDiskCSIDriverAddonName: defaultAzureDiskCSIDriverAddonsConfig,
 			common.AzureFileCSIDriverAddonName: defaultAzureFileCSIDriverAddonsConfig,