Add control-plane node role annotation to cp nodes

Update docs/releases/1.20-NOTES.md Co-authored-by: John Gardiner Myers <jgmyers@proofpoint.com>
kubernetes · Dec 19, 2020 · 8525e3f · 8525e3f
1 parent 409d007
commit 8525e3f
Show file tree

Hide file tree

Showing 114 changed files with 1,444 additions and 812 deletions.
diff --git a/docs/releases/1.20-NOTES.md b/docs/releases/1.20-NOTES.md
@@ -8,6 +8,8 @@
 
 * kOps now use helm3 functions for merging template `--set` and `--values` arguments. This has slightly different behaviour than previous helm2-like logic.
 
+* Following kubeadm, control plane nodes are now labelled with `node-role.kubernetes.io/control-plane=""`
+
 # Breaking changes
 
 * Support for Kubernetes 1.11 and 1.12 has been removed.
@@ -20,6 +22,8 @@
 
 * If you are running `kops toolbox template` in an airgapped environment, you have to set `--channel` to point to a local channel file.
 
+* If your workload targets control plane nodes, you need to change them to select the `node-role.kubernetes.io/control-plane=""` label. You should also add the `node-role.kubernetes.io/control-plane:NoSchedule` toleration to these workloads. This taint will not be added to control plane nodes before kOps 1.21.
+
 # Deprecations
 
 * Support for Kubernetes versions 1.13 and 1.14 are deprecated and will be removed in kOps 1.21.
@@ -28,5 +32,6 @@
 
 * The [manifest based cluster autoscaler addon](https://github.com/kubernetes/kops/tree/master/addons/cluster-autoscaler) has been deprecated in favour of a configurable addon.
 
-# Full change list since 1.19.0 release
+* The `node-role.kubernetes.io/master` and `kubernetes.io/role` labels are deprecated and will be removed from control plane nodes in kOps 1.21
 
+# Full change list since 1.19.0 release
diff --git a/...servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml b/...servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml
@@ -66,6 +66,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master
@@ -144,6 +145,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master
@@ -222,6 +224,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master

diff --git a/.../servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml b/.../servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml
@@ -67,6 +67,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-a
@@ -133,6 +134,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-b
@@ -199,6 +201,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-c

diff --git a/...l/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml b/...l/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml
@@ -74,6 +74,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-a
@@ -152,6 +153,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-b
@@ -230,6 +232,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-c

diff --git a/...ackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml b/...ackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml
@@ -26,6 +26,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-a
@@ -98,6 +99,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-b
@@ -170,6 +172,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-c

diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml
@@ -85,6 +85,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master

diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml
@@ -111,6 +111,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master

diff --git a/...model/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml b/...model/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml
@@ -18,6 +18,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master

diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml
@@ -38,6 +38,7 @@ Metadata:
   ig_generation: "0"
   k8s: cluster
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
+  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master

diff --git a/pkg/model/tests/data/bootstrapscript_0.txt b/pkg/model/tests/data/bootstrapscript_0.txt
@@ -202,6 +202,7 @@ KubeletConfig:
     kubernetes.io/role: master
     label2: value2
     labelname: labelvalue
+    node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
   taints:
   - key1=value1:NoSchedule

diff --git a/pkg/model/tests/data/bootstrapscript_1.txt b/pkg/model/tests/data/bootstrapscript_1.txt
@@ -219,6 +219,7 @@ KubeletConfig:
     kubernetes.io/role: master
     label2: value2
     labelname: labelvalue
+    node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
   taints:
   - key1=value1:NoSchedule

diff --git a/pkg/model/tests/data/bootstrapscript_2.txt b/pkg/model/tests/data/bootstrapscript_2.txt
@@ -219,6 +219,7 @@ KubeletConfig:
     kubernetes.io/role: master
     label2: value2
     labelname: labelvalue
+    node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
   taints:
   - key1=value1:NoSchedule

diff --git a/pkg/nodelabels/builder.go b/pkg/nodelabels/builder.go
@@ -29,16 +29,18 @@ const (
 
 	RoleLabelMaster16 = "node-role.kubernetes.io/master"
 	RoleLabelNode16   = "node-role.kubernetes.io/node"
+
+	RoleLabelControlPlane20 = "node-role.kubernetes.io/control-plane"
 )
 
 // BuildNodeLabels returns the node labels for the specified instance group
 // This moved from the kubelet to a central controller in kubernetes 1.16
 func BuildNodeLabels(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) map[string]string {
-	isMaster := instanceGroup.Spec.Role == kops.InstanceGroupRoleMaster
+	isControlPlane := instanceGroup.Spec.Role == kops.InstanceGroupRoleMaster
 
 	// Merge KubeletConfig for NodeLabels
 	c := &kops.KubeletConfigSpec{}
-	if isMaster {
+	if isControlPlane {
 		reflectutils.JSONMergeStruct(c, cluster.Spec.MasterKubelet)
 	} else {
 		reflectutils.JSONMergeStruct(c, cluster.Spec.Kubelet)
@@ -50,11 +52,12 @@ func BuildNodeLabels(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) m
 
 	nodeLabels := c.NodeLabels
 
-	if isMaster {
+	if isControlPlane {
 		if nodeLabels == nil {
 			nodeLabels = make(map[string]string)
 		}
 		nodeLabels[RoleLabelMaster16] = ""
+		nodeLabels[RoleLabelControlPlane20] = ""
 		nodeLabels[RoleLabelName15] = RoleMasterLabelValue15
 	} else {
 		if nodeLabels == nil {

diff --git a/pkg/nodelabels/builder_test.go b/pkg/nodelabels/builder_test.go
@@ -31,14 +31,14 @@ func TestBuildNodeLabels(t *testing.T) {
 		expected map[string]string
 	}{
 		{
-			name: "RoleMaster",
+			name: "RoleControlPlane",
 			cluster: &kops.Cluster{
 				Spec: kops.ClusterSpec{
 					KubernetesVersion: "v1.9.0",
 					MasterKubelet: &kops.KubeletConfigSpec{
 						NodeLabels: map[string]string{
-							"master1": "master1",
-							"master2": "master2",
+							"controlPlane1": "controlPlane1",
+							"controlPlane2": "controlPlane2",
 						},
 					},
 					Kubelet: &kops.KubeletConfigSpec{
@@ -61,12 +61,13 @@ func TestBuildNodeLabels(t *testing.T) {
 				},
 			},
 			expected: map[string]string{
-				RoleLabelMaster16: "",
-				RoleLabelName15:   RoleMasterLabelValue15,
-				"master1":         "master1",
-				"master2":         "master2",
-				"node1":           "override1",
-				"node3":           "override3",
+				RoleLabelMaster16:       "",
+				RoleLabelControlPlane20: "",
+				RoleLabelName15:         RoleMasterLabelValue15,
+				"controlPlane1":         "controlPlane1",
+				"controlPlane2":         "controlPlane2",
+				"node1":                 "override1",
+				"node3":                 "override3",
 			},
 		},
 		{
@@ -76,8 +77,8 @@ func TestBuildNodeLabels(t *testing.T) {
 					KubernetesVersion: "v1.9.0",
 					MasterKubelet: &kops.KubeletConfigSpec{
 						NodeLabels: map[string]string{
-							"master1": "master1",
-							"master2": "master2",
+							"controlPlane1": "controlPlane1",
+							"controlPlane2": "controlPlane2",
 						},
 					},
 					Kubelet: &kops.KubeletConfigSpec{

diff --git a/.../data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data b/.../data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data
@@ -292,6 +292,7 @@ KubeletConfig:
   networkPluginName: cni
   nodeLabels:
     kubernetes.io/role: master
+    node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
   nonMasqueradeCIDR: 100.64.0.0/10
   podInfraContainerImage: k8s.gcr.io/pause:3.2

diff --git a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf
@@ -185,6 +185,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-bastionuserdata-exam
     propagate_at_launch = true
     value               = "master"
   }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"
     propagate_at_launch = true
@@ -514,35 +519,38 @@ resource "aws_launch_template" "master-us-test-1a-masters-bastionuserdata-exampl
   tag_specifications {
     resource_type = "instance"
     tags = {
-      "KubernetesCluster"                                                            = "bastionuserdata.example.com"
-      "Name"                                                                         = "master-us-test-1a.masters.bastionuserdata.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"             = "master"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = ""
-      "k8s.io/role/master"                                                           = "1"
-      "kops.k8s.io/instancegroup"                                                    = "master-us-test-1a"
-      "kubernetes.io/cluster/bastionuserdata.example.com"                            = "owned"
+      "KubernetesCluster"                                                                   = "bastionuserdata.example.com"
+      "Name"                                                                                = "master-us-test-1a.masters.bastionuserdata.example.com"
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                    = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"        = ""
+      "k8s.io/role/master"                                                                  = "1"
+      "kops.k8s.io/instancegroup"                                                           = "master-us-test-1a"
+      "kubernetes.io/cluster/bastionuserdata.example.com"                                   = "owned"
     }
   }
   tag_specifications {
     resource_type = "volume"
     tags = {
-      "KubernetesCluster"                                                            = "bastionuserdata.example.com"
-      "Name"                                                                         = "master-us-test-1a.masters.bastionuserdata.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"             = "master"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = ""
-      "k8s.io/role/master"                                                           = "1"
-      "kops.k8s.io/instancegroup"                                                    = "master-us-test-1a"
-      "kubernetes.io/cluster/bastionuserdata.example.com"                            = "owned"
+      "KubernetesCluster"                                                                   = "bastionuserdata.example.com"
+      "Name"                                                                                = "master-us-test-1a.masters.bastionuserdata.example.com"
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                    = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"        = ""
+      "k8s.io/role/master"                                                                  = "1"
+      "kops.k8s.io/instancegroup"                                                           = "master-us-test-1a"
+      "kubernetes.io/cluster/bastionuserdata.example.com"                                   = "owned"
     }
   }
   tags = {
-    "KubernetesCluster"                                                            = "bastionuserdata.example.com"
-    "Name"                                                                         = "master-us-test-1a.masters.bastionuserdata.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"             = "master"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = ""
-    "k8s.io/role/master"                                                           = "1"
-    "kops.k8s.io/instancegroup"                                                    = "master-us-test-1a"
-    "kubernetes.io/cluster/bastionuserdata.example.com"                            = "owned"
+    "KubernetesCluster"                                                                   = "bastionuserdata.example.com"
+    "Name"                                                                                = "master-us-test-1a.masters.bastionuserdata.example.com"
+    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                    = "master"
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = ""
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"        = ""
+    "k8s.io/role/master"                                                                  = "1"
+    "kops.k8s.io/instancegroup"                                                           = "master-us-test-1a"
+    "kubernetes.io/cluster/bastionuserdata.example.com"                                   = "owned"
   }
   user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data")
 }

diff --git a/tests/integration/update_cluster/complex/cloudformation.json b/tests/integration/update_cluster/complex/cloudformation.json
@@ -48,6 +48,11 @@
             "Value": "master",
             "PropagateAtLaunch": true
           },
+          {
+            "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane",
+            "Value": "",
+            "PropagateAtLaunch": true
+          },
           {
             "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master",
             "Value": "",
@@ -308,6 +313,10 @@
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
                   "Value": "master"
                 },
+                {
+                  "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane",
+                  "Value": ""
+                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master",
                   "Value": ""
@@ -349,6 +358,10 @@
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
                   "Value": "master"
                 },
+                {
+                  "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane",
+                  "Value": ""
+                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master",
                   "Value": ""

diff --git a/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml
@@ -311,6 +311,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscomplexexamplecom.Properties.
     networkPluginName: kubenet
     nodeLabels:
       kubernetes.io/role: master
+      node-role.kubernetes.io/control-plane: ""
       node-role.kubernetes.io/master: ""
     nonMasqueradeCIDR: 100.64.0.0/10
     podInfraContainerImage: k8s.gcr.io/pause:3.2

diff --git a/.../complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data b/.../complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data
@@ -310,6 +310,7 @@ KubeletConfig:
   networkPluginName: kubenet
   nodeLabels:
     kubernetes.io/role: master
+    node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
   nonMasqueradeCIDR: 100.64.0.0/10
   podInfraContainerImage: k8s.gcr.io/pause:3.2