Merge pull request #13204 from olemarkus/automated-cherry-pick-of-#13…

…203-origin-release-1.23 Automated cherry pick of #13203: Fix CSI migration feature gates
kubernetes · Feb 8, 2022 · 2d3ef3f · 2d3ef3f
2 parents 0ff5b9a + 6fee599
commit 2d3ef3f
Show file tree

Hide file tree

Showing 45 changed files with 283 additions and 22 deletions.
diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml
@@ -32,6 +32,7 @@ contents: |
       - --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key
       - --etcd-servers-overrides=/events#https://127.0.0.1:4002
       - --etcd-servers=https://127.0.0.1:4001
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --insecure-port=0
       - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt
       - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key

diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml
@@ -25,7 +25,7 @@ contents: |
       - --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key
       - --configure-cloud-routes=true
       - --enable-leader-migration=true
-      - --feature-gates=CSIMigrationAWS=true
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/
       - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --leader-elect=true

diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-scheduler.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-scheduler.yaml
@@ -16,6 +16,7 @@ contents: |
       - --authentication-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --config=/var/lib/kube-scheduler/config.yaml
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --leader-elect=true
       - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt
       - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key

diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go
@@ -175,6 +175,27 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error {
 	c.InsecureBindAddress = ""
 	c.InsecurePort = 0
 
+	if clusterSpec.CloudConfig != nil && clusterSpec.CloudConfig.AWSEBSCSIDriver != nil && fi.BoolValue(clusterSpec.CloudConfig.AWSEBSCSIDriver.Enabled) {
+
+		if c.FeatureGates == nil {
+			c.FeatureGates = make(map[string]string)
+		}
+
+		if b.IsKubernetesLT("1.21.0") {
+			if _, found := c.FeatureGates["CSIMigrationAWSComplete"]; !found {
+				c.FeatureGates["CSIMigrationAWSComplete"] = "true"
+			}
+		} else {
+			if _, found := c.FeatureGates["InTreePluginAWSUnregister"]; !found {
+				c.FeatureGates["InTreePluginAWSUnregister"] = "true"
+			}
+		}
+
+		if _, found := c.FeatureGates["CSIMigrationAWS"]; !found {
+			c.FeatureGates["CSIMigrationAWS"] = "true"
+		}
+	}
+
 	return nil
 }
 

diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go
@@ -181,12 +181,12 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 		}
 
 		if b.IsKubernetesLT("1.21.0") {
-			if _, found := clusterSpec.Kubelet.FeatureGates["CSIMigrationAWSComplete"]; !found {
-				clusterSpec.Kubelet.FeatureGates["CSIMigrationAWSComplete"] = "true"
+			if _, found := kcm.FeatureGates["CSIMigrationAWSComplete"]; !found {
+				kcm.FeatureGates["CSIMigrationAWSComplete"] = "true"
 			}
 		} else {
-			if _, found := clusterSpec.Kubelet.FeatureGates["InTreePluginAWSUnregister"]; !found {
-				clusterSpec.Kubelet.FeatureGates["InTreePluginAWSUnregister"] = "true"
+			if _, found := kcm.FeatureGates["InTreePluginAWSUnregister"]; !found {
+				kcm.FeatureGates["InTreePluginAWSUnregister"] = "true"
 			}
 		}
 

diff --git a/pkg/model/components/kubescheduler.go b/pkg/model/components/kubescheduler.go
@@ -57,5 +57,25 @@ func (b *KubeSchedulerOptionsBuilder) BuildOptions(o interface{}) error {
 		}
 	}
 
+	if clusterSpec.CloudConfig != nil && clusterSpec.CloudConfig.AWSEBSCSIDriver != nil && fi.BoolValue(clusterSpec.CloudConfig.AWSEBSCSIDriver.Enabled) {
+
+		if config.FeatureGates == nil {
+			config.FeatureGates = make(map[string]string)
+		}
+
+		if b.IsKubernetesLT("1.21.0") {
+			if _, found := config.FeatureGates["CSIMigrationAWSComplete"]; !found {
+				config.FeatureGates["CSIMigrationAWSComplete"] = "true"
+			}
+		} else {
+			if _, found := config.FeatureGates["InTreePluginAWSUnregister"]; !found {
+				config.FeatureGates["InTreePluginAWSUnregister"] = "true"
+			}
+		}
+
+		if _, found := config.FeatureGates["CSIMigrationAWS"]; !found {
+			config.FeatureGates["CSIMigrationAWS"] = "true"
+		}
+	}
 	return nil
 }
diff --git a/tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml
@@ -167,7 +167,7 @@ Resources.AWSEC2LaunchTemplateapiserverapiserversminimalexamplecom.Properties.La
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: apiserver
   InstanceGroupRole: APIServer
-  NodeupConfigHash: gAJg1M6Qa3y03kN1JD1QD4dwAJR/0jD/oD/Ba0znnso=
+  NodeupConfigHash: PtMYVrjpz/mtwT1TnwIy1pIUlM41zm0y8bQBiW2G8OY=
 
   __EOF_KUBE_ENV
 
@@ -339,6 +339,9 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP
@@ -367,6 +370,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
     configureCloudRoutes: false
     featureGates:
       CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-controller-manager:v1.22.0
     leaderElection:
       leaderElect: true
@@ -378,6 +382,9 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
     image: k8s.gcr.io/kube-proxy:v1.22.0
     logLevel: 2
   kubeScheduler:
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-scheduler:v1.22.0
     leaderElection:
       leaderElect: true
@@ -429,7 +436,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: master-us-test-1a
   InstanceGroupRole: Master
-  NodeupConfigHash: 2f0ZcjFD5Qagywj47m1BGIDkAa3gs8mszC9sXB8Yf/Y=
+  NodeupConfigHash: gmPPQHqhI+jyfmryK+736uIBPaksib1CSDiX3LZgxn4=
 
   __EOF_KUBE_ENV
 

diff --git a/...piservernodes/data/aws_launch_template_apiserver.apiservers.minimal.example.com_user_data b/...piservernodes/data/aws_launch_template_apiserver.apiservers.minimal.example.com_user_data
@@ -166,7 +166,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: apiserver
 InstanceGroupRole: APIServer
-NodeupConfigHash: gAJg1M6Qa3y03kN1JD1QD4dwAJR/0jD/oD/Ba0znnso=
+NodeupConfigHash: PtMYVrjpz/mtwT1TnwIy1pIUlM41zm0y8bQBiW2G8OY=
 
 __EOF_KUBE_ENV
 

diff --git a/...vernodes/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/...vernodes/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
@@ -163,6 +163,9 @@ kubeAPIServer:
   - https://127.0.0.1:4001
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-apiserver:v1.22.0
   kubeletPreferredAddressTypes:
   - InternalIP
@@ -191,6 +194,7 @@ kubeControllerManager:
   configureCloudRoutes: false
   featureGates:
     CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-controller-manager:v1.22.0
   leaderElection:
     leaderElect: true
@@ -202,6 +206,9 @@ kubeProxy:
   image: k8s.gcr.io/kube-proxy:v1.22.0
   logLevel: 2
 kubeScheduler:
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-scheduler:v1.22.0
   leaderElection:
     leaderElect: true
@@ -253,7 +260,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: 2f0ZcjFD5Qagywj47m1BGIDkAa3gs8mszC9sXB8Yf/Y=
+NodeupConfigHash: gmPPQHqhI+jyfmryK+736uIBPaksib1CSDiX3LZgxn4=
 
 __EOF_KUBE_ENV
 

diff --git a/...on/update_cluster/apiservernodes/data/aws_s3_bucket_object_cluster-completed.spec_content b/...on/update_cluster/apiservernodes/data/aws_s3_bucket_object_cluster-completed.spec_content
@@ -68,6 +68,9 @@ spec:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP
@@ -96,6 +99,7 @@ spec:
     configureCloudRoutes: false
     featureGates:
       CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-controller-manager:v1.22.0
     leaderElection:
       leaderElect: true
@@ -121,6 +125,9 @@ spec:
     image: k8s.gcr.io/kube-proxy:v1.22.0
     logLevel: 2
   kubeScheduler:
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-scheduler:v1.22.0
     leaderElection:
       leaderElect: true

diff --git a/...on/update_cluster/apiservernodes/data/aws_s3_bucket_object_nodeupconfig-apiserver_content b/...on/update_cluster/apiservernodes/data/aws_s3_bucket_object_nodeupconfig-apiserver_content
@@ -22,6 +22,9 @@ APIServerConfig:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP

diff --git a/...e_cluster/apiservernodes/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content b/...e_cluster/apiservernodes/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content
@@ -22,6 +22,9 @@ APIServerConfig:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP

diff --git a/...ccm-irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/...ccm-irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
@@ -163,6 +163,9 @@ kubeAPIServer:
   - https://127.0.0.1:4001
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-apiserver:v1.21.0
   kubeletPreferredAddressTypes:
   - InternalIP
@@ -191,6 +194,7 @@ kubeControllerManager:
   configureCloudRoutes: false
   featureGates:
     CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-controller-manager:v1.21.0
   leaderElection:
     leaderElect: true
@@ -201,6 +205,9 @@ kubeProxy:
   image: k8s.gcr.io/kube-proxy:v1.21.0
   logLevel: 2
 kubeScheduler:
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-scheduler:v1.21.0
   leaderElection:
     leaderElect: true
@@ -250,7 +257,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: k2ipJcklA/jkSst9ttsWY48Ivs6MZywueD36MEj0/Dg=
+NodeupConfigHash: DgGmCIGWpoEpkcZx7R00D1/X8ktPb238l9BStImYk7Q=
 
 __EOF_KUBE_ENV
 

diff --git a/...ate_cluster/many-addons-ccm-irsa/data/aws_s3_bucket_object_cluster-completed.spec_content b/...ate_cluster/many-addons-ccm-irsa/data/aws_s3_bucket_object_cluster-completed.spec_content
@@ -92,6 +92,9 @@ spec:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.21.0
     kubeletPreferredAddressTypes:
     - InternalIP
@@ -120,6 +123,7 @@ spec:
     configureCloudRoutes: false
     featureGates:
       CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-controller-manager:v1.21.0
     leaderElection:
       leaderElect: true
@@ -144,6 +148,9 @@ spec:
     image: k8s.gcr.io/kube-proxy:v1.21.0
     logLevel: 2
   kubeScheduler:
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-scheduler:v1.21.0
     leaderElection:
       leaderElect: true

diff --git a/...ter/many-addons-ccm-irsa/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content b/...ter/many-addons-ccm-irsa/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content
@@ -22,6 +22,9 @@ APIServerConfig:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.21.0
     kubeletPreferredAddressTypes:
     - InternalIP

diff --git a/...dons-ccm/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/...dons-ccm/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
@@ -163,6 +163,9 @@ kubeAPIServer:
   - https://127.0.0.1:4001
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-apiserver:v1.21.0
   kubeletPreferredAddressTypes:
   - InternalIP
@@ -191,6 +194,7 @@ kubeControllerManager:
   configureCloudRoutes: false
   featureGates:
     CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-controller-manager:v1.21.0
   leaderElection:
     leaderElect: true
@@ -201,6 +205,9 @@ kubeProxy:
   image: k8s.gcr.io/kube-proxy:v1.21.0
   logLevel: 2
 kubeScheduler:
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-scheduler:v1.21.0
   leaderElection:
     leaderElect: true
@@ -250,7 +257,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: ocC8yqA7amgLRil7MVhsRN9iBEwo0N+bGN/yq3A/8kM=
+NodeupConfigHash: Ok74bQdj+4AILB4r24EfQuF+q5XUqkDLKldgJ0rHaqU=
 
 __EOF_KUBE_ENV
 

diff --git a/...n/update_cluster/many-addons-ccm/data/aws_s3_bucket_object_cluster-completed.spec_content b/...n/update_cluster/many-addons-ccm/data/aws_s3_bucket_object_cluster-completed.spec_content
@@ -91,6 +91,9 @@ spec:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.21.0
     kubeletPreferredAddressTypes:
     - InternalIP
@@ -119,6 +122,7 @@ spec:
     configureCloudRoutes: false
     featureGates:
       CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-controller-manager:v1.21.0
     leaderElection:
       leaderElect: true
@@ -143,6 +147,9 @@ spec:
     image: k8s.gcr.io/kube-proxy:v1.21.0
     logLevel: 2
   kubeScheduler:
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-scheduler:v1.21.0
     leaderElection:
       leaderElect: true

diff --git a/..._cluster/many-addons-ccm/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content b/..._cluster/many-addons-ccm/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content
@@ -22,6 +22,9 @@ APIServerConfig:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.21.0
     kubeletPreferredAddressTypes:
     - InternalIP