Merge pull request #13203 from olemarkus/csi-featuregates

Fix CSI migration feature gates
kubernetes · Feb 8, 2022 · 1470528 · 1470528
2 parents 3d90a82 + 66e3202
commit 1470528
Show file tree

Hide file tree

Showing 57 changed files with 325 additions and 26 deletions.
diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml
@@ -32,6 +32,7 @@ contents: |
       - --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key
       - --etcd-servers-overrides=/events#https://127.0.0.1:4002
       - --etcd-servers=https://127.0.0.1:4001
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt
       - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key
       - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP

diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml
@@ -25,7 +25,7 @@ contents: |
       - --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key
       - --configure-cloud-routes=true
       - --enable-leader-migration=true
-      - --feature-gates=CSIMigrationAWS=true
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/
       - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --leader-elect=true

diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-scheduler.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-scheduler.yaml
@@ -16,6 +16,7 @@ contents: |
       - --authentication-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --config=/var/lib/kube-scheduler/config.yaml
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --leader-elect=true
       - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt
       - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key

diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml
@@ -29,6 +29,7 @@ contents: |
       - --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key
       - --etcd-servers-overrides=/events#https://127.0.0.1:4002
       - --etcd-servers=https://127.0.0.1:4001
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt
       - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key
       - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP

diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml
@@ -29,6 +29,7 @@ contents: |
       - --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key
       - --etcd-servers-overrides=/events#https://127.0.0.1:4002
       - --etcd-servers=https://127.0.0.1:4001
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt
       - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key
       - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP

diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml
@@ -21,7 +21,7 @@ contents: |
       - --cluster-signing-cert-file=/srv/kubernetes/kube-controller-manager/ca.crt
       - --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key
       - --configure-cloud-routes=true
-      - --feature-gates=CSIMigrationAWS=true
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/
       - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --leader-elect=true

diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml
@@ -21,7 +21,7 @@ contents: |
       - --cluster-signing-cert-file=/srv/kubernetes/kube-controller-manager/ca.crt
       - --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key
       - --configure-cloud-routes=true
-      - --feature-gates=CSIMigrationAWS=true
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/
       - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --leader-elect=true

diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-amd64.yaml
@@ -13,6 +13,7 @@ contents: |
       - --authentication-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --config=/var/lib/kube-scheduler/config.yaml
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --leader-elect=true
       - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt
       - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key

diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-arm64.yaml
@@ -13,6 +13,7 @@ contents: |
       - --authentication-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --config=/var/lib/kube-scheduler/config.yaml
+      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
       - --leader-elect=true
       - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt
       - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key

diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go
@@ -177,6 +177,27 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error {
 		}
 	}
 
+	if clusterSpec.CloudConfig != nil && clusterSpec.CloudConfig.AWSEBSCSIDriver != nil && fi.BoolValue(clusterSpec.CloudConfig.AWSEBSCSIDriver.Enabled) {
+
+		if c.FeatureGates == nil {
+			c.FeatureGates = make(map[string]string)
+		}
+
+		if b.IsKubernetesLT("1.21.0") {
+			if _, found := c.FeatureGates["CSIMigrationAWSComplete"]; !found {
+				c.FeatureGates["CSIMigrationAWSComplete"] = "true"
+			}
+		} else {
+			if _, found := c.FeatureGates["InTreePluginAWSUnregister"]; !found {
+				c.FeatureGates["InTreePluginAWSUnregister"] = "true"
+			}
+		}
+
+		if _, found := c.FeatureGates["CSIMigrationAWS"]; !found {
+			c.FeatureGates["CSIMigrationAWS"] = "true"
+		}
+	}
+
 	return nil
 }
 

diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go
@@ -180,12 +180,12 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 		}
 
 		if b.IsKubernetesLT("1.21.0") {
-			if _, found := clusterSpec.Kubelet.FeatureGates["CSIMigrationAWSComplete"]; !found {
-				clusterSpec.Kubelet.FeatureGates["CSIMigrationAWSComplete"] = "true"
+			if _, found := kcm.FeatureGates["CSIMigrationAWSComplete"]; !found {
+				kcm.FeatureGates["CSIMigrationAWSComplete"] = "true"
 			}
 		} else {
-			if _, found := clusterSpec.Kubelet.FeatureGates["InTreePluginAWSUnregister"]; !found {
-				clusterSpec.Kubelet.FeatureGates["InTreePluginAWSUnregister"] = "true"
+			if _, found := kcm.FeatureGates["InTreePluginAWSUnregister"]; !found {
+				kcm.FeatureGates["InTreePluginAWSUnregister"] = "true"
 			}
 		}
 

diff --git a/pkg/model/components/kubescheduler.go b/pkg/model/components/kubescheduler.go
@@ -57,5 +57,25 @@ func (b *KubeSchedulerOptionsBuilder) BuildOptions(o interface{}) error {
 		}
 	}
 
+	if clusterSpec.CloudConfig != nil && clusterSpec.CloudConfig.AWSEBSCSIDriver != nil && fi.BoolValue(clusterSpec.CloudConfig.AWSEBSCSIDriver.Enabled) {
+
+		if config.FeatureGates == nil {
+			config.FeatureGates = make(map[string]string)
+		}
+
+		if b.IsKubernetesLT("1.21.0") {
+			if _, found := config.FeatureGates["CSIMigrationAWSComplete"]; !found {
+				config.FeatureGates["CSIMigrationAWSComplete"] = "true"
+			}
+		} else {
+			if _, found := config.FeatureGates["InTreePluginAWSUnregister"]; !found {
+				config.FeatureGates["InTreePluginAWSUnregister"] = "true"
+			}
+		}
+
+		if _, found := config.FeatureGates["CSIMigrationAWS"]; !found {
+			config.FeatureGates["CSIMigrationAWS"] = "true"
+		}
+	}
 	return nil
 }
diff --git a/tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml
@@ -166,7 +166,7 @@ Resources.AWSEC2LaunchTemplateapiserverapiserversminimalexamplecom.Properties.La
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: apiserver
   InstanceGroupRole: APIServer
-  NodeupConfigHash: AQHPJZNIlLmvJbcdG7J6j41vrzOUaaUonWietUjHJ2c=
+  NodeupConfigHash: lJvIfsBLG+O4Nj5rxkSE6Cre23QYwFVKyErAGDOps3U=
 
   __EOF_KUBE_ENV
 
@@ -338,6 +338,9 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP
@@ -366,6 +369,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
     configureCloudRoutes: false
     featureGates:
       CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-controller-manager:v1.22.0
     leaderElection:
       leaderElect: true
@@ -377,6 +381,9 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
     image: k8s.gcr.io/kube-proxy:v1.22.0
     logLevel: 2
   kubeScheduler:
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-scheduler:v1.22.0
     leaderElection:
       leaderElect: true
@@ -426,7 +433,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: master-us-test-1a
   InstanceGroupRole: Master
-  NodeupConfigHash: SfFU9PJN/YBuvc0tAV+EMD2AfiA8Pk9/m7marXkRIjM=
+  NodeupConfigHash: qHq7EcCRxYCbh/iRszfG6rVcwqWk7x6lI73/xCRJw/A=
 
   __EOF_KUBE_ENV
 

diff --git a/...piservernodes/data/aws_launch_template_apiserver.apiservers.minimal.example.com_user_data b/...piservernodes/data/aws_launch_template_apiserver.apiservers.minimal.example.com_user_data
@@ -165,7 +165,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: apiserver
 InstanceGroupRole: APIServer
-NodeupConfigHash: AQHPJZNIlLmvJbcdG7J6j41vrzOUaaUonWietUjHJ2c=
+NodeupConfigHash: lJvIfsBLG+O4Nj5rxkSE6Cre23QYwFVKyErAGDOps3U=
 
 __EOF_KUBE_ENV
 

diff --git a/...vernodes/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/...vernodes/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
@@ -163,6 +163,9 @@ kubeAPIServer:
   - https://127.0.0.1:4001
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-apiserver:v1.22.0
   kubeletPreferredAddressTypes:
   - InternalIP
@@ -191,6 +194,7 @@ kubeControllerManager:
   configureCloudRoutes: false
   featureGates:
     CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-controller-manager:v1.22.0
   leaderElection:
     leaderElect: true
@@ -202,6 +206,9 @@ kubeProxy:
   image: k8s.gcr.io/kube-proxy:v1.22.0
   logLevel: 2
 kubeScheduler:
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-scheduler:v1.22.0
   leaderElection:
     leaderElect: true
@@ -251,7 +258,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: SfFU9PJN/YBuvc0tAV+EMD2AfiA8Pk9/m7marXkRIjM=
+NodeupConfigHash: qHq7EcCRxYCbh/iRszfG6rVcwqWk7x6lI73/xCRJw/A=
 
 __EOF_KUBE_ENV
 

diff --git a/...on/update_cluster/apiservernodes/data/aws_s3_bucket_object_cluster-completed.spec_content b/...on/update_cluster/apiservernodes/data/aws_s3_bucket_object_cluster-completed.spec_content
@@ -68,6 +68,9 @@ spec:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP
@@ -96,6 +99,7 @@ spec:
     configureCloudRoutes: false
     featureGates:
       CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-controller-manager:v1.22.0
     leaderElection:
       leaderElect: true
@@ -121,6 +125,9 @@ spec:
     image: k8s.gcr.io/kube-proxy:v1.22.0
     logLevel: 2
   kubeScheduler:
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-scheduler:v1.22.0
     leaderElection:
       leaderElect: true

diff --git a/...on/update_cluster/apiservernodes/data/aws_s3_bucket_object_nodeupconfig-apiserver_content b/...on/update_cluster/apiservernodes/data/aws_s3_bucket_object_nodeupconfig-apiserver_content
@@ -22,6 +22,9 @@ APIServerConfig:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP

diff --git a/...e_cluster/apiservernodes/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content b/...e_cluster/apiservernodes/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content
@@ -22,6 +22,9 @@ APIServerConfig:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP

diff --git a/...ccm-irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/...ccm-irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
@@ -163,6 +163,9 @@ kubeAPIServer:
   - https://127.0.0.1:4001
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-apiserver:v1.21.0
   kubeletPreferredAddressTypes:
   - InternalIP
@@ -191,6 +194,7 @@ kubeControllerManager:
   configureCloudRoutes: false
   featureGates:
     CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-controller-manager:v1.21.0
   leaderElection:
     leaderElect: true
@@ -201,6 +205,9 @@ kubeProxy:
   image: k8s.gcr.io/kube-proxy:v1.21.0
   logLevel: 2
 kubeScheduler:
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-scheduler:v1.21.0
   leaderElection:
     leaderElect: true
@@ -248,7 +255,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: KRPfC7gdEszAwdvxrT35RmCa0PKOWQEH3lSEv+scbR0=
+NodeupConfigHash: Iy6mY3H+yC+PFFDzNn21qRYJYT5inh86/86q3DCAJ8g=
 
 __EOF_KUBE_ENV
 

diff --git a/...ate_cluster/many-addons-ccm-irsa/data/aws_s3_bucket_object_cluster-completed.spec_content b/...ate_cluster/many-addons-ccm-irsa/data/aws_s3_bucket_object_cluster-completed.spec_content
@@ -94,6 +94,9 @@ spec:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.21.0
     kubeletPreferredAddressTypes:
     - InternalIP
@@ -122,6 +125,7 @@ spec:
     configureCloudRoutes: false
     featureGates:
       CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-controller-manager:v1.21.0
     leaderElection:
       leaderElect: true
@@ -146,6 +150,9 @@ spec:
     image: k8s.gcr.io/kube-proxy:v1.21.0
     logLevel: 2
   kubeScheduler:
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-scheduler:v1.21.0
     leaderElection:
       leaderElect: true

diff --git a/...ter/many-addons-ccm-irsa/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content b/...ter/many-addons-ccm-irsa/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content
@@ -22,6 +22,9 @@ APIServerConfig:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     image: k8s.gcr.io/kube-apiserver:v1.21.0
     kubeletPreferredAddressTypes:
     - InternalIP

diff --git a/...m-irsa23/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/...m-irsa23/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
@@ -163,6 +163,9 @@ kubeAPIServer:
   - https://127.0.0.1:4001
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-apiserver:v1.23.0
   kubeletPreferredAddressTypes:
   - InternalIP
@@ -191,6 +194,7 @@ kubeControllerManager:
   configureCloudRoutes: false
   featureGates:
     CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-controller-manager:v1.23.0
   leaderElection:
     leaderElect: true
@@ -201,6 +205,9 @@ kubeProxy:
   image: k8s.gcr.io/kube-proxy:v1.23.0
   logLevel: 2
 kubeScheduler:
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   image: k8s.gcr.io/kube-scheduler:v1.23.0
   leaderElection:
     leaderElect: true
@@ -250,7 +257,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: kCFd6y0ozkd4yZPLo0twTr4CSfC70PJCxZzGVdM97vM=
+NodeupConfigHash: iFwdYeYTe1hhOB084iFcT9VKpMkh0G+QjLzd6vpFhnc=
 
 __EOF_KUBE_ENV