Merge pull request #12676 from johngmyers/leader-migration

Migrate to AWS CCM in k8s 1.24
kubernetes · Nov 6, 2021 · e230cc9 · e230cc9
2 parents f49b99f + 3d5d5b3
commit e230cc9
Show file tree

Hide file tree

Showing 12 changed files with 79 additions and 2 deletions.
diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -522,6 +522,9 @@ spec:
                     description: ConfigureCloudRoutes enables CIDRs allocated with
                       to be configured on the cloud provider.
                     type: boolean
+                  enableLeaderMigration:
+                    description: EnableLeaderMigration enables controller leader migration.
+                    type: boolean
                   image:
                     description: Image is the OCI image of the cloud controller manager.
                     type: string
@@ -1876,6 +1879,9 @@ spec:
                       sync loop in the attach-detach controller. This can cause volumes
                       to become mismatched with pods
                     type: boolean
+                  enableLeaderMigration:
+                    description: EnableLeaderMigration enables controller leader migration.
+                    type: boolean
                   enableProfiling:
                     description: EnableProfiling enables profiling via web interface
                       host:port/debug/pprof/

diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml
@@ -23,6 +23,7 @@ contents: |
       - --cluster-signing-cert-file=/srv/kubernetes/kube-controller-manager/ca.crt
       - --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key
       - --configure-cloud-routes=true
+      - --enable-leader-migration=true
       - --feature-gates=CSIMigrationAWS=true
       - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/
       - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig

diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go
@@ -650,6 +650,8 @@ type KubeControllerManagerConfig struct {
 
 	// EnableProfiling enables profiling via web interface host:port/debug/pprof/
 	EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // CloudControllerManagerConfig is the configuration of the cloud controller
@@ -677,6 +679,8 @@ type CloudControllerManagerConfig struct {
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
 	// UseServiceAccountCredentials controls whether we use individual service account credentials for each controller.
 	UseServiceAccountCredentials *bool `json:"useServiceAccountCredentials,omitempty" flag:"use-service-account-credentials"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // KubeSchedulerConfig is the configuration for the kube-scheduler

diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go
@@ -649,6 +649,8 @@ type KubeControllerManagerConfig struct {
 
 	// EnableProfiling enables profiling via web interface host:port/debug/pprof/
 	EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // CloudControllerManagerConfig is the configuration of the cloud controller
@@ -676,6 +678,8 @@ type CloudControllerManagerConfig struct {
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
 	// UseServiceAccountCredentials controls whether we use individual service account credentials for each controller.
 	UseServiceAccountCredentials *bool `json:"useServiceAccountCredentials,omitempty" flag:"use-service-account-credentials"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // KubeSchedulerConfig is the configuration for the kube-scheduler

diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
diff --git a/pkg/apis/kops/v1alpha3/componentconfig.go b/pkg/apis/kops/v1alpha3/componentconfig.go
@@ -647,6 +647,8 @@ type KubeControllerManagerConfig struct {
 
 	// EnableProfiling enables profiling via web interface host:port/debug/pprof/
 	EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // CloudControllerManagerConfig is the configuration of the cloud controller
@@ -674,6 +676,8 @@ type CloudControllerManagerConfig struct {
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
 	// UseServiceAccountCredentials controls whether we use individual service account credentials for each controller.
 	UseServiceAccountCredentials *bool `json:"useServiceAccountCredentials,omitempty" flag:"use-service-account-credentials"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // KubeSchedulerConfig is the configuration for the kube-scheduler

diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go
diff --git a/pkg/model/components/awscloudcontrollermanager.go b/pkg/model/components/awscloudcontrollermanager.go
@@ -38,6 +38,14 @@ func (b *AWSCloudControllerManagerOptionsBuilder) BuildOptions(o interface{}) er
 
 	eccm := clusterSpec.ExternalCloudControllerManager
 
+	if kops.CloudProviderID(clusterSpec.CloudProvider) != kops.CloudProviderAWS {
+		return nil
+	}
+
+	if eccm == nil && b.IsKubernetesGTE("1.24") {
+		eccm = &kops.CloudControllerManagerConfig{}
+	}
+
 	if eccm == nil || kops.CloudProviderID(clusterSpec.CloudProvider) != kops.CloudProviderAWS {
 		return nil
 	}
@@ -91,5 +99,9 @@ func (b *AWSCloudControllerManagerOptionsBuilder) BuildOptions(o interface{}) er
 		}
 	}
 
+	if b.IsKubernetesGTE("1.24") && b.IsKubernetesLT("1.25") {
+		eccm.EnableLeaderMigration = fi.Bool(true)
+	}
+
 	return nil
 }
diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go
@@ -79,7 +79,11 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 	kcm.ClusterName = b.ClusterName
 	switch kops.CloudProviderID(clusterSpec.CloudProvider) {
 	case kops.CloudProviderAWS:
-		kcm.CloudProvider = "aws"
+		if b.IsKubernetesGTE("1.24") {
+			kcm.CloudProvider = "external"
+		} else {
+			kcm.CloudProvider = "aws"
+		}
 
 	case kops.CloudProviderGCE:
 		kcm.CloudProvider = "gce"
@@ -101,7 +105,11 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 		return fmt.Errorf("unknown cloudprovider %q", clusterSpec.CloudProvider)
 	}
 
-	if clusterSpec.ExternalCloudControllerManager != nil {
+	if clusterSpec.ExternalCloudControllerManager == nil {
+		if kcm.CloudProvider == "aws" && b.IsKubernetesGTE("1.23") {
+			kcm.EnableLeaderMigration = fi.Bool(true)
+		}
+	} else {
 		kcm.CloudProvider = "external"
 	}