AWS CSI driver

k8s/crds/kops.k8s.io_clusters.yaml

@@ -214,6 +214,14 @@ spec:
               cloudConfig:
                 description: CloudConfiguration defines the cloud provider configuration
                 properties:
+                  awsEBSCSIDriver:
+                    description: AWSEBSCSIDriver is the config for the AWS EBS CSI
+                      driver
+                    properties:
+                      enabled:
+                        description: Enabled enables the AWS EBS CSI driver
+                        type: boolean
+                    type: object
                   azure:
                     description: Azure cloud-config options
                     properties:

pkg/apis/kops/componentconfig.go

@@ -811,6 +811,14 @@ type CloudConfiguration struct {
 	Openstack *OpenstackConfiguration `json:"openstack,omitempty"`
 	// Azure cloud-config options
 	Azure *AzureConfiguration `json:"azure,omitempty"`
+	// AWSEBSCSIDriver is the config for the AWS EBS CSI driver
+	AWSEBSCSIDriver *AWSEBSCSIDriver `json:"awsEBSCSIDriver,omitempty"`
+}
+
+// AWSEBSCSIDriver is the config for the AWS EBS CSI driver
+type AWSEBSCSIDriver struct {
+	//Enabled enables the AWS EBS CSI driver
+	Enabled *bool `json:"enabled,omitempty"`
 }
 
 // NodeTerminationHandlerConfig determines the node termination handler configuration.

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -810,6 +810,14 @@ type CloudConfiguration struct {
 	Openstack *OpenstackConfiguration `json:"openstack,omitempty"`
 	// Azure cloud-config options
 	Azure *AzureConfiguration `json:"azure,omitempty"`
+	// AWSEBSCSIDriver is the config for the AWS EBS CSI driver
+	AWSEBSCSIDriver *AWSEBSCSIDriver `json:"awsEBSCSIDriver,omitempty"`
+}
+
+// AWSEBSCSIDriver is the config for the AWS EBS CSI driver
+type AWSEBSCSIDriver struct {
+	//Enabled enables the AWS EBS CSI driver
+	Enabled *bool `json:"enabled,omitempty"`
 }
 
 // NodeTerminationHandlerConfig determines the node termination handler configuration.

pkg/apis/kops/validation/aws.go

@@ -38,9 +38,25 @@ func awsValidateCluster(c *kops.Cluster) field.ErrorList {
 		}
 	}
 
+	allErrs = append(allErrs, awsValidateExternalCloudControllerManager(c.Spec)...)
+
 	return allErrs
 }
 
+func awsValidateExternalCloudControllerManager(c kops.ClusterSpec) (allErrs field.ErrorList) {
+
+	if c.ExternalCloudControllerManager != nil {
+		if c.KubeControllerManager == nil || c.KubeControllerManager.ExternalCloudVolumePlugin != "aws" {
+			if c.CloudConfig == nil || c.CloudConfig.AWSEBSCSIDriver == nil || !fi.BoolValue(c.CloudConfig.AWSEBSCSIDriver.Enabled) {
+				allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "externalCloudControllerManager"),
+					"AWS external CCM cannot be used without enabling spec.cloudConfig.AWSEBSCSIDriver or setting spec.kubeControllerManaager.externalCloudVolumePlugin set to `aws`"))
+			}
+		}
+	}
+	return allErrs
+
+}
+
 func awsValidateInstanceGroup(ig *kops.InstanceGroup, cloud awsup.AWSCloud) field.ErrorList {
 	allErrs := field.ErrorList{}
 

pkg/apis/kops/validation/aws_test.go

@@ -26,6 +26,43 @@ import (
 	"k8s.io/kops/pkg/apis/kops"
 )
 
+func TestAWSValidateExternalCloudConfig(t *testing.T) {
+	grid := []struct {
+		Input          kops.ClusterSpec
+		ExpectedErrors []string
+	}{
+		{
+			Input: kops.ClusterSpec{
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
+			},
+			ExpectedErrors: []string{"Forbidden::spec.externalCloudControllerManager"},
+		},
+		{
+			Input: kops.ClusterSpec{
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
+				CloudConfig: &kops.CloudConfiguration{
+					AWSEBSCSIDriver: &kops.AWSEBSCSIDriver{
+						Enabled: fi.Bool(true),
+					},
+				},
+			},
+		},
+		{
+			Input: kops.ClusterSpec{
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
+				KubeControllerManager: &kops.KubeControllerManagerConfig{
+					ExternalCloudVolumePlugin: "aws",
+				},
+			},
+		},
+	}
+	for _, g := range grid {
+		errs := awsValidateExternalCloudControllerManager(g.Input)
+
+		testErrors(t, g.Input, errs, g.ExpectedErrors)
+	}
+}
+
 func TestValidateInstanceGroupSpec(t *testing.T) {
 	grid := []struct {
 		Input          kops.InstanceGroupSpec

pkg/model/components/kubecontrollermanager.go

@@ -101,14 +101,6 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 
 	if clusterSpec.ExternalCloudControllerManager != nil {
 		kcm.CloudProvider = "external"
-
-		// External cloud provider disables KCM volume controllers, so
-		// most users would want to either install CSI or pass
-		// --external-cloud-volume-plugin to the KCM, which runs the
-		// KCM volume controllers.
-		if kcm.ExternalCloudVolumePlugin == "" {
-			klog.Infof("An external cloud controller manager is configured, but ExternalCloudVolumePlugin is not configured for the KCM.  This means a CSI plugin must be installed by the user or else volume management might not work.")
-		}
 	}
 
 	kcm.LogLevel = 2
@@ -158,5 +150,15 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 		kcm.Controllers = []string{"*", "tokencleaner"}
 	}
 
+	if clusterSpec.CloudConfig != nil && clusterSpec.CloudConfig.AWSEBSCSIDriver != nil && fi.BoolValue(clusterSpec.CloudConfig.AWSEBSCSIDriver.Enabled) {
+
+		if kcm.FeatureGates == nil {
+			kcm.FeatureGates = make(map[string]string)
+		}
+		if _, found := kcm.FeatureGates["CSIMigrationAWSComplete"]; !found {
+			kcm.FeatureGates["CSIMigrationAWSComplete"] = "true"
+		}
+	}
+
 	return nil
 }

pkg/model/components/kubelet.go

@@ -213,5 +213,16 @@ func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error {
 		}
 	}
 
+	if clusterSpec.CloudConfig != nil && clusterSpec.CloudConfig.AWSEBSCSIDriver != nil && fi.BoolValue(clusterSpec.CloudConfig.AWSEBSCSIDriver.Enabled) {
+		if _, found := clusterSpec.Kubelet.FeatureGates["CSIMigrationAWSComplete"]; !found {
+			clusterSpec.Kubelet.FeatureGates["CSIMigrationAWSComplete"] = "true"
+		}
+	}
+	if _, found := clusterSpec.Kubelet.FeatureGates["ExperimentalCriticalPodAnnotation"]; !found {
+		if b.IsKubernetesLT("1.16") {
+			clusterSpec.Kubelet.FeatureGates["ExperimentalCriticalPodAnnotation"] = "true"
+		}
+	}
+
 	return nil
 }