AWS CSI driver

kubernetes · Dec 21, 2020 · fe7b5cd · fe7b5cd
1 parent cbb418c
commit fe7b5cd
Show file tree

Hide file tree

Showing 45 changed files with 1,613 additions and 9 deletions.
diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go
@@ -384,6 +384,16 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr
 	cluster := clusterResult.Cluster
 	instanceGroups := clusterResult.InstanceGroups
 
+	//Temporarily to run the e2e with ebs csi driver
+	if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderAWS {
+		cluster.Spec.CloudConfig = &api.CloudConfiguration{
+			AWSEBSCSIDriver: &api.AWSEBSCSIDriver{
+
+				Enabled: fi.Bool(true),
+			},
+		}
+	}
+
 	var masters []*api.InstanceGroup
 	var nodes []*api.InstanceGroup
 	for _, ig := range instanceGroups {

diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -214,6 +214,14 @@ spec:
               cloudConfig:
                 description: CloudConfiguration defines the cloud provider configuration
                 properties:
+                  awsEBSCSIDriver:
+                    description: AWSEBSCSIDriver is the config for the AWS EBS CSI
+                      driver
+                    properties:
+                      enabled:
+                        description: Enabled enables the AWS EBS CSI driver
+                        type: boolean
+                    type: object
                   disableSecurityGroupIngress:
                     description: AWS cloud-config options
                     type: boolean
@@ -328,6 +336,8 @@ spec:
                     description: VSphereUsername is deprecated and will be removed
                       in a later version
                     type: string
+                required:
+                - awsEBSCSIDriver
                 type: object
               cloudControllerManager:
                 description: CloudControllerManagerConfig is the configuration of

diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go
@@ -785,6 +785,14 @@ type CloudConfiguration struct {
 	SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
 	// Openstack cloud-config options
 	Openstack *OpenstackConfiguration `json:"openstack,omitempty"`
+	// AWSEBSCSIDriver is the config for the AWS EBS CSI driver
+	AWSEBSCSIDriver *AWSEBSCSIDriver `json:"awsEBSCSIDriver"`
+}
+
+// AWSEBSCSIDriver is the config for the AWS EBS CSI driver
+type AWSEBSCSIDriver struct {
+	//Enabled enables the AWS EBS CSI driver
+	Enabled *bool `json:"enabled,omitempty"`
 }
 
 // NodeTerminationHandlerConfig determines the node termination handler configuration.

diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go
@@ -784,6 +784,14 @@ type CloudConfiguration struct {
 	SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
 	// Openstack cloud-config options
 	Openstack *OpenstackConfiguration `json:"openstack,omitempty"`
+	// AWSEBSCSIDriver is the config for the AWS EBS CSI driver
+	AWSEBSCSIDriver *AWSEBSCSIDriver `json:"awsEBSCSIDriver"`
+}
+
+// AWSEBSCSIDriver is the config for the AWS EBS CSI driver
+type AWSEBSCSIDriver struct {
+	//Enabled enables the AWS EBS CSI driver
+	Enabled *bool `json:"enabled,omitempty"`
 }
 
 // NodeTerminationHandlerConfig determines the node termination handler configuration.

diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
diff --git a/pkg/apis/kops/validation/aws.go b/pkg/apis/kops/validation/aws.go
@@ -38,9 +38,25 @@ func awsValidateCluster(c *kops.Cluster) field.ErrorList {
 		}
 	}
 
+	allErrs = append(allErrs, awsValidateExternalCloudControllerManager(c.Spec)...)
+
 	return allErrs
 }
 
+func awsValidateExternalCloudControllerManager(c kops.ClusterSpec) (allErrs field.ErrorList) {
+
+	if c.ExternalCloudControllerManager != nil {
+		if c.KubeControllerManager == nil || c.KubeControllerManager.ExternalCloudVolumePlugin != "aws" {
+			if c.CloudConfig == nil || c.CloudConfig.AWSEBSCSIDriver == nil || !fi.BoolValue(c.CloudConfig.AWSEBSCSIDriver.Enabled) {
+				allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "externalCloudControllerManager"),
+					"AWS external CCM cannot be used without enabling spec.cloudConfig.AWSEBSCSIDriver or setting spec.kubeControllerManaager.externalCloudVolumePlugin set to `aws`"))
+			}
+		}
+	}
+	return allErrs
+
+}
+
 func awsValidateInstanceGroup(ig *kops.InstanceGroup, cloud awsup.AWSCloud) field.ErrorList {
 	allErrs := field.ErrorList{}
 

diff --git a/pkg/apis/kops/validation/aws_test.go b/pkg/apis/kops/validation/aws_test.go
@@ -26,6 +26,43 @@ import (
 	"k8s.io/kops/pkg/apis/kops"
 )
 
+func TestAWSValidateExternalCloudConfig(t *testing.T) {
+	grid := []struct {
+		Input          kops.ClusterSpec
+		ExpectedErrors []string
+	}{
+		{
+			Input: kops.ClusterSpec{
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
+			},
+			ExpectedErrors: []string{"Forbidden::spec.externalCloudControllerManager"},
+		},
+		{
+			Input: kops.ClusterSpec{
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
+				CloudConfig: &kops.CloudConfiguration{
+					AWSEBSCSIDriver: &kops.AWSEBSCSIDriver{
+						Enabled: fi.Bool(true),
+					},
+				},
+			},
+		},
+		{
+			Input: kops.ClusterSpec{
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
+				KubeControllerManager: &kops.KubeControllerManagerConfig{
+					ExternalCloudVolumePlugin: "aws",
+				},
+			},
+		},
+	}
+	for _, g := range grid {
+		errs := awsValidateExternalCloudControllerManager(g.Input)
+
+		testErrors(t, g.Input, errs, g.ExpectedErrors)
+	}
+}
+
 func TestValidateInstanceGroupSpec(t *testing.T) {
 	grid := []struct {
 		Input          kops.InstanceGroupSpec

diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go
diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go
@@ -98,14 +98,6 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 
 	if clusterSpec.ExternalCloudControllerManager != nil {
 		kcm.CloudProvider = "external"
-
-		// External cloud provider disables KCM volume controllers, so
-		// most users would want to either install CSI or pass
-		// --external-cloud-volume-plugin to the KCM, which runs the
-		// KCM volume controllers.
-		if kcm.ExternalCloudVolumePlugin == "" {
-			klog.Infof("An external cloud controller manager is configured, but ExternalCloudVolumePlugin is not configured for the KCM.  This means a CSI plugin must be installed by the user or else volume management might not work.")
-		}
 	}
 
 	kcm.LogLevel = 2

diff --git a/pkg/model/tests/data/bootstrapscript_0.txt b/pkg/model/tests/data/bootstrapscript_0.txt
@@ -150,6 +150,7 @@ ensure-install-dir
 
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
+  awsEBSCSIDriver: null
   nodeTags: something
 containerRuntime: docker
 containerd:

diff --git a/pkg/model/tests/data/bootstrapscript_1.txt b/pkg/model/tests/data/bootstrapscript_1.txt
@@ -150,6 +150,7 @@ ensure-install-dir
 
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
+  awsEBSCSIDriver: null
   nodeTags: something
 containerRuntime: docker
 containerd:

diff --git a/pkg/model/tests/data/bootstrapscript_2.txt b/pkg/model/tests/data/bootstrapscript_2.txt
@@ -150,6 +150,7 @@ ensure-install-dir
 
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
+  awsEBSCSIDriver: null
   nodeTags: something
 containerRuntime: docker
 containerd:

diff --git a/pkg/model/tests/data/bootstrapscript_3.txt b/pkg/model/tests/data/bootstrapscript_3.txt
@@ -150,6 +150,7 @@ ensure-install-dir
 
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
+  awsEBSCSIDriver: null
   nodeTags: something
 containerRuntime: docker
 containerd:

diff --git a/pkg/model/tests/data/bootstrapscript_4.txt b/pkg/model/tests/data/bootstrapscript_4.txt
@@ -150,6 +150,7 @@ ensure-install-dir
 
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
+  awsEBSCSIDriver: null
   nodeTags: something
 containerRuntime: docker
 containerd:

diff --git a/pkg/model/tests/data/bootstrapscript_5.txt b/pkg/model/tests/data/bootstrapscript_5.txt
@@ -150,6 +150,7 @@ ensure-install-dir
 
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
+  awsEBSCSIDriver: null
   nodeTags: something
 containerRuntime: docker
 containerd:

diff --git a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml
@@ -9,6 +9,9 @@ spec:
   authorization:
     rbac: {}
   channel: stable
+  cloudConfig:
+    awsEBSCSIDriver:
+      enabled: true
   cloudProvider: aws
   configBase: memfs://tests/complex.example.com
   containerRuntime: docker

diff --git a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml
@@ -10,6 +10,7 @@ spec:
     rbac: {}
   channel: stable
   cloudConfig:
+    awsEBSCSIDriver: null
     gceServiceAccount: test-account@testproject.iam.gserviceaccount.com
   cloudProvider: gce
   configBase: memfs://tests/gce.example.com

diff --git a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml
@@ -9,6 +9,9 @@ spec:
   authorization:
     rbac: {}
   channel: stable
+  cloudConfig:
+    awsEBSCSIDriver:
+      enabled: true
   cloudProvider: aws
   configBase: memfs://tests/ha.example.com
   containerRuntime: docker

diff --git a/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml
@@ -9,6 +9,9 @@ spec:
   authorization:
     rbac: {}
   channel: stable
+  cloudConfig:
+    awsEBSCSIDriver:
+      enabled: true
   cloudProvider: aws
   configBase: memfs://tests/ha.example.com
   containerRuntime: docker