Add FSx for Lustre AutoImport support

kubernetes-sigs · Oct 1, 2020 · a932e67 · a932e67
1 parent eaca84d
commit a932e67
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 0 deletions.
diff --git a/examples/kubernetes/dynamic_provisioning_s3/README.md b/examples/kubernetes/dynamic_provisioning_s3/README.md
@@ -13,12 +13,14 @@ provisioner: fsx.csi.aws.com
 parameters:
   subnetId: subnet-056da83524edbe641
   securityGroupIds: sg-086f61ea73388fb6b
+  autoImportPolicy: NONE
   s3ImportPath: s3://ml-training-data-000
   s3ExportPath: s3://ml-training-data-000/export
   deploymentType: SCRATCH_2
 ```
 * subnetId - the subnet ID that the FSx for Lustre filesystem should be created inside.
 * securityGroupIds - a common separated list of security group IDs that should be attached to the filesystem.
+* autoImportPolicy - the policy FSx will follow that determines how the filesystem is automatically updated with changes made in the linked data repository. For list of acceptable policies, please view the official FSx for Lustre documentation
 * s3ImportPath(Optional) - S3 data repository you want to copy from S3 to persistent volume.
 * s3ExportPath(Optional) - S3 data repository you want to export new or modified files from persistent volume to S3.
 * deploymentType (Optional) - FSx for Lustre supports three deployment types, SCRATCH_1, SCRATCH_2 and PERSISTENT_1. Default: SCRATCH_1.

diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go
@@ -69,6 +69,7 @@ type FileSystemOptions struct {
 	CapacityGiB              int64
 	SubnetId                 string
 	SecurityGroupIds         []string
+	AutoImportPolicy         string
 	S3ImportPath             string
 	S3ExportPath             string
 	DeploymentType           string
@@ -115,6 +116,10 @@ func (c *cloud) CreateFileSystem(ctx context.Context, volumeName string, fileSys
 
 	lustreConfiguration := &fsx.CreateFileSystemLustreConfiguration{}
 
+	if fileSystemOptions.AutoImportPolicy != "" {
+		lustreConfiguration.SetAutoImportPolicy(fileSystemOptions.AutoImportPolicy)
+	}
+
 	if fileSystemOptions.S3ImportPath != "" {
 		lustreConfiguration.SetImportPath(fileSystemOptions.S3ImportPath)
 	}

diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go
@@ -35,6 +35,7 @@ func TestCreateFileSystem(t *testing.T) {
 		subnetId                       = "subnet-056da83524edbe641"
 		securityGroupIds               = []string{"sg-086f61ea73388fb6b", "sg-0145e55e976000c9e"}
 		dnsname                        = "test.fsx.us-west-2.amazoawd.com"
+		autoImportPolicy               = "NEW_CHANGED"
 		s3ImportPath                   = "s3://fsx-s3-data-repository"
 		s3ExportPath                   = "s3://fsx-s3-data-repository/export"
 		deploymentType                 = fsx.LustreDeploymentTypeScratch2
@@ -170,11 +171,13 @@ func TestCreateFileSystem(t *testing.T) {
 					CapacityGiB:      volumeSizeGiB,
 					SubnetId:         subnetId,
 					SecurityGroupIds: securityGroupIds,
+					AutoImportPolicy: autoImportPolicy,
 					S3ImportPath:     s3ImportPath,
 					S3ExportPath:     s3ExportPath,
 				}
 
 				dataRepositoryConfiguration := &fsx.DataRepositoryConfiguration{}
+				dataRepositoryConfiguration.SetAutoImportPolicy(autoImportPolicy)
 				dataRepositoryConfiguration.SetImportPath(s3ImportPath)
 				dataRepositoryConfiguration.SetExportPath(s3ExportPath)
 
@@ -260,6 +263,7 @@ func TestCreateFileSystem(t *testing.T) {
 					CapacityGiB:      volumeSizeGiB,
 					SubnetId:         subnetId,
 					SecurityGroupIds: securityGroupIds,
+					AutoImportPolicy: autoImportPolicy,
 					S3ImportPath:     "s3://bucket1/import",
 					S3ExportPath:     "s3://bucket2/export",
 				}
@@ -442,6 +446,7 @@ func TestDescribeFileSystem(t *testing.T) {
 		fileSystemId        = "fs-1234"
 		volumeSizeGiB int64 = 1200
 		dnsname             = "test.fsx.us-west-2.amazoawd.com"
+		autoImportPolicy    = "NEW_CHANGED"
 		s3ImportPath        = "s3://fsx-s3-data-repository"
 		s3ExportPath        = "s3://fsx-s3-data-repository/export"
 		mountName           = "fsx"
@@ -491,6 +496,7 @@ func TestDescribeFileSystem(t *testing.T) {
 				}
 
 				dataRepositoryConfiguration := &fsx.DataRepositoryConfiguration{}
+				dataRepositoryConfiguration.SetAutoImportPolicy(autoImportPolicy)
 				dataRepositoryConfiguration.SetImportPath(s3ImportPath)
 				dataRepositoryConfiguration.SetExportPath(s3ExportPath)
 

diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go
@@ -42,6 +42,7 @@ const (
 
 	volumeParamsSubnetId                 = "subnetId"
 	volumeParamsSecurityGroupIds         = "securityGroupIds"
+	volumeParamsAutoImportPolicy         = "autoImportPolicy"
 	volumeParamsS3ImportPath             = "s3ImportPath"
 	volumeParamsS3ExportPath             = "s3ExportPath"
 	volumeParamsDeploymentType           = "deploymentType"
@@ -75,6 +76,10 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
 		SecurityGroupIds: strings.Split(securityGroupIds, ","),
 	}
 
+	if val, ok := volumeParams[volumeParamsAutoImportPolicy]; ok {
+		fsOptions.AutoImportPolicy = val
+	}
+
 	if val, ok := volumeParams[volumeParamsS3ImportPath]; ok {
 		fsOptions.S3ImportPath = val
 	}

diff --git a/tests/e2e/dynamic_provisioning_test.go b/tests/e2e/dynamic_provisioning_test.go
@@ -158,6 +158,7 @@ var _ = Describe("[fsx-csi-e2e] Dynamic Provisioning with s3 data repository", f
 						Parameters: map[string]string{
 							"subnetId":         subnetId,
 							"securityGroupIds": strings.Join(securityGroupIds, ","),
+							"autoImportPolicy": "NEW_CHANGED"
 							"s3ImportPath":     fmt.Sprintf("s3://%s", bucketName),
 							"s3ExportPath":     fmt.Sprintf("s3://%s/export", bucketName),
 						},