From 03088a52887247d3a4eb558e5dd64f5b44cab8e7 Mon Sep 17 00:00:00 2001 From: Connor Catlett Date: Fri, 30 Sep 2022 15:27:41 +0000 Subject: [PATCH] Add support for io2 Block Express volumes io2 Block Express, which io2 volumes automatically get upgraded to on certain instance types, has a higher iops limit than normal io2 volumes Signed-off-by: Connor Catlett --- docs/parameters.md | 34 ++++++++++++++++++---------------- pkg/cloud/cloud.go | 26 ++++++++++++++++---------- pkg/cloud/cloud_test.go | 20 ++++++++++++++++++++ pkg/driver/constants.go | 3 +++ pkg/driver/controller.go | 15 +++++++++++++-- 5 files changed, 70 insertions(+), 28 deletions(-) diff --git a/docs/parameters.md b/docs/parameters.md index 822492cbb5..5225c7477c 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -3,25 +3,27 @@ There are several optional parameters that may be passed into `CreateVolumeReque The AWS EBS CSI Driver supports [tagging](tagging.md) through `StorageClass.parameters` (in v1.6.0 and later). -| Parameters | Values | Default | Description | -|-----------------------------|----------------------------------------------------------------|----------|---------------------| -| "csi.storage.k8s.io/fstype" | xfs, ext2, ext3, ext4 | ext4 | File system type that will be formatted during volume creation. This parameter is case sensitive! | -| "type" | io1, io2, gp2, gp3, sc1, st1, standard, sbp1, sbg1 | gp3* | EBS volume type. | -| "iopsPerGB" | | | I/O operations per second per GiB. Can be specified for IO1, IO2, and GP3 volumes. | -| "allowAutoIOPSPerGBIncrease"| true, false | false | When `"true"`, the CSI driver increases IOPS for a volume when `iopsPerGB * ` is too low to fit into IOPS range supported by AWS. This allows dynamic provisioning to always succeed, even when user specifies too small PVC capacity or `iopsPerGB` value. On the other hand, it may introduce additional costs, as such volumes have higher IOPS than requested in `iopsPerGB`.| -| "iops" | | | I/O operations per second. Can be specified for IO1, IO2, and GP3 volumes. | -| "throughput" | | 125 | Throughput in MiB/s. Only effective when gp3 volume type is specified. If empty, it will set to 125MiB/s as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html). | -| "encrypted" | | | Whether the volume should be encrypted or not. Valid values are "true" or "false". | -| "kmsKeyId" | | | The full ARN of the key to use when encrypting the volume. If not specified, AWS will use the default KMS key for the region the volume is in. This will be an auto-generated key called `/aws/ebs` if not changed. | +| Parameters | Values | Default | Description | +|------------------------------|----------------------------------------------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| "csi.storage.k8s.io/fstype" | xfs, ext2, ext3, ext4 | ext4 | File system type that will be formatted during volume creation. This parameter is case sensitive! | +| "type" | io1, io2, gp2, gp3, sc1, st1, standard, sbp1, sbg1 | gp3* | EBS volume type. | +| "iopsPerGB" | | | I/O operations per second per GiB. Can be specified for IO1, IO2, and GP3 volumes. | +| "allowAutoIOPSPerGBIncrease" | true, false | false | When `"true"`, the CSI driver increases IOPS for a volume when `iopsPerGB * ` is too low to fit into IOPS range supported by AWS. This allows dynamic provisioning to always succeed, even when user specifies too small PVC capacity or `iopsPerGB` value. On the other hand, it may introduce additional costs, as such volumes have higher IOPS than requested in `iopsPerGB`. | +| "iops" | | | I/O operations per second. Can be specified for IO1, IO2, and GP3 volumes. | +| "throughput" | | 125 | Throughput in MiB/s. Only effective when gp3 volume type is specified. If empty, it will set to 125MiB/s as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html). | +| "encrypted" | true, false | false | Whether the volume should be encrypted or not. Valid values are "true" or "false". | +| "blockExpress" | true, false | false | Enables the creation of [io2 Block Express volumes](https://aws.amazon.com/ebs/provisioned-iops/#Introducing_io2_Block_Express) by increasing the limit for io2 volumes to 256000 for io volumes. | +| "kmsKeyId" | | | The full ARN of the key to use when encrypting the volume. If not specified, AWS will use the default KMS key for the region the volume is in. This will be an auto-generated key called `/aws/ebs` if not changed. | **Appendix** * `gp3` is currently not supported on outposts. Outpost customers need to use a different type for their volumes. * Unless explicitly noted, all parameters are case insensitive (e.g. "kmsKeyId", "kmskeyid" and any other combination of upper/lowercase characters can be used). -* If the requested IOPs (either directly from `iops` or from `iopsPerGB` multiplied by the volume's capacity) produces a value above the maximum IOPs allowed for the [volume type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html), the IOPS will be capped at the maximum value allowed. If the value is lower than the minimal supported IOPS value per volume, either an error is returned (the default behavior), or the value is increased to fit into the supported range when `allowautoiopspergbincrease` is `"true"`. +* If the requested IOPS (either directly from `iops` or from `iopsPerGB` multiplied by the volume's capacity) produces a value above the maximum IOPS allowed for the [volume type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html), the IOPS will be capped at the maximum value allowed. If the value is lower than the minimal supported IOPS value per volume, either an error is returned (the default behavior), or the value is increased to fit into the supported range when `allowautoiopspergbincrease` is `"true"`. * You may specify either the "iops" or "iopsPerGb" parameters, not both. Specifying both parameters will result in an invalid StorageClass. -| Volume Type | Min total IOPS | Max total IOPS | Max IOPS per GB | -|-----------------------------|----------------------------------------|------------------|-------------------| -| IO1 | 100 | 64000 | 50 | -| IO2 | 100 | 64000 | 500 | -| GP3 | 3000 | 16000 | 500 | +| Volume Type | Min total IOPS | Max total IOPS | Max IOPS per GB | +|----------------------------|----------------|---------------|-------------------| +| io1 | 100 | 64000 | 50 | +| io2 (blockExpress = false) | 100 | 64000 | 500 | +| io2 (blockExpress = true) | 100 | 256000 | 500 | +| gp3 | 3000 | 16000 | 500 | diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 043e22fa0b..8fa93067ca 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -63,15 +63,16 @@ const ( // AWS provisioning limits. // Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html const ( - io1MinTotalIOPS = 100 - io1MaxTotalIOPS = 64000 - io1MaxIOPSPerGB = 50 - io2MinTotalIOPS = 100 - io2MaxTotalIOPS = 64000 - io2MaxIOPSPerGB = 500 - gp3MaxTotalIOPS = 16000 - gp3MinTotalIOPS = 3000 - gp3MaxIOPSPerGB = 500 + io1MinTotalIOPS = 100 + io1MaxTotalIOPS = 64000 + io1MaxIOPSPerGB = 50 + io2MinTotalIOPS = 100 + io2MaxTotalIOPS = 64000 + io2BxMaxTotalIOPS = 256000 + io2MaxIOPSPerGB = 500 + gp3MaxTotalIOPS = 16000 + gp3MinTotalIOPS = 3000 + gp3MaxIOPSPerGB = 500 ) var ( @@ -192,6 +193,7 @@ type DiskOptions struct { AvailabilityZone string OutpostArn string Encrypted bool + BlockExpress bool // KmsKeyID represents a fully qualified resource name to the key to use for encryption. // example: arn:aws:kms:us-east-1:012345678910:key/abcd1234-a123-456a-a12b-a123b4cd56ef KmsKeyID string @@ -316,7 +318,11 @@ func (c *cloud) CreateDisk(ctx context.Context, volumeName string, diskOptions * minIops = io1MinTotalIOPS maxIopsPerGb = io1MaxIOPSPerGB case VolumeTypeIO2: - maxIops = io2MaxTotalIOPS + if diskOptions.BlockExpress { + maxIops = io2BxMaxTotalIOPS + } else { + maxIops = io2MaxTotalIOPS + } minIops = io2MinTotalIOPS maxIopsPerGb = io2MaxIOPSPerGB case VolumeTypeGP3: diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go index 68cb9f6ca1..15f6a7ae87 100644 --- a/pkg/cloud/cloud_test.go +++ b/pkg/cloud/cloud_test.go @@ -496,6 +496,26 @@ func TestCreateDisk(t *testing.T) { }, expErr: nil, }, + { + name: "success: large io2 Block Express with too high iopsPerGB", + volumeName: "vol-test-name", + diskOptions: &DiskOptions{ + CapacityBytes: util.GiBToBytes(3333), + Tags: map[string]string{VolumeNameTagKey: "vol-test", AwsEbsDriverTagKey: "true"}, + VolumeType: VolumeTypeIO2, + IOPSPerGB: 100000, + BlockExpress: true, + }, + expDisk: &Disk{ + VolumeID: "vol-test", + CapacityGiB: 3333, + AvailabilityZone: defaultZone, + }, + expCreateVolumeInput: &ec2.CreateVolumeInput{ + Iops: aws.Int64(256000), + }, + expErr: nil, + }, { name: "success: create volume when zone is snow and add tags", volumeName: "vol-test-name", diff --git a/pkg/driver/constants.go b/pkg/driver/constants.go index 1dc092be55..c5adc3aa98 100644 --- a/pkg/driver/constants.go +++ b/pkg/driver/constants.go @@ -69,6 +69,9 @@ const ( // provisioned volume PVNameKey = "csi.storage.k8s.io/pv/name" + // BlockExpressKey increases the iops limit for io2 volumes to the block express limit + BlockExpressKey = "blockExpress" + // TagKeyPrefix contains the prefix of a volume parameter that designates it as // a tag to be attached to the resource TagKeyPrefix = "tagSpecification" diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 37678f434d..c83044ff25 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -32,9 +32,8 @@ import ( "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util/template" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "k8s.io/klog/v2" ) - + var ( // volumeCaps represents how the volume could be accessed. // It is SINGLE_NODE_WRITER since EBS volume could only be @@ -124,6 +123,7 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol iops int throughput int isEncrypted bool + blockExpress bool kmsKeyID string scTags []string volumeTags = map[string]string{ @@ -174,6 +174,12 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol case PVNameKey: volumeTags[PVNameTag] = value tProps.PVName = value + case BlockExpressKey: + if value == "true" { + blockExpress = true + } else { + blockExpress = false + } default: if strings.HasPrefix(key, TagKeyPrefix) { scTags = append(scTags, value) @@ -189,6 +195,10 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol } } + if volumeType != cloud.VolumeTypeIO2 && blockExpress { + return nil, status.Errorf(codes.InvalidArgument, "Block Express is only supported on io2 volumes") + } + snapshotID := "" volumeSource := req.GetVolumeContentSource() if volumeSource != nil { @@ -241,6 +251,7 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol AvailabilityZone: zone, OutpostArn: outpostArn, Encrypted: isEncrypted, + BlockExpress: blockExpress, KmsKeyID: kmsKeyID, SnapshotID: snapshotID, }