diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index ecc738fe88738..faec8189af150 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -689,6 +689,11 @@ spec: description: SecurityGroupOverride overrides the default security group created by Kops for this IG (AWS only). type: string + spotDurationInMinutes: + description: SpotDurationInMinutes indicates this is a spot-block group, + with the specified value as the spot reservation time + format: int64 + type: integer subnets: description: Subnets is the names of the Subnets (as specified in the Cluster) where machines in this instance group should be placed diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index af441b4512cae..e46ee68290e8a 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -121,6 +121,8 @@ type InstanceGroupSpec struct { Hooks []HookSpec `json:"hooks,omitempty"` // MaxPrice indicates this is a spot-pricing group, with the specified value as our max-price bid MaxPrice *string `json:"maxPrice,omitempty"` + // SpotDurationInMinutes reserves a spot block for the period specified + SpotDurationInMinutes *int64 `json:"spotDurationInMinutes,omitempty"` // AssociatePublicIP is true if we want instances to have a public IP AssociatePublicIP *bool `json:"associatePublicIp,omitempty"` // AdditionalSecurityGroups attaches additional security groups (e.g. i-123456) diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index 021e6a9a12e93..eac6b7dcdd5a3 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -116,6 +116,8 @@ type InstanceGroupSpec struct { Hooks []HookSpec `json:"hooks,omitempty"` // MaxPrice indicates this is a spot-pricing group, with the specified value as our max-price bid MaxPrice *string `json:"maxPrice,omitempty"` + // SpotDurationInMinutes indicates this is a spot-block group, with the specified value as the spot reservation time + SpotDurationInMinutes *int64 `json:"spotDurationInMinutes,omitempty"` // AssociatePublicIP is true if we want instances to have a public IP AssociatePublicIP *bool `json:"associatePublicIp,omitempty"` // AdditionalSecurityGroups attaches additional security groups (e.g. i-123456) diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 5e87782c7d853..1769d3ed1769c 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -3225,6 +3225,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan out.Hooks = nil } out.MaxPrice = in.MaxPrice + out.SpotDurationInMinutes = in.SpotDurationInMinutes out.AssociatePublicIP = in.AssociatePublicIP out.AdditionalSecurityGroups = in.AdditionalSecurityGroups out.CloudLabels = in.CloudLabels @@ -3362,6 +3363,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I out.Hooks = nil } out.MaxPrice = in.MaxPrice + out.SpotDurationInMinutes = in.SpotDurationInMinutes out.AssociatePublicIP = in.AssociatePublicIP out.AdditionalSecurityGroups = in.AdditionalSecurityGroups out.CloudLabels = in.CloudLabels diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index f83aefa4b4370..67966b3f304d4 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1663,6 +1663,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = new(string) **out = **in } + if in.SpotDurationInMinutes != nil { + in, out := &in.SpotDurationInMinutes, &out.SpotDurationInMinutes + *out = new(int64) + **out = **in + } if in.AssociatePublicIP != nil { in, out := &in.AssociatePublicIP, &out.AssociatePublicIP *out = new(bool) diff --git a/pkg/apis/kops/validation/aws.go b/pkg/apis/kops/validation/aws.go index ffe003a21c5ff..00267b89310d9 100644 --- a/pkg/apis/kops/validation/aws.go +++ b/pkg/apis/kops/validation/aws.go @@ -18,6 +18,7 @@ package validation import ( "fmt" + "strconv" "strings" "k8s.io/apimachinery/pkg/util/sets" @@ -48,6 +49,8 @@ func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList { allErrs = append(allErrs, awsValidateAMIforNVMe(field.NewPath(ig.GetName(), "spec", "machineType"), ig)...) + allErrs = append(allErrs, awsValidateSpotDurationInMinute(field.NewPath(ig.GetName(), "spec", "spotDurationInMinutes"), ig)...) + return allErrs } @@ -107,3 +110,13 @@ func awsValidateAMIforNVMe(fieldPath *field.Path, ig *kops.InstanceGroup) field. } return allErrs } + +func awsValidateSpotDurationInMinute(fieldPath *field.Path, ig *kops.InstanceGroup) field.ErrorList { + allErrs := field.ErrorList{} + if ig.Spec.SpotDurationInMinutes != nil { + validSpotDurations := []string{"60", "120", "180", "240", "300", "360"} + spotDurationStr := strconv.FormatInt(*ig.Spec.SpotDurationInMinutes, 10) + allErrs = append(allErrs, IsValidValue(fieldPath, &spotDurationStr, validSpotDurations)...) + } + return allErrs +} diff --git a/pkg/apis/kops/validation/aws_test.go b/pkg/apis/kops/validation/aws_test.go index 93051b842b846..20cd30636c59f 100644 --- a/pkg/apis/kops/validation/aws_test.go +++ b/pkg/apis/kops/validation/aws_test.go @@ -19,6 +19,8 @@ package validation import ( "testing" + "k8s.io/kops/upup/pkg/fi" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kops/pkg/apis/kops" ) @@ -102,6 +104,36 @@ func TestValidateInstanceGroupSpec(t *testing.T) { "Forbidden::test-nodes.spec.machineType", }, }, + { + Input: kops.InstanceGroupSpec{ + SpotDurationInMinutes: fi.Int64(55), + }, + ExpectedErrors: []string{ + "Unsupported value::test-nodes.spec.spotDurationInMinutes", + }, + }, + { + Input: kops.InstanceGroupSpec{ + SpotDurationInMinutes: fi.Int64(380), + }, + ExpectedErrors: []string{ + "Unsupported value::test-nodes.spec.spotDurationInMinutes", + }, + }, + { + Input: kops.InstanceGroupSpec{ + SpotDurationInMinutes: fi.Int64(125), + }, + ExpectedErrors: []string{ + "Unsupported value::test-nodes.spec.spotDurationInMinutes", + }, + }, + { + Input: kops.InstanceGroupSpec{ + SpotDurationInMinutes: fi.Int64(120), + }, + ExpectedErrors: []string{}, + }, } for _, g := range grid { ig := &kops.InstanceGroup{ diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index d9c74786377b1..9e0f3a7c367d9 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1829,6 +1829,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = new(string) **out = **in } + if in.SpotDurationInMinutes != nil { + in, out := &in.SpotDurationInMinutes, &out.SpotDurationInMinutes + *out = new(int64) + **out = **in + } if in.AssociatePublicIP != nil { in, out := &in.AssociatePublicIP, &out.AssociatePublicIP *out = new(bool) diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index fb1d0cb3364d2..dcc611b4a6c61 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -131,6 +131,9 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde if ig.Spec.MixedInstancesPolicy == nil { lt.SpotPrice = lc.SpotPrice } + if ig.Spec.SpotDurationInMinutes != nil { + lt.SpotDurationInMinutes = ig.Spec.SpotDurationInMinutes + } return lt, nil } diff --git a/tests/integration/update_cluster/launch_templates/cloudformation.json b/tests/integration/update_cluster/launch_templates/cloudformation.json index 315c8a5596e0f..cfb4a0e8418e3 100644 --- a/tests/integration/update_cluster/launch_templates/cloudformation.json +++ b/tests/integration/update_cluster/launch_templates/cloudformation.json @@ -602,6 +602,13 @@ "ImageId": "ami-12345678", "InstanceType": "t3.medium", "KeyName": "kubernetes.launchtemplates.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "InstanceMarketOptions": { + "MarketType": "spot", + "SpotOptions": { + "BlockDurationMinutes": 120, + "MaxPrice": "0.1" + } + }, "NetworkInterfaces": [ { "AssociatePublicIpAddress": true, diff --git a/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml b/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml index 62d0b7deaf866..44f663c28ffaa 100644 --- a/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml @@ -71,6 +71,8 @@ spec: minSize: 2 role: Node instanceProtection: true + maxPrice: "0.1" + spotDurationInMinutes: 120 subnets: - us-test-1b --- diff --git a/tests/integration/update_cluster/launch_templates/kubernetes.tf b/tests/integration/update_cluster/launch_templates/kubernetes.tf index e46f6be77f5f1..a30a113d2bdc9 100644 --- a/tests/integration/update_cluster/launch_templates/kubernetes.tf +++ b/tests/integration/update_cluster/launch_templates/kubernetes.tf @@ -611,6 +611,15 @@ resource "aws_launch_template" "nodes-launchtemplates-example-com" { instance_type = "t3.medium" key_name = "${aws_key_pair.kubernetes-launchtemplates-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}" + instance_market_options = { + market_type = "spot" + + spot_options = { + block_duration_minutes = 120 + max_price = "0.1" + } + } + network_interfaces = { associate_public_ip_address = true delete_on_termination = true diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate.go index 87351b4a709c0..a55907e0d59af 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate.go @@ -62,6 +62,8 @@ type LaunchTemplate struct { SecurityGroups []*SecurityGroup // SpotPrice is set to the spot-price bid if this is a spot pricing request SpotPrice string + // SpotDurationInMinutes is set for requesting spot blocks + SpotDurationInMinutes *int64 // Tags are the keypairs to apply to the instance and volume on launch. Tags map[string]string // Tenancy. Can be either default or dedicated. diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go index 45d4a9c287750..39f3a0d75fd0a 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go @@ -62,6 +62,8 @@ type cloudformationLaunchTemplateIAMProfile struct { } type cloudformationLaunchTemplateMarketOptionsSpotOptions struct { + // BlockDurationMinutes is required duration in minutes. This value must be a multiple of 60. + BlockDurationMinutes *int64 `json:"BlockDurationMinutes,omitempty"` // InstancesInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate InstancesInterruptionBehavior *string `json:"InstancesInterruptionBehavior,omitempty"` // MaxPrice is the maximum hourly price you're willing to pay for the Spot Instances @@ -74,7 +76,7 @@ type cloudformationLaunchTemplateMarketOptions struct { // MarketType is the option type MarketType *string `json:"MarketType,omitempty"` // SpotOptions are the set of options - SpotOptions []*cloudformationLaunchTemplateMarketOptionsSpotOptions `json:"Options,omitempty"` + SpotOptions *cloudformationLaunchTemplateMarketOptionsSpotOptions `json:"SpotOptions,omitempty"` } type cloudformationLaunchTemplateBlockDeviceEBS struct { @@ -165,29 +167,33 @@ func (t *LaunchTemplate) RenderCloudformation(target *cloudformation.Cloudformat image = im.ImageId } - cf := &cloudformationLaunchTemplate{ - LaunchTemplateName: fi.String(fi.StringValue(e.Name)), - LaunchTemplateData: &cloudformationLaunchTemplateData{ - EBSOptimized: e.RootVolumeOptimization, - ImageID: image, - InstanceType: e.InstanceType, - NetworkInterfaces: []*cloudformationLaunchTemplateNetworkInterface{ - { - AssociatePublicIPAddress: e.AssociatePublicIP, - DeleteOnTermination: fi.Bool(true), - DeviceIndex: fi.Int(0), - }, + launchTemplateData := &cloudformationLaunchTemplateData{ + EBSOptimized: e.RootVolumeOptimization, + ImageID: image, + InstanceType: e.InstanceType, + NetworkInterfaces: []*cloudformationLaunchTemplateNetworkInterface{ + { + AssociatePublicIPAddress: e.AssociatePublicIP, + DeleteOnTermination: fi.Bool(true), + DeviceIndex: fi.Int(0), }, }, } - data := cf.LaunchTemplateData if e.SpotPrice != "" { - data.MarketOptions = &cloudformationLaunchTemplateMarketOptions{ - MarketType: fi.String("spot"), - SpotOptions: []*cloudformationLaunchTemplateMarketOptionsSpotOptions{{MaxPrice: fi.String(e.SpotPrice)}}, + marketSpotOptions := cloudformationLaunchTemplateMarketOptionsSpotOptions{MaxPrice: fi.String(e.SpotPrice)} + if e.SpotDurationInMinutes != nil { + marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes } + launchTemplateData.MarketOptions = &cloudformationLaunchTemplateMarketOptions{MarketType: fi.String("spot"), SpotOptions: &marketSpotOptions} } + + cf := &cloudformationLaunchTemplate{ + LaunchTemplateName: fi.String(fi.StringValue(e.Name)), + LaunchTemplateData: launchTemplateData, + } + data := cf.LaunchTemplateData + for _, x := range e.SecurityGroups { data.NetworkInterfaces[0].SecurityGroups = append(data.NetworkInterfaces[0].SecurityGroups, x.CloudformationLink()) } diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go index e905965931b4e..427f96a583f91 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go @@ -37,6 +37,8 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) { RootVolumeOptimization: fi.Bool(true), RootVolumeIops: fi.Int64(100), RootVolumeSize: fi.Int64(64), + SpotPrice: "10", + SpotDurationInMinutes: fi.Int64(120), SSHKey: &SSHKey{ Name: fi.String("mykey"), }, @@ -61,6 +63,13 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) { }, "InstanceType": "t2.medium", "KeyName": "mykey", + "InstanceMarketOptions": { + "MarketType": "spot", + "SpotOptions": { + "BlockDurationMinutes": 120, + "MaxPrice": "10" + } + }, "NetworkInterfaces": [ { "AssociatePublicIpAddress": true, diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go index 85ad2df028b4a..6c827080ef834 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go @@ -179,10 +179,14 @@ func (t *LaunchTemplate) RenderTerraform(target *terraform.TerraformTarget, a, e } if e.SpotPrice != "" { + marketSpotOptions := terraformLaunchTemplateMarketOptionsSpotOptions{MaxPrice: fi.String(e.SpotPrice)} + if e.SpotDurationInMinutes != nil { + marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes + } tf.MarketOptions = []*terraformLaunchTemplateMarketOptions{ { MarketType: fi.String("spot"), - SpotOptions: []*terraformLaunchTemplateMarketOptionsSpotOptions{{MaxPrice: fi.String(e.SpotPrice)}}, + SpotOptions: []*terraformLaunchTemplateMarketOptionsSpotOptions{&marketSpotOptions}, }, } } diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go index 3d100b5daee74..38553f87c0e6b 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go @@ -35,6 +35,7 @@ func TestLaunchTemplateTerraformRender(t *testing.T) { InstanceMonitoring: fi.Bool(true), InstanceType: fi.String("t2.medium"), SpotPrice: "0.1", + SpotDurationInMinutes: fi.Int64(60), RootVolumeOptimization: fi.Bool(true), RootVolumeIops: fi.Int64(100), RootVolumeSize: fi.Int64(64), @@ -72,7 +73,8 @@ resource "aws_launch_template" "test" { market_type = "spot" spot_options = { - max_price = "0.1" + block_duration_minutes = 60 + max_price = "0.1" } }