Merge pull request #8802 from thejasbabu/feature/spot-block

Add support for Spot block in launch template

k8s/crds/kops.k8s.io_instancegroups.yaml

@@ -689,6 +689,11 @@ spec:
               description: SecurityGroupOverride overrides the default security group
                 created by Kops for this IG (AWS only).
               type: string
+            spotDurationInMinutes:
+              description: SpotDurationInMinutes indicates this is a spot-block group,
+                with the specified value as the spot reservation time
+              format: int64
+              type: integer
             subnets:
               description: Subnets is the names of the Subnets (as specified in the
                 Cluster) where machines in this instance group should be placed

pkg/apis/kops/instancegroup.go

@@ -121,6 +121,8 @@ type InstanceGroupSpec struct {
 	Hooks []HookSpec `json:"hooks,omitempty"`
 	// MaxPrice indicates this is a spot-pricing group, with the specified value as our max-price bid
 	MaxPrice *string `json:"maxPrice,omitempty"`
+	// SpotDurationInMinutes reserves a spot block for the period specified
+	SpotDurationInMinutes *int64 `json:"spotDurationInMinutes,omitempty"`
 	// AssociatePublicIP is true if we want instances to have a public IP
 	AssociatePublicIP *bool `json:"associatePublicIp,omitempty"`
 	// AdditionalSecurityGroups attaches additional security groups (e.g. i-123456)

pkg/apis/kops/v1alpha2/instancegroup.go

@@ -116,6 +116,8 @@ type InstanceGroupSpec struct {
 	Hooks []HookSpec `json:"hooks,omitempty"`
 	// MaxPrice indicates this is a spot-pricing group, with the specified value as our max-price bid
 	MaxPrice *string `json:"maxPrice,omitempty"`
+	// SpotDurationInMinutes indicates this is a spot-block group, with the specified value as the spot reservation time
+	SpotDurationInMinutes *int64 `json:"spotDurationInMinutes,omitempty"`
 	// AssociatePublicIP is true if we want instances to have a public IP
 	AssociatePublicIP *bool `json:"associatePublicIp,omitempty"`
 	// AdditionalSecurityGroups attaches additional security groups (e.g. i-123456)

pkg/apis/kops/validation/aws.go

@@ -18,6 +18,7 @@ package validation
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -48,6 +49,8 @@ func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList {
 
 	allErrs = append(allErrs, awsValidateAMIforNVMe(field.NewPath(ig.GetName(), "spec", "machineType"), ig)...)
 
+	allErrs = append(allErrs, awsValidateSpotDurationInMinute(field.NewPath(ig.GetName(), "spec", "spotDurationInMinutes"), ig)...)
+
 	return allErrs
 }
 
@@ -107,3 +110,13 @@ func awsValidateAMIforNVMe(fieldPath *field.Path, ig *kops.InstanceGroup) field.
 	}
 	return allErrs
 }
+
+func awsValidateSpotDurationInMinute(fieldPath *field.Path, ig *kops.InstanceGroup) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if ig.Spec.SpotDurationInMinutes != nil {
+		validSpotDurations := []string{"60", "120", "180", "240", "300", "360"}
+		spotDurationStr := strconv.FormatInt(*ig.Spec.SpotDurationInMinutes, 10)
+		allErrs = append(allErrs, IsValidValue(fieldPath, &spotDurationStr, validSpotDurations)...)
+	}
+	return allErrs
+}

pkg/apis/kops/validation/aws_test.go

@@ -19,6 +19,8 @@ package validation
 import (
 	"testing"
 
+	"k8s.io/kops/upup/pkg/fi"
+
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/pkg/apis/kops"
 )
@@ -102,6 +104,36 @@ func TestValidateInstanceGroupSpec(t *testing.T) {
 				"Forbidden::test-nodes.spec.machineType",
 			},
 		},
+		{
+			Input: kops.InstanceGroupSpec{
+				SpotDurationInMinutes: fi.Int64(55),
+			},
+			ExpectedErrors: []string{
+				"Unsupported value::test-nodes.spec.spotDurationInMinutes",
+			},
+		},
+		{
+			Input: kops.InstanceGroupSpec{
+				SpotDurationInMinutes: fi.Int64(380),
+			},
+			ExpectedErrors: []string{
+				"Unsupported value::test-nodes.spec.spotDurationInMinutes",
+			},
+		},
+		{
+			Input: kops.InstanceGroupSpec{
+				SpotDurationInMinutes: fi.Int64(125),
+			},
+			ExpectedErrors: []string{
+				"Unsupported value::test-nodes.spec.spotDurationInMinutes",
+			},
+		},
+		{
+			Input: kops.InstanceGroupSpec{
+				SpotDurationInMinutes: fi.Int64(120),
+			},
+			ExpectedErrors: []string{},
+		},
 	}
 	for _, g := range grid {
 		ig := &kops.InstanceGroup{

pkg/model/awsmodel/autoscalinggroup.go

@@ -131,6 +131,9 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde
 	if ig.Spec.MixedInstancesPolicy == nil {
 		lt.SpotPrice = lc.SpotPrice
 	}
+	if ig.Spec.SpotDurationInMinutes != nil {
+		lt.SpotDurationInMinutes = ig.Spec.SpotDurationInMinutes
+	}
 	return lt, nil
 }
 

tests/integration/update_cluster/launch_templates/cloudformation.json

@@ -602,6 +602,13 @@
           "ImageId": "ami-12345678",
           "InstanceType": "t3.medium",
           "KeyName": "kubernetes.launchtemplates.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
+          "InstanceMarketOptions": {
+            "MarketType": "spot",
+            "SpotOptions": {
+              "BlockDurationMinutes": 120,
+              "MaxPrice": "0.1"
+            }
+          },
           "NetworkInterfaces": [
             {
               "AssociatePublicIpAddress": true,

tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml

@@ -71,6 +71,8 @@ spec:
   minSize: 2
   role: Node
   instanceProtection: true
+  maxPrice: "0.1"
+  spotDurationInMinutes: 120
   subnets:
   - us-test-1b
 ---

tests/integration/update_cluster/launch_templates/kubernetes.tf

@@ -611,6 +611,15 @@ resource "aws_launch_template" "nodes-launchtemplates-example-com" {
   instance_type = "t3.medium"
   key_name      = "${aws_key_pair.kubernetes-launchtemplates-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
 
+  instance_market_options = {
+    market_type = "spot"
+
+    spot_options = {
+      block_duration_minutes = 120
+      max_price              = "0.1"
+    }
+  }
+
   network_interfaces = {
     associate_public_ip_address = true
     delete_on_termination       = true

upup/pkg/fi/cloudup/awstasks/launchtemplate.go

@@ -62,6 +62,8 @@ type LaunchTemplate struct {
 	SecurityGroups []*SecurityGroup
 	// SpotPrice is set to the spot-price bid if this is a spot pricing request
 	SpotPrice string
+	// SpotDurationInMinutes is set for requesting spot blocks
+	SpotDurationInMinutes *int64
 	// Tags are the keypairs to apply to the instance and volume on launch.
 	Tags map[string]string
 	// Tenancy. Can be either default or dedicated.

upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go

@@ -62,6 +62,8 @@ type cloudformationLaunchTemplateIAMProfile struct {
 }
 
 type cloudformationLaunchTemplateMarketOptionsSpotOptions struct {
+	// BlockDurationMinutes is required duration in minutes. This value must be a multiple of 60.
+	BlockDurationMinutes *int64 `json:"BlockDurationMinutes,omitempty"`
 	// InstancesInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate
 	InstancesInterruptionBehavior *string `json:"InstancesInterruptionBehavior,omitempty"`
 	// MaxPrice is the maximum hourly price you're willing to pay for the Spot Instances
@@ -74,7 +76,7 @@ type cloudformationLaunchTemplateMarketOptions struct {
 	// MarketType is the option type
 	MarketType *string `json:"MarketType,omitempty"`
 	// SpotOptions are the set of options
-	SpotOptions []*cloudformationLaunchTemplateMarketOptionsSpotOptions `json:"Options,omitempty"`
+	SpotOptions *cloudformationLaunchTemplateMarketOptionsSpotOptions `json:"SpotOptions,omitempty"`
 }
 
 type cloudformationLaunchTemplateBlockDeviceEBS struct {
@@ -165,29 +167,33 @@ func (t *LaunchTemplate) RenderCloudformation(target *cloudformation.Cloudformat
 		image = im.ImageId
 	}
 
-	cf := &cloudformationLaunchTemplate{
-		LaunchTemplateName: fi.String(fi.StringValue(e.Name)),
-		LaunchTemplateData: &cloudformationLaunchTemplateData{
-			EBSOptimized: e.RootVolumeOptimization,
-			ImageID:      image,
-			InstanceType: e.InstanceType,
-			NetworkInterfaces: []*cloudformationLaunchTemplateNetworkInterface{
-				{
-					AssociatePublicIPAddress: e.AssociatePublicIP,
-					DeleteOnTermination:      fi.Bool(true),
-					DeviceIndex:              fi.Int(0),
-				},
+	launchTemplateData := &cloudformationLaunchTemplateData{
+		EBSOptimized: e.RootVolumeOptimization,
+		ImageID:      image,
+		InstanceType: e.InstanceType,
+		NetworkInterfaces: []*cloudformationLaunchTemplateNetworkInterface{
+			{
+				AssociatePublicIPAddress: e.AssociatePublicIP,
+				DeleteOnTermination:      fi.Bool(true),
+				DeviceIndex:              fi.Int(0),
 			},
 		},
 	}
-	data := cf.LaunchTemplateData
 
 	if e.SpotPrice != "" {
-		data.MarketOptions = &cloudformationLaunchTemplateMarketOptions{
-			MarketType:  fi.String("spot"),
-			SpotOptions: []*cloudformationLaunchTemplateMarketOptionsSpotOptions{{MaxPrice: fi.String(e.SpotPrice)}},
+		marketSpotOptions := cloudformationLaunchTemplateMarketOptionsSpotOptions{MaxPrice: fi.String(e.SpotPrice)}
+		if e.SpotDurationInMinutes != nil {
+			marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes
 		}
+		launchTemplateData.MarketOptions = &cloudformationLaunchTemplateMarketOptions{MarketType: fi.String("spot"), SpotOptions: &marketSpotOptions}
 	}
+
+	cf := &cloudformationLaunchTemplate{
+		LaunchTemplateName: fi.String(fi.StringValue(e.Name)),
+		LaunchTemplateData: launchTemplateData,
+	}
+	data := cf.LaunchTemplateData
+
 	for _, x := range e.SecurityGroups {
 		data.NetworkInterfaces[0].SecurityGroups = append(data.NetworkInterfaces[0].SecurityGroups, x.CloudformationLink())
 	}

upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go

@@ -37,6 +37,8 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) {
 				RootVolumeOptimization: fi.Bool(true),
 				RootVolumeIops:         fi.Int64(100),
 				RootVolumeSize:         fi.Int64(64),
+				SpotPrice:              "10",
+				SpotDurationInMinutes:  fi.Int64(120),
 				SSHKey: &SSHKey{
 					Name: fi.String("mykey"),
 				},
@@ -61,6 +63,13 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) {
           },
           "InstanceType": "t2.medium",
           "KeyName": "mykey",
+          "InstanceMarketOptions": {
+            "MarketType": "spot",
+            "SpotOptions": {
+              "BlockDurationMinutes": 120,
+              "MaxPrice": "10"
+            }
+          },
           "NetworkInterfaces": [
             {
               "AssociatePublicIpAddress": true,

upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go

@@ -179,10 +179,14 @@ func (t *LaunchTemplate) RenderTerraform(target *terraform.TerraformTarget, a, e
 	}
 
 	if e.SpotPrice != "" {
+		marketSpotOptions := terraformLaunchTemplateMarketOptionsSpotOptions{MaxPrice: fi.String(e.SpotPrice)}
+		if e.SpotDurationInMinutes != nil {
+			marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes
+		}
 		tf.MarketOptions = []*terraformLaunchTemplateMarketOptions{
 			{
 				MarketType:  fi.String("spot"),
-				SpotOptions: []*terraformLaunchTemplateMarketOptionsSpotOptions{{MaxPrice: fi.String(e.SpotPrice)}},
+				SpotOptions: []*terraformLaunchTemplateMarketOptionsSpotOptions{&marketSpotOptions},
 			},
 		}
 	}

upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go

@@ -35,6 +35,7 @@ func TestLaunchTemplateTerraformRender(t *testing.T) {
 				InstanceMonitoring:     fi.Bool(true),
 				InstanceType:           fi.String("t2.medium"),
 				SpotPrice:              "0.1",
+				SpotDurationInMinutes:  fi.Int64(60),
 				RootVolumeOptimization: fi.Bool(true),
 				RootVolumeIops:         fi.Int64(100),
 				RootVolumeSize:         fi.Int64(64),
@@ -72,7 +73,8 @@ resource "aws_launch_template" "test" {
     market_type = "spot"
 
     spot_options = {
-      max_price = "0.1"
+      block_duration_minutes = 60
+      max_price              = "0.1"
     }
   }