diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index 40a6bf4dcd57f..46977bc39e377 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -78,6 +78,8 @@ var ( Spotinst = New("Spotinst", Bool(false)) // SpotinstOcean toggles the use of Spotinst Ocean instance group implementation. SpotinstOcean = New("SpotinstOcean", Bool(false)) + // SpotinstHybrid toggles between hybrid and full instance group implementations. + SpotinstHybrid = New("SpotinstHybrid", Bool(false)) // VPCSkipEnableDNSSupport if set will make that a VPC does not need DNSSupport enabled. VPCSkipEnableDNSSupport = New("VPCSkipEnableDNSSupport", Bool(false)) // VSphereCloudProvider enables the vsphere cloud provider diff --git a/pkg/model/awsmodel/BUILD.bazel b/pkg/model/awsmodel/BUILD.bazel index df24fc207282b..a67667c2d65cc 100644 --- a/pkg/model/awsmodel/BUILD.bazel +++ b/pkg/model/awsmodel/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//pkg/featureflag:go_default_library", "//pkg/model:go_default_library", "//pkg/model/defaults:go_default_library", + "//pkg/model/spotinstmodel:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awstasks:go_default_library", "//upup/pkg/fi/fitasks:go_default_library", diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index f56018b960b12..d0ff4ff319ecc 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -20,9 +20,12 @@ import ( "fmt" "strings" + "k8s.io/klog" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model" "k8s.io/kops/pkg/model/defaults" + "k8s.io/kops/pkg/model/spotinstmodel" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" @@ -52,6 +55,13 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { for _, ig := range b.InstanceGroups { name := b.AutoscalingGroupName(ig) + if featureflag.SpotinstHybrid.Enabled() { + if spotinstmodel.ManageInstanceGroup(ig) { + klog.V(2).Infof("Skipping instance group: %q", name) + continue + } + } + // @check if his instancegroup is backed by a fleet and overide with a launch template task, err := func() (fi.Task, error) { switch UseLaunchTemplate(ig) { diff --git a/pkg/model/spotinstmodel/BUILD.bazel b/pkg/model/spotinstmodel/BUILD.bazel index c334778d41558..f95b8085103e7 100644 --- a/pkg/model/spotinstmodel/BUILD.bazel +++ b/pkg/model/spotinstmodel/BUILD.bazel @@ -9,7 +9,6 @@ go_library( "//pkg/apis/kops:go_default_library", "//pkg/featureflag:go_default_library", "//pkg/model:go_default_library", - "//pkg/model/awsmodel:go_default_library", "//pkg/model/defaults:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awstasks:go_default_library", diff --git a/pkg/model/spotinstmodel/instance_group.go b/pkg/model/spotinstmodel/instance_group.go index 164f04f5e2d9e..2ee90360cec84 100644 --- a/pkg/model/spotinstmodel/instance_group.go +++ b/pkg/model/spotinstmodel/instance_group.go @@ -25,7 +25,6 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model" - "k8s.io/kops/pkg/model/awsmodel" "k8s.io/kops/pkg/model/defaults" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" @@ -34,6 +33,10 @@ import ( ) const ( + // InstanceGroupLabelManaged is the metadata label used on the instance + // group to specify that the Spotinst provider should be used to upon creation. + InstanceGroupLabelManaged = "spotinst.io/managed" + // InstanceGroupLabelSpotPercentage is the metadata label used on the // instance group to specify the percentage of Spot instances that // should spin up from the target capacity. @@ -86,7 +89,7 @@ const ( // InstanceGroupModelBuilder configures InstanceGroup objects type InstanceGroupModelBuilder struct { - *awsmodel.AWSModelContext + *model.KopsModelContext BootstrapScript *model.BootstrapScript Lifecycle *fi.Lifecycle @@ -100,8 +103,16 @@ func (b *InstanceGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { var err error for _, ig := range b.InstanceGroups { - klog.V(2).Infof("Building instance group: %q", b.AutoscalingGroupName(ig)) + name := b.AutoscalingGroupName(ig) + if featureflag.SpotinstHybrid.Enabled() { + if !ManageInstanceGroup(ig) { + klog.V(2).Infof("Skipping instance group: %q", name) + continue + } + } + + klog.V(2).Infof("Building instance group: %q", name) switch ig.Spec.Role { // Create both Master and Bastion instance groups as Elastigroups. @@ -582,7 +593,7 @@ func (b *InstanceGroupModelBuilder) buildRootVolumeOpts(ig *kops.InstanceGroup) { typ := fi.StringValue(ig.Spec.RootVolumeType) if typ == "" { - typ = awsmodel.DefaultVolumeType + typ = "gp2" } opts.Type = fi.String(typ) } @@ -791,3 +802,11 @@ func defaultSpotPercentage(ig *kops.InstanceGroup) *float64 { return &percentage } + +// ManageInstanceGroup indicates whether the instance group labeled with +// a metadata label `spotinst.io/managed` which means the Spotinst provider +// should be used to upon creation if the `SpotinstHybrid` feature flag is on. +func ManageInstanceGroup(ig *kops.InstanceGroup) bool { + managed, _ := strconv.ParseBool(ig.ObjectMeta.Labels[InstanceGroupLabelManaged]) + return managed +} diff --git a/pkg/resources/aws/aws.go b/pkg/resources/aws/aws.go index 061f17ce70bb2..45ff62512b39d 100644 --- a/pkg/resources/aws/aws.go +++ b/pkg/resources/aws/aws.go @@ -63,6 +63,7 @@ func ListResourcesAWS(cloud awsup.AWSCloud, clusterName string) (map[string]*res //ListCloudFormationStacks, // EC2 + ListAutoScalingGroups, ListInstances, ListKeypairs, ListSecurityGroups, @@ -88,9 +89,6 @@ func ListResourcesAWS(cloud awsup.AWSCloud, clusterName string) (map[string]*res if featureflag.Spotinst.Enabled() { // Spotinst resources listFunctions = append(listFunctions, ListSpotinstResources) - } else { - // AutoScaling Groups - listFunctions = append(listFunctions, ListAutoScalingGroups) } for _, fn := range listFunctions { diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 0f6d40e34fdff..95f8b9241b0b9 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -723,24 +723,32 @@ func (c *ApplyClusterCmd) Run() error { } switch kops.CloudProviderID(cluster.Spec.CloudProvider) { case kops.CloudProviderAWS: - awsModelContext := &awsmodel.AWSModelContext{ - KopsModelContext: modelContext, - } + { + awsModelContext := &awsmodel.AWSModelContext{ + KopsModelContext: modelContext, + } - if featureflag.Spotinst.Enabled() { - l.Builders = append(l.Builders, &spotinstmodel.InstanceGroupModelBuilder{ + awsModelBuilder := &awsmodel.AutoscalingGroupModelBuilder{ AWSModelContext: awsModelContext, BootstrapScript: bootstrapScriptBuilder, Lifecycle: &clusterLifecycle, SecurityLifecycle: &securityLifecycle, - }) - } else { - l.Builders = append(l.Builders, &awsmodel.AutoscalingGroupModelBuilder{ - AWSModelContext: awsModelContext, - BootstrapScript: bootstrapScriptBuilder, - Lifecycle: &clusterLifecycle, - SecurityLifecycle: &securityLifecycle, - }) + } + + if featureflag.Spotinst.Enabled() { + l.Builders = append(l.Builders, &spotinstmodel.InstanceGroupModelBuilder{ + KopsModelContext: modelContext, + BootstrapScript: bootstrapScriptBuilder, + Lifecycle: &clusterLifecycle, + SecurityLifecycle: &securityLifecycle, + }) + + if featureflag.SpotinstHybrid.Enabled() { + l.Builders = append(l.Builders, awsModelBuilder) + } + } else { + l.Builders = append(l.Builders, awsModelBuilder) + } } case kops.CloudProviderDO: doModelContext := &domodel.DOModelContext{ diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index 834b7e4b70f93..7066ca35b2e5a 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -339,6 +339,12 @@ func NewEC2Filter(name string, values ...string) *ec2.Filter { // DeleteGroup deletes an aws autoscaling group func (c *awsCloudImplementation) DeleteGroup(g *cloudinstances.CloudInstanceGroup) error { if c.spotinst != nil { + if featureflag.SpotinstHybrid.Enabled() { + if _, ok := g.Raw.(*autoscaling.Group); ok { + return deleteGroup(c, g) + } + } + return spotinst.DeleteInstanceGroup(c.spotinst, g) } @@ -403,6 +409,12 @@ func deleteGroup(c AWSCloud, g *cloudinstances.CloudInstanceGroup) error { // DeleteInstance deletes an aws instance func (c *awsCloudImplementation) DeleteInstance(i *cloudinstances.CloudInstanceGroupMember) error { if c.spotinst != nil { + if featureflag.SpotinstHybrid.Enabled() { + if _, ok := i.CloudInstanceGroup.Raw.(*autoscaling.Group); ok { + return deleteInstance(c, i) + } + } + return spotinst.DeleteInstance(c.spotinst, i) } @@ -434,8 +446,23 @@ func deleteInstance(c AWSCloud, i *cloudinstances.CloudInstanceGroupMember) erro // GetCloudGroups returns a groups of instances that back a kops instance groups func (c *awsCloudImplementation) GetCloudGroups(cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, warnUnmatched bool, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) { if c.spotinst != nil { - return spotinst.GetCloudGroups(c.spotinst, cluster, - instancegroups, warnUnmatched, nodes) + sgroups, err := spotinst.GetCloudGroups(c.spotinst, cluster, instancegroups, warnUnmatched, nodes) + if err != nil { + return nil, err + } + + if featureflag.SpotinstHybrid.Enabled() { + agroups, err := getCloudGroups(c, cluster, instancegroups, warnUnmatched, nodes) + if err != nil { + return nil, err + } + + for name, group := range agroups { + sgroups[name] = group + } + } + + return sgroups, nil } return getCloudGroups(c, cluster, instancegroups, warnUnmatched, nodes)