From 5afd4c9a7284e3fa72f0a56632684775c56bc2d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20M=C4=83gheru=C8=99an-Stanciu=20=40magheru=5Fsan?= Date: Fri, 7 Sep 2018 12:01:03 +0200 Subject: [PATCH] Implement support for handling instance termination protection (#275) * Implement support for handling instance termination protection - determine instance termination protection only when we are about to replace the on-demand instances, to avoid unnecessary API calls when scanning instances on all Lambda function runs - write tests for instance termination protection and scale-in protection - define instanceMap type as map[string]*instance --- .../stacks/AutoSpotting/template.yaml | 7 +- core/autoscaling.go | 50 +- core/autoscaling_test.go | 714 ++++++++++++++++-- core/instance.go | 41 +- core/instance_test.go | 50 +- core/mock_test.go | 20 +- core/region.go | 35 +- core/region_test.go | 140 ++++ .../autospotting/autospotting-policy.json | 7 +- 9 files changed, 909 insertions(+), 155 deletions(-) diff --git a/cloudformation/stacks/AutoSpotting/template.yaml b/cloudformation/stacks/AutoSpotting/template.yaml index 37a1fe06..844e33ad 100644 --- a/cloudformation/stacks/AutoSpotting/template.yaml +++ b/cloudformation/stacks/AutoSpotting/template.yaml @@ -218,21 +218,22 @@ Statement: - Action: + - "autoscaling:AttachInstances" - "autoscaling:DescribeAutoScalingGroups" - "autoscaling:DescribeLaunchConfigurations" - - "autoscaling:AttachInstances" + - "autoscaling:DescribeTags" - "autoscaling:DetachInstances" - "autoscaling:TerminateInstanceInAutoScalingGroup" - - "autoscaling:DescribeTags" - "autoscaling:UpdateAutoScalingGroup" - "ec2:CreateTags" + - "ec2:DescribeInstanceAttribute" - "ec2:DescribeInstances" - "ec2:DescribeRegions" - "ec2:DescribeSpotPriceHistory" - "ec2:RunInstances" - "ec2:TerminateInstances" - - "iam:PassRole" - "iam:CreateServiceLinkedRole" + - "iam:PassRole" - "logs:CreateLogGroup" - "logs:CreateLogStream" - "logs:PutLogEvents" diff --git a/core/autoscaling.go b/core/autoscaling.go index ed8e2269..4c737eff 100644 --- a/core/autoscaling.go +++ b/core/autoscaling.go @@ -334,12 +334,11 @@ func (a *autoScalingGroup) process() { if spotInstance == nil { logger.Println("No spot instances were found for ", a.name) - // find any given on-demand instance and try to replace it with a spot one - onDemandInstance := a.getAnyOnDemandInstance() + onDemandInstance := a.getAnyUnprotectedOnDemandInstance() if onDemandInstance == nil { logger.Println(a.region.name, a.name, - "No running on-demand instances were found, nothing to do here...") + "No running unprotected on-demand instances were found, nothing to do here...") return } @@ -385,10 +384,8 @@ func (a *autoScalingGroup) scanInstances() instances { } i.asg, i.region = a, a.region - if inst.ProtectedFromScaleIn == nil { - i.protected = false - } else { - i.protected = *inst.ProtectedFromScaleIn + if inst.ProtectedFromScaleIn != nil { + i.protected = i.protected || *inst.ProtectedFromScaleIn } if i.isSpot() { @@ -426,8 +423,7 @@ func (a *autoScalingGroup) replaceOnDemandInstanceWithSpot( logger.Println(a.name, spotInstanceID, "is in the availability zone", *az, "looking for an on-demand instance there") - // find an on-demand instance from the same AZ as our spot instance - odInst := a.getOnDemandInstanceInAZ(az) + odInst := a.getUnprotectedOnDemandInstanceInAZ(az) if odInst == nil { logger.Println(a.name, "found no on-demand instances that could be", @@ -463,14 +459,11 @@ func (a *autoScalingGroup) replaceOnDemandInstanceWithSpot( // group. It can also filter by AZ and Lifecycle. func (a *autoScalingGroup) getInstance( availabilityZone *string, - onDemand bool, any bool) *instance { - - var retI *instance + onDemand bool, + considerInstanceProtection bool, +) *instance { for i := range a.instances.instances() { - if retI != nil { - continue - } // instance is running if *i.State.Name == ec2.InstanceStateNameRunning { @@ -478,26 +471,33 @@ func (a *autoScalingGroup) getInstance( // the InstanceLifecycle attribute is non-nil only for spot instances, // where it contains the value "spot", if we're looking for on-demand // instances only, then we have to skip the current instance. - if !any && - (onDemand && i.isSpot() || - (!onDemand && !i.isSpot())) { + if (onDemand && i.isSpot()) || (!onDemand && !i.isSpot()) { + debug.Println(a.name, "skipping instance", *i.InstanceId, + "having different lifecycle than what we're looking for") continue } - if i.isProtected() { + + if considerInstanceProtection && (i.isProtectedFromScaleIn() || i.isProtectedFromTermination()) { + debug.Println(a.name, "skipping protected instance", *i.InstanceId) continue } - if (availabilityZone != nil) && - (*availabilityZone != *i.Placement.AvailabilityZone) { + + if (availabilityZone != nil) && (*availabilityZone != *i.Placement.AvailabilityZone) { + debug.Println(a.name, "skipping instance", *i.InstanceId, + "placed in a different AZ than what we're looking for") continue } - retI = i + return i } } - return retI + return nil } -func (a *autoScalingGroup) getOnDemandInstanceInAZ(az *string) *instance { - return a.getInstance(az, true, false) +func (a *autoScalingGroup) getUnprotectedOnDemandInstanceInAZ(az *string) *instance { + return a.getInstance(az, true, true) +} +func (a *autoScalingGroup) getAnyUnprotectedOnDemandInstance() *instance { + return a.getInstance(nil, true, true) } func (a *autoScalingGroup) getAnyOnDemandInstance() *instance { diff --git a/core/autoscaling_test.go b/core/autoscaling_test.go index 8e07d421..e9d536f9 100644 --- a/core/autoscaling_test.go +++ b/core/autoscaling_test.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/davecgh/go-spew/spew" ) func TestGetTagValue(t *testing.T) { @@ -147,7 +148,7 @@ func TestLoadConfOnDemand(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}}, ), maxSize: aws.Int64(10), @@ -166,7 +167,7 @@ func TestLoadConfOnDemand(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -188,7 +189,7 @@ func TestLoadConfOnDemand(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -210,7 +211,7 @@ func TestLoadConfOnDemand(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -280,7 +281,7 @@ func TestLoadConfOnDemand(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -302,7 +303,7 @@ func TestLoadConfOnDemand(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -328,7 +329,7 @@ func TestLoadConfOnDemand(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -355,7 +356,7 @@ func TestLoadConfOnDemand(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -458,7 +459,7 @@ func TestLoadDefaultConf(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -476,7 +477,7 @@ func TestLoadDefaultConf(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -494,7 +495,7 @@ func TestLoadDefaultConf(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -524,7 +525,7 @@ func TestLoadDefaultConf(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -542,7 +543,7 @@ func TestLoadDefaultConf(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -560,7 +561,7 @@ func TestLoadDefaultConf(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -579,7 +580,7 @@ func TestLoadDefaultConf(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -676,7 +677,7 @@ func TestLoadConfigFromTags(t *testing.T) { }, }, asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -930,7 +931,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has no 'running' instance but has some", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, @@ -948,7 +949,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has no 'running' spot instances but has some", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -973,7 +974,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has no 'running' on-demand instances but has some", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -998,7 +999,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has no 'running' on-demand instances in the AZ", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1023,7 +1024,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has some 'running' on-demand instances in the AZ", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1048,7 +1049,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has no 'running' spot instances in the AZ", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1073,7 +1074,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has some 'running' spot instances in any AZ", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1098,7 +1099,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has no 'running' spot instances in any AZ", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameShuttingDown)}, @@ -1123,7 +1124,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has some 'running' on-demand instances in any AZ", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1148,7 +1149,7 @@ func TestAlreadyRunningInstanceCount(t *testing.T) { {name: "ASG has no 'running' on-demand instances in any AZ", asgName: "test-asg", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1212,7 +1213,7 @@ func TestNeedReplaceOnDemandInstances(t *testing.T) { }, {name: "ASG has no instance running - 1 on-demand required", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameShuttingDown)}, @@ -1235,7 +1236,7 @@ func TestNeedReplaceOnDemandInstances(t *testing.T) { }, {name: "ASG has no instance running - 0 on-demand required", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameShuttingDown)}, @@ -1258,7 +1259,7 @@ func TestNeedReplaceOnDemandInstances(t *testing.T) { }, {name: "ASG has not the required on-demand running", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ InstanceId: aws.String("id-1"), @@ -1296,7 +1297,7 @@ func TestNeedReplaceOnDemandInstances(t *testing.T) { }, {name: "ASG has just enough on-demand instances running", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1319,7 +1320,7 @@ func TestNeedReplaceOnDemandInstances(t *testing.T) { }, {name: "ASG has only one remaining instance, less than enough on-demand", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1335,7 +1336,7 @@ func TestNeedReplaceOnDemandInstances(t *testing.T) { }, {name: "ASG has more than enough on-demand instances running but not desired capacity", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1358,7 +1359,7 @@ func TestNeedReplaceOnDemandInstances(t *testing.T) { }, {name: "ASG has more than enough on-demand instances running and desired capacity", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, @@ -1431,7 +1432,7 @@ func TestDetachAndTerminateOnDemandInstance(t *testing.T) { }{ {name: "no err during detach nor terminate", instancesASG: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "1": { Instance: &ec2.Instance{ InstanceId: aws.String("1"), @@ -1459,7 +1460,7 @@ func TestDetachAndTerminateOnDemandInstance(t *testing.T) { }, {name: "err during detach not during terminate", instancesASG: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "1": { Instance: &ec2.Instance{ InstanceId: aws.String("1"), @@ -1487,7 +1488,7 @@ func TestDetachAndTerminateOnDemandInstance(t *testing.T) { }, {name: "no err during detach but error during terminate", instancesASG: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "1": { Instance: &ec2.Instance{ InstanceId: aws.String("1"), @@ -1515,7 +1516,7 @@ func TestDetachAndTerminateOnDemandInstance(t *testing.T) { }, {name: "errors during detach and terminate", instancesASG: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "1": { Instance: &ec2.Instance{ InstanceId: aws.String("1"), @@ -1803,12 +1804,12 @@ func TestScanInstances(t *testing.T) { name string ec2ASG *autoscaling.Group regionInstances *region - expectedInstances map[string]*instance + expectedInstances instanceMap }{ {name: "multiple instances to scan", regionInstances: ®ion{ instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "1": { Instance: &ec2.Instance{ InstanceId: aws.String("1"), @@ -1859,7 +1860,7 @@ func TestScanInstances(t *testing.T) { {InstanceId: aws.String("3")}, }, }, - expectedInstances: map[string]*instance{ + expectedInstances: instanceMap{ "1": { Instance: &ec2.Instance{ InstanceId: aws.String("1"), @@ -1936,11 +1937,13 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { az *string expected *instance }{ - {name: "ASG has no 'running' instance in AZ", + { + name: "ASG has no 'running' instance in the current AZ but only in other AZs", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -1948,6 +1951,7 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { }, "spot-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -1955,6 +1959,7 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { }, "ondemand-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -1962,20 +1967,35 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { }, "ondemand-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, }, }, ), - az: aws.String("1c"), + az: aws.String("1c"), + expected: nil, }, - {name: "ASG has 'running' instance in AZ", + { + name: "ASG has 'running' instance in AZ", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -1983,6 +2003,7 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { }, "spot-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -1990,6 +2011,7 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { }, "ondemand-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -1997,27 +2019,54 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { }, "ondemand-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, }, }, ), az: aws.String("1b"), expected: &instance{ Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, }, }, - {name: "ASG has no instance in AZ", + + { + name: "ASG has 'running' instance in AZ ad we we get error when trying to determine termination protection", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -2025,6 +2074,7 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { }, "spot-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -2032,6 +2082,7 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { }, "ondemand-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -2039,18 +2090,212 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { }, "ondemand-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diaerr: errors.New("error when determining instance termination protection"), + }, + }, + }, }, }, ), - az: aws.String("2a"), + az: aws.String("1b"), + expected: &instance{ + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String(""), + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diaerr: errors.New("error when determining instance termination protection"), + }, + }, + }, + }, }, - {name: "ASG has no instance at all", + + { + name: "ASG has 'running' but protected from termination instance in AZ", + asgInstances: makeInstancesWithCatalog( + instanceMap{ + "spot-stopped": { + Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, + InstanceLifecycle: aws.String("spot"), + }, + }, + "spot-running": { + Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String("spot"), + }, + }, + "ondemand-stopped": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, + InstanceLifecycle: aws.String(""), + }, + }, + "ondemand-running": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String(""), + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + }, + }, + }, + }, + }, + }, + ), + az: aws.String("1b"), + expected: nil, + }, + { + name: "ASG has 'running' but protected from ASG scale-in instance in AZ", + asgInstances: makeInstancesWithCatalog( + instanceMap{ + "spot-stopped": { + Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, + InstanceLifecycle: aws.String("spot"), + }, + }, + "spot-running": { + Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String("spot"), + }, + }, + "ondemand-stopped": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, + InstanceLifecycle: aws.String(""), + }, + }, + "ondemand-running": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String(""), + }, + asg: &autoScalingGroup{ + Group: &autoscaling.Group{ + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("ondemand-running"), + ProtectedFromScaleIn: aws.Bool(true), + }, + }, + }, + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + // not protected from termination + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, + }, + }, + ), + az: aws.String("1b"), + expected: nil, + }, + + { + name: "ASG has no instance in AZ", + asgInstances: makeInstancesWithCatalog( + instanceMap{ + "spot-stopped": { + Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, + InstanceLifecycle: aws.String("spot"), + }, + }, + "spot-running": { + Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String("spot"), + }, + }, + "ondemand-stopped": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, + InstanceLifecycle: aws.String(""), + }, + }, + "ondemand-running": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String(""), + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, + }, + }, + ), + az: aws.String("2a"), + expected: nil, + }, + { + name: "ASG has no instance at all", asgInstances: makeInstances(), az: aws.String("1a"), + expected: nil, }, } @@ -2059,11 +2304,12 @@ func TestGetOnDemandInstanceInAZ(t *testing.T) { a := &autoScalingGroup{ instances: tt.asgInstances, } - returnedInstance := a.getOnDemandInstanceInAZ(tt.az) + returnedInstance := a.getUnprotectedOnDemandInstanceInAZ(tt.az) if !reflect.DeepEqual(returnedInstance, tt.expected) { - t.Errorf("getOnDemandInstanceInAZ received: %+v, expected: %+v", - returnedInstance, - tt.expected) + t.Errorf("%s: getOnDemandInstanceInAZ \nreceived: %+v,\n expected: %+v", + tt.name, + spew.Sdump(returnedInstance), + spew.Sdump(tt.expected)) } }) } @@ -2077,9 +2323,10 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }{ {name: "ASG has no 'running' OnDemand instance", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -2087,6 +2334,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, "spot-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -2094,6 +2342,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, "ondemand-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -2105,9 +2354,10 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, {name: "ASG has one 'running' OnDemand instance", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -2115,6 +2365,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, "spot-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -2122,6 +2373,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, "ondemand-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -2129,6 +2381,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, "ondemand-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), @@ -2138,6 +2391,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { ), expected: []*instance{{ Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), @@ -2146,9 +2400,10 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, {name: "ASG has multiple 'running' OnDemand instances", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -2156,6 +2411,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, "spot-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -2163,6 +2419,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, "ondemand-running1": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running1"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -2170,6 +2427,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, "ondemand-running2": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running2"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), @@ -2180,6 +2438,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { expected: []*instance{ { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running2"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), @@ -2187,6 +2446,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running1"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -2195,7 +2455,7 @@ func TestGetAnyOnDemandInstance(t *testing.T) { }, }, {name: "ASG has no instance at all", - asgInstances: makeInstancesWithCatalog(map[string]*instance{}), + asgInstances: makeInstancesWithCatalog(instanceMap{}), expected: []*instance{}, }, } @@ -2236,9 +2496,10 @@ func TestGetAnySpotInstance(t *testing.T) { }{ {name: "ASG has no 'running' Spot instance", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -2246,6 +2507,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, "ondemand-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("onemand-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), @@ -2253,6 +2515,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, "ondemand-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("onemand-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -2264,9 +2527,10 @@ func TestGetAnySpotInstance(t *testing.T) { }, {name: "ASG has one 'running' Spot instance", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -2274,6 +2538,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, "spot-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -2281,6 +2546,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, "ondemand-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -2288,6 +2554,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, "ondemand-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), @@ -2297,6 +2564,7 @@ func TestGetAnySpotInstance(t *testing.T) { ), expected: []*instance{{ Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -2305,9 +2573,10 @@ func TestGetAnySpotInstance(t *testing.T) { }, {name: "ASG has multiple 'running' Spot instances", asgInstances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-running1": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running1"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -2315,6 +2584,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, "spot-running2": { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running2"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -2322,6 +2592,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, "ondemand-stopped": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-stopped"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameStopped)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, InstanceLifecycle: aws.String(""), @@ -2329,6 +2600,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, "ondemand-running": { Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-running"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String(""), @@ -2339,6 +2611,7 @@ func TestGetAnySpotInstance(t *testing.T) { expected: []*instance{ { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running1"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1a")}, InstanceLifecycle: aws.String("spot"), @@ -2346,6 +2619,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, { Instance: &ec2.Instance{ + InstanceId: aws.String("spot-running2"), State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, InstanceLifecycle: aws.String("spot"), @@ -2354,7 +2628,7 @@ func TestGetAnySpotInstance(t *testing.T) { }, }, {name: "ASG has no instance at all", - asgInstances: makeInstancesWithCatalog(map[string]*instance{}), + asgInstances: makeInstancesWithCatalog(instanceMap{}), expected: []*instance{}, }, } @@ -2405,7 +2679,7 @@ func TestReplaceOnDemandInstanceWithSpot(t *testing.T) { DesiredCapacity: aws.Int64(2), }, instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "ondemand-stopped": { Instance: &ec2.Instance{ InstanceId: aws.String("ondemand-stopped"), @@ -2440,6 +2714,12 @@ func TestReplaceOnDemandInstanceWithSpot(t *testing.T) { ec2: &mockEC2{ tio: nil, tierr: nil, + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + }, + diaerr: nil, }, }, }, @@ -2464,7 +2744,7 @@ func TestReplaceOnDemandInstanceWithSpot(t *testing.T) { }, }, instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-running": { Instance: &ec2.Instance{ InstanceId: aws.String("spot-running"), @@ -2477,6 +2757,12 @@ func TestReplaceOnDemandInstanceWithSpot(t *testing.T) { ec2: &mockEC2{ tio: nil, tierr: nil, + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + }, + diaerr: nil, }, }, }, @@ -2529,7 +2815,7 @@ func TestReplaceOnDemandInstanceWithSpot(t *testing.T) { DesiredCapacity: aws.Int64(2), }, instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "ondemand-running": { Instance: &ec2.Instance{ InstanceId: aws.String("ondemand-running"), @@ -2542,6 +2828,12 @@ func TestReplaceOnDemandInstanceWithSpot(t *testing.T) { ec2: &mockEC2{ tio: nil, tierr: nil, + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + }, + diaerr: nil, }, }, }, @@ -2566,7 +2858,7 @@ func TestReplaceOnDemandInstanceWithSpot(t *testing.T) { }, }, instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-running": { Instance: &ec2.Instance{ InstanceId: aws.String("spot-running"), @@ -2612,7 +2904,7 @@ func TestReplaceOnDemandInstanceWithSpot(t *testing.T) { }, }, instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-running": { Instance: &ec2.Instance{ InstanceId: aws.String("spot-running"), @@ -2667,7 +2959,7 @@ func TestReplaceOnDemandInstanceWithSpot(t *testing.T) { }, }, instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "spot-running": { Instance: &ec2.Instance{ InstanceId: aws.String("spot-running"), @@ -3178,7 +3470,7 @@ func Test_autoScalingGroup_findUnattachedInstanceLaunchedForThisASG(t *testing.T name: "mygroup", region: ®ion{ instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ InstanceId: aws.String("id-1"), @@ -3197,7 +3489,7 @@ func Test_autoScalingGroup_findUnattachedInstanceLaunchedForThisASG(t *testing.T name: "mygroup", region: ®ion{ instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ InstanceId: aws.String("id-1"), @@ -3238,7 +3530,7 @@ func Test_autoScalingGroup_findUnattachedInstanceLaunchedForThisASG(t *testing.T region: ®ion{ instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ InstanceId: aws.String("id-1"), @@ -3291,3 +3583,285 @@ func Test_autoScalingGroup_findUnattachedInstanceLaunchedForThisASG(t *testing.T }) } } + +func Test_autoScalingGroup_getAnyUnprotectedOnDemandInstance(t *testing.T) { + tests := []struct { + name string + asgInstances instances + + want *instance + }{ + { + name: "ASG has unprotected and protected from scale-in instance", + asgInstances: makeInstancesWithCatalog( + instanceMap{ + "ondemand-unprotected": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-unprotected"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, + InstanceLifecycle: aws.String(""), + }, + asg: &autoScalingGroup{ + Group: &autoscaling.Group{ + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("ondemand-unprotected"), + ProtectedFromScaleIn: aws.Bool(false), + }, + }, + }, + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + // not protected from termination + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, + }, + "ondemand-protected-scalein": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-protected-scalein"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String(""), + }, + asg: &autoScalingGroup{ + Group: &autoscaling.Group{ + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("ondemand-protected-scalein"), + ProtectedFromScaleIn: aws.Bool(true), + }, + }, + }, + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + // not protected from termination + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, + }, + }, + ), + want: &instance{ + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-unprotected"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, + InstanceLifecycle: aws.String(""), + }, + asg: &autoScalingGroup{ + Group: &autoscaling.Group{ + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("ondemand-unprotected"), + ProtectedFromScaleIn: aws.Bool(false), + }, + }, + }, + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + // not protected from termination + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, + }, + }, + + { + name: "ASG has unprotected and protected from termination instance", + asgInstances: makeInstancesWithCatalog( + instanceMap{ + "ondemand-unprotected": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-unprotected"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, + InstanceLifecycle: aws.String(""), + }, + asg: &autoScalingGroup{ + Group: &autoscaling.Group{ + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("ondemand-unprotected"), + ProtectedFromScaleIn: aws.Bool(false), + }, + }, + }, + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + // not protected from termination + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, + }, + "ondemand-protected-termination": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-protected-termination"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String(""), + }, + asg: &autoScalingGroup{ + Group: &autoscaling.Group{ + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("ondemand-protected-termination"), + ProtectedFromScaleIn: aws.Bool(false), + }, + }, + }, + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + // not protected from termination + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + }, + }, + }, + }, + }, + }, + ), + want: &instance{ + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-unprotected"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, + InstanceLifecycle: aws.String(""), + }, + asg: &autoScalingGroup{ + Group: &autoscaling.Group{ + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("ondemand-unprotected"), + ProtectedFromScaleIn: aws.Bool(false), + }, + }, + }, + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + // not protected from termination + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, + }, + }, + { + name: "ASG has no unprotected instances in AZ", + asgInstances: makeInstancesWithCatalog( + instanceMap{ + "ondemand-protected-scale-in": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-protected-scale-in"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1c")}, + InstanceLifecycle: aws.String(""), + }, + asg: &autoScalingGroup{ + Group: &autoscaling.Group{ + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("ondemand-protected-scale-in"), + ProtectedFromScaleIn: aws.Bool(true), + }, + }, + }, + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(false), + }, + }, + }, + }, + }, + }, + "ondemand-protected-termination": { + Instance: &ec2.Instance{ + InstanceId: aws.String("ondemand-protected-termination"), + State: &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)}, + Placement: &ec2.Placement{AvailabilityZone: aws.String("1b")}, + InstanceLifecycle: aws.String(""), + }, + asg: &autoScalingGroup{ + Group: &autoscaling.Group{ + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("ondemand-protected-termination"), + ProtectedFromScaleIn: aws.Bool(false), + }, + }, + }, + }, + region: ®ion{ + services: connections{ + ec2: mockEC2{ + diao: &ec2.DescribeInstanceAttributeOutput{ + DisableApiTermination: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + }, + }, + }, + }, + }, + }, + ), + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &autoScalingGroup{ + name: tt.name, + instances: tt.asgInstances, + } + if got := a.getAnyUnprotectedOnDemandInstance(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("autoScalingGroup.getAnyUnprotectedOnDemandInstance() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/core/instance.go b/core/instance.go index 3e3f7ebc..ba480d99 100644 --- a/core/instance.go +++ b/core/instance.go @@ -16,9 +16,11 @@ import ( // The key in this map is the instance ID, useful for quick retrieval of // instance attributes. +type instanceMap map[string]*instance + type instanceManager struct { sync.RWMutex - catalog map[string]*instance + catalog instanceMap } type instances interface { @@ -32,10 +34,10 @@ type instances interface { } func makeInstances() instances { - return &instanceManager{catalog: map[string]*instance{}} + return &instanceManager{catalog: instanceMap{}} } -func makeInstancesWithCatalog(catalog map[string]*instance) instances { +func makeInstancesWithCatalog(catalog instanceMap) instances { return &instanceManager{catalog: catalog} } @@ -44,10 +46,9 @@ func (is *instanceManager) dump() string { defer is.RUnlock() return spew.Sdump(is.catalog) } - func (is *instanceManager) make() { is.Lock() - is.catalog = make(map[string]*instance) + is.catalog = make(instanceMap) is.Unlock() } @@ -134,8 +135,34 @@ func (i *instance) isSpot() bool { *i.InstanceLifecycle == "spot") } -func (i *instance) isProtected() bool { - return i.protected +func (i *instance) isProtectedFromTermination() bool { + + // determine and set the API termination protection field + diaRes, err := i.region.services.ec2.DescribeInstanceAttribute( + &ec2.DescribeInstanceAttributeInput{ + Attribute: aws.String("disableApiTermination"), + InstanceId: i.InstanceId, + }) + + if err == nil && + diaRes.DisableApiTermination != nil && + diaRes.DisableApiTermination.Value != nil { + return *diaRes.DisableApiTermination.Value + } + return false +} + +func (i *instance) isProtectedFromScaleIn() bool { + if i.asg == nil { + return false + } + + for _, inst := range i.asg.Instances { + if *inst.InstanceId == *i.InstanceId { + return *inst.ProtectedFromScaleIn + } + } + return false } func (i *instance) canTerminate() bool { diff --git a/core/instance_test.go b/core/instance_test.go index a6c38594..b55bd739 100644 --- a/core/instance_test.go +++ b/core/instance_test.go @@ -14,7 +14,7 @@ import ( ) func TestMake(t *testing.T) { - expected := map[string]*instance{} + expected := instanceMap{} is := &instanceManager{} is.make() @@ -28,33 +28,33 @@ func TestMake(t *testing.T) { func TestAdd(t *testing.T) { tests := []struct { name string - catalog map[string]*instance - expected map[string]*instance + catalog instanceMap + expected instanceMap }{ {name: "map contains a nil pointer", - catalog: map[string]*instance{ + catalog: instanceMap{ "inst1": {Instance: &ec2.Instance{InstanceId: aws.String("1")}}, "inst2": nil, }, - expected: map[string]*instance{ + expected: instanceMap{ "1": {Instance: &ec2.Instance{InstanceId: aws.String("1")}}, }, }, {name: "map has 1 instance", - catalog: map[string]*instance{ + catalog: instanceMap{ "inst1": {Instance: &ec2.Instance{InstanceId: aws.String("1")}}, }, - expected: map[string]*instance{ + expected: instanceMap{ "1": {Instance: &ec2.Instance{InstanceId: aws.String("1")}}, }, }, {name: "map has several instances", - catalog: map[string]*instance{ + catalog: instanceMap{ "inst1": {Instance: &ec2.Instance{InstanceId: aws.String("1")}}, "inst2": {Instance: &ec2.Instance{InstanceId: aws.String("2")}}, "inst3": {Instance: &ec2.Instance{InstanceId: aws.String("3")}}, }, - expected: map[string]*instance{ + expected: instanceMap{ "1": {Instance: &ec2.Instance{InstanceId: aws.String("1")}}, "2": {Instance: &ec2.Instance{InstanceId: aws.String("2")}}, "3": {Instance: &ec2.Instance{InstanceId: aws.String("3")}}, @@ -79,12 +79,12 @@ func TestAdd(t *testing.T) { func TestGet(t *testing.T) { tests := []struct { name string - catalog map[string]*instance + catalog instanceMap idToGet string expected *instance }{ {name: "map contains the required instance", - catalog: map[string]*instance{ + catalog: instanceMap{ "inst1": {Instance: &ec2.Instance{InstanceId: aws.String("1")}}, "inst2": {Instance: &ec2.Instance{InstanceId: aws.String("2")}}, "inst3": {Instance: &ec2.Instance{InstanceId: aws.String("3")}}, @@ -93,7 +93,7 @@ func TestGet(t *testing.T) { expected: &instance{Instance: &ec2.Instance{InstanceId: aws.String("2")}}, }, {name: "catalog doesn't contain the instance", - catalog: map[string]*instance{ + catalog: instanceMap{ "inst1": {Instance: &ec2.Instance{InstanceId: aws.String("1")}}, "inst2": {Instance: &ec2.Instance{InstanceId: aws.String("2")}}, "inst3": {Instance: &ec2.Instance{InstanceId: aws.String("3")}}, @@ -119,7 +119,7 @@ func TestGet(t *testing.T) { func TestCount(t *testing.T) { tests := []struct { name string - catalog map[string]*instance + catalog instanceMap expected int }{ {name: "map is nil", @@ -127,17 +127,17 @@ func TestCount(t *testing.T) { expected: 0, }, {name: "map is empty", - catalog: map[string]*instance{}, + catalog: instanceMap{}, expected: 0, }, {name: "map has 1 instance", - catalog: map[string]*instance{ + catalog: instanceMap{ "id-1": {}, }, expected: 1, }, {name: "map has several instances", - catalog: map[string]*instance{ + catalog: instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -161,7 +161,7 @@ func TestCount(t *testing.T) { func TestCount64(t *testing.T) { tests := []struct { name string - catalog map[string]*instance + catalog instanceMap expected int64 }{ {name: "map is nil", @@ -169,17 +169,17 @@ func TestCount64(t *testing.T) { expected: 0, }, {name: "map is empty", - catalog: map[string]*instance{}, + catalog: instanceMap{}, expected: 0, }, {name: "map has 1 instance", - catalog: map[string]*instance{ + catalog: instanceMap{ "id-1": {}, }, expected: 1, }, {name: "map has several instances", - catalog: map[string]*instance{ + catalog: instanceMap{ "id-1": {}, "id-2": {}, "id-3": {}, @@ -719,7 +719,7 @@ func TestGetCheapestCompatibleSpotInstanceType(t *testing.T) { asg: &autoScalingGroup{ name: "test-asg", instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ InstanceId: aws.String("id-1"), @@ -793,7 +793,7 @@ func TestGetCheapestCompatibleSpotInstanceType(t *testing.T) { asg: &autoScalingGroup{ name: "test-asg", instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ InstanceId: aws.String("id-1"), @@ -868,7 +868,7 @@ func TestGetCheapestCompatibleSpotInstanceType(t *testing.T) { asg: &autoScalingGroup{ name: "test-asg", instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ InstanceId: aws.String("id-1"), @@ -943,7 +943,7 @@ func TestGetCheapestCompatibleSpotInstanceType(t *testing.T) { asg: &autoScalingGroup{ name: "test-asg", instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ InstanceId: aws.String("id-1"), @@ -1019,7 +1019,7 @@ func TestGetCheapestCompatibleSpotInstanceType(t *testing.T) { asg: &autoScalingGroup{ name: "test-asg", instances: makeInstancesWithCatalog( - map[string]*instance{ + instanceMap{ "id-1": { Instance: &ec2.Instance{ InstanceId: aws.String("id-1"), diff --git a/core/mock_test.go b/core/mock_test.go index 573edcd9..b5b74ed3 100644 --- a/core/mock_test.go +++ b/core/mock_test.go @@ -13,7 +13,7 @@ import ( ) func CheckErrors(t *testing.T, err error, expected error) { - if err != nil && !reflect.DeepEqual(err, expected) { + if err != nil && expected != nil && !reflect.DeepEqual(err, expected) { t.Errorf("Error received: '%v' expected '%v'", err.Error(), expected.Error()) } @@ -28,9 +28,16 @@ type mockEC2 struct { dspho *ec2.DescribeSpotPriceHistoryOutput dspherr error - // Error in DescribeInstancesPages + // DescribeInstancesOutput + dio *ec2.DescribeInstancesOutput + + // DescribeInstancesPages error diperr error + // DescribeInstanceAttribute + diao *ec2.DescribeInstanceAttributeOutput + diaerr error + // Terminate Instance tio *ec2.TerminateInstancesOutput tierr error @@ -44,8 +51,13 @@ func (m mockEC2) DescribeSpotPriceHistory(in *ec2.DescribeSpotPriceHistoryInput) return m.dspho, m.dspherr } -func (m mockEC2) DescribeInstancesPages(in *ec2.DescribeInstancesInput, fn func(*ec2.DescribeInstancesOutput, bool) bool) error { - return m.diperr +func (m mockEC2) DescribeInstancesPages(in *ec2.DescribeInstancesInput, f func(*ec2.DescribeInstancesOutput, bool) bool) error { + f(m.dio, true) + return nil +} + +func (m mockEC2) DescribeInstanceAttribute(in *ec2.DescribeInstanceAttributeInput) (*ec2.DescribeInstanceAttributeOutput, error) { + return m.diao, m.diaerr } func (m mockEC2) TerminateInstances(*ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) { diff --git a/core/region.go b/core/region.go index 19f2063a..35de8f95 100644 --- a/core/region.go +++ b/core/region.go @@ -136,6 +136,22 @@ func splitTagAndValue(value string) *Tag { return nil } +func (r *region) processDescribeInstancesPage(page *ec2.DescribeInstancesOutput, lastPage bool) bool { + logger.Println("Processing page of DescribeInstancesPages for", r.name) + debug.Println(page) + + if len(page.Reservations) > 0 && + page.Reservations[0].Instances != nil { + + for _, res := range page.Reservations { + for _, inst := range res.Instances { + r.addInstance(inst) + } + } + } + return true +} + func (r *region) scanInstances() error { svc := r.services.ec2 input := &ec2.DescribeInstancesInput{ @@ -152,26 +168,9 @@ func (r *region) scanInstances() error { r.instances = makeInstances() - pageNum := 0 err := svc.DescribeInstancesPages( input, - func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { - pageNum++ - logger.Println("Processing page", pageNum, "of DescribeInstancesPages for", r.name) - - debug.Println(page) - if len(page.Reservations) > 0 && - page.Reservations[0].Instances != nil { - - for _, res := range page.Reservations { - for _, inst := range res.Instances { - r.addInstance(inst) - } - } - } - return true - }, - ) + r.processDescribeInstancesPage) if err != nil { return err diff --git a/core/region_test.go b/core/region_test.go index b7e74ea5..2df1ccce 100644 --- a/core/region_test.go +++ b/core/region_test.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" "github.com/cristim/ec2-instances-info" + "github.com/davecgh/go-spew/spew" ) func Test_region_enabled(t *testing.T) { @@ -509,3 +510,142 @@ func TestFilterAsgs(t *testing.T) { }) } } + +func Test_region_scanInstances(t *testing.T) { + + tests := []struct { + name string + regionInfo *region + wantErr bool + wantInstances instances + }{ + { + name: "region with a single instance", + regionInfo: ®ion{ + name: "us-east-1", + conf: &Config{MinOnDemandNumber: 2}, + services: connections{ + ec2: mockEC2{ + diperr: nil, + dio: &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + { + InstanceId: aws.String("id-1"), + InstanceType: aws.String("typeX"), + }, + }, + }, + }, + }, + }, + }, + }, + wantErr: false, + wantInstances: makeInstancesWithCatalog( + instanceMap{ + "id-1": { + Instance: &ec2.Instance{ + InstanceId: aws.String("id-1"), + InstanceType: aws.String("typeX"), + }, + }, + }, + ), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := tt.regionInfo + err := r.scanInstances() + + if (err != nil) != tt.wantErr { + t.Errorf("region.scanInstances() error = %v, wantErr %v", err, tt.wantErr) + } + + for inst := range r.instances.instances() { + wantedInstance := tt.wantInstances.get(*inst.InstanceId).Instance + + if !reflect.DeepEqual(inst.Instance, wantedInstance) { + t.Errorf("region.scanInstances() \nreceived instance data: \n %+v\nexpected: \n %+v", + spew.Sdump(inst.Instance), spew.Sdump(wantedInstance)) + + } + } + + }) + } +} + +func Test_region_processDescribeInstancesPage(t *testing.T) { + type regionFields struct { + name string + instances instances + } + type args struct { + page *ec2.DescribeInstancesOutput + lastPage bool + } + tests := []struct { + name string + regionFields regionFields + args args + want bool + wantInstances instances + }{ + { + name: "region with a single instance", + regionFields: regionFields{ + name: "us-east-1", + instances: makeInstancesWithCatalog(instanceMap{}), + }, + args: args{ + page: &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{{ + InstanceId: aws.String("id-1"), + InstanceType: aws.String("typeX"), + }, + { + InstanceId: aws.String("id-2"), + InstanceType: aws.String("typeY"), + }, + }, + }, + }, + }, + lastPage: true, + }, + want: true, + wantInstances: makeInstancesWithCatalog( + instanceMap{ + "id-1": { + Instance: &ec2.Instance{ + InstanceId: aws.String("id-1"), + InstanceType: aws.String("typeX"), + }, + }, + "id-2": { + Instance: &ec2.Instance{ + InstanceId: aws.String("id-2"), + InstanceType: aws.String("typeY"), + }, + }, + }, + ), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := ®ion{ + name: tt.regionFields.name, + instances: tt.regionFields.instances, + } + if got := r.processDescribeInstancesPage(tt.args.page, tt.args.lastPage); got != tt.want { + t.Errorf("region.processDescribeInstancesPage() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/terraform/autospotting/autospotting-policy.json b/terraform/autospotting/autospotting-policy.json index 61b5bcf5..47d51a95 100644 --- a/terraform/autospotting/autospotting-policy.json +++ b/terraform/autospotting/autospotting-policy.json @@ -3,21 +3,22 @@ "Statement": [ { "Action": [ + "autoscaling:AttachInstances", "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeLaunchConfigurations", - "autoscaling:AttachInstances", + "autoscaling:DescribeTags", "autoscaling:DetachInstances", "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:DescribeTags", "autoscaling:UpdateAutoScalingGroup", "ec2:CreateTags", + "ec2:DescribeInstanceAttribute", "ec2:DescribeInstances", "ec2:DescribeRegions", "ec2:DescribeSpotPriceHistory", "ec2:RunInstances", "ec2:TerminateInstances", - "iam:PassRole", "iam:CreateServiceLinkedRole", + "iam:PassRole", "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"