From 60bf89e430d273341d1ab44638866d2eb2ea7dcb Mon Sep 17 00:00:00 2001 From: Dave Shepherd Date: Fri, 15 Jun 2018 11:47:19 +0100 Subject: [PATCH 1/2] Fix: sensible error when autoscaling group has no instances at all --- src/instance_terminator.js | 2 +- test/unit/instance_terminator.test.js | 54 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/instance_terminator.js b/src/instance_terminator.js index 51bae8b..9a46d5c 100644 --- a/src/instance_terminator.js +++ b/src/instance_terminator.js @@ -118,7 +118,7 @@ function terminateOldestInstanceFrom(matchingAutoscalingGroups) { const instances = autoscalingGroup['Instances']; const healthyInstances = instances.filter(instance => instance['LifecycleState'] == 'InService' && instance['HealthStatus'] == 'Healthy'); - if (healthyInstances.length < desiredCapacity) { + if (healthyInstances.length < desiredCapacity || desiredCapacity < 1) { console.log('Too few healthy instances, ignoring.') var response = { result: 'not enough healthy instances in group' diff --git a/test/unit/instance_terminator.test.js b/test/unit/instance_terminator.test.js index 0c0c37a..ff743b4 100644 --- a/test/unit/instance_terminator.test.js +++ b/test/unit/instance_terminator.test.js @@ -307,6 +307,35 @@ describe( 'instance-terminator', function() { expect(terminateInstance.notCalled, 'terminate instance should not be called').to.be.true; }); }); + + it( `should not terminate an instance across grouped asgs if the group has no instances`, function() { + const terminateInstance = sinon.spy(); + const lookupOldestInstance = sinon.spy(); + const myLambda = proxyquire( '../../src/instance_terminator', { + './aws/autoscaling_handler': { + findAutoscalingGroupsByTag: (tagKey, tagValue) => { + expect(tagKey).to.equal('can-be-terminated'); + expect(tagValue).to.equal('true'); + return new Promise(function (resolve) { + resolve(describeAutoscalingGroupsResponse_withGroupedAsgsWithNoInstances); + }); + }, + terminateInstance: terminateInstance + }, + './aws/ec2_handler': { + lookupOldestInstance: lookupOldestInstance + }, + }); + + return LambdaTester(myLambda.handler) + .event() + .expectResult((result) => { + expect(result).to.have.lengthOf(1); + expect(result).to.have.deep.members([{instanceTerminatorGroupName: 'my-test-group', result: 'not enough healthy instances in group'}]); + expect(lookupOldestInstance.notCalled, 'lookupOldestInstance should not be called').to.be.true; + expect(terminateInstance.notCalled, 'terminate instance should not be called').to.be.true; + }); + }); }); const describeAutoscalingGroupsResponse_withTooFewInstances = [ @@ -451,4 +480,29 @@ const describeAutoscalingGroupsResponse_withBadlyGroupedAsgs = [ Value: 'my-test-group' }] } +] + +const describeAutoscalingGroupsResponse_withGroupedAsgsWithNoInstances = [ + { + AutoScalingGroupName: 'my-asg-grouped-asgs', + MinSize: 0, + MaxSize: 0, + DesiredCapacity: 0, + Instances: [], + Tags: [{ + Key: 'instance-terminator-group', + Value: 'my-test-group' + }] + }, + { + AutoScalingGroupName: 'another-asg-grouped-asgs', + MinSize: 0, + MaxSize: 0, + DesiredCapacity: 0, + Instances: [], + Tags: [{ + Key: 'instance-terminator-group', + Value: 'my-test-group' + }] + } ] \ No newline at end of file From 4ebe3a43ba97787d870d63ccdedbf3c0c48f460c Mon Sep 17 00:00:00 2001 From: Dave Shepherd Date: Mon, 18 Jun 2018 14:29:27 +0100 Subject: [PATCH 2/2] Fix issue where instance-terminator-group tag values were ignored causing instances to be terminated across all grouped autoscaling groups --- src/instance_terminator.js | 9 +- test/unit/instance_terminator.test.js | 139 ++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/src/instance_terminator.js b/src/instance_terminator.js index 9a46d5c..2316f2d 100644 --- a/src/instance_terminator.js +++ b/src/instance_terminator.js @@ -77,7 +77,7 @@ function terminateOldestInstanceFromEachGrouped(autoscalingGroups) { instanceTerminatorGroupNames.forEach(function(instanceTerminatorGroupName) { promises.push(new Promise(function(resolve) { - const matchingAutoscalingGroups = autoscalingGroups.filter(item => containsTag(item['Tags'], 'instance-terminator-group', instanceTerminatorGroupName)); + const matchingAutoscalingGroups = autoscalingGroups.filter(item => containsTagValue(item['Tags'], 'instance-terminator-group', instanceTerminatorGroupName)); console.log('Attempting to terminate instance from: ' + instanceTerminatorGroupName); if (matchingAutoscalingGroups.length < 2) { @@ -177,4 +177,11 @@ function containsTag(tags, tagKey) { if (tag['Key'] == tagKey) return true; }) +} + +function containsTagValue(tags, tagKey, tagValue) { + return tags.some(function (tag) { + if (tag['Key'] == tagKey && tag['Value'] == tagValue) + return true; + }) } \ No newline at end of file diff --git a/test/unit/instance_terminator.test.js b/test/unit/instance_terminator.test.js index ff743b4..539376d 100644 --- a/test/unit/instance_terminator.test.js +++ b/test/unit/instance_terminator.test.js @@ -336,6 +336,85 @@ describe( 'instance-terminator', function() { expect(terminateInstance.notCalled, 'terminate instance should not be called').to.be.true; }); }); + + it( `should terminate one instance when there is more than one grouped asgs`, function() { + const terminateInstance = sinon.spy(); + const myLambda = proxyquire( '../../src/instance_terminator', { + './aws/autoscaling_handler': { + findAutoscalingGroupsByTag: (tagKey, tagValue) => { + expect(tagKey).to.equal('can-be-terminated'); + expect(tagValue).to.equal('true'); + return new Promise(function (resolve) { + resolve(describeAutoscalingGroupsResponse_withMultpleGroupedAsgs); + }); + }, + terminateInstance: terminateInstance + }, + './aws/ec2_handler': { + lookupOldestInstance: (instances) => { + if (instances[0]['InstanceId'] == 'i-grouped-asgs-1') { + expect(instances).to.deep.equal([{ + InstanceId: 'i-grouped-asgs-1', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }, { + InstanceId: 'i-grouped-asgs-2', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }]); + return new Promise(function (resolve) { + resolve({InstanceId: 'i-grouped-asgs-2',LaunchTime: new Date('2018-01-11T09:56:50.000Z')}); + }); + } + if (instances[0]['InstanceId'] == 'i-grouped-asgs-3') { + expect(instances).to.deep.equal([{ + InstanceId: 'i-grouped-asgs-3', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }, { + InstanceId: 'i-grouped-asgs-4', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }]); + return new Promise(function (resolve) { + resolve({InstanceId: 'i-grouped-asgs-4',LaunchTime: new Date('2018-01-11T09:56:50.000Z')}); + }); + } + if (instances[0]['InstanceId'] == 'i-grouped-asgs-2') { + expect(instances).to.deep.equal([{ + InstanceId: 'i-grouped-asgs-2', + LaunchTime: new Date('2018-01-11T09:56:50.000Z') + }, { + InstanceId: 'i-grouped-asgs-4', + LaunchTime: new Date('2018-01-11T09:56:50.000Z') + }]); + return new Promise(function (resolve) { + resolve({InstanceId: 'i-grouped-asgs-4', LaunchTime: new Date('2018-01-11T09:56:50.000Z')}); + }); + } + } + }, + }); + + return LambdaTester(myLambda.handler) + .event() + .expectResult((result) => { + expect(result).to.have.lengthOf(2); + expect(result).to.have.deep.members([ + { + instanceTerminatorGroupName: "my-test-group1", + result: 'instance terminated', + instanceId: 'i-grouped-asgs-4' + },{ + instanceTerminatorGroupName: "my-test-group2", + result: 'instance-terminator-group tag only attached to one autoscaling group' + } + ]); + + expect(terminateInstance.callCount, 'terminate instance called once').to.equal(1); + expect(terminateInstance.calledWith('i-grouped-asgs-4'), 'terminate instance parameters').to.be.true; + }); + }); }); const describeAutoscalingGroupsResponse_withTooFewInstances = [ @@ -505,4 +584,64 @@ const describeAutoscalingGroupsResponse_withGroupedAsgsWithNoInstances = [ Value: 'my-test-group' }] } +] + +const describeAutoscalingGroupsResponse_withMultpleGroupedAsgs = [ + { + AutoScalingGroupName: 'my-asg-grouped-asgs', + MinSize: 2, + MaxSize: 2, + DesiredCapacity: 2, + Instances: [{ + InstanceId: 'i-grouped-asgs-1', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }, { + InstanceId: 'i-grouped-asgs-2', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }], + Tags: [{ + Key: 'instance-terminator-group', + Value: 'my-test-group1' + }] + }, + { + AutoScalingGroupName: 'another-asg-grouped-asgs', + MinSize: 2, + MaxSize: 2, + DesiredCapacity: 2, + Instances: [{ + InstanceId: 'i-grouped-asgs-3', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }, { + InstanceId: 'i-grouped-asgs-4', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }], + Tags: [{ + Key: 'instance-terminator-group', + Value: 'my-test-group1' + }] + }, + { + AutoScalingGroupName: 'third-grouped-asgs', + MinSize: 2, + MaxSize: 2, + DesiredCapacity: 2, + Instances: [{ + InstanceId: 'i-grouped-asgs-5', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }, { + InstanceId: 'i-grouped-asgs-6', + LifecycleState: 'InService', + HealthStatus: 'Healthy' + }], + Tags: [{ + Key: 'instance-terminator-group', + Value: 'my-test-group2' + }] + }, ] \ No newline at end of file