Skip to content
This repository has been archived by the owner on Mar 4, 2021. It is now read-only.

Added the ability to fine-tune the probability on a per-ASG basis. #228

Merged
merged 4 commits into from
Jan 4, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
*/
package com.netflix.simianarmy.basic;

import com.amazonaws.regions.RegionUtils;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,29 @@

import com.amazonaws.services.autoscaling.model.AutoScalingGroup;
import com.amazonaws.services.autoscaling.model.Instance;
import com.amazonaws.services.autoscaling.model.TagDescription;
import com.netflix.simianarmy.GroupType;
import com.netflix.simianarmy.basic.chaos.BasicChaosMonkey;
import com.netflix.simianarmy.basic.chaos.BasicInstanceGroup;
import com.netflix.simianarmy.chaos.ChaosCrawler;
import com.netflix.simianarmy.client.aws.AWSClient;
import com.netflix.simianarmy.tunable.TunableInstanceGroup;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* The Class ASGChaosCrawler. This will crawl for all available AutoScalingGroups associated with the AWS account.
*/
public class ASGChaosCrawler implements ChaosCrawler {

/** The Constant LOGGER. */
private static final Logger LOGGER = LoggerFactory.getLogger(ASGChaosCrawler.class);

/**
* The key of the tag that set the aggression coefficient
*/
private static final String CHAOS_MONKEY_AGGRESSION_COEFFICIENT_KEY = "chaosMonkey.aggressionCoefficient";

/**
* The group types Types.
*/
Expand Down Expand Up @@ -70,13 +83,79 @@ public List<InstanceGroup> groups() {
@Override
public List<InstanceGroup> groups(String... names) {
List<InstanceGroup> list = new LinkedList<InstanceGroup>();

for (AutoScalingGroup asg : awsClient.describeAutoScalingGroups(names)) {
InstanceGroup ig = new BasicInstanceGroup(asg.getAutoScalingGroupName(), Types.ASG, awsClient.region());

InstanceGroup ig = getInstanceGroup(asg, findAggressionCoefficient(asg));

for (Instance inst : asg.getInstances()) {
ig.addInstance(inst.getInstanceId());
}

list.add(ig);
}
return list;
}

/**
* Returns the desired InstanceGroup. If there is no set aggression coefficient, then it
* returns the basic impl, otherwise it returns the tunable impl.
* @param asg The autoscaling group
* @return The appropriate {@link InstanceGroup}
*/
protected InstanceGroup getInstanceGroup(AutoScalingGroup asg, double aggressionCoefficient) {
InstanceGroup instanceGroup;

// if coefficient is 1 then the BasicInstanceGroup is fine, otherwise use Tunable
if (aggressionCoefficient == 1.0) {
instanceGroup = new BasicInstanceGroup(asg.getAutoScalingGroupName(), Types.ASG, awsClient.region());
} else {
TunableInstanceGroup tunable = new TunableInstanceGroup(asg.getAutoScalingGroupName(), Types.ASG, awsClient.region());
tunable.setAggressionCoefficient(aggressionCoefficient);

instanceGroup = tunable;
}

return instanceGroup;
}

/**
* Reads tags on AutoScalingGroup looking for the tag for the aggression coefficient
* and determines the coefficient value. The default value is 1 if there no tag or
* if the value in the tag is not a parsable number.
*
* @param asg The AutoScalingGroup that might have an aggression coefficient tag
* @return The set or default aggression coefficient.
*/
protected double findAggressionCoefficient(AutoScalingGroup asg) {

List<TagDescription> tagDescriptions = asg.getTags();

double aggression = 1.0;

for (TagDescription tagDescription : tagDescriptions) {

if ( CHAOS_MONKEY_AGGRESSION_COEFFICIENT_KEY.equalsIgnoreCase(tagDescription.getKey()) ) {
String value = tagDescription.getValue();

// prevent NPE on parseDouble
if (value == null) {
break;
}

try {
aggression = Double.parseDouble(value);
LOGGER.info("Aggression coefficient of {} found for ASG {}", value, asg.getAutoScalingGroupName());
} catch (NumberFormatException e) {
LOGGER.warn("Unparsable value of {} found in tag {} for ASG {}", value, CHAOS_MONKEY_AGGRESSION_COEFFICIENT_KEY, asg.getAutoScalingGroupName());
aggression = 1.0;
}

// stop looking
break;
}
}

return aggression;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package com.netflix.simianarmy.tunable;

import com.netflix.simianarmy.GroupType;
import com.netflix.simianarmy.basic.chaos.BasicInstanceGroup;

/**
* Allows for individual InstanceGroups to alter the aggressiveness
* of ChaosMonkey.
*
* @author jeffggardner
*
*/
public class TunableInstanceGroup extends BasicInstanceGroup {

public TunableInstanceGroup(String name, GroupType type, String region) {
super(name, type, region);
}

private double aggressionCoefficient = 1.0;

/**
* @return the aggressionCoefficient
*/
public final double getAggressionCoefficient() {
return aggressionCoefficient;
}

/**
* @param aggressionCoefficient the aggressionCoefficient to set
*/
public final void setAggressionCoefficient(double aggressionCoefficient) {
this.aggressionCoefficient = aggressionCoefficient;
}


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package com.netflix.simianarmy.tunable;

import com.netflix.simianarmy.basic.chaos.BasicChaosMonkey;
import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup;

/**
* This class modifies the probability by multiplying the configured
* probability by the aggression coefficient tag on the instance group.
*
* @author jeffggardner
*/
public class TunablyAggressiveChaosMonkey extends BasicChaosMonkey {

public TunablyAggressiveChaosMonkey(Context ctx) {
super(ctx);
}

/**
* Gets the tuned probability value, returns 0 if the group is not
* enabled. Calls getEffectiveProbability and modifies that value if
* the instance group is a TunableInstanceGroup.
*
* @param group The instance group
* @return the effective probability value for the instance group
*/
@Override
protected double getEffectiveProbability(InstanceGroup group) {

if (!isGroupEnabled(group)) {
return 0;
}

double probability = getEffectiveProbabilityFromCfg(group);

// if this instance group is tunable, then factor in the aggression coefficient
if (group instanceof TunableInstanceGroup ) {
TunableInstanceGroup tunable = (TunableInstanceGroup) group;
probability *= tunable.getAggressionCoefficient();
}

return probability;
}
}
5 changes: 4 additions & 1 deletion src/main/resources/chaos.properties
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ simianarmy.chaos.leashed = true
# set to "false" for Opt-In behavior, "true" for Opt-Out behavior
simianarmy.chaos.ASG.enabled = false

# uncomment this line to use tunable aggression
#simianarmy.client.chaos.class = com.netflix.simianarmy.tunable.TunablyAggressiveChaosMonkey

# default probability for all ASGs
simianarmy.chaos.ASG.probability = 1.0

Expand Down Expand Up @@ -94,4 +97,4 @@ simianarmy.chaos.mandatoryTermination.defaultProbability = 0.5
#simianarmy.chaos.notification.body.suffix = \ BodySuffix

# Enable the email subject to be the same as the body, to include terminated instance and group information
#simianarmy.chaos.notification.subject.isBody = true
#simianarmy.chaos.notification.subject.isBody = true
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,21 @@

import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import org.testng.Assert;
import org.testng.annotations.Test;

import com.amazonaws.services.autoscaling.model.AutoScalingGroup;
import com.amazonaws.services.autoscaling.model.Instance;
import com.amazonaws.services.autoscaling.model.TagDescription;
import com.netflix.simianarmy.basic.chaos.BasicInstanceGroup;
import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup;
import com.netflix.simianarmy.client.aws.AWSClient;
import com.netflix.simianarmy.tunable.TunableInstanceGroup;

public class TestASGChaosCrawler {
private final ASGChaosCrawler crawler;
Expand Down Expand Up @@ -86,4 +91,78 @@ public void testGroups() {
Assert.assertEquals(groups.get(1).instances().size(), 1);
Assert.assertEquals(groups.get(1).instances().get(0), "i-123456781");
}

@Test
public void testFindAggressionCoefficient() {
AutoScalingGroup asg1 = mkAsg("asg1", "i-123456780");
Set<TagDescription> tagDescriptions = new HashSet<>();
tagDescriptions.add(makeTunableTag("1.0"));
asg1.setTags(tagDescriptions);

double aggression = crawler.findAggressionCoefficient(asg1);

Assert.assertEquals(aggression, 1.0);
}

@Test
public void testFindAggressionCoefficient_two() {
AutoScalingGroup asg1 = mkAsg("asg1", "i-123456780");
Set<TagDescription> tagDescriptions = new HashSet<>();
tagDescriptions.add(makeTunableTag("2.0"));
asg1.setTags(tagDescriptions);

double aggression = crawler.findAggressionCoefficient(asg1);

Assert.assertEquals(aggression, 2.0);
}

@Test
public void testFindAggressionCoefficient_null() {
AutoScalingGroup asg1 = mkAsg("asg1", "i-123456780");
Set<TagDescription> tagDescriptions = new HashSet<>();
tagDescriptions.add(makeTunableTag(null));
asg1.setTags(tagDescriptions);

double aggression = crawler.findAggressionCoefficient(asg1);

Assert.assertEquals(aggression, 1.0);
}

@Test
public void testFindAggressionCoefficient_unparsable() {
AutoScalingGroup asg1 = mkAsg("asg1", "i-123456780");
Set<TagDescription> tagDescriptions = new HashSet<>();
tagDescriptions.add(makeTunableTag("not a number"));
asg1.setTags(tagDescriptions);

double aggression = crawler.findAggressionCoefficient(asg1);

Assert.assertEquals(aggression, 1.0);
}

private TagDescription makeTunableTag(String value) {
TagDescription desc = new TagDescription();
desc.setKey("chaosMonkey.aggressionCoefficient");
desc.setValue(value);
return desc;
}

@Test
public void testGetInstanceGroup_basic() {
AutoScalingGroup asg = mkAsg("asg1", "i-123456780");

InstanceGroup group = crawler.getInstanceGroup(asg, 1.0);

Assert.assertTrue( (group instanceof BasicInstanceGroup) );
Assert.assertFalse( (group instanceof TunableInstanceGroup) );
}

@Test
public void testGetInstanceGroup_tunable() {
AutoScalingGroup asg = mkAsg("asg1", "i-123456780");

InstanceGroup group = crawler.getInstanceGroup(asg, 2.0);

Assert.assertTrue( (group instanceof TunableInstanceGroup) );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package com.netflix.simianarmy.tunable;

import org.testng.Assert;
import org.testng.annotations.Test;

import com.netflix.simianarmy.GroupType;
import com.netflix.simianarmy.basic.chaos.BasicInstanceGroup;
import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup;
import com.netflix.simianarmy.chaos.TestChaosMonkeyContext;

public class TestTunablyAggressiveChaosMonkey {
private enum GroupTypes implements GroupType {
TYPE_A, TYPE_B
};

@Test
public void testFullProbability_basic() {
TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("fullProbability.properties");

TunablyAggressiveChaosMonkey chaos = new TunablyAggressiveChaosMonkey(ctx);

InstanceGroup basic = new BasicInstanceGroup("basic", GroupTypes.TYPE_A, "region");

double probability = chaos.getEffectiveProbability(basic);

Assert.assertEquals(probability, 1.0);
}

@Test
public void testFullProbability_tuned() {
TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("fullProbability.properties");

TunablyAggressiveChaosMonkey chaos = new TunablyAggressiveChaosMonkey(ctx);

TunableInstanceGroup tuned = new TunableInstanceGroup("basic", GroupTypes.TYPE_A, "region");
tuned.setAggressionCoefficient(0.5);

double probability = chaos.getEffectiveProbability(tuned);

Assert.assertEquals(probability, 0.5);
}
}