Merge pull request #228 from jeffggardner/master

Added the ability to fine-tune the probability on a per-ASG basis.

src/main/java/com/netflix/simianarmy/basic/BasicChaosMonkeyContext.java

@@ -17,7 +17,6 @@
  */
 package com.netflix.simianarmy.basic;
 
-import com.amazonaws.regions.RegionUtils;
 import com.amazonaws.regions.Region;
 import com.amazonaws.regions.Regions;
 import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient;

src/main/java/com/netflix/simianarmy/client/aws/chaos/ASGChaosCrawler.java

@@ -23,16 +23,29 @@
 
 import com.amazonaws.services.autoscaling.model.AutoScalingGroup;
 import com.amazonaws.services.autoscaling.model.Instance;
+import com.amazonaws.services.autoscaling.model.TagDescription;
 import com.netflix.simianarmy.GroupType;
+import com.netflix.simianarmy.basic.chaos.BasicChaosMonkey;
 import com.netflix.simianarmy.basic.chaos.BasicInstanceGroup;
 import com.netflix.simianarmy.chaos.ChaosCrawler;
 import com.netflix.simianarmy.client.aws.AWSClient;
+import com.netflix.simianarmy.tunable.TunableInstanceGroup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The Class ASGChaosCrawler. This will crawl for all available AutoScalingGroups associated with the AWS account.
  */
 public class ASGChaosCrawler implements ChaosCrawler {
 
+    /** The Constant LOGGER. */
+    private static final Logger LOGGER = LoggerFactory.getLogger(ASGChaosCrawler.class);
+
+    /**
+     * The key of the tag that set the aggression coefficient
+     */
+    private static final String CHAOS_MONKEY_AGGRESSION_COEFFICIENT_KEY = "chaosMonkey.aggressionCoefficient";
+
     /**
      * The group types Types.
      */
@@ -70,13 +83,79 @@ public List<InstanceGroup> groups() {
     @Override
     public List<InstanceGroup> groups(String... names) {
         List<InstanceGroup> list = new LinkedList<InstanceGroup>();
+
         for (AutoScalingGroup asg : awsClient.describeAutoScalingGroups(names)) {
-            InstanceGroup ig = new BasicInstanceGroup(asg.getAutoScalingGroupName(), Types.ASG, awsClient.region());
+
+            InstanceGroup ig = getInstanceGroup(asg, findAggressionCoefficient(asg));
+
             for (Instance inst : asg.getInstances()) {
                 ig.addInstance(inst.getInstanceId());
             }
+
             list.add(ig);
         }
         return list;
     }
+
+    /**
+     * Returns the desired InstanceGroup.  If there is no set aggression coefficient, then it
+     * returns the basic impl, otherwise it returns the tunable impl.
+     * @param asg The autoscaling group 
+     * @return The appropriate {@link InstanceGroup}
+     */
+    protected InstanceGroup getInstanceGroup(AutoScalingGroup asg, double aggressionCoefficient) {
+      InstanceGroup instanceGroup;
+
+      // if coefficient is 1 then the BasicInstanceGroup is fine, otherwise use Tunable
+      if (aggressionCoefficient == 1.0) {
+          instanceGroup = new BasicInstanceGroup(asg.getAutoScalingGroupName(), Types.ASG, awsClient.region());
+      } else {
+        TunableInstanceGroup tunable = new TunableInstanceGroup(asg.getAutoScalingGroupName(), Types.ASG, awsClient.region());
+        tunable.setAggressionCoefficient(aggressionCoefficient);
+
+        instanceGroup = tunable;
+      }
+
+      return instanceGroup;
+    }
+
+    /**
+     * Reads tags on AutoScalingGroup looking for the tag for the aggression coefficient 
+     * and determines the coefficient value. The default value is 1 if there no tag or 
+     * if the value in the tag is not a parsable number.
+     * 
+     * @param asg The AutoScalingGroup that might have an aggression coefficient tag
+     * @return The set or default aggression coefficient.
+     */
+    protected double findAggressionCoefficient(AutoScalingGroup asg) {
+
+      List<TagDescription> tagDescriptions = asg.getTags();
+
+      double aggression = 1.0;
+
+      for (TagDescription tagDescription : tagDescriptions) {
+
+        if ( CHAOS_MONKEY_AGGRESSION_COEFFICIENT_KEY.equalsIgnoreCase(tagDescription.getKey()) ) {
+          String value = tagDescription.getValue();
+
+          // prevent NPE on parseDouble
+          if (value == null) {
+            break;
+          }
+
+          try {
+            aggression = Double.parseDouble(value);
+            LOGGER.info("Aggression coefficient of {} found for ASG {}", value, asg.getAutoScalingGroupName());
+          } catch (NumberFormatException e) {
+            LOGGER.warn("Unparsable value of {} found in tag {} for ASG {}", value, CHAOS_MONKEY_AGGRESSION_COEFFICIENT_KEY, asg.getAutoScalingGroupName());
+            aggression = 1.0;
+          }
+
+          // stop looking
+          break;
+        }
+      }
+
+      return aggression;
+    }
 }

src/main/java/com/netflix/simianarmy/tunable/TunableInstanceGroup.java

@@ -0,0 +1,36 @@
+package com.netflix.simianarmy.tunable;
+
+import com.netflix.simianarmy.GroupType;
+import com.netflix.simianarmy.basic.chaos.BasicInstanceGroup;
+
+/**
+ * Allows for individual InstanceGroups to alter the aggressiveness
+ * of ChaosMonkey.
+ * 
+ * @author jeffggardner
+ *
+ */
+public class TunableInstanceGroup extends BasicInstanceGroup {
+
+  public TunableInstanceGroup(String name, GroupType type, String region) {
+    super(name, type, region);
+  }
+
+  private double aggressionCoefficient = 1.0;
+
+  /**
+   * @return the aggressionCoefficient
+   */
+  public final double getAggressionCoefficient() {
+    return aggressionCoefficient;
+  }
+
+  /**
+   * @param aggressionCoefficient the aggressionCoefficient to set
+   */
+  public final void setAggressionCoefficient(double aggressionCoefficient) {
+    this.aggressionCoefficient = aggressionCoefficient;
+  }
+
+
+}

src/main/java/com/netflix/simianarmy/tunable/TunablyAggressiveChaosMonkey.java

@@ -0,0 +1,43 @@
+package com.netflix.simianarmy.tunable;
+
+import com.netflix.simianarmy.basic.chaos.BasicChaosMonkey;
+import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup;
+
+/**
+ * This class modifies the probability by multiplying the configured
+ * probability by the aggression coefficient tag on the instance group.
+ * 
+ * @author jeffggardner
+ */
+public class TunablyAggressiveChaosMonkey extends BasicChaosMonkey {
+
+  public TunablyAggressiveChaosMonkey(Context ctx) {
+    super(ctx);
+  }
+
+  /**
+   * Gets the tuned probability value, returns 0 if the group is not
+   * enabled. Calls getEffectiveProbability and modifies that value if 
+   * the instance group is a TunableInstanceGroup.
+   * 
+   * @param group The instance group
+   * @return the effective probability value for the instance group
+   */
+  @Override
+  protected double getEffectiveProbability(InstanceGroup group) {
+
+    if (!isGroupEnabled(group)) {
+      return 0;
+    }
+
+    double probability = getEffectiveProbabilityFromCfg(group);
+
+    // if this instance group is tunable, then factor in the aggression coefficient
+    if (group instanceof TunableInstanceGroup ) {
+      TunableInstanceGroup tunable = (TunableInstanceGroup) group;
+      probability *= tunable.getAggressionCoefficient();
+    }
+
+    return probability; 
+  }
+}

src/main/resources/chaos.properties

@@ -11,6 +11,9 @@ simianarmy.chaos.leashed = true
 # set to "false" for Opt-In behavior, "true" for Opt-Out behavior
 simianarmy.chaos.ASG.enabled = false
 
+# uncomment this line to use tunable aggression
+#simianarmy.client.chaos.class = com.netflix.simianarmy.tunable.TunablyAggressiveChaosMonkey
+
 # default probability for all ASGs
 simianarmy.chaos.ASG.probability = 1.0
 
@@ -94,4 +97,4 @@ simianarmy.chaos.mandatoryTermination.defaultProbability = 0.5
 #simianarmy.chaos.notification.body.suffix =  \ BodySuffix
 
 # Enable the email subject to be the same as the body, to include terminated instance and group information
-#simianarmy.chaos.notification.subject.isBody = true
+#simianarmy.chaos.notification.subject.isBody = true

src/test/java/com/netflix/simianarmy/client/aws/chaos/TestASGChaosCrawler.java

@@ -25,16 +25,21 @@
 
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
 import com.amazonaws.services.autoscaling.model.AutoScalingGroup;
 import com.amazonaws.services.autoscaling.model.Instance;
+import com.amazonaws.services.autoscaling.model.TagDescription;
+import com.netflix.simianarmy.basic.chaos.BasicInstanceGroup;
 import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup;
 import com.netflix.simianarmy.client.aws.AWSClient;
+import com.netflix.simianarmy.tunable.TunableInstanceGroup;
 
 public class TestASGChaosCrawler {
     private final ASGChaosCrawler crawler;
@@ -86,4 +91,78 @@ public void testGroups() {
         Assert.assertEquals(groups.get(1).instances().size(), 1);
         Assert.assertEquals(groups.get(1).instances().get(0), "i-123456781");
     }
+
+    @Test
+    public void testFindAggressionCoefficient() {
+      AutoScalingGroup asg1 = mkAsg("asg1", "i-123456780");
+      Set<TagDescription> tagDescriptions = new HashSet<>();
+      tagDescriptions.add(makeTunableTag("1.0"));
+      asg1.setTags(tagDescriptions);
+
+      double aggression = crawler.findAggressionCoefficient(asg1);
+
+      Assert.assertEquals(aggression, 1.0);
+    }
+
+    @Test
+    public void testFindAggressionCoefficient_two() {
+      AutoScalingGroup asg1 = mkAsg("asg1", "i-123456780");
+      Set<TagDescription> tagDescriptions = new HashSet<>();
+      tagDescriptions.add(makeTunableTag("2.0"));
+      asg1.setTags(tagDescriptions);
+
+      double aggression = crawler.findAggressionCoefficient(asg1);
+
+      Assert.assertEquals(aggression, 2.0);
+    }
+
+    @Test
+    public void testFindAggressionCoefficient_null() {
+      AutoScalingGroup asg1 = mkAsg("asg1", "i-123456780");
+      Set<TagDescription> tagDescriptions = new HashSet<>();
+      tagDescriptions.add(makeTunableTag(null));
+      asg1.setTags(tagDescriptions);
+
+      double aggression = crawler.findAggressionCoefficient(asg1);
+
+      Assert.assertEquals(aggression, 1.0);
+    }
+
+    @Test
+    public void testFindAggressionCoefficient_unparsable() {
+      AutoScalingGroup asg1 = mkAsg("asg1", "i-123456780");
+      Set<TagDescription> tagDescriptions = new HashSet<>();
+      tagDescriptions.add(makeTunableTag("not a number"));
+      asg1.setTags(tagDescriptions);
+
+      double aggression = crawler.findAggressionCoefficient(asg1);
+
+      Assert.assertEquals(aggression, 1.0);
+    }
+
+    private TagDescription makeTunableTag(String value) {
+      TagDescription desc = new TagDescription();
+      desc.setKey("chaosMonkey.aggressionCoefficient");
+      desc.setValue(value);
+      return desc;
+    }
+
+    @Test 
+    public void testGetInstanceGroup_basic() {
+      AutoScalingGroup asg = mkAsg("asg1", "i-123456780");
+
+      InstanceGroup group = crawler.getInstanceGroup(asg, 1.0);
+
+      Assert.assertTrue( (group instanceof BasicInstanceGroup) );
+      Assert.assertFalse( (group instanceof TunableInstanceGroup) );
+    }
+
+    @Test 
+    public void testGetInstanceGroup_tunable() {
+      AutoScalingGroup asg = mkAsg("asg1", "i-123456780");
+
+      InstanceGroup group = crawler.getInstanceGroup(asg, 2.0);
+
+      Assert.assertTrue( (group instanceof TunableInstanceGroup) );
+    }
 }

src/test/java/com/netflix/simianarmy/tunable/TestTunablyAggressiveChaosMonkey.java

@@ -0,0 +1,42 @@
+package com.netflix.simianarmy.tunable;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import com.netflix.simianarmy.GroupType;
+import com.netflix.simianarmy.basic.chaos.BasicInstanceGroup;
+import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup;
+import com.netflix.simianarmy.chaos.TestChaosMonkeyContext;
+
+public class TestTunablyAggressiveChaosMonkey {
+  private enum GroupTypes implements GroupType {
+    TYPE_A, TYPE_B
+  };
+
+  @Test
+  public void testFullProbability_basic() {
+    TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("fullProbability.properties");
+
+    TunablyAggressiveChaosMonkey chaos = new TunablyAggressiveChaosMonkey(ctx);
+
+    InstanceGroup basic = new BasicInstanceGroup("basic", GroupTypes.TYPE_A, "region");
+
+    double probability = chaos.getEffectiveProbability(basic);
+
+    Assert.assertEquals(probability, 1.0);
+  }
+
+  @Test
+  public void testFullProbability_tuned() {
+    TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("fullProbability.properties");
+
+    TunablyAggressiveChaosMonkey chaos = new TunablyAggressiveChaosMonkey(ctx);
+
+    TunableInstanceGroup tuned = new TunableInstanceGroup("basic", GroupTypes.TYPE_A, "region");
+    tuned.setAggressionCoefficient(0.5);
+
+    double probability = chaos.getEffectiveProbability(tuned);
+
+    Assert.assertEquals(probability, 0.5);
+  }
+}