very basic detection

SingularityBase/src/main/java/com/hubspot/singularity/SingularityDisabledAction.java

@@ -9,12 +9,14 @@ public class SingularityDisabledAction {
   private final SingularityDisabledActionType type;
   private final String message;
   private final Optional<String> user;
+  private final boolean systemGenerated;
 
   @JsonCreator
-  public SingularityDisabledAction(@JsonProperty("type") SingularityDisabledActionType type, @JsonProperty("message") String message, @JsonProperty("user") Optional<String> user) {
+  public SingularityDisabledAction(@JsonProperty("type") SingularityDisabledActionType type, @JsonProperty("message") String message, @JsonProperty("user") Optional<String> user, @JsonProperty("systemGenerated") boolean systemGenerated) {
     this.type = type;
     this.message = message;
     this.user = user;
+    this.systemGenerated = systemGenerated;
   }
 
   public SingularityDisabledActionType getType() {
@@ -29,6 +31,10 @@ public Optional<String> getUser() {
     return user;
   }
 
+  public boolean isSystemGenerated() {
+    return systemGenerated;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -38,14 +44,15 @@ public boolean equals(Object o) {
       return false;
     }
     SingularityDisabledAction that = (SingularityDisabledAction) o;
-    return type == that.type &&
+    return systemGenerated == that.systemGenerated &&
+      type == that.type &&
       Objects.equal(message, that.message) &&
       Objects.equal(user, that.user);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hashCode(type, message, user);
+    return Objects.hashCode(type, message, user, systemGenerated);
   }
 
   @Override
@@ -54,6 +61,7 @@ public String toString() {
       .add("type", type)
       .add("message", message)
       .add("user", user)
+      .add("systemGenerated", systemGenerated)
       .toString();
   }
 }

SingularityBase/src/main/java/com/hubspot/singularity/SingularityDisabledActionType.java

@@ -1,5 +1,5 @@
 package com.hubspot.singularity;
 
 public enum SingularityDisabledActionType {
-  BOUNCE, DEPLOY, SCALE, REMOVE, DECOMMISSION
+  BOUNCE, DEPLOY, SCALE, REMOVE, DECOMMISSION, TASK_RECONCILIATION
 }

SingularityBase/src/main/java/com/hubspot/singularity/SingularityDisasterStats.java

@@ -0,0 +1,99 @@
+package com.hubspot.singularity;
+
+import java.util.Collections;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+
+public class SingularityDisasterStats {
+  private final long timestamp;
+  private final int numActiveTasks;
+  private final int numPendingTasks;
+  private final int numOverdueTasks;
+  private final long avgTaskLagMillis;
+  private final int numActiveSlaves;
+  private final int numLostSlaves;
+
+  @JsonCreator
+  public SingularityDisasterStats(@JsonProperty("timestamp") long timestamp,
+                                  @JsonProperty("numActiveTasks") int numActiveTasks,
+                                  @JsonProperty("numPendingTasks") int numPendingTasks,
+                                  @JsonProperty("numOverdueTasks") int numOverdueTasks,
+                                  @JsonProperty("avgTaskLagMillis") long avgTaskLagMillis,
+                                  @JsonProperty("numActiveSlaves") int numActiveSlaves,
+                                  @JsonProperty("numLostSlaves") int numLostSlaves) {
+    this.timestamp = timestamp;
+    this.numActiveTasks = numActiveTasks;
+    this.numPendingTasks = numPendingTasks;
+    this.numOverdueTasks = numOverdueTasks;
+    this.avgTaskLagMillis = avgTaskLagMillis;
+    this.numActiveSlaves = numActiveSlaves;
+    this.numLostSlaves = numLostSlaves;
+  }
+
+  public long getTimestamp() {
+    return timestamp;
+  }
+
+  public int getNumActiveTasks() {
+    return numActiveTasks;
+  }
+
+  public int getNumPendingTasks() {
+    return numPendingTasks;
+  }
+
+  public int getNumOverdueTasks() {
+    return numOverdueTasks;
+  }
+
+  public long getAvgTaskLagMillis() {
+    return avgTaskLagMillis;
+  }
+
+  public int getNumActiveSlaves() {
+    return numActiveSlaves;
+  }
+
+  public int getNumLostSlaves() {
+    return numLostSlaves;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    SingularityDisasterStats that = (SingularityDisasterStats) o;
+    return timestamp == that.timestamp &&
+      numActiveTasks == that.numActiveTasks &&
+      numPendingTasks == that.numPendingTasks &&
+      numOverdueTasks == that.numOverdueTasks &&
+      avgTaskLagMillis == that.avgTaskLagMillis &&
+      numActiveSlaves == that.numActiveSlaves &&
+      numLostSlaves == that.numLostSlaves;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hashCode(timestamp, numActiveTasks, numPendingTasks, numOverdueTasks, avgTaskLagMillis, numActiveSlaves, numLostSlaves);
+  }
+
+  @Override
+  public String toString() {
+    return Objects.toStringHelper(this)
+      .add("timestamp", timestamp)
+      .add("numActiveTasks", numActiveTasks)
+      .add("numPendingTasks", numPendingTasks)
+      .add("numOverdueTasks", numOverdueTasks)
+      .add("avgTaskLagMillis", avgTaskLagMillis)
+      .add("numActiveSlaves", numActiveSlaves)
+      .add("numLostSlaves", numLostSlaves)
+      .toString();
+  }
+}

SingularityBase/src/main/java/com/hubspot/singularity/SingularityDisasterType.java

@@ -0,0 +1,5 @@
+package com.hubspot.singularity;
+
+public enum SingularityDisasterType {
+  EXCESSIVE_TASK_LAG, LOST_SLAVES;
+}

SingularityService/src/main/java/com/hubspot/singularity/config/DisasterDetectionConfiguration.java

@@ -1,20 +1,49 @@
 package com.hubspot.singularity.config;
 
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 import javax.validation.constraints.NotNull;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableList;
+import com.hubspot.singularity.SingularityDisabledActionType;
 
 public class DisasterDetectionConfiguration {
 
   @JsonProperty
   private boolean enabled = false;
 
+  @JsonProperty
+  private long runEveryMillis = TimeUnit.SECONDS.toMillis(30);
+
+  @JsonProperty
+  private long considerOverdueAfterMillis = TimeUnit.MINUTES.toMillis(1);
+
   @JsonProperty
   @NotNull
-  private long runEveryMillis = TimeUnit.SECONDS.toMillis(15);
+  private List<SingularityDisabledActionType> disableActionsOnDisaster = ImmutableList.of(SingularityDisabledActionType.BOUNCE, SingularityDisabledActionType.DEPLOY, SingularityDisabledActionType.TASK_RECONCILIATION);
+
+  @JsonProperty
+  private boolean checkOverdueTasks = true;
+
+  @JsonProperty
+  private long criticalAvgTaskLagMillis = 300000L;
+
+  @JsonProperty
+  private double criticalOverdueTaskPortion = 0.2;
 
+  @JsonProperty
+  private boolean requireAllConditionsForOverdueTaskDisaster = true;
+
+  @JsonProperty
+  private boolean checkLostSlaves = true;
+
+  @JsonProperty
+  private double criticalLostSlavePortion = 0.2;
+
+  @JsonProperty
+  private long checkLostSlavesInLastMillis = 60000;
 
   public boolean isEnabled() {
     return enabled;
@@ -31,4 +60,76 @@ public long getRunEveryMillis() {
   public void setRunEveryMillis(long runEveryMillis) {
     this.runEveryMillis = runEveryMillis;
   }
+
+  public long getConsiderOverdueAfterMillis() {
+    return considerOverdueAfterMillis;
+  }
+
+  public void setConsiderOverdueAfterMillis(long considerOverdueAfterMillis) {
+    this.considerOverdueAfterMillis = considerOverdueAfterMillis;
+  }
+
+  public List<SingularityDisabledActionType> getDisableActionsOnDisaster() {
+    return disableActionsOnDisaster;
+  }
+
+  public void setDisableActionsOnDisaster(List<SingularityDisabledActionType> disableActionsOnDisaster) {
+    this.disableActionsOnDisaster = disableActionsOnDisaster;
+  }
+
+  public boolean isCheckOverdueTasks() {
+    return checkOverdueTasks;
+  }
+
+  public void setCheckOverdueTasks(boolean checkOverdueTasks) {
+    this.checkOverdueTasks = checkOverdueTasks;
+  }
+
+  public long getCriticalAvgTaskLagMillis() {
+    return criticalAvgTaskLagMillis;
+  }
+
+  public void setCriticalAvgTaskLagMillis(long criticalAvgTaskLagMillis) {
+    this.criticalAvgTaskLagMillis = criticalAvgTaskLagMillis;
+  }
+
+  public double getCriticalOverdueTaskPortion() {
+    return criticalOverdueTaskPortion;
+  }
+
+  public void setCriticalOverdueTaskPortion(double criticalOverdueTaskPortion) {
+    this.criticalOverdueTaskPortion = criticalOverdueTaskPortion;
+  }
+
+  public boolean isRequireAllConditionsForOverdueTaskDisaster() {
+    return requireAllConditionsForOverdueTaskDisaster;
+  }
+
+  public void setRequireAllConditionsForOverdueTaskDisaster(boolean requireAllConditionsForOverdueTaskDisaster) {
+    this.requireAllConditionsForOverdueTaskDisaster = requireAllConditionsForOverdueTaskDisaster;
+  }
+
+  public boolean isCheckLostSlaves() {
+    return checkLostSlaves;
+  }
+
+  public void setCheckLostSlaves(boolean checkLostSlaves) {
+    this.checkLostSlaves = checkLostSlaves;
+  }
+
+  public double getCriticalLostSlavePortion() {
+    return criticalLostSlavePortion;
+  }
+
+  public void setCriticalLostSlavePortion(double criticalLostSlavePortion) {
+    this.criticalLostSlavePortion = criticalLostSlavePortion;
+  }
+
+  public long getCheckLostSlavesInLastMillis() {
+    return checkLostSlavesInLastMillis;
+  }
+
+  public void setCheckLostSlavesInLastMillis(long checkLostSlavesInLastMillis) {
+    this.checkLostSlavesInLastMillis = checkLostSlavesInLastMillis;
+  }
 }

SingularityService/src/main/java/com/hubspot/singularity/data/DisabledActionManager.java → SingularityService/src/main/java/com/hubspot/singularity/data/DisasterManager.java

@@ -2,6 +2,7 @@
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import org.apache.curator.framework.CuratorFramework;
@@ -14,26 +15,34 @@
 import com.hubspot.singularity.SingularityDeleteResult;
 import com.hubspot.singularity.SingularityDisabledAction;
 import com.hubspot.singularity.SingularityDisabledActionType;
+import com.hubspot.singularity.SingularityDisasterStats;
+import com.hubspot.singularity.SingularityDisasterType;
 import com.hubspot.singularity.SingularityUser;
 import com.hubspot.singularity.config.SingularityConfiguration;
 import com.hubspot.singularity.data.transcoders.Transcoder;
 
-public class DisabledActionManager extends CuratorAsyncManager {
-  private static final String DISABLED_ACTIONS_ROOT = "/disabled-actions";
+public class DisasterManager extends CuratorAsyncManager {
+  private static final String DISASTERS_ROOT = "/disasters";
+  private static final String DISABLED_ACTIONS = DISASTERS_ROOT + "/disabled-actions";
+  private static final String ACTIVE_DISASTERS = DISASTERS_ROOT + "/active";
+  private static final String DISASTER_STATS = DISASTERS_ROOT + "/stats";
 
   private static final String MESSAGE_FORMAT = "Cannot perform action %s: %s";
   private static final String DEFAULT_MESSAGE = "Action is currently disabled";
 
   private final Transcoder<SingularityDisabledAction> disabledActionTranscoder;
+  private final Transcoder<SingularityDisasterStats> disasterStatsTranscoder;
 
   @Inject
-  public DisabledActionManager(CuratorFramework curator, SingularityConfiguration configuration, MetricRegistry metricRegistry, Transcoder<SingularityDisabledAction> disabledActionTranscoder) {
+  public DisasterManager(CuratorFramework curator, SingularityConfiguration configuration, MetricRegistry metricRegistry,
+                         Transcoder<SingularityDisabledAction> disabledActionTranscoder, Transcoder<SingularityDisasterStats> disasterStatsTranscoder) {
     super(curator, configuration, metricRegistry);
     this.disabledActionTranscoder = disabledActionTranscoder;
+    this.disasterStatsTranscoder = disasterStatsTranscoder;
   }
 
   private String getActionPath(SingularityDisabledActionType action) {
-    return ZKPaths.makePath(DISABLED_ACTIONS_ROOT, action.name());
+    return ZKPaths.makePath(DISABLED_ACTIONS, action.name());
   }
 
   public boolean isDisabled(SingularityDisabledActionType action) {
@@ -42,14 +51,15 @@ public boolean isDisabled(SingularityDisabledActionType action) {
 
   public SingularityDisabledAction getDisabledAction(SingularityDisabledActionType action) {
     Optional<SingularityDisabledAction> maybeDisabledAction = getData(getActionPath(action), disabledActionTranscoder);
-    return maybeDisabledAction.or(new SingularityDisabledAction(action, String.format(MESSAGE_FORMAT, action, DEFAULT_MESSAGE), Optional.<String>absent()));
+    return maybeDisabledAction.or(new SingularityDisabledAction(action, String.format(MESSAGE_FORMAT, action, DEFAULT_MESSAGE), Optional.<String>absent(), false));
   }
 
-  public SingularityCreateResult disable(SingularityDisabledActionType action, Optional<String> maybeMessage, Optional<SingularityUser> user) {
+  public SingularityCreateResult disable(SingularityDisabledActionType action, Optional<String> maybeMessage, Optional<SingularityUser> user, boolean systemGenerated) {
     SingularityDisabledAction disabledAction = new SingularityDisabledAction(
       action,
       String.format(MESSAGE_FORMAT, action, maybeMessage.or(DEFAULT_MESSAGE)),
-      user.isPresent() ? Optional.of(user.get().getId()) : Optional.<String>absent());
+      user.isPresent() ? Optional.of(user.get().getId()) : Optional.<String>absent(),
+      systemGenerated);
 
     return save(getActionPath(action), disabledAction, disabledActionTranscoder);
   }
@@ -60,10 +70,35 @@ public SingularityDeleteResult enable(SingularityDisabledActionType action) {
 
   public List<SingularityDisabledAction> getDisabledActions() {
     List<String> paths = new ArrayList<>();
-    for (String path : getChildren(DISABLED_ACTIONS_ROOT)) {
-      paths.add(ZKPaths.makePath(DISABLED_ACTIONS_ROOT, path));
+    for (String path : getChildren(DISABLED_ACTIONS)) {
+      paths.add(ZKPaths.makePath(DISABLED_ACTIONS, path));
     }
 
-    return getAsync(DISABLED_ACTIONS_ROOT, paths, disabledActionTranscoder);
+    return getAsync(DISABLED_ACTIONS, paths, disabledActionTranscoder);
+  }
+
+  public void addDisaster(SingularityDisasterType disaster) {
+    create(ZKPaths.makePath(ACTIVE_DISASTERS, disaster.name()));
+  }
+
+  public void removeDisaster(SingularityDisasterType disaster) {
+    delete(ZKPaths.makePath(ACTIVE_DISASTERS, disaster.name()));
+  }
+
+  public List<SingularityDisasterType> getActiveDisasters() {
+    List<String> disasterNames = getChildren(ACTIVE_DISASTERS);
+    List<SingularityDisasterType> disasters = new ArrayList<>();
+    for (String name : disasterNames) {
+      disasters.add(SingularityDisasterType.valueOf(name));
+    }
+    return disasters;
+  }
+
+  public void saveDisasterStats(SingularityDisasterStats stats) {
+    save(DISASTER_STATS, stats, disasterStatsTranscoder);
+  }
+
+  public Optional<SingularityDisasterStats> getDisasterStats() {
+    return getData(DISASTER_STATS, disasterStatsTranscoder);
   }
 }