better data reported

HubSpot · Aug 25, 2016 · d417ed2 · d417ed2
1 parent fca02bb
commit d417ed2
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 4 deletions.
diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDisasterType.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDisasterType.java
@@ -1,5 +1,5 @@
 package com.hubspot.singularity;
 
 public enum SingularityDisasterType {
-  EXCESSIVE_TASK_LAG, LOST_SLAVES, LOST_TASKS;
+  EXCESSIVE_TASK_LAG, LOST_SLAVES, LOST_TASKS, USER_INITIATED;
 }
diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/DisasterManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/DisasterManager.java
@@ -17,6 +17,7 @@
 import com.hubspot.singularity.SingularityDisabledActionType;
 import com.hubspot.singularity.SingularityDisasterStats;
 import com.hubspot.singularity.SingularityDisasterType;
+import com.hubspot.singularity.SingularityDisastersData;
 import com.hubspot.singularity.SingularityUser;
 import com.hubspot.singularity.config.SingularityConfiguration;
 import com.hubspot.singularity.data.transcoders.Transcoder;
@@ -26,6 +27,7 @@ public class DisasterManager extends CuratorAsyncManager {
   private static final String DISABLED_ACTIONS_PATH = DISASTERS_ROOT + "/disabled-actions";
   private static final String ACTIVE_DISASTERS_PATH = DISASTERS_ROOT + "/active";
   private static final String DISASTER_STATS_PATH = DISASTERS_ROOT + "/stats";
+  private static final String PREVIOUS_DISASTER_STATS_PATH = DISASTERS_ROOT + "/previous-stats";
   private static final String DISABLE_AUTOMATED_PATH = DISASTERS_ROOT + "/disabled";
 
   private static final String MESSAGE_FORMAT = "Cannot perform action %s: %s";
@@ -107,6 +109,18 @@ public Optional<SingularityDisasterStats> getDisasterStats() {
     return getData(DISASTER_STATS_PATH, disasterStatsTranscoder);
   }
 
+  public void savePreviousDisasterStats(SingularityDisasterStats stats) {
+    save(PREVIOUS_DISASTER_STATS_PATH, stats, disasterStatsTranscoder);
+  }
+
+  public Optional<SingularityDisasterStats> getPreviousDisasterStats() {
+    return getData(PREVIOUS_DISASTER_STATS_PATH, disasterStatsTranscoder);
+  }
+
+  public SingularityDisastersData getDisastersData() {
+    return new SingularityDisastersData(getDisasterStats(), getPreviousDisasterStats(), getActiveDisasters());
+  }
+
   public void updateActiveDisasters(List<SingularityDisasterType> previouslyActiveDisasters, List<SingularityDisasterType> newActiveDisasters) {
     for (SingularityDisasterType disaster : previouslyActiveDisasters) {
       if (!newActiveDisasters.contains(disaster)) {

diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/DisastersResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/DisastersResource.java
@@ -18,6 +18,7 @@
 import com.hubspot.singularity.SingularityDisabledActionType;
 import com.hubspot.singularity.SingularityDisasterStats;
 import com.hubspot.singularity.SingularityDisasterType;
+import com.hubspot.singularity.SingularityDisastersData;
 import com.hubspot.singularity.SingularityService;
 import com.hubspot.singularity.SingularityUser;
 import com.hubspot.singularity.auth.SingularityAuthorizationHelper;
@@ -43,9 +44,9 @@ public DisastersResource(DisasterManager disasterManager, SingularityAuthorizati
 
   @GET
   @Path("/stats")
-  public Optional<SingularityDisasterStats> disasterStats() {
+  public SingularityDisastersData disasterStats() {
     authorizationHelper.checkAdminAuthorization(user);
-    return disasterManager.getDisasterStats();
+    return disasterManager.getDisastersData();
   }
 
   @GET

diff --git a/...e/src/main/java/com/hubspot/singularity/scheduler/SingularityDisasterDetectionPoller.java b/...e/src/main/java/com/hubspot/singularity/scheduler/SingularityDisasterDetectionPoller.java
@@ -17,6 +17,7 @@
 import com.hubspot.singularity.MachineState;
 import com.hubspot.singularity.SingularityDisasterStats;
 import com.hubspot.singularity.SingularityDisasterType;
+import com.hubspot.singularity.SingularityDisastersData;
 import com.hubspot.singularity.SingularityEmailDestination;
 import com.hubspot.singularity.SingularityEmailType;
 import com.hubspot.singularity.SingularityPendingTask;
@@ -83,6 +84,9 @@ public void runActionOnPoll() {
 
     disasterManager.updateActiveDisasters(previouslyActiveDisasters, newActiveDisasters);
     disasterManager.saveDisasterStats(newStats);
+    if (lastStats.isPresent()) {
+      disasterManager.savePreviousDisasterStats(lastStats.get());
+    }
 
     if (!newActiveDisasters.isEmpty()) {
       if (!disasterManager.isAutomatedDisabledActionsDisabled()) {
@@ -192,7 +196,9 @@ private void queueDisasterEmail(SingularityDisasterStats stats, Optional<Singula
       return;
     }
 
-    final String body = String.format("Disasters detected: %s%nCurrent disaster stats: %s%nPrevious disaster stats: %s", disasters, stats, lastStats);
+    SingularityDisastersData data = new SingularityDisastersData(Optional.of(stats), lastStats, disasters);
+
+    final String body = String.format("New disasters detected. Data: %s ", data);
     final String subject = String.format("Disaster(s) Detected %s", disasters);
 
     smtpSender.queueMail(configuration.getSmtpConfiguration().get().getAdmins(), ImmutableList.<String> of(), subject, body);