From 0c9b09ebc84d885c5816545f511e5ad8922c1543 Mon Sep 17 00:00:00 2001
From: Stephen Salinas <ssalinas@hubspot.com>
Date: Wed, 23 Nov 2016 16:19:26 -0500
Subject: [PATCH] Count losts tasks with a Meter

---
 .../singularity/SingularityMainModule.java     | 10 ++++++++++
 .../SingularityMesosStatusUpdateHandler.java   | 18 ++++++++++++------
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java
index 52fe21eb03..d74cf1f2d5 100644
--- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java
+++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java
@@ -24,6 +24,7 @@
 import org.jets3t.service.impl.rest.httpclient.RestS3Service;
 import org.jets3t.service.security.AWSCredentials;
 
+import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricRegistry;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Optional;
@@ -102,6 +103,8 @@ public class SingularityMainModule implements Module {
 
   public static final String CURRENT_HTTP_REQUEST = "_singularity_current_http_request";
 
+  public static final String LOST_TASKS_METER = "singularity.lost.tasks.meter";
+
   private final SingularityConfiguration configuration;
 
   public SingularityMainModule(final SingularityConfiguration configuration) {
@@ -347,4 +350,11 @@ public Optional<HttpServletRequest> providesUrl(Provider<HttpServletRequest> req
       return Optional.absent();
     }
   }
+
+  @Provides
+  @Singleton
+  @Named(LOST_TASKS_METER)
+  public Meter providesLostTasksMeter(MetricRegistry registry) {
+    return registry.meter("com.hubspot.singularity.lostTasks");
+  }
 }
diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosStatusUpdateHandler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosStatusUpdateHandler.java
index ffe4c37830..cba585d819 100644
--- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosStatusUpdateHandler.java
+++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosStatusUpdateHandler.java
@@ -15,6 +15,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.codahale.metrics.Meter;
 import com.codahale.metrics.annotation.Timed;
 import com.google.common.base.Optional;
 import com.google.common.base.Strings;
@@ -74,6 +75,7 @@ public class SingularityMesosStatusUpdateHandler implements Managed {
     private final SingularityAbort singularityAbort;
     private final SingularityConfiguration configuration;
     private final Multiset<Protos.TaskStatus.Reason> taskLostReasons;
+    private final Meter lostTasksMeter;
 
     private Future statusUpdateFuture;
 
@@ -87,7 +89,8 @@ public SingularityMesosStatusUpdateHandler(TaskManager taskManager, DeployManage
         @Named(SingularityMainModule.STATUS_UPDATE_THREADPOOL_NAME) ScheduledExecutorService executorService,
         SingularityConfiguration configuration,
         SingularityAbort singularityAbort,
-        @Named(SingularityMesosModule.TASK_LOST_REASONS_COUNTER) Multiset<Protos.TaskStatus.Reason> taskLostReasons) {
+        @Named(SingularityMesosModule.TASK_LOST_REASONS_COUNTER) Multiset<Protos.TaskStatus.Reason> taskLostReasons,
+        @Named(SingularityMainModule.LOST_TASKS_METER) Meter lostTasksMeter) {
         this.taskManager = taskManager;
         this.deployManager = deployManager;
         this.requestManager = requestManager;
@@ -105,6 +108,7 @@ public SingularityMesosStatusUpdateHandler(TaskManager taskManager, DeployManage
         this.singularityAbort = singularityAbort;
         this.configuration = configuration;
         this.taskLostReasons = taskLostReasons;
+        this.lostTasksMeter = lostTasksMeter;
         this.handlerStarted = new AtomicBoolean();
 
         this.statusUpdateQueue = new ArrayBlockingQueue<>(configuration.getStatusUpdateQueueCapacity());
@@ -167,9 +171,7 @@ private Optional<String> getStatusMessage(Protos.TaskStatus status, Optional<Sin
     }
 
     private void updateDisasterStats(Protos.TaskStatus status) {
-        if (status.getState() == TaskState.TASK_LOST) {
-            taskLostReasons.add(status.getReason());
-        }
+
     }
 
     private SchedulerDriver getSchedulerDriver() {
@@ -214,10 +216,14 @@ private void unsafeProcessStatusUpdate(Protos.TaskStatus status) {
             return;
         }
 
-        if (configuration.getDisasterDetection().isEnabled()) {
-            updateDisasterStats(status);
+        if (status.getState() == TaskState.TASK_LOST) {
+            lostTasksMeter.mark();
+            if (configuration.getDisasterDetection().isEnabled()) {
+                taskLostReasons.add(status.getReason());
+            }
         }
 
+
         final Optional<SingularityTask> task = taskManager.getTask(taskIdObj);
 
         final boolean isActiveTask = taskManager.isActiveTask(taskId);