Don't operate directly on pending tasks during statusUpdate

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java

@@ -603,16 +603,14 @@ private Optional<PendingType> handleCompletedTaskWithStatistics(Optional<Singula
     if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
       // TODO send not cooldown anymore email
       LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
-      requestState = RequestState.ACTIVE;
       requestManager.exitCooldown(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent());
     }
 
     SingularityPendingRequest pendingRequest = new SingularityPendingRequest(request.getId(), requestDeployState.get().getActiveDeploy().get().getDeployId(),
-      System.currentTimeMillis(), Optional.<String>absent(), pendingType, cmdLineArgsList, Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(),
-      Optional.<String>absent());
+        System.currentTimeMillis(), Optional.absent(), pendingType, cmdLineArgsList, Optional.absent(), Optional.absent(), Optional.absent(),
+        Optional.absent());
 
-    SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId());
-    scheduleTasks(request, requestState, deployStatistics, pendingRequest, getMatchingTaskIds(request, deployKey), maybePendingDeploy);
+    requestManager.addToPendingQueue(pendingRequest);
 
     return Optional.of(pendingType);
   }

SingularityService/src/test/java/com/hubspot/singularity/data/StateManagerTest.java

@@ -39,6 +39,8 @@ public void itDoesntCountCleaningTasks() {
 
     SingularityTask task = taskManager.getActiveTasks().get(0);
     statusUpdate(task, TaskState.TASK_KILLED);
+    scheduler.drainPendingQueue();
+
     taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.BOUNCING, 1L, task.getTaskId(), Optional.absent(), Optional.absent(), Optional.absent()));
     Assert.assertEquals(2, taskManager.getActiveTaskIds().size());
     Assert.assertEquals(0, stateManager.getState(true, false).getOverProvisionedRequests());

SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityDeploysTest.java

@@ -61,6 +61,7 @@ public void testDeployManagerHandlesFailedLBTask() {
 
     statusUpdate(secondTask, TaskState.TASK_FAILED);
     statusUpdate(firstTask, TaskState.TASK_FAILED);
+    scheduler.drainPendingQueue();
 
     deployChecker.checkDeploys();
 

SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java

@@ -24,14 +24,10 @@
 import org.mockito.Mockito;
 
 import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;
 import com.google.inject.Inject;
 import com.hubspot.baragon.models.BaragonRequestState;
-import com.hubspot.mesos.protos.MesosTaskState;
-import com.hubspot.singularity.helpers.MesosProtosUtils;
-import com.hubspot.singularity.helpers.MesosUtils;
 import com.hubspot.mesos.Resources;
 import com.hubspot.mesos.SingularityContainerInfo;
 import com.hubspot.mesos.SingularityContainerType;
@@ -40,6 +36,7 @@
 import com.hubspot.mesos.SingularityDockerPortMapping;
 import com.hubspot.mesos.SingularityPortMappingType;
 import com.hubspot.mesos.SingularityVolume;
+import com.hubspot.mesos.protos.MesosTaskState;
 import com.hubspot.singularity.DeployState;
 import com.hubspot.singularity.ExtendedTaskState;
 import com.hubspot.singularity.LoadBalancerRequestType;
@@ -88,6 +85,8 @@
 import com.hubspot.singularity.api.SingularityScaleRequest;
 import com.hubspot.singularity.data.AbstractMachineManager.StateChangeResult;
 import com.hubspot.singularity.data.SingularityValidator;
+import com.hubspot.singularity.helpers.MesosProtosUtils;
+import com.hubspot.singularity.helpers.MesosUtils;
 import com.hubspot.singularity.mesos.OfferCache;
 import com.hubspot.singularity.mesos.SingularityMesosTaskPrioritizer;
 import com.hubspot.singularity.scheduler.SingularityDeployHealthHelper.DeployHealth;
@@ -1472,6 +1471,7 @@ public void testRunNowScheduledJobDoesNotRetry() {
 
     SingularityTask task = taskManager.getActiveTasks().get(0);
     statusUpdate(task, TaskState.TASK_FAILED);
+    scheduler.drainPendingQueue();
 
     SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get();
 
@@ -1544,6 +1544,7 @@ public void testJobRescheduledWhenItFinishesDuringDecommission() {
     cleaner.drainCleanupQueue();
 
     statusUpdate(task, TaskState.TASK_FINISHED);
+    scheduler.drainPendingQueue();
 
     Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty());
   }
@@ -1675,6 +1676,7 @@ public void testWaitAfterTaskWorks() {
     SingularityTask task = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
 
     statusUpdate(task, TaskState.TASK_FAILED);
+    scheduler.drainPendingQueue();
 
     Assert.assertTrue(taskManager.getPendingTaskIds().get(0).getNextRunAt() - System.currentTimeMillis() < 1000L);
 
@@ -1686,6 +1688,7 @@ public void testWaitAfterTaskWorks() {
     resourceOffers();
 
     statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FAILED);
+    scheduler.drainPendingQueue();
 
     Assert.assertTrue(taskManager.getPendingTaskIds().get(0).getNextRunAt() - System.currentTimeMillis() > 1000L);
     Assert.assertEquals(1, taskManager.getActiveTaskIds().size());

SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java

@@ -393,6 +393,7 @@ protected void killKilledTasks() {
     for (SingularityKilledTaskIdRecord killed : taskManager.getKilledTaskIdRecords()) {
       statusUpdate(taskManager.getTask(killed.getTaskId()).get(), TaskState.TASK_KILLED);
     }
+    scheduler.drainPendingQueue();
   }
 
   protected void finishNewTaskChecksAndCleanup() {