purge dead slaves after a configurable amount of time

SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java

@@ -70,6 +70,8 @@ public class SingularityConfiguration extends Configuration {
 
   private long deleteDeploysFromZkWhenNoDatabaseAfterHours = TimeUnit.DAYS.toHours(14);
 
+  private long deleteDeadSlavesAfterHours = TimeUnit.DAYS.toHours(7);
+
   private long deleteStaleRequestsFromZkWhenNoDatabaseAfterHours = TimeUnit.DAYS.toHours(14);
 
   private long deleteTasksFromZkWhenNoDatabaseAfterHours = TimeUnit.DAYS.toHours(7);
@@ -316,6 +318,14 @@ public long getKillNonLongRunningTasksInCleanupAfterSeconds() {
     return killNonLongRunningTasksInCleanupAfterSeconds;
   }
 
+  public long getDeleteDeadSlavesAfterHours() {
+    return deleteDeadSlavesAfterHours;
+  }
+
+  public void setDeleteDeadSlavesAfterHours(long deleteDeadSlavesAfterHours) {
+    this.deleteDeadSlavesAfterHours = deleteDeadSlavesAfterHours;
+  }
+
   public int getListenerThreadpoolSize() {
     return listenerThreadpoolSize;
   }
@@ -396,6 +406,10 @@ public long getThreadpoolShutdownDelayInSeconds() {
     return threadpoolShutdownDelayInSeconds;
   }
 
+  public void setThreadpoolShutdownDelayInSeconds(long threadpoolShutdownDelayInSeconds) {
+    this.threadpoolShutdownDelayInSeconds = threadpoolShutdownDelayInSeconds;
+  }
+
   public UIConfiguration getUiConfiguration() {
     return uiConfiguration;
   }
@@ -727,4 +741,5 @@ public AuthConfiguration getAuthConfiguration() {
   public void setAuthConfiguration(AuthConfiguration authConfiguration) {
     this.authConfiguration = authConfiguration;
   }
+
 }

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeadSlavePoller.java

@@ -0,0 +1,64 @@
+package com.hubspot.singularity.scheduler;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import javax.inject.Singleton;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.hubspot.mesos.JavaUtils;
+import com.hubspot.singularity.MachineState;
+import com.hubspot.singularity.SingularityDeleteResult;
+import com.hubspot.singularity.SingularitySlave;
+import com.hubspot.singularity.config.SingularityConfiguration;
+import com.hubspot.singularity.data.SlaveManager;
+
+@Singleton
+public class SingularityDeadSlavePoller extends SingularityLeaderOnlyPoller {
+
+  private static final Logger LOG = LoggerFactory.getLogger(SingularityDeadSlavePoller.class);
+
+  private final SlaveManager slaveManager;
+  private final SingularityConfiguration configuration;
+
+  @Inject
+  SingularityDeadSlavePoller(SingularityConfiguration configuration, SlaveManager slaveManager) {
+    super(1, TimeUnit.HOURS);
+
+    this.slaveManager = slaveManager;
+    this.configuration = configuration;
+  }
+
+  @Override
+  public void runActionOnPoll() {
+    final long start = System.currentTimeMillis();
+
+    final List<SingularitySlave> deadSlaves = slaveManager.getObjectsFiltered(MachineState.DEAD);
+
+    if (deadSlaves.isEmpty()) {
+      LOG.trace("No dead slaves");
+      return;
+    }
+
+    int deleted = 0;
+    final long maxDuration = TimeUnit.HOURS.toMillis(configuration.getDeleteDeadSlavesAfterHours());
+
+    for (SingularitySlave deadSlave : slaveManager.getObjectsFiltered(MachineState.DEAD)) {
+      final long duration = System.currentTimeMillis() - deadSlave.getCurrentState().getTimestamp();
+
+      if (duration > maxDuration) {
+        SingularityDeleteResult result = slaveManager.deleteObject(deadSlave.getId());
+
+        deleted++;
+
+        LOG.info("Removing dead slave {} ({}) after {} (max {})", deadSlave.getId(), result, JavaUtils.durationFromMillis(duration), JavaUtils.durationFromMillis(maxDuration));
+      }
+    }
+
+    LOG.debug("Checked {} dead slaves, deleted {} in {}", deadSlaves.size(), deleted, JavaUtils.duration(start));
+  }
+
+}

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularitySchedulerModule.java

@@ -10,6 +10,7 @@ protected void configure() {
     bind(SingularityHealthchecker.class).in(Scopes.SINGLETON);
     bind(SingularityNewTaskChecker.class).in(Scopes.SINGLETON);
     bind(SingularityCleanupPoller.class).in(Scopes.SINGLETON);
+    bind(SingularityDeadSlavePoller.class).in(Scopes.SINGLETON);
     bind(SingularityCooldownPoller.class).in(Scopes.SINGLETON);
     bind(SingularityDeployPoller.class).in(Scopes.SINGLETON);
     bind(SingularityCooldownPoller.class).in(Scopes.SINGLETON);

SingularityService/src/test/java/com/hubspot/singularity/scheduler/SlaveTest.java

@@ -0,0 +1,55 @@
+package com.hubspot.singularity.scheduler;
+
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.hubspot.singularity.MachineState;
+import com.hubspot.singularity.SingularityCuratorTestBase;
+import com.hubspot.singularity.SingularityMachineStateHistoryUpdate;
+import com.hubspot.singularity.SingularitySlave;
+import com.hubspot.singularity.config.SingularityConfiguration;
+import com.hubspot.singularity.data.SlaveManager;
+
+
+public class SlaveTest extends SingularityCuratorTestBase {
+
+  @Inject
+  protected SlaveManager slaveManager;
+
+  @Inject
+  protected SingularityConfiguration configuration;
+
+  @Inject
+  protected SingularityDeadSlavePoller deadSlavePoller;
+
+  @Test
+  public void testDeadSlavesArePurged() {
+    SingularitySlave liveSlave = new SingularitySlave("1", "h1", "r1");
+    SingularitySlave deadSlave = new SingularitySlave("2", "h1", "r1");
+
+    final long now = System.currentTimeMillis();
+
+    liveSlave = liveSlave.changeState(new SingularityMachineStateHistoryUpdate("1", MachineState.ACTIVE, 100, Optional.<String> absent()));
+    deadSlave = deadSlave.changeState(new SingularityMachineStateHistoryUpdate("2", MachineState.DEAD, now - TimeUnit.HOURS.toMillis(10), Optional.<String> absent()));
+
+    slaveManager.saveObject(liveSlave);
+    slaveManager.saveObject(deadSlave);
+
+    deadSlavePoller.runActionOnPoll();
+
+    Assert.assertEquals(1, slaveManager.getObjectsFiltered(MachineState.ACTIVE).size());
+    Assert.assertEquals(1, slaveManager.getObjectsFiltered(MachineState.DEAD).size());
+
+    configuration.setDeleteDeadSlavesAfterHours(1);
+
+    deadSlavePoller.runActionOnPoll();
+
+    Assert.assertEquals(1, slaveManager.getObjectsFiltered(MachineState.ACTIVE).size());
+    Assert.assertEquals(0, slaveManager.getObjectsFiltered(MachineState.DEAD).size());
+  }
+
+}