Merge pull request #1473 from HubSpot/reconciliation_disable

More disabled actions for pollers

SingularityBase/src/main/java/com/hubspot/singularity/SingularityAction.java

@@ -5,13 +5,14 @@ public enum SingularityAction {
   KILL_TASK(true), BOUNCE_TASK(true), RUN_SHELL_COMMAND(true), ADD_METADATA(true),
   DEPLOY(true), CANCEL_DEPLOY(true),
   ADD_WEBHOOK(true), REMOVE_WEBHOOK(true), VIEW_WEBHOOKS(false),
-  TASK_RECONCILIATION(true),
+  TASK_RECONCILIATION(true), STARTUP_TASK_RECONCILIATION(true),
   RUN_HEALTH_CHECKS(true),
   ADD_DISASTER(false), REMOVE_DISASTER(false), DISABLE_ACTION(false), ENABLE_ACTION(false), VIEW_DISASTERS(false),
   FREEZE_SLAVE(true), ACTIVATE_SLAVE(true), DECOMMISSION_SLAVE(true), VIEW_SLAVES(false),
   FREEZE_RACK(true), ACTIVATE_RACK(true), DECOMMISSION_RACK(true), VIEW_RACKS(false),
   SEND_EMAIL(true),
-  PROCESS_OFFERS(true), CACHE_OFFERS(true);
+  PROCESS_OFFERS(true), CACHE_OFFERS(true),
+  RUN_CLEANUP_POLLER(true), RUN_DEPLOY_POLLER(true), RUN_SCHEDULER_POLLER(true), RUN_EXPIRING_ACTION_POLLER(true);
 
   private final boolean canDisable;
 

SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityStartup.java

@@ -19,6 +19,7 @@
 import com.hubspot.mesos.client.MesosClient;
 import com.hubspot.mesos.json.MesosMasterStateObject;
 import com.hubspot.singularity.RequestType;
+import com.hubspot.singularity.SingularityAction;
 import com.hubspot.singularity.SingularityDeployKey;
 import com.hubspot.singularity.SingularityPendingDeploy;
 import com.hubspot.singularity.SingularityPendingRequest;
@@ -33,6 +34,7 @@
 import com.hubspot.singularity.SingularityTaskId;
 import com.hubspot.singularity.SingularityTaskIdHolder;
 import com.hubspot.singularity.data.DeployManager;
+import com.hubspot.singularity.data.DisasterManager;
 import com.hubspot.singularity.data.RequestManager;
 import com.hubspot.singularity.data.TaskManager;
 import com.hubspot.singularity.data.zkmigrations.ZkDataMigrationRunner;
@@ -49,6 +51,7 @@ class SingularityStartup {
   private final TaskManager taskManager;
   private final RequestManager requestManager;
   private final DeployManager deployManager;
+  private final DisasterManager disasterManager;
   private final SingularitySlaveAndRackManager slaveAndRackManager;
   private final SingularityHealthchecker healthchecker;
   private final SingularityNewTaskChecker newTaskChecker;
@@ -57,12 +60,13 @@ class SingularityStartup {
 
   @Inject
   SingularityStartup(MesosClient mesosClient, SingularityHealthchecker healthchecker, SingularityNewTaskChecker newTaskChecker,
-      SingularitySlaveAndRackManager slaveAndRackManager, TaskManager taskManager, RequestManager requestManager, DeployManager deployManager, SingularityTaskReconciliation taskReconciliation,
-      ZkDataMigrationRunner zkDataMigrationRunner) {
+      SingularitySlaveAndRackManager slaveAndRackManager, TaskManager taskManager, RequestManager requestManager, DeployManager deployManager, DisasterManager disasterManager,
+      SingularityTaskReconciliation taskReconciliation, ZkDataMigrationRunner zkDataMigrationRunner) {
     this.mesosClient = mesosClient;
     this.zkDataMigrationRunner = zkDataMigrationRunner;
     this.slaveAndRackManager = slaveAndRackManager;
     this.deployManager = deployManager;
+    this.disasterManager = disasterManager;
     this.requestManager = requestManager;
     this.newTaskChecker = newTaskChecker;
     this.taskManager = taskManager;
@@ -87,7 +91,9 @@ public void startup(MasterInfo masterInfo, SchedulerDriver driver) throws Except
 
     enqueueHealthAndNewTaskChecks();
 
-    taskReconciliation.startReconciliation();
+    if (!disasterManager.isDisabled(SingularityAction.STARTUP_TASK_RECONCILIATION)) {
+      taskReconciliation.startReconciliation();
+    }
 
     LOG.info("Finished startup after {}", JavaUtils.duration(start));
   }

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleanupPoller.java

@@ -5,25 +5,38 @@
 
 import javax.inject.Singleton;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
+import com.hubspot.singularity.SingularityAction;
 import com.hubspot.singularity.config.SingularityConfiguration;
+import com.hubspot.singularity.data.DisasterManager;
 import com.hubspot.singularity.mesos.SingularityMesosModule;
 
 @Singleton
 public class SingularityCleanupPoller extends SingularityLeaderOnlyPoller {
 
+  private static final Logger LOG = LoggerFactory.getLogger(SingularityCleanupPoller.class);
+
   private final SingularityCleaner cleaner;
+  private final DisasterManager disasterManager;
 
   @Inject
-  SingularityCleanupPoller(SingularityConfiguration configuration, SingularityCleaner cleaner, @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock lock) {
+  SingularityCleanupPoller(SingularityConfiguration configuration, SingularityCleaner cleaner, @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock lock, DisasterManager disasterManager) {
     super(configuration.getCleanupEverySeconds(), TimeUnit.SECONDS, lock);
 
     this.cleaner = cleaner;
+    this.disasterManager = disasterManager;
   }
 
   @Override
   public void runActionOnPoll() {
-    cleaner.drainCleanupQueue();
+    if (!disasterManager.isDisabled(SingularityAction.RUN_CLEANUP_POLLER)) {
+      cleaner.drainCleanupQueue();
+    } else {
+      LOG.warn("Cleanup poller is currently disabled");
+    }
   }
 }

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityDeployPoller.java

@@ -11,7 +11,9 @@
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
 import com.hubspot.mesos.JavaUtils;
+import com.hubspot.singularity.SingularityAction;
 import com.hubspot.singularity.config.SingularityConfiguration;
+import com.hubspot.singularity.data.DisasterManager;
 import com.hubspot.singularity.mesos.SingularityMesosModule;
 
 @Singleton
@@ -20,24 +22,30 @@ public class SingularityDeployPoller extends SingularityLeaderOnlyPoller {
   private static final Logger LOG = LoggerFactory.getLogger(SingularityDeployPoller.class);
 
   private final SingularityDeployChecker deployChecker;
+  private final DisasterManager disasterManager;
 
   @Inject
-  SingularityDeployPoller(SingularityDeployChecker deployChecker, SingularityConfiguration configuration, @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock lock) {
+  SingularityDeployPoller(SingularityDeployChecker deployChecker, SingularityConfiguration configuration, @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock lock, DisasterManager disasterManager) {
     super(configuration.getCheckDeploysEverySeconds(), TimeUnit.SECONDS, lock);
 
     this.deployChecker = deployChecker;
+    this.disasterManager = disasterManager;
   }
 
   @Override
   public void runActionOnPoll() {
-    final long start = System.currentTimeMillis();
+    if (!disasterManager.isDisabled(SingularityAction.RUN_DEPLOY_POLLER)) {
+      final long start = System.currentTimeMillis();
 
-    final int numDeploys = deployChecker.checkDeploys();
+      final int numDeploys = deployChecker.checkDeploys();
 
-    if (numDeploys == 0) {
-      LOG.trace("No pending deploys");
+      if (numDeploys == 0) {
+        LOG.trace("No pending deploys");
+      } else {
+        LOG.info("Checked {} deploys in {}", numDeploys, JavaUtils.duration(start));
+      }
     } else {
-      LOG.info("Checked {} deploys in {}", numDeploys, JavaUtils.duration(start));
+      LOG.warn("Deploy poller is currently disabled");
     }
   }
 }

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityExpiringUserActionPoller.java

@@ -21,6 +21,7 @@
 import com.hubspot.singularity.ExtendedTaskState;
 import com.hubspot.singularity.MachineState;
 import com.hubspot.singularity.RequestState;
+import com.hubspot.singularity.SingularityAction;
 import com.hubspot.singularity.SingularityMachineAbstraction;
 import com.hubspot.singularity.SingularityPendingRequest;
 import com.hubspot.singularity.SingularityPendingRequest.PendingType;
@@ -38,6 +39,7 @@
 import com.hubspot.singularity.api.SingularityScaleRequest;
 import com.hubspot.singularity.config.SingularityConfiguration;
 import com.hubspot.singularity.data.DeployManager;
+import com.hubspot.singularity.data.DisasterManager;
 import com.hubspot.singularity.data.RackManager;
 import com.hubspot.singularity.data.RequestManager;
 import com.hubspot.singularity.data.SlaveManager;
@@ -67,10 +69,11 @@ public class SingularityExpiringUserActionPoller extends SingularityLeaderOnlyPo
   private final RackManager rackManager;
   private final List<SingularityExpiringUserActionHandler<?, ?>> handlers;
   private final SingularityConfiguration configuration;
+  private final DisasterManager disasterManager;
 
   @Inject
   SingularityExpiringUserActionPoller(SingularityConfiguration configuration, RequestManager requestManager, DeployManager deployManager, TaskManager taskManager, SlaveManager slaveManager, RackManager rackManager,
-      @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock lock, RequestHelper requestHelper, SingularityMailer mailer) {
+      @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock lock, RequestHelper requestHelper, SingularityMailer mailer, DisasterManager disasterManager) {
     super(configuration.getCheckExpiringUserActionEveryMillis(), TimeUnit.MILLISECONDS, lock);
 
     this.deployManager = deployManager;
@@ -81,6 +84,7 @@ public class SingularityExpiringUserActionPoller extends SingularityLeaderOnlyPo
     this.slaveManager = slaveManager;
     this.rackManager = rackManager;
     this.configuration = configuration;
+    this.disasterManager = disasterManager;
 
     List<SingularityExpiringUserActionHandler<?, ?>> tempHandlers = Lists.newArrayList();
     tempHandlers.add(new SingularityExpiringBounceHandler());
@@ -95,6 +99,10 @@ public class SingularityExpiringUserActionPoller extends SingularityLeaderOnlyPo
 
   @Override
   public void runActionOnPoll() {
+    if (disasterManager.isDisabled(SingularityAction.RUN_EXPIRING_ACTION_POLLER)) {
+      LOG.warn("Expiring user action poller is disabled");
+      return;
+    }
     for (SingularityExpiringUserActionHandler<?, ?> handler : handlers) {
       handler.checkExpiringObjects();
     }

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularitySchedulerPoller.java

@@ -20,7 +20,9 @@
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
 import com.hubspot.mesos.JavaUtils;
+import com.hubspot.singularity.SingularityAction;
 import com.hubspot.singularity.config.SingularityConfiguration;
+import com.hubspot.singularity.data.DisasterManager;
 import com.hubspot.singularity.mesos.OfferCache;
 import com.hubspot.singularity.mesos.SchedulerDriverSupplier;
 import com.hubspot.singularity.mesos.SingularityMesosModule;
@@ -36,19 +38,25 @@ public class SingularitySchedulerPoller extends SingularityLeaderOnlyPoller {
   private final OfferCache offerCache;
   private final SchedulerDriverSupplier schedulerDriverSupplier;
   private final SingularityMesosOfferScheduler offerScheduler;
+  private final DisasterManager disasterManager;
 
   @Inject
   SingularitySchedulerPoller(SingularityMesosOfferScheduler offerScheduler, OfferCache offerCache, SchedulerDriverSupplier schedulerDriverSupplier,
-      SingularityConfiguration configuration, @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock lock) {
+      SingularityConfiguration configuration, @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock lock, DisasterManager disasterManager) {
     super(configuration.getCheckSchedulerEverySeconds(), TimeUnit.SECONDS, lock);
 
     this.offerCache = offerCache;
     this.offerScheduler = offerScheduler;
     this.schedulerDriverSupplier = schedulerDriverSupplier;
+    this.disasterManager = disasterManager;
   }
 
   @Override
   public void runActionOnPoll() {
+    if (disasterManager.isDisabled(SingularityAction.RUN_SCHEDULER_POLLER)) {
+      LOG.warn("Scheduler poller is disabled");
+      return;
+    }
     final long start = System.currentTimeMillis();
 
     List<CachedOffer> cachedOffers = offerCache.checkoutOffers();

SingularityUI/app/components/disasters/NewDisabledActionModal.jsx

@@ -7,7 +7,8 @@ const DISABLED_ACTION_TYPES = [
   'BOUNCE_REQUEST', 'SCALE_REQUEST', 'REMOVE_REQUEST', 'CREATE_REQUEST', 'UPDATE_REQUEST','KILL_TASK', 
   'BOUNCE_TASK', 'RUN_HEALTH_CHECKS', 'RUN_SHELL_COMMAND', 'ADD_METADATA', 'DEPLOY', 'CANCEL_DEPLOY', 'ADD_WEBHOOK', 'REMOVE_WEBHOOK',
   'TASK_RECONCILIATION', 'FREEZE_SLAVE', 'ACTIVATE_SLAVE', 'DECOMMISSION_SLAVE', 'VIEW_SLAVES','FREEZE_RACK',
-  'ACTIVATE_RACK', 'DECOMMISSION_RACK', 'VIEW_RACKS', 'PROCESS_OFFERS', 'CACHE_OFFERS', 'SEND_EMAIL'
+  'ACTIVATE_RACK', 'DECOMMISSION_RACK', 'VIEW_RACKS', 'PROCESS_OFFERS', 'CACHE_OFFERS', 'SEND_EMAIL', 'STARTUP_TASK_RECONCILIATION',
+  'RUN_CLEANUP_POLLER', 'RUN_DEPLOY_POLLER', 'RUN_SCHEDULER_POLLER', 'RUN_EXPIRING_ACTION_POLLER'
 ];
 
 import Utils from '../../utils';