refactoring

HubSpot · Sep 12, 2016 · 727215c · 727215c
1 parent d5dede1
commit 727215c
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 18 deletions.
diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java b/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java
@@ -3,8 +3,8 @@
 public enum TaskCleanupType {
 
   USER_REQUESTED(true, true), USER_REQUESTED_TASK_BOUNCE(false, false), DECOMISSIONING(false, false), SCALING_DOWN(true, false), BOUNCING(false, false), INCREMENTAL_BOUNCE(false, false),
-  DEPLOY_FAILED(true, true), NEW_DEPLOY_SUCCEEDED(true, false), DEPLOY_STEP_FINISHED(true, false), DEPLOY_CANCELED(true, true), DEPLOY_TIMED_OUT(true, true), UNHEALTHY_NEW_TASK(true, true),
-  OVERDUE_NEW_TASK(true, true), USER_REQUESTED_DESTROY(true, true), INCREMENTAL_DEPLOY_FAILED(false, true), INCREMENTAL_DEPLOY_CANCELLED(false, true);
+  DEPLOY_FAILED(true, true), NEW_DEPLOY_SUCCEEDED(true, false), DEPLOY_STEP_FINISHED(true, false), DEPLOY_CANCELED(true, true), TASK_EXCEEDED_TIME_LIMIT(true, true), UNHEALTHY_NEW_TASK(true, true),
+  OVERDUE_NEW_TASK(true, true), USER_REQUESTED_DESTROY(true, true), INCREMENTAL_DEPLOY_FAILED(false, true), INCREMENTAL_DEPLOY_CANCELLED(false, true), PRIORITY_KILL(true, true);
 
   private final boolean killLongRunningTaskInstantly;
   private final boolean killNonLongRunningTaskInstantly;

diff --git a/...ularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/...ularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java
@@ -214,10 +214,10 @@ public class SingularityConfiguration extends Configuration {
 
   private long warnIfScheduledJobIsRunningForAtLeastMillis = TimeUnit.DAYS.toMillis(1);
 
-  @JsonProperty("taskExecutionTimeout")
+  @JsonProperty("taskExecutionTimeLimitMillis")
   @Valid
   @NotNull
-  private long taskExecutionTimeout = TimeUnit.HOURS.toMillis(12);
+  private Optional<Long> taskExecutionTimeLimitMillis = Optional.absent();
 
   private int warnIfScheduledJobIsRunningPastNextRunPct = 200;
 
@@ -609,8 +609,8 @@ public long getWarnIfScheduledJobIsRunningForAtLeastMillis() {
     return warnIfScheduledJobIsRunningForAtLeastMillis;
   }
 
-  public long getTaskExecutionTimeout() {
-    return taskExecutionTimeout;
+  public Optional<Long> getTaskExecutionTimeLimitMillis() {
+    return taskExecutionTimeLimitMillis;
   }
 
   public int getWarnIfScheduledJobIsRunningPastNextRunPct() {
@@ -933,8 +933,8 @@ public void setWarnIfScheduledJobIsRunningForAtLeastMillis(long warnIfScheduledJ
     this.warnIfScheduledJobIsRunningForAtLeastMillis = warnIfScheduledJobIsRunningForAtLeastMillis;
   }
 
-  public SingularityConfiguration setTaskExecutionTimeout(long taskExecutionTimeout) {
-    this.taskExecutionTimeout = taskExecutionTimeout;
+  public SingularityConfiguration setTaskExecutionTimeLimitMillis(Optional<Long> taskExecutionTimeLimitMillis) {
+    this.taskExecutionTimeLimitMillis = taskExecutionTimeLimitMillis;
     return this;
   }
 

diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityJobPoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityJobPoller.java
@@ -72,15 +72,6 @@ public void runActionOnPoll() {
         continue;
       }
 
-      if (start - taskId.getStartedAt() >= configuration.getTaskExecutionTimeout()) {
-        taskManager.createTaskCleanup(new SingularityTaskCleanup(
-            Optional.<String>absent(),
-            TaskCleanupType.DEPLOY_TIMED_OUT,
-            start,
-            taskId,
-            Optional.of("Deploy has reached/exceeded the set timeout"),
-            Optional.of(UUID.randomUUID().toString())));
-      }
 
       requestIdsToLookup.add(taskId.getRequestId());
     }
@@ -91,11 +82,30 @@ public void runActionOnPoll() {
     for (SingularityTaskId taskId : activeTaskIds) {
       SingularityRequestWithState request = idToRequest.get(taskId.getRequestId());
 
-      if (request == null || !request.getRequest().isScheduled() || taskManager.hasNotifiedOverdue(taskId)) {
+      if (request == null) {
         continue;
       }
 
       final long runtime = start - taskId.getStartedAt();
+
+      if (!request.getRequest().getRequestType().isLongRunning() &&
+          configuration.getTaskExecutionTimeLimitMillis().isPresent() &&
+          (start - taskId.getStartedAt()) >= configuration.getTaskExecutionTimeLimitMillis().get()) {
+        taskManager.createTaskCleanup(new SingularityTaskCleanup(
+            Optional.<String>absent(),
+            TaskCleanupType.TASK_EXCEEDED_TIME_LIMIT,
+            start,
+            taskId,
+            Optional.of(String.format("Task has run for %s milliseconds, which exceeds the maximum execution time of %s milliseconds",
+                start - taskId.getStartedAt(),
+                configuration.getTaskExecutionTimeLimitMillis().get())),
+            Optional.of(UUID.randomUUID().toString())));
+      }
+
+      if (!request.getRequest().isScheduled() || taskManager.hasNotifiedOverdue(taskId)) {
+        continue;
+      }
+
       final Optional<Long> expectedRuntime = getExpectedRuntime(request, taskId);
 
       if (!expectedRuntime.isPresent()) {