diff --git a/Docs/reference/api.md b/Docs/reference/api.md index ed9d21bb56..22d84b9aa9 100644 --- a/Docs/reference/api.md +++ b/Docs/reference/api.md @@ -3808,7 +3808,7 @@ Start a new deployment for a Request | name | type | required | description | |------|------|----------|-------------| | expiringSkipHealthchecks | [SingularityExpiringSkipHealthchecks](#model-SingularityExpiringSkipHealthchecks) | optional | | -| state | [RequestState](#model-RequestState) | optional | Allowable values: ACTIVE, DELETED, PAUSED, SYSTEM_COOLDOWN, FINISHED, DEPLOYING_TO_UNPAUSE | +| state | [RequestState](#model-RequestState) | optional | Allowable values: ACTIVE, DELETING, DELETED, PAUSED, SYSTEM_COOLDOWN, FINISHED, DEPLOYING_TO_UNPAUSE | | pendingDeploy | [SingularityDeploy](#model-SingularityDeploy) | optional | | | activeDeploy | [SingularityDeploy](#model-SingularityDeploy) | optional | | | expiringPause | [SingularityExpiringPause](#model-SingularityExpiringPause) | optional | | diff --git a/Docs/reference/apidocs/models.md b/Docs/reference/apidocs/models.md index da804605f6..85357caacb 100644 --- a/Docs/reference/apidocs/models.md +++ b/Docs/reference/apidocs/models.md @@ -792,7 +792,7 @@ Models: | name | type | required | description | |------|------|----------|-------------| | expiringSkipHealthchecks | [SingularityExpiringSkipHealthchecks](models.md#model-SingularityExpiringSkipHealthchecks) | optional | | -| state | [RequestState](models.md#model-RequestState) | optional | Allowable values: ACTIVE, DELETED, PAUSED, SYSTEM_COOLDOWN, FINISHED, DEPLOYING_TO_UNPAUSE | +| state | [RequestState](models.md#model-RequestState) | optional | Allowable values: ACTIVE, DELETING, DELETED, PAUSED, SYSTEM_COOLDOWN, FINISHED, DEPLOYING_TO_UNPAUSE | | pendingDeploy | [SingularityDeploy](models.md#model-SingularityDeploy) | optional | | | activeDeploy | [SingularityDeploy](models.md#model-SingularityDeploy) | optional | | | expiringPause | [SingularityExpiringPause](models.md#model-SingularityExpiringPause) | optional | | diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/RequestState.java b/SingularityBase/src/main/java/com/hubspot/singularity/RequestState.java index ca95b7cf41..c758e85ddc 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/RequestState.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/RequestState.java @@ -2,7 +2,7 @@ public enum RequestState { - ACTIVE(true), DELETED(false), PAUSED(false), SYSTEM_COOLDOWN(true), FINISHED(false), DEPLOYING_TO_UNPAUSE(true); + ACTIVE(true), DELETING(false), DELETED(false), PAUSED(false), SYSTEM_COOLDOWN(true), FINISHED(false), DEPLOYING_TO_UNPAUSE(true); private final boolean isRunnable; diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeleteResult.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeleteResult.java index 0e4092fcbb..8674ba6a2f 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeleteResult.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeleteResult.java @@ -1,5 +1,5 @@ package com.hubspot.singularity; public enum SingularityDeleteResult { - DELETED, DIDNT_EXIST; + DELETED, DIDNT_EXIST } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestHistory.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestHistory.java index 89577e15e0..457544ab50 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestHistory.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestHistory.java @@ -16,7 +16,7 @@ public class SingularityRequestHistory implements Comparable message; public enum RequestHistoryType { - CREATED, UPDATED, DELETED, PAUSED, UNPAUSED, ENTERED_COOLDOWN, EXITED_COOLDOWN, FINISHED, DEPLOYED_TO_UNPAUSE, BOUNCED, SCALED, SCALE_REVERTED; + CREATED, UPDATED, DELETING, DELETED, PAUSED, UNPAUSED, ENTERED_COOLDOWN, EXITED_COOLDOWN, FINISHED, DEPLOYED_TO_UNPAUSE, BOUNCED, SCALED, SCALE_REVERTED; } @JsonCreator diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java b/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java index d05cb3df57..43491a2643 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java @@ -5,7 +5,7 @@ public enum TaskCleanupType { USER_REQUESTED(true, true), USER_REQUESTED_TASK_BOUNCE(false, false), DECOMISSIONING(false, false), SCALING_DOWN(true, false), BOUNCING(false, false), INCREMENTAL_BOUNCE(false, false), DEPLOY_FAILED(true, true), NEW_DEPLOY_SUCCEEDED(true, false), DEPLOY_STEP_FINISHED(true, false), DEPLOY_CANCELED(true, true), TASK_EXCEEDED_TIME_LIMIT(true, true), UNHEALTHY_NEW_TASK(true, true), OVERDUE_NEW_TASK(true, true), USER_REQUESTED_DESTROY(true, true), INCREMENTAL_DEPLOY_FAILED(false, true), INCREMENTAL_DEPLOY_CANCELLED(false, true), PRIORITY_KILL(true, true), REBALANCE_RACKS(false, false), - PAUSING(false, false), PAUSE(true, true), DECOMMISSION_TIMEOUT(true, true); + PAUSING(false, false), PAUSE(true, true), DECOMMISSION_TIMEOUT(true, true), REQUEST_DELETING(true, true); private final boolean killLongRunningTaskInstantly; private final boolean killNonLongRunningTaskInstantly; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityDriverManager.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityDriverManager.java index 4ef0eb3767..cf1a0d840f 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityDriverManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityDriverManager.java @@ -95,7 +95,7 @@ public Protos.Status killAndRecord(SingularityTaskId taskId, Optional maybeCleanupFromRequestAndTask = getTaskCleanupType(requestCleanupType, taskCleanupType); - if (maybeCleanupFromRequestAndTask.isPresent() && maybeCleanupFromRequestAndTask.get() == TaskCleanupType.USER_REQUESTED_DESTROY) { + if (maybeCleanupFromRequestAndTask.isPresent() && (maybeCleanupFromRequestAndTask.get() == TaskCleanupType.USER_REQUESTED_DESTROY || maybeCleanupFromRequestAndTask.get() == TaskCleanupType.REQUEST_DELETING)) { Optional task = taskManager.getTask(taskId); if (task.isPresent()) { if (task.get().getMesosTask().hasExecutor()) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/CuratorManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/CuratorManager.java index faad40f4aa..541abfd94a 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/CuratorManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/CuratorManager.java @@ -138,6 +138,7 @@ protected SingularityDeleteResult delete(String path) { final long start = System.currentTimeMillis(); try { + // moves RequestState to DELETED curator.delete().deletingChildrenIfNeeded().forPath(path); return SingularityDeleteResult.DELETED; } catch (NoNodeException nne) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/RequestManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/RequestManager.java index b427c21cbd..6e818acde1 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/RequestManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/RequestManager.java @@ -10,7 +10,6 @@ import org.slf4j.LoggerFactory; import com.codahale.metrics.MetricRegistry; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableMap; @@ -37,7 +36,6 @@ import com.hubspot.singularity.data.transcoders.Transcoder; import com.hubspot.singularity.event.SingularityEventListener; import com.hubspot.singularity.expiring.SingularityExpiringBounce; -import com.hubspot.singularity.expiring.SingularityExpiringParent; import com.hubspot.singularity.expiring.SingularityExpiringPause; import com.hubspot.singularity.expiring.SingularityExpiringRequestActionParent; import com.hubspot.singularity.expiring.SingularityExpiringScale; @@ -262,6 +260,15 @@ public SingularityCreateResult activate(SingularityRequest request, RequestHisto return save(request, RequestState.ACTIVE, historyType, timestamp, user, message); } + public SingularityCreateResult markDeleting(SingularityRequest request, long timestamp, Optional user, Optional message) { + return save(request, RequestState.DELETING, RequestHistoryType.DELETING, timestamp, user, message); + } + + public SingularityDeleteResult markDeleted(SingularityRequest request, long timestamp, Optional user, Optional message) { + save(request, RequestState.DELETED, RequestHistoryType.DELETED, timestamp, user, message); + return delete(getRequestPath(request.getId())); + } + public List getPendingRequests() { return getAsyncChildren(PENDING_PATH_ROOT, pendingRequestTranscoder); } @@ -323,20 +330,16 @@ public Optional getRequest(String requestId) { return getData(getRequestPath(requestId), requestTranscoder); } - public SingularityDeleteResult deleteRequest(SingularityRequest request, Optional user, Optional actionId, Optional message) { + public void startDeletingRequest(SingularityRequest request, Optional user, Optional actionId, Optional message) { final long now = System.currentTimeMillis(); // delete it no matter if the delete request already exists. createCleanupRequest(new SingularityRequestCleanup(user, RequestCleanupType.DELETING, now, Optional.of(Boolean.TRUE), request.getId(), Optional. absent(), Optional. absent(), message, actionId, Optional.absent())); - saveHistory(new SingularityRequestHistory(now, user, RequestHistoryType.DELETED, request, message)); - - SingularityDeleteResult deleteResult = delete(getRequestPath(request.getId())); - - LOG.info("Request {} deleted ({}) by {} - {}", request.getId(), deleteResult, user, message); + markDeleting(request, System.currentTimeMillis(), user, message); - return deleteResult; + LOG.info("Request {} enqueued for deletion by {} - {}", request.getId(), user, message); } public List getLbCleanupRequests() { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/RequestResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/RequestResource.java index 943df6d2e8..81bdaa4806 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/RequestResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/RequestResource.java @@ -545,7 +545,7 @@ public SingularityRequest deleteRequest(@ApiParam("The request ID to delete.") @ message = deleteRequest.get().getMessage(); } - requestManager.deleteRequest(request, JavaUtils.getUserEmail(user), actionId, message); + requestManager.startDeletingRequest(request, JavaUtils.getUserEmail(user), actionId, message); mailer.sendRequestRemovedMail(request, JavaUtils.getUserEmail(user), message); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java index 0aaecc6611..043c76b610 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityCleaner.java @@ -2,8 +2,10 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -23,6 +25,7 @@ import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.LoadBalancerRequestType; import com.hubspot.singularity.LoadBalancerRequestType.LoadBalancerRequestId; +import com.hubspot.singularity.RequestCleanupType; import com.hubspot.singularity.RequestState; import com.hubspot.singularity.SingularityDeleteResult; import com.hubspot.singularity.SingularityDeploy; @@ -39,6 +42,7 @@ import com.hubspot.singularity.SingularityRequestHistory; import com.hubspot.singularity.SingularityRequestLbCleanup; import com.hubspot.singularity.SingularityRequestWithState; +import com.hubspot.singularity.SingularityShellCommand; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskCleanup; import com.hubspot.singularity.SingularityTaskId; @@ -123,8 +127,7 @@ private boolean shouldKillTask(SingularityTaskCleanup taskCleanup, List maybeHistory = requestHistoryHelper.getLastHistory(requestId); - if (maybeHistory.isPresent() && maybeHistory.get().getRequest().isLoadBalanced() && configuration.isDeleteRemovedRequestsFromLoadBalancer()) { - createLbCleanupRequest(requestId, matchingActiveTaskIds); + if (maybeHistory.isPresent()) { + if (maybeHistory.get().getRequest().isLoadBalanced() && configuration.isDeleteRemovedRequestsFromLoadBalancer()) { + createLbCleanupRequest(requestId, matchingActiveTaskIds); + } + requestManager.markDeleted(maybeHistory.get().getRequest(), start, requestCleanup.getUser(), requestCleanup.getMessage()); } cleanupDeployState(requestCleanup); } @@ -432,7 +436,7 @@ private void bounce(SingularityRequestCleanup requestCleanup, final List activeTaskIds, boolean killActiveTasks) { + private TaskCleanupType pause(SingularityRequestCleanup requestCleanup, Iterable activeTaskIds) { final long start = System.currentTimeMillis(); boolean killTasks = requestCleanup.getKillTasks().or(configuration.isDefaultValueForKillTasksOfPausedRequests()); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { @@ -458,6 +462,24 @@ private TaskCleanupType pause(SingularityRequestCleanup requestCleanup, Iterable return cleanupType; } + private void delete(SingularityRequestCleanup requestCleanup, Iterable activeTaskIds){ + final long start = System.currentTimeMillis(); + + for (SingularityTaskId taskId : activeTaskIds) { + LOG.debug("Adding task {} to cleanup (delete)", taskId.getId()); + + Optional runBeforeKillId = Optional.absent(); + + if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { + SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); + taskManager.saveTaskShellCommandRequestToQueue(shellRequest); + runBeforeKillId = Optional.of(shellRequest.getId()); + } + + taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), TaskCleanupType.REQUEST_DELETING, start, taskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId)); + } + } + private void cleanupDeployState(SingularityRequestCleanup requestCleanup) { SingularityDeleteResult deletePendingDeployResult = deployManager.deletePendingDeploy(requestCleanup.getRequestId()); SingularityDeleteResult deleteRequestDeployStateResult = deployManager.deleteRequestDeployState(requestCleanup.getRequestId()); @@ -536,6 +558,7 @@ private void drainTaskCleanupQueue() { final Multiset incrementalCleaningTasks = HashMultiset.create(cleanupTasks.size()); final Set isBouncing = new HashSet<>(cleanupTasks.size()); + final Map> deletedRequestIdToTaskIds = new HashMap<>(); final List cleaningTasks = Lists.newArrayListWithCapacity(cleanupTasks.size()); for (SingularityTaskCleanup cleanupTask : cleanupTasks) { @@ -546,6 +569,9 @@ private void drainTaskCleanupQueue() { if (cleanupTask.getCleanupType() == TaskCleanupType.BOUNCING || cleanupTask.getCleanupType() == TaskCleanupType.INCREMENTAL_BOUNCE) { isBouncing.add(SingularityDeployKey.fromTaskId(cleanupTask.getTaskId())); } + if (cleanupTask.getCleanupType() == TaskCleanupType.REQUEST_DELETING) { + updateRequestToTaskMap(cleanupTask, deletedRequestIdToTaskIds); + } } LOG.info("Cleaning up {} tasks", cleanupTasks.size()); @@ -555,17 +581,19 @@ private void drainTaskCleanupQueue() { int killedTasks = 0; for (SingularityTaskCleanup cleanupTask : cleanupTasks) { + SingularityTaskId taskId = cleanupTask.getTaskId(); + if (!isValidTask(cleanupTask)) { LOG.info("Couldn't find a matching active task for cleanup task {}, deleting..", cleanupTask); - taskManager.deleteCleanupTask(cleanupTask.getTaskId().getId()); + taskManager.deleteCleanupTask(taskId.getId()); } else if (shouldKillTask(cleanupTask, activeTaskIds, cleaningTasks, incrementalCleaningTasks) && checkLBStateAndShouldKillTask(cleanupTask)) { - driverManager.killAndRecord(cleanupTask.getTaskId(), cleanupTask.getCleanupType(), cleanupTask.getUser()); - - taskManager.deleteCleanupTask(cleanupTask.getTaskId().getId()); + driverManager.killAndRecord(taskId, cleanupTask.getCleanupType(), cleanupTask.getUser()); + cleanupRequestIfNoRemainingTasks(cleanupTask, deletedRequestIdToTaskIds); + taskManager.deleteCleanupTask(taskId.getId()); killedTasks++; } else if (cleanupTask.getCleanupType() == TaskCleanupType.BOUNCING || cleanupTask.getCleanupType() == TaskCleanupType.INCREMENTAL_BOUNCE) { - isBouncing.remove(SingularityDeployKey.fromTaskId(cleanupTask.getTaskId())); + isBouncing.remove(SingularityDeployKey.fromTaskId(taskId)); } } @@ -582,6 +610,35 @@ private void drainTaskCleanupQueue() { LOG.info("Killed {} tasks in {}", killedTasks, JavaUtils.duration(start)); } + private void updateRequestToTaskMap(SingularityTaskCleanup cleanupTask, Map> requestIdToTaskIds) { + SingularityTaskId taskId = cleanupTask.getTaskId(); + List requestTasks = requestIdToTaskIds.get(taskId.getRequestId()); + if (requestTasks != null) { + requestTasks.add(taskId.getId()); + } else { + requestTasks = new ArrayList<>(Collections.singletonList(taskId.getId())); + requestIdToTaskIds.put(taskId.getRequestId(), requestTasks); + } + } + + private void cleanupRequestIfNoRemainingTasks(SingularityTaskCleanup cleanupTask, Map> requestIdToTaskIds) { + String requestId = cleanupTask.getTaskId().getRequestId(); + if (requestIdToTaskIds.get(requestId) == null) { + LOG.info("Couldn't find tasks for requestId {}", requestId); + return; + } + List requestTasks = requestIdToTaskIds.get(requestId); + requestTasks.remove(cleanupTask.getTaskId().getId()); + if (requestTasks.isEmpty()) { + LOG.info("All tasks for requestId {} are now killed", requestId); + requestManager.createCleanupRequest( + new SingularityRequestCleanup( + cleanupTask.getUser(), RequestCleanupType.DELETING, System.currentTimeMillis(), + Optional.of(Boolean.TRUE), requestId, Optional. absent(), + Optional. absent(), cleanupTask.getMessage(), Optional. absent(), Optional.absent())); + } + } + private boolean checkLBStateAndShouldKillTask(SingularityTaskCleanup cleanupTask) { final long start = System.currentTimeMillis(); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java b/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java index 459f19c6e4..8d14cd6d7e 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java @@ -28,11 +28,11 @@ import com.hubspot.singularity.config.HistoryPurgingConfiguration; import com.hubspot.singularity.data.TaskManager; import com.hubspot.singularity.data.history.HistoryManager; -import com.hubspot.singularity.scheduler.SingularitySchedulerTestBase; import com.hubspot.singularity.data.history.SingularityHistoryPurger; import com.hubspot.singularity.data.history.SingularityRequestHistoryPersister; import com.hubspot.singularity.data.history.SingularityTaskHistoryPersister; import com.hubspot.singularity.data.history.TaskHistoryHelper; +import com.hubspot.singularity.scheduler.SingularitySchedulerTestBase; import liquibase.Liquibase; import liquibase.database.Database; @@ -495,19 +495,18 @@ public void testMessage() { List history = historyManager.getRequestHistory(requestId, Optional.of(OrderDirection.DESC), 0, 100); - Assert.assertEquals(3, history.size()); + Assert.assertEquals(4, history.size()); for (SingularityRequestHistory historyItem : history) { if (historyItem.getEventType() == RequestHistoryType.DELETED) { Assert.assertEquals("a msg", historyItem.getMessage().get()); } else if (historyItem.getEventType() == RequestHistoryType.SCALED) { Assert.assertEquals(280, historyItem.getMessage().get().length()); + } else if (historyItem.getEventType() == RequestHistoryType.DELETING) { + Assert.assertEquals("a msg", historyItem.getMessage().get()); } else { Assert.assertTrue(!historyItem.getMessage().isPresent()); } } - } - - } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/HistoryPersisterTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/HistoryPersisterTest.java index 9936e2ed27..2b8b91b24e 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/HistoryPersisterTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/HistoryPersisterTest.java @@ -48,7 +48,7 @@ public void testRequestAgePurging() { Assert.assertTrue(!requestManager.getRequestHistory(requestId).isEmpty()); - requestManager.deleteRequest(request, user, Optional. absent(), Optional. absent()); + requestManager.startDeletingRequest(request, user, Optional. absent(), Optional. absent()); requestManager.deleteHistoryParent(requestId); @@ -79,17 +79,17 @@ public void testRequestCountPurging() { configuration.setMaxRequestsWithHistoryInZkWhenNoDatabase(Optional.of(2)); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(7); - requestManager.deleteRequest(requestOne, user, Optional.absent(), Optional.absent()); + requestManager.startDeletingRequest(requestOne, user, Optional.absent(), Optional.absent()); requestManager.deleteHistoryParent(requestOne.getId()); requestManager.activate(requestOne, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(4), Optional. absent(), Optional. absent()); requestManager.cooldown(requestOne, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3)); - requestManager.deleteRequest(requestTwo, user, Optional.absent(), Optional.absent()); + requestManager.startDeletingRequest(requestTwo, user, Optional.absent(), Optional.absent()); requestManager.deleteHistoryParent(requestTwo.getId()); requestManager.activate(requestTwo, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(4), Optional. absent(), Optional. absent()); requestManager.cooldown(requestTwo, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3)); - requestManager.deleteRequest(requestThree, user, Optional.absent(), Optional.absent()); + requestManager.startDeletingRequest(requestThree, user, Optional.absent(), Optional.absent()); requestManager.deleteHistoryParent(requestThree.getId()); requestManager.activate(requestThree, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(4), Optional. absent(), Optional. absent()); requestManager.cooldown(requestThree, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3)); @@ -198,7 +198,7 @@ public void testPurgingDoesntApplyIfDatabasePresent() { initRequest(); initFirstDeploy(); - requestManager.deleteRequest(request, user, Optional. absent(), Optional. absent()); + requestManager.startDeletingRequest(request, user, Optional. absent(), Optional. absent()); requestManager.deleteHistoryParent(requestId); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java index eb6d9da1ad..e09b662f4c 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java @@ -1,5 +1,6 @@ package com.hubspot.singularity.scheduler; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -177,6 +178,26 @@ public void testTaskKill() { Assert.assertEquals(0, taskManager.getNumActiveTasks()); } + @Test + public void testTaskDestroy() { + initRequest(); + initFirstDeploy(); + + SingularityTask firstTask = startTask(firstDeploy, 1); + SingularityTask secondTask = startTask(firstDeploy, 2); + SingularityTask thirdTask = startTask(firstDeploy, 3); + + taskResource.killTask(secondTask.getTaskId().getId(), Optional.of( + new SingularityKillTaskRequest(Optional.of(true), Optional.of("kill -9 bb"), Optional. absent(), Optional. absent(), Optional. absent()))); + + cleaner.drainCleanupQueue(); + killKilledTasks(); + + Assert.assertEquals(2, taskManager.getNumActiveTasks()); + Assert.assertEquals(0, requestManager.getCleanupRequests().size()); + Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); + } + @Test public void testTaskBounce() { initRequest(); @@ -263,7 +284,7 @@ public void testKilledTaskIdRecords() { initScheduledRequest(); initFirstDeploy(); - SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); + launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); requestResource.deleteRequest(requestId, Optional. absent()); @@ -271,15 +292,12 @@ public void testKilledTaskIdRecords() { cleaner.drainCleanupQueue(); - Assert.assertTrue(requestManager.getCleanupRequests().isEmpty()); Assert.assertTrue(!taskManager.getKilledTaskIdRecords().isEmpty()); + killKilledTasks(); cleaner.drainCleanupQueue(); - Assert.assertTrue(!taskManager.getKilledTaskIdRecords().isEmpty()); - - statusUpdate(firstTask, TaskState.TASK_KILLED); - + Assert.assertTrue(requestManager.getCleanupRequests().isEmpty()); Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty()); } @@ -1328,21 +1346,46 @@ public void testWaitAfterTaskWorks() { @Test public void testRemovedRequestData() { + long now = System.currentTimeMillis(); + initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); - initDeploy(db, System.currentTimeMillis()); + initDeploy(db, now); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.WAITING, deployManager.getPendingDeploys().get(0).getCurrentDeployState()); - requestManager.deleteRequest(request, Optional.absent(), Optional.absent(), Optional.absent()); + requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent()); + requestManager.markDeleted(request, now, Optional.absent(), Optional.absent()); deployChecker.checkDeploys(); SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, firstDeployId).get(); Assert.assertEquals(DeployState.FAILED, deployResult.getDeployState()); Assert.assertTrue(deployResult.getMessage().get().contains("MISSING")); } + @Test + public void testDeletingState() { + initRequest(); + Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); + Assert.assertEquals(1, requestManager.getRequestHistory(requestId).size()); + + requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.of("the cake is a lie")); + Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); + Assert.assertEquals(2, requestManager.getRequestHistory(requestId).size()); + + cleaner.drainCleanupQueue(); + Assert.assertEquals(3, requestManager.getRequestHistory(requestId).size()); + + List historyTypes = new ArrayList<>(); + for (SingularityRequestHistory request : requestManager.getRequestHistory(requestId)) { + historyTypes.add(request.getEventType()); + } + Assert.assertTrue(historyTypes.contains(RequestHistoryType.CREATED)); + Assert.assertTrue(historyTypes.contains(RequestHistoryType.DELETING)); + Assert.assertTrue(historyTypes.contains(RequestHistoryType.DELETED)); + } + @Test public void testMaxTasksPerOffer() { configuration.setMaxTasksPerOffer(3); diff --git a/SingularityUI/app/components/requestDetail/header/RequestActionButtons.jsx b/SingularityUI/app/components/requestDetail/header/RequestActionButtons.jsx index c310d2c9c6..953a9cafbb 100644 --- a/SingularityUI/app/components/requestDetail/header/RequestActionButtons.jsx +++ b/SingularityUI/app/components/requestDetail/header/RequestActionButtons.jsx @@ -160,14 +160,8 @@ const RequestActionButtons = ({requestParent, fetchRequest, fetchRequestHistory, } } - const navigateAwayOnSuccess = (response) => { - if (response.statusCode === 200) { - router.push('/requests'); - } - }; - const removeButton = ( - +