diff --git a/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosExecutorObject.java b/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosExecutorObject.java index 4182681352..dec18c9625 100644 --- a/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosExecutorObject.java +++ b/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosExecutorObject.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; public class MesosExecutorObject { @@ -54,4 +55,16 @@ public String getId() { return id; } + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("directory", directory) + .add("id", id) + .add("container", container) + .add("name", name) + .add("resources", resources) + .add("tasks", tasks) + .add("completedTasks", completedTasks) + .toString(); + } } diff --git a/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosTaskObject.java b/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosTaskObject.java index cdb036620e..5eafa07c58 100644 --- a/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosTaskObject.java +++ b/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosTaskObject.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; import com.hubspot.mesos.SingularityMesosTaskLabel; public class MesosTaskObject { @@ -55,4 +56,17 @@ public String getFrameworkId() { public String getExecutorId() { return executorId; } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("resources", resources) + .add("state", state) + .add("id", id) + .add("name", name) + .add("slaveId", slaveId) + .add("frameworkId", frameworkId) + .add("executorId", executorId) + .toString(); + } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistory.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistory.java index fd27f39fcf..f2d6b17d51 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistory.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskHistory.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; import com.google.common.base.Optional; import com.hubspot.mesos.JavaUtils; @@ -12,6 +13,7 @@ public class SingularityTaskHistory { private final List taskUpdates; private final Optional directory; + private final Optional containerId; private final SingularityTask task; private final List healthcheckResults; private final List loadBalancerUpdates; @@ -19,12 +21,13 @@ public class SingularityTaskHistory { private final List taskMetadata; @JsonCreator - public SingularityTaskHistory(@JsonProperty("taskUpdates") List taskUpdates, @JsonProperty("directory") Optional directory, - @JsonProperty("healthcheckResults") List healthcheckResults, @JsonProperty("task") SingularityTask task, + public SingularityTaskHistory(@JsonProperty("taskUpdates") List taskUpdates, @JsonProperty("directory") Optional directory, @JsonProperty("containerId") Optional containerId, + @JsonProperty("healthcheckResults") List healthcheckResults, @JsonProperty("task") SingularityTask task, @JsonProperty("loadBalancerUpdates") List loadBalancerUpdates, @JsonProperty("shellCommandHistory") List shellCommandHistory, @JsonProperty("taskMetadata") List taskMetadata) { this.directory = directory; + this.containerId = containerId; this.task = task; this.taskUpdates = JavaUtils.nonNullImmutable(taskUpdates); this.healthcheckResults = JavaUtils.nonNullImmutable(healthcheckResults); @@ -41,6 +44,10 @@ public Optional getDirectory() { return directory; } + public Optional getContainerId() { + return containerId; + } + public SingularityTask getTask() { return task; } @@ -68,8 +75,16 @@ public Optional getLastTaskUpdate() { @Override public String toString() { - return "SingularityTaskHistory [taskUpdates=" + taskUpdates + ", directory=" + directory + ", task=" + task + ", healthcheckResults=" + healthcheckResults + ", loadBalancerUpdates=" - + loadBalancerUpdates + ", shellCommandHistory=" + shellCommandHistory + ", taskMetadata=" + taskMetadata + "]"; + return Objects.toStringHelper(this) + .add("taskUpdates", taskUpdates) + .add("directory", directory) + .add("containerId", containerId) + .add("task", task) + .add("healthcheckResults", healthcheckResults) + .add("loadBalancerUpdates", loadBalancerUpdates) + .add("shellCommandHistory", shellCommandHistory) + .add("taskMetadata", taskMetadata) + .add("lastTaskUpdate", getLastTaskUpdate()) + .toString(); } - } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java index 082a52350f..903b6ddef1 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/TaskManager.java @@ -82,6 +82,7 @@ public class TaskManager extends CuratorAsyncManager { private static final String LAST_HEALTHCHECK_KEY = "LAST_HEALTHCHECK"; private static final String DIRECTORY_KEY = "DIRECTORY"; + private static final String CONTAINER_ID_KEY = "CONTAINER_ID"; private static final String TASK_KEY = "TASK"; private static final String NOTIFIED_OVERDUE_TO_FINISH_KEY = "NOTIFIED_OVERDUE_TO_FINISH"; @@ -214,6 +215,10 @@ private String getDirectoryPath(SingularityTaskId taskId) { return ZKPaths.makePath(getHistoryPath(taskId), DIRECTORY_KEY); } + private String getContainerIdPath(SingularityTaskId taskId) { + return ZKPaths.makePath(getHistoryPath(taskId), CONTAINER_ID_KEY); + } + private String getNotifiedOverduePath(SingularityTaskId taskId) { return ZKPaths.makePath(getHistoryPath(taskId), NOTIFIED_OVERDUE_TO_FINISH_KEY); } @@ -272,6 +277,10 @@ public void saveTaskDirectory(SingularityTaskId taskId, String directory) { save(getDirectoryPath(taskId), Optional.of(directory.getBytes(UTF_8))); } + public void saveContainerId(SingularityTaskId taskId, String containerId) { + save(getContainerIdPath(taskId), Optional.of(containerId.getBytes(UTF_8))); + } + @Timed public void saveLastActiveTaskStatus(SingularityTaskStatusHolder taskStatus) { save(getLastActiveTaskStatusPath(taskStatus.getTaskId()), taskStatus, taskStatusTranscoder); @@ -281,6 +290,10 @@ public Optional getDirectory(SingularityTaskId taskId) { return getData(getDirectoryPath(taskId), StringTranscoder.INSTANCE); } + public Optional getContainerId(SingularityTaskId taskId) { + return getData(getContainerIdPath(taskId), StringTranscoder.INSTANCE); + } + public void saveHealthcheckResult(SingularityTaskHealthcheckResult healthcheckResult) { final Optional bytes = Optional.of(healthcheckResultTranscoder.toBytes(healthcheckResult)); @@ -547,6 +560,7 @@ public Optional getTaskHistory(SingularityTaskId taskId) List taskUpdates = getTaskHistoryUpdates(taskId); Optional directory = getDirectory(taskId); + Optional containerId = getContainerId(taskId); List healthchecks = getHealthcheckResults(taskId); List loadBalancerUpdates = Lists.newArrayListWithCapacity(2); @@ -558,7 +572,7 @@ public Optional getTaskHistory(SingularityTaskId taskId) List taskMetadata = getTaskMetadata(taskId); - return Optional.of(new SingularityTaskHistory(taskUpdates, directory, healthchecks, task.get(), loadBalancerUpdates, shellCommandHistory, taskMetadata)); + return Optional.of(new SingularityTaskHistory(taskUpdates, directory, containerId, healthchecks, task.get(), loadBalancerUpdates, shellCommandHistory, taskMetadata)); } private List getTaskShellCommandHistory(SingularityTaskId taskId) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityLogSupport.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosExecutorInfoSupport.java similarity index 56% rename from SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityLogSupport.java rename to SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosExecutorInfoSupport.java index 61d2998d02..fa0a5e71ef 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityLogSupport.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosExecutorInfoSupport.java @@ -27,9 +27,9 @@ import io.dropwizard.lifecycle.Managed; @Singleton -public class SingularityLogSupport implements Managed { +public class SingularityMesosExecutorInfoSupport implements Managed { - private static final Logger LOG = LoggerFactory.getLogger(SingularityLogSupport.class); + private static final Logger LOG = LoggerFactory.getLogger(SingularityMesosExecutorInfoSupport.class); private final MesosClient mesosClient; private final TaskManager taskManager; @@ -37,7 +37,7 @@ public class SingularityLogSupport implements Managed { private final ThreadPoolExecutor logLookupExecutorService; @Inject - public SingularityLogSupport(SingularityConfiguration configuration, MesosClient mesosClient, TaskManager taskManager) { + public SingularityMesosExecutorInfoSupport(SingularityConfiguration configuration, MesosClient mesosClient, TaskManager taskManager) { this.mesosClient = mesosClient; this.taskManager = taskManager; @@ -53,16 +53,16 @@ public void stop() { MoreExecutors.shutdownAndAwaitTermination(logLookupExecutorService, 1, TimeUnit.SECONDS); } - private Optional findDirectory(SingularityTaskId taskId, List executors) { + private Optional findExecutor(SingularityTaskId taskId, List executors) { for (MesosExecutorObject executor : executors) { for (MesosTaskObject executorTask : executor.getTasks()) { if (taskId.getId().equals(executorTask.getId())) { - return Optional.of(executor.getDirectory()); + return Optional.of(executor); } } for (MesosTaskObject executorTask : executor.getCompletedTasks()) { if (taskId.getId().equals(executorTask.getId())) { - return Optional.of(executor.getDirectory()); + return Optional.of(executor); } } } @@ -70,60 +70,71 @@ private Optional findDirectory(SingularityTaskId taskId, List directory = Optional.absent(); + Optional containerId = Optional.absent(); for (MesosSlaveFrameworkObject slaveFramework : slaveState.getFrameworks()) { - directory = findDirectory(task.getTaskId(), slaveFramework.getExecutors()); - if (directory.isPresent()) { + Optional maybeExecutor = findExecutor(task.getTaskId(), slaveFramework.getExecutors()); + if (maybeExecutor.isPresent()) { + directory = Optional.of(maybeExecutor.get().getDirectory()); + containerId = Optional.of(maybeExecutor.get().getContainer()); break; } - directory = findDirectory(task.getTaskId(), slaveFramework.getCompletedExecutors()); - if (directory.isPresent()) { + maybeExecutor = findExecutor(task.getTaskId(), slaveFramework.getCompletedExecutors()); + if (maybeExecutor.isPresent()) { + directory = Optional.of(maybeExecutor.get().getDirectory()); + containerId = Optional.of(maybeExecutor.get().getContainer()); break; } } - if (!directory.isPresent()) { + if (!directory.isPresent() && !containerId.isPresent()) { LOG.warn("Couldn't find matching executor for task {}", task.getTaskId()); return; } - LOG.debug("Found a directory {} for task {}", directory.get(), task.getTaskId()); + LOG.debug("Found a directory {} and container id {} for task {}", directory.or(""), containerId.or(""), task.getTaskId()); - taskManager.saveTaskDirectory(task.getTaskId(), directory.get()); + if (directory.isPresent()) { + taskManager.saveTaskDirectory(task.getTaskId(), directory.get()); + } + if (containerId.isPresent()) { + taskManager.saveContainerId(task.getTaskId(), containerId.get()); + } - LOG.trace("Updated task {} directory in {}", task.getTaskId(), JavaUtils.duration(start)); + LOG.trace("Updated task {} directory and container id in {}", task.getTaskId(), JavaUtils.duration(start)); } @Timed - public void checkDirectory(final SingularityTaskId taskId) { + public void checkDirectoryAndContainerId(final SingularityTaskId taskId) { final Optional maybeDirectory = taskManager.getDirectory(taskId); + final Optional maybeContainerId = taskManager.getContainerId(taskId); - if (maybeDirectory.isPresent()) { - LOG.debug("Already had a directory for task {}, skipping lookup", taskId); + if (maybeDirectory.isPresent() && maybeContainerId.isPresent()) { + LOG.debug("Already had a directory and container id for task {}, skipping lookup", taskId); return; } final Optional task = taskManager.getTask(taskId); if (!task.isPresent()) { - LOG.warn("No task found available for task {}, can't locate directory", taskId); + LOG.warn("No task found available for task {}, can't locate directory or container id", taskId); return; } Runnable cmd = generateLookupCommand(task.get()); - LOG.trace("Enqueing a request to fetch directory for task: {}, current queue size: {}", taskId, logLookupExecutorService.getQueue().size()); + LOG.trace("Enqueing a request to fetch directory and container id for task: {}, current queue size: {}", taskId, logLookupExecutorService.getQueue().size()); logLookupExecutorService.submit(cmd); } @@ -134,9 +145,9 @@ private Runnable generateLookupCommand(final SingularityTask task) { @Override public void run() { try { - loadDirectory(task); + loadDirectoryAndContainer(task); } catch (Throwable t) { - LOG.error("While fetching directory for task: {}", task.getTaskId(), t); + LOG.error("While fetching directory and container id for task: {}", task.getTaskId(), t); } } }; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosModule.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosModule.java index 04d23f2bf8..c4ca7e9cc0 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosModule.java @@ -23,7 +23,7 @@ public class SingularityMesosModule extends AbstractModule { @Override public void configure() { bind(SingularityDriver.class).in(Scopes.SINGLETON); - bind(SingularityLogSupport.class).in(Scopes.SINGLETON); + bind(SingularityMesosExecutorInfoSupport.class).in(Scopes.SINGLETON); bind(SingularityMesosScheduler.class).in(Scopes.SINGLETON); bind(SingularityMesosFrameworkMessageHandler.class).in(Scopes.SINGLETON); bind(SingularityMesosSchedulerDelegator.class).in(Scopes.SINGLETON); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosStatusUpdateHandler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosStatusUpdateHandler.java index beaa2e4a25..ffe4c37830 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosStatusUpdateHandler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosStatusUpdateHandler.java @@ -61,7 +61,7 @@ public class SingularityMesosStatusUpdateHandler implements Managed { private final SingularityHealthchecker healthchecker; private final SingularityNewTaskChecker newTaskChecker; private final SingularitySlaveAndRackManager slaveAndRackManager; - private final SingularityLogSupport logSupport; + private final SingularityMesosExecutorInfoSupport logSupport; private final SingularityScheduler scheduler; private final Provider stateCacheProvider; private final String serverId; @@ -80,7 +80,7 @@ public class SingularityMesosStatusUpdateHandler implements Managed { @Inject public SingularityMesosStatusUpdateHandler(TaskManager taskManager, DeployManager deployManager, RequestManager requestManager, IdTranscoder taskIdTranscoder, SingularityExceptionNotifier exceptionNotifier, SingularityHealthchecker healthchecker, - SingularityNewTaskChecker newTaskChecker, SingularitySlaveAndRackManager slaveAndRackManager, SingularityLogSupport logSupport, SingularityScheduler scheduler, + SingularityNewTaskChecker newTaskChecker, SingularitySlaveAndRackManager slaveAndRackManager, SingularityMesosExecutorInfoSupport logSupport, SingularityScheduler scheduler, Provider stateCacheProvider, @Named(SingularityMainModule.SERVER_ID_PROPERTY) String serverId, SchedulerDriverSupplier schedulerDriverSupplier, @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock schedulerLock, @@ -252,7 +252,7 @@ private void unsafeProcessStatusUpdate(Protos.TaskStatus status) { new SingularityTaskHistoryUpdate(taskIdObj, timestamp, taskState, statusMessage, status.hasReason() ? Optional.of(status.getReason().name()) : Optional.absent()); final SingularityCreateResult taskHistoryUpdateCreateResult = taskManager.saveTaskHistoryUpdate(taskUpdate); - logSupport.checkDirectory(taskIdObj); + logSupport.checkDirectoryAndContainerId(taskIdObj); if (taskState.isDone()) { healthchecker.cancelHealthcheck(taskId); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/SandboxResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/SandboxResource.java index e88d32d248..4bd926161c 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/SandboxResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/SandboxResource.java @@ -39,7 +39,7 @@ import com.hubspot.singularity.data.SandboxManager.SlaveNotFoundException; import com.hubspot.singularity.data.TaskManager; import com.hubspot.singularity.data.history.HistoryManager; -import com.hubspot.singularity.mesos.SingularityLogSupport; +import com.hubspot.singularity.mesos.SingularityMesosExecutorInfoSupport; import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; @@ -51,11 +51,11 @@ public class SandboxResource extends AbstractHistoryResource { public static final String PATH = SingularityService.API_BASE_PATH + "/sandbox"; private final SandboxManager sandboxManager; - private final SingularityLogSupport logSupport; + private final SingularityMesosExecutorInfoSupport logSupport; private final SingularityConfiguration configuration; @Inject - public SandboxResource(HistoryManager historyManager, TaskManager taskManager, SandboxManager sandboxManager, DeployManager deployManager, SingularityLogSupport logSupport, + public SandboxResource(HistoryManager historyManager, TaskManager taskManager, SandboxManager sandboxManager, DeployManager deployManager, SingularityMesosExecutorInfoSupport logSupport, SingularityConfiguration configuration, SingularityAuthorizationHelper authorizationHelper, Optional user) { super(historyManager, taskManager, deployManager, authorizationHelper, user); @@ -69,7 +69,7 @@ private SingularityTaskHistory checkHistory(String taskId) { final SingularityTaskHistory taskHistory = getTaskHistoryRequired(taskIdObj); if (!taskHistory.getDirectory().isPresent()) { - logSupport.checkDirectory(taskIdObj); + logSupport.checkDirectoryAndContainerId(taskIdObj); throw badRequest("Task %s does not have a directory yet - check again soon (enqueued request to refetch)", taskId); } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java b/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java index a0f4df7535..0633980d31 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java @@ -93,9 +93,7 @@ public void blowDBAway() { private SingularityTaskHistory buildTask(long launchTime) { SingularityTask task = prepTask(request, firstDeploy, launchTime, 1); - SingularityTaskHistory taskHistory = new SingularityTaskHistory(null, Optional. absent(), null, task, null, null, null); - - return taskHistory; + return new SingularityTaskHistory(null, Optional. absent(), Optional.absent(), null, task, null, null, null); } private void saveTasks(int num, long launchTime) { diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java index 8e0e9e1592..0be6b077d3 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityTestModule.java @@ -56,7 +56,7 @@ import com.hubspot.singularity.hooks.LoadBalancerClient; import com.hubspot.singularity.mesos.SchedulerDriverSupplier; import com.hubspot.singularity.mesos.SingularityDriver; -import com.hubspot.singularity.mesos.SingularityLogSupport; +import com.hubspot.singularity.mesos.SingularityMesosExecutorInfoSupport; import com.hubspot.singularity.mesos.SingularityMesosModule; import com.hubspot.singularity.resources.DeployResource; import com.hubspot.singularity.resources.PriorityResource; @@ -176,8 +176,8 @@ public HttpServletRequest get() { @Override public void configure(Binder binder) { - SingularityLogSupport logSupport = mock(SingularityLogSupport.class); - binder.bind(SingularityLogSupport.class).toInstance(logSupport); + SingularityMesosExecutorInfoSupport logSupport = mock(SingularityMesosExecutorInfoSupport.class); + binder.bind(SingularityMesosExecutorInfoSupport.class).toInstance(logSupport); SingularityDriver mock = mock(SingularityDriver.class); when(mock.kill((SingularityTaskId) Matchers.any())).thenReturn(Status.DRIVER_RUNNING);