From 0b9ce605fb6c4db67aa2c089fee86ad1b05618d4 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 9 Oct 2015 13:14:33 -0400 Subject: [PATCH 1/2] optionally take start and end time query params to limit s3 log searching --- .../singularity/resources/S3LogResource.java | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java index 7ad619f898..3566de973d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java @@ -19,6 +19,7 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import org.jets3t.service.S3Service; @@ -96,13 +97,20 @@ public S3LogResource(RequestManager requestManager, HistoryManager historyManage this.s3GroupOverride = s3GroupOverride; } - private Collection getS3PrefixesForTask(SingularityTaskId taskId) { + private Collection getS3PrefixesForTask(SingularityTaskId taskId, Optional startArg, Optional endArg) { SingularityTaskHistory history = getTaskHistory(taskId); SimplifiedTaskState taskState = SingularityTaskHistoryUpdate.getCurrentState(history.getTaskUpdates()); - final long start = taskId.getStartedAt(); - final long end = taskState == SimplifiedTaskState.DONE ? Iterables.getLast(history.getTaskUpdates()).getTimestamp() : System.currentTimeMillis(); + long start = taskId.getStartedAt(); + if (startArg.isPresent()) { + start = Math.max(startArg.get(), start); + } + + long end = taskState == SimplifiedTaskState.DONE ? Iterables.getLast(history.getTaskUpdates()).getTimestamp() : System.currentTimeMillis(); + if (endArg.isPresent()) { + end = Math.min(endArg.get(), end); + } Optional tag = Optional.absent(); if (history.getTask().getTaskRequest().getDeploy().getExecutorData().isPresent()) { @@ -120,12 +128,15 @@ private boolean isCurrentDeploy(String requestId, String deployId) { return deployId.equals(deployManager.getInUseDeployId(requestId).orNull()); } - private Collection getS3PrefixesForRequest(String requestId) { + private Collection getS3PrefixesForRequest(String requestId, Optional startArg, Optional endArg) { Optional firstHistory = requestHistoryHelper.getFirstHistory(requestId); checkNotFound(firstHistory.isPresent(), "No request history found for %s", requestId); - final long start = firstHistory.get().getCreatedAt(); + long start = firstHistory.get().getCreatedAt(); + if (startArg.isPresent()) { + start = Math.max(startArg.get(), start); + } Optional lastHistory = requestHistoryHelper.getLastHistory(requestId); @@ -135,6 +146,10 @@ private Collection getS3PrefixesForRequest(String requestId) { end = lastHistory.get().getCreatedAt() + TimeUnit.DAYS.toMillis(1); } + if (endArg.isPresent()) { + end = Math.min(endArg.get(), end); + } + Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(configuration.get().getS3KeyFormat(), requestId, start, end); LOG.trace("Request {} got S3 prefixes {} for start {}, end {}", requestId, prefixes, start, end); @@ -142,10 +157,13 @@ private Collection getS3PrefixesForRequest(String requestId) { return prefixes; } - private Collection getS3PrefixesForDeploy(String requestId, String deployId) { + private Collection getS3PrefixesForDeploy(String requestId, String deployId, Optional startArg, Optional endArg) { SingularityDeployHistory deployHistory = getDeployHistory(requestId, deployId); - final long start = deployHistory.getDeployMarker().getTimestamp(); + long start = deployHistory.getDeployMarker().getTimestamp(); + if (startArg.isPresent()) { + start = Math.max(startArg.get(), start); + } long end = System.currentTimeMillis(); @@ -153,6 +171,10 @@ private Collection getS3PrefixesForDeploy(String requestId, String deplo end = deployHistory.getDeployStatistics().get().getLastFinishAt().get() + TimeUnit.DAYS.toMillis(1); } + if (endArg.isPresent()) { + end = Math.min(endArg.get(), end); + } + Optional tag = Optional.absent(); if (deployHistory.getDeploy().isPresent() && deployHistory.getDeploy().get().getExecutorData().isPresent()) { @@ -235,7 +257,10 @@ private void checkS3() { @GET @Path("/task/{taskId}") @ApiOperation("Retrieve the list of logs stored in S3 for a specific task.") - public List getS3LogsForTask(@ApiParam("The task ID to search for") @PathParam("taskId") String taskId) throws Exception { + public List getS3LogsForTask( + @ApiParam("The task ID to search for") @PathParam("taskId") String taskId, + @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") long start, + @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") long end) throws Exception { checkS3(); SingularityTaskId taskIdObject = getTaskIdObject(taskId); @@ -244,7 +269,7 @@ public List getS3LogsForTask(@ApiParam("The task ID to search final Optional maybeRequest = requestManager.getRequest(taskIdObject.getRequestId()); checkNotFound(maybeRequest.isPresent(), "Request ID %s does not exist", taskIdObject.getRequestId()); authorizationHelper.checkForAuthorization(maybeRequest.get().getRequest(), Optional.absent(), user); - return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForTask(taskIdObject)); + return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForTask(taskIdObject, Optional.fromNullable(start), Optional.fromNullable(end))); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s", taskId); } catch (Throwable t) { @@ -255,14 +280,17 @@ public List getS3LogsForTask(@ApiParam("The task ID to search @GET @Path("/request/{requestId}") @ApiOperation("Retrieve the list of logs stored in S3 for a specific request.") - public List getS3LogsForRequest(@ApiParam("The request ID to search for") @PathParam("requestId") String requestId) throws Exception { + public List getS3LogsForRequest( + @ApiParam("The request ID to search for") @PathParam("requestId") String requestId, + @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") long start, + @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") long end) throws Exception { checkS3(); try { final Optional maybeRequest = requestManager.getRequest(requestId); checkNotFound(maybeRequest.isPresent(), "Request ID %s does not exist", requestId); authorizationHelper.checkForAuthorization(maybeRequest.get().getRequest(), Optional.absent(), user); - return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForRequest(requestId)); + return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForRequest(requestId, Optional.fromNullable(start), Optional.fromNullable(end))); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s", requestId); } catch (Throwable t) { @@ -273,15 +301,18 @@ public List getS3LogsForRequest(@ApiParam("The request ID to s @GET @Path("/request/{requestId}/deploy/{deployId}") @ApiOperation("Retrieve the list of logs stored in S3 for a specific deploy.") - public List getS3LogsForDeploy(@ApiParam("The request ID to search for") @PathParam("requestId") String requestId, - @ApiParam("The deploy ID to search for") @PathParam("deployId") String deployId) throws Exception { + public List getS3LogsForDeploy( + @ApiParam("The request ID to search for") @PathParam("requestId") String requestId, + @ApiParam("The deploy ID to search for") @PathParam("deployId") String deployId, + @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") long start, + @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") long end) throws Exception { checkS3(); try { final Optional maybeRequest = requestManager.getRequest(requestId); checkNotFound(maybeRequest.isPresent(), "Request ID %s does not exist", requestId); authorizationHelper.checkForAuthorization(maybeRequest.get().getRequest(), Optional.absent(), user); - return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForDeploy(requestId, deployId)); + return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForDeploy(requestId, deployId, Optional.fromNullable(start), Optional.fromNullable(end))); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s-%s", requestId, deployId); } catch (Throwable t) { From cce71360ce3782d6a768ef209af9f2a4494ab8ac Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Tue, 13 Oct 2015 13:35:00 -0400 Subject: [PATCH 2/2] change query params to optionals --- .../singularity/resources/S3LogResource.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java index 3566de973d..92485a87f6 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java @@ -259,8 +259,8 @@ private void checkS3() { @ApiOperation("Retrieve the list of logs stored in S3 for a specific task.") public List getS3LogsForTask( @ApiParam("The task ID to search for") @PathParam("taskId") String taskId, - @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") long start, - @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") long end) throws Exception { + @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") Optional start, + @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end) throws Exception { checkS3(); SingularityTaskId taskIdObject = getTaskIdObject(taskId); @@ -269,7 +269,7 @@ public List getS3LogsForTask( final Optional maybeRequest = requestManager.getRequest(taskIdObject.getRequestId()); checkNotFound(maybeRequest.isPresent(), "Request ID %s does not exist", taskIdObject.getRequestId()); authorizationHelper.checkForAuthorization(maybeRequest.get().getRequest(), Optional.absent(), user); - return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForTask(taskIdObject, Optional.fromNullable(start), Optional.fromNullable(end))); + return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForTask(taskIdObject, start, end)); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s", taskId); } catch (Throwable t) { @@ -282,15 +282,15 @@ public List getS3LogsForTask( @ApiOperation("Retrieve the list of logs stored in S3 for a specific request.") public List getS3LogsForRequest( @ApiParam("The request ID to search for") @PathParam("requestId") String requestId, - @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") long start, - @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") long end) throws Exception { + @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") Optional start, + @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end) throws Exception { checkS3(); try { final Optional maybeRequest = requestManager.getRequest(requestId); checkNotFound(maybeRequest.isPresent(), "Request ID %s does not exist", requestId); authorizationHelper.checkForAuthorization(maybeRequest.get().getRequest(), Optional.absent(), user); - return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForRequest(requestId, Optional.fromNullable(start), Optional.fromNullable(end))); + return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForRequest(requestId, start, end)); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s", requestId); } catch (Throwable t) { @@ -304,15 +304,15 @@ public List getS3LogsForRequest( public List getS3LogsForDeploy( @ApiParam("The request ID to search for") @PathParam("requestId") String requestId, @ApiParam("The deploy ID to search for") @PathParam("deployId") String deployId, - @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") long start, - @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") long end) throws Exception { + @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") Optional start, + @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end) throws Exception { checkS3(); try { final Optional maybeRequest = requestManager.getRequest(requestId); checkNotFound(maybeRequest.isPresent(), "Request ID %s does not exist", requestId); authorizationHelper.checkForAuthorization(maybeRequest.get().getRequest(), Optional.absent(), user); - return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForDeploy(requestId, deployId, Optional.fromNullable(start), Optional.fromNullable(end))); + return getS3Logs(maybeRequest.get().getRequest().getGroup(), getS3PrefixesForDeploy(requestId, deployId, start, end)); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s-%s", requestId, deployId); } catch (Throwable t) {