From 6929dc3f44f521d747de6dbef1d09b81da522b4e Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 6 Jan 2017 09:57:29 -0500 Subject: [PATCH 1/2] Optionally skip extra s3 metadata --- .../singularity/resources/S3LogResource.java | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java index d8eab55f3f..80e0b7aef7 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java @@ -199,7 +199,7 @@ private Collection getS3PrefixesForDeploy(S3Configuration s3Configuratio return prefixes; } - private List getS3LogsWithExecutorService(S3Configuration s3Configuration, Optional group, ListeningExecutorService executorService, Collection prefixes) throws InterruptedException, ExecutionException, TimeoutException { + private List getS3LogsWithExecutorService(S3Configuration s3Configuration, Optional group, ListeningExecutorService executorService, Collection prefixes, final boolean excldueMetadata) throws InterruptedException, ExecutionException, TimeoutException { List> futures = Lists.newArrayListWithCapacity(prefixes.size()); final String s3Bucket = (group.isPresent() && s3Configuration.getGroupOverrides().containsKey(group.get())) ? s3Configuration.getGroupOverrides().get(group.get()).getS3Bucket() : s3Configuration.getS3Bucket(); @@ -240,9 +240,13 @@ public SingularityS3Log call() throws Exception { String getUrl = s3Service.createSignedGetUrl(s3Bucket, s3Object.getKey(), expireAt); String downloadUrl = s3Service.createSignedUrl("GET", s3Bucket, s3Object.getKey(), FORCE_DOWNLOAD_S3_PARAMS, null, expireAt.getTime() / 1000, false); - Map objectMetadata = s3Service.getObjectDetails(s3Bucket, s3Object.getKey()).getMetadataMap(); - Optional maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR); - Optional maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR); + Optional maybeStartTime = Optional.absent(); + Optional maybeEndTime = Optional.absent(); + if (!excldueMetadata) { + Map objectMetadata = s3Service.getObjectDetails(s3Bucket, s3Object.getKey()).getMetadataMap(); + maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR); + maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR); + } return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModifiedDate().getTime(), s3Object.getContentLength(), downloadUrl, maybeStartTime, maybeEndTime); } @@ -270,7 +274,7 @@ private Optional getMetadataAsLong(Map objectMetadata, Str } } - private List getS3Logs(S3Configuration s3Configuration, Optional group, Collection prefixes) throws InterruptedException, ExecutionException, TimeoutException { + private List getS3Logs(S3Configuration s3Configuration, Optional group, Collection prefixes, boolean excldueMetadata) throws InterruptedException, ExecutionException, TimeoutException { if (prefixes.isEmpty()) { return Collections.emptyList(); } @@ -279,7 +283,7 @@ private List getS3Logs(S3Configuration s3Configuration, Option new ThreadFactoryBuilder().setNameFormat("S3LogFetcher-%d").build())); try { - List logs = Lists.newArrayList(getS3LogsWithExecutorService(s3Configuration, group, executorService, prefixes)); + List logs = Lists.newArrayList(getS3LogsWithExecutorService(s3Configuration, group, executorService, prefixes, excldueMetadata)); Collections.sort(logs, LOG_COMPARATOR); return logs; } finally { @@ -327,13 +331,14 @@ private Optional getRequestGroup(final String requestId) { public List getS3LogsForTask( @ApiParam("The task ID to search for") @PathParam("taskId") String taskId, @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") Optional start, - @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end) throws Exception { + @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end, + @ApiParam("Exclude custom object metadata") @QueryParam("excludeMetadata") Optional excludeMetadata) throws Exception { checkS3(); SingularityTaskId taskIdObject = getTaskIdObject(taskId); try { - return getS3Logs(configuration.get(), getRequestGroupForTask(taskIdObject), getS3PrefixesForTask(configuration.get(), taskIdObject, start, end)); + return getS3Logs(configuration.get(), getRequestGroupForTask(taskIdObject), getS3PrefixesForTask(configuration.get(), taskIdObject, start, end), excludeMetadata.or(false)); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s", taskId); } catch (Throwable t) { @@ -347,11 +352,12 @@ public List getS3LogsForTask( public List getS3LogsForRequest( @ApiParam("The request ID to search for") @PathParam("requestId") String requestId, @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") Optional start, - @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end) throws Exception { + @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end, + @ApiParam("Exclude custom object metadata") @QueryParam("excludeMetadata") Optional excludeMetadata) throws Exception { checkS3(); try { - return getS3Logs(configuration.get(), getRequestGroup(requestId), getS3PrefixesForRequest(configuration.get(), requestId, start, end)); + return getS3Logs(configuration.get(), getRequestGroup(requestId), getS3PrefixesForRequest(configuration.get(), requestId, start, end), excludeMetadata.or(false)); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s", requestId); } catch (Throwable t) { @@ -366,11 +372,12 @@ public List getS3LogsForDeploy( @ApiParam("The request ID to search for") @PathParam("requestId") String requestId, @ApiParam("The deploy ID to search for") @PathParam("deployId") String deployId, @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") Optional start, - @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end) throws Exception { + @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end, + @ApiParam("Exclude custom object metadata") @QueryParam("excludeMetadata") Optional excludeMetadata) throws Exception { checkS3(); try { - return getS3Logs(configuration.get(), getRequestGroup(requestId), getS3PrefixesForDeploy(configuration.get(), requestId, deployId, start, end)); + return getS3Logs(configuration.get(), getRequestGroup(requestId), getS3PrefixesForDeploy(configuration.get(), requestId, deployId, start, end), excludeMetadata.or(false)); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s-%s", requestId, deployId); } catch (Throwable t) { From 4d9959954e8b303e58c01c836424fb55260d39dd Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 6 Jan 2017 10:15:11 -0500 Subject: [PATCH 2/2] fix typo --- .../java/com/hubspot/singularity/resources/S3LogResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java index 80e0b7aef7..529ea2d0cd 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java @@ -199,7 +199,7 @@ private Collection getS3PrefixesForDeploy(S3Configuration s3Configuratio return prefixes; } - private List getS3LogsWithExecutorService(S3Configuration s3Configuration, Optional group, ListeningExecutorService executorService, Collection prefixes, final boolean excldueMetadata) throws InterruptedException, ExecutionException, TimeoutException { + private List getS3LogsWithExecutorService(S3Configuration s3Configuration, Optional group, ListeningExecutorService executorService, Collection prefixes, final boolean excludeMetadata) throws InterruptedException, ExecutionException, TimeoutException { List> futures = Lists.newArrayListWithCapacity(prefixes.size()); final String s3Bucket = (group.isPresent() && s3Configuration.getGroupOverrides().containsKey(group.get())) ? s3Configuration.getGroupOverrides().get(group.get()).getS3Bucket() : s3Configuration.getS3Bucket(); @@ -242,7 +242,7 @@ public SingularityS3Log call() throws Exception { Optional maybeStartTime = Optional.absent(); Optional maybeEndTime = Optional.absent(); - if (!excldueMetadata) { + if (!excludeMetadata) { Map objectMetadata = s3Service.getObjectDetails(s3Bucket, s3Object.getKey()).getMetadataMap(); maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR); maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR);