From cc69460e91c14eb4ec6419596c63e66b040a783d Mon Sep 17 00:00:00 2001 From: Kevin Dorosh Date: Mon, 9 Jul 2018 11:51:22 -0400 Subject: [PATCH 1/6] Proxy file download from Mesos slave through Singularity - needs testing. --- .../singularity/resources/TaskResource.java | 23 +++++++++++++++++++ .../app/components/taskDetail/TaskDetail.jsx | 12 +++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java index 27fd99ee13..de8c772725 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java @@ -5,6 +5,7 @@ import static com.hubspot.singularity.WebExceptions.checkNotFound; import static com.hubspot.singularity.WebExceptions.notFound; +import java.io.InputStream; import java.util.Collections; import java.util.List; @@ -18,6 +19,8 @@ import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -609,4 +612,24 @@ public List getShellCommandHisotryUpdates( SingularityTaskId taskIdObj = getTaskIdFromStr(taskId); return taskManager.getTaskShellCommandUpdates(new SingularityTaskShellCommandRequestId(taskIdObj, commandName, commandTimestamp)); } + + @GET + @Path("/download/{httpPrefix}/{slaveHostname}/port/{port}/path/{filePath}") + @Produces("application/octet-stream") + @Operation(summary = "Proxy a file download from a Mesos Slave through Singularity to access it over EAA (no VPN needed)") + public Response downloadFileOverProxy( + @Parameter(required = true, description = "Https (if available) or http") String httpPrefix, + @Parameter(required = true, description = "Mesos slave hostname") String slaveHostname, + @Parameter(required = true, description = "Http port for Mesos slave") String httpPort, + @Parameter(required = true, description = "Full file path to file on Mesos slave to be downloaded") String fileFullPath + ) { + + Client client = ClientBuilder.newClient(); + String url = String.format("%s://%s:%s/files/download.json?path=%s", + httpPrefix, slaveHostname, httpPort, fileFullPath); + final InputStream responseStream = client.target(url).request().get(InputStream.class); + + final String headerValue = String.format("attachment, filename=\"%s\"", fileFullPath); + return Response.ok(responseStream).header("Content-Disposition", headerValue).build(); + } } diff --git a/SingularityUI/app/components/taskDetail/TaskDetail.jsx b/SingularityUI/app/components/taskDetail/TaskDetail.jsx index 8ab175cd17..c3ac35f28c 100644 --- a/SingularityUI/app/components/taskDetail/TaskDetail.jsx +++ b/SingularityUI/app/components/taskDetail/TaskDetail.jsx @@ -163,6 +163,16 @@ class TaskDetail extends Component { analyzeFiles(files) { if (files && files.files) { + + let userParam = ''; + if (localStorage.getItem('singularityUserId')) { + if (options.url.includes('?')) { + userParam = `&user=${localStorage.getItem('singularityUserId')}` + } else { + userParam = `?user=${localStorage.getItem('singularityUserId')}` + } + } + for (const file of files.files) { file.isDirectory = file.mode[0] === 'd'; let httpPrefix = 'http'; @@ -179,8 +189,8 @@ class TaskDetail extends Component { } file.fullPath = `${files.fullPathToRoot}/${files.currentDirectory}/${file.name}`; - file.downloadLink = `${httpPrefix}://${files.slaveHostname}:${httpPort}/files/download.json?path=${file.fullPath}`; + file.downloadLink = `${config.apiRoot + options.url + userParam}/download/${httpPrefix}/${files.slaveHostname}/port/${httpPort}/path/${file.fullPath}`; file.isRecentlyModified = Date.now() / 1000 - file.mtime <= RECENTLY_MODIFIED_SECONDS; if (!file.isDirectory) { From 0ad34c5e23a95566c51376baa623db43cbc54b1e Mon Sep 17 00:00:00 2001 From: Kevin Dorosh Date: Tue, 10 Jul 2018 16:55:56 -0400 Subject: [PATCH 2/6] Tested SingularityService server side proxy code. --- .../singularity/resources/TaskResource.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java index de8c772725..04b898b021 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java @@ -614,14 +614,15 @@ public List getShellCommandHisotryUpdates( } @GET - @Path("/download/{httpPrefix}/{slaveHostname}/port/{port}/path/{filePath}") - @Produces("application/octet-stream") - @Operation(summary = "Proxy a file download from a Mesos Slave through Singularity to access it over EAA (no VPN needed)") + @Path("/download/{httpPrefix}/{slaveHostname}/port/{port}/path/{path}") + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @Operation(summary = "Proxy a file download from a Mesos Slave through Singularity") + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) public Response downloadFileOverProxy( - @Parameter(required = true, description = "Https (if available) or http") String httpPrefix, - @Parameter(required = true, description = "Mesos slave hostname") String slaveHostname, - @Parameter(required = true, description = "Http port for Mesos slave") String httpPort, - @Parameter(required = true, description = "Full file path to file on Mesos slave to be downloaded") String fileFullPath + @Parameter(required = true, description = "Https (if available) or http") @PathParam("httpPrefix") String httpPrefix, + @Parameter(required = true, description = "Mesos slave hostname") @PathParam("slaveHostname") String slaveHostname, + @Parameter(required = true, description = "Http port for Mesos slave") @PathParam("port") String httpPort, + @Parameter(required = true, description = "Full file path to file on Mesos slave to be downloaded") @PathParam("path") String fileFullPath ) { Client client = ClientBuilder.newClient(); @@ -632,4 +633,5 @@ public Response downloadFileOverProxy( final String headerValue = String.format("attachment, filename=\"%s\"", fileFullPath); return Response.ok(responseStream).header("Content-Disposition", headerValue).build(); } + } From ec9500cebf475956b68f3e07d01dc2b6f9564782 Mon Sep 17 00:00:00 2001 From: Kevin Dorosh Date: Wed, 11 Jul 2018 12:05:05 -0400 Subject: [PATCH 3/6] Fixed response header. Tested download from UI over proxy in docker local dev -- sucess. --- .../singularity/resources/TaskResource.java | 3 ++- .../app/components/taskDetail/TaskDetail.jsx | 14 ++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java index 04b898b021..4279b99e25 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java @@ -6,6 +6,7 @@ import static com.hubspot.singularity.WebExceptions.notFound; import java.io.InputStream; +import java.nio.file.Paths; import java.util.Collections; import java.util.List; @@ -630,7 +631,7 @@ public Response downloadFileOverProxy( httpPrefix, slaveHostname, httpPort, fileFullPath); final InputStream responseStream = client.target(url).request().get(InputStream.class); - final String headerValue = String.format("attachment, filename=\"%s\"", fileFullPath); + final String headerValue = String.format("attachment; filename=\"%s\"", Paths.get(fileFullPath).getFileName().toString()); return Response.ok(responseStream).header("Content-Disposition", headerValue).build(); } diff --git a/SingularityUI/app/components/taskDetail/TaskDetail.jsx b/SingularityUI/app/components/taskDetail/TaskDetail.jsx index c3ac35f28c..7b3cb1003c 100644 --- a/SingularityUI/app/components/taskDetail/TaskDetail.jsx +++ b/SingularityUI/app/components/taskDetail/TaskDetail.jsx @@ -163,16 +163,6 @@ class TaskDetail extends Component { analyzeFiles(files) { if (files && files.files) { - - let userParam = ''; - if (localStorage.getItem('singularityUserId')) { - if (options.url.includes('?')) { - userParam = `&user=${localStorage.getItem('singularityUserId')}` - } else { - userParam = `?user=${localStorage.getItem('singularityUserId')}` - } - } - for (const file of files.files) { file.isDirectory = file.mode[0] === 'd'; let httpPrefix = 'http'; @@ -188,9 +178,9 @@ class TaskDetail extends Component { file.uiPath = file.name; } - file.fullPath = `${files.fullPathToRoot}/${files.currentDirectory}/${file.name}`; + file.fullPath = encodeURIComponent(`${files.fullPathToRoot}/${files.currentDirectory}/${file.name}`); - file.downloadLink = `${config.apiRoot + options.url + userParam}/download/${httpPrefix}/${files.slaveHostname}/port/${httpPort}/path/${file.fullPath}`; + file.downloadLink = `${config.apiRoot}/tasks/download/${httpPrefix}/${files.slaveHostname}/port/${httpPort}/path/${file.fullPath}`; file.isRecentlyModified = Date.now() / 1000 - file.mtime <= RECENTLY_MODIFIED_SECONDS; if (!file.isDirectory) { From c2d88711176b436a0e1683a83953222b7be0443d Mon Sep 17 00:00:00 2001 From: Kevin Dorosh Date: Wed, 11 Jul 2018 13:10:42 -0400 Subject: [PATCH 4/6] Fix potential null dereference. --- .../com/hubspot/singularity/resources/TaskResource.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java index 4279b99e25..868c4d7f72 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java @@ -631,7 +631,11 @@ public Response downloadFileOverProxy( httpPrefix, slaveHostname, httpPort, fileFullPath); final InputStream responseStream = client.target(url).request().get(InputStream.class); - final String headerValue = String.format("attachment; filename=\"%s\"", Paths.get(fileFullPath).getFileName().toString()); + // Strip file path down to just a file name if we can + java.nio.file.Path filePath = Paths.get(fileFullPath).getFileName(); + String fileName = filePath != null ? filePath.toString() : fileFullPath; + + final String headerValue = String.format("attachment; filename=\"%s\"", fileName); return Response.ok(responseStream).header("Content-Disposition", headerValue).build(); } From d74d5003d2b2de3a255c1a1dc390bdd8f36fdc6f Mon Sep 17 00:00:00 2001 From: Kevin Dorosh Date: Wed, 11 Jul 2018 16:17:37 -0400 Subject: [PATCH 5/6] Injected AsyncHttpClient instead of making new HttpClient and stopped passing params that were in Mesos config. --- .../singularity/resources/TaskResource.java | 51 ++++++++++++------- .../app/components/taskDetail/TaskDetail.jsx | 2 +- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java index 868c4d7f72..005c70c489 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java @@ -6,6 +6,7 @@ import static com.hubspot.singularity.WebExceptions.notFound; import java.io.InputStream; +import java.net.ConnectException; import java.nio.file.Paths; import java.util.Collections; import java.util.List; @@ -20,8 +21,6 @@ import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -73,9 +72,11 @@ import com.hubspot.singularity.api.SingularityTaskMetadataRequest; import com.hubspot.singularity.auth.SingularityAuthorizationHelper; import com.hubspot.singularity.config.ApiPaths; +import com.hubspot.singularity.config.MesosConfiguration; import com.hubspot.singularity.config.SingularityTaskMetadataConfiguration; import com.hubspot.singularity.data.DisasterManager; import com.hubspot.singularity.data.RequestManager; +import com.hubspot.singularity.data.SandboxManager.SlaveNotFoundException; import com.hubspot.singularity.data.SingularityValidator; import com.hubspot.singularity.data.SlaveManager; import com.hubspot.singularity.data.TaskManager; @@ -99,6 +100,8 @@ public class TaskResource extends AbstractLeaderAwareResource { private static final Logger LOG = LoggerFactory.getLogger(TaskResource.class); + private final AsyncHttpClient httpClient; + private final MesosConfiguration configuration; private final TaskManager taskManager; private final RequestManager requestManager; private final SlaveManager slaveManager; @@ -113,7 +116,7 @@ public class TaskResource extends AbstractLeaderAwareResource { @Inject public TaskResource(TaskRequestManager taskRequestManager, TaskManager taskManager, SlaveManager slaveManager, MesosClient mesosClient, SingularityTaskMetadataConfiguration taskMetadataConfiguration, SingularityAuthorizationHelper authorizationHelper, RequestManager requestManager, SingularityValidator validator, DisasterManager disasterManager, - AsyncHttpClient httpClient, LeaderLatch leaderLatch, ObjectMapper objectMapper, RequestHelper requestHelper) { + AsyncHttpClient httpClient, LeaderLatch leaderLatch, ObjectMapper objectMapper, RequestHelper requestHelper, MesosConfiguration configuration) { super(httpClient, leaderLatch, objectMapper); this.taskManager = taskManager; this.taskRequestManager = taskRequestManager; @@ -125,6 +128,8 @@ public TaskResource(TaskRequestManager taskRequestManager, TaskManager taskManag this.validator = validator; this.disasterManager = disasterManager; this.requestHelper = requestHelper; + this.httpClient = httpClient; + this.configuration = configuration; } @GET @@ -615,28 +620,40 @@ public List getShellCommandHisotryUpdates( } @GET - @Path("/download/{httpPrefix}/{slaveHostname}/port/{port}/path/{path}") + @Path("/download/hosts/{slaveHostname}/path/{path}") @Produces(MediaType.APPLICATION_OCTET_STREAM) @Operation(summary = "Proxy a file download from a Mesos Slave through Singularity") - @Consumes(MediaType.APPLICATION_FORM_URLENCODED) public Response downloadFileOverProxy( - @Parameter(required = true, description = "Https (if available) or http") @PathParam("httpPrefix") String httpPrefix, @Parameter(required = true, description = "Mesos slave hostname") @PathParam("slaveHostname") String slaveHostname, - @Parameter(required = true, description = "Http port for Mesos slave") @PathParam("port") String httpPort, @Parameter(required = true, description = "Full file path to file on Mesos slave to be downloaded") @PathParam("path") String fileFullPath ) { + String httpPrefix = configuration.getSlaveHttpsPort().isPresent() ? "https" : "http"; + int httpPort = configuration.getSlaveHttpsPort().isPresent() ? configuration.getSlaveHttpsPort().get() : configuration.getSlaveHttpPort(); - Client client = ClientBuilder.newClient(); - String url = String.format("%s://%s:%s/files/download.json?path=%s", - httpPrefix, slaveHostname, httpPort, fileFullPath); - final InputStream responseStream = client.target(url).request().get(InputStream.class); + String url = String.format("%s://%s:%s/files/download.json", + httpPrefix, slaveHostname, httpPort); - // Strip file path down to just a file name if we can - java.nio.file.Path filePath = Paths.get(fileFullPath).getFileName(); - String fileName = filePath != null ? filePath.toString() : fileFullPath; + try { + final InputStream responseStream = httpClient.prepareGet(url) + .addQueryParameter("path", fileFullPath) + .execute() + .get() + .getResponseBodyAsStream(); + + // Strip file path down to just a file name if we can + java.nio.file.Path filePath = Paths.get(fileFullPath).getFileName(); + String fileName = filePath != null ? filePath.toString() : fileFullPath; + + final String headerValue = String.format("attachment; filename=\"%s\"", fileName); + return Response.ok(responseStream).header("Content-Disposition", headerValue).build(); + + } catch (Exception e) { + if (e.getCause().getClass() == ConnectException.class) { + throw new SlaveNotFoundException(e); + } else { + throw new RuntimeException(e); + } + } - final String headerValue = String.format("attachment; filename=\"%s\"", fileName); - return Response.ok(responseStream).header("Content-Disposition", headerValue).build(); } - } diff --git a/SingularityUI/app/components/taskDetail/TaskDetail.jsx b/SingularityUI/app/components/taskDetail/TaskDetail.jsx index 7b3cb1003c..7724a92514 100644 --- a/SingularityUI/app/components/taskDetail/TaskDetail.jsx +++ b/SingularityUI/app/components/taskDetail/TaskDetail.jsx @@ -180,7 +180,7 @@ class TaskDetail extends Component { file.fullPath = encodeURIComponent(`${files.fullPathToRoot}/${files.currentDirectory}/${file.name}`); - file.downloadLink = `${config.apiRoot}/tasks/download/${httpPrefix}/${files.slaveHostname}/port/${httpPort}/path/${file.fullPath}`; + file.downloadLink = `${config.apiRoot}/tasks/download/hosts/${files.slaveHostname}/path/${file.fullPath}`; file.isRecentlyModified = Date.now() / 1000 - file.mtime <= RECENTLY_MODIFIED_SECONDS; if (!file.isDirectory) { From a76151fe0b596f4791652eba3f61c5f1df6e5217 Mon Sep 17 00:00:00 2001 From: Kevin Dorosh Date: Wed, 11 Jul 2018 17:31:32 -0400 Subject: [PATCH 6/6] Switch to query params. --- .../com/hubspot/singularity/resources/TaskResource.java | 6 +++--- SingularityUI/app/components/taskDetail/TaskDetail.jsx | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java index 005c70c489..bafa943769 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/TaskResource.java @@ -620,12 +620,12 @@ public List getShellCommandHisotryUpdates( } @GET - @Path("/download/hosts/{slaveHostname}/path/{path}") + @Path("/download/") @Produces(MediaType.APPLICATION_OCTET_STREAM) @Operation(summary = "Proxy a file download from a Mesos Slave through Singularity") public Response downloadFileOverProxy( - @Parameter(required = true, description = "Mesos slave hostname") @PathParam("slaveHostname") String slaveHostname, - @Parameter(required = true, description = "Full file path to file on Mesos slave to be downloaded") @PathParam("path") String fileFullPath + @Parameter(required = true, description = "Mesos slave hostname") @QueryParam("slaveHostname") String slaveHostname, + @Parameter(required = true, description = "Full file path to file on Mesos slave to be downloaded") @QueryParam("path") String fileFullPath ) { String httpPrefix = configuration.getSlaveHttpsPort().isPresent() ? "https" : "http"; int httpPort = configuration.getSlaveHttpsPort().isPresent() ? configuration.getSlaveHttpsPort().get() : configuration.getSlaveHttpPort(); diff --git a/SingularityUI/app/components/taskDetail/TaskDetail.jsx b/SingularityUI/app/components/taskDetail/TaskDetail.jsx index 7724a92514..fad4994ece 100644 --- a/SingularityUI/app/components/taskDetail/TaskDetail.jsx +++ b/SingularityUI/app/components/taskDetail/TaskDetail.jsx @@ -178,9 +178,8 @@ class TaskDetail extends Component { file.uiPath = file.name; } - file.fullPath = encodeURIComponent(`${files.fullPathToRoot}/${files.currentDirectory}/${file.name}`); - - file.downloadLink = `${config.apiRoot}/tasks/download/hosts/${files.slaveHostname}/path/${file.fullPath}`; + file.fullPath = `${files.fullPathToRoot}/${files.currentDirectory}/${file.name}`; + file.downloadLink = `${config.apiRoot}/tasks/download?slaveHostname=${files.slaveHostname}&path=${file.fullPath}`; file.isRecentlyModified = Date.now() / 1000 - file.mtime <= RECENTLY_MODIFIED_SECONDS; if (!file.isDirectory) {