Made upload/download generic so that it doesnt have to be function sp…

…ecific (#1629)
apache · Apr 21, 2018 · b4658e2 · b4658e2
1 parent 4f73c54
commit b4658e2
Show file tree

Hide file tree

Showing 6 changed files with 67 additions and 123 deletions.
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
@@ -155,24 +155,17 @@ public Response triggerFunction(final @PathParam("tenant") String tenant,
     }
 
     @POST
-    @Path("/{tenant}/{namespace}/{functionName}/upload")
+    @Path("/upload")
     @Consumes(MediaType.MULTIPART_FORM_DATA)
-    public Response uploadFunction(final @PathParam("tenant") String tenant,
+    public Response uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream,
-                                   final @PathParam("namespace") String namespace,
+                                   final @FormDataParam("path") String path) {
-                                   final @PathParam("functionName") String functionName,
+        return functions.uploadFunction(uploadedInputStream, path);
-                                   final @FormDataParam("data") InputStream uploadedInputStream,
-                                   final @FormDataParam("data") FormDataContentDisposition fileDetail) {
-        return functions.uploadFunction(
-                tenant, namespace, functionName, uploadedInputStream, fileDetail);
     }
 
     @GET
-    @Path("/{tenant}/{namespace}/{functionName}/download")
+    @Path("/download")
-    public Response downloadFunction(final @PathParam("tenant") String tenant,
+    public Response downloadFunction(final @QueryParam("path") String path) {
-                                     final @PathParam("namespace") String namespace,
+        return functions.downloadFunction(path);
-                                     final @PathParam("functionName") String functionName,
-                                     final @QueryParam("filename") String fileName) {
-        return functions.downloadFunction(tenant, namespace, functionName, fileName);
     }
 
 }
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java
@@ -160,38 +160,30 @@ public interface Functions {
     String triggerFunction(String tenant, String namespace, String function, String triggerValue, String triggerFile) throws PulsarAdminException;
 
     /**
-     * Upload Function Code.
+     * Upload Data.
      *
-     * @param tenant
+     * @param sourceFile
-     *            Tenant name
+     *            dataFile that needs to be uploaded
-     * @param namespace
+     * @param path
-     *            Namespace name
+     *            Path where data should be stored
-     * @param function
-     *            Function name
-     * @param fileName
-     *            Code where file is located
      *
      * @throws PulsarAdminException
      *             Unexpected error
      */
-    void uploadFunction(String tenant, String namespace, String function, String fileName) throws PulsarAdminException;
+    void uploadFunction(String sourceFile, String path) throws PulsarAdminException;
 
     /**
      * Download Function Code.
      *
-     * @param tenant
+     * @param destinationFile
-     *            Tenant name
+     *            file where data should be downloaded to
-     * @param namespace
+     * @param path
-     *            Namespace name
+     *            Path where data is located
-     * @param function
-     *            Function name
-     * @param fileName
-     *            Code where function should be downloaded
      *
      * @throws PulsarAdminException
      *             Unexpected error
      */
-    void downloadFunction(String tenant, String namespace, String function, String fileName) throws PulsarAdminException;
+    void downloadFunction(String destinationFile, String path) throws PulsarAdminException;
 
 
 }
diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java
@@ -169,27 +169,27 @@ public String triggerFunction(String tenant, String namespace, String functionNa
     }
 
     @Override
-    public void uploadFunction(String tenant, String namespace, String functionName, String fileName) throws PulsarAdminException {
+    public void uploadFunction(String sourceFile, String path) throws PulsarAdminException {
         try {
             final FormDataMultiPart mp = new FormDataMultiPart();
 
-            mp.bodyPart(new FileDataBodyPart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM_TYPE));
+            mp.bodyPart(new FileDataBodyPart("data", new File(sourceFile), MediaType.APPLICATION_OCTET_STREAM_TYPE));
 
-            request(functions.path(tenant).path(namespace).path(functionName).path("upload"))
+            mp.bodyPart(new FormDataBodyPart("path", path, MediaType.TEXT_PLAIN_TYPE));
+            request(functions.path("upload"))
                     .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA), ErrorData.class);
         } catch (Exception e) {
             throw getApiException(e);
         }
     }
 
     @Override
-    public void downloadFunction(String tenant, String namespace, String function, String path) throws PulsarAdminException {
+    public void downloadFunction(String destinationPath, String path) throws PulsarAdminException {
         try {
-            Path pathToFile = Paths.get(path);
+            InputStream response = request(functions.path("download")
-            InputStream response = request(functions.path(tenant).path(namespace).path(function).path("download")
+                    .queryParam("path", path)).get(InputStream.class);
-                    .queryParam("filename", pathToFile.getFileName().toString())).get(InputStream.class);
             if (response != null) {
-                File targetFile = new File(path);
+                File targetFile = new File(destinationPath);
                 java.nio.file.Files.copy(
                         response,
                         targetFile.toPath(),

diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -693,46 +693,41 @@ void runCmd() throws Exception {
         }
     }
 
-    @Parameters(commandDescription = "Upload Pulsar Function code to Pulsar")
+    @Parameters(commandDescription = "Upload File Data to Pulsar")
-    class UploadFunction extends FunctionCommand {
+    class UploadFunction extends BaseCommand {
         @Parameter(
-                names = "--jar",
+                names = "--sourceFile",
-                description = "Path to the jar file for the function (if the function is written in Java)",
+                description = "The file whose contents need to be uploaded",
-                listConverter = StringConverter.class)
+                listConverter = StringConverter.class, required = true)
-        protected String jarFile;
+        protected String sourceFile;
         @Parameter(
-                names = "--py",
+                names = "--path",
-                description = "Path to the main Python file for the function (if the function is written in Python)",
+                description = "Path where the contents need to be stored",
-                listConverter = StringConverter.class)
+                listConverter = StringConverter.class, required = true)
-        protected String pyFile;
+        protected String path;
-
         @Override
         void runCmd() throws Exception {
-            if (jarFile == null && pyFile == null) {
+            admin.functions().uploadFunction(sourceFile, path);
-                throw new RuntimeException("Either a jar File or a python file needs to be specified");
-            }
-            String userCodeFile;
-            if (jarFile != null) {
-                userCodeFile = jarFile;
-            } else {
-                userCodeFile = pyFile;
-            }
-            admin.functions().uploadFunction(tenant, namespace, functionName, userCodeFile);
             print("Uploaded successfully");
         }
     }
 
-    @Parameters(commandDescription = "Download Pulsar Function code from Pulsar")
+    @Parameters(commandDescription = "Download File Data from Pulsar")
-    class DownloadFunction extends FunctionCommand {
+    class DownloadFunction extends BaseCommand {
+        @Parameter(
+                names = "--destinationFile",
+                description = "The file where downloaded contents need to be stored",
+                listConverter = StringConverter.class, required = true)
+        protected String destinationFile;
         @Parameter(
-                names = "--downloadPath",
+                names = "--path",
-                description = "Path where the file needs to be downloaded)",
+                description = "Path where the contents are to be stored",
                 listConverter = StringConverter.class, required = true)
-        protected String downloadPath;
+        protected String path;
 
         @Override
         void runCmd() throws Exception {
-            admin.functions().downloadFunction(tenant, namespace, functionName, downloadPath);
+            admin.functions().downloadFunction(destinationFile, path);
             print("Downloaded successfully");
         }
     }

diff --git a/...tions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/...tions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
@@ -529,52 +529,32 @@ public Response triggerFunction(final @PathParam("tenant") String tenant,
     }
 
     @POST
-    @Path("/{tenant}/{namespace}/{functionName}/upload")
+    @Path("/upload")
     @Consumes(MediaType.MULTIPART_FORM_DATA)
-    public Response uploadFunction(final @PathParam("tenant") String tenant,
+    public Response uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream,
-                                   final @PathParam("namespace") String namespace,
+                                   final @FormDataParam("path") String path) {
-                                   final @PathParam("functionName") String functionName,
-                                   final @FormDataParam("data") InputStream uploadedInputStream,
-                                   final @FormDataParam("data") FormDataContentDisposition fileDetail) {
         // validate parameters
         try {
-            if (tenant == null) {
+            if (uploadedInputStream == null || path == null) {
-                throw new IllegalArgumentException("Tenant is not provided");
+                throw new IllegalArgumentException("Function Package is not provided " + path);
-            }
-            if (namespace == null) {
-                throw new IllegalArgumentException("Namespace is not provided");
-            }
-            if (functionName == null) {
-                throw new IllegalArgumentException("Function Name is not provided");
-            }
-            if (uploadedInputStream == null || fileDetail == null) {
-                throw new IllegalArgumentException("Function Package is not provided");
             }
         } catch (IllegalArgumentException e) {
-            log.error("Invalid upload function request @ /{}/{}/{}",
+            log.error("Invalid upload function request @ /{}", path, e);
-                    tenant, namespace, functionName, e);
             return Response.status(Status.BAD_REQUEST)
                     .type(MediaType.APPLICATION_JSON)
                     .entity(new ErrorData(e.getMessage())).build();
         }
 
-        String packageLocation = String.format(
-                        "%s/%s/%s/%s",
-                        tenant,
-                        namespace,
-                        functionName,
-                        fileDetail.getFileName());
-
         // Upload to bookkeeper
         try {
-            log.info("Uploading function package to {}", packageLocation);
+            log.info("Uploading function package to {}", path);
 
             Utils.uploadToBookeeper(
                     worker().getDlogNamespace(),
                     uploadedInputStream,
-                    packageLocation);
+                    path);
         } catch (IOException e) {
-            log.error("Error uploading file {}", packageLocation, e);
+            log.error("Error uploading file {}", path, e);
             return Response.serverError()
                     .type(MediaType.APPLICATION_JSON)
                     .entity(new ErrorData(e.getMessage()))
@@ -585,23 +565,14 @@ public Response uploadFunction(final @PathParam("tenant") String tenant,
     }
 
     @GET
-    @Path("/{tenant}/{namespace}/{functionName}/download")
+    @Path("/download")
-    public Response downloadFunction(final @PathParam("tenant") String tenant,
+    public Response downloadFunction(final @QueryParam("path") String path) {
-                                     final @PathParam("namespace") String namespace,
-                                     final @PathParam("functionName") String functionName,
-                                     final @QueryParam("filename") String fileName) {
-        String packageLocation = String.format(
-                "%s/%s/%s/%s",
-                tenant,
-                namespace,
-                functionName,
-                fileName);
         return Response.status(Status.OK).entity(
                 new StreamingOutput() {
                     @Override
                     public void write(final OutputStream output) throws IOException {
                         Utils.downloadFromBookkeeper(worker().getDlogNamespace(),
-                                output, packageLocation);
+                                output, path);
                     }
                 }).build();
     }

diff --git a/...r/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java b/...r/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java
@@ -142,24 +142,17 @@ public Response triggerFunction(final @PathParam("tenant") String tenant,
     }
 
     @POST
-    @Path("/{tenant}/{namespace}/{functionName}/upload")
+    @Path("/upload")
     @Consumes(MediaType.MULTIPART_FORM_DATA)
-    public Response uploadFunction(final @PathParam("tenant") String tenant,
+    public Response uploadFunction(final @FormDataParam("data") InputStream uploadedInputStream,
-                                   final @PathParam("namespace") String namespace,
+                                   final @FormDataParam("path") String path) {
-                                   final @PathParam("functionName") String functionName,
+        return functions.uploadFunction(uploadedInputStream, path);
-                                   final @FormDataParam("data") InputStream uploadedInputStream,
-                                   final @FormDataParam("data") FormDataContentDisposition fileDetail) {
-        return functions.uploadFunction(
-                tenant, namespace, functionName, uploadedInputStream, fileDetail);
     }
 
     @GET
-    @Path("/{tenant}/{namespace}/{functionName}/download")
+    @Path("/download")
-    public Response downloadFunction(final @PathParam("tenant") String tenant,
+    public Response downloadFunction(final @QueryParam("path") String path) {
-                                     final @PathParam("namespace") String namespace,
+        return functions.downloadFunction(path);
-                                     final @PathParam("functionName") String functionName,
-                                     final @QueryParam("filename") String fileName) {
-        return functions.downloadFunction(tenant, namespace, functionName, fileName);
     }
 
 }