Merge pull request #6957 from QualitativeDataRepository/IQSS/6943

use POST for file download to avoid url length limits

doc/sphinx-guides/source/api/dataaccess.rst

@@ -67,6 +67,8 @@ Multiple File ("bundle") download
 
 ``/api/access/datafiles/$id1,$id2,...$idN``
 
+Alternate Form: POST to ``/api/access/datafiles`` with a ``fileIds`` input field containing the same comma separated list of file ids. This is most useful when your list of files surpasses the allowed URL length (varies but can be ~2000 characters).  
+
 Returns the files listed, zipped. 
 
 .. note:: If the request can only be completed partially - if only *some* of the requested files can be served (because of the permissions and/or size restrictions), the file MANIFEST.TXT included in the zipped bundle will have entries specifying the reasons the missing files could not be downloaded. IN THE FUTURE the API will return a 207 status code to indicate that the result was a partial success. (As of writing this - v.4.11 - this hasn't been implemented yet)

src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java

@@ -182,6 +182,7 @@ public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse) {
             mdcLogService.logEntry(entry);
         } catch (CommandException e) {
             //if an error occurs here then download won't happen no need for response recs...
+            logger.warning("Exception writing GuestbookResponse for file: " + guestbookResponse.getDataFile().getId() + " : " + e.getLocalizedMessage());
         }
     }
 
@@ -203,7 +204,7 @@ public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse) {
     // to the API.
     private void redirectToBatchDownloadAPI(String multiFileString, Boolean guestbookRecordsAlreadyWritten, Boolean downloadOriginal){
 
-        String fileDownloadUrl = "/api/access/datafiles/" + multiFileString;
+        String fileDownloadUrl = "/api/access/datafiles";
         if (guestbookRecordsAlreadyWritten && !downloadOriginal){
             fileDownloadUrl += "?gbrecs=true";
         } else if (guestbookRecordsAlreadyWritten && downloadOriginal){
@@ -212,11 +213,7 @@ private void redirectToBatchDownloadAPI(String multiFileString, Boolean guestboo
             fileDownloadUrl += "?format=original";
         }
 
-        try {
-            FacesContext.getCurrentInstance().getExternalContext().redirect(fileDownloadUrl);
-        } catch (IOException ex) {
-            logger.info("Failed to issue a redirect to file download url.");
-        }
+        PrimeFaces.current().executeScript("downloadFiles('"+fileDownloadUrl + "','"+ multiFileString+"');");
 
     }
 

src/main/java/edu/harvard/iq/dataverse/api/Access.java

@@ -103,9 +103,11 @@
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpSession;
 import javax.ws.rs.BadRequestException;
+import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.ForbiddenException;
 import javax.ws.rs.NotFoundException;
+import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.ServiceUnavailableException;
@@ -525,15 +527,34 @@ public DownloadInstance tabularDatafileMetadataPreprocessed(@PathParam("fileId")
     }
 
     /* 
-     * API method for downloading zipped bundles of multiple files:
+     * API method for downloading zipped bundles of multiple files. Uses POST to avoid long lists of file IDs that can make the URL longer than what's supported by browsers/servers
     */
 
-    // TODO: Rather than only supporting looking up files by their database IDs, consider supporting persistent identifiers.
+    // TODO: Rather than only supporting looking up files by their database IDs,
+    // consider supporting persistent identifiers.
+    @Path("datafiles")
+    @POST
+    @Consumes("text/plain")
+    @Produces({ "application/zip" })
+    public Response postDownloadDatafiles(String fileIds, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException {
+
+
+        return downloadDatafiles(fileIds, gbrecs, apiTokenParam, uriInfo, headers, response);
+    }
+    /*
+     * API method for downloading zipped bundles of multiple files:
+     */
+
+    // TODO: Rather than only supporting looking up files by their database IDs,
+    // consider supporting persistent identifiers.
     @Path("datafiles/{fileIds}")
     @GET
     @Produces({"application/zip"})
-    public Response datafiles(@PathParam("fileIds") String fileIds,  @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
+    public Response datafiles(@PathParam("fileIds") String fileIds, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException {
+        return downloadDatafiles(fileIds, gbrecs, apiTokenParam, uriInfo, headers, response);
+    }
 
+    private Response downloadDatafiles(String rawFileIds, boolean gbrecs, String apiTokenParam, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response) throws WebApplicationException /* throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
         long setLimit = systemConfig.getZipDownloadLimit();
         if (!(setLimit > 0L)) {
             setLimit = DataFileZipper.DEFAULT_ZIPFILE_LIMIT;
@@ -543,10 +564,19 @@ public Response datafiles(@PathParam("fileIds") String fileIds,  @QueryParam("gb
 
         logger.fine("setting zip download size limit to " + zipDownloadSizeLimit + " bytes.");
 
-        if (fileIds == null || fileIds.equals("")) {
+        if (rawFileIds == null || rawFileIds.equals("")) {
             throw new BadRequestException();
         }
-
+        final String fileIds;
+        if(rawFileIds.startsWith("fileIds=")) {
+            fileIds = rawFileIds.substring(8); // String "fileIds=" from the front
+        } else {
+            fileIds=rawFileIds;
+        }
+        /* Note - fileIds coming from the POST ends in '\n' and a ',' has been added after the last file id number and before a
+         * final '\n' - this stops the last item from being parsed in the fileIds.split(","); line below.
+         */
+
         String apiToken = (apiTokenParam == null || apiTokenParam.equals("")) 
                 ? headers.getHeaderString(API_KEY_HEADER) 
                 : apiTokenParam;

src/main/webapp/file-download-button-fragment.xhtml

@@ -277,4 +277,13 @@
         <!-- 4.2.1 - TODO: retest this on a dataset with fileRequest enabled and with some files restricted to the user -->
         <span class="glyphicon glyphicon-bullhorn"/> #{fileMetadata.dataFile.fileAccessRequesters.contains(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']}
     </p:commandLink>
+    <script type="text/javascript">
+      function downloadFiles(url, filelist) {
+        filelist=filelist + ','; //Prevents last file from being dropped on server
+        var form = $('<form></form>').attr('action', url).attr('method', 'post').attr('enctype','text/plain');
+        form.append($("<input></input>").attr('type', 'hidden').attr('name', 'fileIds').attr('value', filelist));
+        //Submit and then remove form
+        form.appendTo('body').submit().remove();
+      }
+    </script>
 </ui:composition>