7767 unable to download large guestbooks

doc/sphinx-guides/source/api/native-api.rst

@@ -580,10 +580,32 @@ The fully expanded example above (without environment variables) looks like this
 
 .. code-block:: bash
 
-  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/actions/:publish
+  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/actions/:publish  
 
 You should expect a 200 ("OK") response and JSON output.
 
+Retrieve Guestbook Responses for a Dataverse Collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to retrieve a file containing a list of Guestbook Responses in csv format for Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. If the Dataverse collection has more than one guestbook you may provide the id of a single guestbook as an optional parameter. If no guestbook id is provided the results returned will be the same as pressing the "Download All Responses" button on the Manage Dataset Guestbook page. If the guestbook id is provided then only those responses from that guestbook will be included in the file.  
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=root
+  export GUESTBOOK_ID=1
+
+  curl -O -J -f -H  X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=$GUESTBOOK_ID
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -O -J -f -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1
+
 Datasets
 --------
 

src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java

@@ -128,6 +128,18 @@ public void streamResponsesByDataverseIdAndGuestbookId(OutputStream out, Long da
         // of queries now) -- L.A. 
 
         Map<Integer, Object> customQandAs = mapCustomQuestionAnswersAsStrings(dataverseId, guestbookId);
+
+        List<Object[]> guestbookResults = getGuestbookResults( dataverseId,  guestbookId );
+        // the CSV header:
+        out.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes());
+        for (Object[] result : guestbookResults) {
+            StringBuilder sb = convertGuestbookResponsesToCSV(customQandAs, result);
+            out.write(sb.toString().getBytes());
+            out.flush();
+        }
+    }
+
+    public List<Object[]> getGuestbookResults(Long dataverseId, Long guestbookId ){
 
         String queryString = BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV
                 + " and  o.owner_id = " 
@@ -137,15 +149,15 @@ public void streamResponsesByDataverseIdAndGuestbookId(OutputStream out, Long da
             queryString+= (" and r.guestbook_id = " + guestbookId.toString());
         }
 
-        queryString += ";";
+        queryString += " ORDER by r.id DESC;";
         logger.fine("stream responses query: " + queryString);
 
-        List<Object[]> guestbookResults = em.createNativeQuery(queryString).getResultList();
-
-        // the CSV header:
-        out.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes());
+        return  em.createNativeQuery(queryString).getResultList();
 
-        for (Object[] result : guestbookResults) {
+    }
+
+    public StringBuilder convertGuestbookResponsesToCSV ( Map<Integer, Object> customQandAs, Object[] result) throws IOException {
+
             Integer guestbookResponseId = (Integer)result[0];
 
             StringBuilder sb = new StringBuilder();
@@ -208,36 +220,17 @@ public void streamResponsesByDataverseIdAndGuestbookId(OutputStream out, Long da
 
             // Finally, custom questions and answers, if present:
 
-            // (the old implementation, below, would run one extra query FOR EVERY SINGLE
-            // guestbookresponse entry! -- instead, we are now pre-caching all the 
-            // available custom question responses, with a single native query at 
-            // the top of this method. -- L.A.)
-
-            /*String cqString = "select q.questionstring, r.response  from customquestionresponse r, customquestion q where q.id = r.customquestion_id and r.guestbookResponse_id = " + result[0];
-            List<Object[]> customResponses = em.createNativeQuery(cqString).getResultList();
-            if (customResponses != null) {
-                for (Object[] response : customResponses) {
-                    sb.append(SEPARATOR);
-                    sb.append(response[0]);
-                    sb.append(SEPARATOR);
-                    sb.append(response[1] == null ? "" : response[1]);
-                }
-            }*/
 
             if (customQandAs.containsKey(guestbookResponseId)) {
                 sb.append(customQandAs.get(guestbookResponseId)); 
             } 
 
             sb.append(NEWLINE);
-
-            // Finally, write the line out: 
-            // (i.e., we are writing one guestbook response at a time, thus allowing the 
-            // whole thing to stream in real time -- L.A.)
-            out.write(sb.toString().getBytes());
-            out.flush();
-        }
+        return sb;
+
     }
 
+
     private String formatPersistentIdentifier(String protocol, String authority, String identifier) {
         // Note that the persistent id may be unavailable for this dvObject:
         if (StringUtil.nonEmpty(protocol) && StringUtil.nonEmpty(authority) && StringUtil.nonEmpty(identifier)) {
@@ -349,7 +342,7 @@ private Map<Integer, Object> mapCustomQuestionAnswersAsLists(Long dataverseId, L
         return selectCustomQuestionAnswers(dataverseId, guestbookId, false, firstResponse, lastResponse);
     }
 
-    private Map<Integer, Object> mapCustomQuestionAnswersAsStrings(Long dataverseId, Long guestbookId) {
+    public Map<Integer, Object> mapCustomQuestionAnswersAsStrings(Long dataverseId, Long guestbookId) {
         return selectCustomQuestionAnswers(dataverseId, guestbookId, true, null, null);
     }
 

src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java

@@ -11,6 +11,8 @@
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
 import edu.harvard.iq.dataverse.DvObject;
 import edu.harvard.iq.dataverse.GlobalId;
+import edu.harvard.iq.dataverse.GuestbookResponseServiceBean;
+import edu.harvard.iq.dataverse.GuestbookServiceBean;
 import edu.harvard.iq.dataverse.MetadataBlock;
 import edu.harvard.iq.dataverse.RoleAssignment;
 import static edu.harvard.iq.dataverse.api.AbstractApiBean.error;
@@ -98,10 +100,16 @@
 import javax.ws.rs.core.Response.Status;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import java.io.IOException;
 import java.text.MessageFormat;
+import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.Map;
 import java.util.Optional;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.Context;
 import javax.xml.stream.XMLStreamException;
 
 /**
@@ -114,6 +122,7 @@
 public class Dataverses extends AbstractApiBean {
 
     private static final Logger logger = Logger.getLogger(Dataverses.class.getCanonicalName());
+    private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
 
     @EJB
     ExplicitGroupServiceBean explicitGroupSvc;
@@ -123,6 +132,12 @@ public class Dataverses extends AbstractApiBean {
 
     @EJB
     SettingsServiceBean settingsService;
+
+    @EJB
+    GuestbookResponseServiceBean guestbookResponseService;
+
+    @EJB
+    GuestbookServiceBean guestbookService;
 
     @POST
     public Response addRoot(String body) {
@@ -832,7 +847,49 @@ public Response getGroupByOwnerAndAliasInOwner(@PathParam("identifier") String d
                 req,
                 grpAliasInOwner))));
     }
+
+    @GET
+    @Path("{identifier}/guestbookResponses/")
+    @Produces({"application/download"})
+    public Response getGuestbookResponsesByDataverse(@PathParam("identifier") String dvIdtf,
+            @QueryParam("guestbookId") Long gbId, @Context HttpServletResponse response) {
+
+        try {
+            Dataverse dv = findDataverseOrDie(dvIdtf);
+            User u = findUserOrDie();
+            DataverseRequest req = createDataverseRequest(u);
+            if (permissionSvc.request(req)
+                    .on(dv)
+                    .has(Permission.EditDataverse)) {
+            } else {
+                return error(Status.FORBIDDEN, "Not authorized");
+            }
+
+            String fileTimestamp = dateFormatter.format(new Date());
+            String filename = dv.getAlias() + "_GBResponses_" + fileTimestamp + ".csv";
+
+            response.setHeader("Content-Disposition", "attachment; filename="
+                + filename);
+               ServletOutputStream outputStream = response.getOutputStream();
+
+            Map<Integer, Object> customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId);
+
+            List<Object[]> guestbookResults = guestbookResponseService.getGuestbookResults(dv.getId(), gbId);
+            outputStream.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes());
+            for (Object[] result : guestbookResults) {
+                StringBuilder sb = guestbookResponseService.convertGuestbookResponsesToCSV(customQandAs, result);
+                outputStream.write(sb.toString().getBytes());
+                outputStream.flush();
+            }
+            return Response.ok().build();
+        } catch (IOException io) {
+            return error(Status.BAD_REQUEST, "Failed to produce response file. Exception: " + io.getMessage());
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
 
+    }
+
     @PUT
     @Path("{identifier}/groups/{aliasInOwner}")
     public Response updateGroup(ExplicitGroupDTO groupDto,