IQSS/6763-multi-part upload API calls

doc/release-notes/6763-multipart-uploads.md

@@ -0,0 +1,3 @@
+# Large Data Support (continued)
+
+Installations configured for direct S3 upload will be able to use the [Dataverse Uploader](https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader) to upload large (>5 GB) files to Dataverse.

doc/sphinx-guides/source/developers/big-data-support.rst

@@ -44,7 +44,8 @@ with the contents of the file cors.json as follows:
              {
                 "AllowedOrigins": ["https://<DATAVERSE SERVER>"],
                 "AllowedHeaders": ["*"],
-                "AllowedMethods": ["PUT", "GET"]
+                "AllowedMethods": ["PUT", "GET"],
+                "ExposeHeaders": ["ETag"]
              }
           ]
         }

src/main/java/META-INF/mime.types

@@ -7,7 +7,7 @@ text/comma-separated-values csv CSV
 text/plain txt TXT
 text/xml xml XML
 # Common statistical data formats
-text/tab-separated-values tab TAB tsv TSV
+text/tsv tab TAB tsv TSV
 text/x-fixed-field dat DAT asc ASC
 application/x-rlang-transport Rdata RData rdata RDATA
 type/x-r-syntax r R

src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java

@@ -58,6 +58,7 @@
 import org.primefaces.model.file.UploadedFile;
 import javax.json.Json;
 import javax.json.JsonObject;
+import javax.json.JsonObjectBuilder;
 import javax.json.JsonArray;
 import javax.json.JsonReader;
 import org.apache.commons.httpclient.HttpClient;
@@ -1721,6 +1722,7 @@ public String getRsyncScriptFilename() {
         return rsyncScriptFilename;
     }
 
+    @Deprecated
     public void requestDirectUploadUrl() {
 
 
@@ -1742,6 +1744,38 @@ public void requestDirectUploadUrl() {
     	PrimeFaces.current().executeScript("uploadFileDirectly('" + url + "','" + storageIdentifier + "')");
     }
 
+	public void requestDirectUploadUrls() {
+
+		Map<String, String> paramMap = FacesContext.getCurrentInstance().getExternalContext().getRequestParameterMap();
+
+		String sizeString = paramMap.get("fileSize");
+		long fileSize = Long.parseLong(sizeString);
+
+		S3AccessIO<?> s3io = FileUtil.getS3AccessForDirectUpload(dataset);
+		if (s3io == null) {
+			FacesContext.getCurrentInstance().addMessage(uploadComponentId,
+					new FacesMessage(FacesMessage.SEVERITY_ERROR,
+							BundleUtil.getStringFromBundle("dataset.file.uploadWarning"),
+							"Direct upload not supported for this dataset"));
+		}
+		JsonObjectBuilder urls = null;
+		String storageIdentifier = null;
+		try {
+			storageIdentifier = FileUtil.getStorageIdentifierFromLocation(s3io.getStorageLocation());
+			urls = s3io.generateTemporaryS3UploadUrls(dataset.getGlobalId().asString(), storageIdentifier, fileSize);
+
+		} catch (IOException io) {
+			logger.warning(io.getMessage());
+			FacesContext.getCurrentInstance().addMessage(uploadComponentId,
+					new FacesMessage(FacesMessage.SEVERITY_ERROR,
+							BundleUtil.getStringFromBundle("dataset.file.uploadWarning"),
+							"Issue in connecting to S3 store for direct upload"));
+		}
+
+		PrimeFaces.current().executeScript(
+				"uploadFileDirectly('" + urls.build().toString() + "','" + storageIdentifier + "','" + fileSize + "')");
+	}
+
     public void uploadFinished() {
         // This method is triggered from the page, by the <p:upload ... onComplete=...
         // attribute. 

src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java

@@ -378,7 +378,7 @@ protected AuthenticatedUser findAuthenticatedUserOrDie() throws WrappedResponse
     private AuthenticatedUser findAuthenticatedUserOrDie( String key ) throws WrappedResponse {
         AuthenticatedUser authUser = authSvc.lookupUser(key);
         if ( authUser != null ) {
-            authUser = userSvc.updateLastApiUseTime(authUser);
+             authUser = userSvc.updateLastApiUseTime(authUser);
 
             return authUser;
         }

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -78,10 +78,8 @@
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
 import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
 import edu.harvard.iq.dataverse.S3PackageImporter;
-import static edu.harvard.iq.dataverse.api.AbstractApiBean.error;
 import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO;
 import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
-import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataaccess.S3AccessIO;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
@@ -128,8 +126,10 @@
 import javax.json.Json;
 import javax.json.JsonArray;
 import javax.json.JsonArrayBuilder;
+import javax.json.JsonException;
 import javax.json.JsonObject;
 import javax.json.JsonObjectBuilder;
+import javax.json.JsonReader;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.Consumes;
@@ -153,6 +153,8 @@
 import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
 import org.glassfish.jersey.media.multipart.FormDataParam;
 
+import com.amazonaws.services.s3.model.PartETag;
+
 @Path("datasets")
 public class Datasets extends AbstractApiBean {
 
@@ -1485,6 +1487,7 @@ public Response returnToAuthor(@PathParam("id") String idSupplied, String jsonBo
 
 @GET
 @Path("{id}/uploadsid")
+@Deprecated
 public Response getUploadUrl(@PathParam("id") String idSupplied) {
 	try {
 		Dataset dataset = findDatasetOrDie(idSupplied);
@@ -1494,6 +1497,7 @@ public Response getUploadUrl(@PathParam("id") String idSupplied) {
 			canUpdateDataset = permissionSvc.requestOn(createDataverseRequest(findUserOrDie()), dataset).canIssue(UpdateDatasetVersionCommand.class);
 		} catch (WrappedResponse ex) {
 			logger.info("Exception thrown while trying to figure out permissions while getting upload URL for dataset id " + dataset.getId() + ": " + ex.getLocalizedMessage());
+			throw ex;
 		}
 		if (!canUpdateDataset) {
             return error(Response.Status.FORBIDDEN, "You are not permitted to upload files to this dataset.");
@@ -1520,6 +1524,179 @@ public Response getUploadUrl(@PathParam("id") String idSupplied) {
 		return wr.getResponse();
 	}
 }
+
+@GET
+@Path("{id}/uploadurls")
+public Response getMPUploadUrls(@PathParam("id") String idSupplied, @QueryParam("size") long fileSize) {
+	try {
+		Dataset dataset = findDatasetOrDie(idSupplied);
+
+		boolean canUpdateDataset = false;
+		try {
+			canUpdateDataset = permissionSvc.requestOn(createDataverseRequest(findUserOrDie()), dataset)
+					.canIssue(UpdateDatasetVersionCommand.class);
+		} catch (WrappedResponse ex) {
+			logger.info(
+					"Exception thrown while trying to figure out permissions while getting upload URLs for dataset id "
+							+ dataset.getId() + ": " + ex.getLocalizedMessage());
+			throw ex;
+		}
+		if (!canUpdateDataset) {
+			return error(Response.Status.FORBIDDEN, "You are not permitted to upload files to this dataset.");
+		}
+		S3AccessIO<DataFile> s3io = FileUtil.getS3AccessForDirectUpload(dataset);
+		if (s3io == null) {
+			return error(Response.Status.NOT_FOUND,
+					"Direct upload not supported for files in this dataset: " + dataset.getId());
+		}
+		JsonObjectBuilder response = null;
+		String storageIdentifier = null;
+		try {
+			storageIdentifier = FileUtil.getStorageIdentifierFromLocation(s3io.getStorageLocation());
+			response = s3io.generateTemporaryS3UploadUrls(dataset.getGlobalId().asString(), storageIdentifier, fileSize);
+
+		} catch (IOException io) {
+			logger.warning(io.getMessage());
+			throw new WrappedResponse(io,
+					error(Response.Status.INTERNAL_SERVER_ERROR, "Could not create process direct upload request"));
+		}
+
+		response.add("storageIdentifier", storageIdentifier);
+		return ok(response);
+	} catch (WrappedResponse wr) {
+		return wr.getResponse();
+	}
+}
+
+@DELETE
+@Path("mpupload")
+public Response abortMPUpload(@QueryParam("globalid") String idSupplied, @QueryParam("storageidentifier") String storageidentifier, @QueryParam("uploadid") String uploadId) {
+	try {
+		Dataset dataset = datasetSvc.findByGlobalId(idSupplied);
+		//Allow the API to be used within a session (e.g. for direct upload in the UI)
+		User user =session.getUser();
+		if (!user.isAuthenticated()) {
+			try {
+				user = findAuthenticatedUserOrDie();
+			} catch (WrappedResponse ex) {
+				logger.info(
+						"Exception thrown while trying to figure out permissions while getting aborting upload for dataset id "
+								+ dataset.getId() + ": " + ex.getLocalizedMessage());
+				throw ex;
+			}
+		}
+		boolean allowed = false;
+		if (dataset != null) {
+				allowed = permissionSvc.requestOn(createDataverseRequest(user), dataset)
+						.canIssue(UpdateDatasetVersionCommand.class);
+		} else {
+			/*
+			 * The only legitimate case where a global id won't correspond to a dataset is
+			 * for uploads during creation. Given that this call will still fail unless all
+			 * three parameters correspond to an active multipart upload, it should be safe
+			 * to allow the attempt for an authenticated user. If there are concerns about
+			 * permissions, one could check with the current design that the user is allowed
+			 * to create datasets in some dataverse that is configured to use the storage
+			 * provider specified in the storageidentifier, but testing for the ability to
+			 * create a dataset in a specific dataverse would requiring changing the design
+			 * somehow (e.g. adding the ownerId to this call).
+			 */
+			allowed = true;
+		}
+		if (!allowed) {
+			return error(Response.Status.FORBIDDEN,
+					"You are not permitted to abort file uploads with the supplied parameters.");
+		}
+		try {
+			S3AccessIO.abortMultipartUpload(idSupplied, storageidentifier, uploadId);
+		} catch (IOException io) {
+			logger.warning("Multipart upload abort failed for uploadId: " + uploadId + " storageidentifier="
+					+ storageidentifier + " dataset Id: " + dataset.getId());
+			logger.warning(io.getMessage());
+			throw new WrappedResponse(io,
+					error(Response.Status.INTERNAL_SERVER_ERROR, "Could not abort multipart upload"));
+		}
+		return Response.noContent().build();
+	} catch (WrappedResponse wr) {
+		return wr.getResponse();
+	}
+}
+
+@PUT
+@Path("mpupload")
+public Response completeMPUpload(String partETagBody, @QueryParam("globalid") String idSupplied, @QueryParam("storageidentifier") String storageidentifier, @QueryParam("uploadid") String uploadId)  {
+	try {
+		Dataset dataset = datasetSvc.findByGlobalId(idSupplied);
+		//Allow the API to be used within a session (e.g. for direct upload in the UI)
+		User user =session.getUser();
+		if (!user.isAuthenticated()) {
+			try {
+				user=findAuthenticatedUserOrDie();
+			} catch (WrappedResponse ex) {
+				logger.info(
+						"Exception thrown while trying to figure out permissions to complete mpupload for dataset id "
+								+ dataset.getId() + ": " + ex.getLocalizedMessage());
+				throw ex;
+			}
+		}
+		boolean allowed = false;
+		if (dataset != null) {
+				allowed = permissionSvc.requestOn(createDataverseRequest(user), dataset)
+						.canIssue(UpdateDatasetVersionCommand.class);
+		} else {
+			/*
+			 * The only legitimate case where a global id won't correspond to a dataset is
+			 * for uploads during creation. Given that this call will still fail unless all
+			 * three parameters correspond to an active multipart upload, it should be safe
+			 * to allow the attempt for an authenticated user. If there are concerns about
+			 * permissions, one could check with the current design that the user is allowed
+			 * to create datasets in some dataverse that is configured to use the storage
+			 * provider specified in the storageidentifier, but testing for the ability to
+			 * create a dataset in a specific dataverse would requiring changing the design
+			 * somehow (e.g. adding the ownerId to this call).
+			 */
+			allowed = true;
+		}
+		if (!allowed) {
+			return error(Response.Status.FORBIDDEN,
+					"You are not permitted to complete file uploads with the supplied parameters.");
+		}
+		List<PartETag> eTagList = new ArrayList<PartETag>();
+        logger.info("Etags: " + partETagBody);
+		try {
+			JsonReader jsonReader = Json.createReader(new StringReader(partETagBody));
+			JsonObject object = jsonReader.readObject();
+			jsonReader.close();
+			for(String partNo : object.keySet()) {
+				eTagList.add(new PartETag(Integer.parseInt(partNo), object.getString(partNo)));
+			}
+			for(PartETag et: eTagList) {
+				logger.info("Part: " + et.getPartNumber() + " : " + et.getETag());
+			}
+		} catch (JsonException je) {
+			logger.info("Unable to parse eTags from: " + partETagBody);
+			throw new WrappedResponse(je, error( Response.Status.INTERNAL_SERVER_ERROR, "Could not complete multipart upload"));
+		}
+		try {
+			S3AccessIO.completeMultipartUpload(idSupplied, storageidentifier, uploadId, eTagList);
+		} catch (IOException io) {
+			logger.warning("Multipart upload completion failed for uploadId: " + uploadId +" storageidentifier=" + storageidentifier + " globalId: " + idSupplied);
+			logger.warning(io.getMessage());
+			try {
+				S3AccessIO.abortMultipartUpload(idSupplied, storageidentifier, uploadId);
+			} catch (IOException e) {
+				logger.severe("Also unable to abort the upload (and release the space on S3 for uploadId: " + uploadId +" storageidentifier=" + storageidentifier + " globalId: " + idSupplied);
+				logger.severe(io.getMessage());
+			}
+
+			throw new WrappedResponse(io, error( Response.Status.INTERNAL_SERVER_ERROR, "Could not complete multipart upload")); 
+		}
+		return ok("Multipart Upload completed");
+	} catch (WrappedResponse wr) {
+		return wr.getResponse();
+	}
+}
+
     /**
      * Add a File to an existing Dataset
      *