a few small changes per code review comments (#5498)

IQSS · Feb 4, 2019 · 3034a2e · 3034a2e
1 parent 53f7dca
commit 3034a2e
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 12 deletions.
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -107,6 +107,13 @@
 import javax.ws.rs.core.Response;
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import javax.ws.rs.core.StreamingOutput;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
 
 /*
     Custom API exceptions [NOT YET IMPLEMENTED]

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
@@ -1249,7 +1249,7 @@ public Response updateHashValues(@PathParam("alg") String alg, @QueryParam("num"
 						}
 						if (in == null)
 							logger.warning("Cannot retrieve file.");
-						String currentChecksum = FileUtil.CalculateChecksum(in, df.getChecksumType());
+						String currentChecksum = FileUtil.calculateChecksum(in, df.getChecksumType());
 						if (currentChecksum.equals(df.getChecksumValue())) {
 							logger.fine("Current checksum for datafile: " + df.getFileMetadata().getLabel() + ", "
 									+ df.getIdentifier() + " is valid");
@@ -1263,7 +1263,7 @@ public Response updateHashValues(@PathParam("alg") String alg, @QueryParam("num"
 							}
 							if (in2 == null)
 								logger.warning("Cannot retrieve file to calculate new checksum.");
-							String newChecksum = FileUtil.CalculateChecksum(in2, cType);
+							String newChecksum = FileUtil.calculateChecksum(in2, cType);
 
 							df.setChecksumType(cType);
 							df.setChecksumValue(newChecksum);

diff --git a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordWriter.java
@@ -319,7 +319,7 @@ private DataFile createPackageDataFile(List<File> files) {
                 // add code to generate the manifest, if not present? -- L.A. 
             } else {
                 try {
-                    packageFile.setChecksumValue(FileUtil.CalculateChecksum(checksumManifestPath, packageFile.getChecksumType()));
+                    packageFile.setChecksumValue(FileUtil.calculateChecksum(checksumManifestPath, packageFile.getChecksumType()));
                 } catch (Exception ex) {
                     getJobLogger().log(Level.SEVERE, "Failed to calculate checksum (type "+packageFile.getChecksumType()+") "+ex.getMessage());
                     jobContext.setExitStatus("FAILED");

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataFileZipper.java
@@ -42,6 +42,7 @@ public class DataFileZipper {
     public static long DEFAULT_ZIPFILE_LIMIT = 100 * 1024 * 1024; // 100MB
 
     private static final Logger logger = Logger.getLogger(DataFileZipper.class.getCanonicalName());
+    private static final String MANIFEST_FILE_NAME = "MANIFEST.TXT";
 
     private OutputStream outputStream = null; 
     private ZipOutputStream zipOutputStream = null;
@@ -138,8 +139,13 @@ public long addFileToZipStream(DataFile dataFile, boolean getOriginal) throws IO
 
                 Success = false;
             } else {
+                // If any of the files have non-empty DirectoryLabels we'll 
+                // use them to re-create the folders in the Zipped bundle:
                 String folderName = dataFile.getFileMetadata().getDirectoryLabel(); 
                 if (folderName != null) {
+                    // If any of the saved folder names start with with slashes,
+                    // we want to remove them: 
+                    // (i.e., ///foo/bar will become foo/bar)
                     while (folderName.startsWith("/")) {
                         folderName = folderName.substring(1);
                     }
@@ -208,7 +214,12 @@ public void finalizeZipStream() throws IOException {
         }
 
         if (createManifest) {
-            ZipEntry e = new ZipEntry("MANIFEST.TXT");
+            String manifestEntry = MANIFEST_FILE_NAME; 
+            while (fileNameList.contains(manifestEntry)) {
+                manifestEntry = "0".concat(manifestEntry); 
+            }
+
+            ZipEntry e = new ZipEntry(manifestEntry);
 
             zipOutputStream.putNextEntry(e);
             zipOutputStream.write(fileManifest.getBytes());

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
@@ -34,6 +34,13 @@
 import java.util.Map;
 import javax.imageio.ImageIO;
 import org.apache.commons.io.IOUtils;
+import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
+import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
+import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
+import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
+import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
+import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
+import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
 
 public class DatasetUtil {
 

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -77,6 +77,13 @@
 import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
 
 
 /**
@@ -505,7 +512,7 @@ private static boolean isGraphMLFile(File file) {
     }
 
     // from MD5Checksum.java
-    public static String CalculateChecksum(String datafile, ChecksumType checksumType) {
+    public static String calculateChecksum(String datafile, ChecksumType checksumType) {
 
         FileInputStream fis = null;
         try {
@@ -514,11 +521,11 @@ public static String CalculateChecksum(String datafile, ChecksumType checksumTyp
             throw new RuntimeException(ex);
         }
 
-        return CalculateChecksum(fis, checksumType);
+        return FileUtil.calculateChecksum(fis, checksumType);
     }
 
     // from MD5Checksum.java
-    public static String CalculateChecksum(InputStream in, ChecksumType checksumType) {
+    public static String calculateChecksum(InputStream in, ChecksumType checksumType) {
         MessageDigest md = null;
         try {
             // Use "SHA-1" (toString) rather than "SHA1", for example.
@@ -546,7 +553,7 @@ public static String CalculateChecksum(InputStream in, ChecksumType checksumType
         return checksumDigestToString(md.digest());
     }
 
-    public static String CalculateChecksum(byte[] dataBytes, ChecksumType checksumType) {
+    public static String calculateChecksum(byte[] dataBytes, ChecksumType checksumType) {
         MessageDigest md = null;
         try {
             // Use "SHA-1" (toString) rather than "SHA1", for example.
@@ -1052,7 +1059,7 @@ private static DataFile createSingleDataFile(DatasetVersion version, File tempFi
         try {
             // We persist "SHA1" rather than "SHA-1".
             datafile.setChecksumType(checksumType);
-            datafile.setChecksumValue(CalculateChecksum(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(), datafile.getChecksumType()));
+            datafile.setChecksumValue(calculateChecksum(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(), datafile.getChecksumType()));
         } catch (Exception cksumEx) {
             logger.warning("Could not calculate " + checksumType + " signature for the new file " + fileName);
         }

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
@@ -31,6 +31,12 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
 
 /**
  *
@@ -575,9 +581,9 @@ public void testZipUploadAndDownload() throws IOException {
 
         // And finally, check the md5 checksums of the unzipped file streams: 
 
-        assertEquals(testFileFromZipUploadWithFoldersChecksum1, FileUtil.CalculateChecksum(unzippedFiles.get(testFileFromZipUploadWithFolders1).toByteArray(), DataFile.ChecksumType.MD5));
-        assertEquals(testFileFromZipUploadWithFoldersChecksum2, FileUtil.CalculateChecksum(unzippedFiles.get(testFileFromZipUploadWithFolders2).toByteArray(), DataFile.ChecksumType.MD5)); 
-        assertEquals(testFileFromZipUploadWithFoldersChecksum3, FileUtil.CalculateChecksum(unzippedFiles.get(testFileFromZipUploadWithFolders3).toByteArray(), DataFile.ChecksumType.MD5));
+        assertEquals(testFileFromZipUploadWithFoldersChecksum1, FileUtil.calculateChecksum(unzippedFiles.get(testFileFromZipUploadWithFolders1).toByteArray(), DataFile.ChecksumType.MD5));
+        assertEquals(testFileFromZipUploadWithFoldersChecksum2, FileUtil.calculateChecksum(unzippedFiles.get(testFileFromZipUploadWithFolders2).toByteArray(), DataFile.ChecksumType.MD5)); 
+        assertEquals(testFileFromZipUploadWithFoldersChecksum3, FileUtil.calculateChecksum(unzippedFiles.get(testFileFromZipUploadWithFolders3).toByteArray(), DataFile.ChecksumType.MD5));
 
         System.out.println("MD5 checksums of the unzipped file streams are correct.");