From 3db44862192196891a4d178d0ebce923bb8454ad Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 31 May 2018 16:02:59 -0400 Subject: [PATCH 001/137] Added new UI prototype to Upload Files pg. Code comments include some to-do's to tackle. [ref #4610] --- src/main/java/Bundle.properties | 5 +- .../iq/dataverse/EditDatafilesPage.java | 3 +- src/main/webapp/editFilesFragment.xhtml | 158 ++++++++++++++---- src/main/webapp/editdatafiles.xhtml | 31 +++- 4 files changed, 157 insertions(+), 40 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index fe87288740a..f4ee44d7f9c 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1226,7 +1226,8 @@ dataset.keywordDisplay.title=Keyword dataset.subjectDisplay.title=Subject dataset.contact.tip=Use email button above to contact. dataset.asterisk.tip=Asterisks indicate required fields -dataset.message.uploadFiles=Upload Dataset Files - You can drag and drop files from your desktop, directly into the upload widget. +dataset.message.uploadFiles.label=Upload Dataset Files +dataset.message.uploadFiles.message=For more information about supported file formats, please refer to the User Guide. dataset.message.editMetadata=Edit Dataset Metadata - Add more metadata about this dataset to help others easily find it. dataset.message.editTerms=Edit Dataset Terms - Update this dataset's terms of use. dataset.message.locked.editNotAllowedInReview=Dataset cannot be edited due to In Review dataset lock. @@ -1302,7 +1303,7 @@ file.count={0} {0, choice, 0#Files|1#File|2#Files} file.count.selected={0} {0, choice, 0#Files Selected|1#File Selected|2#Files Selected} file.selectToAddBtn=Select Files to Add file.selectToAdd.tipLimit=File upload limit is {0} bytes per file. -file.selectToAdd.tipMoreInformation=For more information about supported file formats, please refer to the User Guide. +file.selectToAdd.tipMoreInformation=Select or drag and drop files from your desktop, directly into the upload widget. file.selectToAdd.dragdropMsg=Drag and drop files here. file.createUploadDisabled=Once you have saved your dataset, you can upload your data using the "Upload Files" button on the dataset page. For more information about supported file formats, please refer to the User Guide. file.fromDropbox=Upload from Dropbox diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index f3f7a93978c..36f9090eba0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -20,6 +20,7 @@ import edu.harvard.iq.dataverse.ingest.IngestUtil; import edu.harvard.iq.dataverse.search.FileView; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.JsfHelper; import static edu.harvard.iq.dataverse.util.JsfHelper.JH; @@ -551,7 +552,7 @@ public String init() { saveEnabled = true; if (mode == FileEditMode.UPLOAD) { - JH.addMessage(FacesMessage.SEVERITY_INFO, getBundleString("dataset.message.uploadFiles")); + JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.uploadFiles.label"), BundleUtil.getStringFromBundle("dataset.message.uploadFiles.message", Arrays.asList(systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion()))); } if (settingsService.isTrueForKey(SettingsServiceBean.Key.PublicInstall, false)){ diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index cb651d8c5fb..621dcb4d643 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -20,8 +20,46 @@
+ - +
+ + + +
+ + + + + +
+ + +
+ +

+ Help message proving value and info about HTTP upload to the user. +

+ +

+ + + + + + + + +

+ -

- - - - - - - - - - - - -

+ + @@ -198,29 +224,100 @@ invalidSizeMessage="#{bundle['file.edit.error.file_exceeds_limit']}" sequential="true" widgetVar="fileUploadWidget"/> - -
-
-
+
+
+ + +
+ + + + + +
+
+ +

+ Help message proving value and info about Dropbox to the user. +

+ + +
+
+
+ +
+
-
+ +
+ + + + + +
+
+

+ Help message proving value and info about rsync to the user. +

+

+ Warning message about how this option is not available because of uploaded files to this dataset. +

+
+
+
+ +
+ + +
+ + + + + +
+ +
diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index 61ce0ed5c1e..3a3c14b4a7b 100755 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -100,15 +100,28 @@
-
- - - -
- - + + + - + - +
-
+ - -
- - - - - -
-
-

- Help message proving value and info about rsync to the user. + +

+ + +
+
+

+ Help message proving value and info about rsync to the user. +

+

+ Warning message about how this option is not available because of uploaded files to this dataset. +

+
+
+ + + +

+ +

-

- Warning message about how this option is not available because of uploaded files to this dataset. + + +

+ +

-
+ + +

+ + + + + +

+
    +
  1. + +
    2. +
  2. + + + + +
    3. +
  3. + + + +
    4. +
  4. + +
    5. +
+ + +
+
-
+ + + + + + + +
+ +
+
+

#{bundle['file.createUploadDisabled']}

+
+
+ +
+ +
+ +
@@ -547,8 +612,8 @@

#{bundle['file.deleteFileDialog.failed.tip']}

- +
@@ -891,6 +956,10 @@ $('button[id$="compareVersions"]').trigger('click'); } } + function deleteFinished() { + $('button[id$="updateEditDataFilesButtonsForDelete"]').trigger('click'); + $('button[id$="allDeletesFinished"]').trigger('click'); + } function checkFilesSelected() { var count = PF('filesTable').getSelectedRowsCount(); if (count > 0) { diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index 7911080cdf3..6238ca5a091 100755 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -20,6 +20,7 @@ + @@ -44,7 +45,7 @@ -
+
@@ -95,36 +96,37 @@
- - + + + +
-
- - files buttons edit datafiles -
- - - -
+ +
+ +
+ + + +
+ +
-
- - - - - - - + + + +
+
- -
+ + diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index b19ba58fd23..5f734eeff4c 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -90,18 +90,17 @@ - count
- files fragment
+ + + and !widgetWrapper.widgetView }"> From d0e453716ee904a66de1c2a6dfe6f41d6f2c31d9 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 13 Jul 2018 13:50:03 -0400 Subject: [PATCH 004/137] #4610 remove unused code; fix broken test --- src/main/webapp/dataset.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index c101ab6fcec..1dea5e88dc2 100755 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1153,7 +1153,7 @@
- + @@ -1202,7 +1202,7 @@
- +

From fb934f27144bdd96b927c9914d6658b4bfab0eac Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 17 Jul 2018 14:40:58 -0400 Subject: [PATCH 005/137] #4610 Resolve conflicts fix Download --- .../java/edu/harvard/iq/dataverse/DatasetVersion.java | 10 ++++++++++ src/main/webapp/dataset.xhtml | 5 +++-- src/main/webapp/dataverse.xhtml | 4 ++-- src/main/webapp/editFilesFragment.xhtml | 2 +- src/main/webapp/editdatafiles.xhtml | 2 +- src/main/webapp/filesFragment.xhtml | 9 +++++---- src/main/webapp/metadataFragment.xhtml | 8 ++++---- src/main/webapp/themeAndWidgetsFragment.xhtml | 4 ++-- 8 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 876da73d619..da07abe1526 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -478,6 +478,16 @@ public boolean isMinorUpdate() { return true; } + public boolean isHasPackageFile(){ + if (this.fileMetadatas.isEmpty()){ + return false; + } + if(this.fileMetadatas.size() > 1){ + return false; + } + return this.fileMetadatas.get(0).getDataFile().getContentType().equals("application/vnd.dataverse.file-package"); + } + public void updateDefaultValuesFromTemplate(Template template) { if (!template.getDatasetFields().isEmpty()) { this.setDatasetFields(this.copyDatasetFields(template.getDatasetFields())); diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index be02be3dd81..c98a0600f70 100755 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -198,13 +198,14 @@ #{bundle['dataset.editBtn']}

- +
- +
diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index f681d02a830..71649e34fee 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -612,7 +612,7 @@

#{bundle['file.deleteFileDialog.failed.tip']}

-
diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index 6238ca5a091..f993d09394b 100755 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -103,7 +103,7 @@
- +
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index a1b95d5e4bf..40dc7a61fc5 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -111,9 +111,10 @@ + and !widgetWrapper.widgetView + and !DatasetPage.workingVersion.hasPackageFile }"> @@ -364,7 +365,7 @@ -
+
@@ -378,7 +379,7 @@
-
+
diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 524f6fdf6c5..609c998fb3f 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -104,7 +104,7 @@ - +
- +
@@ -222,7 +222,7 @@
- +
diff --git a/src/main/webapp/themeAndWidgetsFragment.xhtml b/src/main/webapp/themeAndWidgetsFragment.xhtml index 315ae30376e..f188cdc4514 100644 --- a/src/main/webapp/themeAndWidgetsFragment.xhtml +++ b/src/main/webapp/themeAndWidgetsFragment.xhtml @@ -7,7 +7,7 @@ - +
@@ -178,7 +178,7 @@
- +
From b5f81e3208460b1964cfd4419e682a866f64633e Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 17 Jul 2018 15:00:30 -0400 Subject: [PATCH 006/137] #4610 fix bundle conflict --- src/main/java/Bundle.properties | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 09b45dc0b66..3740f9a3f09 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1310,13 +1310,8 @@ dataset.privateurl.noPermToCreate=To create a Private URL you must have the foll file.count={0} {0, choice, 0#Files|1#File|2#Files} file.count.selected={0} {0, choice, 0#Files Selected|1#File Selected|2#Files Selected} file.selectToAddBtn=Select Files to Add -<<<<<<< HEAD -file.selectToAdd.tipLimit=File upload limit is {0} bytes per file. -file.selectToAdd.tipMoreInformation=Select or drag and drop files from your desktop, directly into the upload widget. -======= file.selectToAdd.tipLimit=File upload limit is {0} per file. file.selectToAdd.tipMoreInformation=For more information about supported file formats, please refer to the User Guide. ->>>>>>> develop file.selectToAdd.dragdropMsg=Drag and drop files here. file.createUploadDisabled=Once you have saved your dataset, you can upload your data using the "Upload Files" button on the dataset page. For more information about supported file formats, please refer to the User Guide. file.fromDropbox=Upload from Dropbox From e4a6eede17bfbe89e01e52ec832f6425cf52fac6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 17 Jul 2018 17:00:53 -0400 Subject: [PATCH 007/137] #4610 backing out autoupdate changes --- src/main/webapp/dataverse.xhtml | 4 ++-- src/main/webapp/metadataFragment.xhtml | 8 ++++---- src/main/webapp/themeAndWidgetsFragment.xhtml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 18d37032776..d501099b561 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -131,7 +131,7 @@
- +
- +
diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 609c998fb3f..524f6fdf6c5 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -104,7 +104,7 @@ - +
- +
@@ -222,7 +222,7 @@
- +
diff --git a/src/main/webapp/themeAndWidgetsFragment.xhtml b/src/main/webapp/themeAndWidgetsFragment.xhtml index f188cdc4514..315ae30376e 100644 --- a/src/main/webapp/themeAndWidgetsFragment.xhtml +++ b/src/main/webapp/themeAndWidgetsFragment.xhtml @@ -7,7 +7,7 @@ - +
@@ -178,7 +178,7 @@
- +
From 3ba00366b83cbd5acc58c8b3160c4482e88a8e54 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 18 Jul 2018 10:29:21 -0400 Subject: [PATCH 008/137] rename "NATIVE" to "native/http" and update javadoc #4610 --- .../harvard/iq/dataverse/util/SystemConfig.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 5de06559a27..c6dede2e2d8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -826,15 +826,21 @@ public int getPVNumberOfConsecutiveDigitsAllowed() { * * - TransferProtocols * - * There is a good chance these will be consolidated in the future. The word - * "NATIVE" is a bit of placeholder term to mean how Dataverse has - * traditionally handled files, which tends to involve users uploading and - * downloading files using a browser or APIs. + * There is a good chance these will be consolidated in the future. */ public enum FileUploadMethods { + /** + * DCM stands for Data Capture Module. Right now it supports upload over + * rsync+ssh but DCM may support additional methods in the future. + */ RSYNC("dcm/rsync+ssh"), - NATIVE("NATIVE"); + // TODO: Add "native/dropbox"? + /** + * Traditional Dataverse file handling, which tends to involve users + * uploading and downloading files using a browser or APIs. + */ + NATIVE("native/http"); private final String text; From 2fe2e545deede0e972ce4aed0a140cc4a12427d1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 19 Jul 2018 12:08:22 -0400 Subject: [PATCH 009/137] #4610 support use of multiple upload methods --- .../harvard/iq/dataverse/SettingsWrapper.java | 8 +++++ .../iq/dataverse/TemplateServiceBean.java | 19 ++++------- .../iq/dataverse/util/SystemConfig.java | 32 ++++++++++++++++--- src/main/webapp/editFilesFragment.xhtml | 4 +-- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index 2559cd6bc8b..19177d81216 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -150,6 +150,14 @@ public boolean isRsyncDownload() { return systemConfig.isRsyncDownload(); } + public boolean isHTTPUpload(){ + return systemConfig.isHTTPUpload(); + } + + public boolean isDropBoxUpload(){ + return systemConfig.isDropBoxUpload(); + } + public boolean isDataFilePIDSequentialDependent(){ return systemConfig.isDataFilePIDSequentialDependent(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/TemplateServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/TemplateServiceBean.java index 66c2d35fe61..dcbb65443f6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/TemplateServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/TemplateServiceBean.java @@ -55,17 +55,12 @@ public List findDataversesByDefaultTemplateId(Long defaultTemplateId) } public void incrementUsageCount(Long templateId) { - // Use object instead of Native Queries to update count - //also catch/log exceptions so that create dataset will succeed - //Issue 4847 SEK 7/16/2018 - try { - Template toUpdate = em.find(Template.class, templateId); - Long usage = toUpdate.getUsageCount(); - usage++; - toUpdate.setUsageCount(usage); - em.merge(toUpdate); - } catch (Exception e) { - logger.log(Level.INFO, "Increment usage count failed on template id {0}. ", templateId); - } + + Template toUpdate = em.find(Template.class, templateId); + Long usage = toUpdate.getUsageCount(); + usage++; + toUpdate.setUsageCount(usage); + em.merge(toUpdate); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index c6dede2e2d8..28a7835595e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -835,12 +835,17 @@ public enum FileUploadMethods { * rsync+ssh but DCM may support additional methods in the future. */ RSYNC("dcm/rsync+ssh"), - // TODO: Add "native/dropbox"? /** * Traditional Dataverse file handling, which tends to involve users * uploading and downloading files using a browser or APIs. */ - NATIVE("native/http"); + NATIVE("native/http"), + /** + * Traditional Dataverse file handling, which tends to involve users + * uploading and downloading files using a browser or APIs. + */ + + DROPBOX("native/dropbox"); private final String text; @@ -967,14 +972,31 @@ public boolean isPublicInstall(){ } public boolean isRsyncUpload(){ - String uploadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.UploadMethods); - return uploadMethods != null && uploadMethods.toLowerCase().equals(SystemConfig.FileUploadMethods.RSYNC.toString()); + return getUploadMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString()); + } + + public boolean isHTTPUpload(){ + return getUploadMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString()); } + public boolean isDropBoxUpload(){ + return getUploadMethodAvailable(SystemConfig.FileUploadMethods.DROPBOX.toString()); + } + + public boolean isRsyncDownload() { String downloadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.DownloadMethods); - return downloadMethods !=null && downloadMethods.toLowerCase().equals(SystemConfig.FileDownloadMethods.RSYNC.toString()); + return downloadMethods !=null && downloadMethods.toLowerCase().contains(SystemConfig.FileDownloadMethods.RSYNC.toString()); + } + + private Boolean getUploadMethodAvailable(String method){ + String uploadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.UploadMethods); + if (uploadMethods==null){ + return false; + } else { + return Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).contains(method); + } } public boolean isDataFilePIDSequentialDependent(){ diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 71649e34fee..174acf3dfc6 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -27,7 +27,7 @@ -
+
@@ -228,7 +228,7 @@
- +
From ca96daca966edf67e9cad7900adfdbbadbf08093 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 19 Jul 2018 14:07:07 -0400 Subject: [PATCH 010/137] #4610 bundle-ize file upload messages --- src/main/java/Bundle.properties | 5 +++++ src/main/webapp/editFilesFragment.xhtml | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 3740f9a3f09..711f6e2643d 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1316,6 +1316,11 @@ file.selectToAdd.dragdropMsg=Drag and drop files here. file.createUploadDisabled=Once you have saved your dataset, you can upload your data using the "Upload Files" button on the dataset page. For more information about supported file formats, please refer to the User Guide. file.fromDropbox=Upload from Dropbox file.fromDropbox.tip=Files can also be uploaded directly from Dropbox. +file.fromDropbox.help=Help message providing value and info about Dropbox to the user. +file.fromRsync=Upload with rsync+ssh via Data Capture Module (DCM) +file.fromRsync.help=Help message providing value and info about Rsync to the user. +file.fromHTTP=Upload with HTTP via your browser +file.fromHTTP.help=Help message providing value and info about HTTP upload to the user. file.replace.original=Original File file.editFiles=Edit Files file.editFilesSelected=Edit diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 174acf3dfc6..be106e966f5 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -34,7 +34,7 @@ @@ -45,7 +45,7 @@

- Help message proving value and info about HTTP upload to the user. + #{bundle['file.fromHTTP.help']}

@@ -236,7 +236,7 @@

@@ -244,7 +244,7 @@

- Help message proving value and info about Dropbox to the user. + #{bundle['file.fromDropbox.help']}

@@ -268,7 +268,7 @@ @@ -276,7 +276,7 @@

- Help message proving value and info about rsync to the user. + #{bundle['file.fromRsync.help']}

Warning message about how this option is not available because of uploaded files to this dataset. From 939ad70d80b61a9abd339cfd8e2d32ebc3c240eb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Jul 2018 14:31:57 -0400 Subject: [PATCH 011/137] ensure "native" upload is enabled for new install or upgrade #4610 --- scripts/api/setup-all.sh | 1 + scripts/database/upgrades/upgrade_v4.9.1_to_v4.9.2.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 scripts/database/upgrades/upgrade_v4.9.1_to_v4.9.2.sql diff --git a/scripts/api/setup-all.sh b/scripts/api/setup-all.sh index 03cb2252f2c..b76742ab555 100755 --- a/scripts/api/setup-all.sh +++ b/scripts/api/setup-all.sh @@ -54,6 +54,7 @@ curl -X PUT -d "FK2/" "$SERVER/admin/settings/:Shoulder" curl -X PUT -d EZID "$SERVER/admin/settings/:DoiProvider" curl -X PUT -d burrito $SERVER/admin/settings/BuiltinUsers.KEY curl -X PUT -d localhost-only $SERVER/admin/settings/:BlockedApiPolicy +curl -X PUT -d 'native/http' $SERVER/admin/settings/:UploadMethods echo echo "Setting up the admin user (and as superuser)" diff --git a/scripts/database/upgrades/upgrade_v4.9.1_to_v4.9.2.sql b/scripts/database/upgrades/upgrade_v4.9.1_to_v4.9.2.sql new file mode 100644 index 00000000000..a7a7ddeb814 --- /dev/null +++ b/scripts/database/upgrades/upgrade_v4.9.1_to_v4.9.2.sql @@ -0,0 +1 @@ +INSERT INTO setting(name, content) VALUES (':UploadMethods', 'native/http'); From 0a6e2836630f65feebc41789442264b1dc41e813 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Jul 2018 15:07:19 -0400 Subject: [PATCH 012/137] if HTTP upload is disabled via GUI, disable via API also #4610 --- src/main/java/Bundle.properties | 1 + src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 +++++- src/main/java/edu/harvard/iq/dataverse/api/Files.java | 5 ++++- .../dataverse/api/datadeposit/MediaResourceManagerImpl.java | 4 ++++ .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 1 + 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 711f6e2643d..0cdf5930eb3 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1321,6 +1321,7 @@ file.fromRsync=Upload with rsync+ssh via Data Capture Module (DCM) file.fromRsync.help=Help message providing value and info about Rsync to the user. file.fromHTTP=Upload with HTTP via your browser file.fromHTTP.help=Help message providing value and info about HTTP upload to the user. +file.api.httpDisabled=File upload via HTTP is not available for this installation of Dataverse. file.replace.original=Original File file.editFiles=Edit Files file.editFilesSelected=Edit diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index fd547b800dd..3e70c78cc2a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -72,6 +72,7 @@ import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.EjbUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonParseException; @@ -1127,7 +1128,10 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, @FormDataParam("file") final FormDataBodyPart formDataBodyPart ){ - + if (!systemConfig.isHTTPUpload()) { + return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled")); + } + // ------------------------------------- // (1) Get the user from the API key // ------------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 190c5e49c52..2cfe2c679e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -151,7 +151,10 @@ public Response replaceFileInDataset( @FormDataParam("file") FormDataContentDisposition contentDispositionHeader, @FormDataParam("file") final FormDataBodyPart formDataBodyPart ){ - + + if (!systemConfig.isHTTPUpload()) { + return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled")); + } // ------------------------------------- // (1) Get the user from the API key // ------------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java index e28a21a71d2..25f8d1eb1c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java @@ -16,6 +16,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.ByteArrayInputStream; @@ -200,6 +201,9 @@ public DepositReceipt addResource(String uri, Deposit deposit, AuthCredentials a } DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials authCredentials, SwordConfiguration swordConfiguration, boolean shouldReplace) throws SwordError, SwordAuthException, SwordServerException { + if (!systemConfig.isHTTPUpload()) { + throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, BundleUtil.getStringFromBundle("file.api.httpDisabled")); + } AuthenticatedUser user = swordAuth.auth(authCredentials); DataverseRequest dvReq = new DataverseRequest(user, httpRequest); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 28a7835595e..6bc67191d51 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -975,6 +975,7 @@ public boolean isRsyncUpload(){ return getUploadMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString()); } + // Controls if HTTP upload is enabled for both GUI and API. public boolean isHTTPUpload(){ return getUploadMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString()); } From 4a7fcf52711127593e593346f451a0a6be822145 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Jul 2018 15:20:32 -0400 Subject: [PATCH 013/137] remove help messages that aren't providing value #4610 --- src/main/java/Bundle.properties | 3 --- src/main/webapp/editFilesFragment.xhtml | 11 ----------- 2 files changed, 14 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 0cdf5930eb3..cd8af544785 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1316,11 +1316,8 @@ file.selectToAdd.dragdropMsg=Drag and drop files here. file.createUploadDisabled=Once you have saved your dataset, you can upload your data using the "Upload Files" button on the dataset page. For more information about supported file formats, please refer to the User Guide. file.fromDropbox=Upload from Dropbox file.fromDropbox.tip=Files can also be uploaded directly from Dropbox. -file.fromDropbox.help=Help message providing value and info about Dropbox to the user. file.fromRsync=Upload with rsync+ssh via Data Capture Module (DCM) -file.fromRsync.help=Help message providing value and info about Rsync to the user. file.fromHTTP=Upload with HTTP via your browser -file.fromHTTP.help=Help message providing value and info about HTTP upload to the user. file.api.httpDisabled=File upload via HTTP is not available for this installation of Dataverse. file.replace.original=Original File file.editFiles=Edit Files diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index be106e966f5..639faf11c6d 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -44,10 +44,6 @@

-

- #{bundle['file.fromHTTP.help']} -

-

-

- #{bundle['file.fromDropbox.help']} -

-
@@ -275,9 +267,6 @@
-

- #{bundle['file.fromRsync.help']} -

Warning message about how this option is not available because of uploaded files to this dataset.

From 1aab2026a9b6a83298b8d5de5095338cf6b411a2 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Jul 2018 15:36:59 -0400 Subject: [PATCH 014/137] update guides to reflect dual mode #4610 --- .../source/developers/big-data-support.rst | 2 +- .../source/installation/config.rst | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 501c53a3bb5..6cb413ae738 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -21,7 +21,7 @@ Installation instructions can be found at https://github.com/sbgrid/data-capture Once you have installed a DCM, you will need to configure two database settings on the Dataverse side. These settings are documented in the :doc:`/installation/config` section of the Installation Guide: - ``:DataCaptureModuleUrl`` should be set to the URL of a DCM you installed. -- ``:UploadMethods`` should be set to ``dcm/rsync+ssh``. +- ``:UploadMethods`` should include ``dcm/rsync+ssh``. This will allow your Dataverse installation to communicate with your DCM, so that Dataverse can download rsync scripts for your users. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index b8d4678ac0f..d772e3d1cc9 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -26,6 +26,8 @@ The :doc:`/api/native-api` contains a useful but potentially dangerous API endpo By default, all APIs can be operated on remotely and a number of endpoints do not require authentication. https://github.com/IQSS/dataverse/issues/1886 was opened to explore changing these defaults, but until then it is very important to block both the "admin" endpoint (and at least consider blocking ``builtin-users``). For details please see also the section on ``:BlockedApiPolicy`` below. +It's also possible to prevent file uploads via API by adjusting the ``:UploadMethods`` database setting. + Forcing HTTPS +++++++++++++ @@ -520,7 +522,7 @@ Configuration for :doc:`r-rapache-tworavens`. dataverse.dropbox.key +++++++++++++++++++++ -Dropbox provides a Chooser app, which is a Javascript component that allows you to upload files to Dataverse from Dropbox. It is an optional configuration setting, which requires you to pass it an app key. For more information on setting up your Chooser app, visit https://www.dropbox.com/developers/chooser. +Dropbox provides a Chooser app, which is a Javascript component that allows you to upload files to Dataverse from Dropbox. It is an optional configuration setting, which requires you to pass it an app key and configure the ``:UploadMethods`` database setting. For more information on setting up your Chooser app, visit https://www.dropbox.com/developers/chooser. ``./asadmin create-jvm-options "-Ddataverse.dropbox.key={{YOUR_APP_KEY}}"`` @@ -1250,9 +1252,19 @@ The URL for your Repository Storage Abstraction Layer (RSAL) installation. This :UploadMethods ++++++++++++++ -This setting is experimental and to be used with the Data Capture Module (DCM). For now, if you set the upload methods to ``dcm/rsync+ssh`` it will allow your users to download rsync scripts from the DCM. +This setting controls which upload methods are available to users of your installation of Dataverse. The following upload methods are available: + +- ``native/http``: Corresponds to "Upload with HTTP via your browser" and APIs that use HTTP (SWORD and native). +- ``native/dropbox``: Corresponds to "Upload with Dropbox". Note that the JVM option ``dataverse.dropbox.key`` is also required. +- ``dcm/rsync+ssh``: Corresponds to "Upload with rsync+ssh via Data Capture Module (DCM)". A lot of setup is required, as explained in the :doc:`/developers/big-data-support` section of the Dev Guide. + +Out of the box only ``native/http`` is enabled and will work without further configuration. To add multiple upload method, separate them using a comma like this: + +``curl -X PUT -d 'native/http,native/dropbox,dcm/rsync+ssh' http://localhost:8080/api/admin/settings/:UploadMethods`` + +You'll always want at least one upload method, so the easiest way to remove one of them is to simply ``PUT`` just the one you want, like this: -``curl -X PUT -d 'dcm/rsync+ssh' http://localhost:8080/api/admin/settings/:UploadMethods`` +``curl -X PUT -d 'native/http' http://localhost:8080/api/admin/settings/:UploadMethods`` :DownloadMethods ++++++++++++++++ From 67d8296ea5bb91cc38d0517e9bdd7067796116fd Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 23 Jul 2018 15:13:44 -0400 Subject: [PATCH 015/137] #4610 Fix Lock issue and tests --- .../java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 3 +++ .../datacapturemodule/DataCaptureModuleUtil.java | 9 +++++---- .../datacapturemodule/DataCaptureModuleUtilTest.java | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 1dc29cd8ce7..ff618ec91ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -557,6 +557,9 @@ public String init() { } saveEnabled = true; + if (mode == FileEditMode.UPLOAD && workingVersion.getFileMetadatas().isEmpty() && settingsWrapper.isRsyncUpload()) { + setUpRsync(); + } if (mode == FileEditMode.UPLOAD) { JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.uploadFiles.label"), BundleUtil.getStringFromBundle("dataset.message.uploadFiles.message", Arrays.asList(systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion()))); diff --git a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java index d5a883a15a7..bc8994abd41 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtil.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.util.SystemConfig; +import java.util.Arrays; import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonObject; @@ -15,11 +16,11 @@ public class DataCaptureModuleUtil { private static final Logger logger = Logger.getLogger(DataCaptureModuleUtil.class.getCanonicalName()); public static boolean rsyncSupportEnabled(String uploadMethodsSettings) { - logger.fine("uploadMethodsSettings: " + uploadMethodsSettings); - if (uploadMethodsSettings != null && SystemConfig.FileUploadMethods.RSYNC.toString().equals(uploadMethodsSettings)) { - return true; - } else { + logger.fine("uploadMethodsSettings: " + uploadMethodsSettings);; + if (uploadMethodsSettings==null){ return false; + } else { + return Arrays.asList(uploadMethodsSettings.toLowerCase().split("\\s*,\\s*")).contains(SystemConfig.FileUploadMethods.RSYNC.toString()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java index 08b020ec2ca..75252cc3ac2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/datacapturemodule/DataCaptureModuleUtilTest.java @@ -26,9 +26,9 @@ public void testRsyncSupportEnabled() { System.out.println("rsyncSupportEnabled"); assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled(null)); assertEquals(true, DataCaptureModuleUtil.rsyncSupportEnabled("dcm/rsync+ssh")); - // We haven't finalized what the separator will be yet. + // Comma sepratated lists of upload methods are supported. assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE:dcm/rsync+ssh")); - assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE,dcm/rsync+ssh")); + assertEquals(true, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE,dcm/rsync+ssh")); assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("NATIVE")); assertEquals(false, DataCaptureModuleUtil.rsyncSupportEnabled("junk")); } From 2add42be64f5280d727f0f9552d7193852bf58d6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 23 Jul 2018 16:38:24 -0400 Subject: [PATCH 016/137] #4610 Added download render logic to file page --- src/main/webapp/file.xhtml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 5c6eda68571..478ad8c2955 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -171,7 +171,7 @@
+ and !settingsWrapper.rsyncDownload and !FilePage.fileMetadata.dataFile.filePackage}"> @@ -312,7 +312,7 @@ + rendered="#{settingsWrapper.rsyncDownload and FilePage.fileMetadata.dataFile.filePackage }"> @@ -374,7 +374,7 @@
-
+
From 4840c0840c19f878efc2ba747affdef2ff750128 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 23 Jul 2018 16:41:05 -0400 Subject: [PATCH 017/137] #4610 display download button on non-package files --- src/main/webapp/file.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 478ad8c2955..22f7cc35a4d 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -171,7 +171,7 @@
+ and !FilePage.fileMetadata.dataFile.filePackage}"> From 1b930bf3b9c6b8a06e786e730c8fc9b4e2e9fc0c Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 24 Jul 2018 13:44:37 -0400 Subject: [PATCH 018/137] #4610 fix error handling for addFile apis --- src/main/java/Bundle.properties | 1 + .../edu/harvard/iq/dataverse/api/Datasets.java | 18 +++++++++++------- .../edu/harvard/iq/dataverse/api/Files.java | 4 ++++ .../datadeposit/MediaResourceManagerImpl.java | 10 ++++++---- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index cd8af544785..ef7d5ac37f6 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1319,6 +1319,7 @@ file.fromDropbox.tip=Files can also be uploaded directly from Dropbox. file.fromRsync=Upload with rsync+ssh via Data Capture Module (DCM) file.fromHTTP=Upload with HTTP via your browser file.api.httpDisabled=File upload via HTTP is not available for this installation of Dataverse. +file.api.alreadyHasPackageFile=File upload via HTTP disabled since this dataset already contains a package file. file.replace.original=Original File file.editFiles=Edit Files file.editFilesSelected=Edit diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 3e70c78cc2a..32301a8717a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1143,13 +1143,6 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, ResourceBundle.getBundle("Bundle").getString("file.addreplace.error.auth") ); } - //--------------------------------------- - // (1A) Make sure that the upload type is not rsync - // ------------------------------------- - - if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsSvc.getValueForKey(SettingsServiceBean.Key.UploadMethods))) { - return error(Response.Status.METHOD_NOT_ALLOWED, SettingsServiceBean.Key.UploadMethods + " contains " + SystemConfig.FileUploadMethods.RSYNC + ". Please use rsync file upload."); - } // ------------------------------------- @@ -1165,6 +1158,17 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, return wr.getResponse(); } + //------------------------------------ + // (2a) Make sure dataset does not have package file + // + // -------------------------------------- + + if (dataset.getEditVersion().isHasPackageFile()){ + return error(Response.Status.FORBIDDEN, + ResourceBundle.getBundle("Bundle").getString("file.api.alreadyHasPackageFile") + ); + } + // ------------------------------------- // (3) Get the file name and content type diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 2cfe2c679e7..40420f56a2d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -225,6 +225,10 @@ public Response replaceFileInDataset( try { DataFile dataFile = findDataFileOrDie(fileIdOrPersistentId); fileToReplaceId = dataFile.getId(); + + if (dataFile.isFilePackage()) { + return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.alreadyHasPackageFile")); + } } catch (WrappedResponse ex) { String error = BundleUtil.getStringFromBundle("file.addreplace.error.existing_file_to_replace_not_found_by_id", Arrays.asList(fileIdOrPersistentId)); // TODO: Some day, return ex.getResponse() instead. Also run FilesIT and updated expected status code and message. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java index 25f8d1eb1c0..1ac4d553a81 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java @@ -221,12 +221,14 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au } //--------------------------------------- - // Make sure that the upload type is not rsync + // Make sure that the upload type is not rsync - handled above for dual mode // ------------------------------------- - if (DataCaptureModuleUtil.rsyncSupportEnabled(settingsSvc.getValueForKey(SettingsServiceBean.Key.UploadMethods))) { - throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, SettingsServiceBean.Key.UploadMethods + " contains " + SystemConfig.FileUploadMethods.RSYNC + ". Please use rsync file upload."); - } + if (dataset.getEditVersion().isHasPackageFile()) { + throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, BundleUtil.getStringFromBundle("file.api.alreadyHasPackageFile")); + } + + // Right now we are only supporting UriRegistry.PACKAGE_SIMPLE_ZIP but // in the future maybe we'll support other formats? Rdata files? Stata files? /** From 8d2c6406972863493444f53e20e63cae4abde500 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 24 Jul 2018 16:39:36 -0400 Subject: [PATCH 019/137] #4610 fix failing test --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 04c5a140f2b..d9a9fd7d6cb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -59,7 +59,8 @@ public static void setUpClass() { Response removeExcludeEmail = UtilIT.deleteSetting(SettingsServiceBean.Key.ExcludeEmailFromExport); removeExcludeEmail.then().assertThat() .statusCode(200); - + /* With Dual mode, we can no longer mess with upload methods since native is now required for anything to work + Response removeDcmUrl = UtilIT.deleteSetting(SettingsServiceBean.Key.DataCaptureModuleUrl); removeDcmUrl.then().assertThat() .statusCode(200); @@ -67,6 +68,7 @@ public static void setUpClass() { Response removeUploadMethods = UtilIT.deleteSetting(SettingsServiceBean.Key.UploadMethods); removeUploadMethods.then().assertThat() .statusCode(200); + */ } @AfterClass @@ -79,7 +81,7 @@ public static void afterClass() { Response removeExcludeEmail = UtilIT.deleteSetting(SettingsServiceBean.Key.ExcludeEmailFromExport); removeExcludeEmail.then().assertThat() .statusCode(200); - + /* See above Response removeDcmUrl = UtilIT.deleteSetting(SettingsServiceBean.Key.DataCaptureModuleUrl); removeDcmUrl.then().assertThat() .statusCode(200); @@ -87,6 +89,7 @@ public static void afterClass() { Response removeUploadMethods = UtilIT.deleteSetting(SettingsServiceBean.Key.UploadMethods); removeUploadMethods.then().assertThat() .statusCode(200); + */ } @Test From 1debd4edb40415d08056006c9784f6d39ef53ad2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 25 Jul 2018 09:21:33 -0400 Subject: [PATCH 020/137] #4610 limit page refresh on file upload --- src/main/webapp/editFilesFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 639faf11c6d..bd8cc263540 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -209,7 +209,7 @@ disabled="#{!(datasetPage || EditDatafilesPage.showFileUploadComponent())}" fileUploadListener="#{EditDatafilesPage.handleFileUpload}" process="filesTable" - update="@form, :datasetForm, @([id$=filesButtons])" + update=":datasetForm:filesTable, @([id$=filesButtons])" label="#{bundle['file.selectToAddBtn']}" oncomplete="javascript:dataset_fileupload_rebind();uploadFinished(PF('fileUploadWidget'));uploadFinished();" onstart="javascript:uploadWidgetDropRemoveMsg();uploadStarted();" From 68f95b0c2a51af4e83e843a3f70dfc7c4a236b74 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 25 Jul 2018 09:46:42 -0400 Subject: [PATCH 021/137] #4610 fix deprecated autoUpdate tag --- src/main/webapp/dataverse.xhtml | 3 ++- src/main/webapp/metadataFragment.xhtml | 6 ++++-- src/main/webapp/themeAndWidgetsFragment.xhtml | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index d501099b561..3b86148fa3e 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -131,7 +131,8 @@
- + + - + +
- + +
diff --git a/src/main/webapp/themeAndWidgetsFragment.xhtml b/src/main/webapp/themeAndWidgetsFragment.xhtml index 315ae30376e..b1931ae7e5a 100644 --- a/src/main/webapp/themeAndWidgetsFragment.xhtml +++ b/src/main/webapp/themeAndWidgetsFragment.xhtml @@ -7,7 +7,8 @@ - + +
From b35fb5fe9899c659f493447df9662f2638a78739 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 25 Jul 2018 11:07:00 -0400 Subject: [PATCH 022/137] #4610 make dropbox subordinate to native --- .../edu/harvard/iq/dataverse/SettingsWrapper.java | 4 ---- .../harvard/iq/dataverse/util/SystemConfig.java | 14 ++------------ src/main/webapp/editFilesFragment.xhtml | 12 +++--------- 3 files changed, 5 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index 19177d81216..56347eb5430 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -154,10 +154,6 @@ public boolean isHTTPUpload(){ return systemConfig.isHTTPUpload(); } - public boolean isDropBoxUpload(){ - return systemConfig.isDropBoxUpload(); - } - public boolean isDataFilePIDSequentialDependent(){ return systemConfig.isDataFilePIDSequentialDependent(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 6bc67191d51..5e737f16943 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -839,13 +839,8 @@ public enum FileUploadMethods { * Traditional Dataverse file handling, which tends to involve users * uploading and downloading files using a browser or APIs. */ - NATIVE("native/http"), - /** - * Traditional Dataverse file handling, which tends to involve users - * uploading and downloading files using a browser or APIs. - */ - - DROPBOX("native/dropbox"); + NATIVE("native/http"); + private final String text; @@ -980,11 +975,6 @@ public boolean isHTTPUpload(){ return getUploadMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString()); } - public boolean isDropBoxUpload(){ - return getUploadMethodAvailable(SystemConfig.FileUploadMethods.DROPBOX.toString()); - } - - public boolean isRsyncDownload() { String downloadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.DownloadMethods); diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index bd8cc263540..008b0997c4e 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -224,8 +224,8 @@
- -
+ +
@@ -314,15 +314,10 @@ - -
- -
- - +
@@ -332,7 +327,6 @@

#{bundle['file.createUploadDisabled']}

-
From 3b033d683804519bb8730cfbccd90631da86a44e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Jul 2018 11:14:23 -0400 Subject: [PATCH 023/137] update docs to indicate that Dropbox only is not a valid mode #4610 --- doc/sphinx-guides/source/installation/config.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index d772e3d1cc9..f4636c5f891 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1255,12 +1255,11 @@ The URL for your Repository Storage Abstraction Layer (RSAL) installation. This This setting controls which upload methods are available to users of your installation of Dataverse. The following upload methods are available: - ``native/http``: Corresponds to "Upload with HTTP via your browser" and APIs that use HTTP (SWORD and native). -- ``native/dropbox``: Corresponds to "Upload with Dropbox". Note that the JVM option ``dataverse.dropbox.key`` is also required. - ``dcm/rsync+ssh``: Corresponds to "Upload with rsync+ssh via Data Capture Module (DCM)". A lot of setup is required, as explained in the :doc:`/developers/big-data-support` section of the Dev Guide. Out of the box only ``native/http`` is enabled and will work without further configuration. To add multiple upload method, separate them using a comma like this: -``curl -X PUT -d 'native/http,native/dropbox,dcm/rsync+ssh' http://localhost:8080/api/admin/settings/:UploadMethods`` +``curl -X PUT -d 'native/http,dcm/rsync+ssh' http://localhost:8080/api/admin/settings/:UploadMethods`` You'll always want at least one upload method, so the easiest way to remove one of them is to simply ``PUT`` just the one you want, like this: From 7452e66ae92b6d9f3af64ef1a49576221b87775e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Jul 2018 15:55:30 -0400 Subject: [PATCH 024/137] update createUploadDisabled text for rsync #4610 --- src/main/java/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index ef7d5ac37f6..6514ac72a47 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1313,7 +1313,7 @@ file.selectToAddBtn=Select Files to Add file.selectToAdd.tipLimit=File upload limit is {0} per file. file.selectToAdd.tipMoreInformation=For more information about supported file formats, please refer to the User Guide. file.selectToAdd.dragdropMsg=Drag and drop files here. -file.createUploadDisabled=Once you have saved your dataset, you can upload your data using the "Upload Files" button on the dataset page. For more information about supported file formats, please refer to the User Guide. +file.createUploadDisabled=rsync allows you to run a script that transfers your files into or out of Dataverse via SSH. It is useful for extremely large files, or packages containing a large number of files. Once you have saved this dataset, you can upload your data using rsync via the "Upload Files" button on the dataset page. file.fromDropbox=Upload from Dropbox file.fromDropbox.tip=Files can also be uploaded directly from Dropbox. file.fromRsync=Upload with rsync+ssh via Data Capture Module (DCM) From a867f5b7a8c1499cedf27ffb481b71b6d462e626 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Jul 2018 16:08:03 -0400 Subject: [PATCH 025/137] indent rsync help block to match native/http #4610 --- src/main/webapp/editFilesFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 008b0997c4e..060e00aa1c1 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -322,7 +322,7 @@
-
+

#{bundle['file.createUploadDisabled']}

From a818aeada10555b47f1cfc8997b21aef0f55ae72 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 26 Jul 2018 10:22:49 -0400 Subject: [PATCH 026/137] #4610 remove script lookup from Create DS command --- .../engine/command/impl/AbstractCreateDatasetCommand.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java index 9c521625b48..9ebc816a9cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java @@ -133,6 +133,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.index().indexDataset(theDataset, true); ctxt.solrIndex().indexPermissionsOnSelfAndChildren(theDataset.getId()); + /* if (DataCaptureModuleUtil.rsyncSupportEnabled(ctxt.settings().getValueForKey(SettingsServiceBean.Key.UploadMethods))) { logger.fine("Requesting rsync support."); try { @@ -142,8 +143,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { logger.log(Level.WARNING, "Problem getting rsync script: {0}", ex.getLocalizedMessage()); } logger.fine("Done with rsync request."); - } - + }*/ return theDataset; } From 4cc643c7a8e60d726628559f4e629fd09c899e6d Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 30 Jul 2018 17:08:41 -0400 Subject: [PATCH 027/137] Fixed rsync panel collapse toggle. Various code clean up. [ref #4610] --- src/main/java/Bundle.properties | 2 +- src/main/webapp/editFilesFragment.xhtml | 160 +++++++++++------------- 2 files changed, 74 insertions(+), 88 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 6514ac72a47..18dbd0e858f 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -1316,7 +1316,7 @@ file.selectToAdd.dragdropMsg=Drag and drop files here. file.createUploadDisabled=rsync allows you to run a script that transfers your files into or out of Dataverse via SSH. It is useful for extremely large files, or packages containing a large number of files. Once you have saved this dataset, you can upload your data using rsync via the "Upload Files" button on the dataset page. file.fromDropbox=Upload from Dropbox file.fromDropbox.tip=Files can also be uploaded directly from Dropbox. -file.fromRsync=Upload with rsync+ssh via Data Capture Module (DCM) +file.fromRsync=Upload with rsync+SSH via Data Capture Module (DCM) file.fromHTTP=Upload with HTTP via your browser file.api.httpDisabled=File upload via HTTP is not available for this installation of Dataverse. file.api.alreadyHasPackageFile=File upload via HTTP disabled since this dataset already contains a package file. diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 060e00aa1c1..c83aba6ed95 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -27,23 +27,20 @@ -
+
-