4659 oai default set

src/main/java/Bundle.properties

@@ -487,8 +487,10 @@ harvestserver.noSets.how.tip2=Once the service is enabled, you can define collec
 harvestserver.noSets.getStarted=To get started, enable the OAI server and click on the Add Set button. To learn more about Harvesting, visit the <a href="{0}/{1}/user/index.html#index" title="Harvesting - Dataverse User Guide" target="_blank">Harvesting</a> section of the User Guide.
 harvestserver.btn.add=Add Set
 harvestserver.tab.header.spec=OAI setSpec
+harvestserver.tab.col.spec.default=DEFAULT
 harvestserver.tab.header.description=Description
 harvestserver.tab.header.definition=Definition Query
+harvestserver.tab.col.definition.default=All Published Local Datasets
 harvestserver.tab.header.stats=Datasets
 harvestserver.tab.col.stats.empty=No records (empty set)
 harvestserver.tab.col.stats.results={0} {0, choice, 0#datasets|1#dataset|2#datasets} ({1} {1, choice, 0#records|1#record|2#records} exported, {2} marked as deleted)
@@ -506,6 +508,7 @@ harvestserver.newSetDialog.setspec=Name/OAI setSpec
 harvestserver.newSetDialog.setspec.tip=A unique name (OAI setSpec) identifying this set.
 harvestserver.newSetDialog.setspec.helptext=Consists of letters, digits, underscores (_) and dashes (-).
 harvestserver.editSetDialog.setspec.helptext=The name can not be changed once the set has been created.
+harvestserver.editSetDialog.setspec.helptext.default=this is the default, unnamed set
 harvestserver.newSetDialog.setspec.required=Name (OAI setSpec) cannot be empty!
 harvestserver.newSetDialog.setspec.invalid=Name (OAI setSpec) can contain only letters, digits, underscores (_) and dashes (-).
 harvestserver.newSetDialog.setspec.alreadyused=This set name (OAI setSpec) is already used.
@@ -515,6 +518,7 @@ harvestserver.newSetDialog.setspec.superUser.required=Only superusers may create
 harvestserver.newSetDialog.setdescription=Description
 harvestserver.newSetDialog.setdescription.tip=Provide a brief description for this OAI set.
 harvestserver.newSetDialog.setdescription.required=Set description cannot be empty!
+harvestserver.newSetDialog.setdescription.default=The default, "no name" set. The OAI server will serve the records from this set when no "setspec" argument is specified by the client.
 harvestserver.newSetDialog.setquery=Definition Query
 harvestserver.newSetDialog.setquery.tip=Search query that defines the content of the dataset.
 harvestserver.newSetDialog.setquery.helptext=Example query: authorName:king
@@ -1931,14 +1935,15 @@ harvest.delete.error=Selected harvesting client cannot be deleted; unknown excep
 harvest.create.error=Failed to create a new Harvesting Client configuration: no destination dataverse selected.
 harvest.createCommand.error=Harvesting client creation command failed
 harvest.create.fail=Harvesting client creation failed (reason unknown).
-harvest.update.success=Succesfully updated harvesting client
+harvest.update.success=Successfully updated harvesting client
 harvest.save.failure1=Failed to save harvesting client
 harvest.save.failure2=Failed to save harvesting client (reason unknown).
 
 #HarvestingSetsPage.java
 harvest.oaicreate.fail=Failed to create OAI set
+harvest.oaicreate.defaultset.fail=Failed to create the default OAI set
 harvest.oaiupdate.fail=Failed to update OAI set.
-harvest.oaiupdate.success=Succesfully updated OAI set &#34;{0}&#34;.
+harvest.oaiupdate.success=Successfully updated OAI set &#34;{0}&#34;.
 harvest.delete.fail=Failed to delete harvesting set; unknown exception: 
 harvest.reexport.fail=Sorry, could not start re-export on selected OAI set (unknown server error).
 harvest.search.failed=Search failed for the query provided. Message from the Dataverse search server:
@@ -2009,8 +2014,8 @@ permission.fileAccessRejected=File Access request by {0} was rejected.
 permission.roleNotAbleToBeAssigned=The role was not able to be assigned.
 
 #ManageGroupsPage.java
-dataverse.manageGroups.create.success=Succesfully created group {0}. Refresh to update your page.
-dataverse.manageGroups.save.success=Succesfully saved group {0}
+dataverse.manageGroups.create.success=Successfully created group {0}. Refresh to update your page.
+dataverse.manageGroups.save.success=Successfully saved group {0}
 dataverse.manageGroups.delete=The group has been deleted.
 dataverse.manageGroups.nodelete=The explicit group cannot be deleted.
 dataverse.manageGroups.create.fail=Group Creation failed.

src/main/java/edu/harvard/iq/dataverse/HarvestingSetsPage.java

@@ -130,12 +130,12 @@ public String init() {
             return navigationWrapper.notAuthorized();
         }
 
-
         configuredHarvestingSets = oaiSetService.findAll();
         pageMode = PageMode.VIEW;
 
         if (isHarvestingServerEnabled()) {
             oaiServerStatusRadio = oaiServerStatusRadioEnabled;
+            checkIfDefaultSetExists();
         } else {
             oaiServerStatusRadio = oaiServerStatusRadioDisabled;
         }
@@ -144,6 +144,13 @@ public String init() {
         return null; 
     }
 
+    private void checkIfDefaultSetExists() {
+        OAISet defaultSet = oaiSetService.findDefaultSet();
+        if (defaultSet == null) {
+            createDefaultSet();
+        }
+    }
+
     public List<OAISet> getConfiguredOAISets() {
         return configuredHarvestingSets; 
     }
@@ -152,6 +159,11 @@ public void setConfiguredOAISets(List<OAISet> oaiSets) {
         configuredHarvestingSets = oaiSets; 
     }
 
+    public boolean isHasNamedOAISets() {
+        List<OAISet> namedSets = oaiSetService.findAllNamedSets();
+        return namedSets != null && namedSets.size() > 0;
+    }
+
     public boolean isHarvestingServerEnabled() {
         return systemConfig.isOAIServerEnabled();
     }
@@ -162,6 +174,7 @@ public void toggleHarvestingServerStatus() {
         } else {
             systemConfig.enableOAIServer();
             JsfHelper.addSuccessMessage(JH.localize("harvestserver.service.enable.success"));
+            checkIfDefaultSetExists();
         }
     }
 
@@ -218,6 +231,10 @@ public void editSet(OAISet oaiSet) {
         this.setQueryValidated = false;
         this.setQueryResult = -1;
 
+        if (oaiSet.isDefaultSet()) {
+            setSetQueryValidated(true);
+        }
+
         setSelectedSet(oaiSet);
     }
 
@@ -279,7 +296,7 @@ public void saveSet(ActionEvent ae) {
         try {
             oaiSetService.save(oaiSet);
             configuredHarvestingSets = oaiSetService.findAll(); 
-            JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("harvest.oaiupdate.success",   Arrays.asList(oaiSet.getSpec())));
+            JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("harvest.oaiupdate.success",   Arrays.asList(oaiSet.isDefaultSet() ? "default" : oaiSet.getSpec())));
             success = true;
 
         } catch (Exception ex) {
@@ -320,6 +337,42 @@ public void deleteSet() {
 
     }
 
+    private void createDefaultSet() {
+
+        OAISet newOaiSet = new OAISet();
+
+
+        newOaiSet.setSpec("");
+        newOaiSet.setName("");
+        // The default description of the default set. The admin will be  
+        // able to modify it later, if necessary.
+        newOaiSet.setDescription(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setdescription.default"));
+        newOaiSet.setDefinition("");
+
+        boolean success = false;
+
+        try {
+            oaiSetService.save(newOaiSet);
+            configuredHarvestingSets = oaiSetService.findAll();  
+            success = true;
+
+        } catch (Exception ex) {
+            // should be a warning perhaps??
+            JH.addMessage(FacesMessage.SEVERITY_FATAL, BundleUtil.getStringFromBundle("harvest.oaicreate.defaultset.fail"));
+            logger.log(Level.SEVERE, "Failed to create the Default OAI set" + ex.getMessage(), ex);
+        }
+
+        if (success) {
+            OAISet savedSet = oaiSetService.findBySpec(getNewSetSpec());
+            if (savedSet != null) {
+                runSetExport(savedSet);
+                configuredHarvestingSets = oaiSetService.findAll(); 
+            }
+        }
+
+        setPageMode(HarvestingSetsPage.PageMode.VIEW);        
+    }
+
     public boolean isSetSpecValidated() {
         return this.setSpecValidated;
     }
@@ -382,6 +435,10 @@ public boolean isSessionUserAuthenticated() {
     }
 
     public int getSetInfoNumOfDatasets(OAISet oaiSet) {
+        if (oaiSet.isDefaultSet()) {
+            return getSetInfoNumOfExported(oaiSet);
+        }
+
         String query = oaiSet.getDefinition();
 
         try {

src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java

@@ -59,47 +59,27 @@ public class OAIRecordServiceBean implements java.io.Serializable {
     EntityManager em;   
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean");
-
-    /*
-    public void updateOaiRecords() {
-        Date updateTime = new Date();
-        List<OAISet> sets = oaiSetService.findAll();
-
-        for (OAISet oaiSet : sets) {
-            List<Long> studyIds = indexService.query(oaiSet.getDefinition());
-            studyIds = studyService.getVisibleStudies(studyIds, null);
-            studyIds = studyService.getViewableStudies(studyIds);
-            updateOaiRecords( oaiSet.getSpec(), studyIds, updateTime );
-        }
-
-        // also do noset membet
-        List<Long> studyIds = studyService.getAllNonHarvestedStudyIds();
-        studyIds = studyService.getVisibleStudies(studyIds, null);
-        studyIds = studyService.getViewableStudies(studyIds);        
-        updateOaiRecords( null, studyIds, updateTime );
-
-    }   */ 
 
     public void updateOaiRecords(String setName, List<Long> datasetIds, Date updateTime, boolean doExport) {
         updateOaiRecords(setName, datasetIds, updateTime, doExport, logger);
     }
 
     public void updateOaiRecords(String setName, List<Long> datasetIds, Date updateTime, boolean doExport, Logger setUpdateLogger) {
-        
+
         // create Map of OaiRecords
-        List<OAIRecord> oaiRecords = findOaiRecordsBySetName( setName );
-        Map<String,OAIRecord> recordMap = new HashMap<>();
+        List<OAIRecord> oaiRecords = findOaiRecordsBySetName(setName);
+        Map<String, OAIRecord> recordMap = new HashMap<>();
         if (oaiRecords != null) {
             for (OAIRecord record : oaiRecords) {
                 // look for duplicates here? delete?
                 recordMap.put(record.getGlobalId(), record);
             }
         } else {
             setUpdateLogger.fine("Null returned - no records found.");
-        } 
+        }
 
         if (!recordMap.isEmpty()) {
-            setUpdateLogger.fine("Found "+recordMap.size()+" existing records");
+            setUpdateLogger.fine("Found " + recordMap.size() + " existing records");
         } else {
             setUpdateLogger.fine("No records in the set yet.");
         }
@@ -110,32 +90,33 @@ public void updateOaiRecords(String setName, List<Long> datasetIds, Date updateT
                 Dataset dataset = datasetService.find(datasetId);
                 if (dataset == null) {
                     setUpdateLogger.fine("failed to find dataset!");
-                } else {
-                    setUpdateLogger.fine("found dataset.");
+                } else if (dataset.isReleased() && !dataset.isDeaccessioned()) {
+                    // This means this is a published dataset
+                    setUpdateLogger.fine("found published dataset.");
 
-                    // TODO: option to *force* export? 
+                    // if doExport was requested, we'll check if the 
+                    // dataset has been exported since the last time it was 
+                    // published, and try to export if not.
                     if (doExport) {
-                        // TODO: 
-                        // Review this logic - specifically for handling of 
-                        // deaccessioned datasets. -- L.A. 4.5
                         // OK, it looks like we can't rely on .getPublicationDate() - 
                         // as it is essentially the *first publication* date; 
                         // and we are interested in the *last*
-                        
+
                         DatasetVersion releasedVersion = dataset.getReleasedVersion();
                         Date publicationDate = releasedVersion == null ? null : releasedVersion.getReleaseTime();
-                        
+
                         //if (dataset.getPublicationDate() != null
                         //        && (dataset.getLastExportTime() == null
                         //        || dataset.getLastExportTime().before(dataset.getPublicationDate()))) {
-
-                        if (publicationDate != null 
+                        if (publicationDate != null
                                 && (dataset.getLastExportTime() == null
                                 || dataset.getLastExportTime().before(publicationDate))) {
-                        
+
                             setUpdateLogger.fine("Attempting to run export on dataset " + dataset.getGlobalIdString());
                             exportAllFormats(dataset);
                         }
+
+                        // TODO: should probably bail if the export attempt has failed! -- L.A. 4.9.2
                     }
 
                     setUpdateLogger.fine("\"last exported\" timestamp: " + dataset.getLastExportTime());
@@ -315,16 +296,17 @@ public List<OAIRecord> findOaiRecordsNotInThisSet(String setName, Date from, Dat
 
     public List<OAIRecord> findOaiRecordsBySetName(String setName, Date from, Date until, boolean excludeSet) {
 
-        String queryString ="SELECT object(h) from OAIRecord h where h.id is not null";
-        if (setName != null) {
-            if (excludeSet) {
-                queryString += " and h.setName is not null and h.setName != '' and h.setName != :setName";
-            } else {
-                queryString += " and h.setName = :setName";
-            } 
+        if (setName == null) {
+            setName = "";
+        }
+
+        String queryString = "SELECT object(h) from OAIRecord h where h.id is not null";
+        if (excludeSet) {
+            queryString += " and h.setName is not null and h.setName != '' and h.setName != :setName";
         } else {
-            queryString += " and h.setName is null";
+            queryString += " and h.setName = :setName";
         }
+
         queryString += from != null ? " and h.lastUpdateTime >= :from" : "";
         queryString += until != null ? " and h.lastUpdateTime<=:until" : "";
         queryString += " order by h.globalId";

src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISet.java

@@ -62,8 +62,8 @@ public void setId(Long id) {
     private String name;
     @Column(columnDefinition="TEXT", nullable = false, unique=true)
     @Size(max = 30, message = "{setspec.maxLength}")
-    @Pattern.List({@Pattern(regexp = "[a-zA-Z0-9\\_\\-]*", message = "{dataverse.nameIllegalCharacters}"),
-        @Pattern(regexp=".*\\D.*", message="{setspec.notNumber}")})
+    @Pattern.List({@Pattern(regexp = "[a-zA-Z0-9\\_\\-]*", message = "{dataverse.nameIllegalCharacters}")})
+    //    @Pattern(regexp=".*\\D.*", message="{setspec.notNumber}")})
     private String spec;
 
     @Column(columnDefinition="TEXT", nullable = false)
@@ -134,6 +134,10 @@ public Long getVersion() {
     public void setVersion(Long version) {
         this.version = version;
     }
+
+    public boolean isDefaultSet() {
+        return "".equals(this.spec);
+    }
 
     @Override
     public int hashCode() {

src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java

@@ -52,6 +52,9 @@ public class OAISetServiceBean implements java.io.Serializable {
     @EJB
     OAIRecordServiceBean oaiRecordService;
 
+    @EJB 
+    DatasetServiceBean datasetService;
+
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean");
 
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
@@ -81,6 +84,18 @@ public OAISet findBySpec(String spec) {
         }
         return oaiSet;
     }
+
+    // Find the default, "no name" set:
+    public OAISet findDefaultSet() {
+        String query = "SELECT o FROM OAISet o where o.spec = ''";
+        OAISet oaiSet = null;
+        try {
+            oaiSet = (OAISet) em.createQuery(query).getSingleResult();
+        } catch (Exception e) {
+            // Do nothing, just return null. 
+        }
+        return oaiSet;
+    }
 
     public List<OAISet> findAll() {
         try {
@@ -93,6 +108,17 @@ public List<OAISet> findAll() {
         }
     }
 
+    public List<OAISet> findAllNamedSets() {
+        try {
+            logger.info("setService, findAllNamedSets; query: select object(o) from OAISet as o where o.spec != '' order by o.spec");
+            List<OAISet> oaiSets = em.createQuery("select object(o) from OAISet as o where o.spec != '' order by o.spec", OAISet.class).getResultList();
+            logger.info((oaiSets != null ? oaiSets.size() : 0) + " results found.");
+            return oaiSets;
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
     @Asynchronous
     public void remove(Long setId) {
         OAISet oaiSet = find(setId);
@@ -136,8 +162,16 @@ public void exportOaiSet(OAISet oaiSet, Logger exportLogger) {
 
         List<Long> datasetIds;
         try {
-            datasetIds = expandSetQuery(query);
-            exportLogger.info("set query expanded to " + datasetIds.size() + " datasets.");
+            if (!oaiSet.isDefaultSet()) {
+                datasetIds = expandSetQuery(query);
+                exportLogger.info("set query expanded to " + datasetIds.size() + " datasets.");
+            } else {
+                // The default set includes all the local, published datasets. 
+                // findAllLocalDatasetIds() finds the ids of all the local datasets - 
+                // including the unpublished drafts and deaccessioned ones.
+                // Those will be filtered out further down the line. 
+                datasetIds = datasetService.findAllLocalDatasetIds();
+            }
         } catch (OaiSetException ose) {
             datasetIds = null;
         }