NAS-2702 - 1) Changed some logging from log.debug to log.trace. 2) Co…

…rrected the DomainDBDAO.getHarvestInfoForDomainInHarvest() method to select the latest entry, if multiple entries for the same domainconfig in the same harvest (can happen, if the job is resubmitted)

harvester/harvest-scheduler/src/main/java/dk/netarkivet/harvester/scheduler/HarvestJobGenerator.java

@@ -136,7 +136,7 @@ public synchronized void run() {
         void generateJobs(Date timeToGenerateJobsFor) {
             final Iterable<Long> readyHarvestDefinitions = haDefinitionDAO
                     .getReadyHarvestDefinitions(timeToGenerateJobsFor);
-            log.debug("Generating jobs for harvests that should run at time '{}'", timeToGenerateJobsFor);
+            log.trace("Generating jobs for harvests that should run at time '{}'", timeToGenerateJobsFor);
             HarvestChannelDAO hChanDao = HarvestChannelDAO.getInstance();
 
             for (final Long id : readyHarvestDefinitions) {

harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DomainDAO.java

@@ -361,6 +361,11 @@ public abstract Iterator<HarvestInfo> getHarvestInfoBasedOnPreviousHarvestDefini
     public abstract HarvestInfo getHarvestInfoForDomainInHarvest(
             HarvestDefinition harvestDefinition, Domain domain);
 
+    /**
+     * Gets list of all domains in the order expected by the snapshot harvest job generation, that is order by template
+     * name, then byte limit (descending), then domain name.
+     * @param previousHid The harvestDefinitionId of the harvestdefinition that we are continuing. If null, we start from scratch.
+     * @return List of all added domains
+     */
     public abstract Iterator<Domain> getDomainsInSnapshotHarvestOrder(Long previousHid);
-
 }

harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DomainDBDAO.java

@@ -1128,9 +1128,11 @@ public Iterator<Domain> getDomainsInSnapshotHarvestOrder(Long hid) {
                         + " WHERE domains.defaultconfig=configurations.config_id" + " AND configurations.template_id"
                         + "=ordertemplates.template_id" 
                         + " AND configurations.config_id=historyinfo.config_id "
-                        + " AND historyinfo.harvest_id=" + hid
-                        + " ORDER BY" + " ordertemplates.name,"
-                        + " configurations.maxbytes DESC," + " domains.name");
+                        + " AND historyinfo.harvest_id=" + hid);
+                        // NOTE: the ordering has now been skipped to prevent duplicates
+                        //  + " ORDER BY" + " ordertemplates.name," 
+                        //  + " configurations.maxbytes DESC");
+                        // "," + " domains.name");
                 log.info("Retrieved all {} domains harvested in previous snapshot harvest #{}", domainNames.size(), hid);
                 domainNamesWithAttributes = DBUtils.selectStringList(c, // Don't order this - it will be ordered later
                         "SELECT DISTINCT domains.name"
@@ -1619,18 +1621,26 @@ public HarvestInfo getHarvestInfoForDomainInHarvest(final HarvestDefinition harv
                 }
                 long harvestId = harvestDefinition.getOid();
                 Date harvestTime = new Date(res.getTimestamp(6).getTime());
-                HarvestInfo hi;
 
-                hi = new HarvestInfo(harvestId, jobId, domain.getName(), configName, harvestTime, byteCount, objectCount, stopreason);
+                HarvestInfo hi = new HarvestInfo(harvestId, jobId, domain.getName(), configName, harvestTime, byteCount, objectCount, stopreason);
                 infoFoundForDomain.add(hi);
             }
             if (infoFoundForDomain.isEmpty()) {
                 return null;
             } else if (infoFoundForDomain.size() == 1) {
                 return infoFoundForDomain.get(0);
             } else {
-                log.warn("Found {} harvestInfo entries for domain '{}' and harvestdefinition '{}'. Selecting the first", infoFoundForDomain.size(), domain.getName(), harvestDefinition.getName());
-                return infoFoundForDomain.get(0);
+                HarvestInfo selected = infoFoundForDomain.get(0);
+                Long latest = selected.getDate().getTime();
+                for (int i=1; i < infoFoundForDomain.size(); i++) {
+                    if (infoFoundForDomain.get(i).getDate().getTime() > latest) {
+                        latest = infoFoundForDomain.get(i).getDate().getTime();
+                        selected = infoFoundForDomain.get(i);
+                    }
+                }
+                log.warn("Found {} harvestInfo entries for domain '{}' and harvestdefinition '{}'. Selecting the latest entry: {}", infoFoundForDomain.size(), domain.getName(), 
+                        harvestDefinition.getName(), selected);
+                return selected;
             }
         } catch (SQLException e) {
             throw new IOFailure("Error while fetching HarvestInfo for domain '" + domain.getName() + "' in harvest '" + harvestDefinition.getName() + "':", e);