Fixed NAS-2480 - don't insert empty traps in our Heritrix templates

netarchivesuite · Mar 23, 2016 · 4eacf59 · 4eacf59
1 parent 6cae709
commit 4eacf59
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 3 deletions.
diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/Domain.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/Domain.java
@@ -814,6 +814,7 @@ public void setCrawlerTraps(List<String> regExps, boolean strictMode) {
             }
         }
         crawlerTraps = Collections.unmodifiableList(cleanedListOfCrawlerTraps);
+        log.debug("Domain {} has {} crawlertraps", domainName, crawlerTraps.size());
     }
 
     /**

diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DomainDBDAO.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/DomainDBDAO.java
@@ -773,8 +773,16 @@ protected synchronized Domain readKnown(Connection c, String domainName) {
             Domain d = new Domain(domainName);
             d.setComments(comments);
             // don't throw exception if illegal regexps are found.
-            boolean strictMode = false;
-            d.setCrawlerTraps(Arrays.asList(crawlertraps.split("\n")), strictMode);
+            boolean strictMode = false; 
+            String[] traps = crawlertraps.split("\n");
+            List<String> insertList = new ArrayList<String>();
+            for (String trap: traps) {
+                if (!trap.isEmpty()) { // Ignore empty traps (NAS-2480)
+                    insertList.add(trap);
+                }
+            }
+            log.debug("Found {} crawlertraps for domain '{}' in database", insertList.size(), domainName);
+            d.setCrawlerTraps(insertList, strictMode);
             d.setID(domainId);
             d.setEdition(edition);
             if (alias != null) {

diff --git a/...ster/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HeritrixTemplate.java b/...ster/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/HeritrixTemplate.java
@@ -165,7 +165,14 @@ public void setIsActive(boolean isActive) {
 	public void editOrderXMLAddPerDomainCrawlerTraps(DomainConfiguration cfg) {
 		List<String> crawlerTraps = cfg.getCrawlertraps();
 		String elementName = cfg.getDomainName();
-		if (!crawlerTraps.isEmpty()) {
+		int trapCount=crawlerTraps.size();
+		for (String trap: crawlerTraps){
+		    if (trap.isEmpty()) { // Ignore empty traps in the trapcount (NAS-2480)
+		        log.warn("Found empty trap for domain", cfg.getDomainName());
+		        trapCount--; 
+		    }
+		}
+		if (trapCount > 0) {
 			log.info("Inserting {} crawlertraps for domain '{}' into the template", crawlerTraps.size(), elementName);
 			insertCrawlerTraps(elementName, crawlerTraps);
 		}