Enabled "isEnabled" feature in DeDuplicator.

netarchivesuite · Nov 24, 2016 · 6dda348 · 6dda348
1 parent 95eeb7f
commit 6dda348
Showing 1 changed file with 17 additions and 0 deletions.
diff --git a/...ter/heritrix3/heritrix3-extensions/src/main/java/is/hi/bok/deduplicator/DeDuplicator.java b/...ter/heritrix3/heritrix3-extensions/src/main/java/is/hi/bok/deduplicator/DeDuplicator.java
@@ -116,6 +116,14 @@
 @SuppressWarnings({"unchecked"})
 public class DeDuplicator extends Processor implements InitializingBean {
 
+    @Override public boolean getEnabled() {
+        return super.getEnabled();
+    }
+
+    @Override public void setEnabled(boolean enabled) {
+        super.setEnabled(enabled);
+    }
+
     private static Logger logger =
         Logger.getLogger(DeDuplicator.class.getName());
 
@@ -351,6 +359,10 @@ public void setServerCache(ServerCache serverCache) {
 
 
     public void afterPropertiesSet() throws Exception {
+        if (!getEnabled()) {
+            logger.info(this.getClass().getName() + " disabled.");
+            return;
+        }
         // Index location
         String indexLocation = getIndexLocation();
         try {
@@ -393,6 +405,10 @@ public void afterPropertiesSet() throws Exception {
 
 	@Override
 	protected boolean shouldProcess(CrawlURI curi) {
+        if (!getEnabled()) {
+            logger.finest("Not handling " + curi.toString() + ", deduplication disabled.");
+            return false;
+        }
         if (curi.isSuccess() == false) {
             // Early return. No point in doing comparison on failed downloads.
             logger.finest("Not handling " + curi.toString()
@@ -472,6 +488,7 @@ private String getRefersToDate(Document duplicate) {
 
 	@Override
 	protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException {
+
         ProcessResult processResult = ProcessResult.PROCEED; // Default. Continue as normal
 
         logger.finest("Processing " + curi.toString() + "(" +