allow deduplication for ARC-Files in H3

netarchivesuite · Sep 2, 2016 · 7d614d0 · 7d614d0
1 parent f3dd6e0
commit 7d614d0
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/HarvesterSettings.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/HarvesterSettings.java
@@ -731,6 +731,8 @@ public class HarvesterSettings {
     public static String HERITRIX3_ARC_MAXSIZE = "settings.harvester.harvesting.heritrix3.arc.maxFileSizeBytes";
 
     public static String HERITRIX3_ARC_POOL_MAXACTIVE = "settings.harvester.harvesting.heritrix3.arc.poolMaxActive";
+
+    public static String HERITRIX3_ARC_SKIP_IDENTICAL_DIGESTS = "settings.harvester.harvesting.heritrix3.arc.skipIdenticalDigests";
 
     /**
      * <b>settings.harvester.harvesting.heritrix3.warc.template</b>: <br>

diff --git a/...er/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/H3HeritrixTemplate.java b/...er/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/H3HeritrixTemplate.java
@@ -329,15 +329,18 @@ private String getArcWriterProcessor() {
 	    arcWriterBeanBuilder.append(propertyName + "prefix" + valuePrefix
 	            + ARCHIVE_FILE_PREFIX_PLACEHOLDER
 	            + valueSuffix + propertyEnd);
-	    arcWriterBeanBuilder.append(propertyName + "suffix" + valuePrefix
-	            + Settings.get(HarvesterSettings.HERITRIX3_ARC_SUFFIX) 
-	            + valueSuffix + propertyEnd); 
+//	    arcWriterBeanBuilder.append(propertyName + "suffix" + valuePrefix
+//	            + Settings.get(HarvesterSettings.HERITRIX3_ARC_SUFFIX) 
+//	            + valueSuffix + propertyEnd); 
 	    arcWriterBeanBuilder.append(propertyName + "maxFileSizeBytes" + valuePrefix
 	            + Settings.get(HarvesterSettings.HERITRIX3_ARC_MAXSIZE) 
 	            + valueSuffix + propertyEnd); 
 	    arcWriterBeanBuilder.append(propertyName + "poolMaxActive" + valuePrefix
 	            + Settings.get(HarvesterSettings.HERITRIX3_ARC_POOL_MAXACTIVE) 
 	            + valueSuffix + propertyEnd); 
+	    arcWriterBeanBuilder.append(propertyName + "skipIdenticalDigests" + valuePrefix 
+  				+ Settings.get(HarvesterSettings.HERITRIX3_ARC_SKIP_IDENTICAL_DIGESTS)
+  				+ valueSuffix + propertyEnd);