From 047fe3ab04e6e418b3f09e21152418f4c128293e Mon Sep 17 00:00:00 2001 From: csrster Date: Thu, 24 Nov 2016 15:27:13 +0100 Subject: [PATCH] Set enable/disable deduplication in hopefully the right place this time. --- .../harvester/scheduler/jobgen/AbstractJobGenerator.java | 4 ++-- .../dk/netarkivet/harvester/datamodel/H3HeritrixTemplate.java | 4 +++- .../src/main/java/dk/netarkivet/harvester/datamodel/Job.java | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/harvester/harvest-scheduler/src/main/java/dk/netarkivet/harvester/scheduler/jobgen/AbstractJobGenerator.java b/harvester/harvest-scheduler/src/main/java/dk/netarkivet/harvester/scheduler/jobgen/AbstractJobGenerator.java index 8593b75f03..1d933a3ab3 100644 --- a/harvester/harvest-scheduler/src/main/java/dk/netarkivet/harvester/scheduler/jobgen/AbstractJobGenerator.java +++ b/harvester/harvest-scheduler/src/main/java/dk/netarkivet/harvester/scheduler/jobgen/AbstractJobGenerator.java @@ -207,7 +207,7 @@ public boolean canAccept(Job job, DomainConfiguration cfg, DomainConfiguration p protected void editJobOrderXml(Job job) { HeritrixTemplate doc = job.getOrderXMLdoc(); doc.enableOrDisableDeduplication(DEDUPLICATION_ENABLED); - if (DEDUPLICATION_ENABLED) { +/* if (DEDUPLICATION_ENABLED) { // Check that the Deduplicator element is present in the // OrderXMl and enabled. If missing or disabled log a warning @@ -220,7 +220,7 @@ protected void editJobOrderXml(Job job) { doc.removeDeduplicatorIfPresent(); job.setOrderXMLDoc(doc); log.info("Removed DeDuplicator element because Deduplication is disabled"); - } + }*/ } /** diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/H3HeritrixTemplate.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/H3HeritrixTemplate.java index 75b2e160a7..d5af31dfd5 100644 --- a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/H3HeritrixTemplate.java +++ b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/H3HeritrixTemplate.java @@ -531,7 +531,9 @@ public void removeDeduplicatorIfPresent() { } @Override public void enableOrDisableDeduplication(boolean enabled) { - template = template.replace(DEDUPLICATION_ENABLED_PLACEHOLDER, Boolean.toString(enabled).toLowerCase()); + final String replacement = Boolean.toString(enabled).toLowerCase(); + log.debug("Replacing deduplication enabled placeholder {} with {}.", DEDUPLICATION_ENABLED_PLACEHOLDER, replacement); + this.template = template.replace(DEDUPLICATION_ENABLED_PLACEHOLDER, replacement); } // diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/Job.java b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/Job.java index 28fdf67538..495d0239b3 100644 --- a/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/Job.java +++ b/harvester/harvester-core/src/main/java/dk/netarkivet/harvester/datamodel/Job.java @@ -282,6 +282,8 @@ public Job(Long harvestID, DomainConfiguration cfg, HeritrixTemplate orderXMLdoc setAttributes(cfg.getAttributesAndTypes()); + orderXMLdoc.enableOrDisableDeduplication(Settings.getBoolean(HarvesterSettings.DEDUPLICATION_ENABLED)); + status = JobStatus.NEW; }