Fix NAS-2648 - support for crawllogs with old duplicate annotations (…

…also NARK-1338)
netarchivesuite · Oct 20, 2017 · eb12cd4 · eb12cd4
1 parent 72d378b
commit eb12cd4
Show file tree

Hide file tree

Showing 41 changed files with 49 additions and 44 deletions.
diff --git a/archive/archive-core/pom.xml b/archive/archive-core/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>archive</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>archive-core</artifactId>

diff --git a/archive/archive-test/pom.xml b/archive/archive-test/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>archive</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>archive-test</artifactId>

diff --git a/archive/bitpreservation-gui/pom.xml b/archive/bitpreservation-gui/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>archive</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>bitpreservation-gui</artifactId>

diff --git a/archive/pom.xml b/archive/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>netarchivesuite</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>archive</artifactId>

diff --git a/build-tools/pom.xml b/build-tools/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>netarchivesuite</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>build-tools</artifactId>

diff --git a/common/common-core/pom.xml b/common/common-core/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>common</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>common-core</artifactId>

diff --git a/common/common-test/pom.xml b/common/common-test/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>common</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>common-test</artifactId>

diff --git a/common/netarchivesuite-test-utils/pom.xml b/common/netarchivesuite-test-utils/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>common</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/common/pom.xml b/common/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>netarchivesuite</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>common</artifactId>

diff --git a/deploy/deploy-core/pom.xml b/deploy/deploy-core/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>deploy</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>deploy-core</artifactId>

diff --git a/deploy/deploy-test/pom.xml b/deploy/deploy-test/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>deploy</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>deploy-test</artifactId>
@@ -31,7 +31,7 @@
     <dependency>
       <groupId>org.netarchivesuite</groupId>
       <artifactId>common-test</artifactId>
-      <version>5.4-SNAPSHOT</version>
+      <version>5.3.2-RC1</version>
       <scope>test</scope>
     </dependency>
 

diff --git a/deploy/distribution/pom.xml b/deploy/distribution/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>deploy</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>distribution</artifactId>

diff --git a/deploy/pom.xml b/deploy/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>netarchivesuite</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>deploy</artifactId>

diff --git a/harvester/harvest-scheduler/pom.xml b/harvester/harvest-scheduler/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>harvester</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>harvest-scheduler</artifactId>

diff --git a/harvester/harvestchannel-gui/pom.xml b/harvester/harvestchannel-gui/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>harvester</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>harvestchannel-gui</artifactId>

diff --git a/harvester/harvestdefinition-gui/pom.xml b/harvester/harvestdefinition-gui/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>harvester</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>harvestdefinition-gui</artifactId>

diff --git a/harvester/harvester-core/pom.xml b/harvester/harvester-core/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>harvester</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>harvester-core</artifactId>

diff --git a/...er/harvester-core/src/main/java/dk/netarkivet/harvester/indexserver/RawMetadataCache.java b/...er/harvester-core/src/main/java/dk/netarkivet/harvester/indexserver/RawMetadataCache.java
@@ -202,7 +202,7 @@ private void migrateDuplicates(Long id, String replicaUsed, String specifiedPatt
                 log.info("Doing migration for {}", id);
                 try {
                     final List<String> migrationLines = org.apache.commons.io.FileUtils.readLines(migration);
-                    log.info("{} migration records found.", migrationLines.size());
+                    log.info("{} migration records found for job {}", migrationLines.size(), id);
                     for (String line : migrationLines) {
                         String[] splitLine = StringUtils.split(line);
                         lookup.put(new Pair<String, Long>(splitLine[0], Long.parseLong(splitLine[1])),
@@ -223,13 +223,17 @@ private void migrateDuplicates(Long id, String replicaUsed, String specifiedPatt
                 }
                 originalBatchJob.copyResults(crawllog);
                 try {
+                    int matches = 0;
+                    int errors = 0;
                     for (String line :  org.apache.commons.io.FileUtils.readLines(crawllog)) {
                         Matcher m = duplicatePattern.matcher(line);
                         if (m.matches()) {
+                            matches++;
                             Long newOffset = lookup.get(new Pair<String, Long>(m.group(1), Long.parseLong(m.group(2))));
                             if (newOffset == null) {
                                 log.warn("Could not migrate duplicate in " + line);
                                 FileUtils.appendToFile(cacheFileName, line);
+                                errors++;
                             } else {
                                 String newLine = line.substring(0, m.start(2)) + newOffset + line.substring(m.end(2));
                                 newLine = newLine.replace(m.group(1), m.group(1) + ".gz");
@@ -239,6 +243,7 @@ private void migrateDuplicates(Long id, String replicaUsed, String specifiedPatt
                             FileUtils.appendToFile(cacheFileName, line);
                         }
                     }
+                    log.info("Found and migrated {} duplicate lines for job {} with {} errors", matches, id, errors); 
                 } catch (IOException e) {
                     throw new IOFailure("Could not read " + crawllog.getAbsolutePath());
                 } finally {

diff --git a/harvester/harvester-test/pom.xml b/harvester/harvester-test/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>harvester</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>harvester-test</artifactId>

diff --git a/harvester/heritrix1/heritrix1-controller/pom.xml b/harvester/heritrix1/heritrix1-controller/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>heritrix1</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>heritrix1-controller</artifactId>

diff --git a/harvester/heritrix1/heritrix1-extensions/pom.xml b/harvester/heritrix1/heritrix1-extensions/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>heritrix1</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>heritrix1-extensions</artifactId>

diff --git a/harvester/heritrix1/heritrix1-frontier/pom.xml b/harvester/heritrix1/heritrix1-frontier/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>heritrix1</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>heritrix1-frontier</artifactId>

diff --git a/harvester/heritrix1/pom.xml b/harvester/heritrix1/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>harvester</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/harvester/heritrix3/heritrix3-bundler/pom.xml b/harvester/heritrix3/heritrix3-bundler/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>heritrix3</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>heritrix3-bundler</artifactId>

diff --git a/harvester/heritrix3/heritrix3-controller/pom.xml b/harvester/heritrix3/heritrix3-controller/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>heritrix3</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>heritrix3-controller</artifactId>

diff --git a/harvester/heritrix3/heritrix3-extensions/pom.xml b/harvester/heritrix3/heritrix3-extensions/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>heritrix3</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>heritrix3-extensions</artifactId>

diff --git a/harvester/heritrix3/pom.xml b/harvester/heritrix3/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>harvester</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/harvester/history-gui/pom.xml b/harvester/history-gui/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>harvester</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>history-gui</artifactId>

diff --git a/harvester/pom.xml b/harvester/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>netarchivesuite</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>harvester</artifactId>

diff --git a/harvester/qa-gui/pom.xml b/harvester/qa-gui/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>netarchivesuite</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/integration-test/pom.xml b/integration-test/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>netarchivesuite</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>integration-test</artifactId>

diff --git a/integration-test/system-test/pom.xml b/integration-test/system-test/pom.xml
@@ -4,7 +4,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>integration-test</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>system-test</artifactId>

diff --git a/monitor/monitor-core/pom.xml b/monitor/monitor-core/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>monitor</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>monitor-core</artifactId>

diff --git a/monitor/monitor-test/pom.xml b/monitor/monitor-test/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>monitor</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>monitor-test</artifactId>

diff --git a/monitor/pom.xml b/monitor/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>netarchivesuite</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>monitor</artifactId>

diff --git a/monitor/status-gui/pom.xml b/monitor/status-gui/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>monitor</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>status-gui</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -5,7 +5,7 @@
 
   <groupId>org.netarchivesuite</groupId>
   <artifactId>netarchivesuite</artifactId>
-  <version>5.4-SNAPSHOT</version>
+  <version>5.3.2-RC1</version>
   <packaging>pom</packaging>
 
   <name>NetarchiveSuite</name>
@@ -852,7 +852,7 @@
             <dependency>
               <groupId>org.netarchivesuite</groupId>
               <artifactId>build-tools</artifactId>
-              <version>5.4-SNAPSHOT</version>
+              <version>5.3.2-RC1</version>
             </dependency>
           </dependencies>
         </plugin>

diff --git a/wayback/pom.xml b/wayback/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>netarchivesuite</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>wayback</artifactId>

diff --git a/wayback/wayback-indexer/pom.xml b/wayback/wayback-indexer/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.netarchivesuite</groupId>
     <artifactId>wayback</artifactId>
-    <version>5.4-SNAPSHOT</version>
+    <version>5.3.2-RC1</version>
   </parent>
 
   <artifactId>wayback-indexer</artifactId>