NUTCH-1786 CrawlDb should follow db.url.normalizers and db.url.filters

git-svn-id: https://svn.apache.org/repos/asf/nutch/trunk@1597556 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -2,6 +2,8 @@ Nutch Change Log
 
 Nutch Current Development
 
+* NUTCH-1786 CrawlDb should follow db.url.normalizers and db.url.filters (Diaa via markus)
+
 * NUTCH-1757 ParserChecker to take custom metadata as input (jnioche)
 
 * NUTCH-1676 Add rudimentary SSL support to protocol-http (jnioche, markus)

conf/nutch-default.xml

@@ -76,11 +76,11 @@
 
   NOTE: You should also check other related properties:
 
-	http.robots.agents
-	http.agent.description
-	http.agent.url
-	http.agent.email
-	http.agent.version
+    http.robots.agents
+    http.agent.description
+    http.agent.url
+    http.agent.email
+    http.agent.version
 
   and set their values appropriately.
 
@@ -345,7 +345,6 @@
 </property>
 
 <!-- web db properties -->
-
 <property>
   <name>db.fetch.interval.default</name>
   <value>2592000</value>
@@ -449,6 +448,18 @@
   </description>
 </property>
 
+<property>
+    <name>db.url.normalizers</name>
+    <value>false</value>
+    <description>Normalize urls when updating crawldb</description>
+</property>
+
+<property>
+    <name>db.url.filters</name>
+    <value>false</value>
+    <description>Filter urls when updating crawldb</description>
+</property>
+
 <property>
   <name>db.update.max.inlinks</name>
   <value>10000</value>

src/java/org/apache/nutch/crawl/CrawlDb.java

@@ -114,7 +114,9 @@ public void update(Path crawlDb, Path[] segments, boolean normalize, boolean fil
     long end = System.currentTimeMillis();
     LOG.info("CrawlDb update: finished at " + sdf.format(end) + ", elapsed: " + TimingUtil.elapsedTime(start, end));
   }
-
+/*
+ * Configure a new CrawlDb in a temp folder at crawlDb/<rand>
+ */
   public static JobConf createJob(Configuration config, Path crawlDb)
     throws IOException {
     Path newCrawlDb =
@@ -180,12 +182,11 @@ public int run(String[] args) throws Exception {
 
       return -1;
     }
-    boolean normalize = false;
-    boolean filter = false;
+    boolean normalize = getConf().getBoolean(CrawlDbFilter.URL_NORMALIZING, false);
+    boolean filter = getConf().getBoolean(CrawlDbFilter.URL_FILTERING, false);
+    boolean additionsAllowed = getConf().getBoolean(CRAWLDB_ADDITIONS_ALLOWED, true);
     boolean force = false;
-    boolean url404Purging = false;
     final FileSystem fs = FileSystem.get(getConf());
-    boolean additionsAllowed = getConf().getBoolean(CRAWLDB_ADDITIONS_ALLOWED, true);
     HashSet<Path> dirs = new HashSet<Path>();
     for (int i = 1; i < args.length; i++) {
       if (args[i].equals("-normalize")) {