Allow fast-urlfilter to load from HDFS/S3 and support gzipped input [NUTCH-3017]

README.md

@@ -1,9 +1,42 @@
-Apache Nutch README
-===================
+Common Crawl Fork of Apache Nutch
+=================================
+
+Please also have a look at the [Apache Nutch](/apache/nutch) repository and all information about Apache Nutch given below.
+
+Notable additions in Common Crawl's fork of Nutch (not yet pushed to upstream Nutch although this is planned):
+- WARC and CDX writer integrated into Fetcher and able to detect the language of HTML pages using the CLD2 language detector
+- [Generator2](src/java/org/apache/nutch/crawl/Generator2.java): alternative implementation of Generator
+  - allowing to combine per-domain and per-host limits and
+  - optimized to create many (eg. 100) segments in a single job
+
+How to install additional requirements to build this fork of Nutch:
+- [crawler-commons](/crawler-commons/crawler-commons) development snapshot package:
+  ```
+  git clone git@github.com:crawler-commons/crawler-commons.git
+  cd crawler-commons/
+  mvn install
+  ```
+- install the latest public suffix list into `conf/` to ensure that it is definitely used (see #17):
+  ```
+  wget https://publicsuffix.org/list/public_suffix_list.dat -O conf/effective_tld_names.dat
+  ```
+- [Java wrapper for CLD2 language detection](/commoncrawl/language-detection-cld2)
+  ```
+  git clone git@github.com:commoncrawl/language-detection-cld2.git
+  cd language-detection-cld2/
+  mvn install
+  ```
+  For runtime, if WARC language detection is enabled (`warc.detect.language` = true), also the CLD2 shared objects are required, e.g. on Ubuntu
+  ```
+  sudo apt install libcld2-0 libcld2-dev
+  ```
+
+Apache Nutch
+============
 
 <img src="https://nutch.apache.org/assets/img/nutch_logo_tm.png" align="right" width="300" />
 
-For the latest information about Nutch, please visit our website at:
+For the latest information about Nutch, please visit the Nutch website at:
 
    https://nutch.apache.org/
 

build.xml

@@ -126,7 +126,7 @@
     <javac
      encoding="${build.encoding}"
      srcdir="${src.dir}"
-     includes="org/apache/nutch/**/*.java"
+     includes="org/apache/nutch/**/*.java org/commoncrawl/**/*.java"
      destdir="${build.classes}"
      debug="${javac.debug}"
      optimize="${javac.optimize}"
@@ -250,6 +250,7 @@
       <packageset dir="${plugins.dir}/protocol-okhttp/src/java"/>
       <packageset dir="${plugins.dir}/protocol-selenium/src/java"/>
       <packageset dir="${plugins.dir}/publish-rabbitmq/src/java"/>
+      <packageset dir="${plugins.dir}/scoring-adaptive/src/java"/>
       <packageset dir="${plugins.dir}/scoring-depth/src/java"/>
       <packageset dir="${plugins.dir}/scoring-link/src/java"/>
       <packageset dir="${plugins.dir}/scoring-opic/src/java"/>
@@ -456,7 +457,7 @@
     <javac
      encoding="${build.encoding}"
      srcdir="${test.src.dir}"
-     includes="org/apache/nutch/**/*.java"
+     includes="org/apache/nutch/**/*.java org/commoncrawl/**/*.java"
      destdir="${test.build.classes}"
      debug="${javac.debug}"
      optimize="${javac.optimize}"
@@ -735,6 +736,7 @@
       <packageset dir="${plugins.dir}/protocol-okhttp/src/java"/>
       <packageset dir="${plugins.dir}/protocol-selenium/src/java"/>
       <packageset dir="${plugins.dir}/publish-rabbitmq/src/java"/>
+      <packageset dir="${plugins.dir}/scoring-adaptive/src/java"/>
       <packageset dir="${plugins.dir}/scoring-depth/src/java"/>
       <packageset dir="${plugins.dir}/scoring-link/src/java"/>
       <packageset dir="${plugins.dir}/scoring-opic/src/java"/>
@@ -1256,6 +1258,7 @@
         <source path="${plugins.dir}/protocol-okhttp/src/test/" />
         <source path="${plugins.dir}/protocol-selenium/src/java"/>
         <source path="${plugins.dir}/publish-rabbitmq/src/java"/>
+        <source path="${plugins.dir}/scoring-adaptive/src/java"/>
         <source path="${plugins.dir}/scoring-depth/src/java/" />
         <source path="${plugins.dir}/scoring-link/src/java/" />
         <source path="${plugins.dir}/scoring-opic/src/java/" />

conf/adaptive-scoring.txt.template

@@ -0,0 +1,17 @@
+#
+# Configuration file for scoring-adaptive
+#
+# See also properties
+#  scoring.adaptive.sort.by_status.file
+#  scoring.adaptive.factor.fetchtime
+#  scoring.adaptive.penalty.fetch_retry
+#  scoring.adaptive.boost.injected
+#
+# Format:
+#   <status> <tab> <sortvalue>
+# e.g.
+# db_unfetched	.1
+# db_gone	-.5
+#
+# The sort value is added to other sort values (score, fetch time).
+# It may be negative to penalize fetch items.

conf/generate-domain-limits.txt.template

@@ -0,0 +1,15 @@
+#
+# Fetch list limits by domain
+#
+# Note: please register this file using the property `generate.domain.limits.file`.
+#
+# Fields (tab-separated):
+#  1 - domain name
+#  2 - max. number of URLs per domain and segment
+#  3 - max. number of URLs per host (in domain) and segment
+#  4 - max. number of hosts per domain over all segments
+#  5 - max. number of partitions (fetcher tasks) for domain
+#
+# Lines starting with `#` are ignored.
+# 
+# wikipedia.org	5000	500	20	1