NUTCH-2389 Precise data extractor implemented for 2.x

build.xml

@@ -204,6 +204,8 @@
       <packageset dir="${plugins.dir}/indexer-elastic/src/java"/>
       <packageset dir="${plugins.dir}/indexer-hbase/src/java"/>
       <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
+      <packageset dir="${plugins.dir}/jsoup-extractor/src/java"/>
+      <packageset dir="${plugins.dir}/jsoup-extractor/src/test"/>
       <packageset dir="${plugins.dir}/language-identifier/src/java"/>
       <packageset dir="${plugins.dir}/lib-http/src/java"/>
       <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
@@ -662,6 +664,7 @@
       <packageset dir="${plugins.dir}/indexer-elastic2/src/java"/>
       <packageset dir="${plugins.dir}/indexer-hbase/src/java"/>
       <packageset dir="${plugins.dir}/indexer-solr/src/java"/>
+      <packageset dir="${plugins.dir}/jsoup-extractor/src/java"/>
       <packageset dir="${plugins.dir}/language-identifier/src/java"/>
       <packageset dir="${plugins.dir}/lib-http/src/java"/>
       <packageset dir="${plugins.dir}/lib-regex-filter/src/java"/>
@@ -1034,6 +1037,8 @@
         <source path="${basedir}/src/plugin/index-metadata/src/java/" />
         <source path="${basedir}/src/plugin/index-more/src/java/" />
         <source path="${basedir}/src/plugin/index-more/src/test/" />
+        <source path="${basedir}/src/plugin/jsoup-extractor/src/java/" />
+        <source path="${basedir}/src/plugin/jsoup-extractor/src/test/" />
         <source path="${basedir}/src/plugin/language-identifier/src/java/" />
         <source path="${basedir}/src/plugin/language-identifier/src/test/" />
         <source path="${basedir}/src/plugin/lib-http/src/java/" />

conf/jsoup-extractor-example.xml

@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<extractor>
+	<normalizers>
+		<normalizer name="simpleNormalizer" class="org.apache.nutch.core.jsoup.extractor.normalizer.SimpleStringNormalizer" />
+		<normalizer name="viewCountNormalizer" class="org.apache.nutch.parse.jsoup.extractor.ViewCountNormalizer" />
+	</normalizers>
+
+	<documents>
+		<document url-pattern="^https?://(?:www\.)?youtu(?:\.be/|be\.com/watch\?v=)(?:[a-zA-Z0-9_-]{11}).*$" >
+			<field name="title">
+				<css-selector>#eow-title</css-selector>
+				<default>A placeholder Title</default>
+				<normalizer>simpleNormalizer</normalizer>
+			</field>
+			<field name="description">
+				<css-selector>#watch-description-text p#eow-description</css-selector>
+			</field>
+			<field name="uploadTime">
+				<css-selector>.watch-time-text</css-selector>
+			</field>
+			<field name="likeCount">
+				<css-selector>.like-button-renderer-like-button.like-button-renderer-like-button-unclicked span.yt-uix-button-content</css-selector>
+			</field>
+			<field name="dislikeCount">
+				<css-selector>.like-button-renderer-dislike-button.like-button-renderer-dislike-button-unclicked span.yt-uix-button-content</css-selector>
+			</field>
+			<field name="viewCount">
+				<css-selector>.watch-view-count</css-selector>
+				<normalizer>viewCountNormalizer</normalizer>
+			</field>
+			<field name="subscriberCount">
+				<css-selector>.yt-subscriber-count</css-selector>
+			</field>
+			<field name="publisherName">
+				<css-selector>.yt-user-info a</css-selector>
+			</field>
+			<field name="publisherChannel">
+				<css-selector>.yt-user-info a</css-selector>
+				<attribute>abs:href</attribute>
+			</field>
+			<field name="publisherStatus">
+				<css-selector>.yt-user-info span</css-selector>
+				<attribute>aria-label</attribute>
+			</field>
+			<field name="category">
+				<css-selector>.watch-extras-section :nth-child(1) a</css-selector>
+			</field>
+		</document>
+
+		<!-- 
+		<document url-pattern="^https?:/(www\.)?gaana\.com/song/[^/]+/?$" >
+			<field name="title">
+				<css-selector>li[class=s_title][data-type=playSong] a[class=sng_c]</css-selector>
+				<default>A placeholder Title</default>
+			</field>
+			<field name="artist">
+				<css-selector>li[class=s_title][data-type=playSong] a[href^=/artist]</css-selector>
+			</field>
+			<field name="uploadTime">
+				<css-selector>.songdetails_col1 :nth-child(2) :nth-child(2)</css-selector>
+			</field>
+			<field name="duration">
+				<css-selector>.songdetails_col1 :nth-child(3) :nth-child(2)</css-selector>
+			</field>
+			<field name="language">
+				<css-selector>.songdetails_col1 :nth-child(4) :nth-child(2)</css-selector>
+			</field>
+		</document> 
+		-->
+
+	</documents>
+</extractor>

conf/jsoup-extractor.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- 
+	Define your custom normalizer here. Any normalizer class 
+	must have to implement org.apache.nutch.core.jsoup.extractor.normalizer.Normalizable interface
+	Every normalizer must have a name and class attribute. See conf/jsoup-extractor-example.xml for example
+-->
+<normalizers>
+	<normalizer name="simple-string-normalizer" class="org.apache.nutch.core.jsoup.extractor.normalizer.SimpleStringNormalizer" />
+</normalizers>
+
+<documents>
+	<!-- 
+		Every document must have url-pattern attribute which will contain the expected URL regex for filtering 
+	-->
+	<document url-pattern=".*" >
+		<!--
+			A <field> tag can have following properties:
+				1. attribute - 'name', contains the name of the field for indexing (mandatory).
+				2. tag - <css-selector>, contains the jsoup selector-syntax to find content using jsoup select() API (mandatory).
+				3. tag - <attribute>, contains the html attribute name to find content using jsoup attr() API along with select() API
+								      (optional, if <attribute> is defined, select(<css-selector>).attr(<attribute>) will be used to extract content, otherwise, select(<css-selector>).ownText() will be used)
+				4. tag - <default-value>, contains the default value in case nothing found after jsoup selection. This is optional
+				5. tag - <normalizer>, name of the normalizer class defined in <normalizers> section. This is optional.
+
+				See conf/jsoup-extractor-example.xml for example.
+		-->
+		<!--
+			Sample field example
+			<field name="title">
+				<css-selector>#eow-title</css-selector>
+				<default>A placeholder Title</default>
+				<normalizer>simpleNormalizer</normalizer>
+			</field>
+		-->
+	</document>
+</documents>

conf/nutch-default.xml

@@ -1083,6 +1083,15 @@
   </description>
 </property>
 
+<!-- jsoup-extractor properties -->
+
+<property>
+  <name>jsoup.extractor.property.file</name>
+  <value>jsoup-extractor.xml</value>
+  <description>Name of the configuration file which will be used by jsoup-extractor plugin.  
+  </description>
+</property>
+
 <!--
 <property>
   <name>tika.htmlmapper.classname</name>

src/plugin/build.xml

@@ -36,6 +36,7 @@
      <!--<ant dir="indexer-elastic" target="deploy"/>-->
      <ant dir="indexer-elastic2" target="deploy"/>
      <ant dir="indexer-hbase" target="deploy"/>
+     <ant dir="jsoup-extractor" target="deploy"/>
      <ant dir="language-identifier" target="deploy"/>
      <ant dir="lib-http" target="deploy"/>
      <ant dir="lib-nekohtml" target="deploy"/>
@@ -86,6 +87,7 @@
      <ant dir="index-anchor" target="test"/>
      <ant dir="index-basic" target="test"/>
      <ant dir="index-more" target="test"/>
+     <ant dir="jsoup-extractor" target="test"/>
      <ant dir="language-identifier" target="test"/>
      <ant dir="protocol-httpclient" target="test"/>
      <ant dir="protocol-http" target="test"/>
@@ -126,6 +128,7 @@
     <!--<ant dir="indexer-elastic" target="clean"/>-->
     <ant dir="indexer-elastic2" target="clean"/>
     <ant dir="indexer-hbase" target="clean"/>
+    <ant dir="jsoup-extractor" target="clean"/>
     <ant dir="language-identifier" target="clean"/>
     <ant dir="lib-http" target="clean"/>
     <ant dir="lib-nekohtml" target="clean"/>

src/plugin/jsoup-extractor/build.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="jsoup-extractor" default="jar-core">
+
+  <import file="../build-plugin.xml"/>
+
+  <!-- Deploy Unit test dependencies -->
+  <target name="deploy-test">
+    <ant target="deploy" inheritall="false" dir="../nutch-extensionpoints"/>
+  </target>
+
+</project>

src/plugin/jsoup-extractor/ivy.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" ?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<ivy-module version="1.0">
+  <info organisation="org.apache.nutch" module="${ant.project.name}">
+    <license name="Apache 2.0"/>
+    <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+    <description>
+        Apache Nutch
+    </description>
+  </info>
+
+  <configurations>
+    <include file="../../../ivy/ivy-configurations.xml"/>
+  </configurations>
+
+  <publications>
+    <artifact conf="master"/>
+  </publications>
+
+  <dependencies>
+    <dependency org="org.jsoup" name="jsoup" rev="1.10.2"/>
+  </dependencies>
+
+</ivy-module>

src/plugin/jsoup-extractor/plugin.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<plugin
+   id="jsoup-extractor"
+   name="Html Parse and Indexing Filter Plug-in based on Jsoup"
+   version="1.0.0"
+   provider-name="nutch.apache.org">
+
+   <runtime>
+      <library name="jsoup-extractor.jar">
+         <export name="*"/>
+      </library>
+      <library name="jsoup-1.10.2.jar"/>
+      <library name="geronimo-stax-api_1.0_spec-1.0.1.jar"/>
+      <library name="stax-api-1.0.1.jar"/>
+   </runtime>
+
+   <requires>
+      <import plugin="nutch-extensionpoints"/>
+   </requires>
+
+   <extension id="org.apache.nutch.parse.jsoup.extractor"
+              name="JsoupHtmlParser"
+              point="org.apache.nutch.parse.ParseFilter">
+
+      <implementation id="org.apache.nutch.parse.jsoup.extractor.JsoupHtmlParser"
+                      class="org.apache.nutch.parse.jsoup.extractor.JsoupHtmlParser">
+        <parameter name="contentType" value="text/html|application/xhtml+xml"/>
+        <parameter name="pathSuffix" value=""/>
+      </implementation>
+
+   </extension>
+
+   <extension id="org.apache.nutch.indexer.jsoup.extractor" name="JsoupIndexingFilter"
+       point="org.apache.nutch.indexer.IndexingFilter">
+     <implementation id="JsoupIndexingFilter"
+       class="org.apache.nutch.indexer.jsoup.extractor.JsoupIndexingFilter" />
+    </extension>
+
+</plugin>