NUTCH-3015 Add more CI steps to GitHub master-build.yml (#790)

apache · Oct 27, 2023 · 792ed28 · 792ed28
1 parent 8431dcf
commit 792ed28
Show file tree

Hide file tree

Showing 138 changed files with 1,060 additions and 136 deletions.
diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: master pr build
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # every day at midnight
+
+jobs:
+  dependency-check:
+    strategy:
+      matrix:
+        java: ['11']
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+      - name: Dependency check
+        run: ant clean dependency-check -buildfile build.xml
diff --git a/.github/workflows/master-build.yml b/.github/workflows/master-build.yml
@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -13,28 +12,67 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
-name: master pr build
+name: master pull request ci
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    types: [opened, synchronize, reopened]
+    branches: [master]
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  javadoc:
     strategy:
       matrix:
-        java: [ '11' ]
-
+        java: ['11']
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+      - name: Javadoc
+        run: ant clean javadoc -buildfile build.xml
+  rat:
+    strategy:
+      matrix:
+        java: ['11']
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+      - name: Run Apache Rat
+        run: ant clean run-rat -buildfile build.xml
+      - name: Cache unknown licenses
+        run: echo "UNKNOWN_LICENSES=$(sed -n 18p /home/runner/work/nutch/nutch/build/apache-rat-report.txt)" >> $GITHUB_ENV
+      - name: Versions
+        run: |
+          echo $UNKNOWN_LICENSES
+      - name: Fail if any unknown licenses
+        if: ${{ env.UNKNOWN_LICENSES != '0 Unknown Licenses' }}
+        run: exit 1
+  test:
+    strategy:
+      matrix:
+        java: ['11']
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v3
         with:
           java-version: ${{ matrix.java }}
-      - name: Build with Ant
-        run: ant clean nightly javadoc -buildfile build.xml
+          distribution: 'temurin'
+      - name: Test
+        run: ant clean test -buildfile build.xml
diff --git a/.gitignore b/.gitignore
@@ -28,3 +28,4 @@ csvindexwriter
 lib/spotbugs-*
 ivy/dependency-check-ant/*
 .gradle*
+ivy/apache-rat-*
diff --git a/build.xml b/build.xml
@@ -38,7 +38,7 @@
   <property name="maven-javadoc-jar" value="${release.dir}/${artifactId}-${version}-javadoc.jar" />
   <property name="maven-sources-jar" value="${release.dir}/${artifactId}-${version}-sources.jar" />
 
-  <property name="dependency-check-ant.version" value="7.1.1" />
+  <property name="dependency-check-ant.version" value="8.4.2" />
   <property name="dependency-check-ant.home" value="${ivy.dir}/dependency-check-ant" />
   <property name="dependency-check-ant.jar" value="${dependency-check-ant.home}/dependency-check-ant.jar" />
 
@@ -48,7 +48,7 @@
   <property name="spotbugs.home" value="${ivy.dir}/spotbugs-${spotbugs.version}" />
   <property name="spotbugs.jar" value="${spotbugs.home}/lib/spotbugs-ant.jar" />
 
-  <property name="apache-rat.version" value="0.14" />
+  <property name="apache-rat.version" value="0.15" />
   <property name="apache-rat.home" value="${ivy.dir}/apache-rat-${apache-rat.version}" />
   <property name="apache-rat.jar" value="${apache-rat.home}/apache-rat-${apache-rat.version}.jar" />
 
@@ -640,13 +640,15 @@
     </fileset>
   </path>
 
-  <target name="report-vulnerabilities" depends="jar, compile-plugins, dependency-check-ant-download" description="--> check dependencies for security vulnerabilities">
+  <target name="dependency-check" depends="jar, compile-plugins, dependency-check-ant-download" description="--> check dependencies for security vulnerabilities">
     <taskdef resource="dependency-check-taskdefs.properties">
       <classpath refid="dependency-check-ant.path" />
     </taskdef>
     <dependency-check projectname="${name}"
                       reportoutputdirectory="${dependency-check-ant.home}"
-                      reportformat="ALL">
+                      reportformat="ALL"
+                      assemblyAnalyzerEnabled="false"
+                      failBuildOnCVSS="1">
         <suppressionfile path="${dependency-check-ant.home}/dependency-check-suppressions.xml" />
         <retirejsFilter regex="copyright.*jeremy long" />
         <fileset dir="${build.dir}">
@@ -1025,7 +1027,7 @@
 
   <target name="apache-rat-download-unchecked" unless="apache-rat.jar.found"
           description="--> downloads the Apache Rat jar">
-    <get src="https://www.apache.org/dist/creadur/apache-rat-${apache-rat.version}/apache-rat-${apache-rat.version}-bin.tar.gz"
+    <get src="https://archive.apache.org/dist/creadur/apache-rat-${apache-rat.version}/apache-rat-${apache-rat.version}-bin.tar.gz"
          dest="${ivy.dir}/apache-rat-${apache-rat.version}-bin.tar.gz" usetimestamp="false" />
 
     <untar src="${ivy.dir}/apache-rat-${apache-rat.version}-bin.tar.gz"
@@ -1035,8 +1037,8 @@
     <delete file="${ivy.dir}/apache-rat-${apache-rat.version}-bin.tar.gz" />
   </target>
 
-  <target name="rat-sources" depends="init, apache-rat-download"
-    description="--> runs RAT tasks over src/java">
+  <target name="run-rat" depends="init, apache-rat-download"
+    description="--> runs Apache Rat on codebase">
     <taskdef
         uri="antlib:org.apache.rat.anttasks"
         resource="org/apache/rat/anttasks/antlib.xml">
@@ -1047,8 +1049,40 @@
     <rat:report
         reportFile="${build.dir}/apache-rat-report.txt">
       <fileset dir="src">
-        <include name="java/**/*"/>
-        <include name="plugin/**/src/**/*"/>
+        <include name="**"/>
+        <exclude name="plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/langmappings.properties"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/de.test"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/en.test"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/es.test"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/fi.test"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/fr.test"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/it.test"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/nl.test"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/pt.test"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/sv.test"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/test-referencial.txt"/>
+        <exclude name="plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/da.test"/>
+        <exclude name="plugin/parse-tika/sample/ootest.txt"/>
+        <exclude name="plugin/parse-tika/sample/test.rtf"/>
+        <exclude name="plugin/urlfilter-ignoreexempt/data/.donotdelete"/>
+        <exclude name="plugin/urlfilter-automaton/sample/Benchmarks.rules"/>
+        <exclude name="plugin/urlfilter-automaton/sample/Benchmarks.urls"/>
+        <exclude name="plugin/urlfilter-automaton/sample/IntranetCrawling.rules"/>
+        <exclude name="plugin/urlfilter-automaton/sample/IntranetCrawling.urls"/>
+        <exclude name="plugin/urlfilter-automaton/sample/WholeWebCrawling.rules"/>
+        <exclude name="plugin/urlfilter-automaton/sample/WholeWebCrawling.urls"/>
+        <exclude name="plugin/urlfilter-fast/sample/Benchmarks.urls"/>
+        <exclude name="plugin/urlfilter-fast/sample/fast-urlfilter-benchmark.txt"/>
+        <exclude name="plugin/urlfilter-fast/sample/fast-urlfilter-test.txt"/>
+        <exclude name="plugin/urlfilter-fast/sample/test.urls"/>
+        <exclude name="plugin/urlfilter-regex/sample/Benchmarks.rules"/>
+        <exclude name="plugin/urlfilter-regex/sample/Benchmarks.urls"/>
+        <exclude name="plugin/urlfilter-regex/sample/IntranetCrawling.rules"/>
+        <exclude name="plugin/urlfilter-regex/sample/IntranetCrawling.urls"/>
+        <exclude name="plugin/urlfilter-regex/sample/WholeWebCrawling.rules"/>
+        <exclude name="plugin/urlfilter-regex/sample/WholeWebCrawling.urls"/>
+        <exclude name="plugin/urlfilter-regex/sample/nutch1838.rules"/>
+        <exclude name="plugin/urlfilter-regex/sample/nutch1838.urls"/>
       </fileset>
     </rat:report>
   </target>

diff --git a/ivy/dependency-check-ant/dependency-check-suppressions.xml b/ivy/dependency-check-ant/dependency-check-suppressions.xml
@@ -1,8 +1,3 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <suppressions xmlns="https://jeremylong.github.io/DependencyCheck/dependency-suppression.1.1.xsd">
-   <suppress>
-      <notes>only applies to tika-server &lt; 1.18</notes>
-      <gav regex="true">^org\.(apache\.tika:tika-(core|parsers)|gagravarr:vorbis-java-tika):.*$</gav>
-      <cve>CVE-2018-1335</cve>
-   </suppress>
 </suppressions>
diff --git a/src/java/overview.html b/src/java/overview.html
@@ -1,3 +1,19 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+ 
+    http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
 <html>
 <head>
    <title>Apache Nutch</title>

diff --git a/src/plugin/creativecommons/conf/crawl-urlfilter.txt b/src/plugin/creativecommons/conf/crawl-urlfilter.txt
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Creative Commnons crawl filter
 
 # Each non-comment, non-blank line contains a regular expression

diff --git a/src/plugin/creativecommons/conf/nutch-site.xml b/src/plugin/creativecommons/conf/nutch-site.xml
@@ -1,5 +1,21 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
 
 <!-- Creative Commons' Nutch configuration -->
 

diff --git a/src/plugin/creativecommons/data/anchor.html b/src/plugin/creativecommons/data/anchor.html
@@ -1,3 +1,19 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/1999/REC-html401-19991224/loose.dtd">
 <html>
 <head>

diff --git a/src/plugin/creativecommons/data/rdf.html b/src/plugin/creativecommons/data/rdf.html
@@ -1,3 +1,19 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 <html>
  <head>

diff --git a/src/plugin/creativecommons/data/rel.html b/src/plugin/creativecommons/data/rel.html
@@ -1,3 +1,19 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en"><head>
 </head><body>

diff --git a/src/plugin/creativecommons/ivy.xml b/src/plugin/creativecommons/ivy.xml
@@ -1,5 +1,4 @@
 <?xml version="1.0" ?>
-
 <!--
    Licensed to the Apache Software Foundation (ASF) under one or more
    contributor license agreements.  See the NOTICE file distributed with

diff --git a/src/plugin/exchange-jexl/README.md b/src/plugin/exchange-jexl/README.md
@@ -1,3 +1,20 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
 exchange-jexl plugin for Nutch  
 ==============================
 

diff --git a/src/plugin/exchange-jexl/ivy.xml b/src/plugin/exchange-jexl/ivy.xml
@@ -1,5 +1,4 @@
 <?xml version="1.0" ?>
-
 <!--
    Licensed to the Apache Software Foundation (ASF) under one or more
    contributor license agreements.  See the NOTICE file distributed with

diff --git a/src/plugin/feed/ivy.xml b/src/plugin/feed/ivy.xml
@@ -1,5 +1,4 @@
 <?xml version="1.0" ?>
-
 <!--
    Licensed to the Apache Software Foundation (ASF) under one or more
    contributor license agreements.  See the NOTICE file distributed with