[Security] Add configuration for running OWASP Dependency Check for a…

…ll modules (apache#10288) * Add owasp dependency check config Example report: mvn -Pskip-all,core-modules,owasp-dependency-check -pl distribution/server verify * Make it possible to skip shading * Make it possible to skip nar file creation * Skip license and rat checks in skip-all profile * Skip Docker too * Skip tests completely * Skip requirement of tar.gz dependency in presto-distribution * Add bash function for running the dependency check * Remove leftover from experiment
lhotari · Jun 3, 2021 · 612275d · 612275d
1 parent 7bf14b5
commit 612275d
Show file tree

Hide file tree

Showing 13 changed files with 199 additions and 13 deletions.
diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# shell function library for Pulsar CI builds
+
+# lists all available functions in this tool
+function ci_list_functions() {
+  declare -F | awk '{print $NF}' | sort | egrep '^ci_' | sed 's/^ci_//'
+}
+
+# prints thread dumps for all running JVMs
+# used in CI when a job gets cancelled because of a job timeout
+function ci_print_thread_dumps() {
+  for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do
+    echo "----------------------- pid $java_pid -----------------------"
+    cat /proc/$java_pid/cmdline | xargs -0 echo
+    jcmd $java_pid Thread.print -l
+    jcmd $java_pid GC.heap_info
+  done
+  return 0
+}
+
+function _ci_mvn() {
+  mvn -B -ntp "$@"
+}
+
+# runs OWASP Dependency Check for all projects
+function ci_dependency_check() {
+  _ci_mvn -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl '!pulsar-client-tools-test' "$@"
+}
+
+
+if [ -z "$1" ]; then
+  echo "usage: $0 [ci_tool_function_name]"
+  echo "Available ci tool functions:"
+  ci_list_functions
+  exit 1
+fi
+ci_function_name="ci_$1"
+shift
+
+if [[ "$(LC_ALL=C type -t $ci_function_name)" == "function" ]]; then
+  eval "$ci_function_name" "$@"
+else
+  echo "Invalid ci tool function"
+  echo "Available ci tool functions:"
+  ci_list_functions
+  exit 1
+fi
diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml
@@ -59,7 +59,7 @@
         <artifactId>maven-shade-plugin</artifactId>
         <executions>
           <execution>
-            <phase>package</phase>
+            <phase>${shadePluginPhase}</phase>
             <goals>
               <goal>shade</goal>
             </goals>

diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml
@@ -59,7 +59,7 @@
                 <artifactId>maven-shade-plugin</artifactId>
                 <executions>
                     <execution>
-                        <phase>package</phase>
+                        <phase>${shadePluginPhase}</phase>
                         <goals>
                             <goal>shade</goal>
                         </goals>

diff --git a/pom.xml b/pom.xml
@@ -93,7 +93,11 @@ flexible messaging model and an intuitive client API.</description>
     <docker.organization>apachepulsar</docker.organization>
 
     <!-- pin the protobuf-shaded version to make the pulsar build friendly to intellij -->
+    <skipSourceReleaseAssembly>false</skipSourceReleaseAssembly>
+    <skipBuildDistribution>false</skipBuildDistribution>
+    <shadePluginPhase>package</shadePluginPhase>
     <pulsar.protobuf.shaded.version>2.1.0-incubating</pulsar.protobuf.shaded.version>
+    <narPluginPhase>package</narPluginPhase>
 
     <!-- apache commons -->
     <commons-compress.version>1.19</commons-compress.version>
@@ -217,6 +221,7 @@ flexible messaging model and an intuitive client API.</description>
     <wagon-ssh-external.version>2.10</wagon-ssh-external.version>
     <os-maven-plugin.version>1.4.1.Final</os-maven-plugin.version>
     <jacoco-maven-plugin.version>0.8.3</jacoco-maven-plugin.version>
+    <dependency-check-maven.version>6.1.5</dependency-check-maven.version>
 
     <!-- Used to configure rename.netty.native. Libs -->
     <rename.netty.native.libs>rename-netty-native-libs.sh</rename.netty.native.libs>
@@ -1441,7 +1446,7 @@ flexible messaging model and an intuitive client API.</description>
           <executions>
             <execution>
               <id>default-nar</id>
-              <phase>package</phase>
+              <phase>${narPluginPhase}</phase>
               <goals>
                 <goal>nar</goal>
               </goals>
@@ -1611,8 +1616,8 @@ flexible messaging model and an intuitive client API.</description>
         <rename.netty.native.libs>rename-netty-native-libs.cmd</rename.netty.native.libs>
       </properties>
 
-    <!-- Primary Module profile -->
     </profile>
+    <!-- Primary Module profile -->
     <profile>
       <id>main</id>
       <activation>
@@ -1721,6 +1726,101 @@ flexible messaging model and an intuitive client API.</description>
       </modules>
     </profile>
 
+    <profile>
+      <id>skip-all</id>
+      <properties>
+        <maven.main.skip>true</maven.main.skip>
+        <maven.test.skip>true</maven.test.skip>
+        <skipSourceReleaseAssembly>true</skipSourceReleaseAssembly>
+        <skipBuildDistribution>true</skipBuildDistribution>
+        <spotbugs.skip>true</spotbugs.skip>
+        <license.skip>true</license.skip>
+        <rat.skip>true</rat.skip>
+        <assembly.skipAssembly>true</assembly.skipAssembly>
+        <shadePluginPhase>none</shadePluginPhase>
+        <narPluginPhase>none</narPluginPhase>
+        <skipDocker>true</skipDocker>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-compiler-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>default-testCompile</id>
+                <goals>
+                  <goal>testCompile</goal>
+                </goals>
+                <phase>none</phase>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>default-test</id>
+                <goals>
+                  <goal>test</goal>
+                </goals>
+                <phase>none</phase>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>owasp-dependency-check</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.owasp</groupId>
+            <artifactId>dependency-check-maven</artifactId>
+            <version>${dependency-check-maven.version}</version>
+            <configuration>
+              <msbuildAnalyzerEnabled>false</msbuildAnalyzerEnabled>
+              <nodeAnalyzerEnabled>false</nodeAnalyzerEnabled>
+              <yarnAuditAnalyzerEnabled>false</yarnAuditAnalyzerEnabled>
+              <pyDistributionAnalyzerEnabled>false</pyDistributionAnalyzerEnabled>
+              <pyPackageAnalyzerEnabled>false</pyPackageAnalyzerEnabled>
+              <pipAnalyzerEnabled>false</pipAnalyzerEnabled>
+              <pipfileAnalyzerEnabled>false</pipfileAnalyzerEnabled>
+              <retireJsAnalyzerEnabled>false</retireJsAnalyzerEnabled>
+              <msbuildAnalyzerEnabled>false</msbuildAnalyzerEnabled>
+              <mixAuditAnalyzerEnabled>false</mixAuditAnalyzerEnabled>
+              <nugetconfAnalyzerEnabled>false</nugetconfAnalyzerEnabled>
+              <assemblyAnalyzerEnabled>false</assemblyAnalyzerEnabled>
+            </configuration>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>aggregate</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+      <reporting>
+        <plugins>
+          <plugin>
+            <groupId>org.owasp</groupId>
+            <artifactId>dependency-check-maven</artifactId>
+            <version>${dependency-check-maven.version}</version>
+            <reportSets>
+              <reportSet>
+                <reports>
+                  <report>aggregate</report>
+                </reports>
+              </reportSet>
+            </reportSets>
+          </plugin>
+        </plugins>
+      </reporting>
+    </profile>
   </profiles>
 
   <repositories>

diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml
@@ -49,7 +49,7 @@
         <artifactId>maven-shade-plugin</artifactId>
         <executions>
           <execution>
-            <phase>package</phase>
+            <phase>${shadePluginPhase}</phase>
             <goals>
               <goal>shade</goal>
             </goals>

diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml
@@ -48,7 +48,7 @@
         <artifactId>maven-shade-plugin</artifactId>
         <executions>
           <execution>
-            <phase>package</phase>
+            <phase>${shadePluginPhase}</phase>
             <goals>
               <goal>shade</goal>
             </goals>

diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml
@@ -53,7 +53,7 @@
         <artifactId>maven-shade-plugin</artifactId>
         <executions>
           <execution>
-            <phase>package</phase>
+            <phase>${shadePluginPhase}</phase>
             <goals>
               <goal>shade</goal>
             </goals>

diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
@@ -88,7 +88,7 @@
         <artifactId>maven-shade-plugin</artifactId>
         <executions>
           <execution>
-            <phase>package</phase>
+            <phase>${shadePluginPhase}</phase>
             <goals>
               <goal>shade</goal>
             </goals>

diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml
@@ -92,7 +92,7 @@
         <artifactId>maven-shade-plugin</artifactId>
         <executions>
           <execution>
-            <phase>package</phase>
+            <phase>${shadePluginPhase}</phase>
             <goals>
               <goal>shade</goal>
             </goals>

diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml
@@ -48,7 +48,7 @@
                 <artifactId>maven-shade-plugin</artifactId>
                 <executions>
                     <execution>
-                        <phase>package</phase>
+                        <phase>${shadePluginPhase}</phase>
                         <goals>
                             <goal>shade</goal>
                         </goals>

diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml
@@ -159,7 +159,6 @@
             <type>tar.gz</type>
             <scope>provided</scope>
         </dependency>
-
         <dependency>
             <groupId>org.objenesis</groupId>
             <artifactId>objenesis</artifactId>
@@ -358,4 +357,25 @@
             </extension>
         </extensions>
     </build>
+
+  <profiles>
+    <profile>
+      <id>skipBuildDistributionDisabled</id>
+      <activation>
+        <property>
+          <name>skipBuildDistribution</name>
+          <value>false</value>
+        </property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>pulsar-presto-connector</artifactId>
+          <version>${project.version}</version>
+          <type>tar.gz</type>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
 </project>
diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml
@@ -110,7 +110,7 @@
                 <artifactId>maven-shade-plugin</artifactId>
                 <executions>
                     <execution>
-                        <phase>package</phase>
+                        <phase>${shadePluginPhase}</phase>
                         <goals>
                             <goal>shade</goal>
                         </goals>

diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml
@@ -61,7 +61,7 @@
                 <artifactId>maven-shade-plugin</artifactId>
                 <executions>
                     <execution>
-                        <phase>package</phase>
+                        <phase>${shadePluginPhase}</phase>
                         <goals>
                             <goal>shade</goal>
                         </goals>