From f4027aa990b50118e9cfac10fadd1b6dd6681f39 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Sun, 10 Jan 2016 19:59:01 -0800 Subject: [PATCH 01/10] [SPARK-12734][BUILD] Fix Netty exclusion and use Maven Enforcer to prevent future bugs Netty classes are published under multiple artifacts with different names, so our build needs to exclude the `io.netty:netty` and `org.jboss.netty:netty` versions of the Netty artifact. However, our existing exclusions were incomplete, leading to situations where duplicate Netty classes would wind up on the classpath and cause compile errors (or worse). This patch fixes the exclusion issue by adding more exclusions and uses Maven Enforcer's [banned dependencies](https://maven.apache.org/enforcer/enforcer-rules/bannedDependencies.html) rule to prevent these classes from accidentally being reintroduced. I also updated `dev/test-dependencies.sh` to run `mvn validate` so that the enforcer rules can run as part of pull request builds. /cc rxin srowen pwendell. I'd like to backport at least the exclusion portion of this fix to `branch-1.5` in order to fix the documentation publishing job, which fails nondeterministically due to incompatible versions of Netty classes taking precedence on the compile-time classpath. Author: Josh Rosen Author: Josh Rosen Closes #10672 from JoshRosen/enforce-netty-exclusions. --- dev/deps/spark-deps-hadoop-1 | 1 - dev/deps/spark-deps-hadoop-2.2 | 1 - dev/deps/spark-deps-hadoop-2.3 | 1 - dev/deps/spark-deps-hadoop-2.4 | 1 - dev/deps/spark-deps-hadoop-2.6 | 1 - dev/test-dependencies.sh | 16 +++------- examples/pom.xml | 4 +++ pom.xml | 57 +++++++++++++++++++++++++++++++++- 8 files changed, 64 insertions(+), 18 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-1 b/dev/deps/spark-deps-hadoop-1 index c0a0f32c64dc..13e75f1d712f 100644 --- a/dev/deps/spark-deps-hadoop-1 +++ b/dev/deps/spark-deps-hadoop-1 @@ -113,7 +113,6 @@ metrics-graphite-3.1.2.jar metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar -netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index 059b79e70c9a..da653b551dee 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -143,7 +143,6 @@ metrics-graphite-3.1.2.jar metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar -netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index 9cfaacdc0efd..103528c0d738 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -134,7 +134,6 @@ metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar mx4j-3.0.2.jar -netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index 0ee43898a7e5..72c8d295515f 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -135,7 +135,6 @@ metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar mx4j-3.0.2.jar -netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index fb26d452fee1..2cf50a338013 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -142,7 +142,6 @@ metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar mx4j-3.0.2.jar -netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index 129741f13982..13185841b943 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -76,18 +76,10 @@ for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do HADOOP_MODULE_PROFILES="-Phive-thriftserver -Pyarn -Phive" fi echo "Performing Maven install for $HADOOP_PROFILE" - $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE jar:jar install:install -q \ - -pl '!assembly' \ - -pl '!examples' \ - -pl '!external/flume-assembly' \ - -pl '!external/kafka-assembly' \ - -pl '!external/twitter' \ - -pl '!external/flume' \ - -pl '!external/mqtt' \ - -pl '!external/mqtt-assembly' \ - -pl '!external/zeromq' \ - -pl '!external/kafka' \ - -DskipTests + $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE jar:jar jar:test-jar install:install -q + + echo "Performing Maven validate for $HADOOP_PROFILE" + $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE validate -q echo "Generating dependency manifest for $HADOOP_PROFILE" mkdir -p dev/pr-deps diff --git a/examples/pom.xml b/examples/pom.xml index 0d87f370a5cb..1ed0b3b0929d 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -117,6 +117,10 @@ org.jruby jruby-complete + + io.netty + netty + diff --git a/pom.xml b/pom.xml index c751298c9481..eb143a77472a 100644 --- a/pom.xml +++ b/pom.xml @@ -577,6 +577,12 @@ ${akka.group} akka-remote_${scala.binary.version} ${akka.version} + + + io.netty + netty + + ${akka.group} @@ -774,6 +780,10 @@ org.jboss.netty netty + + io.netty + netty + @@ -834,6 +844,10 @@ junit junit + + io.netty + netty + @@ -934,6 +948,10 @@ org.jboss.netty netty + + io.netty + netty + commons-logging commons-logging @@ -958,6 +976,10 @@ org.jboss.netty netty + + io.netty + netty + javax.servlet servlet-api @@ -987,6 +1009,10 @@ org.jboss.netty netty + + io.netty + netty + javax.servlet servlet-api @@ -1015,6 +1041,10 @@ org.jboss.netty netty + + io.netty + netty + javax.servlet servlet-api @@ -1043,6 +1073,10 @@ org.jboss.netty netty + + io.netty + netty + javax.servlet servlet-api @@ -1058,6 +1092,16 @@ zookeeper ${zookeeper.version} ${hadoop.deps.scope} + + + org.jboss.netty + netty + + + io.netty + netty + + org.codehaus.jackson @@ -1774,7 +1818,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 1.4 + 1.4.1 enforce-versions @@ -1789,6 +1833,17 @@ ${java.version} + + + io.netty:netty + org.jboss.netty + + + + io.netty:netty:3.4.0.Final:*:test + + true + From 26df460456721be65eb833cad686606189bccc94 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Sun, 10 Jan 2016 22:28:09 -0800 Subject: [PATCH 02/10] Update pom.xml --- examples/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/pom.xml b/examples/pom.xml index 1ed0b3b0929d..0d87f370a5cb 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -117,10 +117,6 @@ org.jruby jruby-complete - - io.netty - netty - From 9114f8cd62f5f2052973f6c0dea7f39deb65ae88 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Sun, 10 Jan 2016 22:28:58 -0800 Subject: [PATCH 03/10] Update pom.xml --- pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pom.xml b/pom.xml index eb143a77472a..658dc0997d5d 100644 --- a/pom.xml +++ b/pom.xml @@ -948,10 +948,6 @@ org.jboss.netty netty - - io.netty - netty - commons-logging commons-logging From 6fee72f56b3c1b7401460b805e96737fa6800fd3 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Sun, 10 Jan 2016 22:30:04 -0800 Subject: [PATCH 04/10] Update pom.xml --- pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pom.xml b/pom.xml index 658dc0997d5d..4380df3a92d5 100644 --- a/pom.xml +++ b/pom.xml @@ -1093,10 +1093,6 @@ org.jboss.netty netty - - io.netty - netty - From dc94896c12da3a36898b5f0be4a95f76ffc6127b Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Sun, 10 Jan 2016 22:59:04 -0800 Subject: [PATCH 05/10] Update pom.xml --- pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pom.xml b/pom.xml index 4380df3a92d5..81448855cb56 100644 --- a/pom.xml +++ b/pom.xml @@ -1830,10 +1830,6 @@ io.netty:netty org.jboss.netty - - - io.netty:netty:3.4.0.Final:*:test - true From 70f89b6742418aa4f0506bf0eb5be96cb2d6e383 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Sun, 10 Jan 2016 22:20:16 -0800 Subject: [PATCH 06/10] Try to fix tests. --- dev/deps/spark-deps-hadoop-1 | 1 + dev/deps/spark-deps-hadoop-2.2 | 1 + dev/deps/spark-deps-hadoop-2.3 | 1 + dev/deps/spark-deps-hadoop-2.4 | 1 + dev/deps/spark-deps-hadoop-2.6 | 1 + pom.xml | 38 +++++++--------------------------- 6 files changed, 12 insertions(+), 31 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-1 b/dev/deps/spark-deps-hadoop-1 index 13e75f1d712f..c0a0f32c64dc 100644 --- a/dev/deps/spark-deps-hadoop-1 +++ b/dev/deps/spark-deps-hadoop-1 @@ -113,6 +113,7 @@ metrics-graphite-3.1.2.jar metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar +netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index da653b551dee..059b79e70c9a 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -143,6 +143,7 @@ metrics-graphite-3.1.2.jar metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar +netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index 103528c0d738..9cfaacdc0efd 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -134,6 +134,7 @@ metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar mx4j-3.0.2.jar +netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index 72c8d295515f..0ee43898a7e5 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -135,6 +135,7 @@ metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar mx4j-3.0.2.jar +netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 2cf50a338013..fb26d452fee1 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -142,6 +142,7 @@ metrics-json-3.1.2.jar metrics-jvm-3.1.2.jar minlog-1.2.jar mx4j-3.0.2.jar +netty-3.8.0.Final.jar netty-all-4.0.29.Final.jar objenesis-1.2.jar opencsv-2.3.jar diff --git a/pom.xml b/pom.xml index 81448855cb56..5f60dff540d1 100644 --- a/pom.xml +++ b/pom.xml @@ -577,12 +577,6 @@ ${akka.group} akka-remote_${scala.binary.version} ${akka.version} - - - io.netty - netty - - ${akka.group} @@ -780,10 +774,6 @@ org.jboss.netty netty - - io.netty - netty - @@ -844,10 +834,6 @@ junit junit - - io.netty - netty - @@ -972,10 +958,6 @@ org.jboss.netty netty - - io.netty - netty - javax.servlet servlet-api @@ -1005,10 +987,6 @@ org.jboss.netty netty - - io.netty - netty - javax.servlet servlet-api @@ -1037,10 +1015,6 @@ org.jboss.netty netty - - io.netty - netty - javax.servlet servlet-api @@ -1069,10 +1043,6 @@ org.jboss.netty netty - - io.netty - netty - javax.servlet servlet-api @@ -1827,7 +1797,13 @@ - io.netty:netty + org.jboss.netty true From 7e4255340dfa377442fe5de9586deb52719d053e Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Sun, 10 Jan 2016 23:22:16 -0800 Subject: [PATCH 07/10] Update pom.xml --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 5f60dff540d1..d8bb09ae31b4 100644 --- a/pom.xml +++ b/pom.xml @@ -1798,9 +1798,9 @@ From 19a1b8838dd7fd33d1067a7365428631ed93ab27 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Sun, 10 Jan 2016 23:23:28 -0800 Subject: [PATCH 08/10] Update pom.xml --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d8bb09ae31b4..ad31e5f371eb 100644 --- a/pom.xml +++ b/pom.xml @@ -1798,9 +1798,9 @@ From 0225afadc274e5b896790ce307689b368a959893 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Sun, 10 Jan 2016 23:24:37 -0800 Subject: [PATCH 09/10] Update pom.xml --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ad31e5f371eb..ef00c70231c6 100644 --- a/pom.xml +++ b/pom.xml @@ -1799,9 +1799,9 @@ org.jboss.netty From c719fef9513aa31a59a5588cd955119ef3bcca9a Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Mon, 11 Jan 2016 00:25:58 -0800 Subject: [PATCH 10/10] Update test-dependencies.sh --- dev/test-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index 13185841b943..47ae15b52331 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -79,7 +79,7 @@ for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE jar:jar jar:test-jar install:install -q echo "Performing Maven validate for $HADOOP_PROFILE" - $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE validate -q + $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE validate -q echo "Generating dependency manifest for $HADOOP_PROFILE" mkdir -p dev/pr-deps