From ec91ce3c405123818a4c56ef361d9cc82951677d Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Wed, 29 Apr 2015 13:28:09 -0430 Subject: [PATCH 01/19] - Updated protobuf-java version of com.google.protobuf dependancy to fix blocking error when connecting to HDFS via the Hadoop Cloudera HDFS CDH5 (fix for 2.5.0-cdh5.3.3 version) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c85c5feeaf383..81665ef90ce85 100644 --- a/pom.xml +++ b/pom.xml @@ -509,7 +509,7 @@ com.google.protobuf protobuf-java - ${protobuf.version} + 2.5.0 ${hadoop.deps.scope} From 660decce9d3c2300aee493b605da0da8a74b3ea6 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Wed, 29 Apr 2015 14:46:04 -0430 Subject: [PATCH 02/19] - Updated Hadoop dependencies due to inconsistency in the versions. Now the global properties are the ones used by the hadoop-2.2 profile, and the profile was set to empty but kept for backwards compatibility reasons --- pom.xml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 81665ef90ce85..aada6cd7201ca 100644 --- a/pom.xml +++ b/pom.xml @@ -122,9 +122,9 @@ 1.7.10 1.2.17 2.2.0 - 2.4.1 + 2.5.0 ${hadoop.version} - 0.98.7-hadoop1 + 0.98.7-hadoop2 hbase 1.4.0 3.4.5 @@ -155,7 +155,7 @@ ${scala.version} org.scala-lang 3.6.3 - 1.8.8 + 1.9.13 2.4.4 1.1.1.7 1.1.2 @@ -1629,16 +1629,20 @@ - hadoop-2.2 + hadoop-1 - 2.2.0 - 2.5.0 - 0.98.7-hadoop2 - hadoop2 - 1.9.13 + 1.2.0 + 2.4.1 + 0.98.7-hadoop1 + hadoop1 + 1.8.8 + + hadoop-2.2 + + hadoop-2.3 From 7e9955df29b5d5c9cda950636d51da753e6d17ea Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Wed, 29 Apr 2015 15:05:08 -0430 Subject: [PATCH 03/19] - Updated Hadoop dependencies due to inconsistency in the versions. Now the global properties are the ones used by the hadoop-2.2 profile, and the profile was set to empty but kept for backwards compatibility reasons --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aada6cd7201ca..af8edca2567ad 100644 --- a/pom.xml +++ b/pom.xml @@ -509,7 +509,7 @@ com.google.protobuf protobuf-java - 2.5.0 + ${protobuf.version} ${hadoop.deps.scope} From 6b4bfafbe4f98c92ac2fe7aeb5f36a37d27a9678 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Thu, 30 Apr 2015 17:11:08 -0430 Subject: [PATCH 04/19] - Cleanup in hadoop-2.x profiles since they contained mostly redundant stuff. --- pom.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index af8edca2567ad..ee213665ba8e4 100644 --- a/pom.xml +++ b/pom.xml @@ -1647,12 +1647,12 @@ hadoop-2.3 2.3.0 - 2.5.0 + ${protobuf.version} 0.9.3 - 0.98.7-hadoop2 + ${hbase.version} 3.1.1 hadoop2 - 1.9.13 + ${codehaus.jackson.version} @@ -1660,12 +1660,12 @@ hadoop-2.4 2.4.0 - 2.5.0 + ${protobuf.version} 0.9.3 - 0.98.7-hadoop2 + ${hbase.version} 3.1.1 hadoop2 - 1.9.13 + ${codehaus.jackson.version} From 13542929c9cb3ddfec31bbb794e490b44c273df4 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Thu, 30 Apr 2015 17:43:50 -0430 Subject: [PATCH 05/19] - Fixed hadoop-1 version to match jenkins build profile in hadoop1.0 tests and documentation --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ee213665ba8e4..81b72b1ca5ce4 100644 --- a/pom.xml +++ b/pom.xml @@ -1631,7 +1631,7 @@ hadoop-1 - 1.2.0 + 1.0.4 2.4.1 0.98.7-hadoop1 hadoop1 From 287fa2ffc31bb0c9eaf5daf80825ff0093f3f20d Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Thu, 30 Apr 2015 17:47:44 -0430 Subject: [PATCH 06/19] - Updated documentation about specifying the hadoop version in building-spark. Now is clear that Spark will build against Hadoop 2.2.0 by default. - Added Cloudera CDH 5.3.3 without MapReduce example in the building-spark doc. --- docs/building-spark.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/building-spark.md b/docs/building-spark.md index ea79c5bc276d3..01f96755c8752 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -59,7 +59,7 @@ You can fix this by setting the `MAVEN_OPTS` variable as discussed before. # Specifying the Hadoop Version -Because HDFS is not protocol-compatible across versions, if you want to read from HDFS, you'll need to build Spark against the specific HDFS version in your environment. You can do this through the "hadoop.version" property. If unset, Spark will build against Hadoop 1.0.4 by default. Note that certain build profiles are required for particular Hadoop versions: +Because HDFS is not protocol-compatible across versions, if you want to read from HDFS, you'll need to build Spark against the specific HDFS version in your environment. You can do this through the "hadoop.version" property. If unset, Spark will build against Hadoop 2.2.0 by default. Note that certain build profiles are required for particular Hadoop versions: @@ -67,8 +67,8 @@ Because HDFS is not protocol-compatible across versions, if you want to read fro - - + + @@ -83,6 +83,10 @@ mvn -Dhadoop.version=1.2.1 -DskipTests clean package # Cloudera CDH 4.2.0 with MapReduce v1 mvn -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -DskipTests clean package +#Cloudera CDH 5.3.3 without MapReduce + +mvn -Dhadoop.version=2.5.0-cdh5.3.3 -DskipTests clean package + # Apache Hadoop 0.23.x mvn -Phadoop-0.23 -Dhadoop.version=0.23.7 -DskipTests clean package {% endhighlight %} From 70b8344dcad8f6de71bd6356cd6eec375211fdb3 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Thu, 30 Apr 2015 18:27:16 -0430 Subject: [PATCH 07/19] - Fixed typo in the make-distribution.sh file and added hadoop-1 in the Related profiles --- make-distribution.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make-distribution.sh b/make-distribution.sh index cb65932b4abc0..edb277f2b1771 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -57,7 +57,7 @@ while (( "$#" )); do --hadoop) echo "Error: '--hadoop' is no longer supported:" echo "Error: use Maven profiles and options -Dhadoop.version and -Dyarn.version instead." - echo "Error: Related profiles include hadoop-0.23, hdaoop-2.2, hadoop-2.3 and hadoop-2.4." + echo "Error: Related profiles include hadoop-0.23, hadoop-1, hadoop-2.2, hadoop-2.3 and hadoop-2.4." exit_with_usage ;; --with-yarn) From 88a8b88a13a02cbde04792cb63e3c6a81407d915 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Fri, 1 May 2015 12:18:27 -0430 Subject: [PATCH 08/19] - Simplified Hadoop profiles due to new setting of global properties in the pom.xml file - Added comment to specify that the hadoop-2.2 profile is now the default hadoop profile in the pom.xml file - Erased hadoop-2.2 from related hadoop profiles now that is a no-op in the make-distribution.sh file --- make-distribution.sh | 2 +- pom.xml | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/make-distribution.sh b/make-distribution.sh index edb277f2b1771..60d4805d00876 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -57,7 +57,7 @@ while (( "$#" )); do --hadoop) echo "Error: '--hadoop' is no longer supported:" echo "Error: use Maven profiles and options -Dhadoop.version and -Dyarn.version instead." - echo "Error: Related profiles include hadoop-0.23, hadoop-1, hadoop-2.2, hadoop-2.3 and hadoop-2.4." + echo "Error: Related profiles include hadoop-0.23, hadoop-1, hadoop-2.3 and hadoop-2.4." exit_with_usage ;; --with-yarn) diff --git a/pom.xml b/pom.xml index 81b72b1ca5ce4..813b65c8fc1a5 100644 --- a/pom.xml +++ b/pom.xml @@ -1641,18 +1641,16 @@ hadoop-2.2 + hadoop-2.3 2.3.0 - ${protobuf.version} 0.9.3 - ${hbase.version} 3.1.1 hadoop2 - ${codehaus.jackson.version} @@ -1660,12 +1658,9 @@ hadoop-2.4 2.4.0 - ${protobuf.version} 0.9.3 - ${hbase.version} 3.1.1 hadoop2 - ${codehaus.jackson.version} From 199f40b1733015a414eb928b2090f3bf4d0b7a7e Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Fri, 1 May 2015 16:14:30 -0430 Subject: [PATCH 09/19] - Erased unnecessary CDH5-specific note in docs/building-spark.md - Remove example of instance -Phadoop-2.2 -Dhadoop.version=2.2.0 in docs/building-spark.md - Enabled hadoop-2.2 profile when the Hadoop version is 2.2.0, which is now the default .Added comment in the yarn/pom.xml to specify that. --- docs/building-spark.md | 6 ------ yarn/pom.xml | 10 ++++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/building-spark.md b/docs/building-spark.md index 01f96755c8752..5389337f9833b 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -83,10 +83,6 @@ mvn -Dhadoop.version=1.2.1 -DskipTests clean package # Cloudera CDH 4.2.0 with MapReduce v1 mvn -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -DskipTests clean package -#Cloudera CDH 5.3.3 without MapReduce - -mvn -Dhadoop.version=2.5.0-cdh5.3.3 -DskipTests clean package - # Apache Hadoop 0.23.x mvn -Phadoop-0.23 -Dhadoop.version=0.23.7 -DskipTests clean package {% endhighlight %} @@ -96,8 +92,6 @@ You can enable the "yarn" profile and optionally set the "yarn.version" property Examples: {% highlight bash %} -# Apache Hadoop 2.2.X -mvn -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 -DskipTests clean package # Apache Hadoop 2.3.X mvn -Pyarn -Phadoop-2.3 -Dhadoop.version=2.3.0 -DskipTests clean package diff --git a/yarn/pom.xml b/yarn/pom.xml index 7c8c3613e7a05..0e3f104efe5b3 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -109,6 +109,16 @@ 1.9 + + + + hadoop.version + 2.2.0 + + org.mortbay.jetty From a6507792cc12fc03139be825357f22329773c823 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Fri, 1 May 2015 16:20:46 -0430 Subject: [PATCH 10/19] - Default value of avro.mapred.classifier has been set to hadoop2 in pom.xml - Cleaned up hadoop-2.3 and 2.4 profiles due to change in the default set in avro.mapred.classifier in pom.xml --- pom.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 813b65c8fc1a5..e5062252fa462 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 2.0.8 3.1.0 1.7.7 - + hadoop2 0.7.1 1.8.3 1.1.0 @@ -1650,7 +1650,6 @@ 2.3.0 0.9.3 3.1.1 - hadoop2 @@ -1660,7 +1659,6 @@ 2.4.0 0.9.3 3.1.1 - hadoop2 From 0470587ad7af93041e25dcb07954b835d9508a10 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Fri, 1 May 2015 16:36:52 -0430 Subject: [PATCH 11/19] - Erased unnecessary instance of -Phadoop-2.2 -Dhadoop.version=2.2.0 in create-release.sh - Updated how the releases are made in the create-release.sh no that the default hadoop version is the 2.2.0 - Erased unnecessary instance of -Phadoop-2.2 -Dhadoop.version=2.2.0 in scalastyle - Erased unnecessary instance of -Phadoop-2.2 -Dhadoop.version=2.2.0 in run-tests - Better example given in the hadoop-third-party-distributions.md now that the default hadoop version is 2.2.0 --- dev/create-release/create-release.sh | 12 ++++++------ dev/run-tests | 2 +- dev/scalastyle | 4 ++-- docs/hadoop-third-party-distributions.md | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dev/create-release/create-release.sh b/dev/create-release/create-release.sh index 3dbb35f7054a2..16078e0923734 100755 --- a/dev/create-release/create-release.sh +++ b/dev/create-release/create-release.sh @@ -118,14 +118,14 @@ if [[ ! "$@" =~ --skip-publish ]]; then rm -rf $SPARK_REPO - build/mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \ - -Pyarn -Phive -Phive-thriftserver -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \ + build/mvn -DskipTests -Dyarn.version=2.2.0 \ + -Pyarn -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Pkinesis-asl \ clean install ./dev/change-version-to-2.11.sh - build/mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \ - -Dscala-2.11 -Pyarn -Phive -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \ + build/mvn -DskipTests -Dyarn.version=2.2.0 \ + -Dscala-2.11 -Pyarn -Phive -Pspark-ganglia-lgpl -Pkinesis-asl \ clean install ./dev/change-version-to-2.10.sh @@ -228,8 +228,8 @@ if [[ ! "$@" =~ --skip-package ]]; then # We increment the Zinc port each time to avoid OOM's and other craziness if multiple builds # share the same Zinc server. - make_binary_release "hadoop1" "-Phive -Phive-thriftserver -Dhadoop.version=1.0.4" "3030" & - make_binary_release "hadoop1-scala2.11" "-Phive -Dscala-2.11" "3031" & + make_binary_release "hadoop2" "-Phive -Phive-thriftserver -Dhadoop.version=2.2.0" "3030" & + make_binary_release "hadoop2-scala2.11" "-Phive -Dscala-2.11" "3031" & make_binary_release "cdh4" "-Phive -Phive-thriftserver -Dhadoop.version=2.0.0-mr1-cdh4.2.0" "3032" & make_binary_release "hadoop2.3" "-Phadoop-2.3 -Phive -Phive-thriftserver -Pyarn" "3033" & make_binary_release "hadoop2.4" "-Phadoop-2.4 -Phive -Phive-thriftserver -Pyarn" "3034" & diff --git a/dev/run-tests b/dev/run-tests index 861d1671182c2..05eab2f77c047 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -44,7 +44,7 @@ function handle_error () { elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.0" ]; then export SBT_MAVEN_PROFILES_ARGS="-Dhadoop.version=2.0.0-mr1-cdh4.1.1" elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.2" ]; then - export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0" + export SBT_MAVEN_PROFILES_ARGS="-Pyarn" elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.3" ]; then export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Phadoop-2.3 -Dhadoop.version=2.3.0" fi diff --git a/dev/scalastyle b/dev/scalastyle index 4e03f89ed5d5d..bfe9b1fb7df51 100755 --- a/dev/scalastyle +++ b/dev/scalastyle @@ -20,8 +20,8 @@ echo -e "q\n" | build/sbt -Phive -Phive-thriftserver scalastyle > scalastyle.txt echo -e "q\n" | build/sbt -Phive -Phive-thriftserver test:scalastyle >> scalastyle.txt # Check style with YARN built too -echo -e "q\n" | build/sbt -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 scalastyle >> scalastyle.txt -echo -e "q\n" | build/sbt -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 test:scalastyle >> scalastyle.txt +echo -e "q\n" | build/sbt -Pyarn scalastyle >> scalastyle.txt +echo -e "q\n" | build/sbt -Pyarn test:scalastyle >> scalastyle.txt ERRORS=$(cat scalastyle.txt | awk '{if($1~/error/)print}') rm scalastyle.txt diff --git a/docs/hadoop-third-party-distributions.md b/docs/hadoop-third-party-distributions.md index 87dcc58feb494..92722936659a8 100644 --- a/docs/hadoop-third-party-distributions.md +++ b/docs/hadoop-third-party-distributions.md @@ -14,7 +14,7 @@ property. For certain versions, you will need to specify additional profiles. Fo see the guide on [building with maven](building-spark.html#specifying-the-hadoop-version): mvn -Dhadoop.version=1.0.4 -DskipTests clean package - mvn -Phadoop-2.2 -Dhadoop.version=2.2.0 -DskipTests clean package + mvn -Phadoop-2.3 -Dhadoop.version=2.3.0 -DskipTests clean package The table below lists the corresponding `hadoop.version` code for each CDH/HDP release. Note that some Hadoop releases are binary compatible across client versions. This means the pre-built Spark From fda6a51986ed4a656d37539502cd4684c46c8cfe Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Fri, 1 May 2015 17:12:04 -0430 Subject: [PATCH 12/19] - Updated hadoop1 releases in create-release.sh due to changes in the default hadoop version set - Erased unnecessary instance of -Dyarn.version=2.2.0 in create-release.sh - Prettify comment in yarn/pom.xml --- dev/create-release/create-release.sh | 12 ++++++------ yarn/pom.xml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dev/create-release/create-release.sh b/dev/create-release/create-release.sh index 16078e0923734..fafb5c9887d61 100755 --- a/dev/create-release/create-release.sh +++ b/dev/create-release/create-release.sh @@ -118,14 +118,14 @@ if [[ ! "$@" =~ --skip-publish ]]; then rm -rf $SPARK_REPO - build/mvn -DskipTests -Dyarn.version=2.2.0 \ - -Pyarn -Phive -Phive-thriftserver -Pspark-ganglia-lgpl -Pkinesis-asl \ + build/mvn -DskipTests -Pyarn -Phive \ + -Phive-thriftserver -Pspark-ganglia-lgpl -Pkinesis-asl \ clean install ./dev/change-version-to-2.11.sh - build/mvn -DskipTests -Dyarn.version=2.2.0 \ - -Dscala-2.11 -Pyarn -Phive -Pspark-ganglia-lgpl -Pkinesis-asl \ + build/mvn -DskipTests -Pyarn -Phive \ + -Dscala-2.11 -Pspark-ganglia-lgpl -Pkinesis-asl \ clean install ./dev/change-version-to-2.10.sh @@ -228,8 +228,8 @@ if [[ ! "$@" =~ --skip-package ]]; then # We increment the Zinc port each time to avoid OOM's and other craziness if multiple builds # share the same Zinc server. - make_binary_release "hadoop2" "-Phive -Phive-thriftserver -Dhadoop.version=2.2.0" "3030" & - make_binary_release "hadoop2-scala2.11" "-Phive -Dscala-2.11" "3031" & + make_binary_release "hadoop1" "-Phadoop-1 -Phive -Phive-thriftserver" "3030" & + make_binary_release "hadoop1-scala2.11" "-Phadoop-1 -Phive -Dscala-2.11" "3031" & make_binary_release "cdh4" "-Phive -Phive-thriftserver -Dhadoop.version=2.0.0-mr1-cdh4.2.0" "3032" & make_binary_release "hadoop2.3" "-Phadoop-2.3 -Phive -Phive-thriftserver -Pyarn" "3033" & make_binary_release "hadoop2.4" "-Phadoop-2.4 -Phive -Phive-thriftserver -Pyarn" "3034" & diff --git a/yarn/pom.xml b/yarn/pom.xml index 0e3f104efe5b3..2055c27532c70 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -111,7 +111,7 @@ From 668d126bc86e5a71f4a724fd36617e7e5a7ff8bc Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Fri, 1 May 2015 18:49:14 -0430 Subject: [PATCH 13/19] - Moved and sections of the hadoop-2.2 profile in the YARN POM to the YARN profile in the root POM - Erased unnecessary hadoop-2.2 profile from the YARN POM --- pom.xml | 51 ++++++++++++++++++++++++++++++++++++++++++ yarn/pom.xml | 63 +--------------------------------------------------- 2 files changed, 52 insertions(+), 62 deletions(-) diff --git a/pom.xml b/pom.xml index e5062252fa462..e127705bfd6ed 100644 --- a/pom.xml +++ b/pom.xml @@ -1668,6 +1668,57 @@ yarn network/yarn + + 1.9 + + + + + hadoop.version + 2.2.0 + + + + + org.mortbay.jetty + jetty + 6.1.26 + + + org.mortbay.jetty + servlet-api + + + test + + + com.sun.jersey + jersey-core + ${jersey.version} + test + + + com.sun.jersey + jersey-json + ${jersey.version} + test + + + stax + stax-api + + + + + com.sun.jersey + jersey-server + ${jersey.version} + test + + diff --git a/yarn/pom.xml b/yarn/pom.xml index 2055c27532c70..cc9c50cb56efd 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -98,68 +98,7 @@ test - - - - - hadoop-2.2 - - 1.9 - - - - - hadoop.version - 2.2.0 - - - - - org.mortbay.jetty - jetty - 6.1.26 - - - org.mortbay.jetty - servlet-api - - - test - - - com.sun.jersey - jersey-core - ${jersey.version} - test - - - com.sun.jersey - jersey-json - ${jersey.version} - test - - - stax - stax-api - - - - - com.sun.jersey - jersey-server - ${jersey.version} - test - - - - - + target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes From 93f7624224aed23697eb4bad589dcc256d391c19 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Fri, 1 May 2015 19:02:37 -0430 Subject: [PATCH 14/19] - Deleted unnecessary comments and tag on the YARN profile in the main POM --- pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pom.xml b/pom.xml index e127705bfd6ed..0c7e0b98193e5 100644 --- a/pom.xml +++ b/pom.xml @@ -1671,16 +1671,6 @@ 1.9 - - - - hadoop.version - 2.2.0 - - org.mortbay.jetty From 83dc332aea0cfa95f79cb76d46c6d8e15d6b8377 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Fri, 1 May 2015 19:53:28 -0430 Subject: [PATCH 15/19] - Cleaned up the main POM concerning the yarn profile - Erased hadoop-2.2 profile from yarn/pom.xml and its content was integrated into yarn/pom.xml --- pom.xml | 41 ----------------------------------------- yarn/pom.xml | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/pom.xml b/pom.xml index 0c7e0b98193e5..e5062252fa462 100644 --- a/pom.xml +++ b/pom.xml @@ -1668,47 +1668,6 @@ yarn network/yarn - - 1.9 - - - - org.mortbay.jetty - jetty - 6.1.26 - - - org.mortbay.jetty - servlet-api - - - test - - - com.sun.jersey - jersey-core - ${jersey.version} - test - - - com.sun.jersey - jersey-json - ${jersey.version} - test - - - stax - stax-api - - - - - com.sun.jersey - jersey-server - ${jersey.version} - test - - diff --git a/yarn/pom.xml b/yarn/pom.xml index cc9c50cb56efd..76f24d3fff9a2 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -30,6 +30,7 @@ Spark Project YARN yarn + 1.9 @@ -97,6 +98,42 @@ mockito-all test + + org.mortbay.jetty + jetty + 6.1.26 + + + org.mortbay.jetty + servlet-api + + + test + + + com.sun.jersey + jersey-core + ${jersey.version} + test + + + com.sun.jersey + jersey-json + ${jersey.version} + test + + + stax + stax-api + + + + + com.sun.jersey + jersey-server + ${jersey.version} + test + From cbb93e80dc4a79a5219ec6856634b2ac7e9a91a3 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Fri, 1 May 2015 20:55:58 -0430 Subject: [PATCH 16/19] - Added comment related to SPARK-3710 about hadoop-yarn-server-tests in Hadoop 2.2 that fails to pull some needed dependencies --- yarn/pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/yarn/pom.xml b/yarn/pom.xml index 76f24d3fff9a2..00d219f836708 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -86,7 +86,12 @@ jetty-servlet - + + + org.apache.hadoop hadoop-yarn-server-tests From 31bdafad21674fe5bc582fa678753454b04026ad Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Sat, 2 May 2015 10:57:41 -0430 Subject: [PATCH 17/19] - Added missing instances in -Phadoop-1 in create-release.sh, run-tests and in the building-spark documentation --- dev/create-release/create-release.sh | 2 +- dev/run-tests | 4 ++-- docs/building-spark.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/create-release/create-release.sh b/dev/create-release/create-release.sh index fafb5c9887d61..4c1708da1b9fd 100755 --- a/dev/create-release/create-release.sh +++ b/dev/create-release/create-release.sh @@ -230,7 +230,7 @@ if [[ ! "$@" =~ --skip-package ]]; then # share the same Zinc server. make_binary_release "hadoop1" "-Phadoop-1 -Phive -Phive-thriftserver" "3030" & make_binary_release "hadoop1-scala2.11" "-Phadoop-1 -Phive -Dscala-2.11" "3031" & - make_binary_release "cdh4" "-Phive -Phive-thriftserver -Dhadoop.version=2.0.0-mr1-cdh4.2.0" "3032" & + make_binary_release "cdh4" "-Phadoop-1 -Phive -Phive-thriftserver -Dhadoop.version=2.0.0-mr1-cdh4.2.0" "3032" & make_binary_release "hadoop2.3" "-Phadoop-2.3 -Phive -Phive-thriftserver -Pyarn" "3033" & make_binary_release "hadoop2.4" "-Phadoop-2.4 -Phive -Phive-thriftserver -Pyarn" "3034" & make_binary_release "mapr3" "-Pmapr3 -Phive -Phive-thriftserver" "3035" & diff --git a/dev/run-tests b/dev/run-tests index 05eab2f77c047..7eb6448e194ee 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -40,9 +40,9 @@ function handle_error () { { if [ -n "$AMPLAB_JENKINS_BUILD_PROFILE" ]; then if [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop1.0" ]; then - export SBT_MAVEN_PROFILES_ARGS="-Dhadoop.version=1.0.4" + export SBT_MAVEN_PROFILES_ARGS="-Phadoop-1 -Dhadoop.version=1.0.4" elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.0" ]; then - export SBT_MAVEN_PROFILES_ARGS="-Dhadoop.version=2.0.0-mr1-cdh4.1.1" + export SBT_MAVEN_PROFILES_ARGS="-Phadoop-1 -Dhadoop.version=2.0.0-mr1-cdh4.1.1" elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.2" ]; then export SBT_MAVEN_PROFILES_ARGS="-Pyarn" elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.3" ]; then diff --git a/docs/building-spark.md b/docs/building-spark.md index 5389337f9833b..f5d7710466be5 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -78,10 +78,10 @@ For Apache Hadoop versions 1.x, Cloudera CDH "mr1" distributions, and other Hado {% highlight bash %} # Apache Hadoop 1.2.1 -mvn -Dhadoop.version=1.2.1 -DskipTests clean package +mvn -Dhadoop.version=1.2.1 -Phadoop-1 -DskipTests clean package # Cloudera CDH 4.2.0 with MapReduce v1 -mvn -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -DskipTests clean package +mvn -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -Phadoop-1 -DskipTests clean package # Apache Hadoop 0.23.x mvn -Phadoop-0.23 -Dhadoop.version=0.23.7 -DskipTests clean package From 379f50d63629318d1d0689a155a201a220aa54fe Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Wed, 13 May 2015 05:21:14 -0430 Subject: [PATCH 18/19] - Added instances of -Phadoop-2.2 in create-release.sh, run-tests, scalastyle and building-spark.md - Reconstructed docs to not ask users to rely on default behavior --- dev/create-release/create-release.sh | 2 +- dev/run-tests | 2 +- dev/scalastyle | 4 ++-- docs/building-spark.md | 3 +++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dev/create-release/create-release.sh b/dev/create-release/create-release.sh index 4c1708da1b9fd..bc6c48e71bd9f 100755 --- a/dev/create-release/create-release.sh +++ b/dev/create-release/create-release.sh @@ -119,7 +119,7 @@ if [[ ! "$@" =~ --skip-publish ]]; then rm -rf $SPARK_REPO build/mvn -DskipTests -Pyarn -Phive \ - -Phive-thriftserver -Pspark-ganglia-lgpl -Pkinesis-asl \ + -Phive-thriftserver -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \ clean install ./dev/change-version-to-2.11.sh diff --git a/dev/run-tests b/dev/run-tests index 29fc53e837a7b..44d802782c4a4 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -44,7 +44,7 @@ function handle_error () { elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.0" ]; then export SBT_MAVEN_PROFILES_ARGS="-Phadoop-1 -Dhadoop.version=2.0.0-mr1-cdh4.1.1" elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.2" ]; then - export SBT_MAVEN_PROFILES_ARGS="-Pyarn" + export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Phadoop-2.2" elif [ "$AMPLAB_JENKINS_BUILD_PROFILE" = "hadoop2.3" ]; then export SBT_MAVEN_PROFILES_ARGS="-Pyarn -Phadoop-2.3 -Dhadoop.version=2.3.0" fi diff --git a/dev/scalastyle b/dev/scalastyle index bfe9b1fb7df51..7f014c82f14c6 100755 --- a/dev/scalastyle +++ b/dev/scalastyle @@ -20,8 +20,8 @@ echo -e "q\n" | build/sbt -Phive -Phive-thriftserver scalastyle > scalastyle.txt echo -e "q\n" | build/sbt -Phive -Phive-thriftserver test:scalastyle >> scalastyle.txt # Check style with YARN built too -echo -e "q\n" | build/sbt -Pyarn scalastyle >> scalastyle.txt -echo -e "q\n" | build/sbt -Pyarn test:scalastyle >> scalastyle.txt +echo -e "q\n" | build/sbt -Pyarn -Phadoop-2.2 scalastyle >> scalastyle.txt +echo -e "q\n" | build/sbt -Pyarn -Phadoop-2.2 test:scalastyle >> scalastyle.txt ERRORS=$(cat scalastyle.txt | awk '{if($1~/error/)print}') rm scalastyle.txt diff --git a/docs/building-spark.md b/docs/building-spark.md index ca8963e02684f..6e310ff424784 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -89,6 +89,9 @@ Examples: {% highlight bash %} +# Apache Hadoop 2.2.X +mvn -Pyarn -Phadoop-2.2 -DskipTests clean package + # Apache Hadoop 2.3.X mvn -Pyarn -Phadoop-2.3 -Dhadoop.version=2.3.0 -DskipTests clean package From 11670e5baf49489c2d0e394a32865deff8e3a791 Mon Sep 17 00:00:00 2001 From: FavioVazquez Date: Thu, 14 May 2015 04:52:48 -0430 Subject: [PATCH 19/19] - Added missing instance of -Phadoop-2.2 in create-release.sh --- dev/create-release/create-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/create-release/create-release.sh b/dev/create-release/create-release.sh index bc6c48e71bd9f..af4f00054997c 100755 --- a/dev/create-release/create-release.sh +++ b/dev/create-release/create-release.sh @@ -125,7 +125,7 @@ if [[ ! "$@" =~ --skip-publish ]]; then ./dev/change-version-to-2.11.sh build/mvn -DskipTests -Pyarn -Phive \ - -Dscala-2.11 -Pspark-ganglia-lgpl -Pkinesis-asl \ + -Dscala-2.11 -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \ clean install ./dev/change-version-to-2.10.sh
0.23.xhadoop-0.23
1.x to 2.1.x(none)
2.2.xhadoop-2.2
1.x to 2.1.xhadoop-1
2.2.x(none)
2.3.xhadoop-2.3
2.4.xhadoop-2.4