From 88f192dc1c44a8ae166cf6637e905c92b9ad5210 Mon Sep 17 00:00:00 2001
From: Andrew Or <andrewor14@gmail.com>
Date: Mon, 5 May 2014 20:33:22 -0700
Subject: [PATCH 1/2] Add the required special profiles to make-distribution.sh

---
 make-distribution.sh | 47 +++++++++++++++++++++++++-------------------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/make-distribution.sh b/make-distribution.sh
index ff18d01e7a616..e79b5b8ec346b 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -47,7 +47,7 @@ set -o pipefail
 VERSION=$(mvn help:evaluate -Dexpression=project.version 2>/dev/null | grep -v "INFO" | tail -n 1)
 if [ $? != 0 ]; then
     echo -e "You need Maven installed to build Spark."
-    echo -e "Download Maven from https://maven.apache.org."
+    echo -e "Download Maven from https://maven.apache.org/"
     exit -1;
 fi
 
@@ -131,27 +131,34 @@ cd $FWDIR
 
 export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"
 
-if [ "$SPARK_HIVE" == "true" ]; then
-  MAYBE_HIVE="-Phive"
-else
-  MAYBE_HIVE=""
-fi
-
-if [ "$SPARK_YARN" == "true" ]; then
-  if [[ "$SPARK_HADOOP_VERSION" =~ "0.23." ]]; then
-    mvn clean package -DskipTests -Pyarn-alpha -Dhadoop.version=$SPARK_HADOOP_VERSION \
-      -Dyarn.version=$SPARK_HADOOP_VERSION $MAYBE_HIVE -Phadoop-0.23
-  else
-    mvn clean package -DskipTests -Pyarn -Dhadoop.version=$SPARK_HADOOP_VERSION \
-      -Dyarn.version=$SPARK_HADOOP_VERSION $MAYBE_HIVE
-  fi
-else
-  if [[ "$SPARK_HADOOP_VERSION" =~ "0.23." ]]; then
-    mvn clean package -Phadoop-0.23 -DskipTests -Dhadoop.version=$SPARK_HADOOP_VERSION $MAYBE_HIVE
-  else
-    mvn clean package -DskipTests -Dhadoop.version=$SPARK_HADOOP_VERSION $MAYBE_HIVE
+BUILD_COMMAND="mvn clean package"
+
+# Use special profiles for hadoop versions 0.23.x, 2.2.x, 2.3.x, 2.4.x
+if [[ "$SPARK_HADOOP_VERSION" =~ ^0\.23\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-0.23"; fi
+if [[ "$SPARK_HADOOP_VERSION" =~ ^2\.2\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-2.2"; fi
+if [[ "$SPARK_HADOOP_VERSION" =~ ^2\.3\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-2.3"; fi
+if [[ "$SPARK_HADOOP_VERSION" =~ ^2\.4\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-2.4"; fi
+if [[ "$SPARK_HIVE" == "true" ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phive"; fi
+if [[ "$SPARK_YARN" == "true" ]]; then
+  # For hadoop versions 0.23.x to 2.1.x, use the yarn-alpha profile
+  if [[ "$SPARK_HADOOP_VERSION" =~ ^0\.2[3-9]\. ]] ||
+     [[ "$SPARK_HADOOP_VERSION" =~ ^0\.[3-9][0-9]\. ]] ||
+     [[ "$SPARK_HADOOP_VERSION" =~ ^1\.[0-9]\. ]] ||
+     [[ "$SPARK_HADOOP_VERSION" =~ ^2\.[0-1]\. ]]; then
+    BUILD_COMMAND="$BUILD_COMMAND -Pyarn-alpha"
+  # For hadoop versions 2.2.x to 2.4.x, use the yarn profile
+  elif [[ "$SPARK_HADOOP_VERSION" =~ ^2.[2-4]. ]]; then
+    BUILD_COMMAND="$BUILD_COMMAND -Pyarn"
   fi
+  BUILD_COMMAND="$BUILD_COMMAND -Dyarn.version=$SPARK_HADOOP_VERSION"
 fi
+BUILD_COMMAND="$BUILD_COMMAND -Dhadoop.version=$SPARK_HADOOP_VERSION"
+BUILD_COMMAND="$BUILD_COMMAND -DskipTests"
+
+# Actually build the jar
+echo -e "\nBuilding with..."
+echo -e "\$ $BUILD_COMMAND\n"
+${BUILD_COMMAND}
 
 # Make directories
 rm -rf "$DISTDIR"

From 6740126a87afb356e20c299ce72cc4eb834839ed Mon Sep 17 00:00:00 2001
From: Andrew Or <andrewor14@gmail.com>
Date: Mon, 5 May 2014 20:44:37 -0700
Subject: [PATCH 2/2] Generalize the yarn profile to hadoop versions 2.2+

---
 make-distribution.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/make-distribution.sh b/make-distribution.sh
index e79b5b8ec346b..8a63133bc45d4 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -146,8 +146,8 @@ if [[ "$SPARK_YARN" == "true" ]]; then
      [[ "$SPARK_HADOOP_VERSION" =~ ^1\.[0-9]\. ]] ||
      [[ "$SPARK_HADOOP_VERSION" =~ ^2\.[0-1]\. ]]; then
     BUILD_COMMAND="$BUILD_COMMAND -Pyarn-alpha"
-  # For hadoop versions 2.2.x to 2.4.x, use the yarn profile
-  elif [[ "$SPARK_HADOOP_VERSION" =~ ^2.[2-4]. ]]; then
+  # For hadoop versions 2.2+, use the yarn profile
+  elif [[ "$SPARK_HADOOP_VERSION" =~ ^2.[2-9]. ]]; then
     BUILD_COMMAND="$BUILD_COMMAND -Pyarn"
   fi
   BUILD_COMMAND="$BUILD_COMMAND -Dyarn.version=$SPARK_HADOOP_VERSION"