From 83b49b5e2def5df861c21cad1c6c72be3a460e09 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Fri, 9 Jan 2015 16:51:17 -0800 Subject: [PATCH 1/5] Modified make-distribution.sh so that we use curl, not only wget to get tachyon --- build/mvn | 4 ++-- build/sbt-launch-lib.bash | 4 ++-- dev/check-license | 22 +++++++++++----------- make-distribution.sh | 14 ++++++++++++-- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/build/mvn b/build/mvn index 43471f83e904c..4ca5bdbc37d14 100755 --- a/build/mvn +++ b/build/mvn @@ -48,11 +48,11 @@ install_app() { # check if we already have the tarball # check if we have curl installed # download application - [ ! -f "${local_tarball}" ] && [ -n "`which curl 2>/dev/null`" ] && \ + [ ! -f "${local_tarball}" ] && [ -n "`type curl 2>/dev/null`" ] && \ echo "exec: curl ${curl_opts} ${remote_tarball}" && \ curl ${curl_opts} "${remote_tarball}" > "${local_tarball}" # if the file still doesn't exist, lets try `wget` and cross our fingers - [ ! -f "${local_tarball}" ] && [ -n "`which wget 2>/dev/null`" ] && \ + [ ! -f "${local_tarball}" ] && [ -n "`type wget 2>/dev/null`" ] && \ echo "exec: wget ${wget_opts} ${remote_tarball}" && \ wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}" # if both were unsuccessful, exit diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash index f5df439effb01..236d264a92e8f 100755 --- a/build/sbt-launch-lib.bash +++ b/build/sbt-launch-lib.bash @@ -50,9 +50,9 @@ acquire_sbt_jar () { # Download printf "Attempting to fetch sbt\n" JAR_DL="${JAR}.part" - if hash curl 2>/dev/null; then + if type curl &>/dev/null; then (curl --silent ${URL1} > "${JAR_DL}" || curl --silent ${URL2} > "${JAR_DL}") && mv "${JAR_DL}" "${JAR}" - elif hash wget 2>/dev/null; then + elif type wget &>/dev/null; then (wget --quiet ${URL1} -O "${JAR_DL}" || wget --quiet ${URL2} -O "${JAR_DL}") && mv "${JAR_DL}" "${JAR}" else printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n" diff --git a/dev/check-license b/dev/check-license index 72b1013479964..5fef472daf1e9 100755 --- a/dev/check-license +++ b/dev/check-license @@ -27,17 +27,17 @@ acquire_rat_jar () { if [[ ! -f "$rat_jar" ]]; then # Download rat launch jar if it hasn't been downloaded yet if [ ! -f "$JAR" ]; then - # Download - printf "Attempting to fetch rat\n" - JAR_DL="${JAR}.part" - if hash curl 2>/dev/null; then - curl --silent "${URL}" > "$JAR_DL" && mv "$JAR_DL" "$JAR" - elif hash wget 2>/dev/null; then - wget --quiet ${URL} -O "$JAR_DL" && mv "$JAR_DL" "$JAR" - else - printf "You do not have curl or wget installed, please install rat manually.\n" - exit -1 - fi + # Download + printf "Attempting to fetch rat\n" + JAR_DL="${JAR}.part" + if type curl &>/dev/null; then + curl --silent "${URL}" > "$JAR_DL" && mv "$JAR_DL" "$JAR" + elif type wget &>/dev/null; then + wget --quiet ${URL} -O "$JAR_DL" && mv "$JAR_DL" "$JAR" + else + printf "You do not have curl or wget installed, please install rat manually.\n" + exit -1 + fi fi unzip -tq $JAR &> /dev/null diff --git a/make-distribution.sh b/make-distribution.sh index 4e2f400be3053..70b9ef82d98de 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -224,14 +224,24 @@ cp -r "$SPARK_HOME/ec2" "$DISTDIR" if [ "$SPARK_TACHYON" == "true" ]; then TACHYON_VERSION="0.5.0" TACHYON_URL="https://github.com/amplab/tachyon/releases/download/v${TACHYON_VERSION}/tachyon-${TACHYON_VERSION}-bin.tar.gz" + TACHYON_TGZ="tachyon-${TACHYON_VERSION}-bin.tar.gz" TMPD=`mktemp -d 2>/dev/null || mktemp -d -t 'disttmp'` pushd $TMPD > /dev/null echo "Fetching tachyon tgz" - wget "$TACHYON_URL" - tar xf "tachyon-${TACHYON_VERSION}-bin.tar.gz" + TACHYON_DL="${TACHYON_TGZ}.part" + if type curl &>/dev/null; then + curl --silent -k -L "${TACHYON_URL}" > "${TACHYON_DL}" && mv "${TACHYON_DL}" "${TACHYON_TGZ}" + elif type wget &>/dev/null; then + wget --quiet "${TACHYON_URL}" -O "${TACHYON_DL}" && mv "${TACHYON_DL}" "${TACHYON_TGZ}" + else + printf "You do not have curl or wget installed. please install Tachyon manually.\n" + exit -1 + fi + + tar xzf "${TACHYON_TGZ}" cp "tachyon-${TACHYON_VERSION}/core/target/tachyon-${TACHYON_VERSION}-jar-with-dependencies.jar" "$DISTDIR/lib" mkdir -p "$DISTDIR/tachyon/src/main/java/tachyon/web" cp -r "tachyon-${TACHYON_VERSION}"/{bin,conf,libexec} "$DISTDIR/tachyon" From 1e4c7e02fb636bab891a8e8b7fe1770de4a34d07 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 20 Jan 2015 08:44:38 -0800 Subject: [PATCH 2/5] Used "command" command instead of "type" command --- build/mvn | 4 ++-- build/sbt-launch-lib.bash | 4 ++-- dev/check-license | 4 ++-- make-distribution.sh | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/mvn b/build/mvn index 4ca5bdbc37d14..91597d06a22ac 100755 --- a/build/mvn +++ b/build/mvn @@ -48,11 +48,11 @@ install_app() { # check if we already have the tarball # check if we have curl installed # download application - [ ! -f "${local_tarball}" ] && [ -n "`type curl 2>/dev/null`" ] && \ + [ ! -f "${local_tarball}" ] && command -v curl &>/dev/null && \ echo "exec: curl ${curl_opts} ${remote_tarball}" && \ curl ${curl_opts} "${remote_tarball}" > "${local_tarball}" # if the file still doesn't exist, lets try `wget` and cross our fingers - [ ! -f "${local_tarball}" ] && [ -n "`type wget 2>/dev/null`" ] && \ + [ ! -f "${local_tarball}" ] && command -v wget &>/dev/null && \ echo "exec: wget ${wget_opts} ${remote_tarball}" && \ wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}" # if both were unsuccessful, exit diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash index 236d264a92e8f..c209a6104bb91 100755 --- a/build/sbt-launch-lib.bash +++ b/build/sbt-launch-lib.bash @@ -50,9 +50,9 @@ acquire_sbt_jar () { # Download printf "Attempting to fetch sbt\n" JAR_DL="${JAR}.part" - if type curl &>/dev/null; then + if command -v curl &>/dev/null; then (curl --silent ${URL1} > "${JAR_DL}" || curl --silent ${URL2} > "${JAR_DL}") && mv "${JAR_DL}" "${JAR}" - elif type wget &>/dev/null; then + elif command -v wget &>/dev/null; then (wget --quiet ${URL1} -O "${JAR_DL}" || wget --quiet ${URL2} -O "${JAR_DL}") && mv "${JAR_DL}" "${JAR}" else printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n" diff --git a/dev/check-license b/dev/check-license index 5fef472daf1e9..07712cd4d6c7f 100755 --- a/dev/check-license +++ b/dev/check-license @@ -30,9 +30,9 @@ acquire_rat_jar () { # Download printf "Attempting to fetch rat\n" JAR_DL="${JAR}.part" - if type curl &>/dev/null; then + if command -v curl &>/dev/null; then curl --silent "${URL}" > "$JAR_DL" && mv "$JAR_DL" "$JAR" - elif type wget &>/dev/null; then + elif command -v wget &>/dev/null; then wget --quiet ${URL} -O "$JAR_DL" && mv "$JAR_DL" "$JAR" else printf "You do not have curl or wget installed, please install rat manually.\n" diff --git a/make-distribution.sh b/make-distribution.sh index 70b9ef82d98de..c96b88a1ab6fe 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -232,9 +232,9 @@ if [ "$SPARK_TACHYON" == "true" ]; then echo "Fetching tachyon tgz" TACHYON_DL="${TACHYON_TGZ}.part" - if type curl &>/dev/null; then + if command -v curl &>/dev/null; then curl --silent -k -L "${TACHYON_URL}" > "${TACHYON_DL}" && mv "${TACHYON_DL}" "${TACHYON_TGZ}" - elif type wget &>/dev/null; then + elif command -v wget &>/dev/null; then wget --quiet "${TACHYON_URL}" -O "${TACHYON_DL}" && mv "${TACHYON_DL}" "${TACHYON_TGZ}" else printf "You do not have curl or wget installed. please install Tachyon manually.\n" From 2db9fbfedca9c32bd81eea1b28dd8838d510129f Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 20 Jan 2015 10:31:38 -0800 Subject: [PATCH 3/5] Removed redirection from the logic which checks the existence of commands --- build/mvn | 4 ++-- build/sbt-launch-lib.bash | 4 ++-- dev/check-license | 4 ++-- make-distribution.sh | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/mvn b/build/mvn index 91597d06a22ac..a7e27a20dcdc0 100755 --- a/build/mvn +++ b/build/mvn @@ -48,11 +48,11 @@ install_app() { # check if we already have the tarball # check if we have curl installed # download application - [ ! -f "${local_tarball}" ] && command -v curl &>/dev/null && \ + [ ! -f "${local_tarball}" ] && [ $(command -v curl) ] && \ echo "exec: curl ${curl_opts} ${remote_tarball}" && \ curl ${curl_opts} "${remote_tarball}" > "${local_tarball}" # if the file still doesn't exist, lets try `wget` and cross our fingers - [ ! -f "${local_tarball}" ] && command -v wget &>/dev/null && \ + [ ! -f "${local_tarball}" ] && [ $(command -v wget) ] && \ echo "exec: wget ${wget_opts} ${remote_tarball}" && \ wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}" # if both were unsuccessful, exit diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash index c209a6104bb91..5e0c640fa5919 100755 --- a/build/sbt-launch-lib.bash +++ b/build/sbt-launch-lib.bash @@ -50,9 +50,9 @@ acquire_sbt_jar () { # Download printf "Attempting to fetch sbt\n" JAR_DL="${JAR}.part" - if command -v curl &>/dev/null; then + if [ $(command -v curl) ]; then (curl --silent ${URL1} > "${JAR_DL}" || curl --silent ${URL2} > "${JAR_DL}") && mv "${JAR_DL}" "${JAR}" - elif command -v wget &>/dev/null; then + elif [ $(command -v wget) ]; then (wget --quiet ${URL1} -O "${JAR_DL}" || wget --quiet ${URL2} -O "${JAR_DL}") && mv "${JAR_DL}" "${JAR}" else printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n" diff --git a/dev/check-license b/dev/check-license index 07712cd4d6c7f..a006f65710d6d 100755 --- a/dev/check-license +++ b/dev/check-license @@ -30,9 +30,9 @@ acquire_rat_jar () { # Download printf "Attempting to fetch rat\n" JAR_DL="${JAR}.part" - if command -v curl &>/dev/null; then + if [ $(command -v curl) ]; then curl --silent "${URL}" > "$JAR_DL" && mv "$JAR_DL" "$JAR" - elif command -v wget &>/dev/null; then + elif [ $(command -v wget) ]; then wget --quiet ${URL} -O "$JAR_DL" && mv "$JAR_DL" "$JAR" else printf "You do not have curl or wget installed, please install rat manually.\n" diff --git a/make-distribution.sh b/make-distribution.sh index c96b88a1ab6fe..9afac6d18b554 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -232,9 +232,9 @@ if [ "$SPARK_TACHYON" == "true" ]; then echo "Fetching tachyon tgz" TACHYON_DL="${TACHYON_TGZ}.part" - if command -v curl &>/dev/null; then + if [ $(command -v curl) ]; then curl --silent -k -L "${TACHYON_URL}" > "${TACHYON_DL}" && mv "${TACHYON_DL}" "${TACHYON_TGZ}" - elif command -v wget &>/dev/null; then + elif [ $(command -v wget) ]; then wget --quiet "${TACHYON_URL}" -O "${TACHYON_DL}" && mv "${TACHYON_DL}" "${TACHYON_TGZ}" else printf "You do not have curl or wget installed. please install Tachyon manually.\n" From a3e908b52c760175ef7e81f785343cbc2ab87a28 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 20 Jan 2015 12:54:35 -0800 Subject: [PATCH 4/5] Fixed style --- make-distribution.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/make-distribution.sh b/make-distribution.sh index 9afac6d18b554..d39f6d5818286 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -32,6 +32,10 @@ SPARK_HOME="$(cd "`dirname "$0"`"; pwd)" DISTDIR="$SPARK_HOME/dist" SPARK_TACHYON=false +TACHYON_VERSION="0.5.0" +TACHYON_TGZ="tachyon-${TACHYON_VERSION}-bin.tar.gz" +TACHYON_URL="https://github.com/amplab/tachyon/releases/download/v${TACHYON_VERSION}/${TACHYON_TGZ}" + MAKE_TGZ=false NAME=none MVN="$SPARK_HOME/build/mvn" @@ -222,10 +226,6 @@ cp -r "$SPARK_HOME/ec2" "$DISTDIR" # Download and copy in tachyon, if requested if [ "$SPARK_TACHYON" == "true" ]; then - TACHYON_VERSION="0.5.0" - TACHYON_URL="https://github.com/amplab/tachyon/releases/download/v${TACHYON_VERSION}/tachyon-${TACHYON_VERSION}-bin.tar.gz" - TACHYON_TGZ="tachyon-${TACHYON_VERSION}-bin.tar.gz" - TMPD=`mktemp -d 2>/dev/null || mktemp -d -t 'disttmp'` pushd $TMPD > /dev/null From 7cc825591b071c40a3cf121fb9c9a4fc91ff5794 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 20 Jan 2015 14:44:09 -0800 Subject: [PATCH 5/5] Fix to use \$MVN instead of mvn --- make-distribution.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/make-distribution.sh b/make-distribution.sh index d39f6d5818286..6a229ce0ce325 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -97,7 +97,7 @@ done if [ -z "$JAVA_HOME" ]; then # Fall back on JAVA_HOME from rpm, if found - if which rpm &>/dev/null; then + if [ $(command -v rpm) ]; then RPM_JAVA_HOME=$(rpm -E %java_home 2>/dev/null) if [ "$RPM_JAVA_HOME" != "%java_home" ]; then JAVA_HOME=$RPM_JAVA_HOME @@ -111,7 +111,7 @@ if [ -z "$JAVA_HOME" ]; then exit -1 fi -if which git &>/dev/null; then +if [ $(command -v git) ]; then GITREV=$(git rev-parse --short HEAD 2>/dev/null || :) if [ ! -z $GITREV ]; then GITREVSTRING=" (git revision $GITREV)" @@ -119,14 +119,14 @@ if which git &>/dev/null; then unset GITREV fi -if ! which $MVN &>/dev/null; then +if [ ! $(command -v $MVN) ] ; then echo -e "Could not locate Maven command: '$MVN'." echo -e "Specify the Maven command with the --mvn flag" exit -1; fi -VERSION=$(mvn help:evaluate -Dexpression=project.version 2>/dev/null | grep -v "INFO" | tail -n 1) -SPARK_HADOOP_VERSION=$(mvn help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\ +VERSION=$($MVN help:evaluate -Dexpression=project.version 2>/dev/null | grep -v "INFO" | tail -n 1) +SPARK_HADOOP_VERSION=$($MVN help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\ | grep -v "INFO"\ | tail -n 1) SPARK_HIVE=$($MVN help:evaluate -Dexpression=project.activeProfiles -pl sql/hive $@ 2>/dev/null\