From 6c78f5666babdf2b20ae34c47763125fabe1e9e6 Mon Sep 17 00:00:00 2001 From: Michael Armbrust Date: Sat, 29 Aug 2015 16:23:08 -0700 Subject: [PATCH] Update SBT lib to newest version in Spark Moves the sbt scripts to `build/` to match Spark and update it to reflect the new download locations. Author: Michael Armbrust Closes #82 from marmbrus/sbtBuild. --- .gitignore | 2 +- {sbt => build}/sbt | 57 ++++++++++++++++++++++++++++-- {sbt => build}/sbt-launch-lib.bash | 33 ++++++++--------- 3 files changed, 70 insertions(+), 22 deletions(-) rename {sbt => build}/sbt (70%) rename {sbt => build}/sbt-launch-lib.bash (82%) diff --git a/.gitignore b/.gitignore index 706ddde4..21894095 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -sbt/sbt-launch*.jar +build/sbt-launch*.jar target/ .idea/ .idea_modules/ diff --git a/sbt/sbt b/build/sbt similarity index 70% rename from sbt/sbt rename to build/sbt index dd97d0a2..cc3203d7 100755 --- a/sbt/sbt +++ b/build/sbt @@ -1,5 +1,30 @@ #!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# When creating new tests for Spark SQL Hive, the HADOOP_CLASSPATH must contain the hive jars so +# that we can run Hive to generate the golden answer. This is not required for normal development +# or testing. +for i in "$HIVE_HOME"/lib/* +do HADOOP_CLASSPATH="$HADOOP_CLASSPATH:$i" +done +export HADOOP_CLASSPATH + realpath () { ( TARGET_FILE="$1" @@ -63,8 +88,8 @@ Usage: $script_name [options] -Dkey=val pass -Dkey=val directly to the java runtime -J-X pass option -X directly to the java runtime (-J is stripped) - -S-X add -X to sbt's scalacOptions (-J is stripped) - -PmavenProfiles Enable a maven profile for the build. + -S-X add -X to sbt's scalacOptions (-S is stripped) + -PmavenProfiles Enable a maven profile for the build. In the case of duplicated or conflicting options, the order above shows precedence: JAVA_OPTS lowest, command line options highest. @@ -100,4 +125,32 @@ loadConfigFile() { [[ -f "$etc_sbt_opts_file" ]] && set -- $(loadConfigFile "$etc_sbt_opts_file") "$@" [[ -f "$sbt_opts_file" ]] && set -- $(loadConfigFile "$sbt_opts_file") "$@" +exit_status=127 +saved_stty="" + +restoreSttySettings() { + stty $saved_stty + saved_stty="" +} + +onExit() { + if [[ "$saved_stty" != "" ]]; then + restoreSttySettings + fi + exit $exit_status +} + +saveSttySettings() { + saved_stty=$(stty -g 2>/dev/null) + if [[ ! $? ]]; then + saved_stty="" + fi +} + +saveSttySettings +trap onExit INT + run "$@" + +exit_status=$? +onExit diff --git a/sbt/sbt-launch-lib.bash b/build/sbt-launch-lib.bash similarity index 82% rename from sbt/sbt-launch-lib.bash rename to build/sbt-launch-lib.bash index 7f05d2ef..615f8483 100755 --- a/sbt/sbt-launch-lib.bash +++ b/build/sbt-launch-lib.bash @@ -37,10 +37,9 @@ dlog () { } acquire_sbt_jar () { - SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties` - URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar - URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar - JAR=sbt/sbt-launch-${SBT_VERSION}.jar + SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties` + URL1=https://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar + JAR=build/sbt-launch-${SBT_VERSION}.jar sbt_jar=$JAR @@ -50,10 +49,12 @@ acquire_sbt_jar () { # Download printf "Attempting to fetch sbt\n" JAR_DL="${JAR}.part" - if hash curl 2>/dev/null; then - (curl --silent ${URL1} > "${JAR_DL}" || curl --silent ${URL2} > "${JAR_DL}") && mv "${JAR_DL}" "${JAR}" - elif hash wget 2>/dev/null; then - (wget --quiet ${URL1} -O "${JAR_DL}" || wget --quiet ${URL2} -O "${JAR_DL}") && mv "${JAR_DL}" "${JAR}" + if [ $(command -v curl) ]; then + curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\ + mv "${JAR_DL}" "${JAR}" + elif [ $(command -v wget) ]; then + wget --quiet ${URL1} -O "${JAR_DL}" &&\ + mv "${JAR_DL}" "${JAR}" else printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n" exit -1 @@ -81,7 +82,7 @@ execRunner () { echo "" } - exec "$@" + "$@" } addJava () { @@ -104,7 +105,7 @@ addResidual () { residual_args=( "${residual_args[@]}" "$1" ) } addDebugger () { - addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1" + addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1" } # a ham-fisted attempt to move some memory settings in concert @@ -124,7 +125,8 @@ require_arg () { local opt="$2" local arg="$3" if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then - die "$opt requires <$type> argument" + echo "$opt requires <$type> argument" 1>&2 + exit 1 fi } @@ -149,7 +151,7 @@ process_args () { -java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && export JAVA_HOME=$2 && shift 2 ;; -D*) addJava "$1" && shift ;; - -J*) addJava "${1:2}" && shift ;; + -J*) addJava "${1:2}" && shift ;; -P*) enableProfile "$1" && shift ;; *) addResidual "$1" && shift ;; esac @@ -185,10 +187,3 @@ run() { "${sbt_commands[@]}" \ "${residual_args[@]}" } - -runAlternateBoot() { - local bootpropsfile="$1" - shift - addJava "-Dsbt.boot.properties=$bootpropsfile" - run $@ -}