Skip to content
Browse files

Merge the old sbt-launch-lib.bash with the new sbt-launcher jar downl…

…oading logic.

This allows developers to pass options (such as -D) to sbt.  I also modified the SparkBuild to ensure spark specific properties are propagated to forked test JVMs.
  • Loading branch information...
1 parent 12738c1 commit c008b184a34d241b3ed1b314acab7a9a2c960ce7 @marmbrus marmbrus committed Feb 26, 2014
Showing with 315 additions and 51 deletions.
  1. +32 −0 LICENSE
  2. +3 −0 project/SparkBuild.scala
  3. +102 −51 sbt/sbt
  4. +178 −0 sbt/sbt-launch-lib.bash
View
32 LICENSE
@@ -396,3 +396,35 @@ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
+
+
+========================================================================
+For sbt and sbt-launch-lib.bash in sbt/:
+========================================================================
+
+// Generated from http://www.opensource.org/licenses/bsd-license.php
+Copyright (c) 2011, Paul Phillips.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of the author nor the names of its contributors may be
+ used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
View
3 project/SparkBuild.scala
@@ -23,6 +23,8 @@ import AssemblyKeys._
import scala.util.Properties
import org.scalastyle.sbt.ScalastylePlugin.{Settings => ScalaStyleSettings}
+import scala.collection.JavaConversions._
+
// For Sonatype publishing
//import com.jsuereth.pgp.sbtplugin.PgpKeys._
@@ -138,6 +140,7 @@ object SparkBuild extends Build {
fork := true,
javaOptions in Test += "-Dspark.home=" + sparkHome,
javaOptions in Test += "-Dspark.testing=1",
+ javaOptions in Test ++= System.getProperties.filter(_._1 startsWith "spark").map { case (k,v) => s"-D$k=$v" }.toSeq,
javaOptions += "-Xmx3g",
// Show full stack trace and duration in test cases.
testOptions in Test += Tests.Argument("-oDF"),
View
153 sbt/sbt
@@ -1,51 +1,102 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script launches sbt for this project. If present it uses the system
-# version of sbt. If there is no system version of sbt it attempts to download
-# sbt locally.
-SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties`
-URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
-URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
-JAR=sbt/sbt-launch-${SBT_VERSION}.jar
-
-# Download sbt launch jar if it hasn't been downloaded yet
-if [ ! -f ${JAR} ]; then
- # Download
- printf "Attempting to fetch sbt\n"
- JAR_DL=${JAR}.part
- if hash curl 2>/dev/null; then
- (curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
- elif hash wget 2>/dev/null; then
- (wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
- else
- printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
- exit -1
- fi
-fi
-if [ ! -f ${JAR} ]; then
- # We failed to download
- printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
- exit -1
-fi
-printf "Launching sbt from ${JAR}\n"
-java \
- -Xmx1200m -XX:MaxPermSize=350m -XX:ReservedCodeCacheSize=256m \
- -jar ${JAR} \
- "$@"
+#!/usr/bin/env bash
+
+realpath () {
+(
+ TARGET_FILE=$1
+
+ cd $(dirname $TARGET_FILE)
+ TARGET_FILE=$(basename $TARGET_FILE)
+
+ COUNT=0
+ while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
+ do
+ TARGET_FILE=$(readlink $TARGET_FILE)
+ cd $(dirname $TARGET_FILE)
+ TARGET_FILE=$(basename $TARGET_FILE)
+ COUNT=$(($COUNT + 1))
+ done
+
+ echo $(pwd -P)/$TARGET_FILE
+)
+}
+
+. $(dirname $(realpath $0))/sbt-launch-lib.bash
+
+
+declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
+declare -r sbt_opts_file=".sbtopts"
+declare -r etc_sbt_opts_file="/etc/sbt/sbtopts"
+
+usage() {
+ cat <<EOM
+Usage: $script_name [options]
+
+ -h | -help print this message
+ -v | -verbose this runner is chattier
+ -d | -debug set sbt log level to debug
+ -no-colors disable ANSI color codes
+ -sbt-create start sbt even if current directory contains no sbt project
+ -sbt-dir <path> path to global settings/plugins directory (default: ~/.sbt)
+ -sbt-boot <path> path to shared boot directory (default: ~/.sbt/boot in 0.11 series)
+ -ivy <path> path to local Ivy repository (default: ~/.ivy2)
+ -mem <integer> set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem))
+ -no-share use all local caches; no sharing
+ -no-global uses global caches, but does not use global ~/.sbt directory.
+ -jvm-debug <port> Turn on JVM debugging, open at the given port.
+ -batch Disable interactive mode
+
+ # sbt version (default: from project/build.properties if present, else latest release)
+ -sbt-version <version> use the specified version of sbt
+ -sbt-jar <path> use the specified jar as the sbt launcher
+ -sbt-rc use an RC version of sbt
+ -sbt-snapshot use a snapshot version of sbt
+
+ # java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
+ -java-home <path> alternate JAVA_HOME
+
+ # jvm options and output control
+ JAVA_OPTS environment variable, if unset uses "$java_opts"
+ SBT_OPTS environment variable, if unset uses "$default_sbt_opts"
+ .sbtopts if this file exists in the current directory, it is
+ prepended to the runner args
+ /etc/sbt/sbtopts if this file exists, it is prepended to the runner args
+ -Dkey=val pass -Dkey=val directly to the java runtime
+ -J-X pass option -X directly to the java runtime
+ (-J is stripped)
+ -S-X add -X to sbt's scalacOptions (-J is stripped)
+
+In the case of duplicated or conflicting options, the order above
+shows precedence: JAVA_OPTS lowest, command line options highest.
+EOM
+}
+
+process_my_args () {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
+ -no-share) addJava "$noshare_opts" && shift ;;
+ -no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;;
+ -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
+ -sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;;
+ -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
+ -batch) exec </dev/null && shift ;;
+
+ -sbt-create) sbt_create=true && shift ;;
+
+ *) addResidual "$1" && shift ;;
+ esac
+ done
+
+ # Now, ensure sbt version is used.
+ [[ "${sbt_version}XXX" != "XXX" ]] && addJava "-Dsbt.version=$sbt_version"
+}
+
+loadConfigFile() {
+ cat "$1" | sed '/^\#/d'
+}
+
+# if sbtopts files exist, prepend their contents to $@ so it can be processed by this runner
+[[ -f "$etc_sbt_opts_file" ]] && set -- $(loadConfigFile "$etc_sbt_opts_file") "$@"
+[[ -f "$sbt_opts_file" ]] && set -- $(loadConfigFile "$sbt_opts_file") "$@"
+
+run "$@"
View
178 sbt/sbt-launch-lib.bash
@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+#
+
+# A library to simplify using the SBT launcher from other packages.
+# Note: This should be used by tools like giter8/conscript etc.
+
+# TODO - Should we merge the main SBT script with this library?
+
+if test -z "$HOME"; then
+ declare -r script_dir="$(dirname $script_path)"
+else
+ declare -r script_dir="$HOME/.sbt"
+fi
+
+declare -a residual_args
+declare -a java_args
+declare -a scalac_args
+declare -a sbt_commands
+declare java_cmd=java
+
+echoerr () {
+ echo 1>&2 "$@"
+}
+vlog () {
+ [[ $verbose || $debug ]] && echoerr "$@"
+}
+dlog () {
+ [[ $debug ]] && echoerr "$@"
+}
+
+acquire_sbt_jar () {
+ SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties`
+ URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+ URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+ JAR=sbt/sbt-launch-${SBT_VERSION}.jar
+
+ sbt_jar=$JAR
+
+ if [[ ! -f "$sbt_jar" ]]; then
+ # Download sbt launch jar if it hasn't been downloaded yet
+ if [ ! -f ${JAR} ]; then
+ # Download
+ printf "Attempting to fetch sbt\n"
+ JAR_DL=${JAR}.part
+ if hash curl 2>/dev/null; then
+ (curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
+ elif hash wget 2>/dev/null; then
+ (wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
+ else
+ printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
+ exit -1
+ fi
+ fi
+ if [ ! -f ${JAR} ]; then
+ # We failed to download
+ printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
+ exit -1
+ fi
+ printf "Launching sbt from ${JAR}\n"
+ fi
+}
+
+execRunner () {
+ # print the arguments one to a line, quoting any containing spaces
+ [[ $verbose || $debug ]] && echo "# Executing command line:" && {
+ for arg; do
+ if printf "%s\n" "$arg" | grep -q ' '; then
+ printf "\"%s\"\n" "$arg"
+ else
+ printf "%s\n" "$arg"
+ fi
+ done
+ echo ""
+ }
+
+ exec "$@"
+}
+
+addJava () {
+ dlog "[addJava] arg = '$1'"
+ java_args=( "${java_args[@]}" "$1" )
+}
+addSbt () {
+ dlog "[addSbt] arg = '$1'"
+ sbt_commands=( "${sbt_commands[@]}" "$1" )
+}
+addResidual () {
+ dlog "[residual] arg = '$1'"
+ residual_args=( "${residual_args[@]}" "$1" )
+}
+addDebugger () {
+ addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
+}
+
+# a ham-fisted attempt to move some memory settings in concert
+# so they need not be dicked around with individually.
+get_mem_opts () {
+ local mem=${1:-2048}
+ local perm=$(( $mem / 4 ))
+ (( $perm > 256 )) || perm=256
+ (( $perm < 1024 )) || perm=1024
+ local codecache=$(( $perm / 2 ))
+
+ echo "-Xms${mem}m -Xmx${mem}m -XX:MaxPermSize=${perm}m -XX:ReservedCodeCacheSize=${codecache}m"
+}
+
+require_arg () {
+ local type="$1"
+ local opt="$2"
+ local arg="$3"
+ if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
+ die "$opt requires <$type> argument"
+ fi
+}
+
+is_function_defined() {
+ declare -f "$1" > /dev/null
+}
+
+process_args () {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ -h|-help) usage; exit 1 ;;
+ -v|-verbose) verbose=1 && shift ;;
+ -d|-debug) debug=1 && shift ;;
+
+ -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
+ -mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;;
+ -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;;
+ -batch) exec </dev/null && shift ;;
+
+ -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
+ -sbt-version) require_arg version "$1" "$2" && sbt_version="$2" && shift 2 ;;
+ -java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;
+
+ -D*) addJava "$1" && shift ;;
+ -J*) addJava "${1:2}" && shift ;;
+ *) addResidual "$1" && shift ;;
+ esac
+ done
+
+ is_function_defined process_my_args && {
+ myargs=("${residual_args[@]}")
+ residual_args=()
+ process_my_args "${myargs[@]}"
+ }
+}
+
+run() {
+ # no jar? download it.
+ [[ -f "$sbt_jar" ]] || acquire_sbt_jar "$sbt_version" || {
+ # still no jar? uh-oh.
+ echo "Download failed. Obtain the sbt-launch.jar manually and place it at $sbt_jar"
+ exit 1
+ }
+
+ # process the combined args, then reset "$@" to the residuals
+ process_args "$@"
+ set -- "${residual_args[@]}"
+ argumentCount=$#
+
+ # run sbt
+ execRunner "$java_cmd" \
+ ${SBT_OPTS:-$default_sbt_opts} \
+ $(get_mem_opts $sbt_mem) \
+ ${java_opts} \
+ ${java_args[@]} \
+ -jar "$sbt_jar" \
+ "${sbt_commands[@]}" \
+ "${residual_args[@]}"
+}
+
+runAlternateBoot() {
+ local bootpropsfile="$1"
+ shift
+ addJava "-Dsbt.boot.properties=$bootpropsfile"
+ run $@
+}

0 comments on commit c008b18

Please sign in to comment.
Something went wrong with that request. Please try again.