Skip to content

Commit 14cb4e0

Browse files
marmbrusmarmbrus
committed
Project import generated by Copybara.
GitOrigin-RevId: 62fbd592bead1a3592c258a3191e3e603b026377
0 parents  commit 14cb4e0

File tree

62 files changed

+10948
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+10948
-0
lines changed

‎build.sbt

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/* Copyright (C) 2017 Databricks, Inc.
2+
*
3+
* Portions of this software incorporate or are derived from software contained within Apache Spark,
4+
* and this modified software differs from the Apache Spark software provided under the Apache
5+
* License, Version 2.0, a copy of which you may obtain at
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*/
8+
9+
name := "delta"
10+
11+
organization := "com.databricks"
12+
13+
scalaVersion := "2.11.12"
14+
15+
crossScalaVersions := Seq("2.12.8", "2.11.12")
16+
17+
libraryDependencies ++= Seq(
18+
"org.apache.spark" %% "spark-hive" % "2.4.0",
19+
"org.scalatest" %% "scalatest" % "3.0.5" % "test",
20+
"org.apache.spark" %% "spark-sql" % "2.4.0" % "test" classifier "tests",
21+
"org.apache.spark" %% "spark-catalyst" % "2.4.0" % "test" classifier "tests",
22+
"org.apache.spark" %% "spark-core" % "2.4.0" % "test" classifier "tests"
23+
)
24+
25+
// Display full-length stacktraces from ScalaTest:
26+
testOptions in Test += Tests.Argument("-oF")
27+
28+
scalacOptions ++= Seq("-target:jvm-1.7")
29+
30+
javacOptions ++= Seq("-source", "1.7", "-target", "1.7")

‎build/sbt

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
# When creating new tests for Spark SQL Hive, the HADOOP_CLASSPATH must contain the hive jars so
21+
# that we can run Hive to generate the golden answer. This is not required for normal development
22+
# or testing.
23+
if [ -n "$HIVE_HOME" ]; then
24+
for i in "$HIVE_HOME"/lib/*
25+
do HADOOP_CLASSPATH="$HADOOP_CLASSPATH:$i"
26+
done
27+
export HADOOP_CLASSPATH
28+
fi
29+
30+
realpath () {
31+
(
32+
TARGET_FILE="$1"
33+
34+
cd "$(dirname "$TARGET_FILE")"
35+
TARGET_FILE="$(basename "$TARGET_FILE")"
36+
37+
COUNT=0
38+
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
39+
do
40+
TARGET_FILE="$(readlink "$TARGET_FILE")"
41+
cd $(dirname "$TARGET_FILE")
42+
TARGET_FILE="$(basename $TARGET_FILE)"
43+
COUNT=$(($COUNT + 1))
44+
done
45+
46+
echo "$(pwd -P)/"$TARGET_FILE""
47+
)
48+
}
49+
50+
. "$(dirname "$(realpath "$0")")"/sbt-launch-lib.bash
51+
52+
53+
declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
54+
declare -r sbt_opts_file=".sbtopts"
55+
declare -r etc_sbt_opts_file="/etc/sbt/sbtopts"
56+
57+
usage() {
58+
cat <<EOM
59+
Usage: $script_name [options]
60+
61+
-h | -help print this message
62+
-v | -verbose this runner is chattier
63+
-d | -debug set sbt log level to debug
64+
-no-colors disable ANSI color codes
65+
-sbt-create start sbt even if current directory contains no sbt project
66+
-sbt-dir <path> path to global settings/plugins directory (default: ~/.sbt)
67+
-sbt-boot <path> path to shared boot directory (default: ~/.sbt/boot in 0.11 series)
68+
-ivy <path> path to local Ivy repository (default: ~/.ivy2)
69+
-mem <integer> set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem))
70+
-no-share use all local caches; no sharing
71+
-no-global uses global caches, but does not use global ~/.sbt directory.
72+
-jvm-debug <port> Turn on JVM debugging, open at the given port.
73+
-batch Disable interactive mode
74+
75+
# sbt version (default: from project/build.properties if present, else latest release)
76+
-sbt-version <version> use the specified version of sbt
77+
-sbt-jar <path> use the specified jar as the sbt launcher
78+
-sbt-rc use an RC version of sbt
79+
-sbt-snapshot use a snapshot version of sbt
80+
81+
# java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
82+
-java-home <path> alternate JAVA_HOME
83+
84+
# jvm options and output control
85+
JAVA_OPTS environment variable, if unset uses "$java_opts"
86+
SBT_OPTS environment variable, if unset uses "$default_sbt_opts"
87+
.sbtopts if this file exists in the current directory, it is
88+
prepended to the runner args
89+
/etc/sbt/sbtopts if this file exists, it is prepended to the runner args
90+
-Dkey=val pass -Dkey=val directly to the java runtime
91+
-J-X pass option -X directly to the java runtime
92+
(-J is stripped)
93+
-S-X add -X to sbt's scalacOptions (-S is stripped)
94+
-PmavenProfiles Enable a maven profile for the build.
95+
96+
In the case of duplicated or conflicting options, the order above
97+
shows precedence: JAVA_OPTS lowest, command line options highest.
98+
EOM
99+
}
100+
101+
process_my_args () {
102+
while [[ $# -gt 0 ]]; do
103+
case "$1" in
104+
-no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
105+
-no-share) addJava "$noshare_opts" && shift ;;
106+
-no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;;
107+
-sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
108+
-sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;;
109+
-debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
110+
-batch) exec </dev/null && shift ;;
111+
112+
-sbt-create) sbt_create=true && shift ;;
113+
114+
*) addResidual "$1" && shift ;;
115+
esac
116+
done
117+
118+
# Now, ensure sbt version is used.
119+
[[ "${sbt_version}XXX" != "XXX" ]] && addJava "-Dsbt.version=$sbt_version"
120+
}
121+
122+
loadConfigFile() {
123+
cat "$1" | sed '/^\#/d'
124+
}
125+
126+
# if sbtopts files exist, prepend their contents to $@ so it can be processed by this runner
127+
[[ -f "$etc_sbt_opts_file" ]] && set -- $(loadConfigFile "$etc_sbt_opts_file") "$@"
128+
[[ -f "$sbt_opts_file" ]] && set -- $(loadConfigFile "$sbt_opts_file") "$@"
129+
130+
exit_status=127
131+
saved_stty=""
132+
133+
restoreSttySettings() {
134+
stty $saved_stty
135+
saved_stty=""
136+
}
137+
138+
onExit() {
139+
if [[ "$saved_stty" != "" ]]; then
140+
restoreSttySettings
141+
fi
142+
exit $exit_status
143+
}
144+
145+
saveSttySettings() {
146+
saved_stty=$(stty -g 2>/dev/null)
147+
if [[ ! $? ]]; then
148+
saved_stty=""
149+
fi
150+
}
151+
152+
saveSttySettings
153+
trap onExit INT
154+
155+
run "$@"
156+
157+
exit_status=$?
158+
onExit

‎build/sbt-launch-lib.bash

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
#!/usr/bin/env bash
2+
#
3+
4+
# A library to simplify using the SBT launcher from other packages.
5+
# Note: This should be used by tools like giter8/conscript etc.
6+
7+
# TODO - Should we merge the main SBT script with this library?
8+
9+
if test -z "$HOME"; then
10+
declare -r script_dir="$(dirname "$script_path")"
11+
else
12+
declare -r script_dir="$HOME/.sbt"
13+
fi
14+
15+
declare -a residual_args
16+
declare -a java_args
17+
declare -a scalac_args
18+
declare -a sbt_commands
19+
declare -a maven_profiles
20+
21+
if test -x "$JAVA_HOME/bin/java"; then
22+
echo -e "Using $JAVA_HOME as default JAVA_HOME."
23+
echo "Note, this will be overridden by -java-home if it is set."
24+
declare java_cmd="$JAVA_HOME/bin/java"
25+
else
26+
declare java_cmd=java
27+
fi
28+
29+
echoerr () {
30+
echo 1>&2 "$@"
31+
}
32+
vlog () {
33+
[[ $verbose || $debug ]] && echoerr "$@"
34+
}
35+
dlog () {
36+
[[ $debug ]] && echoerr "$@"
37+
}
38+
39+
acquire_sbt_jar () {
40+
SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties`
41+
URL1=https://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
42+
JAR=build/sbt-launch-${SBT_VERSION}.jar
43+
44+
sbt_jar=$JAR
45+
46+
if [[ ! -f "$sbt_jar" ]]; then
47+
# Download sbt launch jar if it hasn't been downloaded yet
48+
if [ ! -f "${JAR}" ]; then
49+
# Download
50+
printf "Attempting to fetch sbt\n"
51+
JAR_DL="${JAR}.part"
52+
if [ $(command -v curl) ]; then
53+
curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\
54+
mv "${JAR_DL}" "${JAR}"
55+
elif [ $(command -v wget) ]; then
56+
wget --quiet ${URL1} -O "${JAR_DL}" &&\
57+
mv "${JAR_DL}" "${JAR}"
58+
else
59+
printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
60+
exit -1
61+
fi
62+
fi
63+
if [ ! -f "${JAR}" ]; then
64+
# We failed to download
65+
printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
66+
exit -1
67+
fi
68+
printf "Launching sbt from ${JAR}\n"
69+
fi
70+
}
71+
72+
execRunner () {
73+
# print the arguments one to a line, quoting any containing spaces
74+
[[ $verbose || $debug ]] && echo "# Executing command line:" && {
75+
for arg; do
76+
if printf "%s\n" "$arg" | grep -q ' '; then
77+
printf "\"%s\"\n" "$arg"
78+
else
79+
printf "%s\n" "$arg"
80+
fi
81+
done
82+
echo ""
83+
}
84+
85+
"$@"
86+
}
87+
88+
addJava () {
89+
dlog "[addJava] arg = '$1'"
90+
java_args=( "${java_args[@]}" "$1" )
91+
}
92+
93+
enableProfile () {
94+
dlog "[enableProfile] arg = '$1'"
95+
maven_profiles=( "${maven_profiles[@]}" "$1" )
96+
export SBT_MAVEN_PROFILES="${maven_profiles[@]}"
97+
}
98+
99+
addSbt () {
100+
dlog "[addSbt] arg = '$1'"
101+
sbt_commands=( "${sbt_commands[@]}" "$1" )
102+
}
103+
addResidual () {
104+
dlog "[residual] arg = '$1'"
105+
residual_args=( "${residual_args[@]}" "$1" )
106+
}
107+
addDebugger () {
108+
addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1"
109+
}
110+
111+
# a ham-fisted attempt to move some memory settings in concert
112+
# so they need not be dicked around with individually.
113+
get_mem_opts () {
114+
local mem=${1:-4000}
115+
local perm=$(( $mem / 4 ))
116+
(( $perm > 256 )) || perm=256
117+
(( $perm < 4096 )) || perm=4096
118+
local codecache=$(( $perm / 2 ))
119+
120+
echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m"
121+
}
122+
123+
require_arg () {
124+
local type="$1"
125+
local opt="$2"
126+
local arg="$3"
127+
if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
128+
echo "$opt requires <$type> argument" 1>&2
129+
exit 1
130+
fi
131+
}
132+
133+
is_function_defined() {
134+
declare -f "$1" > /dev/null
135+
}
136+
137+
process_args () {
138+
while [[ $# -gt 0 ]]; do
139+
case "$1" in
140+
-h|-help) usage; exit 1 ;;
141+
-v|-verbose) verbose=1 && shift ;;
142+
-d|-debug) debug=1 && shift ;;
143+
144+
-ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
145+
-mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;;
146+
-jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;;
147+
-batch) exec </dev/null && shift ;;
148+
149+
-sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
150+
-sbt-version) require_arg version "$1" "$2" && sbt_version="$2" && shift 2 ;;
151+
-java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && export JAVA_HOME=$2 && shift 2 ;;
152+
153+
-D*) addJava "$1" && shift ;;
154+
-J*) addJava "${1:2}" && shift ;;
155+
-P*) enableProfile "$1" && shift ;;
156+
*) addResidual "$1" && shift ;;
157+
esac
158+
done
159+
160+
is_function_defined process_my_args && {
161+
myargs=("${residual_args[@]}")
162+
residual_args=()
163+
process_my_args "${myargs[@]}"
164+
}
165+
}
166+
167+
run() {
168+
# no jar? download it.
169+
[[ -f "$sbt_jar" ]] || acquire_sbt_jar "$sbt_version" || {
170+
# still no jar? uh-oh.
171+
echo "Download failed. Obtain the sbt-launch.jar manually and place it at $sbt_jar"
172+
exit 1
173+
}
174+
175+
# process the combined args, then reset "$@" to the residuals
176+
process_args "$@"
177+
set -- "${residual_args[@]}"
178+
argumentCount=$#
179+
180+
# run sbt
181+
execRunner "$java_cmd" \
182+
${SBT_OPTS:-$default_sbt_opts} \
183+
$(get_mem_opts $sbt_mem) \
184+
${java_opts} \
185+
${java_args[@]} \
186+
-jar "$sbt_jar" \
187+
"${sbt_commands[@]}" \
188+
"${residual_args[@]}"
189+
}

‎project/build.properties

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
sbt.version=0.13.18

0 commit comments

Comments
 (0)