Skip to content
This repository has been archived by the owner on Sep 20, 2022. It is now read-only.

Commit

Permalink
Close #35, #23: [HIVEMALL-31][SPARK] Support Spark-v2.1.0
Browse files Browse the repository at this point in the history
  • Loading branch information
maropu committed Jan 31, 2017
1 parent bfd129c commit c837e51
Show file tree
Hide file tree
Showing 45 changed files with 6,538 additions and 5 deletions.
5 changes: 3 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ notifications:
email: false

script:
- mvn -q scalastyle:check test -Pspark-2.0
# test the spark-1.6 module only in this second run
- mvn -q scalastyle:check test -Pspark-2.1
# test the spark-1.6/2.0 modules only in the following runs
- mvn -q scalastyle:check clean -Pspark-2.0 -pl spark/spark-2.0 -am test -Dtest=none
- mvn -q scalastyle:check clean -Pspark-1.6 -pl spark/spark-1.6 -am test -Dtest=none

after_success:
Expand Down
12 changes: 9 additions & 3 deletions NOTICE
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ Copyright notifications which have been relocated from ASF projects

o hivemall/core/src/main/java/hivemall/utils/math/MathUtils.java#erfInv()

Copyright (C) 2003-2016 The Apache Software Foundation.
Copyright (C) 2003-2016 The Apache Software Foundation.

http://commons.apache.org/proper/commons-math/
Licensed under the Apache License, Version 2.0
Expand All @@ -55,8 +55,14 @@ o hivemall/spark/spark-1.6/extra-src/hive/src/main/scala/org/apache/spark/sql/hi
hivemall/spark/spark-2.0/src/test/scala/org/apache/spark/sql/QueryTest.scala
hivemall/spark/spark-2.0/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
hivemall/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/test/TestHiveSingleton.scala

Copyright (C) 2014-2016 The Apache Software Foundation.
hivemall/spark/spark-2.1/extra-src/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
hivemall/spark/spark-2.1/src/test/scala/org/apache/spark/sql/QueryTest.scala
hivemall/spark/spark-2.1/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
hivemall/spark/spark-2.1/src/test/scala/org/apache/spark/sql/hive/test/TestHiveSingleton.scala
hivemall/spark/spark-2.1/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
hivemall/spark/spark-2.1/src/test/scala/org/apache/spark/sql/test/SQLTestData.scala

Copyright (C) 2014-2017 The Apache Software Foundation.

http://spark.apache.org/
Licensed under the Apache License, Version 2.0
11 changes: 11 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,17 @@
</distributionManagement>

<profiles>
<profile>
<id>spark-2.1</id>
<modules>
<module>spark/spark-2.1</module>
<module>spark/spark-common</module>
</modules>
<properties>
<spark.version>2.1.0</spark.version>
<spark.binary.version>2.1</spark.binary.version>
</properties>
</profile>
<profile>
<id>spark-2.0</id>
<modules>
Expand Down
99 changes: 99 additions & 0 deletions spark/spark-2.1/bin/mvn-zinc
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Copyed from commit 48682f6bf663e54cb63b7e95a4520d34b6fa890b in Apache Spark

# Determine the current working directory
_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Preserve the calling directory
_CALLING_DIR="$(pwd)"
# Options used during compilation
_COMPILE_JVM_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"

# Installs any application tarball given a URL, the expected tarball name,
# and, optionally, a checkable binary path to determine if the binary has
# already been installed
## Arg1 - URL
## Arg2 - Tarball Name
## Arg3 - Checkable Binary
install_app() {
local remote_tarball="$1/$2"
local local_tarball="${_DIR}/$2"
local binary="${_DIR}/$3"
local curl_opts="--progress-bar -L"
local wget_opts="--progress=bar:force ${wget_opts}"

if [ -z "$3" -o ! -f "$binary" ]; then
# check if we already have the tarball
# check if we have curl installed
# download application
[ ! -f "${local_tarball}" ] && [ $(command -v curl) ] && \
echo "exec: curl ${curl_opts} ${remote_tarball}" 1>&2 && \
curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
# if the file still doesn't exist, lets try `wget` and cross our fingers
[ ! -f "${local_tarball}" ] && [ $(command -v wget) ] && \
echo "exec: wget ${wget_opts} ${remote_tarball}" 1>&2 && \
wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
# if both were unsuccessful, exit
[ ! -f "${local_tarball}" ] && \
echo -n "ERROR: Cannot download $2 with cURL or wget; " && \
echo "please install manually and try again." && \
exit 2
cd "${_DIR}" && tar -xzf "$2"
rm -rf "$local_tarball"
fi
}

# Install zinc under the bin/ folder
install_zinc() {
local zinc_path="zinc-0.3.9/bin/zinc"
[ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
install_app \
"http://downloads.typesafe.com/zinc/0.3.9" \
"zinc-0.3.9.tgz" \
"${zinc_path}"
ZINC_BIN="${_DIR}/${zinc_path}"
}

# Setup healthy defaults for the Zinc port if none were provided from
# the environment
ZINC_PORT=${ZINC_PORT:-"3030"}

# Install Zinc for the bin/
install_zinc

# Reset the current working directory
cd "${_CALLING_DIR}"

# Now that zinc is ensured to be installed, check its status and, if its
# not running or just installed, start it
if [ ! -f "${ZINC_BIN}" ]; then
exit -1
fi
if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port ${ZINC_PORT}`" ]; then
export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
"${ZINC_BIN}" -start -port ${ZINC_PORT} &>/dev/null
fi

# Set any `mvn` options if not already present
export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}

# Last, call the `mvn` command as usual
mvn -DzincPort=${ZINC_PORT} "$@"
1 change: 1 addition & 0 deletions spark/spark-2.1/extra-src/README
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Copyed from spark master [commit 908e37bcc10132bb2aa7f80ae694a9df6e40f31a]
Loading

0 comments on commit c837e51

Please sign in to comment.