update build

.travis.yml

@@ -3,24 +3,28 @@ sudo: false
 cache:
   directories:
     - $HOME/.ivy2
+    - $HOME/.cache/spark-versions
 matrix:
   include:
     - jdk: openjdk7
       scala: 2.10.5
-      env: TEST_SPARK_VERSION="2.0.0"
+      env: TEST_SPARK_VERSION="2.0.0" TEST_SPARK_RELEASE="spark-2.0.0-bin-hadoop2.6"
     - jdk: openjdk7
       scala: 2.10.5
-      env: TEST_SPARK_VERSION="2.0.1"
+      env: TEST_SPARK_VERSION="2.0.1" TEST_SPARK_RELEASE="spark-2.0.1-bin-hadoop2.6"
     - jdk: openjdk7
       scala: 2.11.7
-      env: TEST_SPARK_VERSION="2.0.1"
+      env: TEST_SPARK_VERSION="2.0.1" TEST_SPARK_RELEASE="spark-2.0.1-bin-hadoop2.6"
     - jdk: openjdk7
       scala: 2.11.7
-      env: TEST_SPARK_VERSION="2.0.2"
+      env: TEST_SPARK_VERSION="2.0.2" TEST_SPARK_RELEASE="spark-2.0.2-bin-hadoop2.6"
 script:
   - sbt ++$TRAVIS_SCALA_VERSION scalastyle
   - sbt ++$TRAVIS_SCALA_VERSION "test:scalastyle"
   - sbt -Dspark.testVersion=$TEST_SPARK_VERSION ++$TRAVIS_SCALA_VERSION coverage test
   - sbt ++$TRAVIS_SCALA_VERSION assembly
+  - sbt ++$TRAVIS_SCALA_VERSION package
+  - ./bin/setup-spark $TEST_SPARK_RELEASE $HOME/.cache/spark-versions
+  - SPARK_HOME=$HOME/.cache/spark-versions/$TEST_SPARK_RELEASE ./bin/run-python-tests
 after_success:
   - sbt coverageReport coveralls

bin/setup-spark

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+SPARK_RELEASE=$1
+CACHE_DIR=$2
+
+if [[ -z "$SPARK_RELEASE" ]]; then
+  echo "Spark version is invalid: $SPARK_RELEASE, must follow pattern spark-x.y.z-bin-hadoopX.Y"
+  exit 1
+fi
+
+if [[ -z "$CACHE_DIR" ]]; then
+  echo "Cache directory is required"
+  exit 1
+fi
+
+# Prepare cache directory
+if [[ -e "$CACHE_DIR" ]]; then
+  if [[ ! -d "$CACHE_DIR" ]]; then
+    echo "$CACHE_DIR must be a directory"
+    exit 1
+  fi
+else
+  echo "$CACHE_DIR does not exist, creating"
+  mkdir -p $CACHE_DIR
+fi
+
+SPARK_VERSION_DIR="$CACHE_DIR/$SPARK_RELEASE"
+SPARK_VERSION_URL="http://d3kbcqa49mib13.cloudfront.net/$SPARK_RELEASE.tgz"
+
+# Check if cached version is already available
+if [[ -d "$SPARK_VERSION_DIR" ]]; then
+  echo "Using existing Spark cache $SPARK_VERSION_DIR"
+else
+  echo "Cache is not found for $SPARK_VERSION_DIR"
+  echo "Downloading Spark release $SPARK_RELEASE from $SPARK_VERSION_URL"
+  # Remove tar archive that might have been left after unsuccessful download
+  rm -f $SPARK_VERSION_DIR.tgz
+  curl -L "$SPARK_VERSION_URL" > "$SPARK_VERSION_DIR.tgz"
+  echo "Untar archive"
+  tar xf "$SPARK_VERSION_DIR.tgz" -C "$CACHE_DIR"
+  echo "Removing tar archive"
+  rm -f $SPARK_VERSION_DIR.tgz
+  echo "Spark is available at $SPARK_VERSION_DIR"
+fi