simplify build w/ docker

Author: mengxr

.travis.yml

@@ -2,104 +2,32 @@ sudo: required
 
 dist: trusty
 
-services:
-  - docker
-
-language: python
-
-cache:
-  directories:
-    - $HOME/.ivy2
-    - $HOME/.sbt/launchers/
-    - $HOME/.cache/spark-versions
+language: minimal
 
 env:
   global:
-    - SCALA_VERSION=2.11.8
-    - SPARK_VERSION=2.3.1
-    - SPARK_BUILD="spark-${SPARK_VERSION}-bin-hadoop2.7"
-    - SPARK_BUILD_URL="https://dist.apache.org/repos/dist/release/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz"
-    - SPARK_HOME=$HOME/.cache/spark-versions/$SPARK_BUILD
-    - RUN_ONLY_LIGHT_TESTS=True
+    - DOCKER_COMPOSE_VERSION=1.22.0
   matrix:
-    - PYTHON_VERSION=3.6.2 TEST_SUITE=scala-tests
-    - PYTHON_VERSION=3.6.2 TEST_SUITE=python-tests
-    - PYTHON_VERSION=3.6.2 TEST_SUITE=pylint
-    - PYTHON_VERSION=3.5.1 TEST_SUITE=python-tests
-    - PYTHON_VERSION=3.5.1 TEST_SUITE=pylint
-    - PYTHON_VERSION=2.7.13 TEST_SUITE=python-tests
-    - PYTHON_VERSION=2.7.13 TEST_SUITE=pylint
-
-before_install:
-  - ./bin/download_travis_dependencies.sh
-  - ci_env=`bash <(curl -s https://codecov.io/env)`
-  - if [[ "$PYTHON_VERSION" == 2.* ]]; then
-        export CONDA_URL="repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
-        export PYSPARK_PYTHON=python2;
-    else
-        export CONDA_URL="repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh";
-        export PYSPARK_PYTHON=python3;
-    fi
+    - PYTHON_VERSION=3.6 TEST_SUITE=pylint
+    - PYTHON_VERSION=2.7 TEST_SUITE=pylint
+    - PYTHON_VERSION=3.6 TEST_SUITE=python-tests
+    - PYTHON_VERSION=2.7 TEST_SUITE=python-tests
+    - PYTHON_VERSION=3.6 TEST_SUITE=scala-tests
 
-  - docker run $ci_env
-               -e "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64"
-               -e SPARK_VERSION
-               -e SPARK_BUILD
-               -e SCALA_VERSION
-               -e PYTHON_VERSION
-               -e PYSPARK_PYTHON
-               -e SPARK_HOME
-               -e RUN_ONLY_LIGHT_TESTS
-               -e CONDA_URL
-               -d --name ubuntu-test -v $HOME ubuntu:16.04 tail -f /dev/null
-  - docker ps
-
-# See this page: http://conda.pydata.org/docs/travis.html
-install:
-  # install needed ubuntu packages
-  - docker exec -t ubuntu-test bash -c "
-      apt-get update;
-      apt-get upgrade -y;
-      apt-get install -y curl openjdk-8-jdk bzip2"
-
-  # Copy the current dir into the docker container
-  - docker exec -t ubuntu-test bash -c "mkdir -p $TRAVIS_BUILD_DIR"
-  - docker cp `pwd`/. ubuntu-test:$TRAVIS_BUILD_DIR
+services:
+  - docker
 
-  # Download and set up miniconda
-  - docker exec -t ubuntu-test bash -c "
-     curl https://$CONDA_URL >> $HOME/miniconda.sh;
-     bash $HOME/miniconda.sh -b -p $HOME/miniconda;
-     bash $HOME/miniconda.sh -b -p $HOME/miniconda;
-     $HOME/miniconda/bin/conda config --set always_yes yes --set changeps1 no;
-     $HOME/miniconda/bin/conda update -q conda;
-     $HOME/miniconda/bin/conda info -a;
-     $HOME/miniconda/bin/conda create -q -n test-environment python=$PYTHON_VERSION"
+before_install:
+  # update docker compose to the specified version, https://docs.travis-ci.com/user/docker/#using-docker-compose
+  - sudo rm /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
 
- # Activate conda environment ad install required packages
-  - docker exec -t ubuntu-test bash -c "
-      source $HOME/miniconda/bin/activate test-environment;
-      pip install --user -r $TRAVIS_BUILD_DIR/dev/dev-requirements.txt;
-      pip install --user -r $TRAVIS_BUILD_DIR/python/requirements.txt;"
+install :
+  - docker-compose build --build-arg PYTHON_VERSION=$PYTHON_VERSION
+  - docker-compose up -d --scale worker=2
+  - docker-compose exec master bash -c "cd /mnt/sparkdl && build/sbt assembly"
 
 script:
-  - docker cp $HOME/.cache ubuntu-test:$HOME/
-  # build assembly
-  - docker exec -t ubuntu-test bash -c "
-      source $HOME/miniconda/bin/activate test-environment;
-      cd $TRAVIS_BUILD_DIR;
-      ./dev/run.py assembly"
-  # run python style and test suites
-  - docker exec -t ubuntu-test bash -c "
-      source $HOME/miniconda/bin/activate test-environment;
-      cd $TRAVIS_BUILD_DIR;
-      ./dev/run.py $TEST_SUITE"
-
-after_success:
-  # Unfortunately we need to install coverage here even though it's been installed via pip in the virual env.
-  # The codecov bash script executes `which coverage` and that comes up empty unless we install it directly.
-  - docker exec -t ubuntu-test bash -c "
-      source $HOME/miniconda/bin/activate test-environment;
-      $HOME/miniconda/bin/conda install -c anaconda coverage;
-      cd $TRAVIS_BUILD_DIR;
-      bash <(curl -s https://codecov.io/bash)"
+  - docker-compose exec master bash -c "cd /mnt/sparkdl && dev/run.py $TEST_SUITE"

Dockerfile

@@ -0,0 +1,71 @@
+FROM ubuntu:16.04
+
+ARG PYTHON_VERSION=3.6
+
+RUN apt-get update && \
+    apt-get install -y wget bzip2 build-essential openjdk-8-jdk ssh sudo && \
+    apt-get clean
+
+# Add ubuntu user and enable password-less sudo
+RUN useradd -mU -s /bin/bash -G sudo ubuntu && \
+    echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+# Set up SSH. Docker will cache this layer. So the keys end up the same on all containers.
+# We use a non-default location to simulate MLR, which doesn't have passwordless SSH.
+# We still keep the private key for manual SSH during debugging.
+RUN ssh-keygen -q -f ~/.ssh/docker -N "" && \
+    cp ~/.ssh/docker.pub ~/.ssh/authorized_keys
+
+# Install Open MPI
+RUN wget --quiet https://github.com/uber/horovod/files/1596799/openmpi-3.0.0-bin.tar.gz -O /tmp/openmpi.tar.gz && \
+    cd /usr/local && \
+    tar -zxf /tmp/openmpi.tar.gz && \
+    ldconfig && \
+    rm -r /tmp/openmpi.tar.gz
+
+# Install Miniconda.
+RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O /tmp/miniconda.sh && \
+    /bin/bash /tmp/miniconda.sh -b -p /opt/conda
+ENV PATH /opt/conda/bin:$PATH
+
+# Install sparkdl dependencies.
+# Instead of activate the "sparkdl" conda env, we update env variables directly.
+ENV PYTHON_VERSION $PYTHON_VERSION
+ENV PATH /opt/conda/envs/sparkdl/bin:$PATH
+ENV LD_LIBRARY_PATH /opt/conda/envs/sparkdl/lib:$LD_LIBRARY_PATH
+ENV PYTHONPATH /opt/conda/envs/sparkdl/lib/python$PYTHON_VERSION/site-packages:$PYTHONPATH
+COPY ./environment.yml /tmp/environment.yml
+RUN conda create -n sparkdl python=$PYTHON_VERSION && \
+    conda env update -n sparkdl -f /tmp/environment.yml
+
+# Install Spark and update env variables.
+ENV SPARK_VERSION 2.4.0
+ENV SPARK_BUILD "spark-${SPARK_VERSION}-bin-hadoop2.7"
+ENV SPARK_BUILD_URL "https://dist.apache.org/repos/dist/release/spark/spark-2.4.0/${SPARK_BUILD}.tgz"
+RUN wget --quiet $SPARK_BUILD_URL -O /tmp/spark.tgz && \
+    tar -C /opt -xf /tmp/spark.tgz && \
+    mv /opt/$SPARK_BUILD /opt/spark && \
+    rm /tmp/spark.tgz
+ENV SPARK_HOME /opt/spark
+ENV PATH $SPARK_HOME/bin:$PATH
+ENV PYTHONPATH /opt/spark/python/lib/py4j-0.10.7-src.zip:/opt/spark/python/lib/pyspark.zip:$PYTHONPATH
+
+# Declare env variables to run tests.
+ENV SCALA_VERSION 2.11.8
+ENV PYSPARK_PYTHON python
+ENV RUN_ONLY_LIGHT_TESTS True
+
+# Persist important env variables to /etc/environment.
+RUN echo "SCALA_VERSION=$SCALA_VERSION" >> /etc/environment && \
+    echo "SPARK_VERSION=$SPARK_VERSION" >> /etc/environment && \
+    echo "SPARK_HOME=$SPARK_HOME" >> /etc/environment && \
+    echo "PATH=$PATH" > /etc/environment && \
+    echo "PYTHONPATH=$PYTHONPATH" >> /etc/environment && \
+    echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> /etc/environment && \
+    echo "PYSPARK_PYTHON=$PYSPARK_PYTHON" >> /etc/environment && \
+    echo "RUN_ONLY_LIGHT_TESTS=$RUN_ONLY_LIGHT_TESTS" >> /etc/environment
+
+# The sparkdl dir will be mmounted here.
+VOLUME /mnt/sparkdl
+
+ENTRYPOINT service ssh restart && /bin/bash

build.sbt

@@ -3,10 +3,11 @@
 
 import ReleaseTransformations._
 
-val sparkVer = sys.props.getOrElse("spark.version", "2.3.1")
+val sparkVer = sys.props.getOrElse("spark.version", "2.4.0")
 val sparkBranch = sparkVer.substring(0, 3)
 val defaultScalaVer = sparkBranch match {
   case "2.3" => "2.11.8"
+  case "2.4" => "2.11.8"
   case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.")
 }
 val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer)
@@ -16,14 +17,19 @@ sparkVersion := sparkVer
 
 scalaVersion := scalaVer
 
+name := "spark-deep-learning"
+
 spName := "databricks/spark-deep-learning"
 
-// Don't forget to set the version
+organization := "com.databricks"
+
 version := (version in ThisBuild).value + s"-spark$sparkBranch"
 
 // All Spark Packages need a license
 licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0"))
 
+isSnapshot := version.value.contains("-SNAPSHOT")
+
 spAppendScalaVersion := true
 
 // Add Spark components this package depends on, e.g, "mllib", ....
@@ -34,7 +40,7 @@ sparkComponents ++= Seq("mllib-local", "mllib", "sql")
 
 // add any Spark Package dependencies using spDependencies.
 // e.g. spDependencies += "databricks/spark-avro:0.1"
-spDependencies += s"databricks/tensorframes:0.5.0-s_${scalaMajorVersion}"
+spDependencies += s"databricks/tensorframes:0.5.0-s_$scalaMajorVersion"
 
 
 libraryDependencies ++= Seq(
@@ -48,6 +54,7 @@ libraryDependencies ++= Seq(
 
 assemblyMergeStrategy in assembly := {
   case "requirements.txt" => MergeStrategy.concat
+  case "LICENSE-2.0.txt" => MergeStrategy.rename
   case x =>
     val oldStrategy = (assemblyMergeStrategy in assembly).value
     oldStrategy(x)
@@ -80,6 +87,14 @@ autoAPIMappings := true
 
 coverageHighlighting := false
 
+unmanagedResources in Compile += baseDirectory.value / "LICENSE"
+
+unmanagedResourceDirectories in Compile += baseDirectory.value / "python"
+
+includeFilter in unmanagedResources := "requirements.txt" ||
+  new SimpleFileFilter(_.relativeTo(baseDirectory.value / "python")
+    .exists(_.getPath.matches("sparkdl/.*\\.py"))) || "*.png" || "*.jpg" || "*.pb"
+
 // We only use sbt-release to update version numbers for now.
 releaseProcess := Seq[ReleaseStep](
   inquireVersions,
@@ -89,3 +104,6 @@ releaseProcess := Seq[ReleaseStep](
   setNextVersion,
   commitNextVersion
 )
+
+// Skip tests during assembly
+test in assembly := {}

dev/dev-requirements.txt

@@ -3,3 +3,4 @@ argcomplete==1.9.4
 pylint==1.8.4
 prospector==0.12.7
 yapf==0.21.0
+PyYAML==3.13

dev/run.py

@@ -227,7 +227,7 @@ def sbt(*args):
     assert(not missing_env)
     cmd = ("./build/sbt", "-Dspark.version=" + required_env.get("SPARK_VERSION"),
            "-Dscala.version=" + required_env.get("SCALA_VERSION"))
-    call_subprocess(cmd + args)
+    return call_subprocess(cmd + args)
 
 
 def assembly():

docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3'
+services:
+  master:
+    build: .
+    hostname: master
+    environment:
+      - MASTER=spark://master:7077
+    entrypoint: bash -c "service ssh restart && spark-class org.apache.spark.deploy.master.Master -h master"
+    ports:
+    - "4040:4040" # driver UI
+    - "8080:8080" # master UI
+    volumes:
+    - .:/mnt/sparkdl
+  worker:
+    build: .
+    entrypoint: bash -c "service ssh restart && spark-class org.apache.spark.deploy.worker.Worker spark://master:7077"
+    ports:
+    - "8081-8090:8081" # worker UI
+    links:
+    - master
+    volumes:
+    - .:/mnt/sparkdl

docker-resources/ssh_config

@@ -0,0 +1,2 @@
+Host *
+    StrictHostKeyChecking no

environment.yml

@@ -0,0 +1,35 @@
+# To choose a Python version, first create the env with:
+# conda create -n sparkdl python=3.6
+# and then update it
+# conda env update -n sparkdl -f environment.yml
+name: sparkdl
+channels:
+- https://repo.anaconda.com/pkgs/main/linux-64/
+dependencies:
+# runtime
+- six=1.11.0
+- nomkl # do not install MKL because it is large
+- pandas=0.23.4
+- h5py=2.8.0
+- pillow=4.1.1
+- cloudpickle=0.5.2
+- tensorflow=1.10.0
+- keras=2.2.4
+- paramiko=2.4.1
+- wrapt=1.10.11
+# test
+- coverage=4.5.1
+- nose=1.3.7
+- parameterized=0.6.1
+- pylint=1.8.4
+- argcomplete=1.9.4
+- PyYAML=3.13
+- pip:
+  # runtime
+  - horovod==0.15.0
+  # test
+  - prospector==0.12.11
+  - argh==0.26.2
+  - yapf==0.21.0
+  # docs
+  - sphinx

python/requirements.txt

@@ -1,11 +1,16 @@
 # This file should list any python package dependencies.
 coverage>=4.4.1
 h5py>=2.7.0
-keras==2.2.2 # NOTE: this package has only been tested with keras 2.2.2
+keras==2.2.4 # NOTE: this package has only been tested with keras 2.2.4
 nose>=1.3.7  # for testing
 parameterized>=0.6.1 # for testing
 pillow>=4.1.1,<4.2
 pygments>=2.2.0
 tensorflow==1.10.0 # NOTE: this package has only been tested with tensorflow 1.10.0
 pandas>=0.19.1
 six>=1.10.0
+paramiko>=2.4.0
+PyNaCl==1.2.1 # Note: This is a transitive dependency of paramiko v 1.3.0 fails
+cloudpickle>=0.5.2
+horovod==0.15.0
+wrapt==1.10.11

python/tests/transformers/named_image_InceptionV3_test.py

@@ -13,11 +13,13 @@
 # limitations under the License.
 #
 
+import os
 from .named_image_test import NamedImageTransformerBaseTestCase
 
 
 class NamedImageTransformerInceptionV3Test(NamedImageTransformerBaseTestCase):
 
-    __test__ = True
+    # TODO(ML-5165) Enable these tests in a separate target
+    __test__ = os.getenv('RUN_ONLY_LIGHT_TESTS', False) != "True"
     name = "InceptionV3"
     featurizerCompareDigitsExact = 4