ARROW-3367: [INTEGRATION] Port Spark integration test to the docker-compose setup

ci/docker_build_cpp.sh

@@ -17,7 +17,6 @@
 # under the License.
 
 set -e
-set -o xtrace
 
 source_dir=${1:-/arrow/cpp}
 build_dir=${2:-/build/cpp}

ci/docker_build_java.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# /arrow/java is read-only
+mkdir -p /build/java
+
+arrow_src=/build/java/arrow
+
+pushd /arrow
+  rsync -a header java format integration $arrow_src
+popd
+
+pushd $arrow_src/java
+  mvn -DskipTests -Drat.skip=true install
+popd

ci/docker_build_python.sh

@@ -26,15 +26,26 @@ export PYARROW_CMAKE_GENERATOR=Ninja
 export PYARROW_BUILD_TYPE=${PYARROW_BUILD_TYPE:-debug}
 
 # Feature flags
+export SETUPTOOLS_SCM_VERSION_WRITE_TO_PREFIX=$build_dir
 export PYARROW_WITH_ORC=${PYARROW_WITH_ORC:-1}
 export PYARROW_WITH_PARQUET=${PYARROW_WITH_PARQUET:-1}
 export PYARROW_WITH_PLASMA=${PYARROW_WITH_PLASMA:-1}
 
 # Build pyarrow
 pushd ${source_dir}
+  # hacky again, setuptools_scm writes _generated_version.py before pyarrow
+  # directory is created by setuptools
+  mkdir -p $build_dir/pyarrow
 
-python setup.py build --build-temp=${build_dir} \
-                install --single-version-externally-managed \
-                        --record=/build/python/record.txt
+  relative_build_dir=$(realpath --relative-to=. $build_dir)
 
+  # this is a nightmare, but prevents mutating the source directory
+  # which is bind mounted as readonly
+  python setup.py build_ext --build-temp $relative_build_dir \
+                            --build-lib $relative_build_dir \
+                  build_py --build-lib $relative_build_dir \
+                  egg_info --egg-base $relative_build_dir \
+                  install_lib --build-dir $relative_build_dir \
+                  install --single-version-externally-managed \
+                          --record $relative_build_dir/record.txt
 popd

cpp/Dockerfile

@@ -40,7 +40,7 @@ ADD ci/docker_install_conda.sh \
     ci/conda_env_unix.yml \
     /arrow/ci/
 RUN arrow/ci/docker_install_conda.sh && \
-    conda install -c conda-forge \
+    conda install -q -c conda-forge \
         --file arrow/ci/conda_env_cpp.yml \
         --file arrow/ci/conda_env_unix.yml \
         $EXTRA_CONDA_PKGS && \

dev/docker-compose.yml

@@ -31,31 +31,13 @@ services:
     volumes:
       - ../..:/apache-arrow
 
-  spark_integration:
-    build:
-      context: spark_integration
-    volumes:
-      - ../..:/apache-arrow
-
-  dask_integration:
-    build:
-      context: dask_integration
-    volumes:
-      - ../..:/apache-arrow
-
   gen_apidocs:
     build:
       context: ..
       dockerfile: dev/gen_apidocs/Dockerfile
     volumes:
      - ..:/arrow
 
-  iwyu:
-    build:
-      context: iwyu
-    volumes:
-     - ../..:/apache-arrow
-
   run_site:
     build:
       context: run_site

dev/tasks/tests.yml

@@ -38,13 +38,15 @@ groups:
     - docker-iwyu
     - docker-clang-format
     - docker-pandas-master
-    - docker-hdfs-integration
     - docker-dask-integration
+    - docker-hdfs-integration
+    - docker-spark-integration
 
   integration:
     - docker-pandas-master
     - docker-dask-integration
     - docker-hdfs-integration
+    - docker-spark-integration
 
   cpp-python:
     - docker-cpp
@@ -275,6 +277,18 @@ tasks:
         - docker-compose build hdfs-integration
         - docker-compose run hdfs-integration
 
+  docker-spark-integration:
+    platform: linux
+    template: docker-tests/travis.linux.yml
+    params:
+      environment:
+        PYTHON_VERSION: 3.6
+      commands:
+        - docker-compose build cpp
+        - docker-compose build python
+        - docker-compose build spark-integration
+        - docker-compose run spark-integration
+
   docker-pandas-master:
     platform: linux
     template: docker-tests/travis.linux.yml

docker-compose.yml

@@ -21,13 +21,19 @@ version: '3.5'
 
 x-ubuntu-volumes:
   &ubuntu-volumes
-  - .:/arrow:delegated
-  - ${ARROW_DOCKER_CACHE_DIR:-./docker_cache}/ubuntu:/build:delegated
+  - .:/arrow:ro  # ensures that docker won't contaminate the host directory
+  - ubuntu-cache:/build:delegated
 
 x-alpine-volumes:
   &alpine-volumes
-  - .:/arrow:delegated
-  - ${ARROW_DOCKER_CACHE_DIR:-./docker_cache}/alpine:/build:delegated
+  - .:/arrow:ro  # ensures that docker won't contaminate the host directory
+  - alpine-cache:/build:delegated
+
+volumes:
+  ubuntu-cache:
+  alpine-cache:
+  maven-cache:
+  spark-cache:
 
 services:
 
@@ -105,8 +111,8 @@ services:
       context: .
       dockerfile: java/Dockerfile
     volumes:
-      - .:/arrow:delegated
-      - $HOME/.m2:/root/.m2:delegated
+      - .:/arrow:ro  # ensures that docker won't contaminate the host directory
+      - maven-cache:/root/.m2:delegated
 
   js:
     image: arrow:js
@@ -267,14 +273,14 @@ services:
     links:
       - hdfs-namenode:hdfs-namenode
 
-  # TODO(kszucs): pass hdfs client version explicitly as a build argument
   hdfs-integration:
     # Usage:
     #   export PYTHON_VERSION=3.6
     #   docker-compose build cpp
     #   docker-compose build python
     #   docker-compose build hdfs-integration
     #   docker-compose run hdfs-integration
+    image: arrow:hdfs-${HDFS_VERSION:-2.6.5}
     links:
       - hdfs-namenode:hdfs-namenode
       - hdfs-datanode-1:hdfs-datanode-1
@@ -286,6 +292,8 @@ services:
     build:
       context: .
       dockerfile: integration/hdfs/Dockerfile
+      args:
+        HDFS_VERSION: ${HDFS_VERSION:-2.6.5}
     volumes: *ubuntu-volumes
 
   # TODO(kszucs): pass dask version explicitly as a build argument
@@ -296,10 +304,29 @@ services:
     #   docker-compose build python
     #   docker-compose build dask-integration
     #   docker-compose run dask-integration
+    image: arrow:dask-integration
     build:
       context: .
       dockerfile: integration/dask/Dockerfile
     volumes: *ubuntu-volumes
 
+  spark-integration:
+    # Usage:
+    #   export PYTHON_VERSION=3.6
+    #   docker-compose build cpp
+    #   docker-compose build python
+    #   docker-compose build spark-integration
+    #   docker-compose run spark-integration
+    image: arrow:spark-${SPARK_VERSION:-2.4.0}
+    environment:
+      - SPARK_VERSION=${SPARK_VERSION:-2.4.0}
+    build:
+      context: .
+      dockerfile: integration/spark/Dockerfile
+    volumes:
+      - .:/arrow:ro  # ensures that docker won't contaminate the host directory
+      - ubuntu-cache:/build:delegated
+      - maven-cache:/root/.m2:delegated
+      - spark-cache:/spark:delegated
+
   # TODO(kszucs): hive-integration
-  # TODO(kszucs): spark-integration