Run all Python linux builds in a Dockerized Travis CI entry.

.travis.yml

@@ -116,36 +116,14 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1
   - name: "Python 3.6 unit tests w/ Valgrind, conda-forge toolchain, coverage"
-    compiler: gcc
     language: cpp
     os: linux
-    jdk: openjdk8
-    env:
-    # Valgrind is needed for the Plasma store tests
-    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
-    - ARROW_TRAVIS_COVERAGE=1
-    - ARROW_TRAVIS_FLIGHT=1
-    - ARROW_TRAVIS_OPTIONAL_INSTALL=1
-    - ARROW_TRAVIS_PYTHON_DOCS=1
-    - ARROW_TRAVIS_PYTHON_JVM=1
-    - ARROW_TRAVIS_USE_SYSTEM_JAVA=1
-    - ARROW_TRAVIS_USE_TOOLCHAIN=1
-    - ARROW_TRAVIS_VALGRIND=1
-    # TODO(wesm): Run the benchmarks outside of Travis
-    # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1
     before_script:
     - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi
-    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh
+    - docker-compose pull python
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1
-    - export ARROW_TRAVIS_PYTHON_GANDIVA=1
-    # Only run Plasma tests with valgrind in one of the Python builds because
-    # they are slow
-    - export PLASMA_VALGRIND=1
-    - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 || travis_terminate 1
-    - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh
+    - docker-compose run python
+    # TODO(wesm): Run the benchmarks outside of Travis
   - name: "[OS X] C++ w/ XCode 9.3"
     compiler: clang
     osx_image: xcode9.3

ci/conda_env_python.yml

@@ -15,13 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+coverage
 cython=0.29.7
 cloudpickle
 hypothesis
 numpy>=1.14
 pandas
 pytest
-pytest-faulthandler
 pytz
 setuptools
 setuptools_scm=3.2.0

ci/docker_build_and_test_python.sh

@@ -20,4 +20,4 @@ set -e
 
 /arrow/ci/docker_build_cpp.sh
 /arrow/ci/docker_build_python.sh
-pytest -v --pyargs pyarrow
+/arrow/ci/docker_test_python.sh

ci/docker_build_cpp.sh

@@ -52,6 +52,9 @@ cmake -GNinja \
       -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \
       -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \
       -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \
+      -DARROW_OPTIONAL_INSTALL=${ARROW_OPTIONAL_INSTALL:-OFF} \
+      -DARROW_TENSORFLOW=${ARROW_TENSORFLOW:-OFF} \
+      -DARROW_GENERATE_COVERAGE={ARROW_GENERATE_COVERAGE:-OFF} \
       -DPARQUET_REQUIRE_ENCRYPTION=${ARROW_WITH_OPENSSL:-ON} \
       -DCMAKE_CXX_FLAGS=$CXXFLAGS \
       -DBOOST_SOURCE=${ARROW_BOOST_SOURCE:-AUTO} \

ci/docker_build_python.sh

@@ -30,6 +30,8 @@ export SETUPTOOLS_SCM_VERSION_WRITE_TO_PREFIX=$build_dir
 export PYARROW_WITH_ORC=${PYARROW_WITH_ORC:-1}
 export PYARROW_WITH_PARQUET=${PYARROW_WITH_PARQUET:-1}
 export PYARROW_WITH_PLASMA=${PYARROW_WITH_PLASMA:-1}
+export PYARROW_WITH_FLIGHT=${PYARROW_WITH_FLIGHT:-1}
+export PYARROW_WITH_GANDIVA=${PYARROW_WITH_GANDIVA:-1}
 
 # Build pyarrow
 pushd ${source_dir}

ci/docker_test_python.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Set up testing data
+pushd arrow
+git submodule update --init
+popd
+export ARROW_TEST_DATA=/arrow/testing/data
+
+## Set up huge pages for plasma test
+#sudo sysctl -w vm.nr_hugepages=2048
+#sudo mkdir -p /mnt/hugepages
+#sudo mount -t hugetlbfs -o uid=`id -u` -o gid=`id -g` none /mnt/hugepages
+#sudo bash -c "echo `id -g` > /proc/sys/vm/hugetlb_shm_group"
+#sudo bash -c "echo 2048 > /proc/sys/vm/nr_hugepages"
+
+python --version
+
+pytest -v --pyargs pyarrow
+
+# Basic sanity checks
+python -c "import pyarrow.parquet"
+python -c "import pyarrow.plasma"
+python -c "import pyarrow.orc"
+
+# Ensure we do eagerly import pandas (or other expensive imports)
+python < /arrow/python/scripts/test_imports.py
+
+echo "PLASMA_VALGRIND: $PLASMA_VALGRIND"
+
+# Check Cython coverage was correctly captured in $COVERAGE_FILE
+coverage report -i --include="*/lib.pyx"
+coverage report -i --include="*/memory.pxi"
+coverage report -i --include="*/_parquet.pyx"
+
+# Generate XML file for CodeCov
+coverage xml -i -o $SOURCE_DIR/coverage.xml
+
+# Capture C++ coverage info
+pushd $SOURCE_DIR
+lcov --directory . --capture --no-external --output-file coverage-python-tests.info \
+    2>&1 | grep -v "ignoring data for external file"
+lcov --add-tracefile coverage-python-tests.info \
+    --output-file $SOURCE_DIR/coverage.info
+rm coverage-python-tests.info
+popd
+
+# Build docs
+pushd ../cpp/apidoc
+doxygen
+popd
+cd ../docs
+sphinx-build -q -b html -d _build/doctrees -W source _build/html
+
+popd  # $ARROW_PYTHON_DIR
+
+# Check the ASV benchmarking setup.
+# Unfortunately this won't ensure that all benchmarks succeed
+# (see https://github.com/airspeed-velocity/asv/issues/449)
+source deactivate
+conda create -y -q -n pyarrow_asv python=$PYTHON_VERSION
+conda activate pyarrow_asv
+pip install -q git+https://github.com/pitrou/asv.git@customize_commands
+
+pushd $SOURCE_DIR
+# Workaround for https://github.com/airspeed-velocity/asv/issues/631
+git fetch --depth=100 origin master:master
+# Generate machine information (mandatory)
+asv machine --yes
+# Run benchmarks on the changeset being tested
+asv run --no-pull --show-stderr --quick HEAD^!
+popd  # $ARROW_PYTHON_DIR
+
+#- $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh

python/Dockerfile

@@ -19,11 +19,14 @@ FROM arrow:cpp
 
 # install python specific packages
 ARG PYTHON_VERSION=3.6
-COPY ci/conda_env_python.yml /arrow/ci/
+COPY ci/conda_env_python.yml ci/conda_env_sphinx.yml /arrow/ci/
 RUN conda install -q \
         --file arrow/ci/conda_env_python.yml \
+        --file arrow/ci/conda_env_sphinx.yml \
+        --file arrow/ci/conda_env_gandiva.yml \
         python=$PYTHON_VERSION \
         nomkl && \
+    conda install -y -q pytorch torchvision -c soumith && \
     conda clean --all
 
 ENV ARROW_PYTHON=ON \