ARROW-5466: [Java][CI] Dockerize Java CI, run all JDK builds in singl…

…e Travis entry

Since OpenJDK9 has been superseded by OpenJDK11, it is not available in package repositories, so I'm not sure it's worth maintaining a build for this.

I have pushed a pre-built Docker image for this to

https://cloud.docker.com/u/ursalab/repository/docker/ursalab/arrow-ci-java-all-jdks

Author: Wes McKinney <wesm+git@apache.org>

Closes #4761 from wesm/java-dockerify and squashes the following commits:

8b81037 <Wes McKinney> Actually run Java unit tests
cf568a2 <Wes McKinney> Code review feedback
6cb5e3f <Wes McKinney> Build Javadoc in docker-compose job
d9c3900 <Wes McKinney> Run all Java builds in a single Dockerized Travis CI entry

.travis.yml

@@ -199,32 +199,14 @@ matrix:
     - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker-compose pull python-manylinux1; fi
     script:
     - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
-  - name: "Java w/ OpenJDK 8"
-    language: java
-    os: linux
-    jdk: openjdk8
-    before_script:
-    - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
-    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
-    - $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh
-  - name: "Java w/ OpenJDK 9"
-    language: java
-    os: linux
-    jdk: openjdk9
-    before_script:
-    - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
-    script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
-  - name: "Java w/ OpenJDK 11"
-    language: java
+  - name: "Java OpenJDK8 and OpenJDK11"
+    language: cpp
     os: linux
-    jdk: openjdk11
     before_script:
     - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
+    - docker-compose pull java-all-jdks
     script:
-    - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+    - docker-compose run java-all-jdks
   - name: "Integration w/ OpenJDK 8, conda-forge toolchain"
     language: java
     os: linux

ci/docker_build_java.sh

@@ -25,10 +25,23 @@ mkdir -p /build/java
 
 arrow_src=/build/java/arrow
 
+# Remove any pre-existing artifacts
+rm -rf $arrow_src
+
 pushd /arrow
-  rsync -a header java format integration $arrow_src
+rsync -a header java format integration $arrow_src
 popd
 
+JAVA_ARGS=
+if [ "$ARROW_JAVA_RUN_TESTS" != "1" ]; then
+  JAVA_ARGS=-DskipTests
+fi
+
 pushd $arrow_src/java
-  mvn -B -DskipTests -Drat.skip=true install
+mvn -B $JAVA_ARGS -Drat.skip=true install
+
+if [ "$ARROW_JAVADOC" == "1" ]; then
+  export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
+  mvn -B site
+fi
 popd

ci/travis_script_javadoc.sh → ci/docker_java_test_all.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -19,13 +18,16 @@
 
 set -e
 
-source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-JAVA_DIR=${TRAVIS_BUILD_DIR}/java
+export ARROW_TEST_DATA=/arrow/testing/data
 
-pushd $JAVA_DIR
+export ARROW_JAVA_RUN_TESTS=1
 
-export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
-$TRAVIS_MVN -B site
+export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+export ARROW_JAVADOC=1
+bash $SOURCE_DIR/docker_build_java.sh
 
-popd
+export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+export ARROW_JAVADOC=0
+bash $SOURCE_DIR/docker_build_java.sh

docker-compose.yml

@@ -292,6 +292,18 @@ services:
       - .:/arrow:ro  # ensures that docker won't contaminate the host directory
       - maven-cache:/root/.m2:delegated
 
+  java-all-jdks:
+    # Usage:
+    #   docker-compose build java-all-jdks
+    #   docker-compose run java-all-jdks
+    image: ursalab/arrow-ci-java-all-jdks:latest
+    build:
+      context: .
+      dockerfile: java/Dockerfile.all-jdks
+    volumes:
+      - .:/arrow:ro  # ensures that docker won't contaminate the host directory
+      - maven-cache:/root/.m2:delegated
+
   js:
     image: arrow:js
     build:

java/Dockerfile.all-jdks

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:18.04
+
+# install build essentials
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+        wget \
+        software-properties-common \
+        ca-certificates \
+        maven \
+        rsync \
+        tzdata \
+        openjdk-8-jdk \
+        openjdk-11-jdk && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Test all supported JDKs
+CMD ["arrow/ci/docker_java_test_all.sh"]