ARROW-7930: [CI][Python] Test jpype integration

Closes #6484 from pitrou/ARROW-7930-jpype-integration and squashes the following commits:

686ec65 <Antoine Pitrou> ARROW-7930:  Test jpype integration

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>

ci/docker/conda-python-jpype.dockerfile

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG repo
+ARG arch=amd64
+ARG python=3.6
+FROM ${repo}:${arch}-conda-python-${python}
+
+ARG jdk=11
+ARG maven=3.6
+RUN conda install -q \
+        maven=${maven} \
+        openjdk=${jdk} \
+        jpype1 && \
+    conda clean --all

ci/scripts/python_test.sh

@@ -21,6 +21,7 @@ set -ex
 
 arrow_dir=${1}
 
+export ARROW_SOURCE_DIR=${arrow_dir}
 export ARROW_TEST_DATA=${arrow_dir}/testing/data
 export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
 export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}

dev/tasks/tasks.yml

@@ -123,6 +123,7 @@ groups:
     - test-conda-python-3.7-turbodbc-master
     - test-conda-python-3.7-hdfs-2.9.2
     - test-conda-python-3.7-spark-master
+    - test-conda-python-3.8-jpype
     # - test-conda-cpp-hiveserver2
 
   test:
@@ -203,6 +204,7 @@ groups:
     - test-conda-python-3.7-turbodbc-master
     - test-conda-python-3.7-hdfs-2.9.2
     - test-conda-python-3.7-spark-master
+    - test-conda-python-3.8-jpype
     # - test-conda-cpp-hiveserver2
 
   ######################## Tasks to run regularly #############################
@@ -286,6 +288,7 @@ groups:
     - test-conda-python-3.7-turbodbc-master
     - test-conda-python-3.7-hdfs-2.9.2
     - test-conda-python-3.7-spark-master
+    - test-conda-python-3.8-jpype
 
 tasks:
   # arbitrary_task_name:
@@ -2013,6 +2016,20 @@ tasks:
         - docker-compose build --no-cache conda-python-dask
         - docker-compose run conda-python-dask
 
+  test-conda-python-3.8-jpype:
+    ci: circle
+    platform: linux
+    template: docker-tests/circle.linux.yml
+    params:
+      commands:
+        - export PYTHON=3.8
+        - docker-compose pull --ignore-pull-failures conda-cpp
+        - docker-compose pull --ignore-pull-failures conda-python
+        - docker-compose build conda-cpp
+        - docker-compose build conda-python
+        - docker-compose build --no-cache conda-python-jpype
+        - docker-compose run conda-python-jpype
+
   test-conda-python-3.7-turbodbc-latest:
     ci: circle
     platform: linux

docker-compose.yml

@@ -608,6 +608,34 @@ services:
         /arrow/ci/scripts/python_build.sh /arrow /build &&
         /arrow/ci/scripts/integration_dask.sh"]
 
+  conda-python-jpype:
+    # Usage:
+    #   docker-compose build conda-cpp
+    #   docker-compose build conda-python
+    #   docker-compose build conda-python-jpype
+    #   docker-compose run --rm conda-python-jpype
+    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype
+    build:
+      context: .
+      dockerfile: ci/docker/conda-python-jpype.dockerfile
+      cache_from:
+        - ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype
+      args:
+        repo: ${REPO}
+        arch: ${ARCH}
+        python: ${PYTHON}
+    shm_size: *shm-size
+    environment:
+      <<: *ccache
+      ARROW_FLIGHT: "OFF"
+      ARROW_GANDIVA: "OFF"
+    volumes: *conda-volumes
+    command:
+      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
+        /arrow/ci/scripts/python_build.sh /arrow /build &&
+        /arrow/ci/scripts/java_build.sh /arrow /build &&
+        /arrow/ci/scripts/python_test.sh /arrow"]
+
   conda-python-turbodbc:
     # Possible $DASK parameters:
     #  - `latest`: latest release

python/pyarrow/tests/test_jvm.py

@@ -30,18 +30,19 @@
 @pytest.fixture(scope="session")
 def root_allocator():
     # This test requires Arrow Java to be built in the same source tree
-    pom_path = os.path.join(
-        os.path.dirname(__file__), '..', '..', '..',
-        'java', 'pom.xml')
+    try:
+        arrow_dir = os.environ["ARROW_SOURCE_DIR"]
+    except KeyError:
+        arrow_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..')
+    pom_path = os.path.join(arrow_dir, 'java', 'pom.xml')
     tree = ET.parse(pom_path)
     version = tree.getroot().find(
         'POM:version',
         namespaces={
             'POM': 'http://maven.apache.org/POM/4.0.0'
         }).text
     jar_path = os.path.join(
-        os.path.dirname(__file__), '..', '..', '..',
-        'java', 'tools', 'target',
+        arrow_dir, 'java', 'tools', 'target',
         'arrow-tools-{}-jar-with-dependencies.jar'.format(version))
     jar_path = os.getenv("ARROW_TOOLS_JAR", jar_path)
     jpype.startJVM(jpype.getDefaultJVMPath(), "-Djava.class.path=" + jar_path)