apacheGH-38402: [CI][Integration] Provide wrapper scripts for integra…

…tion testing (apache#38403)

### Rationale for this change

It is currently cumbersome to run integration tests from an other repository, as each tested implementation must be built explicitly.

See explanatory comment at apache/arrow-rs#4957 (comment)

### What changes are included in this PR?

Provide a wrapper script to automate building the implementations being tested.

Allow usage of environment variables such as `ARROW_INTEGRATION_CPP`, etc. to selectively disable implementations.

### Are these changes tested?

Yes, by construction.

### Are there any user-facing changes?

No.
* Closes: apache#38402

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

ci/scripts/go_build.sh

@@ -42,7 +42,9 @@ go install -v ./...
 
 popd
 
-if [[ -n "${ARROW_GO_INTEGRATION}" ]]; then
+: ${ARROW_INTEGRATION_GO:=ON}
+
+if [ "${ARROW_INTEGRATION_GO}" == "ON" ]; then
     pushd ${source_dir}/arrow/internal/cdata_integration
 
     case "$(uname)" in

ci/scripts/integration_arrow.sh

@@ -20,11 +20,25 @@
 set -ex
 
 arrow_dir=${1}
+build_dir=${2}
+
 gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration
 
+: ${ARROW_INTEGRATION_CPP:=ON}
+: ${ARROW_INTEGRATION_CSHARP:=ON}
+: ${ARROW_INTEGRATION_GO:=ON}
+: ${ARROW_INTEGRATION_JAVA:=ON}
+: ${ARROW_INTEGRATION_JS:=ON}
+
 pip install -e $arrow_dir/dev/archery[integration]
+
 # For C Data Interface testing
-pip install jpype1 pythonnet
+if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then
+    pip install pythonnet
+fi
+if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then
+    pip install jpype1
+fi
 
 # Get more detailed context on crashes
 export PYTHONFAULTHANDLER=1
@@ -34,11 +48,11 @@ time archery integration \
     --run-c-data \
     --run-ipc \
     --run-flight \
-    --with-cpp=1 \
-    --with-csharp=1 \
-    --with-java=1 \
-    --with-js=1 \
-    --with-go=1 \
+    --with-cpp=$([ "$ARROW_INTEGRATION_CPP" == "ON" ] && echo "1" || echo "0") \
+    --with-csharp=$([ "$ARROW_INTEGRATION_CSHARP" == "ON" ] && echo "1" || echo "0") \
+    --with-go=$([ "$ARROW_INTEGRATION_GO" == "ON" ] && echo "1" || echo "0") \
+    --with-java=$([ "$ARROW_INTEGRATION_JAVA" == "ON" ] && echo "1" || echo "0") \
+    --with-js=$([ "$ARROW_INTEGRATION_JS" == "ON" ] && echo "1" || echo "0") \
     --gold-dirs=$gold_dir/0.14.1 \
     --gold-dirs=$gold_dir/0.17.1 \
     --gold-dirs=$gold_dir/1.0.0-bigendian \

ci/scripts/integration_arrow_build.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+arrow_dir=${1}
+build_dir=${2}
+
+: ${ARROW_INTEGRATION_CPP:=ON}
+: ${ARROW_INTEGRATION_CSHARP:=ON}
+: ${ARROW_INTEGRATION_GO:=ON}
+: ${ARROW_INTEGRATION_JAVA:=ON}
+: ${ARROW_INTEGRATION_JS:=ON}
+
+${arrow_dir}/ci/scripts/rust_build.sh ${arrow_dir} ${build_dir}
+
+if [ "${ARROW_INTEGRATION_CPP}" == "ON" ]; then
+    ${arrow_dir}/ci/scripts/cpp_build.sh ${arrow_dir} ${build_dir}
+fi
+
+if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then
+    ${arrow_dir}/ci/scripts/csharp_build.sh ${arrow_dir} ${build_dir}
+fi
+
+if [ "${ARROW_INTEGRATION_GO}" == "ON" ]; then
+    ${arrow_dir}/ci/scripts/go_build.sh ${arrow_dir} ${build_dir}
+fi
+
+if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then
+    export ARROW_JAVA_CDATA="ON"
+    export JAVA_JNI_CMAKE_ARGS="-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DARROW_JAVA_JNI_ENABLE_C=ON"
+
+    ${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java/$(arch)
+    ${arrow_dir}/ci/scripts/java_build.sh ${arrow_dir} ${build_dir} /tmp/dist/java
+fi
+
+if [ "${ARROW_INTEGRATION_JS}" == "ON" ]; then
+    ${arrow_dir}/ci/scripts/js_build.sh ${arrow_dir} ${build_dir}
+fi

docker-compose.yml

@@ -1733,19 +1733,8 @@ services:
       ARCHERY_INTEGRATION_WITH_RUST: 0
       # Tell Archery where the arrow C++ binaries are located
       ARROW_CPP_EXE_PATH: /build/cpp/debug
-      ARROW_GO_INTEGRATION: 1
-      ARROW_JAVA_CDATA: "ON"
-      JAVA_JNI_CMAKE_ARGS: >-
-        -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF
-        -DARROW_JAVA_JNI_ENABLE_C=ON
     command:
-      ["/arrow/ci/scripts/rust_build.sh /arrow /build &&
-        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
-        /arrow/ci/scripts/csharp_build.sh /arrow /build &&
-        /arrow/ci/scripts/go_build.sh /arrow &&
-        /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/$$(arch) &&
-        /arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java &&
-        /arrow/ci/scripts/js_build.sh /arrow /build &&
+      ["/arrow/ci/scripts/integration_arrow_build.sh /arrow /build &&
         /arrow/ci/scripts/integration_arrow.sh /arrow /build"]
 
   ################################ Docs #######################################