apacheGH-38684: [Integration] Try to strengthen C Data Interface test…

…ing (apache#38846)

### Rationale for this change

C Data Interface integration tests sometimes crash with a complicated traceback involving Java, Go and C# signal handlers.

### What changes are included in this PR?

* Update JDK version in integration build
* Disable signal-based memory management in the JVM so as to improve cooperation with other runtimes
* Set memory limits to the various managed runtimes (Java, .Net, Go)
* Enable some checks in the Go runtime
* Enable debug allocator in Arrow Java

### Are these changes tested?

Yes. I am not able to reproduce any sporadic crash using these changes.

### Are there any user-facing changes?

No.

* Closes: apache#38684

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Antoine Pitrou <antoine@python.org>

.github/workflows/java.yml

@@ -48,7 +48,6 @@ env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
-
   ubuntu:
     name: AMD64 Ubuntu 22.04 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
     runs-on: ubuntu-latest

.github/workflows/java_jni.yml

@@ -48,7 +48,6 @@ env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
-
   docker:
     name: AMD64 manylinux2014 Java JNI
     runs-on: ubuntu-latest

ci/scripts/integration_arrow.sh

@@ -43,6 +43,11 @@ fi
 # Get more detailed context on crashes
 export PYTHONFAULTHANDLER=1
 
+# Due to how Go reads environment variables, we have to set them from the calling
+# process, or they would get ignored.
+# (see https://forum.golangbridge.org/t/are-godebug-and-other-env-vars-ignored-when-loading-a-go-dll-from-foreign-code/33694)
+export GOMEMLIMIT=200MiB
+export GODEBUG=gctrace=1,clobberfree=1
 
 # Rust can be enabled by exporting ARCHERY_INTEGRATION_WITH_RUST=1
 time archery integration \

dev/archery/archery/integration/tester_cpp.py

@@ -167,6 +167,7 @@ def make_c_data_importer(self):
 
 @functools.lru_cache
 def _load_ffi(ffi, lib_path=_ARROW_DLL):
+    os.environ['ARROW_DEBUG_MEMORY_POOL'] = 'trap'
     ffi.cdef(_cpp_c_data_entrypoints)
     dll = ffi.dlopen(lib_path)
     dll.ArrowCpp_CDataIntegration_ExportSchemaFromJson

dev/archery/archery/integration/tester_csharp.py

@@ -38,6 +38,7 @@ def _load_clr():
     global _clr_loaded
     if not _clr_loaded:
         _clr_loaded = True
+        os.environ['DOTNET_GCHeapHardLimit'] = '0xC800000'  # 200 MiB
         import pythonnet
         pythonnet.load("coreclr")
         import clr

dev/archery/archery/integration/tester_go.py

@@ -159,6 +159,9 @@ def make_c_data_importer(self):
 
 @functools.lru_cache
 def _load_ffi(ffi, lib_path=_INTEGRATION_DLL):
+    # NOTE that setting Go environment variables here (such as GODEBUG)
+    # would be ignored by the Go runtime. The environment variables need
+    # to be set from the process calling Archery.
     ffi.cdef(_go_c_data_entrypoints)
     dll = ffi.dlopen(lib_path)
     return dll

dev/archery/archery/integration/tester_java.py

@@ -34,11 +34,13 @@ def load_version_from_pom():
     return version_tag.text
 
 
-# XXX Should we add "-Darrow.memory.debug.allocator=true"? It adds a couple
-# minutes to total CPU usage of the integration test suite.
+# NOTE: we don't add "-Darrow.memory.debug.allocator=true" here as it adds a
+# couple minutes to total CPU usage of the integration test suite
+# (see setup_jpype() below).
 _JAVA_OPTS = [
     "-Dio.netty.tryReflectionSetAccessible=true",
     "-Darrow.struct.conflict.policy=CONFLICT_APPEND",
+    "--add-opens=java.base/java.nio=ALL-UNNAMED",
 ]
 
 _arrow_version = load_version_from_pom()
@@ -80,7 +82,12 @@ def setup_jpype():
     jar_path = f"{_ARROW_TOOLS_JAR}:{_ARROW_C_DATA_JAR}"
     # XXX Didn't manage to tone down the logging level here (DEBUG -> INFO)
     jpype.startJVM(jpype.getDefaultJVMPath(),
-                   "-Djava.class.path=" + jar_path, *_JAVA_OPTS)
+                   "-Djava.class.path=" + jar_path,
+                   # This flag is too heavy for IPC and Flight tests
+                   "-Darrow.memory.debug.allocator=true",
+                   # Reduce internal use of signals by the JVM
+                   "-Xrs",
+                   *_JAVA_OPTS)
 
 
 class _CDataBase:

docker-compose.yml

@@ -1706,7 +1706,8 @@ services:
       args:
         repo: ${REPO}
         arch: ${ARCH}
-        jdk: ${JDK}
+        # Use a newer JDK as it seems to improve stability
+        jdk: 17
         # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should
         # be set to ${MAVEN}
         maven: 3.5