apacheGH-38684: [Integration] Try to strengthen C Data Interface testing

pitrou · Nov 22, 2023 · d761871 · d761871
1 parent 8627921
commit d761871
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 6 deletions.
diff --git a/.env b/.env
@@ -61,7 +61,7 @@ GCC_VERSION=""
 GO=1.19.13
 STATICCHECK=v0.4.5
 HDFS=3.2.1
-JDK=8
+JDK=17
 KARTOTHEK=latest
 # LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
 LLVM=14

diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh
@@ -43,12 +43,12 @@ fi
 # Get more detailed context on crashes
 export PYTHONFAULTHANDLER=1
 
+#     --run-ipc \
+#     --run-flight \
 
 # Rust can be enabled by exporting ARCHERY_INTEGRATION_WITH_RUST=1
 time archery integration \
     --run-c-data \
-    --run-ipc \
-    --run-flight \
     --with-cpp=$([ "$ARROW_INTEGRATION_CPP" == "ON" ] && echo "1" || echo "0") \
     --with-csharp=$([ "$ARROW_INTEGRATION_CSHARP" == "ON" ] && echo "1" || echo "0") \
     --with-go=$([ "$ARROW_INTEGRATION_GO" == "ON" ] && echo "1" || echo "0") \

diff --git a/dev/archery/archery/integration/tester_cpp.py b/dev/archery/archery/integration/tester_cpp.py
@@ -167,6 +167,7 @@ def make_c_data_importer(self):
 
 @functools.lru_cache
 def _load_ffi(ffi, lib_path=_ARROW_DLL):
+    os.environ['ARROW_DEBUG_MEMORY_POOL'] = 'trap'
     ffi.cdef(_cpp_c_data_entrypoints)
     dll = ffi.dlopen(lib_path)
     dll.ArrowCpp_CDataIntegration_ExportSchemaFromJson

diff --git a/dev/archery/archery/integration/tester_csharp.py b/dev/archery/archery/integration/tester_csharp.py
@@ -38,6 +38,8 @@ def _load_clr():
     global _clr_loaded
     if not _clr_loaded:
         _clr_loaded = True
+        os.environ['DOTNET_GCHeapHardLimit'] = '0xC800000'  # 200 MiB
+        os.environ['DOTNET_gcConcurrent'] = '1'
         import pythonnet
         pythonnet.load("coreclr")
         import clr

diff --git a/dev/archery/archery/integration/tester_go.py b/dev/archery/archery/integration/tester_go.py
@@ -159,6 +159,9 @@ def make_c_data_importer(self):
 
 @functools.lru_cache
 def _load_ffi(ffi, lib_path=_INTEGRATION_DLL):
+    # XXX these don't seem to have any effect?
+    os.environ['GOMEMLIMIT'] = '200MiB'
+    os.environ['GODEBUG'] = 'gctrace=1,clobberfree=1'
     ffi.cdef(_go_c_data_entrypoints)
     dll = ffi.dlopen(lib_path)
     return dll

diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py
@@ -34,11 +34,13 @@ def load_version_from_pom():
     return version_tag.text
 
 
-# XXX Should we add "-Darrow.memory.debug.allocator=true"? It adds a couple
-# minutes to total CPU usage of the integration test suite.
+# NOTE: we don't add "-Darrow.memory.debug.allocator=true" here as it adds a
+# couple minutes to total CPU usage of the integration test suite
+# (see setup_jpype() below).
 _JAVA_OPTS = [
     "-Dio.netty.tryReflectionSetAccessible=true",
     "-Darrow.struct.conflict.policy=CONFLICT_APPEND",
+    "--add-opens=java.base/java.nio=ALL-UNNAMED",
 ]
 
 _arrow_version = load_version_from_pom()
@@ -80,7 +82,12 @@ def setup_jpype():
     jar_path = f"{_ARROW_TOOLS_JAR}:{_ARROW_C_DATA_JAR}"
     # XXX Didn't manage to tone down the logging level here (DEBUG -> INFO)
     jpype.startJVM(jpype.getDefaultJVMPath(),
-                   "-Djava.class.path=" + jar_path, *_JAVA_OPTS)
+                   "-Djava.class.path=" + jar_path,
+                   # This flag is too heavy for IPC and Flight tests
+                   "-Darrow.memory.debug.allocator=true",
+                   # Reduce internal use of signals by the JVM
+                   "-Xrs",
+                   *_JAVA_OPTS)
 
 
 class _CDataBase: