diff --git a/.env b/.env index 6746892fd4ed8..44fd0169e448e 100644 --- a/.env +++ b/.env @@ -61,7 +61,7 @@ GCC_VERSION="" GO=1.19.13 STATICCHECK=v0.4.5 HDFS=3.2.1 -JDK=8 +JDK=17 KARTOTHEK=latest # LLVM 12 and GCC 11 reports -Wmismatched-new-delete. LLVM=14 diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh index 6d1d4befa6964..4ea576df1c472 100755 --- a/ci/scripts/integration_arrow.sh +++ b/ci/scripts/integration_arrow.sh @@ -43,12 +43,12 @@ fi # Get more detailed context on crashes export PYTHONFAULTHANDLER=1 +# --run-ipc \ +# --run-flight \ # Rust can be enabled by exporting ARCHERY_INTEGRATION_WITH_RUST=1 time archery integration \ --run-c-data \ - --run-ipc \ - --run-flight \ --with-cpp=$([ "$ARROW_INTEGRATION_CPP" == "ON" ] && echo "1" || echo "0") \ --with-csharp=$([ "$ARROW_INTEGRATION_CSHARP" == "ON" ] && echo "1" || echo "0") \ --with-go=$([ "$ARROW_INTEGRATION_GO" == "ON" ] && echo "1" || echo "0") \ diff --git a/dev/archery/archery/integration/tester_cpp.py b/dev/archery/archery/integration/tester_cpp.py index 02c110c0e20a4..2a47bc830886a 100644 --- a/dev/archery/archery/integration/tester_cpp.py +++ b/dev/archery/archery/integration/tester_cpp.py @@ -167,6 +167,7 @@ def make_c_data_importer(self): @functools.lru_cache def _load_ffi(ffi, lib_path=_ARROW_DLL): + os.environ['ARROW_DEBUG_MEMORY_POOL'] = 'trap' ffi.cdef(_cpp_c_data_entrypoints) dll = ffi.dlopen(lib_path) dll.ArrowCpp_CDataIntegration_ExportSchemaFromJson diff --git a/dev/archery/archery/integration/tester_csharp.py b/dev/archery/archery/integration/tester_csharp.py index 7dca525673ba6..eccc19ea32709 100644 --- a/dev/archery/archery/integration/tester_csharp.py +++ b/dev/archery/archery/integration/tester_csharp.py @@ -38,6 +38,8 @@ def _load_clr(): global _clr_loaded if not _clr_loaded: _clr_loaded = True + os.environ['DOTNET_GCHeapHardLimit'] = '0xC800000' # 200 MiB + os.environ['DOTNET_gcConcurrent'] = '1' import pythonnet pythonnet.load("coreclr") import clr diff --git a/dev/archery/archery/integration/tester_go.py b/dev/archery/archery/integration/tester_go.py index 5368f06a318e5..7d3f5b9c8a17b 100644 --- a/dev/archery/archery/integration/tester_go.py +++ b/dev/archery/archery/integration/tester_go.py @@ -159,6 +159,9 @@ def make_c_data_importer(self): @functools.lru_cache def _load_ffi(ffi, lib_path=_INTEGRATION_DLL): + # XXX these don't seem to have any effect? + os.environ['GOMEMLIMIT'] = '200MiB' + os.environ['GODEBUG'] = 'gctrace=1,clobberfree=1' ffi.cdef(_go_c_data_entrypoints) dll = ffi.dlopen(lib_path) return dll diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py index 5684798d794ad..d71479986c1da 100644 --- a/dev/archery/archery/integration/tester_java.py +++ b/dev/archery/archery/integration/tester_java.py @@ -34,11 +34,13 @@ def load_version_from_pom(): return version_tag.text -# XXX Should we add "-Darrow.memory.debug.allocator=true"? It adds a couple -# minutes to total CPU usage of the integration test suite. +# NOTE: we don't add "-Darrow.memory.debug.allocator=true" here as it adds a +# couple minutes to total CPU usage of the integration test suite +# (see setup_jpype() below). _JAVA_OPTS = [ "-Dio.netty.tryReflectionSetAccessible=true", "-Darrow.struct.conflict.policy=CONFLICT_APPEND", + "--add-opens=java.base/java.nio=ALL-UNNAMED", ] _arrow_version = load_version_from_pom() @@ -80,7 +82,12 @@ def setup_jpype(): jar_path = f"{_ARROW_TOOLS_JAR}:{_ARROW_C_DATA_JAR}" # XXX Didn't manage to tone down the logging level here (DEBUG -> INFO) jpype.startJVM(jpype.getDefaultJVMPath(), - "-Djava.class.path=" + jar_path, *_JAVA_OPTS) + "-Djava.class.path=" + jar_path, + # This flag is too heavy for IPC and Flight tests + "-Darrow.memory.debug.allocator=true", + # Reduce internal use of signals by the JVM + "-Xrs", + *_JAVA_OPTS) class _CDataBase: