ARROW-7119: [C++][CI] Show automatic backtraces

Automatically generate coredumps and backtraces for the failing C++ tests on both Linux (including docker) and macOS.

Closes #6202 from kszucs/coredump and squashes the following commits:

f8200da <Krisztián Szűcs> remove sudo from the comment
1e75d93 <Krisztián Szűcs> resolve review issues
ef5ac74 <Krisztián Szűcs> revert segfault enforcing change
deaca22 <Krisztián Szűcs> remove debug commands
467e000 <Krisztián Szűcs> set ulimit within the same shell
d2af4e0 <Krisztián Szűcs> debug macos coredumps
7eae170 <Krisztián Szűcs> debug macos
01961de <Krisztián Szűcs> macos debug
d0e4f62 <Krisztián Szűcs> remove debug commands
382a5f0 <Krisztián Szűcs> install gdb in more images
95c5384 <Krisztián Szűcs> limit the pattern for the first 15 chars
e75238c <Krisztián Szűcs> debug
76d072b <Krisztián Szűcs> set ulimit
b8db15f <Krisztián Szűcs> no sudo
04fef61 <Krisztián Szűcs> use sysctl
6c8bc4f <Krisztián Szűcs> debug
e8ec7ca <Krisztián Szűcs> remove util_coredump
655db22 <Krisztián Szűcs> workarounds
cf03c0c <Krisztián Szűcs> ARROW_TEST_ULIMIT_CORE: unlimited
8a092cb <Krisztián Szűcs> tune run-tests.sh
7f2ed2d <Krisztián Szűcs> delimiter
e830bc7 <Krisztián Szűcs> debug
187c32e <Krisztián Szűcs> permissions
ada66c0 <Krisztián Szűcs> force segfault
379b6ea <Krisztián Szűcs> enable for python and ruby macos builds
3f4af23 <Krisztián Szűcs> executable flag
b51b1fe <Krisztián Szűcs> on failure
28628ac <Krisztián Szűcs> continue on error to show the coredump
eb1cdb5 <Krisztián Szűcs> coredump on macos

Authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

.env

@@ -18,6 +18,7 @@
 # All of the following environment variables are required to set default values
 # for the parameters in docker-compose.yml.
 
+ULIMIT_CORE=-1
 REPO=apache/arrow-dev
 ARCH=amd64
 CUDA=9.1

.github/workflows/cpp.yml

@@ -50,7 +50,10 @@ jobs:
         run: docker-compose build conda-cpp
       - name: Docker Run
         shell: bash
-        run: docker-compose run conda-cpp
+        run: |
+          sudo sysctl -w kernel.core_pattern="core.%e.%p"
+          ulimit -c unlimited
+          docker-compose run conda-cpp
       - name: Docker Push
         if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
         continue-on-error: true
@@ -137,7 +140,11 @@ jobs:
         run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
       - name: Test
         shell: bash
-        run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
+        run: |
+          sudo sysctl -w kern.coredump=1
+          sudo sysctl -w kern.corefile=core.%N.%P
+          ulimit -c unlimited  # must enable within the same shell
+          ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
 
   windows:
     name: AMD64 ${{ matrix.name }} C++

.github/workflows/cpp_cron.yml

@@ -56,7 +56,10 @@ jobs:
         run: docker-compose build debian-cpp
       - name: Docker Run
         shell: bash
-        run: docker-compose run debian-cpp
+        run: |
+          sudo sysctl -w kernel.core_pattern="core.%e.%p"
+          ulimit -c unlimited
+          docker-compose run debian-cpp
       - name: Docker Push
         if: success() && github.repository == 'apache/arrow'
         continue-on-error: true
@@ -88,7 +91,10 @@ jobs:
         run: docker-compose build fedora-cpp
       - name: Docker Run
         shell: bash
-        run: docker-compose run fedora-cpp
+        run: |
+          sudo sysctl -w kernel.core_pattern="core.%e.%p"
+          ulimit -c unlimited
+          docker-compose run fedora-cpp
       - name: Docker Push
         if: success() && github.repository == 'apache/arrow'
         continue-on-error: true
@@ -120,7 +126,10 @@ jobs:
         run: docker-compose build ubuntu-cpp
       - name: Docker Run
         shell: bash
-        run: docker-compose run ubuntu-cpp
+        run: |
+          sudo sysctl -w kernel.core_pattern="core.%e.%p"
+          ulimit -c unlimited
+          docker-compose run ubuntu-cpp
       # No push step, it's the same image as ubuntu-cpp-sanitizer
 
   ubuntu-cmake32:
@@ -150,7 +159,10 @@ jobs:
           docker-compose build ubuntu-cpp-cmake32
       - name: Docker Run
         shell: bash
-        run: docker-compose run ubuntu-cpp-cmake32
+        run: |
+          sudo sysctl -w kernel.core_pattern="core.%e.%p"
+          ulimit -c unlimited
+          docker-compose run ubuntu-cpp-cmake32
       - name: Docker Push
         if: success() && github.repository == 'apache/arrow'
         continue-on-error: true

ci/docker/conda-cpp.dockerfile

@@ -46,7 +46,10 @@ RUN conda create -n arrow -q \
         --file arrow/ci/conda_env_unix.yml \
         --file arrow/ci/conda_env_cpp.yml \
         --file arrow/ci/conda_env_gandiva.yml \
-        git compilers valgrind && \
+        compilers \
+        gdb \
+        git \
+        valgrind && \
     conda clean --all
 
 # activate the created environment by default

ci/docker/cuda-9.1-cpp.dockerfile

@@ -39,6 +39,7 @@ RUN apt-get update -y -q && \
       flex \
       g++ \
       gcc \
+      gdb \
       git \
       libboost-filesystem-dev \
       libboost-regex-dev \

ci/docker/debian-10-cpp.dockerfile

@@ -29,6 +29,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         cmake \
         g++ \
         gcc \
+        gdb \
         git \
         libbenchmark-dev \
         libboost-all-dev \

ci/docker/ubuntu-14.04-cpp.dockerfile

@@ -31,6 +31,7 @@ RUN apt-get update -y -q && \
         flex \
         g++ \
         gcc \
+        gdb \
         git \
         libbz2-dev \
         libgoogle-glog-dev \

ci/docker/ubuntu-16.04-cpp.dockerfile

@@ -36,6 +36,7 @@ RUN apt-get update -y -q && \
         flex \
         g++ \
         gcc \
+        gdb \
         git \
         libboost-all-dev \
         libbrotli-dev \

ci/docker/ubuntu-18.04-cpp.dockerfile

@@ -61,6 +61,7 @@ RUN apt-get update -y -q && \
         flex \
         g++ \
         gcc \
+        gdb \
         git \
         libbenchmark-dev \
         libboost-filesystem-dev \

ci/scripts/cpp_test.sh

@@ -28,8 +28,6 @@ export ARROW_TEST_DATA=${arrow_dir}/testing/data
 export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data
 export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${LD_LIBRARY_PATH}
 
-pushd ${build_dir}
-
 case "$(uname)" in
   Linux)
     n_jobs=$(nproc)
@@ -41,6 +39,9 @@ case "$(uname)" in
     n_jobs=1
     ;;
 esac
+
+pushd ${build_dir}
+
 ctest --output-on-failure -j${n_jobs}
 
 if [ "${ARROW_FUZZING}" == "ON" ]; then

cpp/build-support/run-test.sh

@@ -60,11 +60,6 @@ rm -f $LOGFILE $LOGFILE.gz
 
 pipe_cmd=cat
 
-# Allow for collecting core dumps.
-ARROW_TEST_ULIMIT_CORE=${ARROW_TEST_ULIMIT_CORE:-0}
-ulimit -c $ARROW_TEST_ULIMIT_CORE
-
-
 function setup_sanitizers() {
   # Sets environment variables for different sanitizers (it configures how) the run_tests. Function works.
 
@@ -124,6 +119,42 @@ function run_test() {
   fi
 }
 
+function print_coredumps() {
+  # The script expects core files relative to the build directory with unique
+  # names per test executable because of the parallel running. So the corefile
+  # patterns must be set with prefix `core.{test-executable}*`:
+  #
+  # In case of macOS:
+  #   sudo sysctl -w kern.corefile=core.%N.%P
+  # On Linux:
+  #   sudo sysctl -w kernel.core_pattern=core.%e.%p
+  #
+  # and the ulimit must be increased:
+  #   ulimit -c unlimited
+
+  # filename is truncated to the first 15 characters in case of linux, so limit
+  # the pattern for the first 15 characters
+  FILENAME=$(basename "${TEST_EXECUTABLE}")
+  FILENAME=$(echo ${FILENAME} | cut -c-15)
+  PATTERN="^core\.${FILENAME}"
+
+  COREFILES=$(ls | grep $PATTERN)
+  if [ -n "$COREFILES" ]; then
+    echo "Found core dump, printing backtrace:"
+
+    for COREFILE in $COREFILES; do
+      # Print backtrace
+      if [ "$(uname)" == "Darwin" ]; then
+        lldb -c "${COREFILE}" --batch --one-line "thread backtrace all -e true"
+      else
+        gdb -c "${COREFILE}" $TEST_EXECUTABLE -ex "thread apply all bt" -ex "set pagination 0" -batch
+      fi
+      # Remove the coredump, regenerate it via running the test case directly
+      rm "${COREFILE}"
+    done
+  fi
+}
+
 function post_process_tests() {
   # If we have a LeakSanitizer report, and XML reporting is configured, add a new test
   # case result to the XML file for the leak report. Otherwise Jenkins won't show
@@ -148,7 +179,7 @@ function run_other() {
 }
 
 if [ $RUN_TYPE = "test" ]; then
-    setup_sanitizers
+  setup_sanitizers
 fi
 
 # Run the actual test.
@@ -200,20 +231,7 @@ if [ $RUN_TYPE = "test" ]; then
   post_process_tests
 fi
 
-# Capture and compress core file and binary.
-COREFILES=$(ls | grep ^core)
-if [ -n "$COREFILES" ]; then
-  echo Found core dump. Saving executable and core files.
-  gzip < $TEST_EXECUTABLE > "$TEST_DEBUGDIR/$TEST_NAME.gz" || exit $?
-  for COREFILE in $COREFILES; do
-    gzip < $COREFILE > "$TEST_DEBUGDIR/$TEST_NAME.$COREFILE.gz" || exit $?
-  done
-  # Pull in any .so files as well.
-  for LIB in $(ldd $TEST_EXECUTABLE | grep $ROOT | awk '{print $3}'); do
-    LIB_NAME=$(basename $LIB)
-    gzip < $LIB > "$TEST_DEBUGDIR/$LIB_NAME.gz" || exit $?
-  done
-fi
+print_coredumps
 
 popd
 rm -Rf $TEST_WORKDIR