From f482f6e09ec040697b347bf7ab8b41fc810632eb Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Fri, 27 Mar 2026 15:14:45 -0700 Subject: [PATCH 01/12] Disable non-odbc workflows --- .github/workflows/cpp_extra.yml | 520 ++++++++++++++++---------------- 1 file changed, 260 insertions(+), 260 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 5c0319897a72..8e95f79b4d6a 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -101,265 +101,265 @@ jobs: with: parent-workflow: cpp_extra - docker: - needs: check-labels - name: ${{ matrix.title }} - runs-on: ${{ matrix.runs-on }} - if: >- - github.event_name != 'workflow_dispatch' && ( - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - include: - - image: alpine-linux-cpp - runs-on: ubuntu-latest - title: AMD64 Alpine Linux - # Disable Large Memory Tests locally - # - image: ubuntu-cpp - # run-options: >- - # -e ARROW_CTEST_TIMEOUT=2000 - # -e ARROW_C_FLAGS_DEBUG="-O1" - # -e ARROW_CXX_FLAGS_DEBUG="-O1" - # -e ARROW_GANDIVA=OFF - # -e ARROW_LARGE_MEMORY_TESTS=ON - # -e BUILD_WARNING_LEVEL=PRODUCTION - # runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/volume=80gb/spot=capacity-optimized" - # title: AMD64 Ubuntu Large Memory Tests - - image: conda-cpp - run-options: >- - -e ARROW_USE_MESON=ON - runs-on: ubuntu-latest - title: AMD64 Ubuntu Meson - # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved - - continue-on-error: true - envs: - - DEBIAN=13 - image: debian-cpp - run-options: >- - -e CMAKE_CXX_STANDARD=23 - runs-on: ubuntu-latest - title: AMD64 Debian C++23 - env: - ARCHERY_DEBUG: 1 - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - steps: - - name: Checkout Arrow - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@v5 - with: - path: .docker - key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} - restore-keys: extra-${{ matrix.image }}- - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - continue-on-error: ${{ matrix.continue-on-error || false }} - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - ENVS: ${{ toJSON(matrix.envs) }} - run: | - # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes - sudo sysctl -w vm.mmap_rnd_bits=28 - source ci/scripts/util_enable_core_dumps.sh - if [ "${ENVS}" != "null" ]; then - echo "${ENVS}" | jq -r '.[]' | while read env; do - echo "${env}" >> .env - done - fi - archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} + # docker: + # needs: check-labels + # name: ${{ matrix.title }} + # runs-on: ${{ matrix.runs-on }} + # if: >- + # github.event_name != 'workflow_dispatch' && ( + # needs.check-labels.outputs.force == 'true' || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) + # timeout-minutes: 75 + # strategy: + # fail-fast: false + # matrix: + # include: + # - image: alpine-linux-cpp + # runs-on: ubuntu-latest + # title: AMD64 Alpine Linux + # # Disable Large Memory Tests locally + # # - image: ubuntu-cpp + # # run-options: >- + # # -e ARROW_CTEST_TIMEOUT=2000 + # # -e ARROW_C_FLAGS_DEBUG="-O1" + # # -e ARROW_CXX_FLAGS_DEBUG="-O1" + # # -e ARROW_GANDIVA=OFF + # # -e ARROW_LARGE_MEMORY_TESTS=ON + # # -e BUILD_WARNING_LEVEL=PRODUCTION + # # runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/volume=80gb/spot=capacity-optimized" + # # title: AMD64 Ubuntu Large Memory Tests + # - image: conda-cpp + # run-options: >- + # -e ARROW_USE_MESON=ON + # runs-on: ubuntu-latest + # title: AMD64 Ubuntu Meson + # # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved + # - continue-on-error: true + # envs: + # - DEBIAN=13 + # image: debian-cpp + # run-options: >- + # -e CMAKE_CXX_STANDARD=23 + # runs-on: ubuntu-latest + # title: AMD64 Debian C++23 + # env: + # ARCHERY_DEBUG: 1 + # ARROW_ENABLE_TIMING_TESTS: OFF + # DOCKER_VOLUME_PREFIX: ".docker/" + # steps: + # - name: Checkout Arrow + # uses: actions/checkout@v6 + # with: + # fetch-depth: 0 + # submodules: recursive + # - name: Cache Docker Volumes + # uses: actions/cache@v5 + # with: + # path: .docker + # key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} + # restore-keys: extra-${{ matrix.image }}- + # - name: Setup Python + # uses: actions/setup-python@v6 + # with: + # python-version: 3 + # - name: Setup Archery + # run: python3 -m pip install -e dev/archery[docker] + # - name: Execute Docker Build + # continue-on-error: ${{ matrix.continue-on-error || false }} + # env: + # ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + # ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + # ENVS: ${{ toJSON(matrix.envs) }} + # run: | + # # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes + # sudo sysctl -w vm.mmap_rnd_bits=28 + # source ci/scripts/util_enable_core_dumps.sh + # if [ "${ENVS}" != "null" ]; then + # echo "${ENVS}" | jq -r '.[]' | while read env; do + # echo "${env}" >> .env + # done + # fi + # archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} + # - name: Docker Push + # if: >- + # success() && + # github.event_name == 'push' && + # github.repository == 'apache/arrow' && + # github.ref_name == 'main' + # env: + # ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + # ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + # continue-on-error: true + # run: archery docker push ${{ matrix.image }} - msvc-arm64: - needs: check-labels - if: >- - github.event_name != 'workflow_dispatch' && ( - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) - name: ARM64 Windows 11 MSVC - uses: ./.github/workflows/cpp_windows.yml - with: - arch: arm64 - os: windows-11-arm - simd-level: NONE + # msvc-arm64: + # needs: check-labels + # if: >- + # github.event_name != 'workflow_dispatch' && ( + # needs.check-labels.outputs.force == 'true' || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) + # name: ARM64 Windows 11 MSVC + # uses: ./.github/workflows/cpp_windows.yml + # with: + # arch: arm64 + # os: windows-11-arm + # simd-level: NONE - jni-linux: - needs: check-labels - name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} - runs-on: ${{ matrix.platform.runs-on }} - if: >- - github.event_name != 'workflow_dispatch' && ( - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) - timeout-minutes: 240 - permissions: - # This is for using GitHub Packages for vcpkg cache - packages: write - strategy: - fail-fast: false - matrix: - platform: - - arch: "amd64" - runs-on: ubuntu-latest - - arch: "arm64v8" - runs-on: ubuntu-24.04-arm - env: - ARCH: ${{ matrix.platform.arch }} - REPO: ghcr.io/${{ github.repository }}-dev - steps: - - name: Checkout Arrow - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@v5 - with: - path: .docker - key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} - restore-keys: jni-${{ matrix.platform.runs-on }}- - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ github.actor }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run cpp-jni - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ github.actor }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - continue-on-error: true - run: archery docker push cpp-jni + # jni-linux: + # needs: check-labels + # name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} + # runs-on: ${{ matrix.platform.runs-on }} + # if: >- + # github.event_name != 'workflow_dispatch' && ( + # needs.check-labels.outputs.force == 'true' || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) + # timeout-minutes: 240 + # permissions: + # # This is for using GitHub Packages for vcpkg cache + # packages: write + # strategy: + # fail-fast: false + # matrix: + # platform: + # - arch: "amd64" + # runs-on: ubuntu-latest + # - arch: "arm64v8" + # runs-on: ubuntu-24.04-arm + # env: + # ARCH: ${{ matrix.platform.arch }} + # REPO: ghcr.io/${{ github.repository }}-dev + # steps: + # - name: Checkout Arrow + # uses: actions/checkout@v6 + # with: + # fetch-depth: 0 + # submodules: recursive + # - name: Free up disk space + # run: | + # ci/scripts/util_free_space.sh + # - name: Cache Docker Volumes + # uses: actions/cache@v5 + # with: + # path: .docker + # key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} + # restore-keys: jni-${{ matrix.platform.runs-on }}- + # - name: Setup Python + # uses: actions/setup-python@v6 + # with: + # python-version: 3 + # - name: Setup Archery + # run: python3 -m pip install -e dev/archery[docker] + # - name: Execute Docker Build + # env: + # ARCHERY_DOCKER_USER: ${{ github.actor }} + # ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" + # run: | + # source ci/scripts/util_enable_core_dumps.sh + # archery docker run cpp-jni + # - name: Docker Push + # if: >- + # success() && + # github.event_name == 'push' && + # github.ref_name == 'main' + # env: + # ARCHERY_DOCKER_USER: ${{ github.actor }} + # ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + # continue-on-error: true + # run: archery docker push cpp-jni - jni-macos: - needs: check-labels - name: JNI macOS - runs-on: macos-14 - if: >- - github.event_name != 'workflow_dispatch' && ( - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) - timeout-minutes: 45 - env: - MACOSX_DEPLOYMENT_TARGET: "14.0" - steps: - - name: Checkout Arrow - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - name: Install dependencies - run: | - brew bundle --file=cpp/Brewfile - # We want to link aws-sdk-cpp statically but Homebrew's - # aws-sdk-cpp provides only shared library. If we have - # Homebrew's aws-sdk-cpp, our build mix Homebrew's - # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's - # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. - brew uninstall aws-sdk-cpp - # We want to use bundled RE2 for static linking. If - # Homebrew's RE2 is installed, its header file may be used. - # We uninstall Homebrew's RE2 to ensure using bundled RE2. - brew uninstall grpc || : # gRPC depends on RE2 - brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - brew uninstall re2 - # We want to use bundled Protobuf for static linking. If - # Homebrew's Protobuf is installed, its library file may be - # used on test We uninstall Homebrew's Protobuf to ensure using - # bundled Protobuf. - brew uninstall protobuf - - name: Prepare ccache - run: | - echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - - name: Cache ccache - uses: actions/cache@v5 - with: - path: ccache - key: jni-macos-${{ hashFiles('cpp/**') }} - restore-keys: jni-macos- - - name: CMake - run: | - cmake \ - -S cpp \ - -B cpp.build \ - --preset=ninja-release-jni-macos \ - -DARROW_BUILD_TESTS=ON \ - -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - - name: Build - run: | - cmake --build cpp.build - - name: Install - run: | - cmake --install cpp.build - - name: Test - env: - ARROW_TEST_DATA: ${{ github.workspace }}/testing/data - PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data - run: | - # MinIO is required - exclude_tests="arrow-s3fs-test" - # unstable - exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel "$(sysctl -n hw.ncpu)" \ - --test-dir "cpp.build" \ - --timeout 300 - - name: Build example - run: | - cmake \ - -S cpp/examples/minimal_build/ \ - -B cpp/examples/minimal_build.build \ - -GNinja \ - -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - cmake --build cpp/examples/minimal_build.build - cd cpp/examples/minimal_build - ../minimal_build.build/arrow-example + # jni-macos: + # needs: check-labels + # name: JNI macOS + # runs-on: macos-14 + # if: >- + # github.event_name != 'workflow_dispatch' && ( + # needs.check-labels.outputs.force == 'true' || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) + # timeout-minutes: 45 + # env: + # MACOSX_DEPLOYMENT_TARGET: "14.0" + # steps: + # - name: Checkout Arrow + # uses: actions/checkout@v6 + # with: + # fetch-depth: 0 + # submodules: recursive + # - name: Install dependencies + # run: | + # brew bundle --file=cpp/Brewfile + # # We want to link aws-sdk-cpp statically but Homebrew's + # # aws-sdk-cpp provides only shared library. If we have + # # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + # brew uninstall aws-sdk-cpp + # # We want to use bundled RE2 for static linking. If + # # Homebrew's RE2 is installed, its header file may be used. + # # We uninstall Homebrew's RE2 to ensure using bundled RE2. + # brew uninstall grpc || : # gRPC depends on RE2 + # brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too + # brew uninstall re2 + # # We want to use bundled Protobuf for static linking. If + # # Homebrew's Protobuf is installed, its library file may be + # # used on test We uninstall Homebrew's Protobuf to ensure using + # # bundled Protobuf. + # brew uninstall protobuf + # - name: Prepare ccache + # run: | + # echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + # - name: Cache ccache + # uses: actions/cache@v5 + # with: + # path: ccache + # key: jni-macos-${{ hashFiles('cpp/**') }} + # restore-keys: jni-macos- + # - name: CMake + # run: | + # cmake \ + # -S cpp \ + # -B cpp.build \ + # --preset=ninja-release-jni-macos \ + # -DARROW_BUILD_TESTS=ON \ + # -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install + # - name: Build + # run: | + # cmake --build cpp.build + # - name: Install + # run: | + # cmake --install cpp.build + # - name: Test + # env: + # ARROW_TEST_DATA: ${{ github.workspace }}/testing/data + # PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data + # run: | + # # MinIO is required + # exclude_tests="arrow-s3fs-test" + # # unstable + # exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" + # exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" + # ctest \ + # --exclude-regex "${exclude_tests}" \ + # --label-regex unittest \ + # --output-on-failure \ + # --parallel "$(sysctl -n hw.ncpu)" \ + # --test-dir "cpp.build" \ + # --timeout 300 + # - name: Build example + # run: | + # cmake \ + # -S cpp/examples/minimal_build/ \ + # -B cpp/examples/minimal_build.build \ + # -GNinja \ + # -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install + # cmake --build cpp/examples/minimal_build.build + # cd cpp/examples/minimal_build + # ../minimal_build.build/arrow-example odbc-linux: needs: check-labels @@ -807,10 +807,10 @@ jobs: report-extra-cpp: if: github.event_name == 'schedule' && always() needs: - - docker - - jni-linux - - jni-macos - - msvc-arm64 + # - docker + # - jni-linux + # - jni-macos + # - msvc-arm64 - odbc-linux - odbc-macos - odbc-msvc From 91e78834b5dc6bf3f98ed4fa45b9c01e38c5c883 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Fri, 27 Mar 2026 14:58:00 -0700 Subject: [PATCH 02/12] Install `netcat` and set `ARROW_FLIGHT_SQL_ODBC_CONN` --- compose.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index bd417c524e4f..927934331531 100644 --- a/compose.yaml +++ b/compose.yaml @@ -558,6 +558,9 @@ services: ARROW_DEPENDENCY_SOURCE: BUNDLED ARROW_DEPENDENCY_USE_SHARED: "OFF" ARROW_FLIGHT_SQL_ODBC: "ON" + # -AL- temp: test 32010 port is reachable. + # -AL- this variable will work after Linux ODBC tests are enabled. + ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=localhost;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" ARROW_GANDIVA: "OFF" ARROW_GCS: "OFF" ARROW_HDFS: "OFF" @@ -572,7 +575,9 @@ services: /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && sudo /arrow/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /usr/local/lib/libarrow_flight_sql_odbc.so && - /arrow/ci/scripts/cpp_test.sh /arrow /build" + /arrow/ci/scripts/cpp_test.sh /arrow /build && + apt-get install -y netcat-openbsd && + nc -vz localhost 32010" ubuntu-cpp-minimal: # Arrow build with minimal components/dependencies From f6a595aacd62e3e0499611b3374f4355052cd7a1 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Fri, 27 Mar 2026 15:21:27 -0700 Subject: [PATCH 03/12] Attempt to enable docker network Update compose.yaml Add executable bash script --- .github/workflows/cpp_extra.yml | 7 +++++++ compose.yaml | 7 +++++++ .../flight/sql/odbc/tests/dremio/docker-compose.yml | 9 +++++++++ .../sql/odbc/tests/dremio/set_up_dremio_instance.sh | 0 4 files changed, 23 insertions(+) mode change 100644 => 100755 cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 8e95f79b4d6a..3952b5615276 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -391,6 +391,13 @@ jobs: with: fetch-depth: 0 submodules: recursive + # -AL- test if below works. + - name: Create Docker Test Network + run: docker network create odbc_net + - name: Set Up Dremio Instance + run: | + docker compose -f cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml up -d + cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh - name: Cache Docker Volumes uses: actions/cache@v5 with: diff --git a/compose.yaml b/compose.yaml index 927934331531..4cce7406603c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -205,6 +205,10 @@ volumes: ubuntu-ccache: name: ${ARCH}-ubuntu-${UBUNTU}-ccache +networks: + odbc_net: + external: true + services: ################################# C++ ####################################### @@ -571,6 +575,9 @@ services: # GH-49651 Link ODBC tests statically on Linux to fix segfault ARROW_TEST_LINKAGE: "static" # Register ODBC before running tests + # -AL- temp: test 32010 port is reachable. + networks: + - odbc_net command: > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && diff --git a/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml b/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml index eaab4d02b731..f824e3b6763a 100644 --- a/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml +++ b/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml @@ -16,6 +16,13 @@ # under the License. # GH-48068 TODO: run remote ODBC tests on Linux +# -AL- todo: move this file's content to `arrow/compose.yaml` after it is proven to work. +# -AL- if move, the `networks:` top level section doesn't need duplication as we already have it in +# `arrow/compose.yaml` + +networks: + odbc_net: + external: true services: dremio: @@ -33,3 +40,5 @@ services: interval: 10s timeout: 5s retries: 30 + networks: + - odbc_net diff --git a/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh b/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh old mode 100644 new mode 100755 From 04dfad7a9681b437d49ba694228efd794c674d88 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Mon, 30 Mar 2026 15:09:06 -0700 Subject: [PATCH 04/12] add commands for testing Dremio instance value --- compose.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 4cce7406603c..6c9737b47df8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -584,7 +584,13 @@ services: sudo /arrow/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /usr/local/lib/libarrow_flight_sql_odbc.so && /arrow/ci/scripts/cpp_test.sh /arrow /build && apt-get install -y netcat-openbsd && - nc -vz localhost 32010" + nc -vz localhost 32010 && + apt-get install -y unixodbc && + isql -k \"$ARROW_FLIGHT_SQL_ODBC_CONN\" < Date: Tue, 31 Mar 2026 11:27:21 -0700 Subject: [PATCH 05/12] Change to use `host.docker.internal` --- compose.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compose.yaml b/compose.yaml index 6c9737b47df8..b9bde2b68853 100644 --- a/compose.yaml +++ b/compose.yaml @@ -564,7 +564,8 @@ services: ARROW_FLIGHT_SQL_ODBC: "ON" # -AL- temp: test 32010 port is reachable. # -AL- this variable will work after Linux ODBC tests are enabled. - ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=localhost;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" + # ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=localhost;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" + ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=host.docker.internal;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" ARROW_GANDIVA: "OFF" ARROW_GCS: "OFF" ARROW_HDFS: "OFF" @@ -578,13 +579,14 @@ services: # -AL- temp: test 32010 port is reachable. networks: - odbc_net + # -AL- nc -vz localhost 32010 did not work. command: > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && sudo /arrow/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /usr/local/lib/libarrow_flight_sql_odbc.so && /arrow/ci/scripts/cpp_test.sh /arrow /build && apt-get install -y netcat-openbsd && - nc -vz localhost 32010 && + nc -vz host.docker.internal 32010 && apt-get install -y unixodbc && isql -k \"$ARROW_FLIGHT_SQL_ODBC_CONN\" < Date: Tue, 7 Apr 2026 14:27:48 -0700 Subject: [PATCH 06/12] Change test to run remote test Update cpp_extra.yml --- .github/workflows/cpp_extra.yml | 8 ++++++++ ci/scripts/cpp_test.sh | 2 +- compose.yaml | 28 ++++++++++++++++++++-------- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 3952b5615276..de285631967c 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -398,6 +398,10 @@ jobs: run: | docker compose -f cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml up -d cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh + - name: Set Up Dremio Instance -AL- TEST TODO REMOVE LATER + run: | + docker ps + docker compose ps - name: Cache Docker Volumes uses: actions/cache@v5 with: @@ -410,6 +414,10 @@ jobs: python-version: 3 - name: Setup Archery run: python3 -m pip install -e dev/archery[docker] + - name: Set Up Dremio Instance -AL- TEST TODO REMOVE LATER + run: | + docker ps + docker compose ps - name: Execute Docker Build env: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index 2f88cdc819b2..abc5ad97e234 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -55,7 +55,7 @@ if ! type minio >/dev/null 2>&1; then fi case "$(uname)" in Linux) - exclude_tests+=("arrow-flight-sql-odbc-test") + # exclude_tests+=("arrow-flight-sql-odbc-test") # -AL- temp disable! n_jobs=$(nproc) ;; Darwin) diff --git a/compose.yaml b/compose.yaml index b9bde2b68853..9b6a70beca01 100644 --- a/compose.yaml +++ b/compose.yaml @@ -585,14 +585,26 @@ services: /arrow/ci/scripts/cpp_build.sh /arrow /build && sudo /arrow/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /usr/local/lib/libarrow_flight_sql_odbc.so && /arrow/ci/scripts/cpp_test.sh /arrow /build && - apt-get install -y netcat-openbsd && - nc -vz host.docker.internal 32010 && - apt-get install -y unixodbc && - isql -k \"$ARROW_FLIGHT_SQL_ODBC_CONN\" < Date: Tue, 7 Apr 2026 15:30:52 -0700 Subject: [PATCH 07/12] Use `localhost` as hostname --- compose.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/compose.yaml b/compose.yaml index 9b6a70beca01..77f6c1e974d0 100644 --- a/compose.yaml +++ b/compose.yaml @@ -564,8 +564,8 @@ services: ARROW_FLIGHT_SQL_ODBC: "ON" # -AL- temp: test 32010 port is reachable. # -AL- this variable will work after Linux ODBC tests are enabled. - # ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=localhost;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" - ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=host.docker.internal;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" + ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=localhost;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" + # ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=host.docker.internal;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" ARROW_GANDIVA: "OFF" ARROW_GCS: "OFF" ARROW_HDFS: "OFF" @@ -584,8 +584,7 @@ services: /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && sudo /arrow/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /usr/local/lib/libarrow_flight_sql_odbc.so && - /arrow/ci/scripts/cpp_test.sh /arrow /build && - /build/cpp/arrow-flight-sql-odbc-test "--gtest_filter=*.*SQLDriverConn*" + /arrow/ci/scripts/cpp_test.sh /arrow /build " #-AL- tried locations From b8b86d5e966fa0e1eedf92d5722a8afa6652ec48 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Tue, 7 Apr 2026 15:36:32 -0700 Subject: [PATCH 08/12] Use `dremio_container` as host --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 77f6c1e974d0..a3bba65b0ee0 100644 --- a/compose.yaml +++ b/compose.yaml @@ -564,7 +564,7 @@ services: ARROW_FLIGHT_SQL_ODBC: "ON" # -AL- temp: test 32010 port is reachable. # -AL- this variable will work after Linux ODBC tests are enabled. - ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=localhost;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" + ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=dremio_container;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" # ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=host.docker.internal;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" ARROW_GANDIVA: "OFF" ARROW_GCS: "OFF" From e1529675ac16f0de5056225ed122a19d493b1037 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Tue, 7 Apr 2026 15:49:50 -0700 Subject: [PATCH 09/12] Clean up PR --- .github/workflows/cpp_extra.yml | 529 +++++++++--------- ci/scripts/cpp_test.sh | 2 +- compose.yaml | 27 +- .../sql/odbc/tests/dremio/docker-compose.yml | 5 - 4 files changed, 263 insertions(+), 300 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index de285631967c..2cabf81a0169 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -101,265 +101,265 @@ jobs: with: parent-workflow: cpp_extra - # docker: - # needs: check-labels - # name: ${{ matrix.title }} - # runs-on: ${{ matrix.runs-on }} - # if: >- - # github.event_name != 'workflow_dispatch' && ( - # needs.check-labels.outputs.force == 'true' || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) - # timeout-minutes: 75 - # strategy: - # fail-fast: false - # matrix: - # include: - # - image: alpine-linux-cpp - # runs-on: ubuntu-latest - # title: AMD64 Alpine Linux - # # Disable Large Memory Tests locally - # # - image: ubuntu-cpp - # # run-options: >- - # # -e ARROW_CTEST_TIMEOUT=2000 - # # -e ARROW_C_FLAGS_DEBUG="-O1" - # # -e ARROW_CXX_FLAGS_DEBUG="-O1" - # # -e ARROW_GANDIVA=OFF - # # -e ARROW_LARGE_MEMORY_TESTS=ON - # # -e BUILD_WARNING_LEVEL=PRODUCTION - # # runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/volume=80gb/spot=capacity-optimized" - # # title: AMD64 Ubuntu Large Memory Tests - # - image: conda-cpp - # run-options: >- - # -e ARROW_USE_MESON=ON - # runs-on: ubuntu-latest - # title: AMD64 Ubuntu Meson - # # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved - # - continue-on-error: true - # envs: - # - DEBIAN=13 - # image: debian-cpp - # run-options: >- - # -e CMAKE_CXX_STANDARD=23 - # runs-on: ubuntu-latest - # title: AMD64 Debian C++23 - # env: - # ARCHERY_DEBUG: 1 - # ARROW_ENABLE_TIMING_TESTS: OFF - # DOCKER_VOLUME_PREFIX: ".docker/" - # steps: - # - name: Checkout Arrow - # uses: actions/checkout@v6 - # with: - # fetch-depth: 0 - # submodules: recursive - # - name: Cache Docker Volumes - # uses: actions/cache@v5 - # with: - # path: .docker - # key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} - # restore-keys: extra-${{ matrix.image }}- - # - name: Setup Python - # uses: actions/setup-python@v6 - # with: - # python-version: 3 - # - name: Setup Archery - # run: python3 -m pip install -e dev/archery[docker] - # - name: Execute Docker Build - # continue-on-error: ${{ matrix.continue-on-error || false }} - # env: - # ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - # ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - # ENVS: ${{ toJSON(matrix.envs) }} - # run: | - # # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes - # sudo sysctl -w vm.mmap_rnd_bits=28 - # source ci/scripts/util_enable_core_dumps.sh - # if [ "${ENVS}" != "null" ]; then - # echo "${ENVS}" | jq -r '.[]' | while read env; do - # echo "${env}" >> .env - # done - # fi - # archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} - # - name: Docker Push - # if: >- - # success() && - # github.event_name == 'push' && - # github.repository == 'apache/arrow' && - # github.ref_name == 'main' - # env: - # ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - # ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - # continue-on-error: true - # run: archery docker push ${{ matrix.image }} + docker: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: >- + github.event_name != 'workflow_dispatch' && ( + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - image: alpine-linux-cpp + runs-on: ubuntu-latest + title: AMD64 Alpine Linux + # Disable Large Memory Tests locally + # - image: ubuntu-cpp + # run-options: >- + # -e ARROW_CTEST_TIMEOUT=2000 + # -e ARROW_C_FLAGS_DEBUG="-O1" + # -e ARROW_CXX_FLAGS_DEBUG="-O1" + # -e ARROW_GANDIVA=OFF + # -e ARROW_LARGE_MEMORY_TESTS=ON + # -e BUILD_WARNING_LEVEL=PRODUCTION + # runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/volume=80gb/spot=capacity-optimized" + # title: AMD64 Ubuntu Large Memory Tests + - image: conda-cpp + run-options: >- + -e ARROW_USE_MESON=ON + runs-on: ubuntu-latest + title: AMD64 Ubuntu Meson + # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved + - continue-on-error: true + envs: + - DEBIAN=13 + image: debian-cpp + run-options: >- + -e CMAKE_CXX_STANDARD=23 + runs-on: ubuntu-latest + title: AMD64 Debian C++23 + env: + ARCHERY_DEBUG: 1 + ARROW_ENABLE_TIMING_TESTS: OFF + DOCKER_VOLUME_PREFIX: ".docker/" + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Cache Docker Volumes + uses: actions/cache@v5 + with: + path: .docker + key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.image }}- + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3 + - name: Setup Archery + run: python3 -m pip install -e dev/archery[docker] + - name: Execute Docker Build + continue-on-error: ${{ matrix.continue-on-error || false }} + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + ENVS: ${{ toJSON(matrix.envs) }} + run: | + # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes + sudo sysctl -w vm.mmap_rnd_bits=28 + source ci/scripts/util_enable_core_dumps.sh + if [ "${ENVS}" != "null" ]; then + echo "${ENVS}" | jq -r '.[]' | while read env; do + echo "${env}" >> .env + done + fi + archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} + - name: Docker Push + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + continue-on-error: true + run: archery docker push ${{ matrix.image }} - # msvc-arm64: - # needs: check-labels - # if: >- - # github.event_name != 'workflow_dispatch' && ( - # needs.check-labels.outputs.force == 'true' || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) - # name: ARM64 Windows 11 MSVC - # uses: ./.github/workflows/cpp_windows.yml - # with: - # arch: arm64 - # os: windows-11-arm - # simd-level: NONE + msvc-arm64: + needs: check-labels + if: >- + github.event_name != 'workflow_dispatch' && ( + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) + name: ARM64 Windows 11 MSVC + uses: ./.github/workflows/cpp_windows.yml + with: + arch: arm64 + os: windows-11-arm + simd-level: NONE - # jni-linux: - # needs: check-labels - # name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} - # runs-on: ${{ matrix.platform.runs-on }} - # if: >- - # github.event_name != 'workflow_dispatch' && ( - # needs.check-labels.outputs.force == 'true' || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) - # timeout-minutes: 240 - # permissions: - # # This is for using GitHub Packages for vcpkg cache - # packages: write - # strategy: - # fail-fast: false - # matrix: - # platform: - # - arch: "amd64" - # runs-on: ubuntu-latest - # - arch: "arm64v8" - # runs-on: ubuntu-24.04-arm - # env: - # ARCH: ${{ matrix.platform.arch }} - # REPO: ghcr.io/${{ github.repository }}-dev - # steps: - # - name: Checkout Arrow - # uses: actions/checkout@v6 - # with: - # fetch-depth: 0 - # submodules: recursive - # - name: Free up disk space - # run: | - # ci/scripts/util_free_space.sh - # - name: Cache Docker Volumes - # uses: actions/cache@v5 - # with: - # path: .docker - # key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} - # restore-keys: jni-${{ matrix.platform.runs-on }}- - # - name: Setup Python - # uses: actions/setup-python@v6 - # with: - # python-version: 3 - # - name: Setup Archery - # run: python3 -m pip install -e dev/archery[docker] - # - name: Execute Docker Build - # env: - # ARCHERY_DOCKER_USER: ${{ github.actor }} - # ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" - # run: | - # source ci/scripts/util_enable_core_dumps.sh - # archery docker run cpp-jni - # - name: Docker Push - # if: >- - # success() && - # github.event_name == 'push' && - # github.ref_name == 'main' - # env: - # ARCHERY_DOCKER_USER: ${{ github.actor }} - # ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - # continue-on-error: true - # run: archery docker push cpp-jni + jni-linux: + needs: check-labels + name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} + runs-on: ${{ matrix.platform.runs-on }} + if: >- + github.event_name != 'workflow_dispatch' && ( + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) + timeout-minutes: 240 + permissions: + # This is for using GitHub Packages for vcpkg cache + packages: write + strategy: + fail-fast: false + matrix: + platform: + - arch: "amd64" + runs-on: ubuntu-latest + - arch: "arm64v8" + runs-on: ubuntu-24.04-arm + env: + ARCH: ${{ matrix.platform.arch }} + REPO: ghcr.io/${{ github.repository }}-dev + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Free up disk space + run: | + ci/scripts/util_free_space.sh + - name: Cache Docker Volumes + uses: actions/cache@v5 + with: + path: .docker + key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} + restore-keys: jni-${{ matrix.platform.runs-on }}- + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3 + - name: Setup Archery + run: python3 -m pip install -e dev/archery[docker] + - name: Execute Docker Build + env: + ARCHERY_DOCKER_USER: ${{ github.actor }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" + run: | + source ci/scripts/util_enable_core_dumps.sh + archery docker run cpp-jni + - name: Docker Push + if: >- + success() && + github.event_name == 'push' && + github.ref_name == 'main' + env: + ARCHERY_DOCKER_USER: ${{ github.actor }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true + run: archery docker push cpp-jni - # jni-macos: - # needs: check-labels - # name: JNI macOS - # runs-on: macos-14 - # if: >- - # github.event_name != 'workflow_dispatch' && ( - # needs.check-labels.outputs.force == 'true' || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - # contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) - # timeout-minutes: 45 - # env: - # MACOSX_DEPLOYMENT_TARGET: "14.0" - # steps: - # - name: Checkout Arrow - # uses: actions/checkout@v6 - # with: - # fetch-depth: 0 - # submodules: recursive - # - name: Install dependencies - # run: | - # brew bundle --file=cpp/Brewfile - # # We want to link aws-sdk-cpp statically but Homebrew's - # # aws-sdk-cpp provides only shared library. If we have - # # Homebrew's aws-sdk-cpp, our build mix Homebrew's - # # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's - # # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. - # brew uninstall aws-sdk-cpp - # # We want to use bundled RE2 for static linking. If - # # Homebrew's RE2 is installed, its header file may be used. - # # We uninstall Homebrew's RE2 to ensure using bundled RE2. - # brew uninstall grpc || : # gRPC depends on RE2 - # brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - # brew uninstall re2 - # # We want to use bundled Protobuf for static linking. If - # # Homebrew's Protobuf is installed, its library file may be - # # used on test We uninstall Homebrew's Protobuf to ensure using - # # bundled Protobuf. - # brew uninstall protobuf - # - name: Prepare ccache - # run: | - # echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - # - name: Cache ccache - # uses: actions/cache@v5 - # with: - # path: ccache - # key: jni-macos-${{ hashFiles('cpp/**') }} - # restore-keys: jni-macos- - # - name: CMake - # run: | - # cmake \ - # -S cpp \ - # -B cpp.build \ - # --preset=ninja-release-jni-macos \ - # -DARROW_BUILD_TESTS=ON \ - # -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - # - name: Build - # run: | - # cmake --build cpp.build - # - name: Install - # run: | - # cmake --install cpp.build - # - name: Test - # env: - # ARROW_TEST_DATA: ${{ github.workspace }}/testing/data - # PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data - # run: | - # # MinIO is required - # exclude_tests="arrow-s3fs-test" - # # unstable - # exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - # exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - # ctest \ - # --exclude-regex "${exclude_tests}" \ - # --label-regex unittest \ - # --output-on-failure \ - # --parallel "$(sysctl -n hw.ncpu)" \ - # --test-dir "cpp.build" \ - # --timeout 300 - # - name: Build example - # run: | - # cmake \ - # -S cpp/examples/minimal_build/ \ - # -B cpp/examples/minimal_build.build \ - # -GNinja \ - # -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - # cmake --build cpp/examples/minimal_build.build - # cd cpp/examples/minimal_build - # ../minimal_build.build/arrow-example + jni-macos: + needs: check-labels + name: JNI macOS + runs-on: macos-14 + if: >- + github.event_name != 'workflow_dispatch' && ( + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')) + timeout-minutes: 45 + env: + MACOSX_DEPLOYMENT_TARGET: "14.0" + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Install dependencies + run: | + brew bundle --file=cpp/Brewfile + # We want to link aws-sdk-cpp statically but Homebrew's + # aws-sdk-cpp provides only shared library. If we have + # Homebrew's aws-sdk-cpp, our build mix Homebrew's + # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's + # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. + brew uninstall aws-sdk-cpp + # We want to use bundled RE2 for static linking. If + # Homebrew's RE2 is installed, its header file may be used. + # We uninstall Homebrew's RE2 to ensure using bundled RE2. + brew uninstall grpc || : # gRPC depends on RE2 + brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too + brew uninstall re2 + # We want to use bundled Protobuf for static linking. If + # Homebrew's Protobuf is installed, its library file may be + # used on test We uninstall Homebrew's Protobuf to ensure using + # bundled Protobuf. + brew uninstall protobuf + - name: Prepare ccache + run: | + echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} + - name: Cache ccache + uses: actions/cache@v5 + with: + path: ccache + key: jni-macos-${{ hashFiles('cpp/**') }} + restore-keys: jni-macos- + - name: CMake + run: | + cmake \ + -S cpp \ + -B cpp.build \ + --preset=ninja-release-jni-macos \ + -DARROW_BUILD_TESTS=ON \ + -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install + - name: Build + run: | + cmake --build cpp.build + - name: Install + run: | + cmake --install cpp.build + - name: Test + env: + ARROW_TEST_DATA: ${{ github.workspace }}/testing/data + PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data + run: | + # MinIO is required + exclude_tests="arrow-s3fs-test" + # unstable + exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" + exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" + ctest \ + --exclude-regex "${exclude_tests}" \ + --label-regex unittest \ + --output-on-failure \ + --parallel "$(sysctl -n hw.ncpu)" \ + --test-dir "cpp.build" \ + --timeout 300 + - name: Build example + run: | + cmake \ + -S cpp/examples/minimal_build/ \ + -B cpp/examples/minimal_build.build \ + -GNinja \ + -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install + cmake --build cpp/examples/minimal_build.build + cd cpp/examples/minimal_build + ../minimal_build.build/arrow-example odbc-linux: needs: check-labels @@ -391,17 +391,12 @@ jobs: with: fetch-depth: 0 submodules: recursive - # -AL- test if below works. - name: Create Docker Test Network run: docker network create odbc_net - name: Set Up Dremio Instance run: | docker compose -f cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml up -d cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh - - name: Set Up Dremio Instance -AL- TEST TODO REMOVE LATER - run: | - docker ps - docker compose ps - name: Cache Docker Volumes uses: actions/cache@v5 with: @@ -414,10 +409,6 @@ jobs: python-version: 3 - name: Setup Archery run: python3 -m pip install -e dev/archery[docker] - - name: Set Up Dremio Instance -AL- TEST TODO REMOVE LATER - run: | - docker ps - docker compose ps - name: Execute Docker Build env: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} @@ -822,10 +813,10 @@ jobs: report-extra-cpp: if: github.event_name == 'schedule' && always() needs: - # - docker - # - jni-linux - # - jni-macos - # - msvc-arm64 + - docker + - jni-linux + - jni-macos + - msvc-arm64 - odbc-linux - odbc-macos - odbc-msvc diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index abc5ad97e234..2f88cdc819b2 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -55,7 +55,7 @@ if ! type minio >/dev/null 2>&1; then fi case "$(uname)" in Linux) - # exclude_tests+=("arrow-flight-sql-odbc-test") # -AL- temp disable! + exclude_tests+=("arrow-flight-sql-odbc-test") n_jobs=$(nproc) ;; Darwin) diff --git a/compose.yaml b/compose.yaml index a3bba65b0ee0..669cabdd9225 100644 --- a/compose.yaml +++ b/compose.yaml @@ -206,6 +206,7 @@ volumes: name: ${ARCH}-ubuntu-${UBUNTU}-ccache networks: + # GH-48068 for Flight SQL ODBC Testing odbc_net: external: true @@ -562,10 +563,7 @@ services: ARROW_DEPENDENCY_SOURCE: BUNDLED ARROW_DEPENDENCY_USE_SHARED: "OFF" ARROW_FLIGHT_SQL_ODBC: "ON" - # -AL- temp: test 32010 port is reachable. - # -AL- this variable will work after Linux ODBC tests are enabled. ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=dremio_container;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" - # ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=host.docker.internal;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" ARROW_GANDIVA: "OFF" ARROW_GCS: "OFF" ARROW_HDFS: "OFF" @@ -576,34 +574,13 @@ services: # GH-49651 Link ODBC tests statically on Linux to fix segfault ARROW_TEST_LINKAGE: "static" # Register ODBC before running tests - # -AL- temp: test 32010 port is reachable. networks: - odbc_net - # -AL- nc -vz localhost 32010 did not work. command: > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && sudo /arrow/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /usr/local/lib/libarrow_flight_sql_odbc.so && - /arrow/ci/scripts/cpp_test.sh /arrow /build - " - -#-AL- tried locations -# /arrow/build/cpp/release/arrow-flight-sql-odbc-test "--gtest_filter=*.*SQLDriverConn*" -# /arrow/build/cpp/arrow-flight-sql-odbc-test "--gtest_filter=*.*SQLDriverConn*" - -# -AL- old commands -# /bin/bash -c " -# /arrow/ci/scripts/cpp_build.sh /arrow /build && -# sudo /arrow/cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc.sh /usr/local/lib/libarrow_flight_sql_odbc.so && -# /arrow/ci/scripts/cpp_test.sh /arrow /build && -# apt-get install -y netcat-openbsd && -# nc -vz host.docker.internal 32010 && -# apt-get install -y unixodbc && -# isql -k \"$ARROW_FLIGHT_SQL_ODBC_CONN\" < Date: Tue, 7 Apr 2026 15:55:46 -0700 Subject: [PATCH 10/12] Test remote instance --- ci/scripts/cpp_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index 2f88cdc819b2..fda73990b2f4 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -55,7 +55,7 @@ if ! type minio >/dev/null 2>&1; then fi case "$(uname)" in Linux) - exclude_tests+=("arrow-flight-sql-odbc-test") + # exclude_tests+=("arrow-flight-sql-odbc-test") # -AL- todo revert after CI connects to remote. n_jobs=$(nproc) ;; Darwin) From ae7b9e93011e3cd6ef7ce8a4eeeebcd003cf1e78 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Tue, 7 Apr 2026 16:24:33 -0700 Subject: [PATCH 11/12] CI can connect to remote instance, revert "Test remote instance" This reverts commit db975bfd30490e21e21b4667f8a004f49347e0ca. --- ci/scripts/cpp_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index fda73990b2f4..2f88cdc819b2 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -55,7 +55,7 @@ if ! type minio >/dev/null 2>&1; then fi case "$(uname)" in Linux) - # exclude_tests+=("arrow-flight-sql-odbc-test") # -AL- todo revert after CI connects to remote. + exclude_tests+=("arrow-flight-sql-odbc-test") n_jobs=$(nproc) ;; Darwin) From 59a7797ebcd2198a3fb8acd90957349368165840 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Tue, 7 Apr 2026 16:27:12 -0700 Subject: [PATCH 12/12] Disable ODBC Debian build * Seeing segfault at ODBC test. Can probably be solved by passing test linkage = static like we did with Ubuntu. Since we don't have capacity to work on Debian now, we can close this. --- .github/workflows/cpp_extra.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 2cabf81a0169..756993f68715 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -377,8 +377,9 @@ jobs: include: - image: ubuntu-cpp-odbc title: AMD64 Ubuntu - - image: debian-cpp-odbc - title: AMD64 Debian + # GH-49582: TODO Enable Debian build for ODBC + # - image: debian-cpp-odbc + # title: AMD64 Debian env: ARCH: amd64 ARCHERY_DEBUG: 1