Merge remote-tracking branch 'upstream/main' into apachegh-38618

.github/workflows/comment_bot.yml

@@ -46,7 +46,7 @@ jobs:
           python-version: 3.8
       - name: Install Archery and Crossbow dependencies
         run: pip install -e arrow/dev/archery[bot]
-      - name: Handle Github comment event
+      - name: Handle GitHub comment event
         env:
           ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN }}
@@ -182,7 +182,7 @@ jobs:
     if: github.event.comment.body == 'take'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

.github/workflows/cpp.yml

@@ -193,12 +193,6 @@ jobs:
           submodules: recursive
       - name: Install Dependencies
         run: |
-          rm -f /usr/local/bin/2to3* || :
-          rm -f /usr/local/bin/idle3* || :
-          rm -f /usr/local/bin/pydoc3* || :
-          rm -f /usr/local/bin/python3* || :
-          rm -f /usr/local/bin/python3-config || :
-          brew update --preinstall || :
           brew bundle --file=cpp/Brewfile
       - name: Install MinIO
         run: |

.github/workflows/csharp.yml

@@ -49,7 +49,7 @@ jobs:
         dotnet: ['7.0.x']
     steps:
       - name: Install C#
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Checkout Arrow
@@ -77,7 +77,7 @@ jobs:
         dotnet: ['7.0.x']
     steps:
       - name: Install C#
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Checkout Arrow
@@ -104,7 +104,7 @@ jobs:
         dotnet: ['7.0.x']
     steps:
       - name: Install C#
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Checkout Arrow

.github/workflows/dev.yml

@@ -96,7 +96,7 @@ jobs:
         with:
           ruby-version: '2.7'
       - name: Install .NET
-        uses: actions/setup-dotnet@3447fd6a9f9e57506b15f895c5b76d3b197dc7c2 # v3.2.0
+        uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4.0.0
         with:
           dotnet-version: '7.0.x'
       - name: Install Dependencies

.github/workflows/dev_pr.yml

@@ -53,7 +53,7 @@ jobs:
         if: |
           (github.event.action == 'opened' ||
            github.event.action == 'edited')
-        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -64,7 +64,7 @@ jobs:
         if: |
           (github.event.action == 'opened' ||
            github.event.action == 'edited')
-        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -75,7 +75,7 @@ jobs:
         if: |
           (github.event.action == 'opened' ||
            github.event.action == 'edited')
-        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           debug: true
           github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/dev_pr/issue_check.js

@@ -103,7 +103,7 @@ async function commentNotStartedTicket(github, context, pullRequestNumber) {
 }
 
 /**
- * Assigns the Github Issue to the PR creator.
+ * Assigns the GitHub Issue to the PR creator.
  *
  * @param {Object} github
  * @param {Object} context

.github/workflows/issue_bot.yml

@@ -33,7 +33,7 @@ jobs:
     if: github.event.issue.pull_request == null
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           script: |
             let split_body = context.payload.issue.body.split('### Component(s)');
@@ -64,7 +64,7 @@ jobs:
               "per_page": 100,
             });
 
-            // this removes non-existent labels
+            // this removes nonexistent labels
             component_labels = component_labels.filter(
               label => repo_labels.data.some(repo_label => repo_label.name === label)
             );

.github/workflows/java.yml

@@ -48,7 +48,6 @@ env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
-
   ubuntu:
     name: AMD64 Ubuntu 22.04 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
     runs-on: ubuntu-latest
@@ -109,7 +108,7 @@ jobs:
         jdk: [11]
     steps:
       - name: Set up Java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'zulu'
           java-version: ${{ matrix.jdk }}
@@ -136,7 +135,7 @@ jobs:
         jdk: [11]
     steps:
       - name: Set up Java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.jdk }}
           distribution: 'temurin'

.github/workflows/java_jni.yml

@@ -48,7 +48,6 @@ env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
-
   docker:
     name: AMD64 manylinux2014 Java JNI
     runs-on: ubuntu-latest

.github/workflows/pr_bot.yml

@@ -40,7 +40,7 @@ jobs:
       - name: 'Download PR review payload'
         id: 'download'
         if: github.event_name == 'workflow_run'
-        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           script: |
             const run_id = "${{ github.event.workflow_run.id }}";

.github/workflows/python.yml

@@ -167,15 +167,7 @@ jobs:
       - name: Install Dependencies
         shell: bash
         run: |
-          rm -f /usr/local/bin/2to3* || :
-          rm -f /usr/local/bin/idle3* || :
-          rm -f /usr/local/bin/pydoc3* || :
-          rm -f /usr/local/bin/python3* || :
-          rm -f /usr/local/bin/python3-config || :
-          brew update --preinstall || :
-          brew install --overwrite git
           brew bundle --file=cpp/Brewfile
-          brew install coreutils
           python -m pip install \
             -r python/requirements-build.txt \
             -r python/requirements-test.txt

.github/workflows/r.yml

@@ -60,7 +60,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        r: ["4.2"]
+        r: ["4.3"]
         ubuntu: [20.04]
         force-tests: ["true"]
     env:
@@ -215,6 +215,9 @@ jobs:
             r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-
       - uses: r-lib/actions/setup-r@v2
         with:
+          # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++
+          # static library. The R is not used here but R 4.1 was the last R to use
+          # Rtools40.
           r-version: "4.1"
           rtools-version: 40
           Ncpus: 2
@@ -234,16 +237,16 @@ jobs:
 
   windows-r:
     needs: [windows-cpp]
-    name: AMD64 Windows R ${{ matrix.config.rversion }} RTools ${{ matrix.config.rtools }}
+    name: AMD64 Windows R ${{ matrix.config.rversion }}
     runs-on: windows-2019
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
     strategy:
       fail-fast: false
       matrix:
         config:
-        - { rtools: 42, rversion: "4.2" }
-        - { rtools: 42, rversion: "devel" }
+        - { rversion: "release" }
+
     env:
       ARROW_R_CXXFLAGS: "-Werror"
       _R_CHECK_TESTS_NLINES_: 0
@@ -255,7 +258,6 @@ jobs:
           fetch-depth: 0
       - run: mkdir r/windows
       - name: Download artifacts
-        if: ${{ matrix.config.rtools == 42 }}
         uses: actions/download-artifact@v3
         with:
           name: libarrow-rtools40-ucrt64.zip
@@ -269,7 +271,6 @@ jobs:
       - uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.config.rversion }}
-          rtools-version: ${{ matrix.config.rtools }}
           Ncpus: 2
       - uses: r-lib/actions/setup-r-dependencies@v2
         env:
@@ -318,7 +319,7 @@ jobs:
             timeout = 3600
           )
       - name: Run lintr
-        if: ${{ matrix.config.rversion == '4.2' }}
+        if: ${{ matrix.config.rversion == 'release' }}
         env:
           NOT_CRAN: "true"
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ruby.yml

@@ -149,13 +149,6 @@ jobs:
       - name: Install Homebrew Dependencies
         shell: bash
         run: |
-          rm -f /usr/local/bin/2to3* || :
-          rm -f /usr/local/bin/idle3* || :
-          rm -f /usr/local/bin/pydoc3* || :
-          rm -f /usr/local/bin/python3* || :
-          rm -f /usr/local/bin/python3-config || :
-          brew update --preinstall || :
-          brew install --overwrite git
           brew bundle --file=cpp/Brewfile
           brew bundle --file=c_glib/Brewfile
       - name: Install Ruby Dependencies

CHANGELOG.md

@@ -1459,7 +1459,7 @@
 * [ARROW-12172](https://issues.apache.org/jira/browse/ARROW-12172) - [Python][Packaging] Pass python version as setuptools pretend version in the macOS wheel builds
 * [ARROW-12178](https://issues.apache.org/jira/browse/ARROW-12178) - [CI] Update setuptools in the ubuntu images
 * [ARROW-12186](https://issues.apache.org/jira/browse/ARROW-12186) - [Rust][DataFusion] Fix regexp_match test
-* [ARROW-12209](https://issues.apache.org/jira/browse/ARROW-12209) - [JS] Copy all src files into the the TypeScript package
+* [ARROW-12209](https://issues.apache.org/jira/browse/ARROW-12209) - [JS] Copy all src files into the TypeScript package
 * [ARROW-12220](https://issues.apache.org/jira/browse/ARROW-12220) - [C++][CI] Thread sanitizer failure
 * [ARROW-12226](https://issues.apache.org/jira/browse/ARROW-12226) - [C++] Fix Address Sanitizer failures
 * [ARROW-12227](https://issues.apache.org/jira/browse/ARROW-12227) - [R] Fix RE2 and median nightly build failures
@@ -11430,7 +11430,7 @@
 * [ARROW-67](https://issues.apache.org/jira/browse/ARROW-67) - C++ metadata flatbuffer serialization and data movement to memory maps
 * [ARROW-68](https://issues.apache.org/jira/browse/ARROW-68) - Better error handling for not fully setup systems
 * [ARROW-70](https://issues.apache.org/jira/browse/ARROW-70) - Add adapt 'lite' DCHECK macros from Kudu as also used in Parquet
-* [ARROW-71](https://issues.apache.org/jira/browse/ARROW-71) - [C++] Add clang-tidy and clang-format to the the tool chain.
+* [ARROW-71](https://issues.apache.org/jira/browse/ARROW-71) - [C++] Add clang-tidy and clang-format to the tool chain.
 * [ARROW-73](https://issues.apache.org/jira/browse/ARROW-73) - Support older CMake versions
 * [ARROW-76](https://issues.apache.org/jira/browse/ARROW-76) - Revise format document to include null count, defer non-nullable arrays to the domain of metadata
 * [ARROW-78](https://issues.apache.org/jira/browse/ARROW-78) - C++: Add constructor for DecimalType

ci/scripts/PKGBUILD

@@ -18,7 +18,7 @@
 _realname=arrow
 pkgbase=mingw-w64-${_realname}
 pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}"
-pkgver=14.0.0.9000
+pkgver=14.0.1.9000
 pkgrel=8000
 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)"
 arch=("any")

ci/scripts/go_build.sh

@@ -21,6 +21,9 @@ set -ex
 
 source_dir=${1}/go
 
+# Need "all=" as per https://github.com/golang/go/issues/42131#issuecomment-713917379
+export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"
+
 pushd ${source_dir}/arrow
 
 if [[ -n "${ARROW_GO_TESTCGO}" ]]; then

ci/scripts/go_cgo_python_test.sh

@@ -21,6 +21,8 @@ set -ex
 
 source_dir=${1}/go
 
+export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"
+
 pushd ${source_dir}/arrow/cdata/test
 
 case "$(uname)" in

ci/scripts/integration_arrow.sh

@@ -43,6 +43,12 @@ fi
 # Get more detailed context on crashes
 export PYTHONFAULTHANDLER=1
 
+# Due to how Go reads environment variables, we have to set them from the calling
+# process, or they would get ignored.
+# (see https://forum.golangbridge.org/t/are-godebug-and-other-env-vars-ignored-when-loading-a-go-dll-from-foreign-code/33694)
+export GOMEMLIMIT=200MiB
+export GODEBUG=gctrace=1,clobberfree=1
+
 # Rust can be enabled by exporting ARCHERY_INTEGRATION_WITH_RUST=1
 time archery integration \
     --run-c-data \

ci/scripts/integration_dask.sh

@@ -32,7 +32,9 @@ python -c "import dask.dataframe"
 # pytest -sv --pyargs dask.bytes.tests.test_local
 
 # The "skip_with_pyarrow_strings" marker is meant to skip automatically, but that doesn't work with --pyargs, so de-selecting manually
-pytest -v --pyargs dask.dataframe.tests.test_dataframe -m "not skip_with_pyarrow_strings"
+# - The 'test_categorize_info' test is failing because of change in StringArray's nbytes and
+#   an upstream fix (https://github.com/apache/arrow/issues/39028)
+pytest -v --pyargs dask.dataframe.tests.test_dataframe -m "not skip_with_pyarrow_strings" -k "not test_categorize_info"
 pytest -v --pyargs dask.dataframe.io.tests.test_orc
 pytest -v --pyargs dask.dataframe.io.tests.test_parquet \
   -m "not skip_with_pyarrow_strings and not xfail_with_pyarrow_strings"

ci/scripts/java_build.sh

@@ -77,20 +77,20 @@ mvn="${mvn} -T 2C"
 
 pushd ${source_dir}
 
-${mvn} install
-
 if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then
-  ${mvn} -Pshade-flatbuffers install
+  mvn="${mvn} -Pshade-flatbuffers"
 fi
 
 if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
-  ${mvn} -Darrow.c.jni.dist.dir=${java_jni_dist_dir} -Parrow-c-data install
+  mvn="${mvn} -Darrow.c.jni.dist.dir=${java_jni_dist_dir} -Parrow-c-data"
 fi
 
 if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
-  ${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni install
+  mvn="${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni"
 fi
 
+${mvn} install
+
 if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then
   # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633
   mkdir -p ${build_dir}/docs/java/reference

ci/scripts/rust_build.sh

@@ -21,6 +21,7 @@ set -e
 
 arrow_dir=${1}
 source_dir=${1}/rust
+build_dir=${2}/rust
 
 # This file is used to build the rust binaries needed for the archery
 # integration tests. Testing of the rust implementation in normal CI is handled
@@ -54,7 +55,7 @@ rustup show
 pushd ${source_dir}
 
 # build only the integration testing binaries
-cargo build -p arrow-integration-testing
+cargo build -p arrow-integration-testing --target-dir ${build_dir}
 
 # Save disk space by removing large temporary build products
 rm -rf target/debug/deps

cmake-format.py

@@ -72,5 +72,5 @@
     first_comment_is_literal = True
 
     # If comment markup is enabled, don't reflow any comment block which
-    # matchesthis (regex) pattern. Default is `None` (disabled).
+    # matches this (regex) pattern. Default is `None` (disabled).
     literal_comment_pattern = None

cpp/CMakePresets.json

@@ -430,6 +430,21 @@
       ],
       "displayName": "Benchmarking build with with everything enabled",
       "cacheVariables": {}
+    },
+    {
+      "name": "fuzzing",
+      "inherits": "base",
+      "displayName": "Debug build with IPC and Parquet fuzzing targets",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++",
+        "ARROW_USE_ASAN": "ON",
+        "ARROW_USE_UBSAN": "ON",
+        "ARROW_IPC": "ON",
+        "ARROW_PARQUET": "ON",
+        "ARROW_FUZZING": "ON"
+      }
     }
   ]
 }