Merge remote-tracking branch 'upstream/main' into apacheGH-15187

.asf.yaml

@@ -19,8 +19,9 @@ github:
   description: "Apache Arrow is a multi-language toolbox for accelerated data interchange and in-memory processing"
   homepage: https://arrow.apache.org/
   collaborators:
-    - assignUser
     - benibus
+    - danepitkin
+    - felipecrv
     - milesgranger
     - toddfarmer
 

.env

@@ -56,7 +56,7 @@ UBUNTU=20.04
 CLANG_TOOLS=14
 CUDA=11.0.3
 DASK=latest
-DOTNET=6.0
+DOTNET=7.0
 GCC_VERSION=""
 GO=1.17
 STATICCHECK=v0.2.2
@@ -92,12 +92,7 @@ DEVTOOLSET_VERSION=
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
 # docker tags more readable.
-#
-# Please also update the crossbow configuration in order to keep the github
-# actions cache up to date for the macOS wheels:
-#   https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
-# vcpkg minimum version "09adfdc8cdad76345b7cc7f3305899e1cbd66297" due to CVE-2022-3786
-VCPKG="2871ddd918cecb9cb642bcb9c56897f397283192"
+VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"    # 2023.04.15 Release
 
 # This must be updated when we update
 # ci/docker/python-wheel-windows-vs2017.dockerfile.

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -39,7 +39,6 @@ body:
         - C#
         - C++
         - C++ - Gandiva
-        - C++ - Plasma
         - Continuous Integration
         - Developer Tools
         - Documentation

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -42,7 +42,6 @@ body:
         - C#
         - C++
         - C++ - Gandiva
-        - C++ - Plasma
         - Continuous Integration
         - Developer Tools
         - Documentation

.github/ISSUE_TEMPLATE/usage_question.yaml

@@ -62,7 +62,6 @@ body:
         - C#
         - C++
         - C++ - Gandiva
-        - C++ - Plasma
         - Continuous Integration
         - Developer Tools
         - Documentation

.github/workflows/cpp.yml

@@ -55,22 +55,39 @@ env:
 jobs:
   docker:
     name: ${{ matrix.title }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
     strategy:
       fail-fast: false
       matrix:
-        image:
-          - conda-cpp
-          - ubuntu-cpp-sanitizer
         include:
-          - image: conda-cpp
+          - arch: amd64
+            clang-tools: "14"
+            image: conda-cpp
+            llvm: "14"
+            runs-on: ubuntu-latest
             title: AMD64 Conda C++
-          - image: ubuntu-cpp-sanitizer
+            ubuntu: "22.04"
+          - arch: amd64
+            clang-tools: "14"
+            image: ubuntu-cpp-sanitizer
+            llvm: "14"
+            runs-on: ubuntu-latest
             title: AMD64 Ubuntu 22.04 C++ ASAN UBSAN
+            ubuntu: "22.04"
+          - arch: arm64v8
+            clang-tools: "10"
+            image: ubuntu-cpp
+            llvm: "10"
+            runs-on: ["self-hosted", "arm", "linux"]
+            title: ARM64 Ubuntu 20.04 C++
+            ubuntu: "20.04"
     env:
-      UBUNTU: "22.04"
+      ARCH: ${{ matrix.arch }}
+      CLANG_TOOLS: ${{ matrix.clang-tools }}
+      LLVM: ${{ matrix.llvm }}
+      UBUNTU: ${{ matrix.ubuntu }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v3
@@ -84,11 +101,10 @@ jobs:
           key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
           restore-keys: ${{ matrix.image }}-
       - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
+        run: |
+          sudo apt install -y --no-install-recommends python3 python3-pip
       - name: Setup Archery
-        run: pip install -e dev/archery[docker]
+        run: python3 -m pip install -e dev/archery[docker]
       - name: Execute Docker Build
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -142,7 +158,6 @@ jobs:
       # TODO(kszucs): link error in the tests
       ARROW_ORC: OFF
       ARROW_PARQUET: ON
-      ARROW_PLASMA: ON
       ARROW_S3: ON
       ARROW_WITH_BROTLI: ON
       ARROW_WITH_BZ2: ON

.github/workflows/dev_pr/link.js

@@ -83,22 +83,16 @@ async function commentGitHubURL(github, context, pullRequestNumber, issueID) {
   // Make the call to ensure issue exists before adding comment
   const issueInfo = await helpers.getGitHubInfo(github, context, issueID, pullRequestNumber);
   const message = "* Closes: #" + issueInfo.number
-  if (await haveComment(github, context, pullRequestNumber, message)) {
-    return;
-  }
-  if (issueInfo){
+  if (issueInfo) {
+    if (context.payload.pull_request.body.includes(message)) {
+      return;
+    }
     await github.pulls.update({
       owner: context.repo.owner,
       repo: context.repo.repo,
       pull_number: pullRequestNumber,
       body: (context.payload.pull_request.body || "") + "\n" + message
     });
-    await github.issues.createComment({
-      owner: context.repo.owner,
-      repo: context.repo.repo,
-      issue_number: pullRequestNumber,
-      body: message
-    });
   }
 }
 

.github/workflows/go.yml

@@ -42,20 +42,31 @@ permissions:
 jobs:
 
   docker:
-    name: AMD64 Debian 11 Go ${{ matrix.go }}
-    runs-on: ubuntu-latest
+    name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
+    runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
         include:
-          - go: 1.17
+          - arch-label: AMD64
+            arch: amd64
+            go: 1.17
+            runs-on: ubuntu-latest
             staticcheck: v0.2.2
-          - go: 1.18
+          - arch-label: AMD64
+            arch: amd64
+            go: 1.18
+            runs-on: ubuntu-latest
             staticcheck: v0.3.3
+          - arch-label: ARM64
+            arch: arm64v8
+            go: 1.17
+            staticcheck: v0.2.2
+            runs-on: ["self-hosted", "arm", "linux"]
     env:
+      ARCH: ${{ matrix.arch }}
       GO: ${{ matrix.go }}
       STATICCHECK: ${{ matrix.staticcheck }}
     steps:
@@ -65,47 +76,57 @@ jobs:
           fetch-depth: 0
           submodules: recursive
       - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
+        run: |
+          sudo apt install -y --no-install-recommends python3 python3-pip
       - name: Setup Archery
-        run: pip install -e dev/archery[docker]
+        run: python3 -m pip install -e dev/archery[docker]
       - name: Execute Docker Build
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         run: archery docker run debian-go
       - name: Docker Push
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
+        if: >-
+          success() &&
+          github.event_name == 'push' &&
+          github.repository == 'apache/arrow'
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
         run: archery docker push debian-go
       - name: Install Go ${{ matrix.go }} for Benchmarks
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
+        if: >-
+          success() &&
+          matrix.arch == 'amd64' &&
+          github.event_name == 'push' &&
+          github.repository == 'apache/arrow'
         uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
           cache: true
           cache-dependency-path: go/go.sum
       - name: Run Benchmarks
-        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
+        if: >-
+          success() &&
+          matrix.arch == 'amd64' &&
+          github.event_name == 'push' &&
+          github.repository == 'apache/arrow'
         env:
           CONBENCH_URL: https://conbench.ursa.dev
           CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
           CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
           CONBENCH_REF: ${{ github.ref_name }}
-          CONBENCH_MACHINE_INFO_NAME: amd64-debian-11
+          CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
         run: |
-          pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
-          python ci/scripts/go_bench_adapt.py
+          python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
+          python3 ci/scripts/go_bench_adapt.py
 
   docker_cgo:
-    name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO
+    name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 15
+    timeout-minutes: 20
     strategy:
       fail-fast: false
       matrix:
@@ -145,7 +166,7 @@ jobs:
 
 
   docker_cgo_python:
-    name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO Python
+    name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 15
@@ -306,6 +327,10 @@ jobs:
         run: brew install apache-arrow pkg-config
       - name: Install staticcheck
         run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }}
+      - name: Add To pkg config path
+        shell: bash
+        run: |
+          echo "PKG_CONFIG_PATH=$(brew --prefix openssl@3)/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)

.github/workflows/pr_bot.yml

@@ -66,6 +66,12 @@ jobs:
         run: |
           unzip pr_review.zip
           echo "pr_review_path=$(pwd)/event.json" >> $GITHUB_OUTPUT
+      - name: Download committers file
+        id: committers
+        run: |
+          url="https://raw.githubusercontent.com/apache/arrow-site/main/_data/committers.yml"
+          curl -sL -o committers.yml $url
+          echo "committers_path=$(pwd)/committers.yml" >> $GITHUB_OUTPUT
       - name: Checkout Arrow
         uses: actions/checkout@v3
         with:
@@ -89,9 +95,11 @@ jobs:
             # workflow_run is executed on PR review. Update to original event.
             archery trigger-bot \
               --event-name "pull_request_review" \
-              --event-payload "${{ steps.extract.outputs.pr_review_path }}"
+              --event-payload "${{ steps.extract.outputs.pr_review_path }}" \
+              --committers-file "${{ steps.committers.outputs.committers_path }}"
           else
             archery trigger-bot \
               --event-name "${{ github.event_name }}" \
-              --event-payload "${{ github.event_path }}"
+              --event-payload "${{ github.event_path }}" \
+              --committers-file "${{ steps.committers.outputs.committers_path }}"
           fi

.github/workflows/pr_review_trigger.yml

@@ -18,6 +18,9 @@
 name: "Label when reviewed"
 on: pull_request_review
 
+permissions:
+  contents: read
+
 jobs:
   # due to GitHub Actions permissions we can't change labels on the pull_request_review
   # workflow. We trigger a new workflow run which will have permissions to add labels.

.github/workflows/python.yml

@@ -135,7 +135,6 @@ jobs:
       ARROW_ORC: ON
       ARROW_PARQUET: ON
       PARQUET_REQUIRE_ENCRYPTION: ON
-      ARROW_PLASMA: ON
       ARROW_PYTHON: ON
       ARROW_S3: ON
       ARROW_SUBSTRAIT: ON

.github/workflows/r_nightly.yml

@@ -58,7 +58,7 @@ jobs:
           fetch-depth: 0
           path: crossbow
           repository: ursacomputing/crossbow
-          ref: master
+          ref: main
       - name: Set up Python
         uses: actions/setup-python@v4
         with:

.github/workflows/ruby.yml

@@ -204,8 +204,7 @@ jobs:
       ARROW_BUILD_TYPE: release
       ARROW_FLIGHT: ON
       ARROW_FLIGHT_SQL: ON
-      # ARROW-17728: SEGV on MinGW
-      ARROW_GANDIVA: OFF
+      ARROW_GANDIVA: ON
       ARROW_GCS: ON
       ARROW_HDFS: OFF
       ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }}

.gitignore

@@ -91,4 +91,10 @@ java-native-cpp/
 # archery files
 dev/archery/build
 
-swift/Arrow/.build
+swift/Arrow/.build
+
+# Go dependencies
+go/vendor
+
+# direnv
+.envrc

.pre-commit-config.yaml

@@ -47,3 +47,8 @@ repos:
           - python
         exclude: vendored
         args: [--config, python/setup.cfg]
+  - repo: https://github.com/MarcoGorelli/cython-lint
+    rev: v0.12.5
+    hooks:
+      - id: cython-lint
+        args: [--no-pycodestyle]