Merge branch 'main' into azure_move

apache · Feb 9, 2024 · 6709d61 · 6709d61
2 parents a744fcf + a0dec7f
commit 6709d61
Show file tree

Hide file tree

Showing 227 changed files with 6,026 additions and 1,053 deletions.
diff --git a/.env b/.env
@@ -65,7 +65,7 @@ JDK=8
 KARTOTHEK=latest
 # LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
 LLVM=14
-MAVEN=3.5.4
+MAVEN=3.6.3
 NODE=18
 NUMBA=latest
 NUMPY=latest
@@ -92,13 +92,13 @@ DEVTOOLSET_VERSION=
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
 # docker tags more readable.
-VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"    # 2023.04.15 Release
+VCPKG="a42af01b72c28a8e1d7b48107b33e4f286a55ef6"    # 2023.11.20 Release
 
 # This must be updated when we update
-# ci/docker/python-wheel-windows-vs2017.dockerfile.
+# ci/docker/python-wheel-windows-vs2019.dockerfile.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2023-08-02
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-02-05
 
 # Use conanio/${CONAN} for "docker-compose run --rm conan". See
 # https://github.com/conan-io/conan-docker-tools#readme for available

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -30,15 +30,10 @@
 # /cpp/
 /cpp/src/arrow/acero @westonpace
 /cpp/src/arrow/adapters/orc @wgtmac
-/cpp/src/arrow/dataset @westonpace
 /cpp/src/arrow/engine @westonpace
 /cpp/src/arrow/flight/ @lidavidm
-/cpp/src/arrow/util/async* @westonpace
-/cpp/src/arrow/util/future* @westonpace
-/cpp/src/arrow/util/thread* @westonpace
 /cpp/src/parquet @wgtmac
-/cpp/src/skyhook @westonpace
-/csharp/ @westonpace
+/csharp/ @curthagenlocher
 /go/ @zeroshade
 /java/ @lidavidm
 /js/ @domoritz @trxcllnt

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -57,37 +57,65 @@ env:
   DOCKER_VOLUME_PREFIX: ".docker/"
 
 jobs:
+  docker-targets:
+    name: Docker targets
+    runs-on: ubuntu-latest
+    outputs:
+      targets: ${{ steps.detect-targets.outputs.targets }}
+    steps:
+      - name: Detect targets
+        id: detect-targets
+        run: |
+          echo "targets<<JSON" >> "$GITHUB_OUTPUT"
+          echo "[" >> "$GITHUB_OUTPUT"
+          cat <<JSON >> "$GITHUB_OUTPUT"
+          {
+            "arch": "amd64",
+            "clang-tools": "14",
+            "image": "conda-cpp",
+            "llvm": "14",
+            "runs-on": "ubuntu-latest",
+            "simd-level": "AVX2",
+            "title": "AMD64 Conda C++ AVX2",
+            "ubuntu": "22.04"
+          },
+          {
+            "arch": "amd64",
+            "clang-tools": "14",
+            "image": "ubuntu-cpp-sanitizer",
+            "llvm": "14",
+            "runs-on": "ubuntu-latest",
+            "title": "AMD64 Ubuntu 22.04 C++ ASAN UBSAN",
+            "ubuntu": "22.04"
+          }
+          JSON
+          if [ "$GITHUB_REPOSITORY_OWNER" = "apache" ]; then
+            echo "," >> "$GITHUB_OUTPUT"
+            cat <<JSON >> "$GITHUB_OUTPUT"
+          {
+            "arch": "arm64v8",
+            "clang-tools": "10",
+            "image": "ubuntu-cpp",
+            "llvm": "10",
+            "runs-on": ["self-hosted", "arm", "linux"],
+            "title": "ARM64 Ubuntu 20.04 C++",
+            "ubuntu": "20.04"
+          }
+          JSON
+          fi
+          echo "]" >> "$GITHUB_OUTPUT"
+          echo "JSON" >> "$GITHUB_OUTPUT"
+
   docker:
     name: ${{ matrix.title }}
+    needs: docker-targets
     runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - arch: amd64
-            clang-tools: "14"
-            image: conda-cpp
-            llvm: "14"
-            runs-on: ubuntu-latest
-            simd-level: AVX2
-            title: AMD64 Conda C++ AVX2
-            ubuntu: "22.04"
-          - arch: amd64
-            clang-tools: "14"
-            image: ubuntu-cpp-sanitizer
-            llvm: "14"
-            runs-on: ubuntu-latest
-            title: AMD64 Ubuntu 22.04 C++ ASAN UBSAN
-            ubuntu: "22.04"
-          - arch: arm64v8
-            clang-tools: "10"
-            image: ubuntu-cpp
-            llvm: "10"
-            runs-on: ["self-hosted", "arm", "linux"]
-            title: ARM64 Ubuntu 20.04 C++
-            ubuntu: "20.04"
+        include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
       ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
@@ -284,10 +312,6 @@ jobs:
             /t REG_DWORD `
             /d 1 `
             /f
-      - name: Installed Packages
-        run: choco list
-      - name: Install Dependencies
-        run: choco install -y --no-progress openssl
       - name: Checkout Arrow
         uses: actions/checkout@v4
         with:

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -43,31 +43,62 @@ permissions:
 
 jobs:
 
+  docker-targets:
+    name: Docker targets
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    outputs:
+      targets: ${{ steps.detect-targets.outputs.targets }}
+    steps:
+      - name: Detect targets
+        id: detect-targets
+        run: |
+          echo "targets<<JSON" >> "$GITHUB_OUTPUT"
+          echo "[" >> "$GITHUB_OUTPUT"
+          cat <<JSON >> "$GITHUB_OUTPUT"
+          {
+            "arch-label": "AMD64",
+            "arch": "amd64",
+            "go": "1.19",
+            "runs-on": "ubuntu-latest"
+          },
+          {
+            "arch-label": "AMD64",
+            "arch": "amd64",
+            "go": "1.20",
+            "runs-on": "ubuntu-latest"
+          }
+          JSON
+          if [ "$GITHUB_REPOSITORY_OWNER" = "apache" ]; then
+            echo "," >> "$GITHUB_OUTPUT"
+            cat <<JSON >> "$GITHUB_OUTPUT"
+          {
+            "arch-label": "ARM64",
+            "arch": "arm64v8",
+            "go": "1.19",
+            "runs-on": ["self-hosted", "arm", "linux"]
+          },
+          {
+            "arch-label": "ARM64",
+            "arch": "arm64v8",
+            "go": "1.20",
+            "runs-on": ["self-hosted", "arm", "linux"]
+          }
+          JSON
+          fi
+          echo "]" >> "$GITHUB_OUTPUT"
+          echo "JSON" >> "$GITHUB_OUTPUT"
+
   docker:
     name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
+    needs: docker-targets
     runs-on: ${{ matrix.runs-on }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - arch-label: AMD64
-            arch: amd64
-            go: 1.19
-            runs-on: ubuntu-latest
-          - arch-label: AMD64
-            arch: amd64
-            go: '1.20'
-            runs-on: ubuntu-latest
-          - arch-label: ARM64
-            arch: arm64v8
-            go: 1.19
-            runs-on: ["self-hosted", "arm", "linux"]
-          - arch-label: ARM64
-            arch: arm64v8
-            go: '1.20'
-            runs-on: ["self-hosted", "arm", "linux"]
+        include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
     env:
       ARCH: ${{ matrix.arch }}
       GO: ${{ matrix.go }}

diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml
@@ -52,7 +52,7 @@ jobs:
       - name: Install ninja-build
         run: sudo apt-get install ninja-build
       - name: Install MATLAB
-        uses: matlab-actions/setup-matlab@v1
+        uses: matlab-actions/setup-matlab@v2
         with:
           release: R2023a
       - name: Install ccache
@@ -85,7 +85,7 @@ jobs:
           # Add the installation directory to the MATLAB Search Path by
           # setting the MATLABPATH environment variable.
           MATLABPATH: matlab/install/arrow_matlab
-        uses: matlab-actions/run-tests@v1
+        uses: matlab-actions/run-tests@v2
         with:
           select-by-folder: matlab/test
   macos:
@@ -100,7 +100,7 @@ jobs:
       - name: Install ninja-build
         run: brew install ninja
       - name: Install MATLAB
-        uses: matlab-actions/setup-matlab@v1
+        uses: matlab-actions/setup-matlab@v2
         with:
           release: R2023a
       - name: Install ccache
@@ -125,7 +125,7 @@ jobs:
           # Add the installation directory to the MATLAB Search Path by
           # setting the MATLABPATH environment variable.
           MATLABPATH: matlab/install/arrow_matlab
-        uses: matlab-actions/run-tests@v1
+        uses: matlab-actions/run-tests@v2
         with:
           select-by-folder: matlab/test 
   windows:
@@ -138,7 +138,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Install MATLAB
-        uses: matlab-actions/setup-matlab@v1
+        uses: matlab-actions/setup-matlab@v2
         with:
           release: R2023a
       - name: Download Timezone Database
@@ -171,6 +171,6 @@ jobs:
           # Add the installation directory to the MATLAB Search Path by
           # setting the MATLABPATH environment variable.
           MATLABPATH: matlab/install/arrow_matlab
-        uses: matlab-actions/run-tests@v1
+        uses: matlab-actions/run-tests@v2
         with:
           select-by-folder: matlab/test