Merge branch 'master' into ARROW-15260_filenames

.dockerignore

@@ -28,6 +28,7 @@
 !ci/**
 !c_glib/Gemfile
 !dev/archery/setup.py
+!dev/release/setup-*.sh
 !docs/requirements*.txt
 !python/requirements*.txt
 !python/manylinux1/**

.env

@@ -55,13 +55,14 @@ UBUNTU=20.04
 CLANG_TOOLS=12
 CUDA=9.1
 DASK=latest
-DOTNET=3.1
+DOTNET=6.0
 GCC_VERSION=""
 GO=1.16
 HDFS=3.2.1
 JDK=8
 KARTOTHEK=latest
-LLVM=12
+# LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
+LLVM=13
 MAVEN=3.5.4
 NODE=16
 NUMPY=latest
@@ -72,11 +73,13 @@ SPARK=master
 TURBODBC=latest
 
 # These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
-ARROW_R_DEV=TRUE
 R_IMAGE=ubuntu-gcc-release
 R_ORG=rhub
-R_PRUNE_DEPS=FALSE
 R_TAG=latest
+
+# Env vars for R builds
+ARROW_R_DEV=TRUE
+R_PRUNE_DEPS=FALSE
 TZ=UTC
 
 # -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
@@ -89,4 +92,6 @@ DEVTOOLSET_VERSION=-1
 # Please also update the crossbow configuration in order to keep the github
 # actions cache up to date for the macOS wheels:
 #   https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
-VCPKG="30465138e"
+# After commit 89295c9 openssl is only available as 3.0.2 which is 
+# incompatible with aws-sdk-cpp 1.8.3 see https://github.com/aws/aws-sdk-cpp/issues/1582 and causes issues on manylinux2010
+VCPKG="89295c9"

.github/workflows/cpp.yml

@@ -126,6 +126,7 @@ jobs:
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      ARROW_GCS: ON
       ARROW_HDFS: ON
       ARROW_HOME: /usr/local
       ARROW_JEMALLOC: ON
@@ -141,6 +142,8 @@ jobs:
       ARROW_WITH_SNAPPY: ON
       ARROW_WITH_ZLIB: ON
       ARROW_WITH_ZSTD: ON
+      # System Abseil installed by Homebrew uses C++ 17
+      CMAKE_CXX_STANDARD: 17
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2
@@ -152,7 +155,11 @@ jobs:
         run: |
           rm -f /usr/local/bin/2to3
           brew update --preinstall
+          brew install --overwrite git
           brew bundle --file=cpp/Brewfile
+      - name: Install Google Cloud Storage Testbench
+        shell: bash
+        run: ci/scripts/install_gcs_testbench.sh default
       - name: Setup ccache
         run: |
           ci/scripts/ccache_setup.sh
@@ -268,6 +275,9 @@ jobs:
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      # google-could-cpp uses _dupenv_s() but it can't be used with msvcrt.
+      # We need to use ucrt to use _dupenv_s().
+      # ARROW_GCS: ON
       ARROW_HDFS: OFF
       ARROW_HOME: /mingw${{ matrix.mingw-n-bits }}
       ARROW_JEMALLOC: OFF

.github/workflows/csharp.yml

@@ -43,7 +43,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['3.1.x']
+        dotnet: ['6.0.x']
     steps:
       - name: Install C#
         uses: actions/setup-dotnet@v1
@@ -71,7 +71,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['3.1.x']
+        dotnet: ['6.0.x']
     steps:
       - name: Install C#
         uses: actions/setup-dotnet@v1
@@ -98,7 +98,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['3.1.x']
+        dotnet: ['6.0.x']
     steps:
       - name: Install C#
         uses: actions/setup-dotnet@v1

.github/workflows/python.yml

@@ -150,6 +150,7 @@ jobs:
         run: |
           rm -f /usr/local/bin/2to3
           brew update --preinstall
+          brew install --overwrite git
           brew bundle --file=cpp/Brewfile
           brew install coreutils
           python3 -mpip install \

.github/workflows/r.yml

@@ -50,7 +50,7 @@ env:
 
 jobs:
   ubuntu:
-    name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }}
+    name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }} Force-Tests ${{ matrix.force-tests }}
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
@@ -59,6 +59,7 @@ jobs:
       matrix:
         r: ["4.1"]
         ubuntu: [20.04]
+        force-tests: ["true", "false"]
     env:
       R: ${{ matrix.r }}
       UBUNTU: ${{ matrix.ubuntu }}
@@ -87,7 +88,7 @@ jobs:
           ulimit -c unlimited
           # Setting a non-default and non-probable Marquesas French Polynesia time
           # it has both with a .45 offset and very very few people who live there.
-          archery docker run -e TZ=MART ubuntu-r
+          archery docker run -e TZ=MART -e ARROW_R_FORCE_TESTS=${{ matrix.force-tests }} ubuntu-r
       - name: Dump install logs
         run: cat r/check/arrow.Rcheck/00install.out
         if: always()

.github/workflows/ruby.yml

@@ -90,6 +90,7 @@ jobs:
           ulimit -c unlimited
           archery docker run \
             -e ARROW_FLIGHT=ON \
+            -e ARROW_GCS=ON \
             -e Protobuf_SOURCE=BUNDLED \
             -e gRPC_SOURCE=BUNDLED \
             ubuntu-ruby
@@ -103,13 +104,16 @@ jobs:
     name: AMD64 MacOS 10.15 GLib & Ruby
     runs-on: macos-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 40
+    timeout-minutes: 60
     strategy:
       fail-fast: false
     env:
+      ARROW_BUILD_STATIC: OFF
       ARROW_BUILD_TESTS: OFF
+      ARROW_BUILD_UTILITIES: OFF
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      ARROW_GCS: ON
       ARROW_GLIB_GTK_DOC: true
       ARROW_GLIB_WERROR: true
       ARROW_HOME: /usr/local
@@ -133,6 +137,7 @@ jobs:
         run: |
           rm -f /usr/local/bin/2to3
           brew update --preinstall
+          brew install --overwrite git
           brew bundle --file=cpp/Brewfile
           brew bundle --file=c_glib/Brewfile
       - name: Install Ruby Dependencies
@@ -173,7 +178,7 @@ jobs:
     name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} GLib & Ruby
     runs-on: windows-2019
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 40
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
@@ -182,12 +187,13 @@ jobs:
         ruby-version:
           - "3.1"
     env:
-      ARROW_BUILD_SHARED: ON
       ARROW_BUILD_STATIC: OFF
       ARROW_BUILD_TESTS: OFF
+      ARROW_BUILD_UTILITIES: OFF
       ARROW_BUILD_TYPE: release
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON
+      ARROW_GCS: ON
       ARROW_HDFS: OFF
       ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }}
       ARROW_JEMALLOC: OFF

LICENSE.txt

@@ -2248,3 +2248,11 @@ SOFTWARE.
 
 The file src/arrow/util/io_util.cc contains code from the CPython project
 which is made available under the Python Software Foundation License Version 2.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency opentelemetry-cpp is statically linked in certain binary
+distributions. opentelemetry-cpp is made available under the Apache License 2.0.
+
+Copyright The OpenTelemetry Authors
+SPDX-License-Identifier: Apache-2.0

c_glib/arrow-glib/array-builder.cpp

@@ -3328,13 +3328,36 @@ gboolean
 garrow_string_array_builder_append_string(GArrowStringArrayBuilder *builder,
                                           const gchar *value,
                                           GError **error)
+{
+  return garrow_string_array_builder_append_string_len(
+    builder,
+    value,
+    static_cast<gint32>(strlen(value)),
+    error);
+}
+
+/**
+ * garrow_string_array_builder_append_string_len:
+ * @builder: A #GArrowStringArrayBuilder.
+ * @value: A string value.
+ * @length: The length of @value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 8.0.0
+ */
+gboolean
+garrow_string_array_builder_append_string_len(GArrowStringArrayBuilder *builder,
+                                              const gchar *value,
+                                              gint32 length,
+                                              GError **error)
 {
   auto arrow_builder =
     static_cast<arrow::StringBuilder *>(
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
 
-  auto status = arrow_builder->Append(value,
-                                      static_cast<gint32>(strlen(value)));
+  auto status = arrow_builder->Append(value, length);
   return garrow_error_check(error,
                             status,
                             "[string-array-builder][append-string]");
@@ -3460,14 +3483,38 @@ gboolean
 garrow_large_string_array_builder_append_string(GArrowLargeStringArrayBuilder *builder,
                                                 const gchar *value,
                                                 GError **error)
+{
+  return garrow_large_string_array_builder_append_string_len(
+    builder,
+    value,
+    static_cast<gint64>(strlen(value)),
+    error);
+}
+
+/**
+ * garrow_large_string_array_builder_append_string_len:
+ * @builder: A #GArrowLargeStringArrayBuilder.
+ * @value: A string value.
+ * @length: The length of @value.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 8.0.0
+ */
+gboolean garrow_large_string_array_builder_append_string_len(
+  GArrowLargeStringArrayBuilder *builder,
+  const gchar *value,
+  gint64 length,
+  GError **error)
 {
   auto arrow_builder =
     static_cast<arrow::LargeStringBuilder *>(
       garrow_array_builder_get_raw(GARROW_ARRAY_BUILDER(builder)));
-  auto status = arrow_builder->Append(value);
+  auto status = arrow_builder->Append(value, length);
   return garrow_error_check(error,
                             status,
-                            "[large-string-array-builder][append-string]");
+                            "[large-string-array-builder][append-string-len]");
 }
 
 /**