ARROW-12590: [C++][R] Update copies of Homebrew files to reflect recent updates

LICENSE.txt

@@ -1990,12 +1990,14 @@ for PyArrow. Ibis is released under the Apache License, Version 2.0.
 
 This project includes code from the autobrew project.
 
-* r/tools/autobrew and dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
-  are based on code from the autobrew project.
+The following files are based on code from the autobrew project:
+* r/tools/autobrew
+* dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
+* dev/tasks/hombrew-formulae/autobrew/apache-arrow-static.rb 
 
 Copyright (c) 2019, Jeroen Ooms
 License: MIT
-Homepage: https://github.com/jeroen/autobrew
+Homepage: https://github.com/autobrew/
 
 --------------------------------------------------------------------------------
 

dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb

@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Cloned from https://github.com/autobrew/homebrew-core/blob/high-sierra/Formula/apache-arrow.rb
+# Upstream any relevant changes (dependencies, build args ...)
+
+class ApacheArrowStatic < Formula
+  desc "Columnar in-memory analytics layer designed to accelerate big data"
+  homepage "https://arrow.apache.org/"
+  url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-8.0.0.9000/apache-arrow-8.0.0.9000.tar.gz"
+  # Uncomment and update to test on a release candidate
+  # mirror "https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-8.0.0-rc1/apache-arrow-8.0.0.tar.gz"
+  sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28"
+  head "https://github.com/apache/arrow.git"
+
+  bottle do
+    sha256 cellar: :any, arm64_big_sur: "ef89d21a110b89840cc6148add685d407e75bd633bc8f79625eb33d00e3694b4"
+    sha256 cellar: :any, big_sur:       "6fcb9f55d44eb61d295a8020e039a0622bdc044b220cfffef855f3e3ab8057a1"
+    sha256 cellar: :any, catalina:      "bf71b17766688077fb9b4e679f309742c16524015287dd3633758c679752c977"
+  end
+
+  depends_on "boost" => :build
+  depends_on "cmake" => :build
+  depends_on "aws-sdk-cpp-static"
+  depends_on "brotli"
+  depends_on "lz4"
+  depends_on "snappy"
+  depends_on "thrift"
+  depends_on "zstd"
+
+  conflicts_with "apache-arrow", because: "both install Arrow"
+
+  def install
+    ENV.cxx11
+    # https://github.com/Homebrew/homebrew-core/issues/94724
+    # https://issues.apache.org/jira/browse/ARROW-15664
+    ENV["HOMEBREW_OPTIMIZATION_LEVEL"] = "O2"
+
+    args = %W[
+      -DARROW_BUILD_SHARED=OFF
+      -DARROW_BUILD_UTILITIES=ON
+      -DARROW_COMPUTE=ON
+      -DARROW_CSV=ON
+      -DARROW_DATASET=ON
+      -DARROW_FILESYSTEM=ON
+      -DARROW_GCS=ON
+      -DARROW_JEMALLOC=ON
+      -DARROW_JSON=ON
+      -DARROW_MIMALLOC=ON
+      -DARROW_PARQUET=ON
+      -DARROW_S3=ON
+      -DARROW_VERBOSE_THIRDPARTY_BUILD=ON
+      -DARROW_WITH_BROTLI=ON
+      -DARROW_WITH_BZ2=ON
+      -DARROW_WITH_LZ4=ON
+      -DARROW_WITH_SNAPPY=ON
+      -DARROW_WITH_ZLIB=ON
+      -DARROW_WITH_ZSTD=ON
+      -DLZ4_HOME=#{Formula["lz4"].prefix}
+      -DPARQUET_BUILD_EXECUTABLES=ON
+      -DTHRIFT_HOME=#{Formula["thrift"].prefix}
+    ]
+
+    mkdir "build"
+    cd "build" do
+      system "cmake", "../cpp", *std_cmake_args, *args
+      system "make"
+      system "make", "install"
+    end
+  end
+
+  test do
+    (testpath/"test.cpp").write <<~EOS
+      #include "arrow/api.h"
+      int main(void) {
+        arrow::int64();
+        return 0;
+      }
+    EOS
+    system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", \
+      "-larrow", "-larrow_bundled_dependencies", "-o", "test"
+    system "./test"
+  end
+end

dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb

@@ -16,6 +16,7 @@
 # under the License.
 
 # https://github.com/autobrew/homebrew-core/blob/master/Formula/apache-arrow.rb
+
 class ApacheArrow < Formula
   desc "Columnar in-memory analytics layer designed to accelerate big data"
   homepage "https://arrow.apache.org/"
@@ -25,15 +26,15 @@ class ApacheArrow < Formula
 
   bottle do
     cellar :any
-    sha256 "a55211ba6f464681b7ca1b48defdad9cfbe1cf6fad8ff9ec875dc5a3c8f3c5ed" => :el_capitan_or_later
+    sha256 "9cd44700798638b5e3ee8774b3929f3fad815290d05572d1f39f01d6423eaad0" => :high_sierra
     root_url "https://autobrew.github.io/bottles"
   end
 
   # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew
   depends_on "boost" => :build
-  depends_on "brotli"
   depends_on "cmake" => :build
   depends_on "aws-sdk-cpp"
+  depends_on "brotli" 
   depends_on "lz4"
   depends_on "snappy"
   depends_on "thrift"
@@ -49,24 +50,20 @@ def install
       -DARROW_DATASET=ON
       -DARROW_FILESYSTEM=ON
       -DARROW_GCS=ON
-      -DARROW_HDFS=OFF
       -DARROW_JEMALLOC=ON
       -DARROW_JSON=ON
       -DARROW_MIMALLOC=ON
       -DARROW_PARQUET=ON
-      -DARROW_PYTHON=OFF
       -DARROW_S3=ON
-      -DARROW_USE_GLOG=OFF
       -DARROW_VERBOSE_THIRDPARTY_BUILD=ON
       -DARROW_WITH_BROTLI=ON
       -DARROW_WITH_BZ2=ON
       -DARROW_WITH_LZ4=ON
       -DARROW_WITH_SNAPPY=ON
       -DARROW_WITH_ZLIB=ON
       -DARROW_WITH_ZSTD=ON
-      -DCMAKE_UNITY_BUILD=OFF
-      -DPARQUET_BUILD_EXECUTABLES=ON
       -DLZ4_HOME=#{Formula["lz4"].prefix}
+      -DPARQUET_BUILD_EXECUTABLES=ON
       -DTHRIFT_HOME=#{Formula["thrift"].prefix}
     ]
 
@@ -86,7 +83,8 @@ def install
         return 0;
       }
     EOS
-    system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", "-larrow", "-o", "test"
+    system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", \
+      "-larrow", "-larrow_bundled_dependencies", "-o", "test"
     system "./test"
   end
 end

dev/tasks/macros.jinja

@@ -226,6 +226,21 @@ on:
       done
 {% endmacro %}
 
+{%- macro pin_brew_formulae(is_fork = false) -%}
+{#- This macro expects the cwd to be arrow/r -#}
+      # Copy all *brew formulae
+      cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow*.rb tools/
+
+      # Pin the git commit in the formula to match
+      cd tools
+      if [ "{{ is_fork }}" == "true" ]; then
+        sed -i.bak -E -e 's/apache\/arrow.git"$/{{ arrow.github_repo.split("/") | join("\/") }}.git", :revision => "'"{{ arrow.head }}"'"/' apache-arrow*.rb
+      else
+        sed -i.bak -E -e 's/arrow.git"$/arrow.git", :revision => "'"{{ arrow.head }}"'"/' apache-arrow*.rb
+      fi
+      rm -f apache-arrow*.rb.bak
+{% endmacro %}
+
 {%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date +%Y%m%d)\"\'' ) -%}
   - name: Modify version
     shell: bash
@@ -236,16 +251,8 @@ on:
         DESCRIPTION
       head DESCRIPTION
       rm -f DESCRIPTION.bak
-      cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb tools/apache-arrow.rb
 
-      # Pin the git commit in the formula to match
-      cd tools
-      if [ "{{ is_fork }}" == "true" ]; then
-        sed -i.bak -E -e 's/apache\/arrow.git"$/{{ arrow.github_repo.split("/") | join("\/") }}.git", :revision => "'"{{ arrow.head }}"'"/' apache-arrow.rb
-      else
-        sed -i.bak -E -e 's/arrow.git"$/arrow.git", :revision => "'"{{ arrow.head }}"'"/' apache-arrow.rb
-      fi
-      rm -f apache-arrow.rb.bak
+      {{ pin_brew_formulae(is_fork) }}
 {% endmacro %}
 
 {%- macro github_test_r_src_pkg() -%}
@@ -309,3 +316,11 @@ on:
       # Set envvar for later steps by appending to $GITHUB_ENV
       write(paste0("R_PROFILE_USER=", profile_path), file = Sys.getenv("GITHUB_ENV"), append = TRUE)
   {% endmacro %}
+
+{# Detect if we are using a fork or the upstream repo #}
+  {% set is_upstream_b = arrow.github_repo == 'apache/arrow' %}
+  {# use filter to cast to string and convert to lowercase to match yaml boolean #}
+  {% set is_fork = (not is_upstream_b)|lower %}
+
+{% set r_release = '4.2' %}
+{% set r_oldrel = '4.1' %}

dev/tasks/r/github.macos.autobrew.yml

@@ -16,44 +16,60 @@
 # under the License.
 
 {% import 'macros.jinja' as macros with context %}
+{% set is_fork = macros.is_fork %}
 
 {{ macros.github_header() }}
 
 jobs:
   autobrew:
-    name: "Autobrew"
-    runs-on: macOS-10.15
+    name: "Autobrew {{ "${{ matrix.platform }}" }}"
+    runs-on: {{ "${{ matrix.platform }}" }}
+    strategy:
+      fail-fast: false
+      matrix:
+          platform:
+            - macos-11
+            - macos-10.13 # self-hosted
+          r-version:
+            - {{ macros.r_release }}
+            - {{ macros.r_oldrel }}
     steps:
       {{ macros.github_checkout_arrow()|indent }}
-
       - name: Configure autobrew script
         run: |
+          # minio is pre-installed on the self-hosted 10.13 runer
+          if [ {{ '${{ matrix.platform }}' }} != macos-10.13  ]; then
+            # install minio for tests
+            brew install minio
+          fi
           cd arrow/r
-          # Put the formula inside r/ so that it's included in the package build
-          cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb tools/apache-arrow.rb
-          # Pin the current commit in the formula to test so that we're not always pulling from master
-          sed -i.bak -E -e 's@https://github.com/apache/arrow.git"$@{{ arrow.remote }}.git", :revision => "{{ arrow.head }}"@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak
-          # Sometimes crossbow gives a remote URL with .git and sometimes not. Make sure there's only one
-          sed -i.bak -E -e 's@.git.git@.git@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak
-          # Get minio for S3 testing
-          brew install minio
-      - uses: r-lib/actions/setup-r@v1
-      - name: Install dependencies
-        run: |
-          install.packages("remotes")
-          remotes::install_deps("arrow/r", dependencies = TRUE)
-          remotes::install_cran(c("rcmdcheck", "sys", "sessioninfo"))
-        shell: Rscript {0}
-      - name: Session info
+          {{ macros.pin_brew_formulae(is_fork)|indent }}
+      - uses: r-lib/actions/setup-r@v2
+        if: matrix.platform != 'macos-10.13'
+        with:
+          r-version: {{ '${{ matrix.r-version }}' }}
+          use-public-rspm: true
+      - name: Setup R
+        if: matrix.platform == 'macos-10.13'
         run: |
-          options(width = 100)
-          pkgs <- installed.packages()[, "Package"]
-          sessioninfo::session_info(pkgs, include_base = TRUE)
-        shell: Rscript {0}
+          # rig is a system utility that allows for switching
+          # between pre-installed R version on the self-hosted runners
+          rig default {{ '${{ matrix.r-version }}' }}
+          rig system setup-user-lib
+          rig system add-pak 
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          working-directory: 'arrow/r'
+          extra-packages: |
+            any::rcmdcheck
+            any::sys
+            any::readr
       - name: Check
         env:
           _R_CHECK_CRAN_INCOMING_: false
+          NOT_CRAN: true
           ARROW_USE_PKG_CONFIG: false
+          ARROW_R_DEV: true
         run: arrow/ci/scripts/r_test.sh arrow
       - name: Dump install logs
         run: cat arrow/r/check/arrow.Rcheck/00install.out

dev/tasks/r/github.packages.yml

@@ -21,12 +21,7 @@
 # crossbow submit --param custom_version=8.5.3 r-binary-packages
 # if the param is unset defaults to the usual Ymd naming scheme
 {% set package_version = custom_version|default("\\2.\'\"$(date +%Y%m%d)\"\'") %}
-# We need this as boolean and string
-{% set is_upstream_b = arrow.github_repo == 'apache/arrow' %}
-# use filter to cast to string and convert to lowercase to match yaml boolean
-{% set is_fork = (not is_upstream_b)|lower %}
-{% set is_upstream = is_upstream_b|lower %}
-
+{% set is_fork = macros.is_fork %}
 
 {{ macros.github_header() }}
 

r/tools/autobrew

@@ -19,8 +19,15 @@
 export HOMEBREW_NO_ANALYTICS=1
 export HOMEBREW_NO_AUTO_UPDATE=1
 
-# Official Homebrew no longer supports El-Capitan
-UPSTREAM_ORG="autobrew"
+if [[ ${OSTYPE:6} -ge 20 ]]; then
+  # We are on a modern enough macOS, we can use the real brew
+  UPSTREAM_ORG="homebrew"
+  PKG_BREW_NAME="$PKG_BREW_NAME-static"
+else
+  # Official Homebrew no longer supports El-Capitan
+  # so we need to use the forked autobrew version of brew that supports old macOSes
+  UPSTREAM_ORG="autobrew"
+fi
 
 if [ "$DISABLE_AUTOBREW" ]; then return 0; fi
 AUTOBREW=${TMPDIR-/tmp}
@@ -36,6 +43,13 @@ curl -fsSL https://github.com/$UPSTREAM_ORG/brew/tarball/master | tar xz --strip
 export HOMEBREW_CACHE="$AUTOBREW"
 LOCAL_FORMULA="tools/${PKG_BREW_NAME}.rb"
 if [ -f "$LOCAL_FORMULA" ]; then
+  if [[ ${OSTYPE:6} -ge 20 ]]; then
+    $BREW tap
+
+    # Tap https://github.com/autobrew/homebrew-cran so that we can get dependencies from there
+    $BREW tap autobrew/cran
+  fi
+
   # Use the local brew formula and install --HEAD
   $BREW deps -n "$LOCAL_FORMULA" 2>/dev/null
   BREW_DEPS=$($BREW deps -n "$LOCAL_FORMULA" 2>/dev/null)