ARROW-14107: [R][CI] Parallelize Windows CI jobs

.github/workflows/r.yml

@@ -170,33 +170,26 @@ jobs:
         continue-on-error: true
         run: archery docker push r
 
-  windows:
-    name: AMD64 Windows RTools ${{ matrix.rtools }}
+  windows-cpp:
+    name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }}
     runs-on: windows-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        rtools: [35, 40]
-    env:
-      ARROW_R_CXXFLAGS: "-Werror"
-      _R_CHECK_TESTS_NLINES_: 0
+        config:
+        - { rtools: 35, arch: 'mingw32' }
+        - { rtools: 35, arch: 'mingw64' }
+        - { rtools: 40, arch: 'mingw32' }
+        - { rtools: 40, arch: 'mingw64' }
+        - { rtools: 40, arch: 'ucrt64' }
     steps:
       - run: git config --global core.autocrlf false
       - name: Checkout Arrow
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Make R tests verbose
-        # If you get a segfault/mysterious test Execution halted,
-        # make this `true` to see where it dies.
-        if: false
-        shell: cmd
-        run: |
-          cd r/tests
-          sed -i.bak -E -e 's/"arrow"/"arrow", reporter = "location"/' testthat.R
-          rm -f testthat.R.bak
       - name: Setup ccache
         shell: bash
         run: |
@@ -211,30 +204,134 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ccache
-          key: r-${{ matrix.rtools }}-ccache-mingw-${{ hashFiles('cpp/**') }}
-          restore-keys: r-${{ matrix.rtools }}-ccache-mingw-
+          key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ hashFiles('cpp/**') }}
+          restore-keys: r-${{ matrix.config.rtools }}-ccache-mingw-
       # We use the makepkg-mingw setup that is included in rtools40 even when
       # we use the rtools35 compilers, so we always install R 4.0/Rtools40
       - uses: r-lib/actions/setup-r@master
         with:
-          rtools-version: 40
           r-version: "4.1"
+          rtools-version: 40
           Ncpus: 2
       - uses: r-lib/actions/setup-r@master
-        if: ${{ matrix.rtools == 35 }}
+        if: ${{ matrix.config.rtools == 35 }}
         with:
           rtools-version: 35
           r-version: "3.6"
           Ncpus: 2
       - name: Build Arrow C++
         shell: bash
         env:
-          RTOOLS_VERSION: ${{ matrix.rtools }}
+          RTOOLS_VERSION: ${{ matrix.config.rtools }}
+          MINGW_ARCH: ${{ matrix.config.arch }}
         run: ci/scripts/r_windows_build.sh
+      - name: Rename libarrow.zip
+        # So that they're unique when multiple are downloaded in the next step
+        shell: bash
+        run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
       - uses: actions/upload-artifact@v1
         with:
-          name: Rtools ${{ matrix.rtools }} Arrow C++
-          path: libarrow.zip
+          name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
+          path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
+      # We can remove this when we drop support for Rtools 3.5.
+      - name: Ensure using system tar in actions/cache
+        run: |
+          Write-Output "${Env:windir}\System32" | `
+            Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
+  windows-r:
+    needs: [windows-cpp]
+    name: AMD64 Windows R RTools ${{ matrix.rtools }}
+    runs-on: windows-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        rtools: [35, 40]
+    env:
+      ARROW_R_CXXFLAGS: "-Werror"
+      _R_CHECK_TESTS_NLINES_: 0
+    steps:
+      - run: git config --global core.autocrlf false
+      - name: Checkout Arrow
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - run: mkdir r/windows
+      - name: Download artifacts
+        if: ${{ matrix.rtools == 35 }}
+        uses: actions/download-artifact@v2
+        with:
+          name: libarrow-rtools35-mingw32.zip
+          path: r/windows
+      - name: Download artifacts
+        if: ${{ matrix.rtools == 35 }}
+        uses: actions/download-artifact@v2
+        with:
+          name: libarrow-rtools35-mingw64.zip
+          path: r/windows
+      - name: Download artifacts
+        if: ${{ matrix.rtools == 40 }}
+        uses: actions/download-artifact@v2
+        with:
+          name: libarrow-rtools40-mingw32.zip
+          path: r/windows
+      - name: Download artifacts
+        if: ${{ matrix.rtools == 40 }}
+        uses: actions/download-artifact@v2
+        with:
+          name: libarrow-rtools40-mingw64.zip
+          path: r/windows
+      - name: Download artifacts
+        if: ${{ matrix.rtools == 40 }}
+        uses: actions/download-artifact@v2
+        with:
+          name: libarrow-rtools40-ucrt64.zip
+          path: r/windows
+      - name: Unzip and rezip libarrows
+        shell: bash
+        run: |
+          cd r/windows
+          ls *.zip | xargs -n 1 unzip -uo
+          rm -rf *.zip
+      - name: Setup ccache
+        shell: bash
+        run: |
+          ci/scripts/ccache_setup.sh
+          echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV
+      # We must enable actions/cache before r-lib/actions/setup-r to ensure
+      # using system tar instead of tar provided by Rtools.
+      # We can use tar provided by Rtools when we drop support for Rtools 3.5.
+      # Because Rtools 4.0 or later has zstd. actions/cache requires zstd
+      # when tar is GNU tar.
+      - name: Cache ccache
+        uses: actions/cache@v2
+        with:
+          path: ccache
+          key: r-${{ matrix.rtools }}-ccache-mingw-${{ hashFiles('cpp/**') }}
+          restore-keys: r-${{ matrix.rtools }}-ccache-mingw-
+      - uses: r-lib/actions/setup-r@master
+        if: ${{ matrix.rtools == 40 }}
+        with:
+          r-version: "4.1"
+          rtools-version: 40
+          Ncpus: 2
+      - uses: r-lib/actions/setup-r@master
+        if: ${{ matrix.rtools == 35 }}
+        with:
+          rtools-version: 35
+          r-version: "3.6"
+          Ncpus: 2
+      - name: Make R tests verbose
+        # If you get a segfault/mysterious test Execution halted,
+        # make this `true` to see where it dies.
+        if: false
+        shell: cmd
+        run: |
+          cd r/tests
+          sed -i.bak -E -e 's/"arrow"/"arrow", reporter = "location"/' testthat.R
+          rm -f testthat.R.bak
       - name: Install R package dependencies
         shell: Rscript {0}
         run: |
@@ -244,13 +341,19 @@ jobs:
       - name: Check
         shell: Rscript {0}
         run: |
+          # Because we do R CMD build and r/windows is in .Rbuildignore,
+          # assemble the libarrow.zip file and pass it as an env var
+          setwd("r/windows")
+          zip("libarrow.zip", ".")
+          setwd("..")
+
           Sys.setenv(
-            RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "libarrow.zip"),
+            RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"),
             MAKEFLAGS = paste0("-j", parallel::detectCores()),
             ARROW_R_DEV = TRUE,
             "_R_CHECK_FORCE_SUGGESTS_" = FALSE
           )
-          rcmdcheck::rcmdcheck("r",
+          rcmdcheck::rcmdcheck(".",
             build_args = '--no-build-vignettes',
             args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'),
             error_on = 'warning',
@@ -259,7 +362,7 @@ jobs:
           )
       - name: Dump install logs
         shell: cmd
-        run: cat check/arrow.Rcheck/00install.out
+        run: cat r/check/arrow.Rcheck/00install.out
         if: always()
       # We can remove this when we drop support for Rtools 3.5.
       - name: Ensure using system tar in actions/cache

ci/scripts/r_windows_build.sh

@@ -29,6 +29,9 @@ if [ "$RTOOLS_VERSION" = "35" ]; then
   pacman --noconfirm -Syy
   # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5)
   RWINLIB_LIB_DIR="lib-4.9.3"
+  # This is the default (will build for each arch) but we can set up CI to
+  # do these in parallel
+  : ${MINGW_ARCH:="mingw32 mingw64"}
 else
   # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN
   # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf
@@ -38,9 +41,11 @@ else
 
   pacman --noconfirm -Syy
   RWINLIB_LIB_DIR="lib"
-  export MINGW_ARCH="mingw32 mingw64 ucrt64"
+  : ${MINGW_ARCH:="mingw32 mingw64 ucrt64"}
 fi
 
+export MINGW_ARCH
+
 cp $ARROW_HOME/ci/scripts/PKGBUILD .
 printenv
 makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild
@@ -57,41 +62,46 @@ cd build
 # This may vary by system/CI provider
 MSYS_LIB_DIR="/c/rtools40"
 
-ls $MSYS_LIB_DIR/mingw64/lib/
-ls $MSYS_LIB_DIR/mingw32/lib/
-
-# Untar the three builds we made
+# Untar the builds we made
 ls *.xz | xargs -n 1 tar -xJf
 mkdir -p $DST_DIR
 # Grab the headers from one, either one is fine
 # (if we're building twice to combine old and new toolchains, this may already exist)
 if [ ! -d $DST_DIR/include ]; then
-  mv mingw64/include $DST_DIR
+  mv $(echo $MINGW_ARCH | cut -d ' ' -f 1)/include $DST_DIR
 fi
 
-# Make the rest of the directory structure
-# lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5)
-mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/x64
-mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/i386
-# lib is for the new gcc 8 toolchain (Rtools 4.0)
-mkdir -p $DST_DIR/lib/x64
-mkdir -p $DST_DIR/lib/i386
-
-# Move the 64-bit versions of libarrow into the expected location
-mv mingw64/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/x64
-# Same for the 32-bit versions
-mv mingw32/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/i386
+# mingw64 -> x64
+# mingw32 -> i386
+# ucrt64 -> x64-ucrt
 
-# These may be from https://dl.bintray.com/rtools/backports/
-cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x64
-cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386
+if [ -d mingw64/lib/ ]; then
+  ls $MSYS_LIB_DIR/mingw64/lib/
+  # Make the rest of the directory structure
+  # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5)
+  mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/x64
+  # lib is for the new gcc 8 toolchain (Rtools 4.0)
+  mkdir -p $DST_DIR/lib/x64
+  # Move the 64-bit versions of libarrow into the expected location
+  mv mingw64/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/x64
+  # These may be from https://dl.bintray.com/rtools/backports/
+  cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x64
+  # These are from https://dl.bintray.com/rtools/mingw{32,64}/
+  cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/x64
+fi
 
-# These are from https://dl.bintray.com/rtools/mingw{32,64}/
-cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/x64
-cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/i386
+# Same for the 32-bit versions
+if [ -d mingw32/lib/ ]; then
+  ls $MSYS_LIB_DIR/mingw32/lib/
+  mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/i386
+  mkdir -p $DST_DIR/lib/i386
+  mv mingw32/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/i386
+  cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386
+  cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/i386
+fi
 
 # Do the same also for ucrt64
-if [ "$RTOOLS_VERSION" != "35" ]; then
+if [ -d ucrt64/lib/ ]; then
   ls $MSYS_LIB_DIR/ucrt64/lib/
   mkdir -p $DST_DIR/lib/x64-ucrt
   mv ucrt64/lib/*.a $DST_DIR/lib/x64-ucrt