ARROW-7987: [CI][R] Fix for verbose nightly builds

Verbose builds trigger a compiler flag check to fail, so make sure that we don't run that check if building verbosely. And make sure we can test both ways: it's important that at least one nightly check the flags, but verbosity is otherwise more useful for debugging build failures.

Closes #6518 from nealrichardson/fix-debug-ci and squashes the following commits:

5653f1f <Neal Richardson> Quote
dfff471 <Neal Richardson> Export all the things
49f1a4a <Neal Richardson> See if this was causing our problems
7dc5af3 <Neal Richardson> oops again
c3f26a5 <Neal Richardson> I will regret this
1ec0691 <Neal Richardson> Try setting _R_CHECK_COMPILATION_FLAGS_KNOWN_ because --as-cran overwrites the other
9faef23 <Neal Richardson> Oops
d15f04a <Neal Richardson> Explicit
f78b4ec <Neal Richardson> If verbose don't check flags, and set nightlies to test some builds each way

Authored-by: Neal Richardson <neal.p.richardson@gmail.com>
Signed-off-by: Neal Richardson <neal.p.richardson@gmail.com>

.env

@@ -38,6 +38,7 @@ HDFS=2.9.2
 SPARK=master
 DOTNET=2.1
 R=3.6
+ARROW_R_DEV=TRUE
 # These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
 R_ORG=rhub
 R_IMAGE=ubuntu-gcc-release

ci/scripts/r_test.sh

@@ -24,15 +24,18 @@ source_dir=${1}/r
 
 pushd ${source_dir}
 
+printenv
+
 if [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then
   export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
   export R_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
 fi
-if [ "$ARROW_R_CXXFLAGS" != "" ]; then
-  export _R_CHECK_COMPILATION_FLAGS_=FALSE
+export _R_CHECK_COMPILATION_FLAGS_KNOWN_=${ARROW_R_CXXFLAGS}
+if [ "$ARROW_R_DEV" = "TRUE" ]; then
+  # These are used in the Arrow C++ build and are not a problem
+  export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_} -Wno-attributes -msse4.2"
 fi
 export TEST_R_WITH_ARROW=TRUE
-export ARROW_R_DEV=TRUE # For log verbosity
 export _R_CHECK_TESTS_NLINES_=0
 export _R_CHECK_CRAN_INCOMING_REMOTE_=FALSE
 export _R_CHECK_LIMIT_CORES_=FALSE
@@ -41,7 +44,9 @@ export VERSION=$(grep ^Version DESCRIPTION | sed s/Version:\ //)
 # Make sure we aren't writing to the home dir (CRAN _hates_ this but there is no official check)
 BEFORE=$(ls -alh ~/)
 
-${R_BIN} -e "rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check')"
+# Conditionally run --as-cran because crossbow jobs aren't using _R_CHECK_COMPILATION_FLAGS_KNOWN_
+# (maybe an R version thing, needs 3.6.2?)
+${R_BIN} -e "rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', if (!identical(Sys.getenv('ARROW_R_DEV'), 'TRUE')) '--as-cran', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check')"
 
 AFTER=$(ls -alh ~/)
 if [ "$BEFORE" != "$AFTER" ]; then

dev/tasks/r/azure.linux.yml

@@ -46,6 +46,8 @@ jobs:
         R_ORG={{ r_org }}
         R_IMAGE={{ r_image }}
         R_TAG={{ r_tag }}
+        # we have to export this (right?) because we need it in the build env
+        export ARROW_R_DEV={{ verbose }}
         docker-compose run r
       displayName: Docker run
 

dev/tasks/tasks.yml

@@ -1836,6 +1836,7 @@ tasks:
       r_org: rhub
       r_image: debian-gcc-devel
       r_tag: latest
+      verbose: "TRUE"
 
   test-r-rhub-ubuntu-gcc-release:
     ci: azure
@@ -1845,6 +1846,7 @@ tasks:
       r_org: rhub
       r_image: ubuntu-gcc-release
       r_tag: latest
+      verbose: "TRUE"
 
   test-r-rstudio-r-base-3.6-bionic:
     ci: azure
@@ -1854,6 +1856,8 @@ tasks:
       r_org: rstudio
       r_image: r-base
       r_tag: 3.6-bionic
+      # Turn off verbosity on some tests in order to check compiler flags
+      verbose: "FALSE"
 
   test-r-rstudio-r-base-3.6-centos6:
     ci: azure
@@ -1863,6 +1867,7 @@ tasks:
       r_org: rstudio
       r_image: r-base
       r_tag: 3.6-centos6
+      verbose: "FALSE"
 
   test-r-rstudio-r-base-3.6-opensuse15:
     ci: azure
@@ -1872,6 +1877,7 @@ tasks:
       r_org: rstudio
       r_image: r-base
       r_tag: 3.6-opensuse15
+      verbose: "TRUE"
 
   test-r-rstudio-r-base-3.6-opensuse42:
     ci: azure
@@ -1881,6 +1887,7 @@ tasks:
       r_org: rstudio
       r_image: r-base
       r_tag: 3.6-opensuse24
+      verbose: "TRUE"
 
   test-ubuntu-18.04-r-3.6:
     ci: circle

docker-compose.yml

@@ -795,6 +795,7 @@ services:
       - .:/arrow:delegated
     command: >
       /bin/bash -c "
+        export ARROW_R_DEV=${ARROW_R_DEV} &&
         /arrow/ci/scripts/r_test.sh /arrow"
 
   ubuntu-r-sanitizer: