ARROW-6841: [C++] Migrate to LLVM 8

.env

@@ -24,8 +24,10 @@ ARCH=amd64
 CUDA=9.1
 DEBIAN=10
 UBUNTU=18.04
-FEDORA=29
+FEDORA=30
 PYTHON=3.6
+LLVM=8
+CLANG_TOOLS=8
 RUST=nightly-2019-11-14
 GO=1.12
 NODE=11

.github/workflows/cpp_cron.yml

@@ -78,7 +78,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        fedora: [29]
+        fedora: [30]
     env:
       FEDORA: ${{ matrix.fedora }}
     steps:

.github/workflows/python_cron.yml

@@ -75,7 +75,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        fedora: [29]
+        fedora: [30]
     env:
       FEDORA: ${{ matrix.fedora }}
     steps:

ci/conda_env_gandiva.yml

@@ -15,6 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-clangdev=7
-llvmdev=7
+clangdev=8
+llvmdev=8
 re2

ci/docker/debian-10-cpp.dockerfile

@@ -18,14 +18,19 @@
 ARG arch=amd64
 FROM ${arch}/debian:10
 
-# install build essentials
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get update -y -q && \
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN \
+  echo "deb http://deb.debian.org/debian buster-backports main" > \
+    /etc/apt/sources.list.d/backports.list
+
+ARG llvm
+RUN apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         autoconf \
         ca-certificates \
         ccache \
-        clang-7 \
+        clang-${llvm} \
         cmake \
         g++ \
         gcc \
@@ -46,7 +51,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         libssl-dev \
         libthrift-dev \
         libzstd-dev \
-        llvm-7-dev \
+        llvm-${llvm}-dev \
         make \
         ninja-build \
         pkg-config \

ci/docker/fedora-29-cpp.dockerfile → ci/docker/fedora-30-cpp.dockerfile

@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch
-FROM ${arch}/fedora:29
+FROM ${arch}/fedora:30
 
 # install dependencies
 RUN dnf update -y && \
@@ -55,7 +55,7 @@ RUN dnf update -y && \
 
 # * c-ares cmake config is not installed on Fedora but gRPC needs it
 #   when built via ExternalProject: https://bugzilla.redhat.com/show_bug.cgi?id=1687844
-# * protobuf libraries in Fedora 29 are too old for gRPC
+# * protobuf libraries in Fedora 30 are too old for gRPC
 ENV ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \
     ARROW_DATASET=ON \

ci/docker/linux-apt-docs.dockerfile

@@ -46,7 +46,8 @@ RUN apt-get update -y && \
         pandoc \
         r-base=${r}* \
         rsync \
-        ruby-dev && \
+        ruby-dev \
+        wget && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

ci/docker/linux-apt-jni.dockerfile

@@ -18,13 +18,28 @@
 ARG base
 FROM ${base}
 
-# install build essentials
-RUN export DEBIAN_FRONTEND=noninteractive && \
+# pipefail is enabled for proper error detection in the `wget | apt-key add`
+# step
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ARG llvm
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+      apt-transport-https \
+      lsb-release \
+      software-properties-common \
+      wget && \
+    code_name=$(lsb_release --codename --short) && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    apt-add-repository -y \
+      "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${llvm} main" && \
     apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         ca-certificates \
         ccache \
-        clang-7 \
+        clang-${llvm} \
         cmake \
         git \
         g++ \
@@ -37,15 +52,14 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         libre2-dev \
         libsnappy-dev \
         libssl-dev \
-        llvm-7-dev \
+        llvm-${llvm}-dev \
         make \
         ninja-build \
         pkg-config \
         protobuf-compiler \
         rapidjson-dev \
         tzdata \
-        zlib1g-dev \
-        wget && \
+        zlib1g-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

ci/docker/linux-apt-lint.dockerfile

@@ -19,15 +19,17 @@ ARG base
 FROM hadolint/hadolint:v1.17.2 AS hadolint
 FROM ${base}
 
+ARG clang_tools
 RUN apt-get update && \
     apt-get install -y -q \
-        clang-7 \
-        clang-format-7 \
-        clang-tidy-7 \
-        clang-tools-7 \
+        clang-${clang_tools} \
+        clang-format-${clang_tools} \
+        clang-tidy-${clang_tools} \
+        clang-tools-${clang_tools} \
         cmake \
         curl \
-        libclang-7-dev \
+        libclang-${clang_tools}-dev \
+        llvm-${clang_tools}-dev \
         openjdk-11-jdk-headless \
         python3 \
         python3-dev \
@@ -41,7 +43,7 @@ COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
 
 # IWYU
 COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/
-RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local 7
+RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools}
 
 # Rust linter
 ARG rust=nightly-2019-09-25

ci/docker/ubuntu-16.04-cpp.dockerfile

@@ -21,19 +21,21 @@ FROM ${arch}/ubuntu:16.04
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 ENV DEBIAN_FRONTEND noninteractive
+
+ARG llvm
 RUN apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         apt-transport-https \
         software-properties-common \
         wget && \
     wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-    apt-add-repository -y "deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main" && \
+    apt-add-repository -y "deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-${llvm} main" && \
     apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
         autoconf \
         ca-certificates \
         ccache \
-        clang-7 \
+        clang-${llvm} \
         cmake \
         g++ \
         gcc \
@@ -46,7 +48,7 @@ RUN apt-get update -y -q && \
         liblz4-dev \
         libre2-dev \
         libssl-dev \
-        llvm-7-dev \
+        llvm-${llvm}-dev \
         make \
         ninja-build \
         pkg-config \

ci/docker/ubuntu-18.04-cpp.dockerfile

@@ -24,29 +24,19 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update -y -q && \
-    apt-get install -y -q --no-install-recommends \
-        wget \
-        software-properties-common \
-        gpg-agent && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists*
-
-# Installs LLVM toolchain, for gandiva and testing other compilers
+# Installs LLVM toolchain, for Gandiva and testing other compilers
 #
 # Note that this is installed before the base packages to improve iteration
-# while debugging package list with docker build due to slow download speed of
-# llvm compared to ubuntu apt mirrors.
-ARG llvm_version=7
-ARG llvm_apt_url="http://apt.llvm.org/bionic/"
-ARG llvm_apt_arch="llvm-toolchain-bionic-${llvm_version}"
-RUN wget -q -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-    apt-add-repository -y --update "deb ${llvm_apt_url} ${llvm_apt_arch} main" && \
+# while debugging package list with docker build.
+ARG clang_tools
+ARG llvm
+RUN apt-get update -y -q && \
     apt-get install -y -q --no-install-recommends \
-        clang-${llvm_version} \
-        clang-format-${llvm_version} \
-        clang-tidy-${llvm_version} \
-        llvm-${llvm_version}-dev && \
+        clang-${clang_tools} \
+        clang-${llvm} \
+        clang-format-${clang_tools} \
+        clang-tidy-${clang_tools} \
+        llvm-${llvm}-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
 

ci/scripts/install_iwyu.sh

@@ -20,19 +20,19 @@ set -eu
 
 source_dir=${1:-/tmp/iwyu}
 install_prefix=${2:-/usr/local}
-llvm_major=${3:-7}
+clang_tools_version=${3:-7}
 
-git clone --single-branch --branch "clang_${llvm_major}.0" \
+git clone --single-branch --branch "clang_${clang_tools_version}.0" \
     https://github.com/include-what-you-use/include-what-you-use.git ${source_dir}
 
 mkdir -p ${source_dir}/build
 pushd ${source_dir}/build
 
 # Build IWYU for current Clang
-export CC=clang-${llvm_major}
-export CXX=clang++-${llvm_major}
+export CC=clang-${clang_tools_version}
+export CXX=clang++-${clang_tools_version}
 
-cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${llvm_major} \
+cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${clang_tools_version} \
       -DCMAKE_INSTALL_PREFIX=${install_prefix} \
       ${source_dir}
 make -j4

cpp/Brewfile

@@ -25,7 +25,7 @@ brew "flatbuffers"
 brew "git"
 brew "glog"
 brew "grpc"
-brew "llvm@7"
+brew "llvm@8"
 brew "lz4"
 brew "ninja"
 brew "numpy"

cpp/CMakeLists.txt

@@ -92,40 +92,48 @@ set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(ARROW_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
 set(ARROW_DOC_DIR "share/doc/${PROJECT_NAME}")
 
-set(ARROW_LLVM_VERSION "7.0")
-# TODO: Remove this when we drop LLVM 7 support.
-set(ARROW_LLVM_VERSION_FALLBACK "7.1")
-string(
-  REGEX
-  REPLACE "^([0-9]+)\\.[0-9]+" "\\1" ARROW_LLVM_MAJOR_VERSION "${ARROW_LLVM_VERSION}")
-string(
-  REGEX
-  REPLACE "^[0-9]+\\.([0-9]+)" "\\1" ARROW_LLVM_MINOR_VERSION "${ARROW_LLVM_VERSION}")
+set(ARROW_LLVM_VERSIONS "8" "7")
+list(GET ARROW_LLVM_VERSIONS 0 ARROW_LLVM_VERSION_PRIMARY)
+string(REGEX
+       REPLACE "^([0-9]+)(\\..+)?" "\\1" ARROW_LLVM_VERSION_PRIMARY_MAJOR
+               "${ARROW_LLVM_VERSION_PRIMARY}")
+
+file(READ ${CMAKE_CURRENT_SOURCE_DIR}/../.env ARROW_ENV)
+string(REGEX MATCH "CLANG_TOOLS=[^\n]+" ARROW_ENV_CLANG_TOOLS_VERSION "${ARROW_ENV}")
+string(REGEX
+       REPLACE "^CLANG_TOOLS=" "" ARROW_CLANG_TOOLS_VERSION
+               "${ARROW_ENV_CLANG_TOOLS_VERSION}")
+string(REGEX
+       REPLACE "^([0-9]+)(\\..+)?" "\\1" ARROW_CLANG_TOOLS_VERSION_MAJOR
+               "${ARROW_CLANG_TOOLS_VERSION}")
 
 if(APPLE)
   find_program(BREW_BIN brew)
   if(BREW_BIN)
-    execute_process(COMMAND ${BREW_BIN} --prefix "llvm@${ARROW_LLVM_MAJOR_VERSION}"
+    execute_process(COMMAND ${BREW_BIN} --prefix
+                            "llvm@${ARROW_LLVM_VERSION_PRIMARY_MAJOR}"
                     OUTPUT_VARIABLE LLVM_BREW_PREFIX
                     OUTPUT_STRIP_TRAILING_WHITESPACE)
     if(NOT LLVM_BREW_PREFIX)
       execute_process(COMMAND ${BREW_BIN} --prefix llvm
                       OUTPUT_VARIABLE LLVM_BREW_PREFIX
                       OUTPUT_STRIP_TRAILING_WHITESPACE)
     endif()
+
+    execute_process(COMMAND ${BREW_BIN} --prefix "llvm@${ARROW_CLANG_TOOLS_MAJOR_VERSION}"
+                    OUTPUT_VARIABLE CLANG_TOOLS_BREW_PREFIX
+                    OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if(NOT CLANG_TOOLS_BREW_PREFIX)
+      execute_process(COMMAND ${BREW_BIN} --prefix llvm
+                      OUTPUT_VARIABLE CLANG_TOOLS_BREW_PREFIX
+                      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    endif()
   endif()
 endif()
 
 find_package(ClangTools)
-if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
-  # Generate a Clang compile_commands.json "compilation database" file for use
-  # with various development tools, such as Vim's YouCompleteMe plugin.
-  # See http://clang.llvm.org/docs/JSONCompilationDatabase.html
-  set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
-endif()
-
 find_package(InferTools)
-if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR INFER_FOUND)
+if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND OR INFER_FOUND)
   # Generate a Clang compile_commands.json "compilation database" file for use
   # with various development tools, such as Vim's YouCompleteMe plugin.
   # See http://clang.llvm.org/docs/JSONCompilationDatabase.html