ARROW-8207: [Packaging][wheel] Use LLVM 8 in manylinux2010 and manyli…

…nux2014 Closes #6714 from kou/wheel-llvm8 Authored-by: Sutou Kouhei <kou@clear-code.com> Signed-off-by: Antoine Pitrou <antoine@python.org>
apache · Mar 25, 2020 · 2be95a9 · 2be95a9
1 parent 1d439ac
commit 2be95a9
Show file tree

Hide file tree

Showing 9 changed files with 93 additions and 108 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -712,6 +712,8 @@ services:
       dockerfile: Dockerfile-x86_64_base
       cache_from:
         - ${REPO}:amd64-centos-5.11-python-manylinux1
+      args:
+        llvm: ${LLVM}
     shm_size: *shm-size
     environment:
       PYARROW_PARALLEL: 3
@@ -729,6 +731,8 @@ services:
       dockerfile: Dockerfile-x86_64_base_2010
       cache_from:
         - ${REPO}:amd64-centos-6.10-python-manylinux2010
+      args:
+        llvm: ${LLVM}
     shm_size: *shm-size
     environment:
       PYARROW_PARALLEL: 3
@@ -746,6 +750,8 @@ services:
       dockerfile: Dockerfile-x86_64_base_2014
       cache_from:
         - ${REPO}:amd64-centos-7.7-python-manylinux2014
+      args:
+        llvm: ${LLVM}
     shm_size: *shm-size
     environment:
       PYARROW_PARALLEL: 3

diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base
@@ -89,11 +89,9 @@ ENV GLOG_HOME /usr
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
 
+ARG llvm
 ADD scripts/build_llvm.sh /
-RUN /build_llvm.sh
-
-ADD scripts/build_clang.sh /
-RUN /build_clang.sh
+RUN /build_llvm.sh ${llvm}
 
 ADD scripts/build_rapidjson.sh /
 RUN /build_rapidjson.sh

diff --git a/python/manylinux1/Dockerfile-x86_64_ubuntu b/python/manylinux1/Dockerfile-x86_64_ubuntu
@@ -84,9 +84,6 @@ ADD scripts/build_glog.sh /
 RUN /build_glog.sh
 ENV GLOG_HOME /usr
 
-
+ARG llvm
 ADD scripts/build_llvm.sh /
-RUN /build_llvm.sh
-
-ADD scripts/build_clang.sh /
-RUN /build_clang.sh
+RUN /build_llvm.sh ${llvm}
diff --git a/python/manylinux1/scripts/build_clang.sh b/python/manylinux1/scripts/build_clang.sh
diff --git a/python/manylinux1/scripts/build_llvm.sh b/python/manylinux1/scripts/build_llvm.sh
@@ -1,4 +1,5 @@
 #!/bin/bash -ex
+#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,17 +17,28 @@
 # specific language governing permissions and limitations
 # under the License.
 
+LLVM_VERSION_MAJOR=$1
+
 source /multibuild/manylinux_utils.sh
 
-export LLVM_VERSION="8.0.1"
+detect_llvm_version() {
+  curl -sL https://api.github.com/repos/llvm/llvm-project/releases | \
+    grep tag_name | \
+    grep -o "llvmorg-${LLVM_VERSION_MAJOR}[^\"]*" | \
+    sed -e "s/^llvmorg-//g" | \
+    head -n 1
+}
+
+LLVM_VERSION=$(detect_llvm_version)
 curl -sL https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz -o llvm-${LLVM_VERSION}.src.tar.xz
 unxz llvm-${LLVM_VERSION}.src.tar.xz
 tar xf llvm-${LLVM_VERSION}.src.tar
 pushd llvm-${LLVM_VERSION}.src
 mkdir build
 pushd build
-cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \
+cmake \
     -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=/usr \
     -DLLVM_TARGETS_TO_BUILD=host \
     -DLLVM_INCLUDE_DOCS=OFF \
     -DLLVM_INCLUDE_EXAMPLES=OFF \
@@ -45,3 +57,26 @@ ninja install
 popd
 popd
 rm -rf llvm-${LLVM_VERSION}.src.tar.xz llvm-${LLVM_VERSION}.src.tar llvm-${LLVM_VERSION}.src
+
+
+# clang is only used to precompile Gandiva bitcode
+curl -sL https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/cfe-${LLVM_VERSION}.src.tar.xz -o cfe-${LLVM_VERSION}.src.tar.xz
+unxz cfe-${LLVM_VERSION}.src.tar.xz
+tar xf cfe-${LLVM_VERSION}.src.tar
+pushd cfe-${LLVM_VERSION}.src
+mkdir build
+pushd build
+cmake \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=/usr \
+    -DCLANG_INCLUDE_TESTS=OFF \
+    -DCLANG_INCLUDE_DOCS=OFF \
+    -DLLVM_INCLUDE_TESTS=OFF \
+    -DLLVM_INCLUDE_DOCS=OFF \
+    -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON \
+    -GNinja \
+    ..
+ninja -w dupbuild=warn install # both clang and llvm builds generate llvm-config file
+popd
+popd
+rm -rf cfe-${LLVM_VERSION}.src.tar.xz cfe-${LLVM_VERSION}.src.tar cfe-${LLVM_VERSION}.src
diff --git a/python/manylinux201x/Dockerfile-x86_64_base_2010 b/python/manylinux201x/Dockerfile-x86_64_base_2010
@@ -34,12 +34,9 @@ RUN /build_zlib.sh
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
 
+ARG llvm
 ADD scripts/build_llvm.sh /
-RUN /build_llvm.sh
-
-# clang is only used to precompile Gandiva bitcode
-ADD scripts/build_clang.sh /
-RUN /build_clang.sh
+RUN /build_llvm.sh ${llvm}
 
 ADD scripts/build_openssl.sh /
 RUN /build_openssl.sh

diff --git a/python/manylinux201x/Dockerfile-x86_64_base_2014 b/python/manylinux201x/Dockerfile-x86_64_base_2014
@@ -34,12 +34,9 @@ RUN /build_zlib.sh
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
 
+ARG llvm
 ADD scripts/build_llvm.sh /
-RUN /build_llvm.sh
-
-# clang is only used to precompile Gandiva bitcode
-ADD scripts/build_clang.sh /
-RUN /build_clang.sh
+RUN /build_llvm.sh ${llvm}
 
 ADD scripts/build_openssl.sh /
 RUN /build_openssl.sh

diff --git a/python/manylinux201x/scripts/build_clang.sh b/python/manylinux201x/scripts/build_clang.sh
diff --git a/python/manylinux201x/scripts/build_llvm.sh b/python/manylinux201x/scripts/build_llvm.sh
@@ -16,19 +16,29 @@
 # specific language governing permissions and limitations
 # under the License.
 
+LLVM_VERSION_MAJOR=$1
+
 source /multibuild/manylinux_utils.sh
 
-LLVM_VERSION="7.0.1"
-PREFIX="/usr/local"
+detect_llvm_version() {
+  curl -sL https://api.github.com/repos/llvm/llvm-project/releases | \
+    grep tag_name | \
+    grep -o "llvmorg-${LLVM_VERSION_MAJOR}[^\"]*" | \
+    sed -e "s/^llvmorg-//g" | \
+    head -n 1
+}
+
+LLVM_VERSION=$(detect_llvm_version)
 
-curl -sL https://releases.llvm.org/${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz -o llvm-${LLVM_VERSION}.src.tar.xz
+curl -sL https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-${LLVM_VERSION}.src.tar.xz -o llvm-${LLVM_VERSION}.src.tar.xz
 unxz llvm-${LLVM_VERSION}.src.tar.xz
 tar xf llvm-${LLVM_VERSION}.src.tar
 pushd llvm-${LLVM_VERSION}.src
 mkdir build
 pushd build
-cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \
+cmake \
     -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=/usr \
     -DLLVM_TARGETS_TO_BUILD=host \
     -DLLVM_INCLUDE_DOCS=OFF \
     -DLLVM_INCLUDE_EXAMPLES=OFF \
@@ -39,10 +49,38 @@ cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \
     -DLLVM_ENABLE_RTTI=ON \
     -DLLVM_ENABLE_OCAMLDOC=OFF \
     -DLLVM_USE_INTEL_JITEVENTS=ON \
+    -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON \
     -DPYTHON_EXECUTABLE="$(cpython_path 3.6 16)/bin/python" \
     -GNinja \
     ..
 ninja install
 popd
 popd
 rm -rf llvm-${LLVM_VERSION}.src.tar.xz llvm-${LLVM_VERSION}.src.tar llvm-${LLVM_VERSION}.src
+
+
+# clang is only used to precompile Gandiva bitcode
+curl -sL https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/cfe-${LLVM_VERSION}.src.tar.xz -o cfe-${LLVM_VERSION}.src.tar.xz
+unxz cfe-${LLVM_VERSION}.src.tar.xz
+tar xf cfe-${LLVM_VERSION}.src.tar
+pushd cfe-${LLVM_VERSION}.src
+mkdir build
+pushd build
+cmake \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=/usr \
+    -DCLANG_INCLUDE_TESTS=OFF \
+    -DCLANG_INCLUDE_DOCS=OFF \
+    -DCLANG_ENABLE_ARCMT=OFF \
+    -DCLANG_ENABLE_STATIC_ANALYZER=OFF \
+    -DLLVM_ENABLE_RTTI=ON \
+    -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON \
+    -DLLVM_INCLUDE_TESTS=OFF \
+    -DLLVM_INCLUDE_DOCS=OFF \
+    -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON \
+    -GNinja \
+    ..
+ninja -w dupbuild=warn install # both clang and llvm builds generate llvm-config file
+popd
+popd
+rm -rf cfe-${LLVM_VERSION}.src.tar.xz cfe-${LLVM_VERSION}.src.tar cfe-${LLVM_VERSION}.src