ARROW-7869: [Python] Remove boost::system and boost::filesystem from …

…Python wheels

Those dependencies are only required by C++ unit tests.

Closes #6441 from pitrou/ARROW-7869-prune-boost-in-wheels and squashes the following commits:

ebae1df <Antoine Pitrou> Reformat CMake files
7abc484 <Antoine Pitrou> - no need to bundle boost::regex except for manylinux1 - avoid pointless download of huge Tensorflow wheel
c3d4efb <Antoine Pitrou> ARROW-7869:  Remove boost::system and boost::filesystem from Python wheels

Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Wes McKinney <wesm+git@apache.org>

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -639,11 +639,10 @@ set(Boost_ADDITIONAL_VERSIONS
     "1.60.0"
     "1.60")
 
-if(ARROW_BUILD_INTEGRATION
-   OR ARROW_BUILD_TESTS
-   OR ARROW_HDFS
-   OR ARROW_GANDIVA
-   OR ARROW_PARQUET)
+# - Parquet requires boost only with gcc 4.8 (because of missing std::regex).
+# - Gandiva has a compile-time (header-only) dependency on Boost, not runtime.
+# - Tests needs Boost at runtime.
+if(ARROW_BUILD_INTEGRATION OR ARROW_BUILD_TESTS OR ARROW_GANDIVA OR ARROW_PARQUET)
   set(ARROW_BOOST_REQUIRED TRUE)
 else()
   set(ARROW_BOOST_REQUIRED FALSE)

dev/tasks/tasks.yml

@@ -22,11 +22,9 @@ groups:
   ############################# Packaging tasks ###############################
 
   conda:
-    - conda-linux-gcc-py27
     - conda-linux-gcc-py36
     - conda-linux-gcc-py37
     - conda-linux-gcc-py38
-    - conda-osx-clang-py27
     - conda-osx-clang-py36
     - conda-osx-clang-py37
     - conda-osx-clang-py38
@@ -35,14 +33,10 @@ groups:
     - conda-win-vs2015-py38
 
   wheel:
-    - wheel-manylinux1-cp27m
-    - wheel-manylinux1-cp27mu
     - wheel-manylinux1-cp35m
     - wheel-manylinux1-cp36m
     - wheel-manylinux1-cp37m
     - wheel-manylinux1-cp38
-    - wheel-manylinux2010-cp27m
-    - wheel-manylinux2010-cp27mu
     - wheel-manylinux2010-cp35m
     - wheel-manylinux2010-cp36m
     - wheel-manylinux2010-cp37m
@@ -51,7 +45,6 @@ groups:
     - wheel-manylinux2014-cp36m
     - wheel-manylinux2014-cp37m
     - wheel-manylinux2014-cp38
-    - wheel-osx-cp27m
     - wheel-osx-cp35m
     - wheel-osx-cp36m
     - wheel-osx-cp37m
@@ -222,11 +215,9 @@ groups:
   ######################## Tasks to run regularly #############################
 
   nightly:
-    - conda-linux-gcc-py27
     - conda-linux-gcc-py36
     - conda-linux-gcc-py37
     - conda-linux-gcc-py38
-    - conda-osx-clang-py27
     - conda-osx-clang-py36
     - conda-osx-clang-py37
     - conda-osx-clang-py38
@@ -317,16 +308,6 @@ tasks:
 
   ############################## Conda Linux ##################################
 
-  conda-linux-gcc-py27:
-    ci: azure
-    platform: linux
-    template: conda-recipes/azure.linux.yml
-    params:
-      config: linux_python2.7
-    artifacts:
-      - arrow-cpp-{no_rc_version}-py27(h[a-z0-9]+)_0.tar.bz2
-      - pyarrow-{no_rc_version}-py27(h[a-z0-9]+)_0.tar.bz2
-
   conda-linux-gcc-py36:
     ci: azure
     platform: linux
@@ -359,16 +340,6 @@ tasks:
 
   ############################## Conda OSX ####################################
 
-  conda-osx-clang-py27:
-    ci: azure
-    platform: osx
-    template: conda-recipes/azure.osx.yml
-    params:
-      config: osx_python2.7
-    artifacts:
-      - arrow-cpp-{no_rc_version}-py27(h[a-z0-9]+)_0.tar.bz2
-      - pyarrow-{no_rc_version}-py27(h[a-z0-9]+)_0.tar.bz2
-
   conda-osx-clang-py36:
     ci: azure
     platform: osx

python/CMakeLists.txt

@@ -348,10 +348,8 @@ if(PYARROW_BUNDLE_ARROW_CPP)
       # disable autolinking in boost
       add_definitions(-DBOOST_ALL_NO_LIB)
     endif()
-    find_package(Boost COMPONENTS system filesystem regex REQUIRED)
+    find_package(Boost COMPONENTS regex REQUIRED)
     bundle_boost_lib(Boost_REGEX_LIBRARY)
-    bundle_boost_lib(Boost_FILESYSTEM_LIBRARY)
-    bundle_boost_lib(Boost_SYSTEM_LIBRARY)
   endif()
 
   if(MSVC)

python/manylinux201x/build_arrow.sh

@@ -50,11 +50,10 @@ export PYARROW_WITH_HDFS=1
 export PYARROW_WITH_PARQUET=1
 export PYARROW_WITH_PLASMA=1
 export PYARROW_BUNDLE_ARROW_CPP=1
-export PYARROW_BUNDLE_BOOST=1
-export PYARROW_BOOST_NAMESPACE=arrow_boost
+# Boost is only a compile-time dependency for wheels => no need to bundle .so's
+export PYARROW_BUNDLE_BOOST=0
 export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/arrow-dist/lib/pkgconfig
 
-export PYARROW_CMAKE_OPTIONS='-DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist'
 # Ensure the target directory exists
 mkdir -p /io/dist
 

python/requirements-wheel.txt

@@ -3,7 +3,4 @@ cython
 numpy>=1.16
 pandas
 setuptools_scm==3.2.0
-# TODO: TensorFlow doesn't support Python 3.8 yet.
-# See https://github.com/tensorflow/tensorflow/issues/33374
-tensorflow; python_version >= "3.0" and python_version < "3.8"
 wheel

python/setup.py

@@ -370,14 +370,6 @@ def append_cmake_bool(value, varname):
                 if self.with_parquet and not self.with_static_parquet:
                     move_shared_libs(build_prefix, build_lib, "parquet")
                 if not self.with_static_boost and self.bundle_boost:
-                    move_shared_libs(
-                        build_prefix, build_lib,
-                        "{}_filesystem".format(self.boost_namespace),
-                        implib_required=False)
-                    move_shared_libs(
-                        build_prefix, build_lib,
-                        "{}_system".format(self.boost_namespace),
-                        implib_required=False)
                     move_shared_libs(
                         build_prefix, build_lib,
                         "{}_regex".format(self.boost_namespace),