ARROW-7917: [C++] Find Python 3 in CMake configuration

Use the new FindPython3 package if available. Closes #6498 from pitrou/ARROW-7917-find-python3 and squashes the following commits: f45b7f7 <Antoine Pitrou> Use distutils.sysconfig dfc8201 <Antoine Pitrou> Apply review comments b3c01eb <Antoine Pitrou> Ensure we use the first Python on PATH 7395f83 <Antoine Pitrou> ARROW-7917: Find Python 3 in CMake configuration Authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
apache · Mar 3, 2020 · c707d8b · c707d8b
1 parent 467129e
commit c707d8b
Show file tree

Hide file tree

Showing 8 changed files with 205 additions and 93 deletions.
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -137,7 +137,17 @@ endif()
 include(DefineOptions)
 
 # Needed for linting targets, etc.
-find_package(PythonInterp)
+if(${CMAKE_VERSION} VERSION_LESS "3.12.0")
+  find_package(PythonInterp)
+else()
+  # Use the first Python installation on PATH, not the newest one
+  set(Python3_FIND_STRATEGY "LOCATION")
+  # On Windows, use registry last, not first
+  set(Python3_FIND_REGISTRY "LAST")
+
+  find_package(Python3)
+  set(PYTHON_EXECUTABLE ${Python3_EXECUTABLE})
+endif()
 
 if(ARROW_USE_CCACHE)
   find_program(CCACHE_FOUND ccache)

diff --git a/cpp/cmake_modules/FindNumPy.cmake b/cpp/cmake_modules/FindNumPy.cmake
@@ -38,12 +38,8 @@
 #
 #============================================================================
 
-# Finding NumPy involves calling the Python interpreter
-if(NumPy_FIND_REQUIRED)
-    find_package(PythonInterp REQUIRED)
-else()
-    find_package(PythonInterp)
-endif()
+# Legacy code for CMake < 3.15.0.  The primary point of entry should be
+# FindPython3Alt.cmake.
 
 if(NOT PYTHONINTERP_FOUND)
     set(NUMPY_FOUND FALSE)

diff --git a/cpp/cmake_modules/FindPython3Alt.cmake b/cpp/cmake_modules/FindPython3Alt.cmake
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This module finds the libraries corresponding to the Python 3 interpreter
+# and the NumPy package, and sets the following variables:
+# - PYTHON_EXECUTABLE
+# - PYTHON_INCLUDE_DIRS
+# - PYTHON_LIBRARIES
+# - PYTHON_OTHER_LIBS
+# - NUMPY_INCLUDE_DIRS
+
+# Need CMake 3.15 or later for Python3_FIND_STRATEGY
+if(${CMAKE_VERSION} VERSION_LESS "3.15.0")
+  # Use deprecated Python- and NumPy-finding code
+  if(Python3Alt_FIND_REQUIRED)
+    find_package(PythonLibsNew REQUIRED)
+    find_package(NumPy REQUIRED)
+  else()
+    find_package(PythonLibsNew)
+    find_package(NumPy)
+  endif()
+  find_package_handle_standard_args(Python3Alt
+                                    REQUIRED_VARS
+                                    PYTHON_EXECUTABLE
+                                    PYTHON_LIBRARIES
+                                    PYTHON_INCLUDE_DIRS
+                                    NUMPY_INCLUDE_DIRS)
+  return()
+endif()
+
+if(Python3Alt_FIND_REQUIRED)
+  find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED)
+else()
+  find_package(Python3 COMPONENTS Interpreter Development NumPy)
+endif()
+
+if(NOT Python3_FOUND)
+  return()
+endif()
+
+set(PYTHON_EXECUTABLE ${Python3_EXECUTABLE})
+set(PYTHON_INCLUDE_DIRS ${Python3_INCLUDE_DIRS})
+set(PYTHON_LIBRARIES ${Python3_LIBRARIES})
+set(PYTHON_OTHER_LIBS)
+
+get_target_property(NUMPY_INCLUDE_DIRS Python3::NumPy INTERFACE_INCLUDE_DIRECTORIES)
+
+# CMake's python3_add_library() doesn't apply the required extension suffix,
+# detect it ourselves.
+# (https://gitlab.kitware.com/cmake/cmake/issues/20408)
+execute_process(
+  COMMAND "${PYTHON_EXECUTABLE}" "-c"
+          "from distutils import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))"
+  RESULT_VARIABLE _PYTHON_RESULT
+  OUTPUT_VARIABLE _PYTHON_STDOUT
+  ERROR_VARIABLE _PYTHON_STDERR)
+
+if(NOT _PYTHON_RESULT MATCHES 0)
+  if(Python3Alt_FIND_REQUIRED)
+    message(FATAL_ERROR "Python 3 config failure:\n${_PYTHON_STDERR}")
+  endif()
+endif()
+
+string(STRIP ${_PYTHON_STDOUT} _EXT_SUFFIX)
+
+function(PYTHON_ADD_MODULE name)
+  python3_add_library(${name} MODULE ${ARGN})
+  set_target_properties(${name} PROPERTIES SUFFIX ${_EXT_SUFFIX})
+endfunction()
+
+find_package_handle_standard_args(Python3Alt
+                                  REQUIRED_VARS
+                                  PYTHON_EXECUTABLE
+                                  PYTHON_LIBRARIES
+                                  PYTHON_INCLUDE_DIRS
+                                  NUMPY_INCLUDE_DIRS)
diff --git a/cpp/cmake_modules/FindPythonLibsNew.cmake b/cpp/cmake_modules/FindPythonLibsNew.cmake
@@ -57,6 +57,9 @@
 # (To distribute this file outside of CMake, substitute the full
 #  License text for the above reference.)
 
+# Legacy code for CMake < 3.15.0.  The primary point of entry should be
+# FindPython3Alt.cmake.
+
 # Use the Python interpreter to find the libs.
 if(PythonLibsNew_FIND_REQUIRED)
     find_package(PythonInterp REQUIRED)

diff --git a/cpp/cmake_modules/UseCython.cmake b/cpp/cmake_modules/UseCython.cmake
@@ -58,119 +58,127 @@
 #=============================================================================
 
 # Configuration options.
-set( CYTHON_ANNOTATE OFF
-  CACHE BOOL "Create an annotated .html file when compiling *.pyx." )
-set( CYTHON_NO_DOCSTRINGS OFF
-  CACHE BOOL "Strip docstrings from the compiled module." )
-set( CYTHON_FLAGS "" CACHE STRING
-  "Extra flags to the cython compiler." )
-mark_as_advanced( CYTHON_ANNOTATE CYTHON_NO_DOCSTRINGS CYTHON_FLAGS)
+set(CYTHON_ANNOTATE OFF CACHE BOOL "Create an annotated .html file when compiling *.pyx.")
+set(CYTHON_NO_DOCSTRINGS OFF CACHE BOOL "Strip docstrings from the compiled module.")
+set(CYTHON_FLAGS "" CACHE STRING "Extra flags to the cython compiler.")
+mark_as_advanced(CYTHON_ANNOTATE CYTHON_NO_DOCSTRINGS CYTHON_FLAGS)
 
-find_package( PythonLibsNew REQUIRED )
+find_package(Python3Alt REQUIRED)
 
 # (using another C++ extension breaks coverage)
-set( CYTHON_CXX_EXTENSION "cpp" )
-set( CYTHON_C_EXTENSION "c" )
+set(CYTHON_CXX_EXTENSION "cpp")
+set(CYTHON_C_EXTENSION "c")
 
 # Create a *.c or *.cpp file from a *.pyx file.
 # Input the generated file basename.  The generate files will put into the variable
 # placed in the "generated_files" argument. Finally all the *.py and *.pyx files.
-function( compile_pyx _name pyx_target_name generated_files pyx_file)
+function(compile_pyx
+         _name
+         pyx_target_name
+         generated_files
+         pyx_file)
   # Default to assuming all files are C.
-  set( cxx_arg "" )
-  set( extension ${CYTHON_C_EXTENSION} )
-  set( pyx_lang "C" )
-  set( comment "Compiling Cython C source for ${_name}..." )
+  set(cxx_arg "")
+  set(extension ${CYTHON_C_EXTENSION})
+  set(pyx_lang "C")
+  set(comment "Compiling Cython C source for ${_name}...")
 
-  get_filename_component( pyx_file_basename "${pyx_file}" NAME_WE )
+  get_filename_component(pyx_file_basename "${pyx_file}" NAME_WE)
 
   # Determine if it is a C or C++ file.
-  get_source_file_property( property_is_cxx ${pyx_file} CYTHON_IS_CXX )
-  if( ${property_is_cxx} )
-    set( cxx_arg "--cplus" )
-    set( extension ${CYTHON_CXX_EXTENSION} )
-    set( pyx_lang "CXX" )
-    set( comment "Compiling Cython CXX source for ${_name}..." )
+  get_source_file_property(property_is_cxx ${pyx_file} CYTHON_IS_CXX)
+  if(${property_is_cxx})
+    set(cxx_arg "--cplus")
+    set(extension ${CYTHON_CXX_EXTENSION})
+    set(pyx_lang "CXX")
+    set(comment "Compiling Cython CXX source for ${_name}...")
   endif()
-  get_source_file_property( pyx_location ${pyx_file} LOCATION )
+  get_source_file_property(pyx_location ${pyx_file} LOCATION)
 
-  set ( output_file "${_name}.${extension}" )
+  set(output_file "${_name}.${extension}")
 
   # Set additional flags.
-  if( CYTHON_ANNOTATE )
-    set( annotate_arg "--annotate" )
+  if(CYTHON_ANNOTATE)
+    set(annotate_arg "--annotate")
   endif()
 
-  if( CYTHON_NO_DOCSTRINGS )
-    set( no_docstrings_arg "--no-docstrings" )
+  if(CYTHON_NO_DOCSTRINGS)
+    set(no_docstrings_arg "--no-docstrings")
   endif()
 
   if(NOT WIN32)
-      if( "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" OR
-            "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo" )
-          set( cython_debug_arg "--gdb" )
-      endif()
+    if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug"
+       OR "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo")
+      set(cython_debug_arg "--gdb")
+    endif()
   endif()
 
   # Determining generated file names.
-  get_source_file_property( property_is_public ${pyx_file} CYTHON_PUBLIC )
-  get_source_file_property( property_is_api ${pyx_file} CYTHON_API )
-  if( ${property_is_api} )
-      set( _generated_files "${output_file}" "${_name}.h" "${name}_api.h")
-  elseif( ${property_is_public} )
-      set( _generated_files "${output_file}" "${_name}.h")
+  get_source_file_property(property_is_public ${pyx_file} CYTHON_PUBLIC)
+  get_source_file_property(property_is_api ${pyx_file} CYTHON_API)
+  if(${property_is_api})
+    set(_generated_files "${output_file}" "${_name}.h" "${name}_api.h")
+  elseif(${property_is_public})
+    set(_generated_files "${output_file}" "${_name}.h")
   else()
-      set( _generated_files "${output_file}")
+    set(_generated_files "${output_file}")
   endif()
-  set_source_files_properties( ${_generated_files} PROPERTIES GENERATED TRUE )
+  set_source_files_properties(${_generated_files} PROPERTIES GENERATED TRUE)
 
-  if (NOT WIN32)
+  if(NOT WIN32)
     # Cython creates a lot of compiler warning detritus on clang
-    set_source_files_properties(${_generated_files} PROPERTIES
-      COMPILE_FLAGS -Wno-unused-function)
+    set_source_files_properties(${_generated_files} PROPERTIES COMPILE_FLAGS
+                                -Wno-unused-function)
   endif()
 
-  set( ${generated_files} ${_generated_files} PARENT_SCOPE )
+  set(${generated_files} ${_generated_files} PARENT_SCOPE)
 
   # Add the command to run the compiler.
-  add_custom_target(${pyx_target_name}
-    COMMAND ${PYTHON_EXECUTABLE} -m cython ${cxx_arg}
-    ${annotate_arg} ${no_docstrings_arg} ${cython_debug_arg}
-    ${CYTHON_FLAGS}
-    # Necessary for Cython code coverage
-    --working ${CMAKE_CURRENT_SOURCE_DIR}
-    --output-file "${CMAKE_CURRENT_BINARY_DIR}/${output_file}"
-    "${CMAKE_CURRENT_SOURCE_DIR}/${pyx_file}"
+  add_custom_target(
+    ${pyx_target_name}
+    COMMAND ${PYTHON_EXECUTABLE}
+            -m
+            cython
+            ${cxx_arg}
+            ${annotate_arg}
+            ${no_docstrings_arg}
+            ${cython_debug_arg}
+            ${CYTHON_FLAGS}
+            # Necessary for Cython code coverage
+            --working
+            ${CMAKE_CURRENT_SOURCE_DIR}
+            --output-file
+            "${CMAKE_CURRENT_BINARY_DIR}/${output_file}"
+            "${CMAKE_CURRENT_SOURCE_DIR}/${pyx_file}"
     DEPENDS ${pyx_location}
-    # Do not specify byproducts for now since they don't work with the older
-    # version of cmake available in the apt repositories.
-    #BYPRODUCTS ${_generated_files}
-    COMMENT ${comment}
-    )
+            # Do not specify byproducts for now since they don't work with the older
+            # version of cmake available in the apt repositories.
+            #BYPRODUCTS ${_generated_files}
+    COMMENT ${comment})
 
   # Remove their visibility to the user.
-  set( corresponding_pxd_file "" CACHE INTERNAL "" )
-  set( header_location "" CACHE INTERNAL "" )
-  set( pxd_location "" CACHE INTERNAL "" )
+  set(corresponding_pxd_file "" CACHE INTERNAL "")
+  set(header_location "" CACHE INTERNAL "")
+  set(pxd_location "" CACHE INTERNAL "")
 endfunction()
 
 # cython_add_module( <name> src1 src2 ... srcN )
 # Build the Cython Python module.
-function( cython_add_module _name pyx_target_name generated_files)
-  set( pyx_module_source "" )
-  set( other_module_sources "" )
-  foreach( _file ${ARGN} )
-    if( ${_file} MATCHES ".*\\.py[x]?$" )
-      list( APPEND pyx_module_source ${_file} )
+function(cython_add_module _name pyx_target_name generated_files)
+  set(pyx_module_source "")
+  set(other_module_sources "")
+  foreach(_file ${ARGN})
+    if(${_file} MATCHES ".*\\.py[x]?$")
+      list(APPEND pyx_module_source ${_file})
     else()
-      list( APPEND other_module_sources ${_file} )
+      list(APPEND other_module_sources ${_file})
     endif()
   endforeach()
-  compile_pyx( ${_name} ${pyx_target_name} _generated_files ${pyx_module_source} )
-  set( ${generated_files} ${_generated_files} PARENT_SCOPE )
-  include_directories( ${PYTHON_INCLUDE_DIRS} )
-  python_add_module( ${_name} ${_generated_files} ${other_module_sources} )
-  add_dependencies( ${_name} ${pyx_target_name})
+  compile_pyx(${_name} ${pyx_target_name} _generated_files ${pyx_module_source})
+  set(${generated_files} ${_generated_files} PARENT_SCOPE)
+  include_directories(${PYTHON_INCLUDE_DIRS})
+  python_add_module(${_name} ${_generated_files} ${other_module_sources})
+  add_dependencies(${_name} ${pyx_target_name})
 endfunction()
 
-include( CMakeParseArguments )
+include(CMakeParseArguments)
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
@@ -19,8 +19,7 @@
 # arrow_python
 #
 
-find_package(PythonLibsNew REQUIRED)
-find_package(NumPy REQUIRED)
+find_package(Python3Alt REQUIRED)
 
 add_custom_target(arrow_python-all)
 add_custom_target(arrow_python)