[Python] Modernize build system

This is a rewrite of the packaging of the Python bindings. The new packaging supports building the Python bindings both as part of a standard CMake build, as well as against a previously installed version of XRootD without the Python bindings. A new setup.py at the top level has been created to replace the old one from packaging/wheel. It can be used to drive the main CMake build using pip to create source and binary distributions of XRootD. Closes: #1768, #1807 #1833, #1844, #2001, #2002.
xrootd · Jun 6, 2023 · 39585c3 · 39585c3
1 parent 4df2c73
commit 39585c3
Show file tree

Hide file tree

Showing 20 changed files with 362 additions and 573 deletions.
diff --git a/.gitignore b/.gitignore
@@ -59,4 +59,3 @@ test/testconfig.sh
 xrootd.spec
 dist
 *.egg-info
-bindings/python/VERSION
diff --git a/packaging/wheel/MANIFEST.in → MANIFEST.in b/packaging/wheel/MANIFEST.in → MANIFEST.in
@@ -1,11 +1,13 @@
 include *.sh *.py *.in
-include CMakeLists.txt VERSION README COPYING* LICENSE
+include CMakeLists.txt
+include COPYING* LICENSE
+include VERSION README
 
 recursive-include bindings *
 recursive-include cmake *
+recursive-include docs *
 recursive-include packaging *
 recursive-include src *
 recursive-include tests *
 recursive-include ups *
 recursive-include utils *
-recursive-include docs *
diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt
@@ -1,109 +1,27 @@
+cmake_minimum_required(VERSION 3.16...3.25)
 
-set(SETUP_PY_IN    "${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in")
-set(SETUP_PY       "${CMAKE_CURRENT_BINARY_DIR}/setup.py")
-set(DEPS           "${CMAKE_CURRENT_SOURCE_DIR}/libs/__init__.py")
-set(OUTPUT         "${CMAKE_CURRENT_BINARY_DIR}/python_bindings")
-set(XRD_SRCINCDIR  "${CMAKE_SOURCE_DIR}/src")
-set(XRD_BININCDIR  "${CMAKE_BINARY_DIR}/src")
-set(XRDCL_LIBDIR   "${CMAKE_BINARY_DIR}/src/XrdCl")
-set(XRD_LIBDIR     "${CMAKE_BINARY_DIR}/src")
-set(XRDCL_INSTALL  "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+project(PyXRootD LANGUAGES CXX)
 
-if( PYPI_BUILD )
-  set(XRDCL_RPATH  "$ORIGIN/${CMAKE_INSTALL_LIBDIR}")
-else()
-  set(XRDCL_RPATH  "$ORIGIN/../../..")
-endif()
-
-if( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" )
-  if( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.7 )
-    message( "clang 3.5" )
-    set( CLANG_PROHIBITED ", '-Wp,-D_FORTIFY_SOURCE=2', '-fstack-protector-strong'" )
-  endif()
-  if( ( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 6.0 ) OR ( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0 ) )
-    message( "clang 6.0" )
-    set( CLANG_PROHIBITED ", '-fcf-protection'" )
-  endif()
-  if( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0 )
-    message( "clang > 7.0" )
-    set( CLANG_PROHIBITED ", '-fstack-clash-protection'" )
-  endif()
-endif()
-
-configure_file(${SETUP_PY_IN} ${SETUP_PY})
+find_package(Python REQUIRED COMPONENTS Interpreter Development)
 
-string(FIND "${PIP_OPTIONS}" "--prefix" PIP_OPTIONS_PREFIX_POSITION)
-if( "${PIP_OPTIONS_PREFIX_POSITION}" EQUAL "-1" )
-  string(APPEND PIP_OPTIONS " --prefix \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}")
+if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR OR PYPI_BUILD)
+  add_subdirectory(src)
 else()
-  message(WARNING
-        " The pip option --prefix has been set in '${PIP_OPTIONS}' which will change"
-        " it from its default value of '--prefix \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}'."
-        " Make sure this is intentional and that you understand the effects."
-        )
-endif()
-
-# Check it the Python interpreter has a valid version of pip
-execute_process(
-        COMMAND ${Python_EXECUTABLE} -m pip --version
-        RESULT_VARIABLE VALID_PIP_EXIT_CODE
-        OUTPUT_QUIET
-)
-
-if ( NOT ${VALID_PIP_EXIT_CODE} EQUAL 0 )
-  # Attempt to still install with deprecated invocation of setup.py
-  message(WARNING
-         " ${Python_EXECUTABLE} does not have a valid pip associated with it."
-         " It is recommended that you install a version of pip to install Python"
-         " packages and bindings. If you are unable to install a version of pip"
-         " through a package manager or with your Python build try using the PyPA's"
-         " get-pip.py bootstrapping script ( https://github.com/pypa/get-pip ).\n"
-         " The installation of the Python bindings will attempt to continue using"
-         " the deprecated method of `${Python_EXECUTABLE} setup.py install`."
-         )
-
-  # https://docs.python.org/3/install/#splitting-the-job-up
-  add_custom_command(OUTPUT ${OUTPUT}
-                     COMMAND ${Python_EXECUTABLE} ${SETUP_PY} --verbose build
-                     DEPENDS ${DEPS})
+  configure_file(setup.py setup.py)
+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/VERSION "${XRootD_VERSION_STRING}")
 
-  add_custom_target(python_target ALL DEPENDS ${OUTPUT} XrdCl)
+  option(INSTALL_PYTHON_BINDINGS "Install Python bindings" TRUE)
 
-  # Get the distribution name on Debian families
-  execute_process( COMMAND grep -i ^NAME= /etc/os-release
-                   OUTPUT_VARIABLE DEB_DISTRO )
-  STRING(REGEX REPLACE "^NAME=\"" "" DEB_DISTRO "${DEB_DISTRO}")
-  STRING(REGEX REPLACE "\".*"     "" DEB_DISTRO "${DEB_DISTRO}")
+  if(INSTALL_PYTHON_BINDINGS)
+    set(PIP_OPTIONS "" CACHE STRING "Install options for pip")
 
-  if( DEB_DISTRO STREQUAL "Debian" OR DEB_DISTRO STREQUAL "Ubuntu" )
-    set(PYTHON_LAYOUT  "unix" CACHE STRING "Python installation layout (deb or unix)")
-    set(DEB_INSTALL_ARGS "--install-layout ${PYTHON_LAYOUT}")
-  endif()
-
-  install(
-    CODE
-    "EXECUTE_PROCESS(
-      RESULT_VARIABLE INSTALL_STATUS
-      COMMAND /usr/bin/env ${XROOTD_PYBUILD_ENV} ${Python_EXECUTABLE} ${SETUP_PY} install \
-                --verbose \
-                --prefix \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX} \
-                ${DEB_INSTALL_ARGS}
-    )
-    if(NOT INSTALL_STATUS EQUAL 0)
-      message(FATAL_ERROR \"Failed to install Python bindings\")
-    endif()
-  ")
-else()
-  install(
-    CODE
-    "EXECUTE_PROCESS(
-      RESULT_VARIABLE INSTALL_STATUS
-      COMMAND /usr/bin/env ${XROOTD_PYBUILD_ENV} ${Python_EXECUTABLE} -m pip install \
-                ${PIP_OPTIONS} \
-                ${CMAKE_CURRENT_BINARY_DIR}
-      )
+    install(CODE "
+      execute_process(COMMAND ${Python_EXECUTABLE} -m pip install ${PIP_OPTIONS}
+        --prefix \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX} ${CMAKE_CURRENT_BINARY_DIR}
+        RESULT_VARIABLE INSTALL_STATUS)
       if(NOT INSTALL_STATUS EQUAL 0)
         message(FATAL_ERROR \"Failed to install Python bindings\")
       endif()
-  ")
+    ")
+  endif()
 endif()
diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in
@@ -1,4 +1,4 @@
-include README.rst
+include CMakeLists.txt
 recursive-include tests *
 recursive-include examples *.py
 recursive-include docs *

diff --git a/bindings/python/README b/bindings/python/README
@@ -0,0 +1,6 @@
+# XRootD Python Bindings
+
+This is a set of simple but pythonic bindings for XRootD. It is designed to make
+it easy to interface with the XRootD client, by writing Python instead of having
+to write C++.
+
diff --git a/bindings/python/README.rst b/bindings/python/README.rst
diff --git a/bindings/python/VERSION b/bindings/python/VERSION
@@ -0,0 +1 @@
+../../VERSION
diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml
@@ -0,0 +1 @@
+../../pyproject.toml
diff --git a/bindings/python/setup.py b/bindings/python/setup.py
@@ -0,0 +1,147 @@
+import os
+import platform
+import subprocess
+import sys
+
+from setuptools import setup, Extension
+from setuptools.command.build_ext import build_ext
+from subprocess import check_call, check_output
+
+try:
+    from shutil import which
+except ImportError:
+    from distutils.spawn import find_executable as which
+
+srcdir = '${CMAKE_CURRENT_SOURCE_DIR}'
+
+cmdline_args = []
+
+# Check for unexpanded srcdir to determine if this is part
+# of a regular CMake build or a Python build using setup.py.
+
+if not srcdir.startswith('$'):
+    # When building the Python bindings as part of a standard CMake build,
+    # propagate down which cmake command to use, and the build type, C++
+    # compiler, build flags, and how to link libXrdCl from the main build.
+
+    cmake = '${CMAKE_COMMAND}'
+
+    cmdline_args += [
+        '-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}',
+        '-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}',
+        '-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}',
+        '-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}',
+        '-DXRootD_CLIENT_LIBRARY=${CMAKE_BINARY_DIR}/src/XrdCl/libXrdCl${CMAKE_SHARED_LIBRARY_SUFFIX}',
+        '-DXRootD_INCLUDE_DIR=${CMAKE_SOURCE_DIR}/src;${CMAKE_BINARY_DIR}/src',
+    ]
+else:
+    srcdir = '.'
+
+    cmake = which("cmake3") or which("cmake")
+
+    for arg in sys.argv:
+        if arg.startswith('-D'):
+            cmdline_args.append(arg)
+
+    for arg in cmdline_args:
+        sys.argv.remove(arg)
+
+def get_version():
+    version = '${XRootD_VERSION_STRING}'
+
+    if version.startswith('$'):
+        try:
+            version = open('VERSION').read().strip()
+
+            if version.startswith('$'):
+                output = check_output(['git', 'describe'])
+                version = output.decode().strip()
+        except:
+            version = None
+            pass
+
+    if version is None:
+        from datetime import date
+        version = '5.6-rc' + date.today().strftime("%Y%m%d")
+
+    if version.startswith('v'):
+        version = version[1:]
+
+    # Sanitize version to conform to PEP 440
+    # https://www.python.org/dev/peps/pep-0440
+    version = version.replace('-rc', 'rc')
+    version = version.replace('-g', '+git.')
+    version = version.replace('-', '.post', 1)
+    version = version.replace('-', '.')
+
+    return version
+
+class CMakeExtension(Extension):
+    def __init__(self, name, src=srcdir, sources=[], **kwa):
+        Extension.__init__(self, name, sources=sources, **kwa)
+        self.src = os.path.abspath(src)
+
+class CMakeBuild(build_ext):
+    def build_extensions(self):
+        if cmake is None:
+            raise RuntimeError('Cannot find CMake executable')
+
+        for ext in self.extensions:
+            extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
+
+            # Use relative RPATHs to ensure the correct libraries are picked up.
+            # The RPATH below covers most cases where a non-standard path is
+            # used for installation. It allows to find libXrdCl with a relative
+            # path from the site-packages directory. Build with install RPATH
+            # because libraries are installed by Python/pip not CMake, so CMake
+            # cannot fix the install RPATH later on.
+
+            cmake_args = [
+                '-DPython_EXECUTABLE={}'.format(sys.executable),
+                '-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE',
+                '-DCMAKE_INSTALL_RPATH=$ORIGIN/../../../../$LIB',
+                '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}/{}'.format(self.build_temp, ext.name),
+                '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}/{}'.format(extdir, ext.name),
+            ]
+
+            cmake_args += cmdline_args
+
+            if not os.path.exists(self.build_temp):
+                os.makedirs(self.build_temp)
+
+            check_call([cmake, ext.src, '-B', self.build_temp] + cmake_args)
+            check_call([cmake, '--build', self.build_temp, '--parallel'])
+
+version = get_version()
+
+setup(name='xrootd',
+      version=version,
+      description='XRootD Python bindings',
+      author='XRootD Developers',
+      author_email='xrootd-dev@slac.stanford.edu',
+      url='http://xrootd.org',
+      download_url='https://github.com/xrootd/xrootd/archive/v%s.tar.gz' % version,
+      keywords=['XRootD', 'network filesystem'],
+      license='LGPLv3+',
+      long_description=open(srcdir + '/README').read(),
+      long_description_content_type='text/plain',
+      packages = ['XRootD', 'XRootD.client', 'pyxrootd'],
+      package_dir = {
+        'pyxrootd'     : srcdir + '/src',
+        'XRootD'       : srcdir + '/libs',
+        'XRootD/client': srcdir + '/libs/client',
+      },
+      ext_modules= [ CMakeExtension('pyxrootd') ],
+      cmdclass={ 'build_ext': CMakeBuild },
+      zip_safe=False,
+      classifiers=[
+          "Intended Audience :: Information Technology",
+          "Intended Audience :: Science/Research",
+          "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)",
+          "Operating System :: MacOS",
+          "Operating System :: POSIX :: Linux",
+          "Operating System :: Unix",
+          "Programming Language :: C++",
+          "Programming Language :: Python",
+      ]
+     )