From 0c3c20e72e4ea1522badc4cf61ed8059f36cfa6c Mon Sep 17 00:00:00 2001 From: Guilherme Amadio Date: Thu, 1 Jun 2023 16:16:29 +0200 Subject: [PATCH] [Python] Modernize build system This is a rewrite of the packaging of the Python bindings. The new packaging supports building the Python bindings both as part of a standard CMake build, as well as against a previously installed version of XRootD without the Python bindings. A new setup.py at the top level has been created to replace the old one from packaging/wheel. It can be used to drive the main CMake build using pip to create source and binary distributions of XRootD. Closes: #1768, #1807 #1833, #1844, #2001, #2002. --- .gitignore | 1 - packaging/wheel/MANIFEST.in => MANIFEST.in | 6 +- bindings/python/CMakeLists.txt | 114 ++----------- bindings/python/MANIFEST.in | 2 +- bindings/python/README | 6 + bindings/python/README.rst | 16 -- bindings/python/VERSION | 1 + bindings/python/pyproject.toml | 1 + bindings/python/setup.py | 147 ++++++++++++++++ bindings/python/setup.py.in | 93 ---------- bindings/python/src/CMakeLists.txt | 49 ++++++ cmake/XRootDDefaults.cmake | 2 - packaging/debian/rules | 6 +- packaging/rhel/xrootd.spec.in | 3 +- packaging/wheel/TestCXX14.txt | 95 ----------- packaging/wheel/has_c++14.sh | 12 -- packaging/wheel/install.sh | 61 ------- packaging/wheel/setup.py | 188 --------------------- pyproject.toml | 3 + setup.py | 120 +++++++++++++ 20 files changed, 353 insertions(+), 573 deletions(-) rename packaging/wheel/MANIFEST.in => MANIFEST.in (76%) create mode 100644 bindings/python/README delete mode 100644 bindings/python/README.rst create mode 120000 bindings/python/VERSION create mode 120000 bindings/python/pyproject.toml create mode 100644 bindings/python/setup.py delete mode 100644 bindings/python/setup.py.in create mode 100644 bindings/python/src/CMakeLists.txt delete mode 100644 packaging/wheel/TestCXX14.txt delete mode 100755 packaging/wheel/has_c++14.sh delete mode 100755 packaging/wheel/install.sh delete mode 100644 packaging/wheel/setup.py create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 6959934d9b6..adfe0558ac7 100644 --- a/.gitignore +++ b/.gitignore @@ -59,4 +59,3 @@ test/testconfig.sh xrootd.spec dist *.egg-info -bindings/python/VERSION diff --git a/packaging/wheel/MANIFEST.in b/MANIFEST.in similarity index 76% rename from packaging/wheel/MANIFEST.in rename to MANIFEST.in index c8c539facbd..c39b67a5b56 100644 --- a/packaging/wheel/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,13 @@ include *.sh *.py *.in -include CMakeLists.txt VERSION README COPYING* LICENSE +include CMakeLists.txt +include COPYING* LICENSE +include VERSION README recursive-include bindings * recursive-include cmake * +recursive-include docs * recursive-include packaging * recursive-include src * recursive-include tests * recursive-include ups * recursive-include utils * -recursive-include docs * diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 2c5168475f4..d29b35cc9f6 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -1,109 +1,27 @@ +cmake_minimum_required(VERSION 3.16...3.25) -set(SETUP_PY_IN "${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in") -set(SETUP_PY "${CMAKE_CURRENT_BINARY_DIR}/setup.py") -set(DEPS "${CMAKE_CURRENT_SOURCE_DIR}/libs/__init__.py") -set(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/python_bindings") -set(XRD_SRCINCDIR "${CMAKE_SOURCE_DIR}/src") -set(XRD_BININCDIR "${CMAKE_BINARY_DIR}/src") -set(XRDCL_LIBDIR "${CMAKE_BINARY_DIR}/src/XrdCl") -set(XRD_LIBDIR "${CMAKE_BINARY_DIR}/src") -set(XRDCL_INSTALL "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") +project(PyXRootD LANGUAGES CXX) -if( PYPI_BUILD ) - set(XRDCL_RPATH "$ORIGIN/${CMAKE_INSTALL_LIBDIR}") -else() - set(XRDCL_RPATH "$ORIGIN/../../..") -endif() - -if( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" ) - if( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.7 ) - message( "clang 3.5" ) - set( CLANG_PROHIBITED ", '-Wp,-D_FORTIFY_SOURCE=2', '-fstack-protector-strong'" ) - endif() - if( ( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 6.0 ) OR ( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0 ) ) - message( "clang 6.0" ) - set( CLANG_PROHIBITED ", '-fcf-protection'" ) - endif() - if( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0 ) - message( "clang > 7.0" ) - set( CLANG_PROHIBITED ", '-fstack-clash-protection'" ) - endif() -endif() - -configure_file(${SETUP_PY_IN} ${SETUP_PY}) +find_package(Python REQUIRED COMPONENTS Interpreter Development) -string(FIND "${PIP_OPTIONS}" "--prefix" PIP_OPTIONS_PREFIX_POSITION) -if( "${PIP_OPTIONS_PREFIX_POSITION}" EQUAL "-1" ) - string(APPEND PIP_OPTIONS " --prefix \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}") +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR OR PYPI_BUILD) + add_subdirectory(src) else() - message(WARNING - " The pip option --prefix has been set in '${PIP_OPTIONS}' which will change" - " it from its default value of '--prefix \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}'." - " Make sure this is intentional and that you understand the effects." - ) -endif() - -# Check it the Python interpreter has a valid version of pip -execute_process( - COMMAND ${Python_EXECUTABLE} -m pip --version - RESULT_VARIABLE VALID_PIP_EXIT_CODE - OUTPUT_QUIET -) - -if ( NOT ${VALID_PIP_EXIT_CODE} EQUAL 0 ) - # Attempt to still install with deprecated invocation of setup.py - message(WARNING - " ${Python_EXECUTABLE} does not have a valid pip associated with it." - " It is recommended that you install a version of pip to install Python" - " packages and bindings. If you are unable to install a version of pip" - " through a package manager or with your Python build try using the PyPA's" - " get-pip.py bootstrapping script ( https://github.com/pypa/get-pip ).\n" - " The installation of the Python bindings will attempt to continue using" - " the deprecated method of `${Python_EXECUTABLE} setup.py install`." - ) - - # https://docs.python.org/3/install/#splitting-the-job-up - add_custom_command(OUTPUT ${OUTPUT} - COMMAND ${Python_EXECUTABLE} ${SETUP_PY} --verbose build - DEPENDS ${DEPS}) + configure_file(setup.py setup.py) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/VERSION "${XRootD_VERSION_STRING}") - add_custom_target(python_target ALL DEPENDS ${OUTPUT} XrdCl) + option(INSTALL_PYTHON_BINDINGS "Install Python bindings" TRUE) - # Get the distribution name on Debian families - execute_process( COMMAND grep -i ^NAME= /etc/os-release - OUTPUT_VARIABLE DEB_DISTRO ) - STRING(REGEX REPLACE "^NAME=\"" "" DEB_DISTRO "${DEB_DISTRO}") - STRING(REGEX REPLACE "\".*" "" DEB_DISTRO "${DEB_DISTRO}") + if(INSTALL_PYTHON_BINDINGS) + set(PIP_OPTIONS "" CACHE STRING "Install options for pip") - if( DEB_DISTRO STREQUAL "Debian" OR DEB_DISTRO STREQUAL "Ubuntu" ) - set(PYTHON_LAYOUT "unix" CACHE STRING "Python installation layout (deb or unix)") - set(DEB_INSTALL_ARGS "--install-layout ${PYTHON_LAYOUT}") - endif() - - install( - CODE - "EXECUTE_PROCESS( - RESULT_VARIABLE INSTALL_STATUS - COMMAND /usr/bin/env ${XROOTD_PYBUILD_ENV} ${Python_EXECUTABLE} ${SETUP_PY} install \ - --verbose \ - --prefix \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX} \ - ${DEB_INSTALL_ARGS} - ) - if(NOT INSTALL_STATUS EQUAL 0) - message(FATAL_ERROR \"Failed to install Python bindings\") - endif() - ") -else() - install( - CODE - "EXECUTE_PROCESS( - RESULT_VARIABLE INSTALL_STATUS - COMMAND /usr/bin/env ${XROOTD_PYBUILD_ENV} ${Python_EXECUTABLE} -m pip install \ - ${PIP_OPTIONS} \ - ${CMAKE_CURRENT_BINARY_DIR} - ) + install(CODE " + execute_process(COMMAND ${Python_EXECUTABLE} -m pip install ${PIP_OPTIONS} + --prefix \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX} ${CMAKE_CURRENT_BINARY_DIR} + RESULT_VARIABLE INSTALL_STATUS) if(NOT INSTALL_STATUS EQUAL 0) message(FATAL_ERROR \"Failed to install Python bindings\") endif() - ") + ") + endif() endif() diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in index e9c2fda97b8..2dd8b3ebfb0 100644 --- a/bindings/python/MANIFEST.in +++ b/bindings/python/MANIFEST.in @@ -1,4 +1,4 @@ -include README.rst +include CMakeLists.txt recursive-include tests * recursive-include examples *.py recursive-include docs * diff --git a/bindings/python/README b/bindings/python/README new file mode 100644 index 00000000000..059782ec5f0 --- /dev/null +++ b/bindings/python/README @@ -0,0 +1,6 @@ +# XRootD Python Bindings + +This is a set of simple but pythonic bindings for XRootD. It is designed to make +it easy to interface with the XRootD client, by writing Python instead of having +to write C++. + diff --git a/bindings/python/README.rst b/bindings/python/README.rst deleted file mode 100644 index cf5c2bd0129..00000000000 --- a/bindings/python/README.rst +++ /dev/null @@ -1,16 +0,0 @@ -Prerequisites (incomplete): xrootd - -# Installing on OSX -Setup should succeed if: - - xrootd is installed on your system - - xrootd was installed via homebrew - - you're installing the bindings package with the same version number as the - xrootd installation (`xrootd -v`). - -If you have xrootd installed and the installation still fails, do -`XRD_LIBDIR=XYZ; XRD_INCDIR=ZYX; pip install xrootd` -where XYZ and ZYX are the paths to the XRootD library and include directories on your system. - -## How to find the lib and inc directories -To find the library directory, search your system for "libXrd*" files. -The include directory should contain a file named "XrdVersion.hh", so search for that. diff --git a/bindings/python/VERSION b/bindings/python/VERSION new file mode 120000 index 00000000000..558194c5a5a --- /dev/null +++ b/bindings/python/VERSION @@ -0,0 +1 @@ +../../VERSION \ No newline at end of file diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml new file mode 120000 index 00000000000..00c904eb842 --- /dev/null +++ b/bindings/python/pyproject.toml @@ -0,0 +1 @@ +../../pyproject.toml \ No newline at end of file diff --git a/bindings/python/setup.py b/bindings/python/setup.py new file mode 100644 index 00000000000..226607c2756 --- /dev/null +++ b/bindings/python/setup.py @@ -0,0 +1,147 @@ +import os +import platform +import subprocess +import sys + +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext +from subprocess import check_call, check_output + +try: + from shutil import which +except ImportError: + from distutils.spawn import find_executable as which + +srcdir = '${CMAKE_CURRENT_SOURCE_DIR}' + +cmdline_args = [] + +# Check for unexpanded srcdir to determine if this is part +# of a regular CMake build or a Python build using setup.py. + +if not srcdir.startswith('$'): + # When building the Python bindings as part of a standard CMake build, + # propagate down which cmake command to use, and the build type, C++ + # compiler, build flags, and how to link libXrdCl from the main build. + + cmake = '${CMAKE_COMMAND}' + + cmdline_args += [ + '-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}', + '-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}', + '-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}', + '-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}', + '-DXRootD_CLIENT_LIBRARY=${CMAKE_BINARY_DIR}/src/XrdCl/libXrdCl${CMAKE_SHARED_LIBRARY_SUFFIX}', + '-DXRootD_INCLUDE_DIR=${CMAKE_SOURCE_DIR}/src;${CMAKE_BINARY_DIR}/src', + ] +else: + srcdir = '.' + + cmake = which("cmake3") or which("cmake") + + for arg in sys.argv: + if arg.startswith('-D'): + cmdline_args.append(arg) + + for arg in cmdline_args: + sys.argv.remove(arg) + +def get_version(): + version = '${XRootD_VERSION_STRING}' + + if version.startswith('$'): + try: + version = open('VERSION').read().strip() + + if version.startswith('$'): + output = check_output(['git', 'describe']) + version = output.decode().strip() + except: + version = None + pass + + if version is None: + from datetime import date + version = '5.6-rc' + date.today().strftime("%Y%m%d") + + if version.startswith('v'): + version = version[1:] + + # Sanitize version to conform to PEP 440 + # https://www.python.org/dev/peps/pep-0440 + version = version.replace('-rc', 'rc') + version = version.replace('-g', '+git.') + version = version.replace('-', '.post', 1) + version = version.replace('-', '.') + + return version + +class CMakeExtension(Extension): + def __init__(self, name, src=srcdir, sources=[], **kwa): + Extension.__init__(self, name, sources=sources, **kwa) + self.src = os.path.abspath(src) + +class CMakeBuild(build_ext): + def build_extensions(self): + if cmake is None: + raise RuntimeError('Cannot find CMake executable') + + for ext in self.extensions: + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + + # Use relative RPATHs to ensure the correct libraries are picked up. + # The RPATH below covers most cases where a non-standard path is + # used for installation. It allows to find libXrdCl with a relative + # path from the site-packages directory. Build with install RPATH + # because libraries are installed by Python/pip not CMake, so CMake + # cannot fix the install RPATH later on. + + cmake_args = [ + '-DPython_EXECUTABLE={}'.format(sys.executable), + '-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE', + '-DCMAKE_INSTALL_RPATH=$ORIGIN/../../../../$LIB', + '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}/{}'.format(self.build_temp, ext.name), + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}/{}'.format(extdir, ext.name), + ] + + cmake_args += cmdline_args + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + check_call([cmake, ext.src, '-B', self.build_temp] + cmake_args) + check_call([cmake, '--build', self.build_temp, '--parallel']) + +version = get_version() + +setup(name='xrootd', + version=version, + description='XRootD Python bindings', + author='XRootD Developers', + author_email='xrootd-dev@slac.stanford.edu', + url='http://xrootd.org', + download_url='https://github.com/xrootd/xrootd/archive/v%s.tar.gz' % version, + keywords=['XRootD', 'network filesystem'], + license='LGPLv3+', + long_description=open(srcdir + '/README').read(), + long_description_content_type='text/plain', + packages = ['XRootD', 'XRootD.client', 'pyxrootd'], + package_dir = { + 'pyxrootd' : srcdir + '/src', + 'XRootD' : srcdir + '/libs', + 'XRootD/client': srcdir + '/libs/client', + }, + ext_modules= [ CMakeExtension('pyxrootd') ], + cmdclass={ 'build_ext': CMakeBuild }, + zip_safe=False, + classifiers=[ + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + "Operating System :: Unix", + "Programming Language :: C++", + "Programming Language :: Python", + ] + ) diff --git a/bindings/python/setup.py.in b/bindings/python/setup.py.in deleted file mode 100644 index 7d02d07278e..00000000000 --- a/bindings/python/setup.py.in +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import print_function - -from setuptools import setup, Extension -# sysconfig with setuptools v48.0.0+ is incompatible for Python 3.6 only, so fall back to distutils. -# FIXME: When support for Python 3.6 is dropped simplify this -import sys - -if sys.version_info < (3, 7): - from distutils import sysconfig -else: - import sysconfig - -from os import getenv, walk, path -import subprocess - -# Remove the "-Wstrict-prototypes" compiler option, which isn't valid for C++. -cfg_vars = sysconfig.get_config_vars() -opt = cfg_vars["OPT"] -cfg_vars["OPT"] = " ".join( flag for flag in opt.split() if flag not in ['-Wstrict-prototypes' ${CLANG_PROHIBITED} ] ) + " --std=c++14" - -cflags = cfg_vars["CFLAGS"] -cfg_vars["CFLAGS"] = " ".join( flag for flag in cflags.split() if flag not in ['-Wstrict-prototypes' ${CLANG_PROHIBITED} ] ) + " --std=c++14" - -# pypy doesn't define PY_CFLAGS so skip it if it's missing -if "PY_CFLAGS" in cfg_vars: - py_cflags = cfg_vars["PY_CFLAGS"] - cfg_vars["PY_CFLAGS"] = " ".join( flag for flag in py_cflags.split() if flag not in ['-Wstrict-prototypes' ${CLANG_PROHIBITED} ] ) + " --std=c++14" - -ccl=cfg_vars["CC"].split() -ccl[0]="${CMAKE_C_COMPILER}" -cfg_vars["CC"] = " ".join(ccl) -cxxl=cfg_vars["CXX"].split() -cxxl[0]="${CMAKE_CXX_COMPILER}" -cfg_vars["CXX"] = " ".join(cxxl) -cfg_vars["PY_CXXFLAGS"] = "${CMAKE_CXX_FLAGS}" - -# Make the RPATH relative to the python module -cfg_vars['LDSHARED'] = cfg_vars['LDSHARED'] + " -Wl,-rpath,${XRDCL_RPATH}" - -sources = list() -depends = list() - -for dirname, dirnames, filenames in walk('${CMAKE_CURRENT_SOURCE_DIR}/src'): - for filename in filenames: - if filename.endswith('.cc'): - sources.append(path.join(dirname, filename)) - elif filename.endswith('.hh'): - depends.append(path.join(dirname, filename)) - -xrdcllibdir = "${XRDCL_LIBDIR}" -xrdlibdir = "${XRD_LIBDIR}" -xrdsrcincdir = "${XRD_SRCINCDIR}" -xrdbinincdir = "${XRD_BININCDIR}" - -version = "${XRootD_VERSION_STRING}" - -# Sanitize version to conform to PEP 440 -# https://www.python.org/dev/peps/pep-0440 - -version = version.replace('-rc', 'rc') -version = version.replace('-g', '+git.') -version = version.replace('-', '.post', 1) -version = version.replace('-', '.') - -print('XRootD library dir: ', xrdlibdir) -print('XRootD src include dir:', xrdsrcincdir) -print('XRootD bin include dir:', xrdbinincdir) -print('Version: ', version) - -setup( name = 'xrootd', - version = version, - author = 'XRootD Developers', - author_email = 'xrootd-dev@slac.stanford.edu', - url = 'http://xrootd.org', - license = 'LGPLv3+', - description = "XRootD Python bindings", - long_description = "XRootD Python bindings", - packages = ['pyxrootd', 'XRootD', 'XRootD.client'], - package_dir = {'pyxrootd' : '${CMAKE_CURRENT_SOURCE_DIR}/src', - 'XRootD' : '${CMAKE_CURRENT_SOURCE_DIR}/libs', - 'XRootD.client': '${CMAKE_CURRENT_SOURCE_DIR}/libs/client'}, - ext_modules = [ - Extension( - 'pyxrootd.client', - sources = sources, - depends = depends, - libraries = ['XrdCl', 'XrdUtils', 'dl'], - extra_compile_args = ['-g'], - include_dirs = [xrdsrcincdir, xrdbinincdir], - library_dirs = [xrdlibdir, xrdcllibdir] - ) - ] - ) diff --git a/bindings/python/src/CMakeLists.txt b/bindings/python/src/CMakeLists.txt new file mode 100644 index 00000000000..b3f4fe3f216 --- /dev/null +++ b/bindings/python/src/CMakeLists.txt @@ -0,0 +1,49 @@ +Python_add_library(client MODULE WITH_SOABI + # headers + AsyncResponseHandler.hh + ChunkIterator.hh + Conversions.hh + PyXRootD.hh + PyXRootDCopyProcess.hh + PyXRootDCopyProgressHandler.hh + PyXRootDEnv.hh + PyXRootDFile.hh + PyXRootDFileSystem.hh + PyXRootDFinalize.hh + PyXRootDURL.hh + Utils.hh + # sources + PyXRootDCopyProcess.cc + PyXRootDCopyProgressHandler.cc + PyXRootDFile.cc + PyXRootDFileSystem.cc + PyXRootDModule.cc + PyXRootDURL.cc + Utils.cc +) + +target_compile_options(client PRIVATE -w) # TODO: fix build warnings + +if(APPLE) + set(CMAKE_MACOSX_RPATH TRUE) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + set_target_properties(client PROPERTIES + INSTALL_NAME_DIR "@rpath" INSTALL_RPATH "@loader_path") +endif() + +# Avoid a call to find_package(XRootD) in order to be able to override +# variables when building the module as part of a standard CMake build. + +if(TARGET XrdCl) + target_link_libraries(client PRIVATE XrdCl) +else() + find_library(XRootD_CLIENT_LIBRARY NAMES XrdCl) + find_path(XRootD_INCLUDE_DIR XrdVersion.hh PATH_SUFFIXES include/xrootd) + + # The client library makes use of private XRootD headers, so add the + # extra include for it to allow building the Python bindings against + # a pre-installed XRootD. + + target_link_libraries(client PRIVATE ${XRootD_CLIENT_LIBRARY}) + target_include_directories(client PRIVATE ${XRootD_INCLUDE_DIR} ${XRootD_INCLUDE_DIR}/private) +endif() diff --git a/cmake/XRootDDefaults.cmake b/cmake/XRootDDefaults.cmake index f9bccf6549d..f7223ceadd2 100644 --- a/cmake/XRootDDefaults.cmake +++ b/cmake/XRootDDefaults.cmake @@ -19,8 +19,6 @@ option( ENABLE_XRDCL "Enable XRootD client." option( ENABLE_TESTS "Enable unit tests." FALSE ) option( ENABLE_HTTP "Enable HTTP component." TRUE ) option( ENABLE_PYTHON "Enable python bindings." TRUE ) -# As PIP_OPTIONS uses the cache, make sure to clean cache if rebuilding (e.g. cmake --build --clean-first) -SET(PIP_OPTIONS "" CACHE STRING "pip options used during the Python bindings install.") option( XRDCL_ONLY "Build only the client and necessary dependencies" FALSE ) option( XRDCL_LIB_ONLY "Build only the client libraries and necessary dependencies" FALSE ) option( PYPI_BUILD "The project is being built for PyPI release" FALSE ) diff --git a/packaging/debian/rules b/packaging/debian/rules index 44203c03974..df662e1c07c 100755 --- a/packaging/debian/rules +++ b/packaging/debian/rules @@ -1,12 +1,12 @@ #!/usr/bin/make -f +export DH_VERBOSE=1 export PYBUILD_NAME=xrootd -# --install-layout deb %: - dh $@ --builddirectory=build --destdir=deb_packages --with python3 + dh $@ --builddirectory=build --destdir=deb_packages --with python3 --buildsystem=cmake override_dh_auto_configure: - dh_auto_configure -- -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_LIBDIR=lib/$(shell dpkg-architecture -qDEB_HOST_MULTIARCH) -DPython_EXECUTABLE=/usr/bin/python3 -DCMAKE_SKIP_INSTALL_RPATH=ON -DENABLE_XRDCLHTTP=TRUE + dh_auto_configure -- -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_LIBDIR=lib/$(shell dpkg-architecture -qDEB_HOST_MULTIARCH) -DPython_EXECUTABLE=/usr/bin/python3 -DCMAKE_SKIP_INSTALL_RPATH=ON -DENABLE_XRDCLHTTP=TRUE -DINSTALL_PYTHON_BINDINGS=TRUE -DPIP_OPTIONS='--no-deps --disable-pip-version-check --verbose' override_dh_install: install -D -m 644 packaging/common/client.conf deb_packages/etc/xrootd/client.conf diff --git a/packaging/rhel/xrootd.spec.in b/packaging/rhel/xrootd.spec.in index 86757efb1ae..7a229600311 100644 --- a/packaging/rhel/xrootd.spec.in +++ b/packaging/rhel/xrootd.spec.in @@ -532,7 +532,8 @@ cmake \ %if %{?_with_isal:1}%{!?_with_isal:0} -DENABLE_XRDEC=TRUE \ %endif - -DUSER_VERSION=v%{version} \ + -DXRootD_VERSION_STRING=v%{version} \ + -DINSTALL_PYTHON_BINDINGS=FALSE \ ../ make -i VERBOSE=1 %{?_smp_mflags} diff --git a/packaging/wheel/TestCXX14.txt b/packaging/wheel/TestCXX14.txt deleted file mode 100644 index ef1e98e0bbf..00000000000 --- a/packaging/wheel/TestCXX14.txt +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################### -# Test with cmake if the compiler supports all features of C++14 -############################################################################### - -cmake_minimum_required(VERSION 3.16...3.25 FATAL_ERROR) -project(TestCXX14 CXX) - -message("Your C++ compiler supports these C++14 features:") - -foreach( feature ${CMAKE_CXX_COMPILE_FEATURES} ) - - if( feature STREQUAL "cxx_std_14" ) - message( "${feature}" ) - set( has_cxx_std_14 TRUE ) - endif() - - if( feature STREQUAL "cxx_aggregate_default_initializers" ) - message( "${feature}" ) - set( has_cxx_aggregate_default_initializers TRUE ) - endif() - - if( feature STREQUAL "cxx_attribute_deprecated" ) - message( "${feature}" ) - set( has_cxx_attribute_deprecated TRUE ) - endif() - - if( feature STREQUAL "cxx_binary_literals" ) - message( "${feature}" ) - set( has_cxx_binary_literals TRUE ) - endif() - - if( feature STREQUAL "cxx_contextual_conversions" ) - message( "${feature}" ) - set( has_cxx_contextual_conversions TRUE ) - endif() - - if( feature STREQUAL "cxx_decltype_auto" ) - message( "${feature}" ) - set( has_cxx_decltype_auto TRUE ) - endif() - - if( feature STREQUAL "cxx_digit_separators" ) - message( "${feature}" ) - set( has_cxx_digit_separators TRUE ) - endif() - - if( feature STREQUAL "cxx_generic_lambdas" ) - message( "${feature}" ) - set( has_cxx_generic_lambdas TRUE ) - endif() - - if( feature STREQUAL "cxx_lambda_init_captures" ) - message( "${feature}" ) - set( has_cxx_lambda_init_captures TRUE ) - endif() - - if( feature STREQUAL "cxx_relaxed_constexpr" ) - message( "${feature}" ) - set( has_cxx_relaxed_constexpr TRUE ) - endif() - - if( feature STREQUAL "cxx_return_type_deduction" ) - message( "${feature}" ) - set( has_cxx_return_type_deduction TRUE ) - endif() - - if( feature STREQUAL "cxx_variable_templates" ) - message( "${feature}" ) - set( has_cxx_variable_templates TRUE ) - endif() - -endforeach() - -if( has_cxx_std_14 AND - has_cxx_aggregate_default_initializers AND - has_cxx_attribute_deprecated AND - has_cxx_binary_literals AND - has_cxx_contextual_conversions AND - has_cxx_decltype_auto AND - has_cxx_digit_separators AND - has_cxx_generic_lambdas AND - has_cxx_lambda_init_captures AND - has_cxx_relaxed_constexpr AND - has_cxx_return_type_deduction AND - has_cxx_variable_templates ) - - message( "We have full C++14 support." ) - set( has_full_cxx14 TRUE ) - -endif() - -if( NOT has_full_cxx14 ) - message( FATAL_ERROR "The compiler DOES NOT support all features of C++14!." ) -endif() - \ No newline at end of file diff --git a/packaging/wheel/has_c++14.sh b/packaging/wheel/has_c++14.sh deleted file mode 100755 index 25c883fc988..00000000000 --- a/packaging/wheel/has_c++14.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -mkdir has_c++14.tmp -cp packaging/wheel/TestCXX14.txt has_c++14.tmp/CMakeLists.txt -cd has_c++14.tmp -mkdir build -cd build -cmake3 .. -has_cxx14=$? -cd ../.. -rm -rf has_c++14.tmp -exit $has_cxx14 \ No newline at end of file diff --git a/packaging/wheel/install.sh b/packaging/wheel/install.sh deleted file mode 100755 index 897b21cee13..00000000000 --- a/packaging/wheel/install.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -startdir="$(pwd)" -mkdir xrootdbuild -cd xrootdbuild - -# build only client -# build python bindings -# set install prefix -# set the respective version of python -# replace the default BINDIR with a custom one that can be easily removed afterwards -# (for the python bindings we don't want to install the binaries) -CMAKE_ARGS="-DPYPI_BUILD=TRUE -DXRDCL_LIB_ONLY=TRUE -DENABLE_PYTHON=TRUE -DCMAKE_INSTALL_PREFIX=$1/pyxrootd -DXRD_PYTHON_REQ_VERSION=$2 -DCMAKE_INSTALL_BINDIR=$startdir/xrootdbuild/bin" - -if [ "$5" = "true" ]; then - source /opt/rh/devtoolset-7/enable -fi - -cmake_path=$4 -$cmake_path .. $CMAKE_ARGS - -res=$? -if [ "$res" -ne "0" ]; then - exit 1 -fi - -make -j8 -res=$? -if [ "$res" -ne "0" ]; then - exit 1 -fi - -cd src -make install -res=$? -if [ "$res" -ne "0" ]; then - exit 1 -fi - -cd ../bindings/python - -# Determine if shutil.which is available for a modern Python package install -# (shutil.which was added in Python 3.3, so any version of Python 3 now will have it) -# TODO: Drop support for Python 3.3 and simplify to pip approach -${6} -c 'from shutil import which' &> /dev/null # $6 holds the python sys.executable -shutil_which_available=$? -if [ "${shutil_which_available}" -ne "0" ]; then - ${6} setup.py install ${3} - res=$? -else - ${6} -m pip install ${3} . - res=$? -fi -unset shutil_which_available - -if [ "$res" -ne "0" ]; then - exit 1 -fi - -cd $startdir -rm -r xrootdbuild diff --git a/packaging/wheel/setup.py b/packaging/wheel/setup.py deleted file mode 100644 index 29be651cee2..00000000000 --- a/packaging/wheel/setup.py +++ /dev/null @@ -1,188 +0,0 @@ -from setuptools import setup -from setuptools.command.install import install -from setuptools.command.sdist import sdist -from wheel.bdist_wheel import bdist_wheel - -# shutil.which was added in Python 3.3 -# c.f. https://docs.python.org/3/library/shutil.html#shutil.which -try: - from shutil import which -except ImportError: - from distutils.spawn import find_executable as which - -import subprocess -import sys - -def get_version(): - try: - version = open('VERSION').read().strip() - except: - from datetime import date - version = 'v5.6-rc' + date.today().strftime("%Y%m%d") - - return version - -def binary_exists(name): - """Check whether `name` is on PATH.""" - return which(name) is not None - -def check_cmake3(path): - args = (path, "--version") - popen = subprocess.Popen(args, stdout=subprocess.PIPE) - popen.wait() - output = popen.stdout.read().decode("utf-8") - prefix_len = len( "cmake version " ) - version = output[prefix_len:].split( '.' ) - return int( version[0] ) >= 3 - -def cmake_exists(): - """Check whether CMAKE is on PATH.""" - path = which('cmake') - if path is not None: - if check_cmake3(path): return True, path - path = which('cmake3') - return path is not None, path - -def is_rhel7(): - """check if we are running on rhel7 platform""" - try: - f = open( '/etc/redhat-release', "r" ) - txt = f.read().split() - i = txt.index( 'release' ) + 1 - return txt[i][0] == '7' - except IOError: - return False - except ValueError: - return False - -def has_devtoolset(): - """check if devtoolset-7 is installed""" - import subprocess - args = ( "/usr/bin/rpm", "-q", "devtoolset-7-gcc-c++" ) - popen = subprocess.Popen(args, stdout=subprocess.PIPE) - rc = popen.wait() - return rc == 0 - -def has_cxx14(): - """check if C++ compiler supports C++14""" - import subprocess - popen = subprocess.Popen("./has_c++14.sh", stdout=subprocess.PIPE) - rc = popen.wait() - return rc == 0 - -# def python_dependency_name( py_version_short, py_version_nodot ): -# """ find the name of python dependency """ -# # this is the path to default python -# path = which( 'python' ) -# from os.path import realpath -# # this is the real default python after resolving symlinks -# real = realpath(path) -# index = real.find( 'python' ) + len( 'python' ) -# # this is the version of default python -# defaultpy = real[index:] -# if defaultpy != py_version_short: -# return 'python' + py_version_nodot -# return 'python' - -class CustomInstall(install): - def run(self): - - py_version_short = self.config_vars['py_version_short'] - py_version_nodot = self.config_vars['py_version_nodot'] - - cmake_bin, cmake_path = cmake_exists() - make_bin = binary_exists( 'make' ) - comp_bin = binary_exists( 'c++' ) or binary_exists( 'g++' ) or binary_exists( 'clang' ) - - import pkgconfig - zlib_dev = pkgconfig.exists( 'zlib' ) - openssl_dev = pkgconfig.exists( 'openssl' ) - uuid_dev = pkgconfig.exists( 'uuid' ) - - if is_rhel7(): - if has_cxx14(): - devtoolset7 = True # we only care about devtoolset7 if the compiler does not support C++14 - need_devtoolset = "false" - else: - devtoolset7 = has_devtoolset() - need_devtoolset = "true" - else: - devtoolset7 = True # we only care about devtoolset7 on rhel7 - need_devtoolset = "false" - - pyname = None - if py_version_nodot[0] == '3': - python_dev = pkgconfig.exists( 'python3' ) or pkgconfig.exists( 'python' + py_version_nodot ); - pyname = 'python3' - else: - python_dev = pkgconfig.exists( 'python' ); - pyname = 'python' - - missing_dep = not ( cmake_bin and make_bin and comp_bin and zlib_dev and openssl_dev and python_dev and uuid_dev and devtoolset7 ) - - if missing_dep: - print( 'Some dependencies are missing:') - if not cmake_bin: print('\tcmake (version 3) is missing!') - if not make_bin: print('\tmake is missing!') - if not comp_bin: print('\tC++ compiler is missing (g++, c++, clang, etc.)!') - if not zlib_dev: print('\tzlib development package is missing!') - if not openssl_dev: print('\topenssl development package is missing!') - if not python_dev: print('\t{} development package is missing!'.format(pyname) ) - if not uuid_dev: print('\tuuid development package is missing') - if not devtoolset7: print('\tdevtoolset-7-gcc-c++ package is missing') - raise Exception( 'Dependencies missing!' ) - - useropt = '' - command = ['./install.sh'] - if self.user: - prefix = self.install_usersite - useropt = '--user' - else: - prefix = self.install_platlib - command.append(prefix) - command.append( py_version_short ) - command.append( useropt ) - command.append( cmake_path ) - command.append( need_devtoolset ) - command.append( sys.executable ) - rc = subprocess.call(command) - if rc: - raise Exception( 'Install step failed!' ) - - -class CustomDist(sdist): - def write_version_to_file(self): - - version = get_version() - with open('bindings/python/VERSION', 'w') as vi: - vi.write(version) - - def run(self): - self.write_version_to_file() - sdist.run(self) - - -class CustomWheelGen(bdist_wheel): - # Do not generate wheel - def run(self): - pass - -version = get_version() -setup_requires=[ 'pkgconfig' ] - -setup( - name = 'xrootd', - version = version, - author = 'XRootD Developers', - author_email = 'xrootd-dev@slac.stanford.edu', - url = 'http://xrootd.org', - license = 'LGPLv3+', - description = "XRootD with Python bindings", - long_description = "XRootD with Python bindings", - setup_requires = setup_requires, - cmdclass = { - 'install': CustomInstall, - 'sdist': CustomDist, - 'bdist_wheel': CustomWheelGen - } -) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..b0f076532a0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py new file mode 100644 index 00000000000..dac6967face --- /dev/null +++ b/setup.py @@ -0,0 +1,120 @@ +import os +import platform +import subprocess +import sys + +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext +from subprocess import check_call, check_output + +try: + from shutil import which +except ImportError: + from distutils.spawn import find_executable as which + +cmdline_args = [] + +for arg in sys.argv: + if arg.startswith('-D'): + cmdline_args.append(arg) + +for arg in cmdline_args: + sys.argv.remove(arg) + +cmake = which("cmake3") or which("cmake") + +def get_version(): + try: + version = open('VERSION').read().strip() + + if version.startswith('$'): + output = check_output(['git', 'describe']) + version = output.decode().strip() + except: + version = None + pass + + if version is None: + from datetime import date + version = '5.6-rc' + date.today().strftime("%Y%m%d") + + if version.startswith('v'): + version = version[1:] + + # Sanitize version to conform to PEP 440 + # https://www.python.org/dev/peps/pep-0440 + version = version.replace('-rc', 'rc') + version = version.replace('-g', '+git.') + version = version.replace('-', '.post', 1) + version = version.replace('-', '.') + + return version + +class CMakeExtension(Extension): + def __init__(self, name, src='.', sources=[], **kwa): + Extension.__init__(self, name, sources=sources, **kwa) + self.src = os.path.abspath(src) + +class CMakeBuild(build_ext): + def build_extensions(self): + if cmake is None: + raise RuntimeError('Cannot find CMake executable') + + for ext in self.extensions: + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + + # Use $ORIGIN RPATH to ensure that the client library can load + # libXrdCl which will be installed in the same directory. Build + # with install RPATH because libraries are installed by Python/pip + # not CMake, so CMake cannot fix the install RPATH later on. + + cmake_args = [ + '-DPython_EXECUTABLE={}'.format(sys.executable), + '-DCMAKE_INSTALL_RPATH=$ORIGIN', + '-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE', + '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}/{}'.format(self.build_temp, ext.name), + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}/{}'.format(extdir, ext.name), + '-DENABLE_PYTHON=1', '-DENABLE_XRDCL=1', '-DXRDCL_LIB_ONLY=1', '-DPYPI_BUILD=1' + ] + + cmake_args += cmdline_args + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + check_call([cmake, ext.src, '-B', self.build_temp] + cmake_args) + check_call([cmake, '--build', self.build_temp, '--parallel']) + +version = get_version() + +setup(name='xrootd', + version=version, + description='eXtended ROOT daemon', + author='XRootD Developers', + author_email='xrootd-dev@slac.stanford.edu', + url='http://xrootd.org', + download_url='https://github.com/xrootd/xrootd/archive/v%s.tar.gz' % version, + keywords=['XRootD', 'network filesystem'], + license='LGPLv3+', + long_description=open('README').read(), + long_description_content_type='text/plain', + packages = ['XRootD', 'XRootD.client', 'pyxrootd'], + package_dir = { + 'pyxrootd' : 'bindings/python/src', + 'XRootD' : 'bindings/python/libs', + 'XRootD/client': 'bindings/python/libs/client', + }, + ext_modules= [ CMakeExtension('pyxrootd') ], + cmdclass={ 'build_ext': CMakeBuild }, + zip_safe=False, + classifiers=[ + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + "Operating System :: Unix", + "Programming Language :: C++", + "Programming Language :: Python", + ] + )