From c6f5c5ddcc021099febae7cb1194fedea01fd056 Mon Sep 17 00:00:00 2001 From: Matus Valo Date: Mon, 20 Dec 2021 17:40:05 +0100 Subject: [PATCH] Make "new_build_ext" the new "build_ext" (GH-4498) This also solves a difficulty with the Cython import in setuptools' build_ext. We need to inherit from the one in distutils, so that setuptools can inherit from us. That leads to a circular dependency that goes either way depending on which gets imported first by users, and in what way (from-import or module import). This is built to match the code in https://github.com/pypa/setuptools/blob/9f1822ee910df3df930a98ab99f66d18bb70659b/setuptools/command/build_ext.py#L14-L21 Closes https://github.com/cython/cython/issues/3541 --- Cython/Distutils/build_ext.py | 139 ++++++++++++++---- .../source_files_and_compilation.rst | 2 +- pyximport/pyxbuild.py | 2 +- setup.py | 4 +- .../build/build_ext_cython_c_in_temp.srctree | 30 ++++ tests/build/build_ext_cython_cplus.srctree | 34 +++++ .../build_ext_cython_include_dirs.srctree | 50 +++++++ 7 files changed, 231 insertions(+), 30 deletions(-) create mode 100644 tests/build/build_ext_cython_c_in_temp.srctree create mode 100644 tests/build/build_ext_cython_cplus.srctree create mode 100644 tests/build/build_ext_cython_include_dirs.srctree diff --git a/Cython/Distutils/build_ext.py b/Cython/Distutils/build_ext.py index efedf59780b..0ab979b8399 100644 --- a/Cython/Distutils/build_ext.py +++ b/Cython/Distutils/build_ext.py @@ -1,43 +1,130 @@ import sys +import os -if 'setuptools' in sys.modules: - try: - from setuptools.command.build_ext import build_ext as _build_ext - except ImportError: - # We may be in the process of importing setuptools, which tries - # to import this. - from distutils.command.build_ext import build_ext as _build_ext -else: - from distutils.command.build_ext import build_ext as _build_ext +try: + from __builtin__ import basestring +except ImportError: + basestring = str + +# Always inherit from the "build_ext" in distutils since setuptools already imports +# it from Cython if available, and does the proper distutils fallback otherwise. +# https://github.com/pypa/setuptools/blob/9f1822ee910df3df930a98ab99f66d18bb70659b/setuptools/command/build_ext.py#L16 +# setuptools imports Cython's "build_ext", so make sure we go first. +_build_ext_module = sys.modules.get('setuptools.command.build_ext') +if _build_ext_module is None: + import distutils.command.build_ext as _build_ext_module + +# setuptools remembers the original distutils "build_ext" as "_du_build_ext" +_build_ext = getattr(_build_ext_module, '_du_build_ext', None) +if _build_ext is None: + _build_ext = getattr(_build_ext_module, 'build_ext', None) +if _build_ext is None: + from distutils.command.build_ext import build_ext as _build_ext -class new_build_ext(_build_ext, object): - user_options = _build_ext.user_options[:] - boolean_options = _build_ext.boolean_options[:] +class build_ext(_build_ext, object): - user_options.extend([ + user_options = _build_ext.user_options + [ + ('cython-cplus', None, + "generate C++ source files"), + ('cython-create-listing', None, + "write errors to a listing file"), + ('cython-line-directives', None, + "emit source line directives"), + ('cython-include-dirs=', None, + "path to the Cython include files" + _build_ext.sep_by), ('cython-c-in-temp', None, "put generated C files in temp directory"), - ]) + ('cython-gen-pxi', None, + "generate .pxi file for public declarations"), + ('cython-directives=', None, + "compiler directive overrides"), + ('cython-gdb', None, + "generate debug information for cygdb"), + ('cython-compile-time-env', None, + "cython compile time environment"), + ] - boolean_options.extend([ - 'cython-c-in-temp' - ]) + boolean_options = _build_ext.boolean_options + [ + 'cython-cplus', 'cython-create-listing', 'cython-line-directives', + 'cython-c-in-temp', 'cython-gdb', + ] def initialize_options(self): - _build_ext.initialize_options(self) + super(build_ext, self).initialize_options() + self.cython_cplus = 0 + self.cython_create_listing = 0 + self.cython_line_directives = 0 + self.cython_include_dirs = None + self.cython_directives = None self.cython_c_in_temp = 0 + self.cython_gen_pxi = 0 + self.cython_gdb = False + self.cython_compile_time_env = None + + def finalize_options(self): + super(build_ext, self).finalize_options() + if self.cython_include_dirs is None: + self.cython_include_dirs = [] + elif isinstance(self.cython_include_dirs, basestring): + self.cython_include_dirs = \ + self.cython_include_dirs.split(os.pathsep) + if self.cython_directives is None: + self.cython_directives = {} + + def get_extension_attr(self, extension, option_name, default=False): + return getattr(self, option_name) or getattr(extension, option_name, default) def build_extension(self, ext): from Cython.Build.Dependencies import cythonize - if self.cython_c_in_temp: - build_dir = self.build_temp - else: - build_dir = None - new_ext = cythonize(ext,force=self.force, quiet=self.verbose == 0, build_dir=build_dir)[0] + + # Set up the include_path for the Cython compiler: + # 1. Start with the command line option. + # 2. Add in any (unique) paths from the extension + # cython_include_dirs (if Cython.Distutils.extension is used). + # 3. Add in any (unique) paths from the extension include_dirs + includes = list(self.cython_include_dirs) + for include_dir in getattr(ext, 'cython_include_dirs', []): + if include_dir not in includes: + includes.append(include_dir) + + # In case extension.include_dirs is a generator, evaluate it and keep + # result + ext.include_dirs = list(ext.include_dirs) + for include_dir in ext.include_dirs + list(self.include_dirs): + if include_dir not in includes: + includes.append(include_dir) + + # Set up Cython compiler directives: + # 1. Start with the command line option. + # 2. Add in any (unique) entries from the extension + # cython_directives (if Cython.Distutils.extension is used). + directives = dict(self.cython_directives) + if hasattr(ext, "cython_directives"): + directives.update(ext.cython_directives) + + if self.get_extension_attr(ext, 'cython_cplus'): + ext.language = 'c++' + + options = { + 'use_listing_file': self.get_extension_attr(ext, 'cython_create_listing'), + 'emit_linenums': self.get_extension_attr(ext, 'cython_line_directives'), + 'include_path': includes, + 'compiler_directives': directives, + 'build_dir': self.build_temp if self.get_extension_attr(ext, 'cython_c_in_temp') else None, + 'generate_pxi': self.get_extension_attr(ext, 'cython_gen_pxi'), + 'gdb_debug': self.get_extension_attr(ext, 'cython_gdb'), + 'c_line_in_traceback': not getattr(ext, 'no_c_in_traceback', 0), + 'compile_time_env': self.get_extension_attr(ext, 'cython_compile_time_env', default=None), + } + + new_ext = cythonize( + ext,force=self.force, quiet=self.verbose == 0, **options + )[0] + ext.sources = new_ext.sources - super(new_build_ext, self).build_extension(ext) + super(build_ext, self).build_extension(ext) -# This will become new_build_ext in the future. -from .old_build_ext import old_build_ext as build_ext +# backward compatibility +new_build_ext = build_ext diff --git a/docs/src/userguide/source_files_and_compilation.rst b/docs/src/userguide/source_files_and_compilation.rst index edc5b1d22f0..b5855609772 100644 --- a/docs/src/userguide/source_files_and_compilation.rst +++ b/docs/src/userguide/source_files_and_compilation.rst @@ -415,7 +415,7 @@ Cython's build_ext module which runs ``cythonize`` as part of the build process: setup( extensions = [Extension("*", ["*.pyx"])], - cmdclass={'build_ext': Cython.Build.new_build_ext}, + cmdclass={'build_ext': Cython.Build.build_ext}, ... ) diff --git a/pyximport/pyxbuild.py b/pyximport/pyxbuild.py index ee3dabf410e..61f9747eec4 100644 --- a/pyximport/pyxbuild.py +++ b/pyximport/pyxbuild.py @@ -10,7 +10,7 @@ from distutils.extension import Extension from distutils.util import grok_environment_error try: - from Cython.Distutils.build_ext import new_build_ext as build_ext + from Cython.Distutils.build_ext import build_ext HAS_CYTHON = True except ImportError: HAS_CYTHON = False diff --git a/setup.py b/setup.py index 1d3fb96126e..f2cbec14be8 100755 --- a/setup.py +++ b/setup.py @@ -159,7 +159,7 @@ def compile_cython_modules(profile=False, coverage=False, compile_more=False, cy # XXX hack around setuptools quirk for '*.pyx' sources extensions[-1].sources[0] = pyx_source_file - from Cython.Distutils.build_ext import new_build_ext + from Cython.Distutils.build_ext import build_ext from Cython.Compiler.Options import get_directive_defaults get_directive_defaults().update( language_level=2, @@ -175,7 +175,7 @@ def compile_cython_modules(profile=False, coverage=False, compile_more=False, cy sys.stderr.write("Enabled line tracing and profiling for the Cython binary modules\n") # not using cythonize() directly to let distutils decide whether building extensions was requested - add_command_class("build_ext", new_build_ext) + add_command_class("build_ext", build_ext) setup_args['ext_modules'] = extensions diff --git a/tests/build/build_ext_cython_c_in_temp.srctree b/tests/build/build_ext_cython_c_in_temp.srctree new file mode 100644 index 00000000000..bbc89a3dead --- /dev/null +++ b/tests/build/build_ext_cython_c_in_temp.srctree @@ -0,0 +1,30 @@ + +PYTHON setup.py build_ext --inplace --cython-c-in-temp +PYTHON -c 'import mymodule; assert mymodule.test_string == "TEST"' +PYTHON check_paths.py + +############# setup.py ############# + +from Cython.Distutils.extension import Extension +from Cython.Build import build_ext +from distutils.core import setup + +setup( + name='Hello world app', + ext_modules = [ + Extension( + name = 'mymodule', + sources=['mymodule.pyx'], + ) + ], + cmdclass={'build_ext': build_ext}, +) + +######## mymodule.pyx ######## + +test_string = "TEST" + +######## check_paths.py ######## + +import os +assert not os.path.exists("mymodule.c") diff --git a/tests/build/build_ext_cython_cplus.srctree b/tests/build/build_ext_cython_cplus.srctree new file mode 100644 index 00000000000..73a61df04cd --- /dev/null +++ b/tests/build/build_ext_cython_cplus.srctree @@ -0,0 +1,34 @@ +# tag: cpp + +PYTHON setup.py build_ext --inplace --cython-cplus +PYTHON -c "import a; a.use_vector([1,2,3])" + +######## setup.py ######## + +from Cython.Distutils.extension import Extension +from Cython.Build import build_ext +from distutils.core import setup + +setup( + name='Hello world app', + ext_modules = [ + Extension( + name = 'a', + sources=['a.pyx'], + ) + ], + cmdclass={'build_ext': build_ext}, +) + +######## a.pyx ######## + +from libcpp.vector cimport vector + +def use_vector(L): + try: + v = new vector[int]() + for a in L: + v.push_back(a) + return v.size() + finally: + del v diff --git a/tests/build/build_ext_cython_include_dirs.srctree b/tests/build/build_ext_cython_include_dirs.srctree new file mode 100644 index 00000000000..8af883720fa --- /dev/null +++ b/tests/build/build_ext_cython_include_dirs.srctree @@ -0,0 +1,50 @@ + +PYTHON setup.py build_ext --inplace --cython-include-dirs=./headers1 --include-dirs=./headers2 +PYTHON -c 'import mymodule; assert mymodule.test_string == "TEST"; assert mymodule.header_value1 == 1; assert mymodule.header_value2 == 2; assert mymodule.header_value3 == 3; assert mymodule.header_value4 == 4' + +############# setup.py ############# + +from Cython.Distutils.extension import Extension +from Cython.Build import build_ext +from distutils.core import setup + +setup( + name='Hello world app', + ext_modules = [ + Extension( + name = 'mymodule', + sources=['mymodule.pyx'], + cython_include_dirs=['headers3'], + include_dirs=['./headers4'] + ) + ], + cmdclass={'build_ext': build_ext}, +) + +######## mymodule.pyx ######## + +include "myheader1.pxi" +include "myheader2.pxi" +include "myheader3.pxi" +include "myheader4.pxi" +header_value1 = test_value1 +header_value2 = test_value2 +header_value3 = test_value3 +header_value4 = test_value4 +test_string = "TEST" + +######## headers1/myheader1.pxi ######## + +cdef int test_value1 = 1 + +######## headers2/myheader2.pxi ######## + +cdef int test_value2 = 2 + +######## headers3/myheader3.pxi ######## + +cdef int test_value3 = 3 + +######## headers4/myheader4.pxi ######## + +cdef int test_value4 = 4