diff --git a/data/test.schema.json b/data/test.schema.json index a809388b4798..98ae44eeb983 100644 --- a/data/test.schema.json +++ b/data/test.schema.json @@ -26,9 +26,11 @@ "exe", "shared_lib", "python_lib", + "python_limited_lib", "pdb", "implib", "py_implib", + "py_limited_implib", "implibempty", "expr" ] diff --git a/docs/markdown/Python-module.md b/docs/markdown/Python-module.md index f67262abfd4e..d011b227533f 100644 --- a/docs/markdown/Python-module.md +++ b/docs/markdown/Python-module.md @@ -101,6 +101,10 @@ the addition of the following: `/usr/lib/site-packages`. When subdir is passed to this method, it will be appended to that location. This keyword argument is mutually exclusive with `install_dir` +- `limited_api`: *since 1.3.0* A string containing the Python version + of the [Py_LIMITED_API](C-https://docs.python.org/3/c-api/stable.html) that + the extension targets. For example, '3.7' to target Python 3.7's version of + the limited API. Additionally, the following diverge from [[shared_module]]'s default behavior: diff --git a/docs/markdown/snippets/python_extension_module_limited_api.md b/docs/markdown/snippets/python_extension_module_limited_api.md new file mode 100644 index 000000000000..f5da9699d9ce --- /dev/null +++ b/docs/markdown/snippets/python_extension_module_limited_api.md @@ -0,0 +1,5 @@ +## Support targeting Python's limited C API + +The Python module's `extension_module` function has gained the ability +to build extensions which target Python's limited C API via a new keyword +argument: `limited_api`. diff --git a/mesonbuild/dependencies/python.py b/mesonbuild/dependencies/python.py index 1607728883df..efb904eca471 100644 --- a/mesonbuild/dependencies/python.py +++ b/mesonbuild/dependencies/python.py @@ -44,6 +44,7 @@ class PythonIntrospectionDict(TypedDict): paths: T.Dict[str, str] platform: str suffix: str + limited_api_suffix: str variables: T.Dict[str, str] version: str @@ -94,6 +95,7 @@ def __init__(self, name: str, command: T.Optional[T.List[str]] = None, 'paths': {}, 'platform': 'sentinel', 'suffix': 'sentinel', + 'limited_api_suffix': 'sentinel', 'variables': {}, 'version': '0.0', } @@ -197,7 +199,7 @@ def __init__(self, name: str, environment: 'Environment', if self.link_libpython: # link args if mesonlib.is_windows(): - self.find_libpy_windows(environment) + self.find_libpy_windows(environment, limited_api=False) else: self.find_libpy(environment) else: @@ -259,7 +261,7 @@ def get_windows_python_arch(self) -> T.Optional[str]: mlog.log(f'Unknown Windows Python platform {self.platform!r}') return None - def get_windows_link_args(self) -> T.Optional[T.List[str]]: + def get_windows_link_args(self, limited_api: bool) -> T.Optional[T.List[str]]: if self.platform.startswith('win'): vernum = self.variables.get('py_version_nodot') verdot = self.variables.get('py_version_short') @@ -277,6 +279,8 @@ def get_windows_link_args(self) -> T.Optional[T.List[str]]: else: libpath = Path(f'python{vernum}.dll') else: + if limited_api: + vernum = vernum[0] libpath = Path('libs') / f'python{vernum}.lib' # For a debug build, pyconfig.h may force linking with # pythonX_d.lib (see meson#10776). This cannot be avoided @@ -317,7 +321,7 @@ def get_windows_link_args(self) -> T.Optional[T.List[str]]: return None return [str(lib)] - def find_libpy_windows(self, env: 'Environment') -> None: + def find_libpy_windows(self, env: 'Environment', limited_api: bool = False) -> None: ''' Find python3 libraries on Windows and also verify that the arch matches what we are building for. @@ -332,7 +336,7 @@ def find_libpy_windows(self, env: 'Environment') -> None: self.is_found = False return # This can fail if the library is not found - largs = self.get_windows_link_args() + largs = self.get_windows_link_args(limited_api) if largs is None: self.is_found = False return diff --git a/mesonbuild/modules/python.py b/mesonbuild/modules/python.py index 879c5487c729..1d4ae8858414 100644 --- a/mesonbuild/modules/python.py +++ b/mesonbuild/modules/python.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import annotations -import copy, json, os, shutil +import copy, json, os, shutil, re import typing as T from . import ExtensionModule, ModuleInfo @@ -32,8 +32,9 @@ InvalidArguments, typed_pos_args, typed_kwargs, KwargInfo, FeatureNew, FeatureNewKwargs, disablerIfNotFound ) -from ..mesonlib import MachineChoice +from ..mesonlib import MachineChoice, OptionKey from ..programs import ExternalProgram, NonExistingExternalProgram +from ..compilers.detect import detect_c_compiler if T.TYPE_CHECKING: from typing_extensions import TypedDict, NotRequired @@ -62,8 +63,7 @@ class ExtensionModuleKw(SharedModuleKw): subdir: NotRequired[T.Optional[str]] - -mod_kwargs = {'subdir'} +mod_kwargs = {'subdir', 'limited_api'} mod_kwargs.update(known_shmod_kwargs) mod_kwargs -= {'name_prefix', 'name_suffix'} @@ -112,6 +112,7 @@ def _get_path(self, state: T.Optional['ModuleState'], key: str) -> None: _PURE_KW = KwargInfo('pure', (bool, NoneType)) _SUBDIR_KW = KwargInfo('subdir', str, default='') +_LIMITED_API_KW = KwargInfo('limited_api', str, default='', since='1.3.0') _DEFAULTABLE_SUBDIR_KW = KwargInfo('subdir', (str, NoneType)) class PythonInstallation(ExternalProgramHolder): @@ -122,6 +123,7 @@ def __init__(self, python: 'PythonExternalProgram', interpreter: 'Interpreter'): assert isinstance(prefix, str), 'for mypy' self.variables = info['variables'] self.suffix = info['suffix'] + self.limited_api_suffix = info['limited_api_suffix'] self.paths = info['paths'] self.pure = python.pure self.platlib_install_path = os.path.join(prefix, python.platlib) @@ -146,7 +148,7 @@ def __init__(self, python: 'PythonExternalProgram', interpreter: 'Interpreter'): @permittedKwargs(mod_kwargs) @typed_pos_args('python.extension_module', str, varargs=(str, mesonlib.File, CustomTarget, CustomTargetIndex, GeneratedList, StructuredSources, ExtractedObjects, BuildTarget)) - @typed_kwargs('python.extension_module', *_MOD_KWARGS, _DEFAULTABLE_SUBDIR_KW, allow_unknown=True) + @typed_kwargs('python.extension_module', *_MOD_KWARGS, _DEFAULTABLE_SUBDIR_KW, _LIMITED_API_KW, allow_unknown=True) def extension_module_method(self, args: T.Tuple[str, T.List[BuildTargetSource]], kwargs: ExtensionModuleKw) -> 'SharedModule': if 'install_dir' in kwargs: if kwargs['subdir'] is not None: @@ -159,9 +161,11 @@ def extension_module_method(self, args: T.Tuple[str, T.List[BuildTargetSource]], kwargs['install_dir'] = self._get_install_dir_impl(False, subdir) + target_suffix = self.suffix + new_deps = mesonlib.extract_as_list(kwargs, 'dependencies') - has_pydep = any(isinstance(dep, _PythonDependencyBase) for dep in new_deps) - if not has_pydep: + pydep = next((dep for dep in new_deps if isinstance(dep, _PythonDependencyBase)), None) + if pydep is None: pydep = self._dependency_method_impl({}) if not pydep.found(): raise mesonlib.MesonException('Python dependency not found') @@ -169,15 +173,59 @@ def extension_module_method(self, args: T.Tuple[str, T.List[BuildTargetSource]], FeatureNew.single_use('python_installation.extension_module with implicit dependency on python', '0.63.0', self.subproject, 'use python_installation.dependency()', self.current_node) + + limited_api_version = kwargs.pop('limited_api') + if limited_api_version != '': + + target_suffix = self.limited_api_suffix + + limited_api_version_hex = self._convert_api_version_to_py_version_hex(limited_api_version) + if mesonlib.version_compare(limited_api_version, '<3.2'): + raise InvalidArguments(f'Python Limited API version invalid: {limited_api_version} (must be greater than 3.2)') + if mesonlib.version_compare(limited_api_version, '>' + pydep.version): + raise InvalidArguments(f'Python Limited API version too high: {limited_api_version} (detected {pydep.version})') + + new_c_args = mesonlib.extract_as_list(kwargs, 'c_args') + new_c_args.append(f'-DPy_LIMITED_API={limited_api_version_hex}') + kwargs['c_args'] = new_c_args + + # When compiled under MSVC, Python's PC/pyconfig.h forcibly inserts pythonMAJOR.MINOR.lib + # into the linker path when not running in debug mode via a series #pragma comment(lib, "") + # directives. We manually override these here as this interferes with the intended + # use of the 'limited_api' kwarg + c_compiler = detect_c_compiler(self.interpreter.environment, MachineChoice.BUILD) + if c_compiler.get_id() == 'msvc': + pydep_copy = copy.copy(pydep) + pydep_copy.find_libpy_windows(self.env, limited_api=True) + if not pydep_copy.found(): + raise mesonlib.MesonException('Python dependency supporting limited API not found') + + new_deps.remove(pydep) + new_deps.append(pydep_copy) + + pyver = pydep.version.replace('.', '') + python_windows_debug_link_exception = f'/NODEFAULTLIB:python{pyver}_d.lib' + python_windows_release_link_exception = f'/NODEFAULTLIB:python{pyver}.lib' + + new_link_args = mesonlib.extract_as_list(kwargs, 'link_args') + + is_debug = self.interpreter.environment.coredata.options[OptionKey('debug')].value + if is_debug: + new_link_args.append(python_windows_debug_link_exception) + else: + new_link_args.append(python_windows_release_link_exception) + + kwargs['link_args'] = new_link_args + kwargs['dependencies'] = new_deps # msys2's python3 has "-cpython-36m.dll", we have to be clever # FIXME: explain what the specific cleverness is here - split, suffix = self.suffix.rsplit('.', 1) + split, target_suffix = target_suffix.rsplit('.', 1) args = (args[0] + split, args[1]) kwargs['name_prefix'] = '' - kwargs['name_suffix'] = suffix + kwargs['name_suffix'] = target_suffix if 'gnu_symbol_visibility' not in kwargs and \ (self.is_pypy or mesonlib.version_compare(self.version, '>=3.9')): @@ -185,6 +233,18 @@ def extension_module_method(self, args: T.Tuple[str, T.List[BuildTargetSource]], return self.interpreter.build_target(self.current_node, args, kwargs, SharedModule) + def _convert_api_version_to_py_version_hex(self, api_version: str) -> str: + python_api_version_format = re.compile(r'[0-9]\.[0-9]{1,2}') + decimal_match = python_api_version_format.fullmatch(api_version) + if not decimal_match: + raise InvalidArguments(f'Python API version invalid: "{api_version}".') + + version_components = api_version.split('.') + major = int(version_components[0]) + minor = int(version_components[1]) + + return '0x{:02x}{:02x}0000'.format(major, minor) + def _dependency_method_impl(self, kwargs: TYPE_kwargs) -> Dependency: for_machine = self.interpreter.machine_from_native_kwarg(kwargs) identifier = get_dep_identifier(self._full_path(), kwargs) diff --git a/mesonbuild/scripts/python_info.py b/mesonbuild/scripts/python_info.py index 9c3a0791ac90..e877dc746031 100755 --- a/mesonbuild/scripts/python_info.py +++ b/mesonbuild/scripts/python_info.py @@ -65,6 +65,20 @@ def links_against_libpython(): else: suffix = variables.get('EXT_SUFFIX') +limited_api_suffix = None +if sys.version_info >= (3, 2): + try: + from importlib.machinery import EXTENSION_SUFFIXES + limited_api_suffix = EXTENSION_SUFFIXES[1] + except Exception: + pass + +# pypy supports modules targetting the limited api but +# does not use a special suffix to distinguish them: +# https://doc.pypy.org/en/latest/cpython_differences.html#permitted-abi-tags-in-extensions +if '__pypy__' in sys.builtin_module_names: + limited_api_suffix = suffix + print(json.dumps({ 'variables': variables, 'paths': paths, @@ -76,4 +90,5 @@ def links_against_libpython(): 'is_venv': sys.prefix != variables['base_prefix'], 'link_libpython': links_against_libpython(), 'suffix': suffix, + 'limited_api_suffix': limited_api_suffix })) diff --git a/run_project_tests.py b/run_project_tests.py index facf1e98f6af..27020caef9fe 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -148,7 +148,7 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa canonical_compiler = 'msvc' python_suffix = python.info['suffix'] - + python_limited_suffix = python.info['limited_api_suffix'] has_pdb = False if self.language in {'c', 'cpp'}: has_pdb = canonical_compiler == 'msvc' @@ -167,7 +167,7 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa return None # Handle the different types - if self.typ in {'py_implib', 'python_lib', 'python_file'}: + if self.typ in {'py_implib', 'py_limited_implib', 'python_lib', 'python_limited_lib', 'python_file'}: val = p.as_posix() val = val.replace('@PYTHON_PLATLIB@', python.platlib) val = val.replace('@PYTHON_PURELIB@', python.purelib) @@ -176,6 +176,8 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa return p if self.typ == 'python_lib': return p.with_suffix(python_suffix) + if self.typ == 'python_limited_lib': + return p.with_suffix(python_limited_suffix) if self.typ == 'py_implib': p = p.with_suffix(python_suffix) if env.machines.host.is_windows() and canonical_compiler == 'msvc': @@ -184,6 +186,14 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa return p.with_suffix('.dll.a') else: return None + if self.typ == 'py_limited_implib': + p = p.with_suffix(python_limited_suffix) + if env.machines.host.is_windows() and canonical_compiler == 'msvc': + return p.with_suffix('.lib') + elif env.machines.host.is_windows() or env.machines.host.is_cygwin(): + return p.with_suffix('.dll.a') + else: + return None elif self.typ in {'file', 'dir'}: return p elif self.typ == 'shared_lib': diff --git a/test cases/python/9 extmodule limited api/limited.c b/test cases/python/9 extmodule limited api/limited.c new file mode 100644 index 000000000000..0d1c718200ba --- /dev/null +++ b/test cases/python/9 extmodule limited api/limited.c @@ -0,0 +1,19 @@ +#include + +#ifndef Py_LIMITED_API +#error Py_LIMITED_API must be defined. +#elif Py_LIMITED_API != 0x03070000 +#error Wrong value for Py_LIMITED_API +#endif + +static struct PyModuleDef limited_module = { + PyModuleDef_HEAD_INIT, + "limited_api_test", + NULL, + -1, + NULL +}; + +PyMODINIT_FUNC PyInit_limited(void) { + return PyModule_Create(&limited_module); +} diff --git a/test cases/python/9 extmodule limited api/meson.build b/test cases/python/9 extmodule limited api/meson.build new file mode 100644 index 000000000000..68afc96996cb --- /dev/null +++ b/test cases/python/9 extmodule limited api/meson.build @@ -0,0 +1,16 @@ +project('Python limited api', 'c', + default_options : ['buildtype=release', 'werror=true']) + +py_mod = import('python') +py = py_mod.find_installation() + +ext_mod_limited = py.extension_module('limited', + 'limited.c', + limited_api: '3.7', + install: true, +) + +ext_mod = py.extension_module('not_limited', + 'not_limited.c', + install: true, +) diff --git a/test cases/python/9 extmodule limited api/not_limited.c b/test cases/python/9 extmodule limited api/not_limited.c new file mode 100644 index 000000000000..169c33b24eef --- /dev/null +++ b/test cases/python/9 extmodule limited api/not_limited.c @@ -0,0 +1,17 @@ +#include + +#ifdef Py_LIMITED_API +#error Py_LIMITED_API must not be defined. +#endif + +static struct PyModuleDef not_limited_module = { + PyModuleDef_HEAD_INIT, + "not_limited_api_test", + NULL, + -1, + NULL +}; + +PyMODINIT_FUNC PyInit_not_limited(void) { + return PyModule_Create(¬_limited_module); +} diff --git a/test cases/python/9 extmodule limited api/test.json b/test cases/python/9 extmodule limited api/test.json new file mode 100644 index 000000000000..06a170623858 --- /dev/null +++ b/test cases/python/9 extmodule limited api/test.json @@ -0,0 +1,8 @@ +{ + "installed": [ + {"type": "python_limited_lib", "file": "usr/@PYTHON_PLATLIB@/limited"}, + {"type": "py_limited_implib", "file": "usr/@PYTHON_PLATLIB@/limited"}, + {"type": "python_lib", "file": "usr/@PYTHON_PLATLIB@/not_limited"}, + {"type": "py_implib", "file": "usr/@PYTHON_PLATLIB@/not_limited"} + ] +}