Add 'limited_api' kwarg to python.extension_module

This commit adds a new keyword arg to extension_module() that enables
a user to target the Python Limited API, declaring the version of the
limited API that they wish to target.

A new unittest has been added to test this functionality.

data/test.schema.json

@@ -26,9 +26,11 @@
               "exe",
               "shared_lib",
               "python_lib",
+              "python_limited_lib",
               "pdb",
               "implib",
               "py_implib",
+              "py_limited_implib",
               "implibempty",
               "expr"
             ]

docs/markdown/Python-module.md

@@ -101,6 +101,10 @@ the addition of the following:
   `/usr/lib/site-packages`. When subdir is passed to this method,
   it will be appended to that location. This keyword argument is
   mutually exclusive with `install_dir`
+- `limited_api`: *since 1.3.0* A string containing the Python version
+  of the [Py_LIMITED_API](C-https://docs.python.org/3/c-api/stable.html) that
+  the extension targets. For example, '3.7' to target Python 3.7's version of
+  the limited API.
 
 Additionally, the following diverge from [[shared_module]]'s default behavior:
 

docs/markdown/snippets/python_extension_module_limited_api.md

@@ -0,0 +1,5 @@
+## Support targeting Python's limited C API
+
+The Python module's `extension_module` function has gained the ability
+to build extensions which target Python's limited C API via a new keyword
+argument: `limited_api`.

mesonbuild/dependencies/python.py

@@ -44,6 +44,7 @@ class PythonIntrospectionDict(TypedDict):
         paths: T.Dict[str, str]
         platform: str
         suffix: str
+        limited_api_suffix: str
         variables: T.Dict[str, str]
         version: str
 
@@ -94,6 +95,7 @@ def __init__(self, name: str, command: T.Optional[T.List[str]] = None,
             'paths': {},
             'platform': 'sentinel',
             'suffix': 'sentinel',
+            'limited_api_suffix': 'sentinel',
             'variables': {},
             'version': '0.0',
         }
@@ -197,7 +199,7 @@ def __init__(self, name: str, environment: 'Environment',
         if self.link_libpython:
             # link args
             if mesonlib.is_windows():
-                self.find_libpy_windows(environment)
+                self.find_libpy_windows(environment, limited_api=False)
             else:
                 self.find_libpy(environment)
         else:
@@ -259,7 +261,7 @@ def get_windows_python_arch(self) -> T.Optional[str]:
         mlog.log(f'Unknown Windows Python platform {self.platform!r}')
         return None
 
-    def get_windows_link_args(self) -> T.Optional[T.List[str]]:
+    def get_windows_link_args(self, limited_api: bool) -> T.Optional[T.List[str]]:
         if self.platform.startswith('win'):
             vernum = self.variables.get('py_version_nodot')
             verdot = self.variables.get('py_version_short')
@@ -277,6 +279,8 @@ def get_windows_link_args(self) -> T.Optional[T.List[str]]:
                     else:
                         libpath = Path(f'python{vernum}.dll')
                 else:
+                    if limited_api:
+                        vernum = vernum[0]
                     libpath = Path('libs') / f'python{vernum}.lib'
                     # For a debug build, pyconfig.h may force linking with
                     # pythonX_d.lib (see meson#10776). This cannot be avoided
@@ -317,7 +321,7 @@ def get_windows_link_args(self) -> T.Optional[T.List[str]]:
             return None
         return [str(lib)]
 
-    def find_libpy_windows(self, env: 'Environment') -> None:
+    def find_libpy_windows(self, env: 'Environment', limited_api: bool = False) -> None:
         '''
         Find python3 libraries on Windows and also verify that the arch matches
         what we are building for.
@@ -332,7 +336,7 @@ def find_libpy_windows(self, env: 'Environment') -> None:
             self.is_found = False
             return
         # This can fail if the library is not found
-        largs = self.get_windows_link_args()
+        largs = self.get_windows_link_args(limited_api)
         if largs is None:
             self.is_found = False
             return

mesonbuild/modules/python.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 from __future__ import annotations
 
-import copy, json, os, shutil
+import copy, json, os, shutil, re
 import typing as T
 
 from . import ExtensionModule, ModuleInfo
@@ -32,8 +32,9 @@
     InvalidArguments, typed_pos_args, typed_kwargs, KwargInfo,
     FeatureNew, FeatureNewKwargs, disablerIfNotFound
 )
-from ..mesonlib import MachineChoice
+from ..mesonlib import MachineChoice, OptionKey
 from ..programs import ExternalProgram, NonExistingExternalProgram
+from ..compilers.detect import detect_c_compiler
 
 if T.TYPE_CHECKING:
     from typing_extensions import TypedDict, NotRequired
@@ -62,8 +63,7 @@ class ExtensionModuleKw(SharedModuleKw):
 
         subdir: NotRequired[T.Optional[str]]
 
-
-mod_kwargs = {'subdir'}
+mod_kwargs = {'subdir', 'limited_api'}
 mod_kwargs.update(known_shmod_kwargs)
 mod_kwargs -= {'name_prefix', 'name_suffix'}
 
@@ -112,6 +112,7 @@ def _get_path(self, state: T.Optional['ModuleState'], key: str) -> None:
 
 _PURE_KW = KwargInfo('pure', (bool, NoneType))
 _SUBDIR_KW = KwargInfo('subdir', str, default='')
+_LIMITED_API_KW = KwargInfo('limited_api', str, default='', since='1.3.0')
 _DEFAULTABLE_SUBDIR_KW = KwargInfo('subdir', (str, NoneType))
 
 class PythonInstallation(ExternalProgramHolder):
@@ -122,6 +123,7 @@ def __init__(self, python: 'PythonExternalProgram', interpreter: 'Interpreter'):
         assert isinstance(prefix, str), 'for mypy'
         self.variables = info['variables']
         self.suffix = info['suffix']
+        self.limited_api_suffix = info['limited_api_suffix']
         self.paths = info['paths']
         self.pure = python.pure
         self.platlib_install_path = os.path.join(prefix, python.platlib)
@@ -146,7 +148,7 @@ def __init__(self, python: 'PythonExternalProgram', interpreter: 'Interpreter'):
 
     @permittedKwargs(mod_kwargs)
     @typed_pos_args('python.extension_module', str, varargs=(str, mesonlib.File, CustomTarget, CustomTargetIndex, GeneratedList, StructuredSources, ExtractedObjects, BuildTarget))
-    @typed_kwargs('python.extension_module', *_MOD_KWARGS, _DEFAULTABLE_SUBDIR_KW, allow_unknown=True)
+    @typed_kwargs('python.extension_module', *_MOD_KWARGS, _DEFAULTABLE_SUBDIR_KW, _LIMITED_API_KW, allow_unknown=True)
     def extension_module_method(self, args: T.Tuple[str, T.List[BuildTargetSource]], kwargs: ExtensionModuleKw) -> 'SharedModule':
         if 'install_dir' in kwargs:
             if kwargs['subdir'] is not None:
@@ -159,32 +161,90 @@ def extension_module_method(self, args: T.Tuple[str, T.List[BuildTargetSource]],
 
             kwargs['install_dir'] = self._get_install_dir_impl(False, subdir)
 
+        target_suffix = self.suffix
+
         new_deps = mesonlib.extract_as_list(kwargs, 'dependencies')
-        has_pydep = any(isinstance(dep, _PythonDependencyBase) for dep in new_deps)
-        if not has_pydep:
+        pydep = next((dep for dep in new_deps if isinstance(dep, _PythonDependencyBase)), None)
+        if pydep is None:
             pydep = self._dependency_method_impl({})
             if not pydep.found():
                 raise mesonlib.MesonException('Python dependency not found')
             new_deps.append(pydep)
             FeatureNew.single_use('python_installation.extension_module with implicit dependency on python',
                                   '0.63.0', self.subproject, 'use python_installation.dependency()',
                                   self.current_node)
+
+        limited_api_version = kwargs.pop('limited_api')
+        if limited_api_version != '':
+
+            target_suffix = self.limited_api_suffix
+
+            limited_api_version_hex = self._convert_api_version_to_py_version_hex(limited_api_version)
+            if mesonlib.version_compare(limited_api_version, '<3.2'):
+                raise InvalidArguments(f'Python Limited API version invalid: {limited_api_version} (must be greater than 3.2)')
+            if mesonlib.version_compare(limited_api_version, '>' + pydep.version):
+                raise InvalidArguments(f'Python Limited API version too high: {limited_api_version} (detected {pydep.version})')
+
+            new_c_args = mesonlib.extract_as_list(kwargs, 'c_args')
+            new_c_args.append(f'-DPy_LIMITED_API={limited_api_version_hex}')
+            kwargs['c_args'] = new_c_args
+
+            # When compiled under MSVC, Python's PC/pyconfig.h forcibly inserts pythonMAJOR.MINOR.lib
+            # into the linker path when not running in debug mode via a series #pragma comment(lib, "")
+            # directives. We manually override these here as this interferes with the intended
+            # use of the 'limited_api' kwarg
+            c_compiler = detect_c_compiler(self.interpreter.environment, MachineChoice.BUILD)
+            if c_compiler.get_id() == 'msvc':
+                pydep_copy = copy.copy(pydep)
+                pydep_copy.find_libpy_windows(self.env, limited_api=True)
+                if not pydep_copy.found():
+                    raise mesonlib.MesonException('Python dependency supporting limited API not found')
+
+                new_deps.remove(pydep)
+                new_deps.append(pydep_copy)
+
+                pyver = pydep.version.replace('.', '')
+                python_windows_debug_link_exception = f'/NODEFAULTLIB:python{pyver}_d.lib'
+                python_windows_release_link_exception = f'/NODEFAULTLIB:python{pyver}.lib'
+
+                new_link_args = mesonlib.extract_as_list(kwargs, 'link_args')
+
+                is_debug = self.interpreter.environment.coredata.options[OptionKey('debug')].value
+                if is_debug:
+                    new_link_args.append(python_windows_debug_link_exception)
+                else:
+                    new_link_args.append(python_windows_release_link_exception)
+
+                kwargs['link_args'] = new_link_args
+
         kwargs['dependencies'] = new_deps
 
         # msys2's python3 has "-cpython-36m.dll", we have to be clever
         # FIXME: explain what the specific cleverness is here
-        split, suffix = self.suffix.rsplit('.', 1)
+        split, target_suffix = target_suffix.rsplit('.', 1)
         args = (args[0] + split, args[1])
 
         kwargs['name_prefix'] = ''
-        kwargs['name_suffix'] = suffix
+        kwargs['name_suffix'] = target_suffix
 
         if 'gnu_symbol_visibility' not in kwargs and \
                 (self.is_pypy or mesonlib.version_compare(self.version, '>=3.9')):
             kwargs['gnu_symbol_visibility'] = 'inlineshidden'
 
         return self.interpreter.build_target(self.current_node, args, kwargs, SharedModule)
 
+    def _convert_api_version_to_py_version_hex(self, api_version: str) -> str:
+        python_api_version_format = re.compile(r'[0-9]\.[0-9]{1,2}')
+        decimal_match = python_api_version_format.fullmatch(api_version)
+        if not decimal_match:
+            raise InvalidArguments(f'Python API version invalid: "{api_version}".')
+
+        version_components = api_version.split('.')
+        major = int(version_components[0])
+        minor = int(version_components[1])
+
+        return '0x{:02x}{:02x}0000'.format(major, minor)
+
     def _dependency_method_impl(self, kwargs: TYPE_kwargs) -> Dependency:
         for_machine = self.interpreter.machine_from_native_kwarg(kwargs)
         identifier = get_dep_identifier(self._full_path(), kwargs)

mesonbuild/scripts/python_info.py

@@ -65,6 +65,20 @@ def links_against_libpython():
 else:
     suffix = variables.get('EXT_SUFFIX')
 
+limited_api_suffix = None
+if sys.version_info >= (3, 2):
+    try:
+        from importlib.machinery import EXTENSION_SUFFIXES
+        limited_api_suffix = EXTENSION_SUFFIXES[1]
+    except Exception:
+        pass
+
+# pypy supports modules targetting the limited api but
+# does not use a special suffix to distinguish them:
+# https://doc.pypy.org/en/latest/cpython_differences.html#permitted-abi-tags-in-extensions
+if '__pypy__' in sys.builtin_module_names:
+    limited_api_suffix = suffix
+
 print(json.dumps({
   'variables': variables,
   'paths': paths,
@@ -76,4 +90,5 @@ def links_against_libpython():
   'is_venv': sys.prefix != variables['base_prefix'],
   'link_libpython': links_against_libpython(),
   'suffix': suffix,
+  'limited_api_suffix': limited_api_suffix
 }))

run_project_tests.py

@@ -148,7 +148,7 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
             canonical_compiler = 'msvc'
 
         python_suffix = python.info['suffix']
-
+        python_limited_suffix = python.info['limited_api_suffix']
         has_pdb = False
         if self.language in {'c', 'cpp'}:
             has_pdb = canonical_compiler == 'msvc'
@@ -167,7 +167,7 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
             return None
 
         # Handle the different types
-        if self.typ in {'py_implib', 'python_lib', 'python_file'}:
+        if self.typ in {'py_implib', 'py_limited_implib', 'python_lib', 'python_limited_lib', 'python_file'}:
             val = p.as_posix()
             val = val.replace('@PYTHON_PLATLIB@', python.platlib)
             val = val.replace('@PYTHON_PURELIB@', python.purelib)
@@ -176,6 +176,8 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
                 return p
             if self.typ == 'python_lib':
                 return p.with_suffix(python_suffix)
+            if self.typ == 'python_limited_lib':
+                return p.with_suffix(python_limited_suffix)
             if self.typ == 'py_implib':
                 p = p.with_suffix(python_suffix)
                 if env.machines.host.is_windows() and canonical_compiler == 'msvc':
@@ -184,6 +186,14 @@ def get_path(self, compiler: str, env: environment.Environment) -> T.Optional[Pa
                     return p.with_suffix('.dll.a')
                 else:
                     return None
+            if self.typ == 'py_limited_implib':
+                p = p.with_suffix(python_limited_suffix)
+                if env.machines.host.is_windows() and canonical_compiler == 'msvc':
+                    return p.with_suffix('.lib')
+                elif env.machines.host.is_windows() or env.machines.host.is_cygwin():
+                    return p.with_suffix('.dll.a')
+                else:
+                    return None
         elif self.typ in {'file', 'dir'}:
             return p
         elif self.typ == 'shared_lib':

test cases/python/9 extmodule limited api/limited.c

@@ -0,0 +1,19 @@
+#include <Python.h>
+
+#ifndef Py_LIMITED_API
+#error Py_LIMITED_API must be defined.
+#elif Py_LIMITED_API != 0x03070000
+#error Wrong value for Py_LIMITED_API
+#endif
+
+static struct PyModuleDef limited_module = {
+   PyModuleDef_HEAD_INIT,
+   "limited_api_test",
+   NULL,
+   -1,
+   NULL
+};
+
+PyMODINIT_FUNC PyInit_limited(void) {
+    return PyModule_Create(&limited_module);
+}

test cases/python/9 extmodule limited api/meson.build

@@ -0,0 +1,16 @@
+project('Python limited api', 'c',
+  default_options : ['buildtype=release', 'werror=true'])
+
+py_mod = import('python')
+py = py_mod.find_installation()
+
+ext_mod_limited = py.extension_module('limited',
+  'limited.c',
+  limited_api: '3.7',
+  install: true,
+)
+
+ext_mod = py.extension_module('not_limited',
+  'not_limited.c',
+  install: true,
+)

test cases/python/9 extmodule limited api/not_limited.c

@@ -0,0 +1,17 @@
+#include <Python.h>
+
+#ifdef Py_LIMITED_API
+#error Py_LIMITED_API must not be defined.
+#endif
+
+static struct PyModuleDef not_limited_module = {
+   PyModuleDef_HEAD_INIT,
+   "not_limited_api_test",
+   NULL,
+   -1,
+   NULL
+};
+
+PyMODINIT_FUNC PyInit_not_limited(void) {
+    return PyModule_Create(&not_limited_module);
+}

test cases/python/9 extmodule limited api/test.json

@@ -0,0 +1,8 @@
+{
+  "installed": [
+    {"type": "python_limited_lib", "file": "usr/@PYTHON_PLATLIB@/limited"},
+    {"type": "py_limited_implib", "file": "usr/@PYTHON_PLATLIB@/limited"},
+    {"type": "python_lib", "file": "usr/@PYTHON_PLATLIB@/not_limited"},
+    {"type": "py_implib", "file": "usr/@PYTHON_PLATLIB@/not_limited"}
+  ]
+}