diff --git a/.gitignore b/.gitignore index af85c9a39..5598c22d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .mesonpy-native-file.ini +.mesonpy/ docs/_build *.pyc .cache/ diff --git a/docs/index.rst b/docs/index.rst index ecf43aac6..d9ed3df2d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,6 +49,56 @@ frontend, like ``builddir``, to specify the build directory to use/re-use. You can find more information about them in the `build options page`_. +Editable installs +----------------- + +Editable installs allow you to install the project in such a way where you can +edit the project source and have those changes be reflected in the installed +module, without having to explicitly rebuild and reinstall. + +The way it works on ``meson-python`` specifically, is that when you import a +module of the project, it will be rebuilt on the fly. This means that project +imports will take more time than usual. + +You can use pip to install the project in editable mode. + + +.. code-block:: + + python -m pip install -e . + + +It might be helpful to see the output of the Meson_ commands. This is offered +as a **provisional** feature, meaning it is subject to change. + +If you want to temporarily enable the output, you can set the +``MESONPY_EDITABLE_VERBOSE`` environment variable to a non-empty value. If this +environment variable is present during import, the Meson_ commands and their +output will be printed. + + +.. code-block:: + + MESONPY_EDITABLE_VERBOSE=1 python my_script.py + + + +This behavior can also be enabled by default by passing the ``editable-verbose`` +config setting when installing the project. + + +.. code-block:: + + python -m pip install -e . --config-settings editable-verbose=true + + +This way, you won't need to always set ``MESONPY_EDITABLE_VERBOSE`` environment +variable, the Meson_ commands and their output will always be printed. + +The ``MESONPY_EDITABLE_VERBOSE`` won't have any effect during the project +install step. + + How does it work? ================= diff --git a/meson.build b/meson.build index 9018de9ba..831c72107 100644 --- a/meson.build +++ b/meson.build @@ -10,7 +10,9 @@ endif py.install_sources( 'mesonpy/__init__.py', 'mesonpy/_compat.py', + 'mesonpy/_editable.py', 'mesonpy/_elf.py', + 'mesonpy/_introspection.py', 'mesonpy/_tags.py', 'mesonpy/_util.py', 'mesonpy/_wheelfile.py', diff --git a/mesonpy/__init__.py b/mesonpy/__init__.py index ac8901c50..44a05b021 100644 --- a/mesonpy/__init__.py +++ b/mesonpy/__init__.py @@ -42,14 +42,21 @@ else: import tomllib +if sys.version_info >= (3, 10): + import importlib.resources as importlib_resources +else: + import importlib_resources + import mesonpy._compat import mesonpy._elf +import mesonpy._introspection import mesonpy._tags import mesonpy._util import mesonpy._wheelfile from mesonpy._compat import ( - Collection, Iterator, Literal, Mapping, ParamSpec, Path, typing_get_args + Collection, Iterable, Iterator, Literal, Mapping, ParamSpec, Path, + cached_property, typing_get_args ) @@ -57,12 +64,6 @@ import pyproject_metadata # noqa: F401 -if sys.version_info >= (3, 8): - from functools import cached_property -else: - cached_property = lambda x: property(functools.lru_cache(maxsize=None)(x)) # noqa: E731 - - __version__ = '0.13.0.dev0' @@ -107,7 +108,7 @@ def _init_colors() -> Dict[str, str]: _STYLES = _init_colors() # holds the color values, should be _COLORS or _NO_COLORS -_EXTENSION_SUFFIXES = frozenset(importlib.machinery.EXTENSION_SUFFIXES) +_EXTENSION_SUFFIXES = importlib.machinery.EXTENSION_SUFFIXES.copy() _EXTENSION_SUFFIX_REGEX = re.compile(r'^\.(?:(?P[^.]+)\.)?(?:so|pyd|dll)$') assert all(re.match(_EXTENSION_SUFFIX_REGEX, x) for x in _EXTENSION_SUFFIXES) @@ -138,6 +139,16 @@ def _setup_cli() -> None: colorama.init() # fix colors on windows +def _as_python_declaration(value: Any) -> str: + if isinstance(value, str): + return f"r'{value}'" + elif isinstance(value, os.PathLike): + return _as_python_declaration(os.fspath(value)) + elif isinstance(value, Iterable): + return '[' + ', '.join(map(_as_python_declaration, value)) + ']' + raise NotImplementedError(f'Unsupported type: {type(value)}') + + class Error(RuntimeError): def __str__(self) -> str: return str(self.args[0]) @@ -198,11 +209,15 @@ def _has_extension_modules(self) -> bool: # Assume that all code installed in {platlib} is Python ABI dependent. return bool(self._wheel_files['platlib']) + @property + def normalized_name(self) -> str: + return self._project.name.replace('-', '_') + @property def basename(self) -> str: """Normalized wheel name and version (eg. meson_python-1.0.0).""" return '{distribution}-{version}'.format( - distribution=self._project.name.replace('-', '_'), + distribution=self.normalized_name, version=self._project.version, ) @@ -284,19 +299,6 @@ def entrypoints_txt(self) -> bytes: return text.encode() - @property - def _debian_python(self) -> bool: - """Check if we are running on Debian-patched Python.""" - try: - import distutils - try: - import distutils.command.install - except ModuleNotFoundError: - raise ModuleNotFoundError('Unable to import distutils, please install python3-distutils') - return 'deb_system' in distutils.command.install.INSTALL_SCHEMES - except ModuleNotFoundError: - return False - @cached_property def _stable_abi(self) -> Optional[str]: """Determine stabe ABI compatibility. @@ -334,6 +336,32 @@ def _stable_abi(self) -> Optional[str]: return stable[0] return None + @property + def top_level_modules(self) -> Collection[str]: + modules = set() + for type_ in self._wheel_files: + for path, _ in self._wheel_files[type_]: + top_part = path.parts[0] + # file module + if top_part.endswith('.py'): + modules.add(top_part[:-3]) + else: + # native module + for extension in _EXTENSION_SUFFIXES: + if top_part.endswith(extension): + modules.add(top_part[:-len(extension)]) + # XXX: We assume the order in _EXTENSION_SUFFIXES + # goes from more specific to last, so we go + # with the first match we find. + break + else: # nobreak + # skip Windows import libraries + if top_part.endswith('.a'): + continue + # package module + modules.add(top_part) + return modules + def _is_native(self, file: Union[str, pathlib.Path]) -> bool: """Check if file is a native file.""" self._project.build() # the project needs to be built for this :/ @@ -384,11 +412,9 @@ def _map_from_heuristics(self, origin: pathlib.Path, destination: pathlib.Path) origin file and the Meson destination path. """ warnings.warn('Using heuristics to map files to wheel, this may result in incorrect locations') - sys_vars = sysconfig.get_config_vars().copy() - sys_vars['base'] = sys_vars['platbase'] = sys.base_prefix - sys_paths = sysconfig.get_paths(vars=sys_vars) + sys_paths = mesonpy._introspection.SYSCONFIG_PATHS # Try to map to Debian dist-packages - if self._debian_python: + if mesonpy._introspection.DEBIAN_PYTHON: search_path = origin while search_path != search_path.parent: search_path = search_path.parent @@ -472,7 +498,7 @@ def _install_path( Some files might need to be fixed up to set the RPATH to the internal library directory on Linux wheels for eg. """ - location = os.fspath(destination).replace(os.path.sep, '/') + location = destination.as_posix() counter.update(location) # fix file @@ -503,24 +529,27 @@ def _install_path( wheel_file.write(origin, location) + def _wheel_write_metadata(self, whl: mesonpy._wheelfile.WheelFile) -> None: + # add metadata + whl.writestr(f'{self.distinfo_dir}/METADATA', self._project.metadata) + whl.writestr(f'{self.distinfo_dir}/WHEEL', self.wheel) + if self.entrypoints_txt: + whl.writestr(f'{self.distinfo_dir}/entry_points.txt', self.entrypoints_txt) + + # add license (see https://github.com/mesonbuild/meson-python/issues/88) + if self._project.license_file: + whl.write( + self._source_dir / self._project.license_file, + f'{self.distinfo_dir}/{os.path.basename(self._project.license_file)}', + ) + def build(self, directory: Path) -> pathlib.Path: self._project.build() # ensure project is built wheel_file = pathlib.Path(directory, f'{self.name}.whl') with mesonpy._wheelfile.WheelFile(wheel_file, 'w') as whl: - # add metadata - whl.writestr(f'{self.distinfo_dir}/METADATA', self._project.metadata) - whl.writestr(f'{self.distinfo_dir}/WHEEL', self.wheel) - if self.entrypoints_txt: - whl.writestr(f'{self.distinfo_dir}/entry_points.txt', self.entrypoints_txt) - - # add license (see https://github.com/mesonbuild/meson-python/issues/88) - if self._project.license_file: - whl.write( - self._source_dir / self._project.license_file, - f'{self.distinfo_dir}/{os.path.basename(self._project.license_file)}', - ) + self._wheel_write_metadata(whl) print('{light_blue}{bold}Copying files to wheel...{reset}'.format(**_STYLES)) with mesonpy._util.cli_counter( @@ -547,6 +576,64 @@ def build(self, directory: Path) -> pathlib.Path: return wheel_file + def build_editable(self, directory: Path, verbose: bool = False) -> pathlib.Path: + self._project.build() # ensure project is built + + wheel_file = pathlib.Path(directory, f'{self.name}.whl') + + install_path = self._source_dir / '.mesonpy' / 'editable' / 'install' + rebuild_commands = self._project.build_commands(install_path) + + import_paths = set() + for name, raw_path in mesonpy._introspection.SYSCONFIG_PATHS.items(): + if name not in ('purelib', 'platlib'): + continue + path = pathlib.Path(raw_path) + import_paths.add(install_path / path.relative_to(path.anchor)) + + install_path.mkdir(parents=True, exist_ok=True) + + with mesonpy._wheelfile.WheelFile(wheel_file, 'w') as whl: + self._wheel_write_metadata(whl) + whl.writestr( + f'{self.distinfo_dir}/direct_url.json', + self._source_dir.as_uri().encode(), + ) + + # install hook module + hook_module_name = f'_mesonpy_hook_{self.normalized_name.replace(".", "_")}' + hook_install_code = textwrap.dedent(f''' + MesonpyFinder.install( + project_name={_as_python_declaration(self._project.name)}, + hook_name={_as_python_declaration(hook_module_name)}, + project_path={_as_python_declaration(self._source_dir)}, + build_path={_as_python_declaration(self._build_dir)}, + import_paths={_as_python_declaration(import_paths)}, + top_level_modules={_as_python_declaration(self.top_level_modules)}, + rebuild_commands={_as_python_declaration(rebuild_commands)}, + verbose={verbose}, + ) + ''').strip().encode() + whl.writestr( + f'{hook_module_name}.py', + (importlib_resources.files('mesonpy') / '_editable.py').read_bytes() + hook_install_code, + ) + # install .pth file + whl.writestr( + f'{self.normalized_name}-editable-hook.pth', + f'import {hook_module_name}'.encode(), + ) + + # install non-code schemes + for scheme in self._SCHEME_MAP: + if scheme in ('purelib', 'platlib', 'mesonpy-libs'): + continue + for destination, origin in self._wheel_files[scheme]: + destination = pathlib.Path(self.data_dir, scheme, destination) + whl.write(origin, destination.as_posix()) + + return wheel_file + MesonArgsKeys = Literal['dist', 'setup', 'compile', 'install'] MesonArgs = Mapping[MesonArgsKeys, List[str]] @@ -566,10 +653,12 @@ def __init__( # noqa: C901 working_dir: Path, build_dir: Optional[Path] = None, meson_args: Optional[MesonArgs] = None, + editable_verbose: bool = False, ) -> None: self._source_dir = pathlib.Path(source_dir).absolute() self._working_dir = pathlib.Path(working_dir).absolute() self._build_dir = pathlib.Path(build_dir).absolute() if build_dir else (self._working_dir / 'build') + self._editable_verbose = editable_verbose self._install_dir = self._working_dir / 'install' self._meson_native_file = self._source_dir / '.mesonpy-native-file.ini' self._meson_cross_file = self._source_dir / '.mesonpy-cross-file.ini' @@ -643,8 +732,8 @@ def __init__( # noqa: C901 self._meson_args[key].extend(value) # make sure the build dir exists - self._build_dir.mkdir(exist_ok=True) - self._install_dir.mkdir(exist_ok=True) + self._build_dir.mkdir(exist_ok=True, parents=True) + self._install_dir.mkdir(exist_ok=True, parents=True) # write the native file native_file_data = textwrap.dedent(f''' @@ -683,14 +772,13 @@ def _get_config_key(self, key: str) -> Any: def _proc(self, *args: str) -> None: """Invoke a subprocess.""" print('{cyan}{bold}+ {}{reset}'.format(' '.join(args), **_STYLES)) - r = subprocess.run(list(args), env=self._env) + r = subprocess.run(list(args), env=self._env, cwd=self._build_dir) if r.returncode != 0: raise SystemExit(r.returncode) def _meson(self, *args: str) -> None: """Invoke Meson.""" - with mesonpy._util.chdir(self._build_dir): - return self._proc('meson', *args) + return self._proc('meson', *args) def _configure(self, reconfigure: bool = False) -> None: """Configure Meson project. @@ -698,6 +786,7 @@ def _configure(self, reconfigure: bool = False) -> None: We will try to reconfigure the build directory if possible to avoid expensive rebuilds. """ + sys_paths = mesonpy._introspection.SYSCONFIG_PATHS setup_args = [ f'--prefix={sys.base_prefix}', os.fspath(self._source_dir), @@ -706,6 +795,15 @@ def _configure(self, reconfigure: bool = False) -> None: # TODO: Allow configuring these arguments '-Ddebug=false', '-Doptimization=2', + + # XXX: This should not be needed, but Meson is using the wrong paths + # in some scenarios, like on macOS. + # https://github.com/mesonbuild/meson-python/pull/87#discussion_r1047041306 + '--python.purelibdir', + sys_paths['purelib'], + '--python.platlibdir', + sys_paths['platlib'], + # user args *self._meson_args['setup'], ] @@ -766,11 +864,24 @@ def _wheel_builder(self) -> _WheelBuilder: self._copy_files, ) + def build_commands(self, install_dir: Optional[pathlib.Path] = None) -> Sequence[Sequence[str]]: + return ( + ('meson', 'compile', *self._meson_args['compile'],), + ( + 'meson', + 'install', + '--only-changed', + '--destdir', + os.fspath(install_dir or self._install_dir), + *self._meson_args['install'], + ), + ) + @functools.lru_cache(maxsize=None) def build(self) -> None: """Trigger the Meson build.""" - self._meson('compile', *self._meson_args['compile'],) - self._meson('install', '--destdir', os.fspath(self._install_dir), *self._meson_args['install'],) + for cmd in self.build_commands(): + self._meson(*cmd[1:]) @classmethod @contextlib.contextmanager @@ -779,10 +890,11 @@ def with_temp_working_dir( source_dir: Path = os.path.curdir, build_dir: Optional[Path] = None, meson_args: Optional[MesonArgs] = None, + editable_verbose: bool = False, ) -> Iterator[Project]: """Creates a project instance pointing to a temporary working directory.""" with tempfile.TemporaryDirectory(prefix='.mesonpy-', dir=os.fspath(source_dir)) as tmpdir: - yield cls(source_dir, tmpdir, build_dir, meson_args) + yield cls(source_dir, tmpdir, build_dir, meson_args, editable_verbose) @functools.lru_cache() def _info(self, name: str) -> Dict[str, Any]: @@ -855,16 +967,23 @@ def metadata(self) -> bytes: """Project metadata.""" # the rest of the keys are only available when using PEP 621 metadata if not self.pep621: - return textwrap.dedent(f''' + data = textwrap.dedent(f''' Metadata-Version: 2.1 Name: {self.name} Version: {self.version} - ''').strip().encode() + ''').strip() + return data.encode() + # re-import pyproject_metadata to raise ModuleNotFoundError if it is really missing import pyproject_metadata # noqa: F401, F811 assert self._metadata - # use self.version as the version may be dynamic -- fetched from Meson + core_metadata = self._metadata.as_rfc822() + # use self.version as the version may be dynamic -- fetched from Meson + # + # we need to overwrite this field in the RFC822 field as + # pyproject_metadata removes 'version' from the dynamic fields when + # giving it a value via the dataclass core_metadata.headers['Version'] = [self.version] return bytes(core_metadata) @@ -950,13 +1069,16 @@ def sdist(self, directory: Path) -> pathlib.Path: return sdist - def wheel(self, directory: Path) -> pathlib.Path: # noqa: F811 + def wheel(self, directory: Path) -> pathlib.Path: """Generates a wheel (binary distribution) in the specified directory.""" - wheel = self._wheel_builder.build(self._build_dir) + file = self._wheel_builder.build(directory) + assert isinstance(file, pathlib.Path) + return file - final_wheel = pathlib.Path(directory, wheel.name) - shutil.move(os.fspath(wheel), final_wheel) - return final_wheel + def editable(self, directory: Path) -> pathlib.Path: + file = self._wheel_builder.build_editable(directory, self._editable_verbose) + assert isinstance(file, pathlib.Path) + return file @contextlib.contextmanager @@ -995,7 +1117,7 @@ def _validate_string_collection(key: str) -> None: meson_args_cli_keys = tuple(f'{key}-args' for key in meson_args_keys) for key in config_settings: - known_keys = ('builddir', *meson_args_cli_keys) + known_keys = ('builddir', 'editable-verbose', *meson_args_cli_keys) if key not in known_keys: import difflib matches = difflib.get_close_matches(key, known_keys, n=3) @@ -1014,6 +1136,7 @@ def _validate_string_collection(key: str) -> None: key: config_settings.get(f'{key}-args', ()) for key in meson_args_keys }), + editable_verbose=bool(config_settings.get('editable-verbose')) ) as project: yield project @@ -1114,3 +1237,29 @@ def build_wheel( out = pathlib.Path(wheel_directory) with _project(config_settings) as project: return project.wheel(out).name + + +@_pyproject_hook +def build_editable( + wheel_directory: str, + config_settings: Optional[Dict[Any, Any]] = None, + metadata_directory: Optional[str] = None, +) -> str: + _setup_cli() + + # force set a permanent builddir + if not config_settings: + config_settings = {} + if 'builddir' not in config_settings: + config_settings['builddir'] = os.path.join('.mesonpy', 'editable', 'build') + + out = pathlib.Path(wheel_directory) + with _project(config_settings) as project: + return project.editable(out).name + + +@_pyproject_hook +def get_requires_for_build_editable( + config_settings: Optional[Dict[str, str]] = None, +) -> List[str]: + return get_requires_for_build_wheel() diff --git a/mesonpy/_compat.py b/mesonpy/_compat.py index 852dbeade..4db7bec32 100644 --- a/mesonpy/_compat.py +++ b/mesonpy/_compat.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: 2021 Quansight, LLC # SPDX-FileCopyrightText: 2021 Filipe Laíns +import functools import os import pathlib import sys @@ -30,6 +31,12 @@ from typing_extensions import get_args as typing_get_args +if sys.version_info >= (3, 8): + from functools import cached_property +else: + cached_property = lambda x: property(functools.lru_cache(maxsize=None)(x)) # noqa: E731 + + Path = Union[str, os.PathLike] @@ -43,6 +50,7 @@ def is_relative_to(path: pathlib.Path, other: Union[pathlib.Path, str]) -> bool: __all__ = [ + 'cached_property', 'is_relative_to', 'typing_get_args', 'Collection', diff --git a/mesonpy/_editable.py b/mesonpy/_editable.py new file mode 100644 index 000000000..f3690c3ec --- /dev/null +++ b/mesonpy/_editable.py @@ -0,0 +1,180 @@ +import functools +import importlib.abc +import os +import subprocess +import sys +import warnings + +from types import ModuleType +from typing import List, Mapping, Optional, Union + + +if sys.version_info >= (3, 9): + from collections.abc import Sequence +else: + from typing import Sequence + + +# This file should be standalone! +# It is copied during the editable hook installation. + + +_COLORS = { + 'cyan': '\33[36m', + 'yellow': '\33[93m', + 'light_blue': '\33[94m', + 'bold': '\33[1m', + 'dim': '\33[2m', + 'underline': '\33[4m', + 'reset': '\33[0m', +} +_NO_COLORS = {color: '' for color in _COLORS} + + +def _init_colors() -> Mapping[str, str]: + """Detect if we should be using colors in the output. We will enable colors + if running in a TTY, and no environment variable overrides it. Setting the + NO_COLOR (https://no-color.org/) environment variable force-disables colors, + and FORCE_COLOR forces color to be used, which is useful for thing like + Github actions. + """ + if 'NO_COLOR' in os.environ: + if 'FORCE_COLOR' in os.environ: + warnings.warn('Both NO_COLOR and FORCE_COLOR environment variables are set, disabling color') + return _NO_COLORS + elif 'FORCE_COLOR' in os.environ or sys.stdout.isatty(): + return _COLORS + return _NO_COLORS + + +_STYLES = _init_colors() # holds the color values, should be _COLORS or _NO_COLORS + + +class MesonpyFinder(importlib.abc.MetaPathFinder): + """Custom loader that whose purpose is to detect when the import system is + trying to load our modules, and trigger a rebuild. After triggering a + rebuild, we return None in find_spec, letting the normal finders pick up the + modules. + """ + + def __init__( + self, + project_name: str, + hook_name: str, + project_path: str, + build_path: str, + import_paths: List[str], + top_level_modules: List[str], + rebuild_commands: List[List[str]], + verbose: bool = False, + ) -> None: + self._project_name = project_name + self._hook_name = hook_name + self._project_path = project_path + self._build_path = build_path + self._import_paths = import_paths + self._top_level_modules = top_level_modules + self._rebuild_commands = rebuild_commands + self._verbose = verbose + + for path in (self._project_path, self._build_path): + if not os.path.isdir(path): + raise ImportError( + f'{path} is not a directory, but it is required to rebuild ' + f'"{self._project_name}", which is installed in editable ' + 'mode. Please reinstall the project to get it back to ' + 'working condition. If there are any issues uninstalling ' + 'this installation, you can manually remove ' + f'{self._hook_name} and {os.path.basename(__file__)}, ' + f'located in {os.path.dirname(__file__)}.' + ) + + def __repr__(self) -> str: + return f'{self.__class__.__name__}({self._project_path})' + + def _debug(self, msg: str) -> None: + if self._verbose: + print(msg.format(**_STYLES)) + + def _proc(self, command: List[str]) -> None: + # skip editable hook installation in subprocesses, as during the build + # commands the module we are rebuilding might be imported, causing a + # rebuild loop + # see https://github.com/mesonbuild/meson-python/pull/87#issuecomment-1342548894 + env = os.environ.copy() + env['_MESONPY_EDITABLE_SKIP'] = os.pathsep.join(( + env.get('_MESONPY_EDITABLE_SKIP', ''), + self._project_path, + )) + + if self._verbose: + subprocess.check_call(command, cwd=self._build_path, env=env) + else: + subprocess.check_output(command, cwd=self._build_path, env=env) + + @functools.lru_cache(maxsize=1) + def rebuild(self) -> None: + self._debug(f'{{cyan}}{{bold}}+ rebuilding {self._project_path}{{reset}}') + for command in self._rebuild_commands: + self._proc(command) + self._debug('{cyan}{bold}+ successfully rebuilt{reset}') + + def find_spec( + self, + fullname: str, + path: Optional[Sequence[Union[str, bytes]]], + target: Optional[ModuleType] = None, + ) -> None: + # if it's one of our modules, trigger a rebuild + if fullname.split('.', maxsplit=1)[0] in self._top_level_modules: + self.rebuild() + # prepend the project path to sys.path, so that the normal finder + # can find our modules + # we prepend so that our path comes before the current path (if + # the interpreter is run with -m), see gh-239 + if sys.path[:len(self._import_paths)] != self._import_paths: + for path in self._import_paths: + if path in sys.path: + sys.path.remove(path) + sys.path = self._import_paths + sys.path + # return none (meaning we "didn't find" the module) and let the normal + # finders find/import it + return None + + @classmethod + def install( + cls, + project_name: str, + hook_name: str, + project_path: str, + build_path: str, + import_paths: List[str], + top_level_modules: List[str], + rebuild_commands: List[List[str]], + verbose: bool = False, + ) -> None: + if project_path in os.environ.get('_MESONPY_EDITABLE_SKIP', '').split(os.pathsep): + return + if os.environ.get('MESONPY_EDITABLE_VERBOSE', ''): + verbose = True + # install our finder + finder = cls( + project_name, + hook_name, + project_path, + build_path, + import_paths, + top_level_modules, + rebuild_commands, + verbose, + ) + if finder not in sys.meta_path: + # prepend our finder to sys.meta_path, so that it is queried before + # the normal finders, and can trigger a project rebuild + sys.meta_path.insert(0, finder) + # we add the project path to sys.path later, so that we can prepend + # after the current directory is prepended (when -m is used) + # see gh-239 + + +# generated hook install below diff --git a/mesonpy/_introspection.py b/mesonpy/_introspection.py new file mode 100644 index 000000000..1b214df82 --- /dev/null +++ b/mesonpy/_introspection.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: MIT + +import sys +import sysconfig +import warnings + +from mesonpy._compat import Mapping + + +def debian_python() -> bool: + """Check if we are running on Debian-patched Python.""" + if sys.version_info >= (3, 10): + return 'deb_system' in sysconfig.get_scheme_names() + try: + import distutils + try: + import distutils.command.install + except ModuleNotFoundError: + raise ModuleNotFoundError('Unable to import distutils, please install python3-distutils') + return 'deb_system' in distutils.command.install.INSTALL_SCHEMES + except ModuleNotFoundError: + return False + + +DEBIAN_PYTHON = debian_python() + + +def debian_distutils_paths() -> Mapping[str, str]: + # https://ffy00.github.io/blog/02-python-debian-and-the-install-locations/ + assert sys.version_info < (3, 12) and DEBIAN_PYTHON + + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=DeprecationWarning) + import distutils.dist + + distribution = distutils.dist.Distribution() + install_cmd = distribution.get_command_obj('install') + install_cmd.install_layout = 'deb' # type: ignore[union-attr] + install_cmd.finalize_options() # type: ignore[union-attr] + + return { + 'data': install_cmd.install_data, # type: ignore[union-attr] + 'platlib': install_cmd.install_platlib, # type: ignore[union-attr] + 'purelib': install_cmd.install_purelib, # type: ignore[union-attr] + 'scripts': install_cmd.install_scripts, # type: ignore[union-attr] + } + + +def sysconfig_paths() -> Mapping[str, str]: + sys_vars = sysconfig.get_config_vars().copy() + sys_vars['base'] = sys_vars['platbase'] = sys.base_prefix + if DEBIAN_PYTHON: + if sys.version_info >= (3, 10, 3): + return sysconfig.get_paths('deb_system', vars=sys_vars) + else: + return debian_distutils_paths() + return sysconfig.get_paths(vars=sys_vars) + + +SYSCONFIG_PATHS = sysconfig_paths() + + +__all__ = [ + 'DEBIAN_PYTHON', + 'SYSCONFIG_PATHS', +] diff --git a/mesonpy/_tags.py b/mesonpy/_tags.py index cdfa68094..ff2b7bb38 100644 --- a/mesonpy/_tags.py +++ b/mesonpy/_tags.py @@ -29,7 +29,7 @@ def get_interpreter_tag() -> str: def _get_config_var(name: str, default: Union[str, int, None] = None) -> Union[str, int, None]: - value = sysconfig.get_config_var(name) + value: Union[str, int, None] = sysconfig.get_config_var(name) if value is None: return default return value diff --git a/noxfile.py b/noxfile.py index 8c2f73a98..47ddbb080 100644 --- a/noxfile.py +++ b/noxfile.py @@ -30,7 +30,7 @@ def docs(session): @nox.session(python='3.7') def mypy(session): - session.install('mypy==0.981') + session.install('mypy==0.991') session.run('mypy', '-p', 'mesonpy') diff --git a/pyproject.toml b/pyproject.toml index deb250f10..3d86fa93e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ requires = [ 'pyproject-metadata>=0.6.1', 'tomli>=1.0.0; python_version<"3.11"', 'typing-extensions>=3.7.4; python_version<"3.10"', + 'importlib_resources>=5.0.0; python_version<"3.10"', ] [project] @@ -29,6 +30,7 @@ dependencies = [ 'pyproject-metadata>=0.6.1', # not a hard dependency, only needed for projects that use PEP 621 metadata 'tomli>=1.0.0; python_version<"3.11"', 'typing-extensions>=3.7.4; python_version<"3.10"', + 'importlib_resources>=5.0.0; python_version<"3.10"', ] dynamic = [ @@ -71,6 +73,11 @@ multi_line_output = 5 known_first_party = 'mesonpy' +[tool.coverage.run] +disable_warnings = [ + 'couldnt-parse', +] + [tool.coverage.html] show_contexts = true diff --git a/tests/conftest.py b/tests/conftest.py index 9b91f5b64..aadecddd5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -89,15 +89,17 @@ def ensure_directories(self, env_dir): self.executable = context.env_exe return context + def python(self, *args: str): + return subprocess.check_output([self.executable, *args]).decode() + + def pip(self, *args: str): + return self.python('-m', 'pip', *args) + @pytest.fixture() -def venv(): - path = pathlib.Path(tempfile.mkdtemp(prefix='mesonpy-test-venv-')) - venv = VEnv(path) - try: - yield venv - finally: - shutil.rmtree(path) +def venv(tmp_path_factory): + path = pathlib.Path(tmp_path_factory.mktemp('mesonpy-test-venv')) + return VEnv(path) def generate_package_fixture(package): @@ -124,12 +126,21 @@ def fixture(tmp_path_session): return fixture +def generate_editable_fixture(package): + @pytest.fixture(scope='session') + def fixture(tmp_path_session): + with chdir(package_dir / package), in_git_repo_context(): + return tmp_path_session / mesonpy.build_editable(tmp_path_session) + return fixture + + # inject {package,sdist,wheel}_* fixtures (https://github.com/pytest-dev/pytest/issues/2424) for package in os.listdir(package_dir): normalized = package.replace('-', '_') globals()[f'package_{normalized}'] = generate_package_fixture(package) globals()[f'sdist_{normalized}'] = generate_sdist_fixture(package) globals()[f'wheel_{normalized}'] = generate_wheel_fixture(package) + globals()[f'editable_{normalized}'] = generate_editable_fixture(package) @pytest.fixture(autouse=True, scope='session') diff --git a/tests/packages/imports-itself-during-build/meson.build b/tests/packages/imports-itself-during-build/meson.build new file mode 100644 index 000000000..75a5d14bd --- /dev/null +++ b/tests/packages/imports-itself-during-build/meson.build @@ -0,0 +1,15 @@ +project( + 'imports-itself-during-build', 'c', + version: '1.0.0', +) + +py_mod = import('python') +py = py_mod.find_installation() + +py.install_sources('pure.py') +py.extension_module( + 'plat', 'plat.c', + install: true, +) + +run_command(py, '-c', 'import pure') diff --git a/tests/packages/imports-itself-during-build/plat.c b/tests/packages/imports-itself-during-build/plat.c new file mode 100644 index 000000000..ae22f71b5 --- /dev/null +++ b/tests/packages/imports-itself-during-build/plat.c @@ -0,0 +1,24 @@ +#include + +static PyObject* foo(PyObject* self) +{ + return PyUnicode_FromString("bar"); +} + +static PyMethodDef methods[] = { + {"foo", (PyCFunction)foo, METH_NOARGS, NULL}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, + "plat", + NULL, + -1, + methods, +}; + +PyMODINIT_FUNC PyInit_plat(void) +{ + return PyModule_Create(&module); +} diff --git a/tests/packages/imports-itself-during-build/pure.py b/tests/packages/imports-itself-during-build/pure.py new file mode 100644 index 000000000..d84c8b24a --- /dev/null +++ b/tests/packages/imports-itself-during-build/pure.py @@ -0,0 +1,2 @@ +def foo(): + return 'bar' diff --git a/tests/packages/imports-itself-during-build/pyproject.toml b/tests/packages/imports-itself-during-build/pyproject.toml new file mode 100644 index 000000000..d6f3b6861 --- /dev/null +++ b/tests/packages/imports-itself-during-build/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python'] diff --git a/tests/packages/module-types/file.py b/tests/packages/module-types/file.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/packages/module-types/meson.build b/tests/packages/module-types/meson.build new file mode 100644 index 000000000..5c5b52535 --- /dev/null +++ b/tests/packages/module-types/meson.build @@ -0,0 +1,21 @@ +project( + 'module-types', 'c', + version: '1.0.0', +) + +py_mod = import('python') +py = py_mod.find_installation() + +py.install_sources('file.py') +py.install_sources( + 'package' / '__init__.py', + subdir: 'package', +) +py.install_sources( + 'namespace' / 'data.py', + subdir: 'namespace', +) +py.extension_module( + 'native', 'native.c', + install: true, +) diff --git a/tests/packages/module-types/namespace/data.py b/tests/packages/module-types/namespace/data.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/packages/module-types/native.c b/tests/packages/module-types/native.c new file mode 100644 index 000000000..ca30346d2 --- /dev/null +++ b/tests/packages/module-types/native.c @@ -0,0 +1,24 @@ +#include + +static PyObject* foo(PyObject* self) +{ + return PyUnicode_FromString("bar"); +} + +static PyMethodDef methods[] = { + {"foo", (PyCFunction)foo, METH_NOARGS, NULL}, + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, + "native", + NULL, + -1, + methods, +}; + +PyMODINIT_FUNC PyInit_native(void) +{ + return PyModule_Create(&module); +} diff --git a/tests/packages/module-types/package/__init__.py b/tests/packages/module-types/package/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/packages/module-types/pyproject.toml b/tests/packages/module-types/pyproject.toml new file mode 100644 index 000000000..d6f3b6861 --- /dev/null +++ b/tests/packages/module-types/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python'] diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 93086e778..b3d77c65f 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -146,13 +146,8 @@ def test_configure_data(wheel_configure_data): @pytest.mark.skipif(platform.system() != 'Linux', reason='Unsupported on this platform for now') def test_local_lib(venv, wheel_link_against_local_lib): - subprocess.run( - [venv.executable, '-m', 'pip', 'install', wheel_link_against_local_lib], - check=True) - output = subprocess.run( - [venv.executable, '-c', 'import example; print(example.example_sum(1, 2))'], - stdout=subprocess.PIPE, - check=True).stdout + venv.pip('install', wheel_link_against_local_lib) + output = venv.python('-c', 'import example; print(example.example_sum(1, 2))') assert int(output) == 3 @@ -225,3 +220,32 @@ def test_entrypoints(wheel_full_metadata): [gui_scripts] example-gui = example:gui ''').strip() + + +def test_top_level_modules(package_module_types): + with mesonpy.Project.with_temp_working_dir() as project: + assert set(project._wheel_builder.top_level_modules) == { + 'file', + 'package', + 'namespace', + 'native', + } + + +def test_editable( + package_imports_itself_during_build, + editable_imports_itself_during_build, + venv, +): + venv.pip('install', os.fspath(editable_imports_itself_during_build)) + + assert venv.python('-c', 'import plat; print(plat.foo())').strip() == 'bar' + + plat = package_imports_itself_during_build / 'plat.c' + plat_text = plat.read_text() + try: + plat.write_text(plat_text.replace('bar', 'something else')) + + assert venv.python('-c', 'import plat; print(plat.foo())').strip() == 'something else' + finally: + plat.write_text(plat_text)