From 3c47c8dbbdcc80b900c3a0ff9a91d55daaf7005c Mon Sep 17 00:00:00 2001 From: Thomas Rausch Date: Fri, 28 Nov 2025 01:59:22 +0100 Subject: [PATCH] hatchling support wip, package scanner, first steps with testing --- .gitignore | 3 + plux/build/discovery.py | 100 +++++++++++- plux/build/hatchling.py | 135 +++++++++++++++- plux/cli/cli.py | 5 +- tests/build/__init__.py | 0 tests/build/test_discovery.py | 146 ++++++++++++++++++ .../hatch/namespace_package/pyproject.toml | 31 ++++ .../namespace_package/test_project/plugins.py | 6 + .../test_project/subpkg/__init__.py | 0 .../test_project/subpkg/plugins.py | 6 + .../hatch_manual_build_mode/pyproject.toml | 28 ++++ .../test_project/__init__.py | 0 .../test_project/plugins.py | 6 + .../test_project/subpkg/__init__.py | 0 .../test_project/subpkg/plugins.py | 6 + tests/cli/projects/manual_build_mode/plux.ini | 4 - tests/cli/test_hatch.py | 52 +++++++ 17 files changed, 519 insertions(+), 9 deletions(-) create mode 100644 tests/build/__init__.py create mode 100644 tests/build/test_discovery.py create mode 100644 tests/cli/projects/hatch/namespace_package/pyproject.toml create mode 100644 tests/cli/projects/hatch/namespace_package/test_project/plugins.py create mode 100644 tests/cli/projects/hatch/namespace_package/test_project/subpkg/__init__.py create mode 100644 tests/cli/projects/hatch/namespace_package/test_project/subpkg/plugins.py create mode 100644 tests/cli/projects/hatch_manual_build_mode/pyproject.toml create mode 100644 tests/cli/projects/hatch_manual_build_mode/test_project/__init__.py create mode 100644 tests/cli/projects/hatch_manual_build_mode/test_project/plugins.py create mode 100644 tests/cli/projects/hatch_manual_build_mode/test_project/subpkg/__init__.py create mode 100644 tests/cli/projects/hatch_manual_build_mode/test_project/subpkg/plugins.py create mode 100644 tests/cli/test_hatch.py diff --git a/.gitignore b/.gitignore index e60091a..ca9349c 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,6 @@ venv.bak/ # don't ignore build package !plux/build +!tests/build + +plux.ini diff --git a/plux/build/discovery.py b/plux/build/discovery.py index 2a8b44f..c94b9a3 100644 --- a/plux/build/discovery.py +++ b/plux/build/discovery.py @@ -5,11 +5,12 @@ import importlib import inspect import logging +import os +import pkgutil import typing as t from fnmatch import fnmatchcase +from pathlib import Path from types import ModuleType -import os -import pkgutil from plux import PluginFinder, PluginSpecResolver, PluginSpec @@ -56,6 +57,101 @@ def path(self) -> str: raise NotImplementedError +class SimplePackageFinder(PackageFinder): + """ + A package finder that uses a heuristic to find python packages within a given path. It iterates over all + subdirectories in the path and returns every directory that contains a ``__init__.py`` file. It will include the + root package in the list of results, so if your tree looks like this:: + + mypkg + ├── __init__.py + ├── subpkg1 + │ ├── __init__.py + │ └── nested_subpkg1 + │ └── __init__.py + └── subpkg2 + └── __init__.py + + and you instantiate SimplePackageFinder("mypkg"), it will return:: + + [ + "mypkg", + "mypkg.subpkg1", + "mypkg.subpkg2", + "mypkg.subpkg1.nested_subpkg1, + ] + + If the root is not a package, say if you have a ``src/`` layout, and you pass "src/mypkg" as ``path`` it will omit + everything in the preceding path that's not a package. + """ + + def __init__(self, path: str): + self._path = path + + @property + def path(self) -> str: + return self._path + + def find_packages(self) -> t.Iterable[str]: + """ + Find all Python packages in the given path. + + Returns a list of package names in the format "pkg", "pkg.subpkg", etc. + """ + path = self.path + if not os.path.isdir(path): + return [] + + result = [] + + # Get the absolute path to handle relative paths correctly + abs_path = os.path.abspath(path) + + # Check if the root directory is a package + root_is_package = self._looks_like_package(abs_path) + + # Walk through the directory tree + for root, dirs, files in os.walk(abs_path): + # Skip directories that don't look like packages + if not self._looks_like_package(root): + continue + + # Determine the base directory for relative path calculation + # If the root is not a package, we use the root directory itself as the base + # This ensures we don't include the root directory name in the package names + if root_is_package: + base_dir = os.path.dirname(abs_path) + else: + base_dir = abs_path + + # Convert the path to a module name + rel_path = os.path.relpath(root, base_dir) + if rel_path == ".": + # If we're at the root and it's a package, use the directory name + rel_path = os.path.basename(abs_path) + + # Skip invalid package names (those containing dots in the path) + if "." in os.path.basename(rel_path): + continue + + module_name = self._path_to_module(rel_path) + result.append(module_name) + + # Sort the results for consistent output + return sorted(result) + + def _looks_like_package(self, path: str) -> bool: + return os.path.exists(os.path.join(path, "__init__.py")) + + @staticmethod + def _path_to_module(path: str): + """ + Convert a path to a Python module to its module representation + Example: plux/core/test -> plux.core.test + """ + return ".".join(Path(path).with_suffix("").parts) + + class PluginFromPackageFinder(PluginFinder): """ Finds Plugins from packages that are resolved by the given ``PackageFinder``. Under the hood this uses a diff --git a/plux/build/hatchling.py b/plux/build/hatchling.py index 10509b5..a405610 100644 --- a/plux/build/hatchling.py +++ b/plux/build/hatchling.py @@ -1,6 +1,137 @@ +import logging +import os +import typing as t +from pathlib import Path + +from hatchling.builders.config import BuilderConfig +from hatchling.builders.wheel import WheelBuilder + +from plux.build.config import EntrypointBuildMode +from plux.build.discovery import PackageFinder, Filter, MatchAllFilter, SimplePackageFinder from plux.build.project import Project +LOG = logging.getLogger(__name__) + + +def _path_to_module(path): + """ + Convert a path to a Python module to its module representation. + + Example: plux/core/test -> plux.core.test + """ + return ".".join(Path(path).with_suffix("").parts) + + +class HatchlingPackageFinder(PackageFinder): + """ + Uses hatchling's BuilderConfig abstraction to enumerate packages. + + TODO: this might not be 100% correct and needs more thorough testing with different scenarios. + """ + + builder_config: BuilderConfig + exclude: Filter + include: Filter + + def __init__( + self, + builder_config: BuilderConfig, + exclude: list[str] | None = None, + include: list[str] | None = None, + ): + self.builder_config = builder_config + self.exclude = Filter(exclude or []) + self.include = Filter(include) if include else MatchAllFilter() + + def find_packages(self) -> t.Iterable[str]: + """ + Hatchling-specific algorithm to find packages. Unlike setuptools, hatchling does not provide a package discovery + and only provides a config, so this implements our own heuristic to detect packages and namespace packages. + + :return: An Iterable of Packages + """ + # packages in hatch are defined as file system paths, whereas find_packages expects modules + package_paths = list(self.builder_config.packages) + + # unlike setuptools, hatch does not return all subpackages by default. instead, these are + # top-level package paths, so we need to recurse and use the ``__init__.py`` heuristic to find + # packages. + all_packages = [] + for relative_package_path in package_paths: + package_name = os.path.basename(relative_package_path) + + package_path = os.path.join( + self.path, relative_package_path + ) # build package path within sources root + if not os.path.isdir(package_path): + continue + + is_namespace_package = not os.path.exists(os.path.join(package_path, "__init__.py")) + found_packages = SimplePackageFinder(package_path).find_packages() + + if is_namespace_package: + # If it's a namespace package, we need to do two things. First, we include it explicitly as a + # top-level package to the list of found packages. Second, since``SimplePackageFinder`` will not + # consider it a package, it will only return subpackages, so we need to prepend the namespace package + # as a namespace to the package names. + all_packages.append(package_name) + found_packages = [f"{package_name}.{found_package}" for found_package in found_packages] + + all_packages.extend(found_packages) + + # now do the filtering like the plux PackageFinder API expects + packages = self.filter_packages(all_packages) + + # de-duplicate and sort + return sorted(set(packages)) + + @property + def path(self) -> str: + if not self.builder_config.sources: + where = self.builder_config.root + else: + if self.builder_config.sources[""]: + where = self.builder_config.sources[""] + else: + LOG.warning("plux doesn't know how to resolve multiple sources directories") + where = self.builder_config.root + + return where + + def filter_packages(self, packages: t.Iterable[str]) -> t.Iterable[str]: + return [item for item in packages if not self.exclude(item) and self.include(item)] + class HatchlingProject(Project): - # TODO: implement me - pass + def __init__(self, workdir: str = None): + super().__init__(workdir) + + if self.config.entrypoint_build_mode != EntrypointBuildMode.MANUAL: + raise NotImplementedError( + "Hatchling integration currently only works with entrypoint_build_mode=manual" + ) + + # FIXME it's unclear whether this will really hold for all configs. most configs we assume will build a wheel, + # and any associated package configuration will come from there. so currently we make the wheel build config + # the source of truth, but we should revisit this once we know more about hatch build configurations. + self.builder = WheelBuilder(workdir) + + def create_package_finder(self) -> PackageFinder: + return HatchlingPackageFinder( + self.hatchling_config, + exclude=self.config.exclude, + include=self.config.include, + ) + + @property + def hatchling_config(self) -> BuilderConfig: + return self.builder.config + + def find_plux_index_file(self) -> Path: + # TODO: extend as soon as we support EntryPointBuildMode = build-hook + return Path(self.hatchling_config.root, self.config.entrypoint_static_file) + + def find_entry_point_file(self) -> Path: + # TODO: we'll assume that `pip install -e .` is used, and therefore the entrypoints file will be in the + # .dist-info metadata directory + raise NotImplementedError diff --git a/plux/cli/cli.py b/plux/cli/cli.py index fcf769b..164a4e5 100644 --- a/plux/cli/cli.py +++ b/plux/cli/cli.py @@ -16,6 +16,7 @@ def _get_build_backend() -> str | None: # TODO: should read this from the project configuration instead somehow. + return "hatchling" try: import setuptools # noqa @@ -45,7 +46,9 @@ def _load_project(args: argparse.Namespace) -> Project: return SetuptoolsProject(workdir) elif backend == "hatchling": - raise NotImplementedError("Hatchling is not yet supported as build backend") + from plux.build.hatchling import HatchlingProject + + return HatchlingProject(workdir) else: raise RuntimeError( "No supported build backend found. Plux needs either setuptools or hatchling to work." diff --git a/tests/build/__init__.py b/tests/build/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/build/test_discovery.py b/tests/build/test_discovery.py new file mode 100644 index 0000000..ddb1fd2 --- /dev/null +++ b/tests/build/test_discovery.py @@ -0,0 +1,146 @@ +import os +import shutil +from pathlib import Path + +import pytest + +from plux.build.discovery import SimplePackageFinder + + +@pytest.fixture +def chdir(): + """Change the working directory to the given path temporarily for the test.""" + prev = os.getcwd() + yield os.chdir + os.chdir(prev) + + +class TestSimplePackageFinder: + @pytest.fixture + def nested_package_tree(self, tmp_path) -> Path: + # nested directory structure + dirs = [ + tmp_path / "src" / "mypkg" / "subpkg1", + tmp_path / "src" / "mypkg" / "subpkg2", + tmp_path / "src" / "mypkg" / "subpkg1" / "nested_subpkg1", + tmp_path / "src" / "mypkg_sibling", + tmp_path / "src" / "notapkg", + tmp_path / "src" / "not.a.pkg", # this is an invalid package name and will break imports + ] + files = [ + tmp_path / "src" / "mypkg" / "__init__.py", + tmp_path / "src" / "mypkg" / "subpkg1" / "__init__.py", + tmp_path / "src" / "mypkg" / "subpkg2" / "__init__.py", + tmp_path / "src" / "mypkg" / "subpkg1" / "nested_subpkg1" / "__init__.py", + tmp_path / "src" / "mypkg_sibling" / "__init__.py", + tmp_path / "src" / "notapkg" / "__init__.txt", + tmp_path / "src" / "not.a.pkg" / "__init__.py", + ] + + for d in dirs: + d.mkdir(parents=True, exist_ok=True) + for f in files: + f.touch(exist_ok=True) + + return tmp_path + + def test_package_discovery(self, nested_package_tree, chdir): + # change into the directory for the test + chdir(nested_package_tree / "src") + + # this emulates what a typical hatchling src-tree config would look like + finder = SimplePackageFinder("mypkg") + + assert finder.find_packages() == [ + "mypkg", + "mypkg.subpkg1", + "mypkg.subpkg1.nested_subpkg1", + "mypkg.subpkg2", + ] + + def test_package_discovery_in_current_dir(self, nested_package_tree, chdir): + # change into the actual package directory, so the path just becomes "." + chdir(nested_package_tree / "src" / "mypkg") + + # this might be equivalent to os.curdir + finder = SimplePackageFinder(".") + + assert finder.find_packages() == [ + "mypkg", + "mypkg.subpkg1", + "mypkg.subpkg1.nested_subpkg1", + "mypkg.subpkg2", + ] + + def test_package_discovery_with_src_folder(self, nested_package_tree, chdir): + # change into the directory for the test + chdir(nested_package_tree) + + # this emulates what a typical hatchling src-tree config would look like + finder = SimplePackageFinder("src") + + assert finder.find_packages() == [ + "mypkg", + "mypkg.subpkg1", + "mypkg.subpkg1.nested_subpkg1", + "mypkg.subpkg2", + "mypkg_sibling", + ] + + def test_package_discovery_with_src_folder_unconventional(self, nested_package_tree, chdir): + # change into the directory for the test + chdir(nested_package_tree) + + # make sure there's no special consideration of "src" as a convention + shutil.move(nested_package_tree / "src", nested_package_tree / "sources") + + # this emulates what a typical hatchling src-tree config would look like + finder = SimplePackageFinder("sources") + + assert finder.find_packages() == [ + "mypkg", + "mypkg.subpkg1", + "mypkg.subpkg1.nested_subpkg1", + "mypkg.subpkg2", + "mypkg_sibling", + ] + + def test_package_discovery_with_nested_src_dir(self, nested_package_tree, chdir): + chdir(nested_package_tree) + + # create a root path + root = nested_package_tree / "root" + root.mkdir(parents=True, exist_ok=True) + + # move everything s.t. the path is now "root/src/" + shutil.move(nested_package_tree / "src", root) + + # should still work in the same way + finder = SimplePackageFinder("root/src/mypkg") + + assert finder.find_packages() == [ + "mypkg", + "mypkg.subpkg1", + "mypkg.subpkg1.nested_subpkg1", + "mypkg.subpkg2", + ] + + # make sure it finds the sibling if not pointed to the pkg dir directly + finder = SimplePackageFinder("root/src") + + assert finder.find_packages() == [ + "mypkg", + "mypkg.subpkg1", + "mypkg.subpkg1.nested_subpkg1", + "mypkg.subpkg2", + "mypkg_sibling", + ] + + def test_package_discovery_in_empty_dir(self, tmp_path, chdir): + path = tmp_path / "empty" + path.mkdir() + chdir(tmp_path) + + finder = SimplePackageFinder("empty") + + assert finder.find_packages() == [] diff --git a/tests/cli/projects/hatch/namespace_package/pyproject.toml b/tests/cli/projects/hatch/namespace_package/pyproject.toml new file mode 100644 index 0000000..1326fd0 --- /dev/null +++ b/tests/cli/projects/hatch/namespace_package/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["hatchling", "wheel"] +build-backend = "hatchling.build" + +[project] +name = "test-project" +authors = [ + { name = "LocalStack Contributors", email = "info@localstack.cloud" } +] +version = "0.1.0" +description = "A test project to test plux with pyproject.toml projects and manual build mode" +dependencies = [ + "plux", + "build", +] +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: Python :: 3", +] +dynamic = [ + "entry-points", +] + +[tool.hatch.build.targets.wheel] +packages = ["test_project"] + +[tool.hatch.dynamic] +entry-points = { file = ["plux.ini"] } + +[tool.plux] +entrypoint_build_mode = "manual" diff --git a/tests/cli/projects/hatch/namespace_package/test_project/plugins.py b/tests/cli/projects/hatch/namespace_package/test_project/plugins.py new file mode 100644 index 0000000..3a0b1b8 --- /dev/null +++ b/tests/cli/projects/hatch/namespace_package/test_project/plugins.py @@ -0,0 +1,6 @@ +from plugin import Plugin + + +class MyPlugin(Plugin): + namespace = "plux.test.plugins" + name = "myplugin" diff --git a/tests/cli/projects/hatch/namespace_package/test_project/subpkg/__init__.py b/tests/cli/projects/hatch/namespace_package/test_project/subpkg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/projects/hatch/namespace_package/test_project/subpkg/plugins.py b/tests/cli/projects/hatch/namespace_package/test_project/subpkg/plugins.py new file mode 100644 index 0000000..163edf8 --- /dev/null +++ b/tests/cli/projects/hatch/namespace_package/test_project/subpkg/plugins.py @@ -0,0 +1,6 @@ +from plugin import Plugin + + +class MyNestedPlugin(Plugin): + namespace = "plux.test.plugins" + name = "mynestedplugin" diff --git a/tests/cli/projects/hatch_manual_build_mode/pyproject.toml b/tests/cli/projects/hatch_manual_build_mode/pyproject.toml new file mode 100644 index 0000000..b03f740 --- /dev/null +++ b/tests/cli/projects/hatch_manual_build_mode/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = ["hatchling", "wheel"] +build-backend = "hatchling.build" + +[project] +name = "test-project" +authors = [ + { name = "LocalStack Contributors", email = "info@localstack.cloud" } +] +version = "0.1.0" +description = "A test project to test plux with pyproject.toml projects and manual build mode" +dependencies = [ + "plux", + "build", +] +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: Python :: 3", +] +dynamic = [ + "entry-points", +] + +[tool.hatch.dynamic] +entry-points = { file = ["plux.ini"] } + +[tool.plux] +entrypoint_build_mode = "manual" diff --git a/tests/cli/projects/hatch_manual_build_mode/test_project/__init__.py b/tests/cli/projects/hatch_manual_build_mode/test_project/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/projects/hatch_manual_build_mode/test_project/plugins.py b/tests/cli/projects/hatch_manual_build_mode/test_project/plugins.py new file mode 100644 index 0000000..3a0b1b8 --- /dev/null +++ b/tests/cli/projects/hatch_manual_build_mode/test_project/plugins.py @@ -0,0 +1,6 @@ +from plugin import Plugin + + +class MyPlugin(Plugin): + namespace = "plux.test.plugins" + name = "myplugin" diff --git a/tests/cli/projects/hatch_manual_build_mode/test_project/subpkg/__init__.py b/tests/cli/projects/hatch_manual_build_mode/test_project/subpkg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/projects/hatch_manual_build_mode/test_project/subpkg/plugins.py b/tests/cli/projects/hatch_manual_build_mode/test_project/subpkg/plugins.py new file mode 100644 index 0000000..163edf8 --- /dev/null +++ b/tests/cli/projects/hatch_manual_build_mode/test_project/subpkg/plugins.py @@ -0,0 +1,6 @@ +from plugin import Plugin + + +class MyNestedPlugin(Plugin): + namespace = "plux.test.plugins" + name = "mynestedplugin" diff --git a/tests/cli/projects/manual_build_mode/plux.ini b/tests/cli/projects/manual_build_mode/plux.ini index 9d11ce7..e69de29 100644 --- a/tests/cli/projects/manual_build_mode/plux.ini +++ b/tests/cli/projects/manual_build_mode/plux.ini @@ -1,4 +0,0 @@ -[plux.test.plugins] -mynestedplugin = mysrc.subpkg.plugins:MyNestedPlugin -myplugin = mysrc.plugins:MyPlugin - diff --git a/tests/cli/test_hatch.py b/tests/cli/test_hatch.py new file mode 100644 index 0000000..9ad838c --- /dev/null +++ b/tests/cli/test_hatch.py @@ -0,0 +1,52 @@ +import os.path +import sys + + +def test_discover_with_ini_output(tmp_path): + from plux.__main__ import main + + project = os.path.join(os.path.dirname(__file__), "projects", "hatch_manual_build_mode") + os.chdir(project) + + out_path = tmp_path / "plux.ini" + + sys.path.append(project) + try: + try: + main(["--workdir", project, "discover", "--format", "ini", "--output", str(out_path)]) + except SystemExit: + pass + finally: + sys.path.remove(project) + + lines = out_path.read_text().strip().splitlines() + assert lines == [ + "[plux.test.plugins]", + "mynestedplugin = test_project.subpkg.plugins:MyNestedPlugin", + "myplugin = test_project.plugins:MyPlugin", + ] + + +def test_discover_with_ini_output_namespace_Package(tmp_path): + from plux.__main__ import main + + project = os.path.join(os.path.dirname(__file__), "projects", "hatch", "namespace_package") + os.chdir(project) + + out_path = tmp_path / "plux.ini" + + sys.path.append(project) + try: + try: + main(["--workdir", project, "discover", "--format", "ini", "--output", str(out_path)]) + except SystemExit: + pass + finally: + sys.path.remove(project) + + lines = out_path.read_text().strip().splitlines() + assert lines == [ + "[plux.test.plugins]", + "mynestedplugin = test_project.subpkg.plugins:MyNestedPlugin", + "myplugin = test_project.plugins:MyPlugin", + ]