add compatibility for editable wheels and src layout (#20)

Makefile

@@ -15,6 +15,7 @@ clean:
 	rm -rf build/
 	rm -rf .eggs/
 	rm -rf *.egg-info/
+	rm -rf .venv
 
 clean-dist: clean
 	rm -rf dist/

README.md

@@ -23,7 +23,7 @@ It provides tools to load plugins from entry points at run time, and to discover
   In the simplest case, that can just be the Plugin's class).
 * `Plugin`: an object that exposes a `should_load` and `load` method.
   Note that it does not function as a domain object (it does not hold the plugins lifecycle state, like initialized, loaded, etc..., or other metadata of the Plugin)
-* `PluginFinder`: finds plugins, either at build time (by scanning the modules using `pkgutil` and `setuptools`) or at run time (reading entrypoints of the distribution using [stevedore](https://docs.openstack.org/stevedore/latest/))
+* `PluginFinder`: finds plugins, either at build time (by scanning the modules using `pkgutil` and `setuptools`) or at run time (reading entrypoints of the distribution using [importlib](https://docs.python.org/3/library/importlib.metadata.html#entry-points))
 * `PluginManager`: manages the run time lifecycle of a Plugin, which has three states:
   * resolved: the entrypoint pointing to the PluginSpec was imported and the `PluginSpec` instance was created
   * init: the `PluginFactory` of the `PluginSpec` was successfully invoked
@@ -33,7 +33,7 @@ It provides tools to load plugins from entry points at run time, and to discover
 
 ### Loading Plugins
 
-At run time, a `PluginManager` uses a `PluginFinder` that in turn uses stevedore to scan the available entrypoints for things that look like a `PluginSpec`.
+At run time, a `PluginManager` uses a `PluginFinder` that in turn uses importlib to scan the available entrypoints for things that look like a `PluginSpec`.
 With `PluginManager.load(name: str)` or `PluginManager.load_all()`, plugins within the namespace that are discoverable in entrypoints can be loaded.
 If an error occurs at any state of the lifecycle, the `PluginManager` informs the `PluginLifecycleListener` about it, but continues operating.
 

plux/build/setuptools.py

@@ -8,13 +8,27 @@
 import re
 import sys
 import typing as t
+from pathlib import Path
 
 import setuptools
+
+try:
+    from setuptools.command.editable_wheel import editable_wheel
+except ImportError:
+    # this means we're probably working on an old setuptools version, perhaps because we are on an older
+    # Python version.
+    class editable_wheel:
+        def run(self):
+            raise NotImplementedError("Compatibility with editable wheels requires Python 3.10")
+
+        def _ensure_dist_info(self, *args, **kwargs):
+            pass
+
 from setuptools.command.egg_info import InfoCommon, write_entries
 
-from plux.core.entrypoint import EntryPointDict, to_entry_point_dict, discover_entry_points
+from plux.core.entrypoint import EntryPointDict, discover_entry_points
 from plux.core.plugin import PluginFinder, PluginSpec
-from plux.runtime.metadata import Distribution  # FIXME
+from plux.runtime.metadata import Distribution, entry_points_from_metadata_path
 from .discovery import ModuleScanningPluginFinder
 
 LOG = logging.getLogger(__name__)
@@ -47,9 +61,56 @@ def run(self) -> None:
         update_entrypoints(self.distribution, ep)
 
 
+def patch_editable_wheel_command():
+    """
+    This patch creates a way for plux to maintain a link between the entry_points.txt contained in source
+    distribution's egg-info, and the installed dist-info directory created by the editable_wheel command.
+    This is so we can resolve entry points re-generated by ``python -m plux plugins`` from the source code,
+    while using editable installs via ``pip install -e .`` into the venv.
+
+    For instance, when you use an editable install, the path will look something like::
+
+        ~/my-project/my_project/
+        ~/my-project/my_project.egg-info/
+        ~/my-project/.venv/
+        ~/my-project/.venv/lib/python3.11/site-packages/my_project-0.0.1.dev0.dist-info
+
+    The .egg-info directory will correctly contain an entry-points.txt, but not the generated dist-info
+    directory in site-packages. This is a problem under certain circumstances because importlib may not
+    find the entry points. The reasons why dist-info doesn't contain the entry_points.txt are because of
+    how pip works. We could make sure that we correctly create an entry_points.txt during the building
+    process, but that would ultimately not help, as we would have to ``pip install -e .`` again every time
+    the plugins change. So a dynamic approach is better, which the linking facilitates. During runtime,
+    simply follow the link left behind in ``entry_points_editable.txt`` to the real ``entry_points.txt`` in
+    the .egg-info directory.
+
+    TODO: this is a hacky patch that relies on setuptools internals. It would be better to find a clean way
+    to completely overwrite the ``editable_wheel`` command, perhaps via the command class resolution
+    mechanism. i looked into that for an hour or so and concluded that this is an acceptable workaround for
+    now.
+    """
+    _ensure_dist_info_orig = editable_wheel._ensure_dist_info
+
+    def _ensure_dist_info(self):
+        _ensure_dist_info_orig(self)
+        # then we can create a link to the original file
+        # this is the egg info dir from the distribution source (the pip install -e target)
+
+        # create what is basically an application-layer symlink to the original entry points
+        target = Path(self.dist_info_dir, "entry_points_editable.txt")
+        target.write_text(os.path.join(find_egg_info_dir(), "entry_points.txt"))
+
+    editable_wheel._ensure_dist_info = _ensure_dist_info
+
+
+# patch will be applied implicitly through the entry point that loads the ``plugins`` command.
+patch_editable_wheel_command()
+
+
 def find_plugins(where=".", exclude=(), include=("*",)) -> EntryPointDict:
     """
-    Utility for setup.py that collects all plugins from the specified path, and creates a dictionary for entry_points.
+    Utility for setup.py that collects all plugins from the specified path, and creates a dictionary for
+    entry_points.
 
     For example:
 
@@ -111,12 +172,13 @@ def load_entry_points(
         ...
     )
 
-    This is a hack for installing from source distributions. When running pip install on a source distribution,
-    the egg_info directory is always re-built, even though it comes with the source distribution package data. This
-    also means the entry points are resolved, and in extent, `find_plugins` is called, which is problematic at this
-    point, because find_plugins will scan the code, and that will fail if requirements aren't yet installed,
-    which they aren't when running pip install. However, since source distributions package the .egg-info directory,
-    we can read the entry points from there instead, acting as sort of a cache.
+    This is a hack for installing from source distributions. When running pip install on a source
+    distribution, the egg_info directory is always re-built, even though it comes with the source
+    distribution package data. This also means the entry points are resolved, and in extent, `find_plugins`
+    is called, which is problematic at this point, because find_plugins will scan the code, and that will
+    fail if requirements aren't yet installed, which they aren't when running pip install. However,
+    since source distributions package the .egg-info directory, we can read the entry points from there
+    instead, acting as sort of a cache.
 
     :param where: the file path to look for plugins (default, the current working dir)
     :param exclude: the glob patterns to exclude
@@ -142,12 +204,11 @@ def entry_points_from_egg_info(egg_info_dir: str) -> EntryPointDict:
     """
     Reads the entry_points.txt from a distribution meta dir (e.g., the .egg-info directory).
     """
-    dist = Distribution.at(egg_info_dir)
-    return to_entry_point_dict(dist.entry_points)
+    return entry_points_from_metadata_path(egg_info_dir)
 
 
 def _has_entry_points_cache() -> bool:
-    egg_info_dir = _get_egg_info_dir()
+    egg_info_dir = find_egg_info_dir()
     if not egg_info_dir:
         return False
 
@@ -158,13 +219,13 @@ def _has_entry_points_cache() -> bool:
 
 
 def _should_read_existing_egg_info() -> t.Tuple[bool, t.Optional[str]]:
-    # we want to read the .egg-info dir only if it exists, and if we are creating the egg_info or installing it with
-    # pip install -e (which calls 'setup.py develop')
+    # we want to read the .egg-info dir only if it exists, and if we are creating the egg_info or
+    # installing it with pip install -e (which calls 'setup.py develop')
 
     if not (_is_pip_build_context() or _is_local_build_context()):
         return False, None
 
-    egg_info_dir = _get_egg_info_dir()
+    egg_info_dir = find_egg_info_dir()
     if not egg_info_dir:
         return False, None
 
@@ -175,8 +236,8 @@ def _should_read_existing_egg_info() -> t.Tuple[bool, t.Optional[str]]:
 
 
 def _is_pip_build_context():
-    # when pip builds packages or wheels from source distributions, it creates a temporary directory with a marker
-    # file that we can use to determine whether we are in such a build context.
+    # when pip builds packages or wheels from source distributions, it creates a temporary directory with a
+    # marker file that we can use to determine whether we are in such a build context.
     for f in os.listdir(os.getcwd()):
         if f == "pip-delete-this-directory.txt":
             return True
@@ -185,7 +246,8 @@ def _is_pip_build_context():
 
 
 def _is_local_build_context():
-    # when installing with `pip install -e` pip also tries to create a "modern-metadata" (dist-info) directory.
+    # when installing with `pip install -e` pip also tries to create a "modern-metadata" (dist-info)
+    # directory.
     if "dist_info" in sys.argv:
         try:
             i = sys.argv.index("--egg-base")
@@ -195,23 +257,28 @@ def _is_local_build_context():
         if "pip-modern-metadata" in sys.argv[i + 1]:
             return True
 
-    # it's unfortunately not really distinguishable whether or not a user calls `python setup.py develop` in the
-    # project, or calls `pip install -e ..` to install the project from somewhere else.
+    # it's unfortunately not really distinguishable whether or not a user calls `python setup.py develop`
+    # in the project, or calls `pip install -e ..` to install the project from somewhere else.
     if len(sys.argv) > 1 and sys.argv[1] in ["egg_info", "develop"]:
         return True
 
     return False
 
 
-def _get_egg_info_dir() -> t.Optional[str]:
+def find_egg_info_dir() -> t.Optional[str]:
     """
     Heuristic to find the .egg-info dir of the current build context.
     """
-    cwd = os.getcwd()
-    for d in os.listdir(cwd):
-        if d.endswith(".egg-info"):
-            return os.path.join(cwd, d)
-
+    workdir = os.getcwd()
+    distribution = get_distribution_from_workdir(workdir)
+    dirs = distribution.package_dir
+    egg_base = (dirs or {}).get("", workdir)
+    if not egg_base:
+        return None
+    egg_info_dir = _to_filename(_safe_name(distribution.get_name())) + ".egg-info"
+    candidate = os.path.join(workdir, egg_base, egg_info_dir)
+    if os.path.exists(candidate):
+        return candidate
     return None
 
 

plux/cli/cli.py

@@ -6,9 +6,10 @@
 import json
 import logging
 import os
+import sys
 
 from plux.build.setuptools import (
-    _get_egg_info_dir,
+    find_egg_info_dir,
     find_plugins,
     get_distribution_from_workdir,
     get_plux_json_path,
@@ -37,7 +38,7 @@ def discover(args):
 
 
 def show(args):
-    egg_info_dir = _get_egg_info_dir()
+    egg_info_dir = find_egg_info_dir()
     if not egg_info_dir:
         print("no *.egg-info directory")
         return
@@ -51,6 +52,17 @@ def show(args):
         print(fd.read())
 
 
+def resolve(args):
+    for p in sys.path:
+        print(f"path = {p}")
+    from plux import PluginManager
+
+    manager = PluginManager(namespace=args.namespace)
+
+    for spec in manager.list_plugin_specs():
+        print(f"{spec.namespace}:{spec.name} = {spec.factory.__module__}:{spec.factory.__name__}")
+
+
 def _pprint_plux_json(plux_json):
     print(json.dumps(plux_json, indent=2))
 
@@ -76,6 +88,13 @@ def main(argv=None):
     discover_parser = subparsers.add_parser("discover", help="Discover plugins and print them")
     discover_parser.set_defaults(func=discover)
 
+    # Subparser for the 'resolve' subcommand
+    resolve_parser = subparsers.add_parser(
+        "resolve", help="Resolve a plugin namespace and list all its plugins"
+    )
+    resolve_parser.add_argument("--namespace", help="the plugin namespace", required=True)
+    resolve_parser.set_defaults(func=resolve)
+
     # Subparser for the 'discover' subcommand
     show_parser = subparsers.add_parser("show", help="Show entrypoints that were generated")
     show_parser.set_defaults(func=show)