Support new mkdocstrings_handlers namespace

docs/handlers/overview.md

@@ -19,18 +19,16 @@ For *mkdocstrings*, a custom handler package would have the following structure:
 
 ```
 📁 your_repository
-└─╴📁 mkdocstrings
-   └─╴📁 handlers
-      └─╴📄 custom_handler.py
+└─╴📁 mkdocstrings_handlers
+   └─╴📁 custom_handler
+      ├─╴📁 templates
+      │  ├─╴📁 material
+      │  ├─╴📁 mkdocs
+      │  └─╴📁 readthedocs
+      └─╴📄 __init__.py
 ```
 
-**Note the absence of `__init__.py` modules!**
-
-If you name you handler after an existing handler,
-it will overwrite it!
-For example, it means you can overwrite the Python handler
-to change how it works or to add functionality,
-by naming your handler module `python.py`.
+**Note the absence of `__init__.py` module in `mkdocstrings_handlers`!**
 
 ### Code
 
@@ -42,7 +40,7 @@ See the documentation for
 [`BaseRenderer`][mkdocstrings.handlers.base.BaseRenderer].
 
 Check out how the
-[Python handler](https://github.com/pawamoy/mkdocstrings/blob/master/src/mkdocstrings/handlers/python.py)
+[Python handler](https://github.com/mkdocstrings/python/blob/master/src/mkdocstrings_handlers/python)
 is written for inspiration.
 
 You must implement a `get_handler` method at the module level.
@@ -54,8 +52,8 @@ will be passed to this function when getting your handler.
 
 ### Templates
 
-You renderer's implementation should normally be backed by templates, which go
-to the directory `mkdocstrings/handlers/custom_handler/some_theme`.
+Your renderer's implementation should normally be backed by templates, which go
+to the directory `mkdocstrings_handlers/custom_handler/templates/some_theme`.
 (`custom_handler` here should be replaced with the actual name of your handler,
 and `some_theme` should be the name of an actual MkDocs theme that you support,
 e.g. `material`).
@@ -73,11 +71,27 @@ one of the other theme directories in case they're exactly the same as in the
 fallback theme.
 
 If your theme's HTML requires CSS to go along with it, put it into a file named
-`mkdocstrings/handlers/custom_handler/some_theme/style.css`, then this will be
+`mkdocstrings_handlers/custom_handler/templates/some_theme/style.css`, then this will be
 included into the final site automatically if this handler is ever used.
 Alternatively, you can put the CSS as a string into the `extra_css` variable of
 your renderer.
 
+Finally, it's possible to entirely omit templates, and tell *mkdocstrings*
+to use the templates of another handler. In you renderer, override the
+`get_templates_dir()` method to return the other handlers templates path:
+
+```python
+from pathlib import Path
+from mkdocstrings.handlers.base import BaseRenderer
+
+
+class CobraRenderer(BaseRenderer):
+    def get_templates_dir(self, handler: str) -> Path:
+        # use the python handler templates
+        # (it assumes the python handler is installed)
+        return super().get_templates_dir("python")
+```
+
 ### Usage
 
 When a custom handler is installed, it is then available to *mkdocstrings*.

src/mkdocstrings/handlers/base.py

@@ -9,8 +9,9 @@
 """
 
 import importlib
-import sys
+import warnings
 from abc import ABC, abstractmethod
+from contextlib import suppress
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Sequence
 from xml.etree.ElementTree import Element, tostring
@@ -72,20 +73,30 @@ class BaseRenderer(ABC):
     fallback_theme: str = ""
     extra_css = ""
 
-    def __init__(self, directory: str, theme: str, custom_templates: Optional[str] = None) -> None:
+    def __init__(self, handler: str, theme: str, custom_templates: Optional[str] = None, directory: str = None) -> None:
         """Initialize the object.
 
         If the given theme is not supported (it does not exist), it will look for a `fallback_theme` attribute
         in `self` to use as a fallback theme.
 
         Arguments:
-            directory: The name of the directory containing the themes for this renderer.
+            handler: The name of the handler.
             theme: The name of theme to use.
             custom_templates: Directory containing custom templates.
+            directory: Deprecated and renamed as `handler`.
         """
+        # TODO: remove at some point
+        if directory:
+            warnings.warn(
+                "The 'directory' keyword parameter is deprecated and renamed 'handler'. ",
+                DeprecationWarning,
+            )
+            if not handler:
+                handler = directory
+
         paths = []
 
-        themes_dir = self.get_templates_dir() / directory
+        themes_dir = self.get_templates_dir(handler)
         paths.append(themes_dir / theme)
 
         if self.fallback_theme and self.fallback_theme != theme:
@@ -98,7 +109,7 @@ def __init__(self, directory: str, theme: str, custom_templates: Optional[str] =
                 break
 
         if custom_templates is not None:
-            paths.insert(0, Path(custom_templates) / directory / theme)
+            paths.insert(0, Path(custom_templates) / handler / theme)
 
         self.env = Environment(
             autoescape=True,
@@ -123,29 +134,52 @@ def render(self, data: CollectorItem, config: dict) -> str:
             The rendered template as HTML.
         """  # noqa: DAR202 (excess return section)
 
-    def get_templates_dir(self) -> Path:
+    def get_templates_dir(self, handler: str) -> Path:
         """Return the path to the handler's templates directory.
 
-        Override this method if your handler is for example compiled from C
-        and does not expose a module directly on the file system from
-        which we can infer the templates directory path.
+        Override to customize how the templates directory is found.
+
+        Arguments:
+            handler: The name of the handler to get the templates directory of.
+
+        Raises:
+            FileNotFoundError: When the templates directory cannot be found.
 
         Returns:
             The templates directory path.
         """
-        # Namespace packages can span multiple locations.
-        # This class can be derived, so we must find the templates path
-        # based on the file path the derived class was defined in.
-        # To do this, we first get the module of this derived class:
-        module = sys.modules[self.__class__.__module__]  # noqa: WPS609
-        # Then we can get the module path:
-        module_path = Path(module.__file__)  # type: ignore[arg-type]  # noqa: WPS609
-        # Now we can go up to the "mkdocstrings" folder,
-        # and one down to the "templates" folder:
-        templates_dir = module_path.parent.resolve()
-        while templates_dir.name != "mkdocstrings":
-            templates_dir = templates_dir.parent
-        return templates_dir / "templates"
+        # Templates can be found in 2 different logical locations:
+        # - in mkdocstrings_handlers/HANDLER/templates: our new migration target
+        # - in mkdocstrings/templates/HANDLER: current situation, this should be avoided
+        # These two other locations are forbidden:
+        # - in mkdocstrings_handlers/templates/HANDLER: sub-namespace packages are too annoying to deal with
+        # - in mkdocstrings/handlers/HANDLER/templates: not currently supported,
+        #   and mkdocstrings will stop being a namespace
+
+        with suppress(ModuleNotFoundError):  # TODO: catch at some point to warn about missing handlers
+            import mkdocstrings_handlers
+
+            for path in mkdocstrings_handlers.__path__:  # noqa: WPS609
+                theme_path = Path(path, handler, "templates")
+                if theme_path.exists():
+                    return theme_path
+
+        # TODO: remove import and loop at some point,
+        # as mkdocstrings will stop being a namespace package
+        import mkdocstrings
+
+        for path in mkdocstrings.__path__:  # noqa: WPS609,WPS440
+            theme_path = Path(path, "templates", handler)
+            if theme_path.exists():
+                if handler != "python":
+                    warnings.warn(
+                        "Exposing templates in the mkdocstrings.templates namespace is deprecated. "
+                        "Put them in a templates folder inside your handler package instead.",
+                        DeprecationWarning,
+                    )
+                return theme_path
+
+        raise FileNotFoundError(f"Can't find 'templates' folder for handler '{handler}'")
 
     def get_anchors(self, data: CollectorItem) -> Sequence[str]:
         """Return the possible identifiers (HTML anchors) for a collected item.
@@ -419,7 +453,17 @@ def get_handler(self, name: str, handler_config: Optional[dict] = None) -> BaseH
         if name not in self._handlers:
             if handler_config is None:
                 handler_config = self.get_handler_config(name)
-            module = importlib.import_module(f"mkdocstrings.handlers.{name}")
+            try:
+                module = importlib.import_module(f"mkdocstrings_handlers.{name}")
+            except ModuleNotFoundError:
+                module = importlib.import_module(f"mkdocstrings.handlers.{name}")
+                if name != "python":
+                    warnings.warn(
+                        DeprecationWarning(
+                            "Using the mkdocstrings.handlers namespace is deprecated. "
+                            "Handlers must now use the mkdocstrings_handlers namespace."
+                        )
+                    )
             self._handlers[name] = module.get_handler(
                 self._config["theme_name"],
                 self._config["mkdocstrings"]["custom_templates"],