Updata Sphinx environment mocking

docs/using/use_api.md

@@ -236,3 +236,42 @@ print(document.pformat())
                     <paragraph>
                         definition
 ```
+
+You can also set Sphinx configuration *via* `sphinx_conf`. This is a dictionary representation of the contents of the Sphinx `conf.py`.
+
+```{warning}
+This feature is only meant for simple testing.
+It will fail for extensions that require the full
+Sphinx build process and/or access to external files.
+```
+
+`````python
+from myst_parser import text_to_tokens, render_tokens
+from myst_parser.docutils_renderer import SphinxRenderer
+
+root = text_to_tokens("""
+````{tabs}
+
+```{tab} Apples
+
+Apples are green, or sometimes red.
+```
+````
+""")
+
+document = render_tokens(root, SphinxRenderer, load_sphinx_env=True, sphinx_conf={"extensions": ["sphinx_tabs.tabs"]})
+print(document.pformat())
+`````
+
+```xml
+<document source="notset">
+    <container classes="sphinx-tabs">
+        <container>
+            <a classes="item">
+                <container>
+                    <paragraph>
+                        Apples
+            <container classes="ui bottom attached sphinx-tab tab segment sphinx-data-tab-0-0 active">
+                <paragraph>
+                    Apples are green, or sometimes red.
+```

myst_parser/__init__.py

@@ -1,10 +1,9 @@
-from myst_parser.block_tokens import Document
-
 __version__ = "0.2.0"
 
 
-def text_to_tokens(text: str) -> Document:
+def text_to_tokens(text: str):
     """Convert some text to the MyST base AST."""
+    from myst_parser.block_tokens import Document
     from myst_parser.ast_renderer import AstRenderer
 
     # this loads the MyST specific token parsers

myst_parser/docutils_renderer.py

@@ -5,7 +5,6 @@
 from pathlib import Path
 import re
 import sys
-from types import ModuleType
 from typing import List, Optional
 from urllib.parse import urlparse, unquote
 
@@ -513,7 +512,7 @@ def render_directive(self, token):
             )
             msg_node += nodes.literal_block(content, content)
             result = [msg_node]
-        except (AttributeError, NotImplementedError) as exc:
+        except MockingError as exc:
             error = self.reporter.error(
                 "Directive '{}' cannot be mocked:\n{}: {}".format(
                     name, exc.__class__.__name__, exc
@@ -542,7 +541,25 @@ class SphinxRenderer(DocutilsRenderer):
     """
 
     def __init__(self, *args, **kwargs):
+        """Intitalise SphinxRenderer
+
+        :param load_sphinx_env: load a basic sphinx environment,
+            when using the renderer as a context manager outside if `sphinx-build`
+        :param sphinx_conf: a dictionary representation of the sphinx `conf.py`
+        :param sphinx_srcdir: a path to a source directory
+          (for example, can be used for `include` statements)
+
+        To use this renderer in a 'standalone' fashion::
+
+            from myst_parser.block_tokens import Document
+
+            with SphinxRenderer(load_sphinx_env=True, sphinx_conf={}) as renderer:
+                renderer.render(Document("source text"))
+
+        """
         self.load_sphinx_env = kwargs.pop("load_sphinx_env", False)
+        self.sphinx_conf = kwargs.pop("sphinx_conf", None)
+        self.sphinx_srcdir = kwargs.pop("sphinx_srcdir", None)
         super().__init__(*args, **kwargs)
 
     def handle_cross_reference(self, token, destination):
@@ -565,34 +582,61 @@ def handle_cross_reference(self, token, destination):
         with self.current_node_context(text_node):
             self.render_children(token)
 
-    def mock_sphinx_env(self):
+    def mock_sphinx_env(self, configuration=None, sourcedir=None):
         """Load sphinx roles, directives, etc."""
         from sphinx.application import builtin_extensions, Sphinx
         from sphinx.config import Config
         from sphinx.environment import BuildEnvironment
         from sphinx.events import EventManager
         from sphinx.project import Project
         from sphinx.registry import SphinxComponentRegistry
+        from sphinx.util.tags import Tags
 
         class MockSphinx(Sphinx):
-            def __init__(self):
+            """Minimal sphinx init to load roles and directives."""
+
+            def __init__(self, confoverrides=None, srcdir=None):
+                self.extensions = {}
                 self.registry = SphinxComponentRegistry()
-                self.config = Config({}, {})
-                self.events = EventManager(self)
                 self.html_themes = {}
-                self.extensions = {}
+                self.events = EventManager(self)
+                self.tags = Tags(None)
+                self.config = Config({}, confoverrides or {})
+                self.config.pre_init_values()
+                self._init_i18n()
                 for extension in builtin_extensions:
                     self.registry.load_extension(self, extension)
                 # fresh env
-                self.doctreedir = "/doctreedir/"
-                self.srcdir = "/srcdir/"
-                self.project = Project(srcdir="", source_suffix=".md")
+                self.doctreedir = None
+                self.srcdir = srcdir
+                self.confdir = None
+                self.outdir = None
+                self.project = Project(srcdir=srcdir, source_suffix=".md")
                 self.project.docnames = ["mock_docname"]
                 self.env = BuildEnvironment()
                 self.env.setup(self)
                 self.env.temp_data["docname"] = "mock_docname"
-
-        app = MockSphinx()
+                self.builder = None
+
+                if not confoverrides:
+                    return
+
+                # this code is only required for more complex parsing with extensions
+                for extension in self.config.extensions:
+                    self.setup_extension(extension)
+                buildername = "dummy"
+                self.preload_builder(buildername)
+                self.config.init_values()
+                self.events.emit("config-inited", self.config)
+                import tempfile
+
+                with tempfile.TemporaryDirectory() as tempdir:
+                    # creating a builder attempts to make the doctreedir
+                    self.doctreedir = tempdir
+                    self.builder = self.create_builder(buildername)
+                self.doctreedir = None
+
+        app = MockSphinx(confoverrides=configuration, srcdir=sourcedir)
         self.document.settings.env = app.env
         return app
 
@@ -606,20 +650,18 @@ def __enter__(self):
         if not self.load_sphinx_env:
             return super().__enter__()
 
-        # TODO ideally we would just use the sphinx funtctions directly,
-        # but I don't think context managers can be used in `__enter__` methods?
-
         # store currently loaded roles/directives, so we can revert on exit
         self._directives = copy.copy(directives._directives)
         self._roles = copy.copy(roles._roles)
         # Monkey-patch directive and role dispatch,
         # so that sphinx domain-specific markup takes precedence.
-        # (see `sphinx.util.docutils.sphinx_domains`)
-        self._directive_func = directives.directive
-        self._role_func = roles.role
-        directives.directive = self.lookup_directive
-        roles.role = self.lookup_role
-        self._env = self.mock_sphinx_env().env
+        self._env = self.mock_sphinx_env(
+            configuration=self.sphinx_conf, sourcedir=self.sphinx_srcdir
+        ).env
+        from sphinx.util.docutils import sphinx_domains
+
+        self._sphinx_domains = sphinx_domains(self._env)
+        self._sphinx_domains.enable()
 
         return super().__enter__()
 
@@ -629,74 +671,23 @@ def __exit__(self, exception_type, exception_val, traceback):
         # revert loaded roles/directives
         directives._directives = self._directives
         roles._roles = self._roles
+        self._directives = None
+        self._roles = None
         # unregister nodes (see `sphinx.util.docutils.docutils_namespace`)
         from sphinx.util.docutils import additional_nodes, unregister_node
 
         for node in list(additional_nodes):
             unregister_node(node)
             additional_nodes.discard(node)
         # revert directive/role function (see `sphinx.util.docutils.sphinx_domains`)
-        directives.directive = self._directive_func
-        roles.role = self._role_func
+        self._sphinx_domains.disable()
+        self._sphinx_domains = None
+        self._env = None
         return super().__exit__(exception_type, exception_val, traceback)
 
-    def lookup_domain_element(self, type: str, name: str):
-        """Lookup a markup element (directive or role), given its name which can
-        be a full name (with domain).
 
-        Taken from: `sphinx.util.docutils.sphinx_domains`
-        """
-        name = name.lower()
-        # explicit domain given?
-        if ":" in name:
-            domain_name, name = name.split(":", 1)
-            if domain_name in self._env.domains:
-                domain = self._env.get_domain(domain_name)
-                element = getattr(domain, type)(name)
-                if element is not None:
-                    return element, []
-        # else look in the default domain
-        else:
-            def_domain = self._env.temp_data.get("default_domain")
-            if def_domain is not None:
-                element = getattr(def_domain, type)(name)
-                if element is not None:
-                    return element, []
-
-        # always look in the std domain
-        element = getattr(self._env.get_domain("std"), type)(name)
-        if element is not None:
-            return element, []
-
-        from sphinx.util.docutils import ElementLookupError
-
-        raise ElementLookupError
-
-    def lookup_directive(
-        self, directive_name: str, language_module: ModuleType, document: nodes.document
-    ):
-        """Taken from: `sphinx.util.docutils.sphinx_domains`."""
-        from sphinx.util.docutils import ElementLookupError
-
-        try:
-            return self.lookup_domain_element("directive", directive_name)
-        except ElementLookupError:
-            return self._directive_func(directive_name, language_module, document)
-
-    def lookup_role(
-        self,
-        role_name: str,
-        language_module: ModuleType,
-        lineno: int,
-        reporter: Reporter,
-    ):
-        """Taken from: `sphinx.util.docutils.sphinx_domains`."""
-        from sphinx.util.docutils import ElementLookupError
-
-        try:
-            return self.lookup_domain_element("role", role_name)
-        except ElementLookupError:
-            return self._role_func(role_name, language_module, lineno, reporter)
+class MockingError(Exception):
+    """An exception to signal an error during mocking of docutils components."""
 
 
 class MockInliner:
@@ -735,9 +726,9 @@ def __getattr__(self, name):
             msg = "{cls} has not yet implemented attribute '{name}'".format(
                 cls=type(self).__name__, name=name
             )
-            raise NotImplementedError(msg).with_traceback(sys.exc_info()[2])
+            raise MockingError(msg).with_traceback(sys.exc_info()[2])
         msg = "{cls} has no attribute {name}".format(cls=type(self).__name__, name=name)
-        raise AttributeError(msg).with_traceback(sys.exc_info()[2])
+        raise MockingError(msg).with_traceback(sys.exc_info()[2])
 
 
 class MockState:
@@ -872,9 +863,9 @@ def __getattr__(self, name):
             msg = "{cls} has not yet implemented attribute '{name}'".format(
                 cls=type(self).__name__, name=name
             )
-            raise NotImplementedError(msg).with_traceback(sys.exc_info()[2])
+            raise MockingError(msg).with_traceback(sys.exc_info()[2])
         msg = "{cls} has no attribute {name}".format(cls=type(self).__name__, name=name)
-        raise AttributeError(msg).with_traceback(sys.exc_info()[2])
+        raise MockingError(msg).with_traceback(sys.exc_info()[2])
 
 
 class MockStateMachine:
@@ -913,9 +904,9 @@ def __getattr__(self, name):
             msg = "{cls} has not yet implemented attribute '{name}'".format(
                 cls=type(self).__name__, name=name
             )
-            raise NotImplementedError(msg).with_traceback(sys.exc_info()[2])
+            raise MockingError(msg).with_traceback(sys.exc_info()[2])
         msg = "{cls} has no attribute {name}".format(cls=type(self).__name__, name=name)
-        raise AttributeError(msg).with_traceback(sys.exc_info()[2])
+        raise MockingError(msg).with_traceback(sys.exc_info()[2])
 
 
 class MockIncludeDirective:

tests/test_renderers/test_roles_directives.py

@@ -249,7 +249,7 @@ def test_sphinx_directives(sphinx_renderer, name, directive):
     """See https://docutils.sourceforge.io/docs/ref/rst/directives.html"""
     # TODO make sure all directives from domains are added (std and rst are done)
     # (some were erroneously added to roles)
-    if name == "include":
+    if name in ["include", "literalinclude"]:
         # this is tested in the sphinx build level tests
         return
     if name in [