Merge 36480b3 into dfaa70e

myst_parser/block_tokens.py

@@ -80,7 +80,7 @@ def read(cls, lines):
 class Document(block_token.BlockToken):
     """Document token."""
 
-    def __init__(self, lines):
+    def __init__(self, lines, start_line=0, inc_front_matter=True):
 
         self.footnotes = {}
         block_token._root_node = self
@@ -89,11 +89,11 @@ def __init__(self, lines):
         if isinstance(lines, str):
             lines = lines.splitlines(keepends=True)
         lines = [line if line.endswith("\n") else "{}\n".format(line) for line in lines]
-        start_line = 0
         self.children = []
         if lines and lines[0].startswith("---"):
             front_matter = FrontMatter(lines)
-            self.children.append(front_matter)
+            if inc_front_matter:
+                self.children.append(front_matter)
             start_line = front_matter.range[1]
             lines = lines[start_line:]
         self.children.extend(tokenize(lines, start_line))

myst_parser/docutils_renderer.py

@@ -1,22 +1,23 @@
 from contextlib import contextmanager
 from itertools import chain
 from os.path import splitext
+from pathlib import Path
 import re
 import sys
 from typing import List, Optional
-from unittest import mock
 from urllib.parse import urlparse, unquote
 
 from docutils import nodes
 from docutils.frontend import OptionParser
 from docutils.languages import get_language
 from docutils.parsers.rst import directives, Directive, DirectiveError, roles
 from docutils.parsers.rst import Parser as RSTParser
+from docutils.parsers.rst.directives.misc import Include
 from docutils.parsers.rst.states import RSTStateMachine, Body, Inliner
 from docutils.utils import new_document, Reporter
 import yaml
 
-from mistletoe import Document, block_token, span_token
+from mistletoe import block_token, span_token
 from mistletoe.base_renderer import BaseRenderer
 
 from myst_parser import span_tokens as myst_span_tokens
@@ -70,6 +71,13 @@ def new_document(self, source_name="") -> nodes.document:
         settings = OptionParser(components=(RSTParser,)).get_default_values()
         return new_document(source_name, settings=settings)
 
+    def nested_render_text(self, text: str, lineno: int):
+        """Render unparsed text."""
+        token = myst_block_tokens.Document(
+            text, start_line=lineno, inc_front_matter=False
+        )
+        self.render(token)
+
     def render_children(self, token):
         for child in token.children:
             self.render(child)
@@ -440,7 +448,12 @@ def render_directive(self, token):
             name, self.language_module, self.document
         )  # type: (Directive, list)
         if not directive_class:
-            self.current_node += messages
+            error = self.reporter.error(
+                "Unknown directive type '{}'\n".format(name),
+                # nodes.literal_block(content, content),
+                line=token.range[0],
+            )
+            self.current_node += [error] + messages
             return
 
         try:
@@ -457,25 +470,36 @@ def render_directive(self, token):
             return
 
         # initialise directive
-        state_machine = MockStateMachine(self, token.range[0])
-        directive_instance = directive_class(
-            name=name,
-            # the list of positional arguments
-            arguments=arguments,
-            # a dictionary mapping option names to values
-            # TODO option parsing
-            options=options,
-            # the directive content line by line
-            content=body_lines,
-            # the absolute line number of the first line of the directive
-            lineno=token.range[0],
-            # the line offset of the first line of the content
-            content_offset=0,
-            # a string containing the entire directive
-            block_text="\n".join(body_lines),
-            state=MockState(self, state_machine, token.range[0]),
-            state_machine=state_machine,
-        )
+        if issubclass(directive_class, Include):
+            directive_instance = MockIncludeDirective(
+                self,
+                name=name,
+                klass=directive_class,
+                arguments=arguments,
+                options=options,
+                body=body_lines,
+                lineno=token.range[0],
+            )
+        else:
+            state_machine = MockStateMachine(self, token.range[0])
+            state = MockState(self, state_machine, token.range[0])
+            directive_instance = directive_class(
+                name=name,
+                # the list of positional arguments
+                arguments=arguments,
+                # a dictionary mapping option names to values
+                options=options,
+                # the directive content line by line
+                content=body_lines,
+                # the absolute line number of the first line of the directive
+                lineno=token.range[0],
+                # the line offset of the first line of the content
+                content_offset=0,
+                # a string containing the entire directive
+                block_text="\n".join(body_lines),
+                state=state,
+                state_machine=state_machine,
+            )
 
         # run directive
         try:
@@ -486,9 +510,14 @@ def render_directive(self, token):
             )
             msg_node += nodes.literal_block(content, content)
             result = [msg_node]
-        except (AttributeError, NotImplementedError):
-            # TODO deal with directives that call unimplemented methods of State/Machine
-            raise
+        except (AttributeError, NotImplementedError) as error:
+            error = self.reporter.error(
+                "Directive '{}' cannot be mocked:\n{}".format(name, error),
+                nodes.literal_block(content, content),
+                line=token.range[0],
+            )
+            self.current_node += [error]
+            return
         assert isinstance(
             result, list
         ), 'Directive "{}" must return a list of nodes.'.format(name)
@@ -572,9 +601,7 @@ def __init__(self, renderer: DocutilsRenderer, lineno: int):
         if not hasattr(self.reporter, "get_source_and_line"):
             # TODO this is called by some roles,
             # but I can't see how that would work in RST?
-            self.reporter.get_source_and_line = mock.Mock(
-                return_value=(self.document.source, lineno)
-            )
+            self.reporter.get_source_and_line = lambda l: (self.document.source, l)
         self.parent = renderer.current_node
         self.language = renderer.language_module
         self.rfc_url = "rfc%d.html"
@@ -640,12 +667,9 @@ def nested_parse(
     ):
         current_match_titles = self.state_machine.match_titles
         self.state_machine.match_titles = match_titles
-        nested_renderer = self._renderer.__class__(
-            document=self.document, current_node=node
-        )
+        with self._renderer.current_node_context(node):
+            self._renderer.nested_render_text(block, self._lineno)
         self.state_machine.match_titles = current_match_titles
-        # TODO deal with starting line number
-        nested_renderer.render(Document(block))
 
     def inline_text(self, text: str, lineno: int):
         # TODO return messages?
@@ -654,7 +678,11 @@ def inline_text(self, text: str, lineno: int):
         renderer = self._renderer.__class__(
             document=self.document, current_node=paragraph
         )
-        renderer.render(Document(text))
+        renderer.render(
+            myst_block_tokens.Document(
+                text, start_line=self._lineno, inc_front_matter=False
+            )
+        )
         textnodes = []
         if paragraph.children:
             # first child should be paragraph
@@ -747,10 +775,15 @@ def __init__(self, renderer: DocutilsRenderer, lineno: int):
         self.node = renderer.current_node
         self.match_titles = True
 
+        # TODO to allow to access like attributes like input_lines,
+        # we would need to store the input lines,
+        # probably via the `Document` token,
+        # and maybe self._lines = lines[:], then for AstRenderer,
+        # ignore private attributes
+
     def get_source_and_line(self, lineno: Optional[int] = None):
         """Return (source, line) tuple for current or given line number."""
-        # TODO return correct line source
-        return "", lineno or self._lineno
+        return self.document.source, lineno or self._lineno
 
     def __getattr__(self, name):
         """This method is only be called if the attribute requested has not
@@ -766,6 +799,160 @@ def __getattr__(self, name):
         raise AttributeError(msg).with_traceback(sys.exc_info()[2])
 
 
+class MockIncludeDirective:
+    """This directive uses a lot of statemachine logic that is not yet mocked.
+    Therefore, we treat it as a special case (at least for now).
+
+    See:
+    https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment
+    """
+
+    def __init__(
+        self,
+        renderer: DocutilsRenderer,
+        name: str,
+        klass: Include,
+        arguments: list,
+        options: dict,
+        body: List[str],
+        lineno: int,
+    ):
+        self.renderer = renderer
+        self.document = renderer.document
+        self.name = name
+        self.klass = klass
+        self.arguments = arguments
+        self.options = options
+        self.body = body
+        self.lineno = lineno
+
+    def run(self):
+
+        from docutils.parsers.rst.directives.body import CodeBlock, NumberLines
+
+        if not self.document.settings.file_insertion_enabled:
+            raise DirectiveError(2, 'Directive "{}" disabled.'.format(self.name))
+
+        source_dir = Path(self.document["source"]).absolute().parent
+        include_arg = "".join([s.strip() for s in self.arguments[0].splitlines()])
+
+        if include_arg.startswith("<") and include_arg.endswith(">"):
+            path = Path(self.klass.standard_include_path).joinpath(include_arg[1:-1])
+        else:
+            path = Path(include_arg)
+        path = source_dir.joinpath(path)
+
+        # read file
+        encoding = self.options.get("encoding", self.document.settings.input_encoding)
+        error_handler = self.document.settings.input_encoding_error_handler
+        # tab_width = self.options.get("tab-width", self.document.settings.tab_width)
+        try:
+            file_content = path.read_text(encoding=encoding, errors=error_handler)
+        except Exception as error:
+            raise DirectiveError(
+                4,
+                'Directive "{}": error reading file: {}\n{error}.'.format(
+                    self.name, path, error
+                ),
+            )
+
+        # get required section of text
+        startline = self.options.get("start-line", None)
+        endline = self.options.get("end-line", None)
+        file_content = "\n".join(file_content.splitlines()[startline:endline])
+        startline = startline or 0
+        for split_on_type in ["start-after", "end-before"]:
+            split_on = self.options.get(split_on_type, None)
+            if not split_on:
+                continue
+            split_index = file_content.find(split_on)
+            if split_index < 0:
+                raise DirectiveError(
+                    4,
+                    'Directive "{}"; option "{}": text not found "{}".'.format(
+                        self.name, split_on_type, split_on
+                    ),
+                )
+            if split_on_type == "start-after":
+                startline += split_index + len(split_on)
+                file_content = file_content[split_index + len(split_on) :]
+            else:
+                file_content = file_content[:split_index]
+
+        if "literal" in self.options:
+            literal_block = nodes.literal_block(
+                file_content, source=str(path), classes=self.options.get("class", [])
+            )
+            literal_block.line = 1
+            self.add_name(literal_block)
+            if "number-lines" in self.options:
+                try:
+                    startline = int(self.options["number-lines"] or 1)
+                except ValueError:
+                    raise DirectiveError(
+                        3, ":number-lines: with non-integer " "start value"
+                    )
+                endline = startline + len(file_content.splitlines())
+                if file_content.endswith("\n"):
+                    file_content = file_content[:-1]
+                tokens = NumberLines([([], file_content)], startline, endline)
+                for classes, value in tokens:
+                    if classes:
+                        literal_block += nodes.inline(value, value, classes=classes)
+                    else:
+                        literal_block += nodes.Text(value)
+            else:
+                literal_block += nodes.Text(file_content)
+            return [literal_block]
+        if "code" in self.options:
+            self.options["source"] = str(path)
+            state_machine = MockStateMachine(self.renderer, self.lineno)
+            state = MockState(self.renderer, state_machine, self.lineno)
+            codeblock = CodeBlock(
+                name=self.name,
+                arguments=[self.options.pop("code")],
+                options=self.options,
+                content=file_content.splitlines(),
+                lineno=self.lineno,
+                content_offset=0,
+                block_text=file_content,
+                state=state,
+                state_machine=state_machine,
+            )
+            return codeblock.run()
+
+        # Here we perform a nested render, but temporarily setup the document/reporter
+        # with the correct document path and lineno for the included file.
+        source = self.renderer.document["source"]
+        rsource = self.renderer.reporter.source
+        line_func = getattr(self.renderer.reporter, "get_source_and_line", None)
+        try:
+            self.renderer.document["source"] = str(path)
+            self.renderer.reporter.source = str(path)
+            self.renderer.reporter.get_source_and_line = lambda l: (str(path), l)
+            self.renderer.nested_render_text(file_content, startline)
+        finally:
+            self.renderer.document["source"] = source
+            self.renderer.reporter.source = rsource
+            if line_func is not None:
+                self.renderer.reporter.get_source_and_line = line_func
+            else:
+                del self.renderer.reporter.get_source_and_line
+        return []
+
+    def add_name(self, node):
+        """Append self.options['name'] to node['names'] if it exists.
+
+        Also normalize the name string and register it as explicit target.
+        """
+        if "name" in self.options:
+            name = nodes.fully_normalize_name(self.options.pop("name"))
+            if "name" in node:
+                del node["name"]
+            node["names"].append(name)
+            self.renderer.document.note_explicit_target(node, node)
+
+
 def dict_to_docinfo(data):
     """Render a key/val pair as a docutils field node."""
     # TODO this data could be used to support default option values for directives

tests/test_renderers/test_roles_directives.py

@@ -277,7 +277,10 @@ def test_docutils_directives(renderer, name, directive):
 )
 def test_sphinx_directives(sphinx_renderer, name, directive):
     """See https://docutils.sourceforge.io/docs/ref/rst/directives.html"""
-    if name in ["csv-table", "meta", "include"]:
+    if name == "include":
+        # this is tested in the sphinx build level tests
+        return
+    if name in ["csv-table", "meta"]:
         # TODO fix skips
         pytest.skip("awaiting fix")
     arguments = " ".join(directive["args"])

tests/test_sphinx/sourcedirs/includes/conf.py

@@ -0,0 +1,2 @@
+extensions = ["myst_parser"]
+exclude_patterns = ["_build", "*.inc.md", "**/*.inc.md"]

tests/test_sphinx/sourcedirs/includes/include1.inc.md

@@ -0,0 +1,10 @@
+---
+orphan: true
+---
+(inc_header)=
+## A Sub-Heading in Include
+
+Some text with *syntax*
+
+```{include} subfolder/include2.inc.md
+```

tests/test_sphinx/sourcedirs/includes/include_code.py

@@ -0,0 +1,2 @@
+def a_func(param):
+    print(param)