diff --git a/myst_parser/block_tokens.py b/myst_parser/block_tokens.py
index de9eb5c4..0738bbbd 100644
--- a/myst_parser/block_tokens.py
+++ b/myst_parser/block_tokens.py
@@ -80,7 +80,7 @@ def read(cls, lines):
class Document(block_token.BlockToken):
"""Document token."""
- def __init__(self, lines):
+ def __init__(self, lines, start_line=0, inc_front_matter=True):
self.footnotes = {}
block_token._root_node = self
@@ -89,11 +89,11 @@ def __init__(self, lines):
if isinstance(lines, str):
lines = lines.splitlines(keepends=True)
lines = [line if line.endswith("\n") else "{}\n".format(line) for line in lines]
- start_line = 0
self.children = []
if lines and lines[0].startswith("---"):
front_matter = FrontMatter(lines)
- self.children.append(front_matter)
+ if inc_front_matter:
+ self.children.append(front_matter)
start_line = front_matter.range[1]
lines = lines[start_line:]
self.children.extend(tokenize(lines, start_line))
diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
index bcd484c6..9838b15f 100644
--- a/myst_parser/docutils_renderer.py
+++ b/myst_parser/docutils_renderer.py
@@ -1,6 +1,7 @@
from contextlib import contextmanager
from itertools import chain
from os.path import splitext
+from pathlib import Path
import re
import sys
from typing import List, Optional
@@ -12,11 +13,12 @@
from docutils.languages import get_language
from docutils.parsers.rst import directives, Directive, DirectiveError, roles
from docutils.parsers.rst import Parser as RSTParser
+from docutils.parsers.rst.directives.misc import Include
from docutils.parsers.rst.states import RSTStateMachine, Body, Inliner
from docutils.utils import new_document, Reporter
import yaml
-from mistletoe import Document, block_token, span_token
+from mistletoe import block_token, span_token
from mistletoe.base_renderer import BaseRenderer
from myst_parser import span_tokens as myst_span_tokens
@@ -70,6 +72,13 @@ def new_document(self, source_name="") -> nodes.document:
settings = OptionParser(components=(RSTParser,)).get_default_values()
return new_document(source_name, settings=settings)
+ def nested_render_text(self, text: str, lineno: int):
+ """Render unparsed text."""
+ token = myst_block_tokens.Document(
+ text, start_line=lineno, inc_front_matter=False
+ )
+ self.render(token)
+
def render_children(self, token):
for child in token.children:
self.render(child)
@@ -457,25 +466,36 @@ def render_directive(self, token):
return
# initialise directive
- state_machine = MockStateMachine(self, token.range[0])
- directive_instance = directive_class(
- name=name,
- # the list of positional arguments
- arguments=arguments,
- # a dictionary mapping option names to values
- # TODO option parsing
- options=options,
- # the directive content line by line
- content=body_lines,
- # the absolute line number of the first line of the directive
- lineno=token.range[0],
- # the line offset of the first line of the content
- content_offset=0,
- # a string containing the entire directive
- block_text="\n".join(body_lines),
- state=MockState(self, state_machine, token.range[0]),
- state_machine=state_machine,
- )
+ if issubclass(directive_class, Include):
+ directive_instance = MockIncludeDirective(
+ self,
+ name=name,
+ klass=directive_class,
+ arguments=arguments,
+ options=options,
+ body=body_lines,
+ lineno=token.range[0],
+ )
+ else:
+ state_machine = MockStateMachine(self, token.range[0])
+ state = MockState(self, state_machine, token.range[0])
+ directive_instance = directive_class(
+ name=name,
+ # the list of positional arguments
+ arguments=arguments,
+ # a dictionary mapping option names to values
+ options=options,
+ # the directive content line by line
+ content=body_lines,
+ # the absolute line number of the first line of the directive
+ lineno=token.range[0],
+ # the line offset of the first line of the content
+ content_offset=0,
+ # a string containing the entire directive
+ block_text="\n".join(body_lines),
+ state=state,
+ state_machine=state_machine,
+ )
# run directive
try:
@@ -486,9 +506,14 @@ def render_directive(self, token):
)
msg_node += nodes.literal_block(content, content)
result = [msg_node]
- except (AttributeError, NotImplementedError):
- # TODO deal with directives that call unimplemented methods of State/Machine
- raise
+ except (AttributeError, NotImplementedError) as error:
+ error = self.reporter.error(
+ "Directive '{}' cannot be mocked:\n{}".format(name, error),
+ nodes.literal_block(content, content),
+ line=token.range[0],
+ )
+ self.current_node += [error]
+ return
assert isinstance(
result, list
), 'Directive "{}" must return a list of nodes.'.format(name)
@@ -640,12 +665,9 @@ def nested_parse(
):
current_match_titles = self.state_machine.match_titles
self.state_machine.match_titles = match_titles
- nested_renderer = self._renderer.__class__(
- document=self.document, current_node=node
- )
+ with self._renderer.current_node_context(node):
+ self._renderer.nested_render_text(block, self._lineno)
self.state_machine.match_titles = current_match_titles
- # TODO deal with starting line number
- nested_renderer.render(Document(block))
def inline_text(self, text: str, lineno: int):
# TODO return messages?
@@ -654,7 +676,11 @@ def inline_text(self, text: str, lineno: int):
renderer = self._renderer.__class__(
document=self.document, current_node=paragraph
)
- renderer.render(Document(text))
+ renderer.render(
+ myst_block_tokens.Document(
+ text, start_line=self._lineno, inc_front_matter=False
+ )
+ )
textnodes = []
if paragraph.children:
# first child should be paragraph
@@ -747,10 +773,15 @@ def __init__(self, renderer: DocutilsRenderer, lineno: int):
self.node = renderer.current_node
self.match_titles = True
+ # TODO to allow to access like attributes like input_lines,
+ # we would need to store the input lines,
+ # probably via the `Document` token,
+ # and maybe self._lines = lines[:], then for AstRenderer,
+ # ignore private attributes
+
def get_source_and_line(self, lineno: Optional[int] = None):
"""Return (source, line) tuple for current or given line number."""
- # TODO return correct line source
- return "", lineno or self._lineno
+ return self.document.source, lineno or self._lineno
def __getattr__(self, name):
"""This method is only be called if the attribute requested has not
@@ -766,6 +797,148 @@ def __getattr__(self, name):
raise AttributeError(msg).with_traceback(sys.exc_info()[2])
+class MockIncludeDirective:
+ """This directive uses a lot of statemachine logic that is not yet mocked.
+ Therefore, we treat it as a special case (at least for now).
+
+ See:
+ https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment
+ """
+
+ def __init__(
+ self,
+ renderer: DocutilsRenderer,
+ name: str,
+ klass: Include,
+ arguments: list,
+ options: dict,
+ body: List[str],
+ lineno: int,
+ ):
+ self.renderer = renderer
+ self.document = renderer.document
+ self.name = name
+ self.klass = klass
+ self.arguments = arguments
+ self.options = options
+ self.body = body
+ self.lineno = lineno
+
+ def run(self):
+
+ from docutils.parsers.rst.directives.body import CodeBlock, NumberLines
+
+ if not self.document.settings.file_insertion_enabled:
+ raise DirectiveError(2, 'Directive "{}" disabled.'.format(self.name))
+
+ source_dir = Path(self.document["source"]).absolute().parent
+ include_arg = "".join([s.strip() for s in self.arguments[0].splitlines()])
+
+ if include_arg.startswith("<") and include_arg.endswith(">"):
+ path = Path(self.klass.standard_include_path).joinpath(include_arg[1:-1])
+ else:
+ path = Path(include_arg)
+ path = source_dir.joinpath(path)
+
+ # read file
+ encoding = self.options.get("encoding", self.document.settings.input_encoding)
+ error_handler = self.document.settings.input_encoding_error_handler
+ # tab_width = self.options.get("tab-width", self.document.settings.tab_width)
+ try:
+ file_content = path.read_text(encoding=encoding, errors=error_handler)
+ except Exception as error:
+ raise DirectiveError(
+ 4,
+ 'Directive "{}": error reading file: {}\n{error}.'.format(
+ self.name, path, error
+ ),
+ )
+
+ # get required section of text
+ startline = self.options.get("start-line", None)
+ endline = self.options.get("end-line", None)
+ file_content = "\n".join(file_content.splitlines()[startline:endline])
+ for split_on_type in ["start-after", "end-before"]:
+ split_on = self.options.get(split_on_type, None)
+ if not split_on:
+ continue
+ split_index = file_content.find(split_on)
+ if split_index < 0:
+ raise DirectiveError(
+ 4,
+ 'Directive "{}"; option "{}": text not found "{}".'.format(
+ self.name, split_on_type, split_on
+ ),
+ )
+ if split_on_type == "start-after":
+ file_content = file_content[split_index + len(split_on) :]
+ else:
+ file_content = file_content[:split_index]
+
+ if "literal" in self.options:
+ literal_block = nodes.literal_block(
+ file_content, source=str(path), classes=self.options.get("class", [])
+ )
+ literal_block.line = 1
+ self.add_name(literal_block)
+ if "number-lines" in self.options:
+ try:
+ startline = int(self.options["number-lines"] or 1)
+ except ValueError:
+ raise DirectiveError(
+ 3, ":number-lines: with non-integer " "start value"
+ )
+ endline = startline + len(file_content.splitlines())
+ if file_content.endswith("\n"):
+ file_content = file_content[:-1]
+ tokens = NumberLines([([], file_content)], startline, endline)
+ for classes, value in tokens:
+ if classes:
+ literal_block += nodes.inline(value, value, classes=classes)
+ else:
+ literal_block += nodes.Text(value)
+ else:
+ literal_block += nodes.Text(file_content)
+ return [literal_block]
+ if "code" in self.options:
+ self.options["source"] = str(path)
+ state_machine = MockStateMachine(self.renderer, self.lineno)
+ state = MockState(self.renderer, state_machine, self.lineno)
+ codeblock = CodeBlock(
+ name=self.name,
+ arguments=[self.options.pop("code")],
+ options=self.options,
+ content=file_content.splitlines(),
+ lineno=self.lineno,
+ content_offset=0,
+ block_text=file_content,
+ state=state,
+ state_machine=state_machine,
+ )
+ return codeblock.run()
+
+ source = self.renderer.document["source"]
+ try:
+ self.renderer.document["source"] = str(path)
+ self.renderer.nested_render_text(file_content, self.lineno)
+ finally:
+ self.renderer.document["source"] = source
+
+ return []
+
+ def add_name(self, node):
+ """Append self.options['name'] to node['names'] if it exists.
+
+ Also normalize the name string and register it as explicit target.
+ """
+ if "name" in self.options:
+ name = nodes.fully_normalize_name(self.options.pop("name"))
+ if "name" in node:
+ del node["name"]
+ node["names"].append(name)
+ self.renderer.document.note_explicit_target(node, node)
+
+
def dict_to_docinfo(data):
"""Render a key/val pair as a docutils field node."""
# TODO this data could be used to support default option values for directives
diff --git a/tests/test_renderers/test_roles_directives.py b/tests/test_renderers/test_roles_directives.py
index 66242c31..c672ebbb 100644
--- a/tests/test_renderers/test_roles_directives.py
+++ b/tests/test_renderers/test_roles_directives.py
@@ -277,7 +277,10 @@ def test_docutils_directives(renderer, name, directive):
)
def test_sphinx_directives(sphinx_renderer, name, directive):
"""See https://docutils.sourceforge.io/docs/ref/rst/directives.html"""
- if name in ["csv-table", "meta", "include"]:
+ if name == "include":
+ # this is tested in the sphinx build level tests
+ return
+ if name in ["csv-table", "meta"]:
# TODO fix skips
pytest.skip("awaiting fix")
arguments = " ".join(directive["args"])
diff --git a/tests/test_sphinx/sourcedirs/includes/conf.py b/tests/test_sphinx/sourcedirs/includes/conf.py
new file mode 100644
index 00000000..4cf4e938
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/includes/conf.py
@@ -0,0 +1,2 @@
+extensions = ["myst_parser"]
+exclude_patterns = ["_build", "*.inc.md", "**/*.inc.md"]
diff --git a/tests/test_sphinx/sourcedirs/includes/example1.jpg b/tests/test_sphinx/sourcedirs/includes/example1.jpg
new file mode 100644
index 00000000..1c1e83ea
Binary files /dev/null and b/tests/test_sphinx/sourcedirs/includes/example1.jpg differ
diff --git a/tests/test_sphinx/sourcedirs/includes/include1.inc.md b/tests/test_sphinx/sourcedirs/includes/include1.inc.md
new file mode 100644
index 00000000..35b43ec5
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/includes/include1.inc.md
@@ -0,0 +1,10 @@
+---
+orphan: true
+---
+(inc_header)=
+## A Sub-Heading in Include
+
+Some text with *syntax*
+
+```{include} subfolder/include2.inc.md
+```
diff --git a/tests/test_sphinx/sourcedirs/includes/include_code.py b/tests/test_sphinx/sourcedirs/includes/include_code.py
new file mode 100644
index 00000000..31589545
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/includes/include_code.py
@@ -0,0 +1,2 @@
+def a_func(param):
+ print(param)
diff --git a/tests/test_sphinx/sourcedirs/includes/include_literal.txt b/tests/test_sphinx/sourcedirs/includes/include_literal.txt
new file mode 100644
index 00000000..5bb9418a
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/includes/include_literal.txt
@@ -0,0 +1,6 @@
+This should be *literal*
+
+Lots
+of
+lines
+so we can select some
diff --git a/tests/test_sphinx/sourcedirs/includes/index.md b/tests/test_sphinx/sourcedirs/includes/index.md
new file mode 100644
index 00000000..dc904e37
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/includes/index.md
@@ -0,0 +1,31 @@
+# Main Title
+
+```{include} include1.inc.md
+```
+
+{ref}`inc_header`
+
+```{include} include_code.py
+:code: python
+```
+
+```{include} include_code.py
+:code: python
+:number-lines: 0
+```
+
+```{include} include_literal.txt
+:literal:
+```
+
+```{include} include_literal.txt
+:literal:
+:name: literal_ref
+:start-line: 2
+:end-before: lines
+:number-lines: 0
+```
+
+### A Sub-sub-Heading
+
+some more text
diff --git a/tests/test_sphinx/sourcedirs/includes/subfolder/example2.jpg b/tests/test_sphinx/sourcedirs/includes/subfolder/example2.jpg
new file mode 100644
index 00000000..1c1e83ea
Binary files /dev/null and b/tests/test_sphinx/sourcedirs/includes/subfolder/example2.jpg differ
diff --git a/tests/test_sphinx/sourcedirs/includes/subfolder/include2.inc.md b/tests/test_sphinx/sourcedirs/includes/subfolder/include2.inc.md
new file mode 100644
index 00000000..1ccbbb48
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/includes/subfolder/include2.inc.md
@@ -0,0 +1,15 @@
+## A Sub-Heading in Nested Include
+
+Some other text with **syntax**
+
+This relative path will refer to the importing file:
+
+```{figure} example1.jpg
+Caption
+```
+
+This absolute path will refer to the project root (where the `conf.py` is):
+
+```{figure} /subfolder/example2.jpg
+Caption
+```
diff --git a/tests/test_sphinx/test_sphinx_builds.py b/tests/test_sphinx/test_sphinx_builds.py
index 37a17a72..765085d4 100644
--- a/tests/test_sphinx/test_sphinx_builds.py
+++ b/tests/test_sphinx/test_sphinx_builds.py
@@ -33,9 +33,11 @@ def test_basic(app, status, warning, get_sphinx_app_output):
"""
import os
import pathlib
+import pickle
import shutil
import pytest
+from docutils.nodes import document
from sphinx.testing.path import path
@@ -83,6 +85,35 @@ def read(
return read
+@pytest.fixture
+def get_sphinx_app_doctree(file_regression):
+ def read(
+ app,
+ filename="index.doctree",
+ folder="doctrees",
+ encoding="utf-8",
+ regress=False,
+ ):
+
+ outpath = path(os.path.join(str(app.srcdir), "_build", folder, filename))
+ if not outpath.exists():
+ raise IOError("no output file exists: {}".format(outpath))
+
+ with open(outpath, "rb") as handle:
+ doctree = pickle.load(handle) # type: document
+
+ # convert absolute filenames
+ for node in doctree.traverse(lambda n: "source" in n):
+ node["source"] = pathlib.Path(node["source"]).name
+
+ if regress:
+ file_regression.check(doctree.pformat(), extension=".xml")
+
+ return doctree
+
+ return read
+
+
@pytest.mark.sphinx(
buildername="html", srcdir=os.path.join(SOURCE_DIR, "basic"), freshenv=True
)
@@ -95,3 +126,25 @@ def test_basic(app, status, warning, get_sphinx_app_output, remove_sphinx_builds
assert warnings == ""
get_sphinx_app_output(app, filename="content.html", regress_html=True)
+
+
+@pytest.mark.sphinx(
+ buildername="html", srcdir=os.path.join(SOURCE_DIR, "includes"), freshenv=True
+)
+def test_includes(
+ app,
+ status,
+ warning,
+ get_sphinx_app_doctree,
+ get_sphinx_app_output,
+ remove_sphinx_builds,
+):
+ """Test of include directive."""
+ app.build()
+
+ assert "build succeeded" in status.getvalue() # Build succeeded
+ warnings = warning.getvalue().strip()
+ assert warnings == ""
+
+ get_sphinx_app_doctree(app, filename="index.doctree", regress=True)
+ get_sphinx_app_output(app, filename="index.html", regress_html=True)
diff --git a/tests/test_sphinx/test_sphinx_builds/test_includes.html b/tests/test_sphinx/test_sphinx_builds/test_includes.html
new file mode 100644
index 00000000..511243a4
--- /dev/null
+++ b/tests/test_sphinx/test_sphinx_builds/test_includes.html
@@ -0,0 +1,118 @@
+
+
+
+
+
+ Main Title
+
+
+
+
+
+ A Sub-Heading in Include
+
+
+
+ Some text with
+
+ syntax
+
+
+
+
+
+ A Sub-Heading in Nested Include
+
+
+
+ Some other text with
+
+ syntax
+
+
+
+ This relative path will refer to the importing file:
+
+
+
+ This absolute path will refer to the project root (where the
+
+
+ conf.py
+
+
+ is):
+
+
+
+
+
+ A Sub-Heading in Include
+
+
+
+
+
+
def a_func ( param ):
+ print ( param )
+
+
+
+
1 def a_func ( param ):
+2 print ( param )
+
+
+
This should be * literal *
+
+Lots
+of
+lines
+so we can select some
+
+
+
+
1 Lots
+2 of
+
+
+ A Sub-sub-Heading
+
+
+
+ some more text
+
+
+
+
+
+
+
diff --git a/tests/test_sphinx/test_sphinx_builds/test_includes.xml b/tests/test_sphinx/test_sphinx_builds/test_includes.xml
new file mode 100644
index 00000000..6e9c7966
--- /dev/null
+++ b/tests/test_sphinx/test_sphinx_builds/test_includes.xml
@@ -0,0 +1,105 @@
+
+
+
+ Main Title
+
+
+
+ A Sub-Heading in Include
+
+ Some text with
+
+ syntax
+
+
+ A Sub-Heading in Nested Include
+
+ Some other text with
+
+ syntax
+
+ This relative path will refer to the importing file:
+
+
+
+ Caption
+
+ This absolute path will refer to the project root (where the
+
+ conf.py
+ is):
+
+
+
+ Caption
+
+
+
+ inc_header
+
+
+ def
+
+
+ a_func
+
+ (
+
+ param
+
+ ):
+
+
+
+ print
+
+ (
+
+ param
+
+ )
+
+
+ 1
+
+ def
+
+
+ a_func
+
+ (
+
+ param
+
+ ):
+
+
+ 2
+
+
+ print
+
+ (
+
+ param
+
+ )
+
+ This should be *literal*
+
+ Lots
+ of
+ lines
+ so we can select some
+
+
+ 1
+ Lots
+
+ 2
+ of
+
+
+ A Sub-sub-Heading
+
+ some more text