Update docutils renderer to handle new token Position class.

myst_parser/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.6.0a2"
+__version__ = "0.6.0"
 
 
 def text_to_tokens(text: str):

myst_parser/docutils_renderer.py

@@ -107,15 +107,20 @@ def new_document(self, source_path="notset") -> nodes.document:
     def add_line_and_source_path(self, node, token):
         """Copy the line number and document source path to the docutils node."""
         try:
-            node.line = token.position[0] + 1
+            node.line = token.position.line_start + 1
         except (AttributeError, TypeError):
             pass
         node.source = self.document["source"]
 
-    def nested_render_text(self, text: str, lineno: int):
+    def nested_render_text(self, text: str, lineno: int, token):
         """Render unparsed text."""
-        # TODO propogate SourceLines metadata from parent document
-        lines = SourceLines(text, start_line=lineno, standardize_ends=True)
+        lines = SourceLines(
+            text,
+            start_line=lineno,
+            uri=self.document["source"],
+            metadata=token.position.data,
+            standardize_ends=True,
+        )
         doc_token = myst_block_tokens.Document.read(
             lines, front_matter=True, reset_definitions=False
         )
@@ -190,7 +195,7 @@ def render_front_matter(self, token):
             data = token.get_data()
         except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error:
             msg_node = self.reporter.error(
-                "Front matter block:\n" + str(error), line=token.position[0]
+                "Front matter block:\n" + str(error), line=token.position.line_start
             )
             msg_node += nodes.literal_block(token.content, token.content)
             self.current_node += [msg_node]
@@ -538,7 +543,7 @@ def render_role(self, token):
         name = token.role_name
         # TODO role name white/black lists
         try:
-            lineno = token.position[0]
+            lineno = token.position.line_start
         except (AttributeError, TypeError):
             lineno = 0
         inliner = MockInliner(self, lineno)
@@ -565,7 +570,7 @@ def render_directive(self, token):
         name = token.language[1:-1]
         # TODO directive name white/black lists
         content = token.children[0].content
-        self.document.current_line = token.position[0]
+        self.document.current_line = token.position.line_start
 
         # get directive class
         directive_class, messages = directives.directive(
@@ -575,7 +580,7 @@ def render_directive(self, token):
             error = self.reporter.error(
                 "Unknown directive type '{}'\n".format(name),
                 # nodes.literal_block(content, content),
-                line=token.position[0],
+                line=token.position.line_start,
             )
             self.current_node += [error] + messages
             return
@@ -588,7 +593,7 @@ def render_directive(self, token):
             error = self.reporter.error(
                 "Directive '{}':\n{}".format(name, error),
                 nodes.literal_block(content, content),
-                line=token.position[0],
+                line=token.position.line_start,
             )
             self.current_node += [error]
             return
@@ -602,11 +607,13 @@ def render_directive(self, token):
                 arguments=arguments,
                 options=options,
                 body=body_lines,
-                lineno=token.position[0],
+                token=token,
             )
         else:
-            state_machine = MockStateMachine(self, token.position[0])
-            state = MockState(self, state_machine, token.position[0], token=token)
+            state_machine = MockStateMachine(self, token.position.line_start)
+            state = MockState(
+                self, state_machine, token.position.line_start, token=token
+            )
             directive_instance = directive_class(
                 name=name,
                 # the list of positional arguments
@@ -616,7 +623,7 @@ def render_directive(self, token):
                 # the directive content line by line
                 content=StringList(body_lines, self.document["source"]),
                 # the absolute line number of the first line of the directive
-                lineno=token.position[0],
+                lineno=token.position.line_start,
                 # the line offset of the first line of the content
                 content_offset=0,  # TODO get content offset from `parse_directive_text`
                 # a string containing the entire directive
@@ -630,7 +637,7 @@ def render_directive(self, token):
             result = directive_instance.run()
         except DirectiveError as error:
             msg_node = self.reporter.system_message(
-                error.level, error.msg, line=token.position[0]
+                error.level, error.msg, line=token.position.line_start
             )
             msg_node += nodes.literal_block(content, content)
             result = [msg_node]
@@ -640,7 +647,7 @@ def render_directive(self, token):
                     name, exc.__class__.__name__, exc
                 ),
                 nodes.literal_block(content, content),
-                line=token.position[0],
+                line=token.position.line_start,
             )
             self.current_node += [error]
             return
@@ -867,7 +874,7 @@ def __init__(
         renderer: DocutilsRenderer,
         state_machine: "MockStateMachine",
         lineno: int,
-        token=None,
+        token,
     ):
         self._renderer = renderer
         self._lineno = lineno
@@ -898,7 +905,9 @@ def nested_parse(
         current_match_titles = self.state_machine.match_titles
         self.state_machine.match_titles = match_titles
         with self._renderer.current_node_context(node):
-            self._renderer.nested_render_text(block, self._lineno + input_offset)
+            self._renderer.nested_render_text(
+                block, self._lineno + input_offset, token=self._token
+            )
         self.state_machine.match_titles = current_match_titles
 
     def inline_text(self, text: str, lineno: int):
@@ -914,8 +923,13 @@ def inline_text(self, text: str, lineno: int):
             current_node=paragraph,
             parse_context=get_parse_context(),
         )
-        # TODO propogate SourceLines metadata from parent document
-        lines = SourceLines(text, start_line=self._lineno, standardize_ends=True)
+        lines = SourceLines(
+            text,
+            start_line=self._lineno,
+            uri=self.document["source"],
+            metadata=self._token.position.data,
+            standardize_ends=True,
+        )
         doc_token = myst_block_tokens.Document.read(
             lines, front_matter=False, reset_definitions=False
         )
@@ -1062,7 +1076,7 @@ def __init__(
         arguments: list,
         options: dict,
         body: List[str],
-        lineno: int,
+        token,
     ):
         self.renderer = renderer
         self.document = renderer.document
@@ -1071,7 +1085,8 @@ def __init__(
         self.arguments = arguments
         self.options = options
         self.body = body
-        self.lineno = lineno
+        self.lineno = token.position.line_start
+        self.token = token
 
     def run(self):
 
@@ -1163,7 +1178,7 @@ def run(self):
         if "code" in self.options:
             self.options["source"] = str(path)
             state_machine = MockStateMachine(self.renderer, self.lineno)
-            state = MockState(self.renderer, state_machine, self.lineno)
+            state = MockState(self.renderer, state_machine, self.lineno, self.token)
             codeblock = CodeBlock(
                 name=self.name,
                 arguments=[self.options.pop("code")],
@@ -1186,7 +1201,7 @@ def run(self):
             self.renderer.document["source"] = str(path)
             self.renderer.reporter.source = str(path)
             self.renderer.reporter.get_source_and_line = lambda l: (str(path), l)
-            self.renderer.nested_render_text(file_content, startline)
+            self.renderer.nested_render_text(file_content, startline, token=self.token)
         finally:
             self.renderer.document["source"] = source
             self.renderer.reporter.source = rsource

myst_parser/sphinx_parser.py

@@ -182,9 +182,7 @@ def parse(self, inputstring: str, document: nodes.document):
             # Log to sphinx (e.g. to warn of duplicate link/footnote definitions)
             renderer.parse_context.logger = SPHINX_LOGGER
             lines = SourceLines(
-                inputstring,
-                metadata={"docname": self.env.docname},
-                standardize_ends=True,
+                inputstring, uri=document["source"], standardize_ends=True
             )
             doc = Document.read(lines)
             renderer.render(doc)

setup.py

@@ -36,7 +36,7 @@
     ],
     keywords="markdown lexer parser development docutils sphinx",
     python_requires=">=3.6",
-    install_requires=["mistletoe-ebp~=0.10.0a3"],
+    install_requires=["mistletoe-ebp~=0.10"],
     extras_require={
         "sphinx": ["pyyaml", "docutils>=0.15", "sphinx>=2,<3"],
         "code_style": ["flake8<3.8.0,>=3.7.0", "black", "pre-commit==1.17.0"],

tests/test_syntax/test_ast.py

@@ -20,7 +20,7 @@ def test_render_tokens():
     assert root.children, root.children
 
 
-def test_walk(json_renderer):
+def test_walk(json_renderer, data_regression):
     doc = Document.read(
         dedent(
             """\
@@ -31,27 +31,7 @@ def test_walk(json_renderer):
         )
     )
     tree = [(repr(t.node), repr(t.parent), t.depth) for t in doc.walk()]
-    assert tree == [
-        (
-            "Paragraph(children=2, position=(1, 1))",
-            "Document(children=2, link_definitions=0, footnotes=0, "
-            "footref_order=0, front_matter=None)",
-            1,
-        ),
-        (
-            "Paragraph(children=2, position=(3, 3))",
-            "Document(children=2, link_definitions=0, footnotes=0, "
-            "footref_order=0, front_matter=None)",
-            1,
-        ),
-        ("RawText()", "Paragraph(children=2, position=(1, 1))", 2),
-        ("Strong(children=1)", "Paragraph(children=2, position=(1, 1))", 2),
-        ("RawText()", "Paragraph(children=2, position=(3, 3))", 2),
-        ("Link(target='link', title='')", "Paragraph(children=2, position=(3, 3))", 2),
-        ("RawText()", "Strong(children=1)", 3),
-        ("Emphasis(children=1)", "Link(target='link', title='')", 3),
-        ("RawText()", "Emphasis(children=1)", 4),
-    ]
+    data_regression.check(tree)
 
 
 @pytest.mark.parametrize(

tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml

@@ -3,21 +3,29 @@ children:
   - children:
     - content: +
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: EscapeSequence
   - content: ++
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []

tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml

@@ -4,19 +4,25 @@ children:
     - children:
       - content: item
         position:
-        - 1
-        - 1
+          data: {}
+          line_end: 1
+          line_start: 1
+          uri: null
         type: RawText
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: Paragraph
     leader: '-'
     loose: false
     next_marker: null
     position:
-    - 0
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 0
+      uri: null
     prepend: 2
     type: ListItem
   loose: false

tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml

@@ -4,13 +4,17 @@ children:
 
       '
     position:
-    - 0
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 0
+      uri: null
     type: RawText
   language: ''
   position:
-  - 0
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 0
+    uri: null
   type: BlockCode
 footnotes: {}
 footref_order: []

tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml

@@ -2,12 +2,16 @@ children:
 - children:
   - content: a +++
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []