Propogate line ranges to span tokens

myst_parser/block_tokens.py

@@ -1,10 +1,12 @@
 import re
+from typing import List, Union
 
 from mistletoe import block_token, span_token
 import mistletoe.block_tokenizer as tokenizer
-
 from mistletoe.block_token import tokenize, Footnote  # noqa: F401
 
+from myst_parser import traverse
+
 """
 Tokens to be included in the parsing process, in the order specified.
 """
@@ -73,7 +75,22 @@ def __repr__(self):
 class Document(block_token.BlockToken):
     """Document token."""
 
-    def __init__(self, lines, start_line=0, inc_front_matter=True, store_lines=False):
+    def __init__(
+        self,
+        lines: Union[str, List[str]],
+        start_line: int = 0,
+        inc_front_matter: bool = True,
+        store_lines: bool = False,
+        propogate_range: bool = True,
+    ):
+        """Parse lines to a syntax token and its (recursive) children.
+
+        :param lines: string or list of strings
+        :param start_line: the initial line (used for nested parsing)
+        :param inc_front_matter: search for an initial YAML block front matter block
+        :param store_lines: store the lines on the token (as `token._lines`)
+        :param propogate_range: traverse the final syntax tree and add missing ranges
+        """
 
         self.footnotes = {}
         self._start_line = start_line
@@ -96,6 +113,16 @@ def __init__(self, lines, start_line=0, inc_front_matter=True, store_lines=False
             lines = lines[start_line:]
         self.children.extend(tokenize(lines, start_line))
 
+        if propogate_range:
+            # TODO this is a placeholder for implementing span level range storage
+            # (with start/end character attributes)
+            for result in traverse(self):
+                if not hasattr(result.node, "range"):
+                    try:
+                        result.node.range = result.parent.range
+                    except AttributeError:
+                        pass
+
         span_token._root_node = None
         block_token._root_node = None
 

myst_parser/docutils_renderer.py

@@ -69,6 +69,7 @@ def new_document(self, source_path="notset") -> nodes.document:
         return new_document(source_path, settings=settings)
 
     def add_line_and_source_path(self, node, token):
+        """Copy the line number and document source path to the docutils node."""
         try:
             node.line = token.range[0] + 1
         except AttributeError:
@@ -97,7 +98,6 @@ def current_node_context(self, node, append: bool = False):
         self.current_node = current_node
 
     def render_document(self, token):
-        # TODO deal with footnotes
         self.footnotes.update(token.footnotes)
         self.render_children(token)
         return self.document
@@ -148,6 +148,7 @@ def render_target(self, token):
         name = nodes.fully_normalize_name(text)
         target = nodes.target(text)
         target["names"].append(name)
+        self.add_line_and_source_path(target, token)
         self.document.note_explicit_target(target, self.current_node)
         self.current_node.append(target)
 
@@ -167,11 +168,13 @@ def render_line_break(self, token):
 
     def render_strong(self, token):
         node = nodes.strong()
+        self.add_line_and_source_path(node, token)
         with self.current_node_context(node, append=True):
             self.render_children(token)
 
     def render_emphasis(self, token):
         node = nodes.emphasis()
+        self.add_line_and_source_path(node, token)
         with self.current_node_context(node, append=True):
             self.render_children(token)
 
@@ -193,6 +196,7 @@ def render_thematic_break(self, token):
     def render_block_break(self, token):
         block_break = nodes.comment(token.content, token.content)
         block_break["classes"] += ["block_break"]
+        self.add_line_and_source_path(block_break, token)
         self.current_node.append(block_break)
 
     def render_math(self, token):
@@ -202,6 +206,7 @@ def render_math(self, token):
         else:
             content = token.content[1:-1]
             node = nodes.math(content, content)
+        self.add_line_and_source_path(node, token)
         self.current_node.append(node)
 
     def render_block_code(self, token):
@@ -236,6 +241,7 @@ def render_code_fence(self, token):
     def render_inline_code(self, token):
         text = token.children[0].content
         node = nodes.literal(text, text)
+        self.add_line_and_source_path(node, token)
         self.current_node.append(node)
 
     def _is_section_level(self, level, section):
@@ -297,6 +303,7 @@ def handle_cross_reference(self, token, destination):
 
     def render_link(self, token):
         ref_node = nodes.reference()
+        self.add_line_and_source_path(ref_node, token)
         # Check destination is supported for cross-linking and remove extension
         # TODO escape urls?
         destination = token.target
@@ -309,8 +316,6 @@ def render_link(self, token):
         # if ext.replace('.', '') in self.supported:
         #     destination = destination.replace(ext, '')
         ref_node["refuri"] = destination
-        # TODO get line of Link token (requires upstream mistletoe improvements)
-        # ref_node.line = self._get_line(token)
         if token.title:
             ref_node["title"] = token.title
         next_node = ref_node
@@ -334,6 +339,7 @@ def render_link(self, token):
 
     def render_image(self, token):
         img_node = nodes.image()
+        self.add_line_and_source_path(img_node, token)
         img_node["uri"] = token.src
 
         img_node["alt"] = ""
@@ -361,17 +367,15 @@ def render_list(self, token):
         else:
             list_node = nodes.bullet_list()
         # TODO deal with token.loose?
-        # TODO list range
-        # list_node.line = token.range[0]
+        self.add_line_and_source_path(list_node, token)
 
         self.current_node.append(list_node)
         with self.current_node_context(list_node):
             self.render_children(token)
 
     def render_list_item(self, token: myst_block_tokens.ListItem):
         item_node = nodes.list_item()
-        # TODO list item range
-        # node.line = token.range[0]
+        self.add_line_and_source_path(item_node, token)
         self.current_node.append(item_node)
         with self.current_node_context(item_node):
             self.render_children(token)
@@ -419,6 +423,7 @@ def render_auto_link(self, token):
         else:
             refuri = escape_url(token.target)
         ref_node = nodes.reference(token.target, token.target, refuri=refuri)
+        self.add_line_and_source_path(ref_node, token)
         self.current_node.append(ref_node)
 
     def render_html_span(self, token):
@@ -431,7 +436,10 @@ def render_role(self, token):
         content = token.children[0].content
         name = token.name
         # TODO role name white/black lists
-        lineno = 0  # TODO get line number
+        try:
+            lineno = token.range[0]
+        except AttributeError:
+            lineno = 0
         inliner = MockInliner(self, lineno)
         role_func, messages = roles.role(
             name, self.language_module, lineno, self.reporter
@@ -585,8 +593,7 @@ def handle_cross_reference(self, token, destination):
             refexplicit=len(token.children) > 0,
             refwarn=True,
         )
-        # TODO also not correct sourcepos
-        # wrap_node.line = self._get_line(token)
+        self.add_line_and_source_path(wrap_node, token)
         if token.title:
             wrap_node["title"] = token.title
         self.current_node.append(wrap_node)

tests/test_renderers/test_docutils.py

@@ -14,6 +14,8 @@ def render_token(
 ):
     render_func = renderer_mock.render_map[token_name]
     children = mock.MagicMock(spec=list) if children else None
+    if "range" not in kwargs:
+        kwargs["range"] = (0, 0)
     mock_token = mock.Mock(children=children, **kwargs)
     without_attrs = without_attrs or []
     for attr in without_attrs:

tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml

@@ -2,9 +2,18 @@ children:
 - children:
   - children:
     - content: +
+      range:
+      - 1
+      - 1
       type: RawText
+    range:
+    - 1
+    - 1
     type: EscapeSequence
   - content: ++
+    range:
+    - 1
+    - 1
     type: RawText
   range:
   - 1

tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml

@@ -3,6 +3,9 @@ children:
   - children:
     - children:
       - content: item
+        range:
+        - 1
+        - 1
         type: RawText
       range:
       - 1

tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml

@@ -3,6 +3,9 @@ children:
   - content: '+++
 
       '
+    range:
+    - 0
+    - 1
     type: RawText
   language: ''
   range:

tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml

@@ -1,6 +1,9 @@
 children:
 - children:
   - content: a +++
+    range:
+    - 1
+    - 1
     type: RawText
   range:
   - 1

tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml

@@ -2,9 +2,18 @@ children:
 - children:
   - children:
     - content: '%'
+      range:
+      - 1
+      - 1
       type: RawText
+    range:
+    - 1
+    - 1
     type: EscapeSequence
   - content: ' comment'
+    range:
+    - 1
+    - 1
     type: RawText
   range:
   - 1

tests/test_syntax/test_ast/test_comment_follows_list_strings5_.yml

@@ -3,6 +3,9 @@ children:
   - children:
     - children:
       - content: item
+        range:
+        - 1
+        - 1
         type: RawText
       range:
       - 1

tests/test_syntax/test_ast/test_comment_indent_4_strings2_.yml

@@ -3,6 +3,9 @@ children:
   - content: '% comment
 
       '
+    range:
+    - 0
+    - 1
     type: RawText
   language: ''
   range:

tests/test_syntax/test_ast/test_comment_inline_strings4_.yml

@@ -1,6 +1,9 @@
 children:
 - children:
   - content: a % comment
+    range:
+    - 1
+    - 1
     type: RawText
   range:
   - 1

tests/test_syntax/test_ast/test_link_references_ref_escape_strings3_.yml

@@ -1,6 +1,9 @@
 children:
 - children:
   - content: '[ref]'
+    range:
+    - 1
+    - 1
     type: RawText
   range:
   - 1
@@ -9,9 +12,18 @@ children:
 - children:
   - children:
     - content: '['
+      range:
+      - 3
+      - 3
       type: RawText
+    range:
+    - 3
+    - 3
     type: EscapeSequence
   - content: 'ref]: https://google.com "title"'
+    range:
+    - 3
+    - 3
     type: RawText
   range:
   - 3

tests/test_syntax/test_ast/test_link_references_ref_first_strings0_.yml

@@ -2,7 +2,13 @@ children:
 - children:
   - children:
     - content: ref
+      range:
+      - 1
+      - 1
       type: RawText
+    range:
+    - 1
+    - 1
     target: https://google.com
     title: title
     type: Link

tests/test_syntax/test_ast/test_link_references_ref_last_strings1_.yml

@@ -2,7 +2,13 @@ children:
 - children:
   - children:
     - content: ref
+      range:
+      - 3
+      - 3
       type: RawText
+    range:
+    - 3
+    - 3
     target: https://google.com
     title: title
     type: Link

tests/test_syntax/test_ast/test_link_references_ref_syntax_strings2_.yml

@@ -3,8 +3,17 @@ children:
   - children:
     - children:
       - content: syntax
+        range:
+        - 1
+        - 1
         type: RawText
+      range:
+      - 1
+      - 1
       type: Emphasis
+    range:
+    - 1
+    - 1
     target: https://google.com
     title: title
     type: Link

tests/test_syntax/test_ast/test_math_basic_strings0_.yml

@@ -1,6 +1,9 @@
 children:
 - children:
   - content: $a$
+    range:
+    - 1
+    - 1
     type: Math
   range:
   - 1

tests/test_syntax/test_ast/test_math_contains_special_chars_strings1_.yml

@@ -1,6 +1,9 @@
 children:
 - children:
   - content: $a`{_*-%$
+    range:
+    - 1
+    - 1
     type: Math
   range:
   - 1