Skip to content

Commit

Permalink
Propogate line ranges to span tokens
Browse files Browse the repository at this point in the history
  • Loading branch information
chrisjsewell committed Mar 4, 2020
1 parent 625430c commit d1daf87
Show file tree
Hide file tree
Showing 48 changed files with 366 additions and 12 deletions.
31 changes: 29 additions & 2 deletions myst_parser/block_tokens.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import re
from typing import List, Union

from mistletoe import block_token, span_token
import mistletoe.block_tokenizer as tokenizer

from mistletoe.block_token import tokenize, Footnote # noqa: F401

from myst_parser import traverse

"""
Tokens to be included in the parsing process, in the order specified.
"""
Expand Down Expand Up @@ -73,7 +75,22 @@ def __repr__(self):
class Document(block_token.BlockToken):
"""Document token."""

def __init__(self, lines, start_line=0, inc_front_matter=True, store_lines=False):
def __init__(
self,
lines: Union[str, List[str]],
start_line: int = 0,
inc_front_matter: bool = True,
store_lines: bool = False,
propogate_range: bool = True,
):
"""Parse lines to a syntax token and its (recursive) children.
:param lines: string or list of strings
:param start_line: the initial line (used for nested parsing)
:param inc_front_matter: search for an initial YAML block front matter block
:param store_lines: store the lines on the token (as `token._lines`)
:param propogate_range: traverse the final syntax tree and add missing ranges
"""

self.footnotes = {}
self._start_line = start_line
Expand All @@ -96,6 +113,16 @@ def __init__(self, lines, start_line=0, inc_front_matter=True, store_lines=False
lines = lines[start_line:]
self.children.extend(tokenize(lines, start_line))

if propogate_range:
# TODO this is a placeholder for implementing span level range storage
# (with start/end character attributes)
for result in traverse(self):
if not hasattr(result.node, "range"):
try:
result.node.range = result.parent.range
except AttributeError:
pass

span_token._root_node = None
block_token._root_node = None

Expand Down
27 changes: 17 additions & 10 deletions myst_parser/docutils_renderer.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def new_document(self, source_path="notset") -> nodes.document:
return new_document(source_path, settings=settings)

def add_line_and_source_path(self, node, token):
"""Copy the line number and document source path to the docutils node."""
try:
node.line = token.range[0] + 1
except AttributeError:
Expand Down Expand Up @@ -97,7 +98,6 @@ def current_node_context(self, node, append: bool = False):
self.current_node = current_node

def render_document(self, token):
# TODO deal with footnotes
self.footnotes.update(token.footnotes)
self.render_children(token)
return self.document
Expand Down Expand Up @@ -148,6 +148,7 @@ def render_target(self, token):
name = nodes.fully_normalize_name(text)
target = nodes.target(text)
target["names"].append(name)
self.add_line_and_source_path(target, token)
self.document.note_explicit_target(target, self.current_node)
self.current_node.append(target)

Expand All @@ -167,11 +168,13 @@ def render_line_break(self, token):

def render_strong(self, token):
node = nodes.strong()
self.add_line_and_source_path(node, token)
with self.current_node_context(node, append=True):
self.render_children(token)

def render_emphasis(self, token):
node = nodes.emphasis()
self.add_line_and_source_path(node, token)
with self.current_node_context(node, append=True):
self.render_children(token)

Expand All @@ -193,6 +196,7 @@ def render_thematic_break(self, token):
def render_block_break(self, token):
block_break = nodes.comment(token.content, token.content)
block_break["classes"] += ["block_break"]
self.add_line_and_source_path(block_break, token)
self.current_node.append(block_break)

def render_math(self, token):
Expand All @@ -202,6 +206,7 @@ def render_math(self, token):
else:
content = token.content[1:-1]
node = nodes.math(content, content)
self.add_line_and_source_path(node, token)
self.current_node.append(node)

def render_block_code(self, token):
Expand Down Expand Up @@ -236,6 +241,7 @@ def render_code_fence(self, token):
def render_inline_code(self, token):
text = token.children[0].content
node = nodes.literal(text, text)
self.add_line_and_source_path(node, token)
self.current_node.append(node)

def _is_section_level(self, level, section):
Expand Down Expand Up @@ -297,6 +303,7 @@ def handle_cross_reference(self, token, destination):

def render_link(self, token):
ref_node = nodes.reference()
self.add_line_and_source_path(ref_node, token)
# Check destination is supported for cross-linking and remove extension
# TODO escape urls?
destination = token.target
Expand All @@ -309,8 +316,6 @@ def render_link(self, token):
# if ext.replace('.', '') in self.supported:
# destination = destination.replace(ext, '')
ref_node["refuri"] = destination
# TODO get line of Link token (requires upstream mistletoe improvements)
# ref_node.line = self._get_line(token)
if token.title:
ref_node["title"] = token.title
next_node = ref_node
Expand All @@ -334,6 +339,7 @@ def render_link(self, token):

def render_image(self, token):
img_node = nodes.image()
self.add_line_and_source_path(img_node, token)
img_node["uri"] = token.src

img_node["alt"] = ""
Expand Down Expand Up @@ -361,17 +367,15 @@ def render_list(self, token):
else:
list_node = nodes.bullet_list()
# TODO deal with token.loose?
# TODO list range
# list_node.line = token.range[0]
self.add_line_and_source_path(list_node, token)

self.current_node.append(list_node)
with self.current_node_context(list_node):
self.render_children(token)

def render_list_item(self, token: myst_block_tokens.ListItem):
item_node = nodes.list_item()
# TODO list item range
# node.line = token.range[0]
self.add_line_and_source_path(item_node, token)
self.current_node.append(item_node)
with self.current_node_context(item_node):
self.render_children(token)
Expand Down Expand Up @@ -419,6 +423,7 @@ def render_auto_link(self, token):
else:
refuri = escape_url(token.target)
ref_node = nodes.reference(token.target, token.target, refuri=refuri)
self.add_line_and_source_path(ref_node, token)
self.current_node.append(ref_node)

def render_html_span(self, token):
Expand All @@ -431,7 +436,10 @@ def render_role(self, token):
content = token.children[0].content
name = token.name
# TODO role name white/black lists
lineno = 0 # TODO get line number
try:
lineno = token.range[0]
except AttributeError:
lineno = 0
inliner = MockInliner(self, lineno)
role_func, messages = roles.role(
name, self.language_module, lineno, self.reporter
Expand Down Expand Up @@ -585,8 +593,7 @@ def handle_cross_reference(self, token, destination):
refexplicit=len(token.children) > 0,
refwarn=True,
)
# TODO also not correct sourcepos
# wrap_node.line = self._get_line(token)
self.add_line_and_source_path(wrap_node, token)
if token.title:
wrap_node["title"] = token.title
self.current_node.append(wrap_node)
Expand Down
2 changes: 2 additions & 0 deletions tests/test_renderers/test_docutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ def render_token(
):
render_func = renderer_mock.render_map[token_name]
children = mock.MagicMock(spec=list) if children else None
if "range" not in kwargs:
kwargs["range"] = (0, 0)
mock_token = mock.Mock(children=children, **kwargs)
without_attrs = without_attrs or []
for attr in without_attrs:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,18 @@ children:
- children:
- children:
- content: +
range:
- 1
- 1
type: RawText
range:
- 1
- 1
type: EscapeSequence
- content: ++
range:
- 1
- 1
type: RawText
range:
- 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ children:
- children:
- children:
- content: item
range:
- 1
- 1
type: RawText
range:
- 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ children:
- content: '+++
'
range:
- 0
- 1
type: RawText
language: ''
range:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
children:
- children:
- content: a +++
range:
- 1
- 1
type: RawText
range:
- 1
Expand Down
9 changes: 9 additions & 0 deletions tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,18 @@ children:
- children:
- children:
- content: '%'
range:
- 1
- 1
type: RawText
range:
- 1
- 1
type: EscapeSequence
- content: ' comment'
range:
- 1
- 1
type: RawText
range:
- 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ children:
- children:
- children:
- content: item
range:
- 1
- 1
type: RawText
range:
- 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ children:
- content: '% comment
'
range:
- 0
- 1
type: RawText
language: ''
range:
Expand Down
3 changes: 3 additions & 0 deletions tests/test_syntax/test_ast/test_comment_inline_strings4_.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
children:
- children:
- content: a % comment
range:
- 1
- 1
type: RawText
range:
- 1
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
children:
- children:
- content: '[ref]'
range:
- 1
- 1
type: RawText
range:
- 1
Expand All @@ -9,9 +12,18 @@ children:
- children:
- children:
- content: '['
range:
- 3
- 3
type: RawText
range:
- 3
- 3
type: EscapeSequence
- content: 'ref]: https://google.com "title"'
range:
- 3
- 3
type: RawText
range:
- 3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,13 @@ children:
- children:
- children:
- content: ref
range:
- 1
- 1
type: RawText
range:
- 1
- 1
target: https://google.com
title: title
type: Link
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,13 @@ children:
- children:
- children:
- content: ref
range:
- 3
- 3
type: RawText
range:
- 3
- 3
target: https://google.com
title: title
type: Link
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,17 @@ children:
- children:
- children:
- content: syntax
range:
- 1
- 1
type: RawText
range:
- 1
- 1
type: Emphasis
range:
- 1
- 1
target: https://google.com
title: title
type: Link
Expand Down
3 changes: 3 additions & 0 deletions tests/test_syntax/test_ast/test_math_basic_strings0_.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
children:
- children:
- content: $a$
range:
- 1
- 1
type: Math
range:
- 1
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
children:
- children:
- content: $a`{_*-%$
range:
- 1
- 1
type: Math
range:
- 1
Expand Down
Loading

0 comments on commit d1daf87

Please sign in to comment.