Add position to span tokens

This is a placeholder for actually implementing span level position recording in a later commit

mistletoe/__init__.py

@@ -2,7 +2,7 @@
 Make mistletoe easier to import.
 """
 
-__version__ = "0.9.4a1"
+__version__ = "0.9.4a2"
 __all__ = [
     "renderers",
     "base_elements",

mistletoe/renderers/base.py

@@ -46,8 +46,9 @@ class BaseRenderer:
 
     :param render_map: maps tokens to their corresponding render functions.
     :type render_map: dict
-    :param extras: a list of custom tokens to be added to the parsing process.
-    :type extras: list
+    :param parse_context: container for the instatiated tokens,
+        and other global parsing variables. These will be re-instatiated on `__enter__`
+
     """
 
     default_block_tokens = (
@@ -92,8 +93,9 @@ def __init__(self, find_blocks=None, find_spans=None):
         for token in chain(
             self.parse_context.block_tokens, self.parse_context.span_tokens
         ):
-            render_func = getattr(self, self._cls_to_func(token.__name__))
-            self.render_map[token.__name__] = render_func
+            if token.__name__ not in self.render_map:
+                render_func = getattr(self, self._cls_to_func(token.__name__))
+                self.render_map[token.__name__] = render_func
 
         self.link_definitions = {}
 

mistletoe/span_tokens.py

@@ -2,7 +2,7 @@
 Built-in span-level token classes.
 """
 import re
-from typing import Pattern
+from typing import Pattern, Tuple
 
 import attr
 
@@ -66,7 +66,14 @@ class InlineCode(SpanToken):
     parse_inner = False
     parse_group = 2
 
-    children = attr.ib(metadata={"doc": "a single RawText node for alternative text."})
+    children = attr.ib(
+        repr=False, metadata={"doc": "a single RawText node for alternative text."}
+    )
+    position: Tuple[int, int] = attr.ib(
+        default=None,
+        repr=False,
+        metadata={"doc": "Line position in source text (start, end)"},
+    )
 
     @classmethod
     def read(cls, match: Pattern):
@@ -88,7 +95,12 @@ class Image(SpanToken):
 
     src: str = attr.ib(metadata={"doc": "image source"})
     title: str = attr.ib(default=None, metadata={"doc": "image title"})
-    children = attr.ib(factory=list, metadata={"doc": "alternative text."})
+    children = attr.ib(factory=list, repr=False, metadata={"doc": "alternative text."})
+    position: Tuple[int, int] = attr.ib(
+        default=None,
+        repr=False,
+        metadata={"doc": "Line position in source text (start, end)"},
+    )
 
     @classmethod
     def read(cls, match: Pattern):
@@ -104,7 +116,12 @@ class Link(SpanToken):
 
     target: str = attr.ib(metadata={"doc": "link target"})
     title: str = attr.ib(default=None, metadata={"doc": "link title"})
-    children = attr.ib(factory=list, metadata={"doc": "link text."})
+    children = attr.ib(factory=list, repr=False, metadata={"doc": "link text."})
+    position: Tuple[int, int] = attr.ib(
+        default=None,
+        repr=False,
+        metadata={"doc": "Line position in source text (start, end)"},
+    )
 
     @classmethod
     def read(cls, match: Pattern):
@@ -128,7 +145,14 @@ class AutoLink(SpanToken):
 
     target: str = attr.ib(metadata={"doc": "link target"})
     mailto: bool = attr.ib(metadata={"doc": "if the link is an email"})
-    children = attr.ib(metadata={"doc": "a single RawText node for alternative text."})
+    children = attr.ib(
+        repr=False, metadata={"doc": "a single RawText node for alternative text."}
+    )
+    position: Tuple[int, int] = attr.ib(
+        default=None,
+        repr=False,
+        metadata={"doc": "Line position in source text (start, end)"},
+    )
 
     @classmethod
     def read(cls, match: Pattern):
@@ -154,7 +178,14 @@ class EscapeSequence(SpanToken):
     parse_inner = False
     precedence = 2
 
-    children = attr.ib(metadata={"doc": "a single RawText node for alternative text."})
+    children = attr.ib(
+        repr=False, metadata={"doc": "a single RawText node for alternative text."}
+    )
+    position: Tuple[int, int] = attr.ib(
+        default=None,
+        repr=False,
+        metadata={"doc": "Line position in source text (start, end)"},
+    )
 
     @classmethod
     def read(cls, match: Pattern):
@@ -176,8 +207,13 @@ class LineBreak(SpanToken):
     parse_inner = False
     parse_group = 0
 
-    content: bool = attr.ib(default="", metadata={"doc": "raw content."})
+    content: bool = attr.ib(default="", repr=False, metadata={"doc": "raw content."})
     soft: bool = attr.ib(metadata={"doc": "if the break is soft or hard."})
+    position: Tuple[int, int] = attr.ib(
+        default=None,
+        repr=False,
+        metadata={"doc": "Line position in source text (start, end)"},
+    )
 
     @classmethod
     def read(cls, match: Pattern):
@@ -195,7 +231,14 @@ class RawText(SpanToken):
     instead of a match object. Also, all recursions should bottom out here.
     """
 
-    content: bool = attr.ib(metadata={"doc": "raw string content of the token"})
+    content: bool = attr.ib(
+        repr=False, metadata={"doc": "raw string content of the token"}
+    )
+    position: Tuple[int, int] = attr.ib(
+        default=None,
+        repr=False,
+        metadata={"doc": "Line position in source text (start, end)"},
+    )
 
     @classmethod
     def read(cls, content: str):

test/test_block_token/test_atx_heading_enclosing_hashes.yml

@@ -2,6 +2,7 @@
     children:
     - RawText:
         content: heading 3
+        position: null
     level: 1
     position:
     - 1

test/test_block_token/test_atx_heading_heading_in_paragraph.yml

@@ -2,13 +2,15 @@
     children:
     - RawText:
         content: foo
+        position: null
     position:
     - 1
     - 1
 - Heading:
     children:
     - RawText:
         content: heading
+        position: null
     level: 1
     position:
     - 2
@@ -17,6 +19,7 @@
     children:
     - RawText:
         content: bar
+        position: null
     position:
     - 3
     - 3

test/test_block_token/test_atx_heading_match.yml

@@ -2,6 +2,7 @@
     children:
     - RawText:
         content: heading 3
+        position: null
     level: 3
     position:
     - 1

test/test_block_token/test_atx_heading_too_many_hashes.yml

@@ -2,6 +2,7 @@
     children:
     - RawText:
         content: '####### paragraph'
+        position: null
     position:
     - 1
     - 1

test/test_block_token/test_block_code_match.yml

@@ -6,6 +6,7 @@
           mkdir test
 
           '
+        position: null
     language: ''
     position:
     - 0

test/test_block_token/test_doc_read_auto_splitlines.yml

@@ -4,16 +4,21 @@ Document:
       children:
       - RawText:
           content: some
+          position: null
       - LineBreak:
           content: ''
+          position: null
           soft: true
       - RawText:
           content: continual
+          position: null
       - LineBreak:
           content: ''
+          position: null
           soft: true
       - RawText:
           content: lines
+          position: null
       position:
       - 1
       - 3

test/test_block_token/test_doc_read_basic.yml

@@ -4,6 +4,7 @@ Document:
       children:
       - RawText:
           content: heading
+          position: null
       level: 1
       position:
       - 1
@@ -12,18 +13,24 @@ Document:
       children:
       - RawText:
           content: paragraph
+          position: null
       - LineBreak:
           content: ''
+          position: null
           soft: true
       - RawText:
           content: with
+          position: null
       - LineBreak:
           content: ''
+          position: null
           soft: true
       - InlineCode:
           children:
           - RawText:
               content: code
+              position: null
+          position: null
       position:
       - 3
       - 5

test/test_block_token/test_doc_read_front_matter_ignore.yml

@@ -8,6 +8,7 @@ Document:
       children:
       - RawText:
           content: 'a: 1'
+          position: null
       level: 2
       position:
       - 2

test/test_block_token/test_fenced_code_lazy_continuation.yml

@@ -8,6 +8,7 @@
           mkdir test
 
           '
+        position: null
     language: sh
     position:
     - 1

test/test_block_token/test_fenced_code_mixed_tick_tilda.yml

@@ -9,6 +9,7 @@
           ```
 
           '
+        position: null
     language: markdown
     position:
     - 1

test/test_block_token/test_fenced_code_no_wrapping_newlines.yml

@@ -3,6 +3,7 @@
     children:
     - RawText:
         content: hey
+        position: null
     language: ''
     position:
     - 1
@@ -11,6 +12,7 @@
     children:
     - RawText:
         content: paragraph
+        position: null
     position:
     - 4
     - 4

test/test_block_token/test_fenced_code_unclosed.yml

@@ -3,6 +3,7 @@
     children:
     - RawText:
         content: hey
+        position: null
     language: ''
     position:
     - 1

test/test_block_token/test_fenced_code_with_ticks.yml

@@ -7,6 +7,7 @@
           mkdir test
 
           '
+        position: null
     language: sh
     position:
     - 1

test/test_block_token/test_fenced_code_with_tildas.yml

@@ -7,6 +7,7 @@
           mkdir test
 
           '
+        position: null
     language: sh
     position:
     - 1

test/test_block_token/test_list_item_allowed_markers.yml

@@ -6,6 +6,7 @@
             children:
             - RawText:
                 content: foo
+                position: null
             position:
             - 1
             - 1
@@ -31,6 +32,7 @@
             children:
             - RawText:
                 content: bar
+                position: null
             position:
             - 2
             - 2
@@ -56,6 +58,7 @@
             children:
             - RawText:
                 content: baz
+                position: null
             position:
             - 3
             - 3
@@ -81,6 +84,7 @@
             children:
             - RawText:
                 content: item 1
+                position: null
             position:
             - 4
             - 4
@@ -106,6 +110,7 @@
             children:
             - RawText:
                 content: item 2
+                position: null
             position:
             - 5
             - 5
@@ -131,6 +136,7 @@
             children:
             - RawText:
                 content: item x
+                position: null
             position:
             - 6
             - 6

test/test_block_token/test_list_item_before_block_code.yml

@@ -6,11 +6,14 @@
             children:
             - RawText:
                 content: foo
+                position: null
             - LineBreak:
                 content: ''
+                position: null
                 soft: true
             - RawText:
                 content: bar
+                position: null
             position:
             - 1
             - 2
@@ -20,6 +23,7 @@
                 content: 'baz
 
                   '
+                position: null
             language: ''
             position:
             - 3