Upgrade to mistletoe-ebp~=0.9.2 (#108)

`mistletoe-ebp` v0.9 introduces a major API improvement, particularly for block tokens (how they are parsed/stored) and capture of source positions. This allows the majority of the code in `myst_parser/block_tokens.py` to be removed, in favour of the upstream implementations.

Breaking changes:
- `range` field renamed to `position`
- `ASTRenderer` renamed to `JsonRenderer`
- `FrontMatter` is stored as an attribute of `Document` (rather than a child)
- source text is now parsed by `Document.read(text)`, instead of `Document(text)`

.travis.yml

@@ -2,12 +2,12 @@ language: python
 cache: pip
 matrix:
     include:
-      - python: 3.5
-        env: TEST_TYPE="pytest"
       - python: 3.6
         env: TEST_TYPE="pytest"
       - python: 3.7
         env: TEST_TYPE="pytest" PYPI_DEPLOY=true
+      - python: 3.8
+        env: TEST_TYPE="pytest"
       - python: 3.7
         env: TEST_TYPE="pre-commit"
 install:

docs/conf.py

@@ -105,25 +105,31 @@ def run_apidoc(app):
 autodoc_member_order = "bysource"
 
 nitpick_ignore = [
-    ("py:class", "mistletoe.ast_renderer.ASTRenderer"),
-    ("py:class", "mistletoe.block_token.BlockToken"),
-    ("py:class", "mistletoe.block_token.BlockCode"),
-    ("py:class", "mistletoe.block_token.Heading"),
-    ("py:class", "mistletoe.block_token.Quote"),
-    ("py:class", "mistletoe.block_token.CodeFence"),
-    ("py:class", "mistletoe.block_token.Document"),
-    ("py:class", "mistletoe.block_token.List"),
-    ("py:class", "mistletoe.block_token.ListItem"),
-    ("py:class", "mistletoe.block_token.Table"),
-    ("py:class", "mistletoe.block_token.Footnote"),
-    ("py:class", "mistletoe.block_token.Paragraph"),
-    ("py:class", "mistletoe.block_token.ThematicBreak"),
-    ("py:class", "mistletoe.block_token.HTMLBlock"),
-    ("py:class", "mistletoe.base_renderer.BaseRenderer"),
-    ("py:class", "mistletoe.html_renderer.HTMLRenderer"),
-    ("py:class", "mistletoe.span_token.SpanToken"),
-    ("py:class", "mistletoe.span_token.InlineCode"),
+    ("py:class", "mistletoe.base_elements.Token"),
+    ("py:class", "mistletoe.base_elements.BlockToken"),
+    ("py:class", "mistletoe.block_tokens.BlockCode"),
+    ("py:class", "mistletoe.block_tokens.Heading"),
+    ("py:class", "mistletoe.block_tokens.Quote"),
+    ("py:class", "mistletoe.block_tokens.CodeFence"),
+    ("py:class", "mistletoe.block_tokens.Document"),
+    ("py:class", "mistletoe.block_tokens.List"),
+    ("py:class", "mistletoe.block_tokens.ListItem"),
+    ("py:class", "mistletoe.block_tokens.Table"),
+    ("py:class", "mistletoe.block_tokens.TableRow"),
+    ("py:class", "mistletoe.block_tokens.Footnote"),
+    ("py:class", "mistletoe.block_tokens.Paragraph"),
+    ("py:class", "mistletoe.block_tokens.ThematicBreak"),
+    ("py:class", "mistletoe.block_tokens.HTMLBlock"),
+    ("py:class", "mistletoe.renderers.json.JsonRenderer"),
+    ("py:class", "mistletoe.renderers.base.BaseRenderer"),
+    ("py:class", "mistletoe.renderers.html.HTMLRenderer"),
+    ("py:class", "mistletoe.base_elements.SpanToken"),
+    ("py:class", "mistletoe.span_tokens.InlineCode"),
     ("py:class", "docutils.parsers.Parser"),
+    ("py:class", "Tuple"),
+    ("py:class", "ForwardRef"),
+    ("py:class", "NoneType"),
+    ("py:class", "TableRow"),
 ]
 
 

docs/using/use_api.md

@@ -40,7 +40,7 @@ root
 ```
 
 ```python
-MyST.Document(blocks=3)
+Document(children=3, link_definitions=0, front_matter=None)
 ```
 
 All non-terminal tokens may contain children:
@@ -50,9 +50,9 @@ root.children
 ```
 
 ```python
-[MyST.Paragraph(range=(2, 2),children=2),
- MyST.List(items=1),
- MyST.Quote(range=(6, 6),children=1)]
+[Paragraph(children=2, position=(2, 2)),
+ List(children=1, loose=False, start_at=1, position=(3, 4)),
+ Quote(children=1, position=(6, 6))]
 ```
 
 Then each token has attributes specific to its type:
@@ -63,17 +63,24 @@ list_token.__dict__
 ```
 
 ```python
-{'children': [MyST.ListItem(children=1)], 'loose': False, 'start': 1}
+{'children': [{'children': [{'children': [RawText()], 'position': [4, 4]}],
+   'loose': False,
+   'leader': '1.',
+   'prepend': 3,
+   'next_marker': None,
+   'position': [3, 4]}],
+ 'loose': False,
+ 'start_at': 1,
+ 'position': [3, 4]}
 ```
 
-You can also recursively traverse the syntax tree, yielding `TraverseResult`s that contain the element, its parent and depth from the source token:
+You can also recursively traverse the syntax tree, yielding `WalkItem`s that contain the element, its parent and depth from the source token:
 
 ```python
 from pprint import pprint
-from myst_parser import traverse
 tree = [
     (t.parent.__class__.__name__, t.node.__class__.__name__, t.depth)
-    for t in traverse(root)
+    for t in root.walk()
 ]
 pprint(tree)
 ```
@@ -94,43 +101,58 @@ pprint(tree)
  ('Emphasis', 'RawText', 4)]
 ```
 
-## AST Renderer
+## JSON Renderer
 
-The `myst_parser.ast_renderer.AstRenderer` converts a token to a nested dictionary representation.
+The `myst_parser.json_renderer.JsonRenderer` converts a token to a nested dictionary representation.
 
 ```python
-from pprint import pprint
+from json import loads
 from myst_parser import render_tokens
-from myst_parser.ast_renderer import AstRenderer
+from myst_parser.json_renderer import JsonRenderer
 
-pprint(render_tokens(root, AstRenderer))
+pprint(loads(render_tokens(root, JsonRenderer)))
 ```
 
 ```python
-{'children': [{'children': [{'content': "Here's some ", 'type': 'RawText'},
+{'children': [{'children': [{'content': "Here's some ",
+                             'position': [2, 2],
+                             'type': 'RawText'},
                             {'children': [{'content': 'text',
+                                           'position': [2, 2],
                                            'type': 'RawText'}],
+                             'position': [2, 2],
                              'type': 'Emphasis'}],
-               'range': (2, 2),
+               'position': [2, 2],
                'type': 'Paragraph'},
               {'children': [{'children': [{'children': [{'content': 'a list',
+                                                         'position': [4, 4],
                                                          'type': 'RawText'}],
-                                           'range': (1, 1),
+                                           'position': [4, 4],
                                            'type': 'Paragraph'}],
                              'leader': '1.',
                              'loose': False,
+                             'next_marker': None,
+                             'position': [3, 4],
                              'prepend': 3,
                              'type': 'ListItem'}],
                'loose': False,
-               'start': 1,
+               'position': [3, 4],
+               'start_at': 1,
                'type': 'List'},
-              {'children': [{'children': [{'content': 'a quote',
-                                           'type': 'RawText'}],
-                             'range': (7, 7),
+              {'children': [{'children': [{'content': 'a ',
+                                           'position': [7, 7],
+                                           'type': 'RawText'},
+                                          {'children': [{'content': 'quote',
+                                                         'position': [7, 7],
+                                                         'type': 'RawText'}],
+                                           'position': [7, 7],
+                                           'type': 'Emphasis'}],
+                             'position': [7, 7],
                              'type': 'Paragraph'}],
-               'range': (6, 6),
+               'position': [6, 6],
                'type': 'Quote'}],
- 'footnotes': {},
+ 'front_matter': None,
+ 'link_definitions': {},
  'type': 'Document'}
 ```
 

myst_parser/__init__.py

@@ -1,16 +1,14 @@
-from .utils import traverse  # noqa: F401
-
-__version__ = "0.3.2"
+__version__ = "0.4.0"
 
 
 def text_to_tokens(text: str):
     """Convert some text to the MyST base AST."""
     from myst_parser.block_tokens import Document
-    from myst_parser.ast_renderer import AstRenderer
+    from myst_parser.json_renderer import JsonRenderer
 
     # this loads the MyST specific token parsers
-    with AstRenderer():
-        return Document(text)
+    with JsonRenderer():
+        return Document.read(text)
 
 
 def render_tokens(root_token, renderer, **kwargs):
@@ -39,7 +37,7 @@ def parse_text(text: str, output_type: str, **kwargs):
     from myst_parser.block_tokens import Document
 
     with renderer_cls(**kwargs) as renderer:
-        return renderer.render(Document(text))
+        return renderer.render(Document.read(text))
 
 
 def setup(app):