From 019fb77b924950f0038f81b83d838721e525e681 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Tue, 10 Mar 2020 12:40:50 +0000
Subject: [PATCH 01/18] Update JSON/HTML renderers

---
 myst_parser/html_renderer.py                  | 38 +++++--------------
 myst_parser/json_renderer.py                  |  2 +
 setup.py                                      |  2 +-
 tests/test_syntax/test_ast.py                 |  6 ++-
 .../test_block_break_basic_strings0_.yml      |  2 +
 .../test_block_break_escaped_strings3_.yml    |  2 +
 ...k_following_content_no_space_strings8_.yml |  2 +
 ...lock_break_following_content_strings5_.yml |  2 +
 ..._block_break_following_space_strings6_.yml |  2 +
 ...est_block_break_follows_list_strings7_.yml |  2 +
 .../test_block_break_indent_2_strings1_.yml   |  2 +
 .../test_block_break_indent_4_strings2_.yml   |  2 +
 .../test_block_break_inline_strings4_.yml     |  2 +
 .../test_ast/test_comment_basic_strings0_.yml |  2 +
 .../test_comment_escaped_strings3_.yml        |  2 +
 .../test_comment_follows_list_strings5_.yml   |  2 +
 .../test_comment_indent_2_strings1_.yml       |  2 +
 .../test_comment_indent_4_strings2_.yml       |  2 +
 .../test_comment_inline_strings4_.yml         |  2 +
 .../test_front_matter_basic_strings0_.yml     |  2 +
 ...t_link_references_ref_escape_strings3_.yml |  2 +
 ...st_link_references_ref_first_strings0_.yml |  2 +
 ...est_link_references_ref_last_strings1_.yml |  2 +
 ...t_link_references_ref_syntax_strings2_.yml |  2 +
 .../test_ast/test_role_basic_strings0_.yml    |  2 +
 .../test_ast/test_role_escaped_strings3_.yml  |  2 +
 .../test_role_external_code_strings11_.yml    |  2 +
 .../test_role_external_emphasis_strings7_.yml |  2 +
 .../test_role_external_math_strings9_.yml     |  2 +
 .../test_ast/test_role_indent_2_strings1_.yml |  2 +
 .../test_ast/test_role_indent_4_strings2_.yml |  2 +
 .../test_ast/test_role_inline_strings4_.yml   |  2 +
 .../test_role_internal_code_strings10_.yml    |  2 +
 .../test_role_internal_emphasis_strings6_.yml |  2 +
 .../test_role_internal_math_strings8_.yml     |  2 +
 .../test_ast/test_role_multiple_strings5_.yml |  2 +
 tests/test_syntax/test_ast/test_table.yml     |  2 +
 .../test_ast/test_target_basic_strings0_.yml  |  2 +
 .../test_target_escaped_strings3_.yml         |  2 +
 ...est_target_external_emphasis_strings6_.yml |  2 +
 .../test_target_indent_2_strings1_.yml        |  2 +
 .../test_target_indent_4_strings2_.yml        |  2 +
 .../test_ast/test_target_inline_strings4_.yml |  2 +
 ...est_target_internal_emphasis_strings5_.yml |  2 +
 44 files changed, 96 insertions(+), 32 deletions(-)

diff --git a/myst_parser/html_renderer.py b/myst_parser/html_renderer.py
index 02fd83d5..f9f368c1 100644
--- a/myst_parser/html_renderer.py
+++ b/myst_parser/html_renderer.py
@@ -1,5 +1,3 @@
-from textwrap import dedent
-
 from mistletoe import block_tokens, block_tokens_ext, span_tokens, span_tokens_ext
 from mistletoe.renderers import html as html_renderer
 
@@ -26,6 +24,7 @@ class HTMLRenderer(html_renderer.HTMLRenderer):
         BlockBreak,
         List,
         block_tokens_ext.Table,
+        block_tokens_ext.Footnote,
         block_tokens.LinkDefinition,
         Paragraph,
     )
@@ -37,6 +36,7 @@ class HTMLRenderer(html_renderer.HTMLRenderer):
         span_tokens.AutoLink,
         Target,
         span_tokens.CoreTokens,
+        span_tokens_ext.FootReference,
         span_tokens_ext.Math,
         # TODO there is no matching core element in docutils for strikethrough
         # span_tokens_ext.Strikethrough,
@@ -61,7 +61,9 @@ def __init__(
         :param as_standalone: return the HTML body within a minmal HTML page
         :param add_css: if as_standalone=True, CSS to add to the header
         """
-        super().__init__(find_blocks=find_blocks, find_spans=find_spans)
+        super().__init__(
+            find_blocks=find_blocks, find_spans=find_spans, as_standalone=False
+        )
 
         self.mathjax_src = ""
         if add_mathjax:
@@ -70,8 +72,8 @@ def __init__(
                 '"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js'
                 '?config=TeX-MML-AM_CHTML"></script>\n'
             )
-        self.as_standalone = as_standalone
-        self.add_css = add_css
+        self._as_standalone = as_standalone
+        self._add_css = add_css
 
     def render_document(self, token):
         """
@@ -85,9 +87,9 @@ def render_document(self, token):
                 "</div>\n"
             ).format(self.escape_html(token.front_matter.content))
         body = front_matter + super().render_document(token) + self.mathjax_src
-        if not self.as_standalone:
+        if not self._as_standalone:
             return body
-        return minimal_html_page(body, css=self.add_css or "")
+        return html_renderer.minimal_html_page(body, css=self._add_css or "")
 
     def render_code_fence(self, token):
         if token.language and token.language.startswith("{"):
@@ -130,25 +132,3 @@ def render_math(self, token):
         if token.content.startswith("$$"):
             return self.render_raw_text(token)
         return "${}$".format(self.render_raw_text(token))
-
-
-def minimal_html_page(
-    body: str, css: str = "", title: str = "Standalone HTML", lang: str = "en"
-):
-    return dedent(
-        """\
-    <!DOCTYPE html>
-    <html lang="{lang}">
-    <head>
-    <meta charset="utf-8">
-    <title>{title}</title>
-    <style>
-    {css}
-    </style>
-    </head>
-    <body>
-    {body}
-    </body>
-    </html>
-    """
-    ).format(title=title, lang=lang, css=css, body=body)
diff --git a/myst_parser/json_renderer.py b/myst_parser/json_renderer.py
index 02146912..0164fbbf 100644
--- a/myst_parser/json_renderer.py
+++ b/myst_parser/json_renderer.py
@@ -21,6 +21,7 @@ class JsonRenderer(json.JsonRenderer):
         BlockBreak,
         List,
         block_tokens_ext.Table,
+        block_tokens_ext.Footnote,
         block_tokens.LinkDefinition,
         Paragraph,
     )
@@ -32,6 +33,7 @@ class JsonRenderer(json.JsonRenderer):
         span_tokens.AutoLink,
         Target,
         span_tokens.CoreTokens,
+        span_tokens_ext.FootReference,
         span_tokens_ext.Math,
         # TODO there is no matching core element in docutils for strikethrough
         # span_tokens_ext.Strikethrough,
diff --git a/setup.py b/setup.py
index 32febce5..28a57d4c 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@
     ],
     keywords="markdown lexer parser development docutils sphinx",
     python_requires=">=3.6",
-    install_requires=["mistletoe-ebp~=0.9.4"],
+    install_requires=["mistletoe-ebp~=0.10.0a1"],
     extras_require={
         "sphinx": ["pyyaml", "docutils>=0.15", "sphinx>=2,<3"],
         "code_style": ["flake8<3.8.0,>=3.7.0", "black", "pre-commit==1.17.0"],
diff --git a/tests/test_syntax/test_ast.py b/tests/test_syntax/test_ast.py
index 841c2bec..22aa9cb8 100644
--- a/tests/test_syntax/test_ast.py
+++ b/tests/test_syntax/test_ast.py
@@ -34,12 +34,14 @@ def test_walk(json_renderer):
     assert tree == [
         (
             "Paragraph(children=2, position=(1, 1))",
-            "Document(children=2, link_definitions=0, front_matter=None)",
+            "Document(children=2, link_definitions=0, footnotes=0, "
+            "footref_order=0, front_matter=None)",
             1,
         ),
         (
             "Paragraph(children=2, position=(3, 3))",
-            "Document(children=2, link_definitions=0, front_matter=None)",
+            "Document(children=2, link_definitions=0, footnotes=0, "
+            "footref_order=0, front_matter=None)",
             1,
         ),
         ("RawText()", "Paragraph(children=2, position=(1, 1))", 2),
diff --git a/tests/test_syntax/test_ast/test_block_break_basic_strings0_.yml b/tests/test_syntax/test_ast/test_block_break_basic_strings0_.yml
index 3dd8a015..5be5f834 100644
--- a/tests/test_syntax/test_ast/test_block_break_basic_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_basic_strings0_.yml
@@ -5,6 +5,8 @@ children:
   - 1
   raw: +++
   type: BlockBreak
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml b/tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml
index 2add8025..d5d61fc4 100644
--- a/tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml
@@ -19,6 +19,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_block_break_following_content_no_space_strings8_.yml b/tests/test_syntax/test_ast/test_block_break_following_content_no_space_strings8_.yml
index fc86b009..f4612227 100644
--- a/tests/test_syntax/test_ast/test_block_break_following_content_no_space_strings8_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_following_content_no_space_strings8_.yml
@@ -5,6 +5,8 @@ children:
   - 1
   raw: +++a
   type: BlockBreak
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_block_break_following_content_strings5_.yml b/tests/test_syntax/test_ast/test_block_break_following_content_strings5_.yml
index 63446083..fccca83f 100644
--- a/tests/test_syntax/test_ast/test_block_break_following_content_strings5_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_following_content_strings5_.yml
@@ -5,6 +5,8 @@ children:
   - 1
   raw: +++ a
   type: BlockBreak
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_block_break_following_space_strings6_.yml b/tests/test_syntax/test_ast/test_block_break_following_space_strings6_.yml
index c624d377..2ab79154 100644
--- a/tests/test_syntax/test_ast/test_block_break_following_space_strings6_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_following_space_strings6_.yml
@@ -5,6 +5,8 @@ children:
   - 1
   raw: '+++   '
   type: BlockBreak
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml b/tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml
index bde66025..a590a643 100644
--- a/tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml
@@ -31,6 +31,8 @@ children:
   - 2
   raw: +++
   type: BlockBreak
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_block_break_indent_2_strings1_.yml b/tests/test_syntax/test_ast/test_block_break_indent_2_strings1_.yml
index d7535e5f..43e9760b 100644
--- a/tests/test_syntax/test_ast/test_block_break_indent_2_strings1_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_indent_2_strings1_.yml
@@ -5,6 +5,8 @@ children:
   - 1
   raw: '  +++'
   type: BlockBreak
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml b/tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml
index 2705528e..98523da9 100644
--- a/tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml
@@ -12,6 +12,8 @@ children:
   - 0
   - 1
   type: BlockCode
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml b/tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml
index f7f04312..df82821f 100644
--- a/tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml
@@ -9,6 +9,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_comment_basic_strings0_.yml b/tests/test_syntax/test_ast/test_comment_basic_strings0_.yml
index d10825e0..a06b0a0c 100644
--- a/tests/test_syntax/test_ast/test_comment_basic_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_comment_basic_strings0_.yml
@@ -5,6 +5,8 @@ children:
   - 1
   raw: '% comment'
   type: LineComment
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml b/tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml
index dda432b7..6a7046ec 100644
--- a/tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml
@@ -19,6 +19,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_comment_follows_list_strings5_.yml b/tests/test_syntax/test_ast/test_comment_follows_list_strings5_.yml
index 55884319..b4a71b04 100644
--- a/tests/test_syntax/test_ast/test_comment_follows_list_strings5_.yml
+++ b/tests/test_syntax/test_ast/test_comment_follows_list_strings5_.yml
@@ -31,6 +31,8 @@ children:
   - 2
   raw: '% comment'
   type: LineComment
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_comment_indent_2_strings1_.yml b/tests/test_syntax/test_ast/test_comment_indent_2_strings1_.yml
index 35f0d600..2634ada8 100644
--- a/tests/test_syntax/test_ast/test_comment_indent_2_strings1_.yml
+++ b/tests/test_syntax/test_ast/test_comment_indent_2_strings1_.yml
@@ -5,6 +5,8 @@ children:
   - 1
   raw: '  % comment'
   type: LineComment
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_comment_indent_4_strings2_.yml b/tests/test_syntax/test_ast/test_comment_indent_4_strings2_.yml
index be09c528..d9f99897 100644
--- a/tests/test_syntax/test_ast/test_comment_indent_4_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_comment_indent_4_strings2_.yml
@@ -12,6 +12,8 @@ children:
   - 0
   - 1
   type: BlockCode
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_comment_inline_strings4_.yml b/tests/test_syntax/test_ast/test_comment_inline_strings4_.yml
index 40af44cd..900d8192 100644
--- a/tests/test_syntax/test_ast/test_comment_inline_strings4_.yml
+++ b/tests/test_syntax/test_ast/test_comment_inline_strings4_.yml
@@ -9,6 +9,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml b/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml
index 7fd01dbe..d8bcc8a7 100644
--- a/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml
@@ -1,4 +1,6 @@
 children: []
+footnotes: {}
+footref_order: []
 front_matter:
   content: 'a: b
 
diff --git a/tests/test_syntax/test_ast/test_link_references_ref_escape_strings3_.yml b/tests/test_syntax/test_ast/test_link_references_ref_escape_strings3_.yml
index c66a9b58..84d2901a 100644
--- a/tests/test_syntax/test_ast/test_link_references_ref_escape_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_link_references_ref_escape_strings3_.yml
@@ -29,6 +29,8 @@ children:
   - 3
   - 3
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_link_references_ref_first_strings0_.yml b/tests/test_syntax/test_ast/test_link_references_ref_first_strings0_.yml
index 94b3c13b..5574254a 100644
--- a/tests/test_syntax/test_ast/test_link_references_ref_first_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_link_references_ref_first_strings0_.yml
@@ -16,6 +16,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions:
   ref:
diff --git a/tests/test_syntax/test_ast/test_link_references_ref_last_strings1_.yml b/tests/test_syntax/test_ast/test_link_references_ref_last_strings1_.yml
index aff79617..2d6c7a22 100644
--- a/tests/test_syntax/test_ast/test_link_references_ref_last_strings1_.yml
+++ b/tests/test_syntax/test_ast/test_link_references_ref_last_strings1_.yml
@@ -16,6 +16,8 @@ children:
   - 3
   - 3
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions:
   ref:
diff --git a/tests/test_syntax/test_ast/test_link_references_ref_syntax_strings2_.yml b/tests/test_syntax/test_ast/test_link_references_ref_syntax_strings2_.yml
index ed617607..149193b4 100644
--- a/tests/test_syntax/test_ast/test_link_references_ref_syntax_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_link_references_ref_syntax_strings2_.yml
@@ -21,6 +21,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions:
   '*syntax*':
diff --git a/tests/test_syntax/test_ast/test_role_basic_strings0_.yml b/tests/test_syntax/test_ast/test_role_basic_strings0_.yml
index ad1ef02e..a0c64bb6 100644
--- a/tests/test_syntax/test_ast/test_role_basic_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_role_basic_strings0_.yml
@@ -15,6 +15,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_escaped_strings3_.yml b/tests/test_syntax/test_ast/test_role_escaped_strings3_.yml
index 83392565..bedd3106 100644
--- a/tests/test_syntax/test_ast/test_role_escaped_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_role_escaped_strings3_.yml
@@ -29,6 +29,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_external_code_strings11_.yml b/tests/test_syntax/test_ast/test_role_external_code_strings11_.yml
index 6d342449..a96ef5ec 100644
--- a/tests/test_syntax/test_ast/test_role_external_code_strings11_.yml
+++ b/tests/test_syntax/test_ast/test_role_external_code_strings11_.yml
@@ -14,6 +14,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_external_emphasis_strings7_.yml b/tests/test_syntax/test_ast/test_role_external_emphasis_strings7_.yml
index bbaac0bd..098aab85 100644
--- a/tests/test_syntax/test_ast/test_role_external_emphasis_strings7_.yml
+++ b/tests/test_syntax/test_ast/test_role_external_emphasis_strings7_.yml
@@ -20,6 +20,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_external_math_strings9_.yml b/tests/test_syntax/test_ast/test_role_external_math_strings9_.yml
index d8266ba5..480c75a3 100644
--- a/tests/test_syntax/test_ast/test_role_external_math_strings9_.yml
+++ b/tests/test_syntax/test_ast/test_role_external_math_strings9_.yml
@@ -9,6 +9,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_indent_2_strings1_.yml b/tests/test_syntax/test_ast/test_role_indent_2_strings1_.yml
index ad1ef02e..a0c64bb6 100644
--- a/tests/test_syntax/test_ast/test_role_indent_2_strings1_.yml
+++ b/tests/test_syntax/test_ast/test_role_indent_2_strings1_.yml
@@ -15,6 +15,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_indent_4_strings2_.yml b/tests/test_syntax/test_ast/test_role_indent_4_strings2_.yml
index 14a08c17..01fea79d 100644
--- a/tests/test_syntax/test_ast/test_role_indent_4_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_role_indent_4_strings2_.yml
@@ -12,6 +12,8 @@ children:
   - 0
   - 1
   type: BlockCode
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_inline_strings4_.yml b/tests/test_syntax/test_ast/test_role_inline_strings4_.yml
index 8a66cb35..6b379e82 100644
--- a/tests/test_syntax/test_ast/test_role_inline_strings4_.yml
+++ b/tests/test_syntax/test_ast/test_role_inline_strings4_.yml
@@ -20,6 +20,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_internal_code_strings10_.yml b/tests/test_syntax/test_ast/test_role_internal_code_strings10_.yml
index e02d4984..ae266cbe 100644
--- a/tests/test_syntax/test_ast/test_role_internal_code_strings10_.yml
+++ b/tests/test_syntax/test_ast/test_role_internal_code_strings10_.yml
@@ -15,6 +15,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_internal_emphasis_strings6_.yml b/tests/test_syntax/test_ast/test_role_internal_emphasis_strings6_.yml
index 5ad535ec..e08dc4a7 100644
--- a/tests/test_syntax/test_ast/test_role_internal_emphasis_strings6_.yml
+++ b/tests/test_syntax/test_ast/test_role_internal_emphasis_strings6_.yml
@@ -15,6 +15,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_internal_math_strings8_.yml b/tests/test_syntax/test_ast/test_role_internal_math_strings8_.yml
index bba9afda..0f246ade 100644
--- a/tests/test_syntax/test_ast/test_role_internal_math_strings8_.yml
+++ b/tests/test_syntax/test_ast/test_role_internal_math_strings8_.yml
@@ -15,6 +15,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_role_multiple_strings5_.yml b/tests/test_syntax/test_ast/test_role_multiple_strings5_.yml
index 5b9a28d8..7d2de519 100644
--- a/tests/test_syntax/test_ast/test_role_multiple_strings5_.yml
+++ b/tests/test_syntax/test_ast/test_role_multiple_strings5_.yml
@@ -31,6 +31,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_table.yml b/tests/test_syntax/test_ast/test_table.yml
index fb38e4a8..eaeb2910 100644
--- a/tests/test_syntax/test_ast/test_table.yml
+++ b/tests/test_syntax/test_ast/test_table.yml
@@ -98,6 +98,8 @@ children:
   - 1
   - 3
   type: Table
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_target_basic_strings0_.yml b/tests/test_syntax/test_ast/test_target_basic_strings0_.yml
index fa02463a..9a56ae97 100644
--- a/tests/test_syntax/test_ast/test_target_basic_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_target_basic_strings0_.yml
@@ -15,6 +15,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_target_escaped_strings3_.yml b/tests/test_syntax/test_ast/test_target_escaped_strings3_.yml
index 570b1a47..a2a6ba9f 100644
--- a/tests/test_syntax/test_ast/test_target_escaped_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_target_escaped_strings3_.yml
@@ -19,6 +19,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_target_external_emphasis_strings6_.yml b/tests/test_syntax/test_ast/test_target_external_emphasis_strings6_.yml
index b7f34614..dc9e8216 100644
--- a/tests/test_syntax/test_ast/test_target_external_emphasis_strings6_.yml
+++ b/tests/test_syntax/test_ast/test_target_external_emphasis_strings6_.yml
@@ -20,6 +20,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_target_indent_2_strings1_.yml b/tests/test_syntax/test_ast/test_target_indent_2_strings1_.yml
index fa02463a..9a56ae97 100644
--- a/tests/test_syntax/test_ast/test_target_indent_2_strings1_.yml
+++ b/tests/test_syntax/test_ast/test_target_indent_2_strings1_.yml
@@ -15,6 +15,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_target_indent_4_strings2_.yml b/tests/test_syntax/test_ast/test_target_indent_4_strings2_.yml
index 6fc3fe8b..380cb9a5 100644
--- a/tests/test_syntax/test_ast/test_target_indent_4_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_target_indent_4_strings2_.yml
@@ -12,6 +12,8 @@ children:
   - 0
   - 1
   type: BlockCode
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_target_inline_strings4_.yml b/tests/test_syntax/test_ast/test_target_inline_strings4_.yml
index 4995edef..6bf8f917 100644
--- a/tests/test_syntax/test_ast/test_target_inline_strings4_.yml
+++ b/tests/test_syntax/test_ast/test_target_inline_strings4_.yml
@@ -20,6 +20,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_target_internal_emphasis_strings5_.yml b/tests/test_syntax/test_ast/test_target_internal_emphasis_strings5_.yml
index a4a8187c..fdb332ce 100644
--- a/tests/test_syntax/test_ast/test_target_internal_emphasis_strings5_.yml
+++ b/tests/test_syntax/test_ast/test_target_internal_emphasis_strings5_.yml
@@ -15,6 +15,8 @@ children:
   - 1
   - 1
   type: Paragraph
+footnotes: {}
+footref_order: []
 front_matter: null
 link_definitions: {}
 type: Document

From a833b9287908ac969fa283d1e41f41f0c2a332f6 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Tue, 10 Mar 2020 12:51:26 +0000
Subject: [PATCH 02/18] Apply fix from #113 (propogate definitions)

---
 myst_parser/docutils_renderer.py      |  7 +++++--
 tests/test_renderers/test_docutils.py | 26 ++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
index 50e11e1f..d5b346a1 100644
--- a/myst_parser/docutils_renderer.py
+++ b/myst_parser/docutils_renderer.py
@@ -106,7 +106,7 @@ def add_line_and_source_path(self, node, token):
     def nested_render_text(self, text: str, lineno: int):
         """Render unparsed text."""
         token = myst_block_tokens.Document.read(
-            text, start_line=lineno, front_matter=True
+            text, start_line=lineno, front_matter=True, reset_definitions=False
         )
         # TODO think if this is the best way: here we consume front matter,
         # but then remove it. this is for example if includes have front matter
@@ -836,7 +836,10 @@ def inline_text(self, text: str, lineno: int):
         )
         renderer.render(
             myst_block_tokens.Document.read(
-                text, start_line=self._lineno, front_matter=False
+                text,
+                start_line=self._lineno,
+                front_matter=False,
+                reset_definitions=False,
             )
         )
         textnodes = []
diff --git a/tests/test_renderers/test_docutils.py b/tests/test_renderers/test_docutils.py
index db6f872a..ff633bda 100644
--- a/tests/test_renderers/test_docutils.py
+++ b/tests/test_renderers/test_docutils.py
@@ -349,6 +349,32 @@ def test_block_quotes(renderer):
     )
 
 
+def test_link_def_in_directive(renderer):
+    renderer.render(
+        Document.read(
+            dedent(
+                """\
+            ```{note}
+            [a]
+            ```
+
+            [a]: link
+            """
+            )
+        )
+    )
+    assert renderer.document.pformat() == dedent(
+        """\
+    <document source="notset">
+        <note>
+            <paragraph>
+                <pending_xref refdomain="True" refexplicit="True" reftarget="link" reftype="any" refwarn="True">
+                    <literal classes="xref any">
+                        a
+    """  # noqa: E501
+    )
+
+
 def test_full_run(sphinx_renderer, file_regression):
     string = dedent(
         """\

From c9730e3a608560e180d8f3609e3ec59094c1e436 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Tue, 10 Mar 2020 14:58:30 +0000
Subject: [PATCH 03/18] Add test for nested links

Related to #113
---
 tests/test_renderers/test_docutils.py         | 25 +++++++++++++++++++
 .../test_link_def_in_directive_nested.xml     | 10 ++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 tests/test_renderers/test_docutils/test_link_def_in_directive_nested.xml

diff --git a/tests/test_renderers/test_docutils.py b/tests/test_renderers/test_docutils.py
index ff633bda..5ee09100 100644
--- a/tests/test_renderers/test_docutils.py
+++ b/tests/test_renderers/test_docutils.py
@@ -375,6 +375,31 @@ def test_link_def_in_directive(renderer):
     )
 
 
+def test_link_def_in_directive_nested(renderer, file_regression):
+    # TODO document or 'fix' the fact that [ref2] here isn't resolved
+    renderer.render(
+        Document.read(
+            dedent(
+                """\
+            ```{note}
+            [ref1]: link
+            ```
+
+            ```{note}
+            [ref1]
+            [ref2]
+            ```
+
+            ```{note}
+            [ref2]: link
+            ```
+            """
+            )
+        )
+    )
+    file_regression.check(renderer.document.pformat(), extension=".xml")
+
+
 def test_full_run(sphinx_renderer, file_regression):
     string = dedent(
         """\
diff --git a/tests/test_renderers/test_docutils/test_link_def_in_directive_nested.xml b/tests/test_renderers/test_docutils/test_link_def_in_directive_nested.xml
new file mode 100644
index 00000000..3864506f
--- /dev/null
+++ b/tests/test_renderers/test_docutils/test_link_def_in_directive_nested.xml
@@ -0,0 +1,10 @@
+<document source="notset">
+    <note>
+    <note>
+        <paragraph>
+            <pending_xref refdomain="True" refexplicit="True" reftarget="link" reftype="any" refwarn="True">
+                <literal classes="xref any">
+                    ref1
+            
+            [ref2]
+    <note>

From 455d8f09fdf12123cd203804b832accfd21012d4 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Tue, 10 Mar 2020 17:42:01 +0000
Subject: [PATCH 04/18] Add initial implementation of footnotes

---
 myst_parser/block_tokens.py                   |   2 +-
 myst_parser/docutils_renderer.py              |  72 +++++++++--
 tests/test_renderers/test_docutils.py         |  28 ++++
 .../test_sphinx/sourcedirs/footnotes/conf.py  |   2 +
 .../sourcedirs/footnotes/footnote_md.md       |  25 ++++
 .../sourcedirs/footnotes/footnote_rst.rst     |  26 ++++
 .../test_sphinx/sourcedirs/footnotes/index.md |   4 +
 tests/test_sphinx/test_sphinx_builds.py       |  21 +++
 .../test_sphinx_builds/test_footnotes.html    | 122 ++++++++++++++++++
 .../test_sphinx_builds/test_footnotes.xml     |  51 ++++++++
 10 files changed, 341 insertions(+), 12 deletions(-)
 create mode 100644 tests/test_sphinx/sourcedirs/footnotes/conf.py
 create mode 100644 tests/test_sphinx/sourcedirs/footnotes/footnote_md.md
 create mode 100644 tests/test_sphinx/sourcedirs/footnotes/footnote_rst.rst
 create mode 100644 tests/test_sphinx/sourcedirs/footnotes/index.md
 create mode 100644 tests/test_sphinx/test_sphinx_builds/test_footnotes.html
 create mode 100644 tests/test_sphinx/test_sphinx_builds/test_footnotes.xml

diff --git a/myst_parser/block_tokens.py b/myst_parser/block_tokens.py
index d138fdcc..954b9f18 100644
--- a/myst_parser/block_tokens.py
+++ b/myst_parser/block_tokens.py
@@ -9,7 +9,7 @@
 
 
 @autodoc
-@attr.s(slots=True, kw_only=True)
+@attr.s(slots=False, kw_only=True)
 class Document(block_tokens.Document):
     """Document token."""
 
diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
index d5b346a1..1e2de183 100644
--- a/myst_parser/docutils_renderer.py
+++ b/myst_parser/docutils_renderer.py
@@ -20,6 +20,7 @@
 
 from mistletoe import block_tokens, block_tokens_ext, span_tokens, span_tokens_ext
 from mistletoe.renderers.base import BaseRenderer
+from mistletoe.parse_context import get_parse_context
 
 from myst_parser import block_tokens as myst_block_tokens
 from myst_parser import span_tokens as myst_span_tokens
@@ -44,6 +45,7 @@ class DocutilsRenderer(BaseRenderer):
         myst_block_tokens.BlockBreak,
         myst_block_tokens.List,
         block_tokens_ext.Table,
+        block_tokens_ext.Footnote,
         block_tokens.LinkDefinition,
         myst_block_tokens.Paragraph,
     )
@@ -55,6 +57,7 @@ class DocutilsRenderer(BaseRenderer):
         span_tokens.AutoLink,
         myst_span_tokens.Target,
         span_tokens.CoreTokens,
+        span_tokens_ext.FootReference,
         span_tokens_ext.Math,
         # TODO there is no matching core element in docutils for strikethrough
         # span_tokens_ext.Strikethrough,
@@ -105,13 +108,15 @@ def add_line_and_source_path(self, node, token):
 
     def nested_render_text(self, text: str, lineno: int):
         """Render unparsed text."""
-        token = myst_block_tokens.Document.read(
+        doc_token = myst_block_tokens.Document.read(
             text, start_line=lineno, front_matter=True, reset_definitions=False
         )
         # TODO think if this is the best way: here we consume front matter,
         # but then remove it. this is for example if includes have front matter
-        token.front_matter = None
-        self.render(token)
+        doc_token.front_matter = None
+        # we mark the token as nested so that footnotes etc aren't rendered
+        doc_token.is_nested = True
+        self.render(doc_token)
 
     def render_children(self, token):
         for child in token.children:
@@ -127,11 +132,27 @@ def current_node_context(self, node, append: bool = False):
         yield
         self.current_node = current_node
 
-    def render_document(self, token):
+    def render_document(self, token: block_tokens.Document):
         self.link_definitions.update(token.link_definitions)
         if token.front_matter:
             self.render_front_matter(token.front_matter)
         self.render_children(token)
+
+        if getattr(token, "is_nested", False):
+            # if the document is nested in another, we don't want to output footnotes
+            return self.document
+
+        # we use the footnotes stored in the global context,
+        # rather than those stored on the document,
+        # since additional references may have been made in nested parses
+        foot_refs = get_parse_context().foot_references
+        footnotes = get_parse_context().foot_definitions
+        if foot_refs:
+            self.current_node.append(nodes.transition())
+        for footref in foot_refs:
+            if footref in footnotes:
+                self.render_footnote(footnotes[footref])
+
         return self.document
 
     def render_front_matter(self, token):
@@ -161,6 +182,37 @@ def render_front_matter(self, token):
         docinfo = dict_to_docinfo(data)
         self.current_node.append(docinfo)
 
+    def render_footnote(self, token: block_tokens_ext.Footnote):
+        footnote = nodes.footnote()
+        self.add_line_and_source_path(footnote, token)
+        # footnote += nodes.label('', token.target)
+        footnote["names"].append(token.target)
+        footnote["auto"] = 1
+        self.document.note_autofootnote(footnote)
+        self.document.note_explicit_target(footnote, footnote)
+        # TODO for now we wrap the content (which are list of spans tokens)
+        # in a paragraph, but eventually upstream in mistletoe this will already be
+        # block level tokens
+        self.current_node.append(footnote)
+        paragraph = nodes.paragraph("")
+        self.add_line_and_source_path(paragraph, token)
+        footnote.append(paragraph)
+        with self.current_node_context(paragraph, append=False):
+            self.render_children(token)
+
+    def render_foot_reference(self, token):
+        """Footnote references are added as auto-numbered,
+        .i.e. `[^a]` is read as rST `[#a]_`
+        """
+        refnode = nodes.footnote_reference("[^{}]".format(token.target))
+        self.add_line_and_source_path(refnode, token)
+        refnode["auto"] = 1
+        refnode["refname"] = token.target
+        # refnode += nodes.Text(token.target)
+        self.document.note_autofootnote_ref(refnode)
+        self.document.note_footnote_ref(refnode)
+        self.current_node.append(refnode)
+
     def render_paragraph(self, token):
         if len(token.children) == 1 and isinstance(
             token.children[0], myst_span_tokens.Target
@@ -834,14 +886,12 @@ def inline_text(self, text: str, lineno: int):
         renderer = self._renderer.__class__(
             document=self.document, current_node=paragraph
         )
-        renderer.render(
-            myst_block_tokens.Document.read(
-                text,
-                start_line=self._lineno,
-                front_matter=False,
-                reset_definitions=False,
-            )
+        doc_token = myst_block_tokens.Document.read(
+            text, start_line=self._lineno, front_matter=False, reset_definitions=False
         )
+        # we mark the token as nested so that footnotes etc aren't rendered
+        doc_token.is_nested = True
+        renderer.render(doc_token)
         textnodes = []
         if paragraph.children:
             # first child should be paragraph
diff --git a/tests/test_renderers/test_docutils.py b/tests/test_renderers/test_docutils.py
index 5ee09100..9510e1b0 100644
--- a/tests/test_renderers/test_docutils.py
+++ b/tests/test_renderers/test_docutils.py
@@ -400,6 +400,34 @@ def test_link_def_in_directive_nested(renderer, file_regression):
     file_regression.check(renderer.document.pformat(), extension=".xml")
 
 
+def test_footnotes(renderer):
+    renderer.render(
+        Document.read(
+            dedent(
+                """\
+            [^a]
+
+            [^a]: footnote*text*
+            """
+            )
+        )
+    )
+    print(renderer.document.pformat())
+    assert renderer.document.pformat() == dedent(
+        """\
+    <document source="notset">
+        <paragraph>
+            <footnote_reference auto="1" ids="id1" refname="a">
+        <transition>
+        <footnote auto="1" ids="a" names="a">
+            <paragraph>
+                footnote
+                <emphasis>
+                    text
+    """
+    )
+
+
 def test_full_run(sphinx_renderer, file_regression):
     string = dedent(
         """\
diff --git a/tests/test_sphinx/sourcedirs/footnotes/conf.py b/tests/test_sphinx/sourcedirs/footnotes/conf.py
new file mode 100644
index 00000000..e1c5009b
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/footnotes/conf.py
@@ -0,0 +1,2 @@
+extensions = ["myst_parser"]
+exclude_patterns = ["_build"]
diff --git a/tests/test_sphinx/sourcedirs/footnotes/footnote_md.md b/tests/test_sphinx/sourcedirs/footnotes/footnote_md.md
new file mode 100644
index 00000000..6d2e40a7
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/footnotes/footnote_md.md
@@ -0,0 +1,25 @@
+# Footnotes with Markdown
+
+[^c]
+
+```{note}
+[^d]
+```
+
+[^a]
+
+[^a]: some footnote *text*
+
+[^b]: a footnote before its reference
+
+[^b]
+
+[^c]: a footnote referenced first
+
+[^d]: a footnote referenced in a directive
+
+[^123] [^123]
+
+[^123]: multiple references footnote
+
+[^x]: an unreferenced footnote
diff --git a/tests/test_sphinx/sourcedirs/footnotes/footnote_rst.rst b/tests/test_sphinx/sourcedirs/footnotes/footnote_rst.rst
new file mode 100644
index 00000000..219f98ac
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/footnotes/footnote_rst.rst
@@ -0,0 +1,26 @@
+Footnotes with rST
+------------------
+
+[#c]_
+
+.. note::
+
+    [#d]_
+
+[#a]_
+
+.. [#a] some footnote *text*
+
+.. [#b] a footnote before its reference
+
+[#b]_
+
+.. [#c] a footnote referenced first
+
+.. [#d] a footnote referenced in a directive
+
+[#123]_ [#123]_
+
+.. [#123] multiple references footnote
+
+.. [#x] an unreferenced footnote
diff --git a/tests/test_sphinx/sourcedirs/footnotes/index.md b/tests/test_sphinx/sourcedirs/footnotes/index.md
new file mode 100644
index 00000000..4e4bc088
--- /dev/null
+++ b/tests/test_sphinx/sourcedirs/footnotes/index.md
@@ -0,0 +1,4 @@
+```{toctree}
+footnote_rst
+footnote_md
+```
diff --git a/tests/test_sphinx/test_sphinx_builds.py b/tests/test_sphinx/test_sphinx_builds.py
index bfcf3b51..aa7f0431 100644
--- a/tests/test_sphinx/test_sphinx_builds.py
+++ b/tests/test_sphinx/test_sphinx_builds.py
@@ -44,3 +44,24 @@ def test_includes(
 
     get_sphinx_app_doctree(app, filename="index.doctree", regress=True)
     get_sphinx_app_output(app, filename="index.html", regress_html=True)
+
+
+@pytest.mark.sphinx(
+    buildername="html", srcdir=os.path.join(SOURCE_DIR, "footnotes"), freshenv=True
+)
+def test_footnotes(
+    app,
+    status,
+    warning,
+    get_sphinx_app_doctree,
+    get_sphinx_app_output,
+    remove_sphinx_builds,
+):
+    """Test of include directive."""
+    app.build()
+
+    assert "build succeeded" in status.getvalue()  # Build succeeded
+    warnings = warning.getvalue().strip()
+    assert warnings == ""
+    get_sphinx_app_doctree(app, filename="footnote_md.doctree", regress=True)
+    get_sphinx_app_output(app, filename="footnote_md.html", regress_html=True)
diff --git a/tests/test_sphinx/test_sphinx_builds/test_footnotes.html b/tests/test_sphinx/test_sphinx_builds/test_footnotes.html
new file mode 100644
index 00000000..66acbda1
--- /dev/null
+++ b/tests/test_sphinx/test_sphinx_builds/test_footnotes.html
@@ -0,0 +1,122 @@
+<div class="documentwrapper">
+ <div class="bodywrapper">
+  <div class="body" role="main">
+   <div class="section" id="footnotes-with-markdown">
+    <h1>
+     Footnotes with Markdown
+     <a class="headerlink" href="#footnotes-with-markdown" title="Permalink to this headline">
+      ¶
+     </a>
+    </h1>
+    <p>
+     <a class="footnote-reference brackets" href="#c" id="id1">
+      1
+     </a>
+    </p>
+    <div class="admonition note">
+     <p class="admonition-title">
+      Note
+     </p>
+     <p>
+      <a class="footnote-reference brackets" href="#d" id="id2">
+       5
+      </a>
+     </p>
+    </div>
+    <p>
+     <a class="footnote-reference brackets" href="#a" id="id3">
+      2
+     </a>
+    </p>
+    <p>
+     <a class="footnote-reference brackets" href="#b" id="id4">
+      3
+     </a>
+    </p>
+    <p>
+     <a class="footnote-reference brackets" href="#id7" id="id5">
+      4
+     </a>
+     <a class="footnote-reference brackets" href="#id7" id="id6">
+      4
+     </a>
+    </p>
+    <hr class="docutils"/>
+    <dl class="footnote brackets">
+     <dt class="label" id="c">
+      <span class="brackets">
+       <a class="fn-backref" href="#id1">
+        1
+       </a>
+      </span>
+     </dt>
+     <dd>
+      <p>
+       a footnote referenced first
+      </p>
+     </dd>
+     <dt class="label" id="a">
+      <span class="brackets">
+       <a class="fn-backref" href="#id3">
+        2
+       </a>
+      </span>
+     </dt>
+     <dd>
+      <p>
+       some footnote
+       <em>
+        text
+       </em>
+      </p>
+     </dd>
+     <dt class="label" id="b">
+      <span class="brackets">
+       <a class="fn-backref" href="#id4">
+        3
+       </a>
+      </span>
+     </dt>
+     <dd>
+      <p>
+       a footnote before its reference
+      </p>
+     </dd>
+     <dt class="label" id="id7">
+      <span class="brackets">
+       4
+      </span>
+      <span class="fn-backref">
+       (
+       <a href="#id5">
+        1
+       </a>
+       ,
+       <a href="#id6">
+        2
+       </a>
+       )
+      </span>
+     </dt>
+     <dd>
+      <p>
+       multiple references footnote
+      </p>
+     </dd>
+     <dt class="label" id="d">
+      <span class="brackets">
+       <a class="fn-backref" href="#id2">
+        5
+       </a>
+      </span>
+     </dt>
+     <dd>
+      <p>
+       a footnote referenced in a directive
+      </p>
+     </dd>
+    </dl>
+   </div>
+  </div>
+ </div>
+</div>
diff --git a/tests/test_sphinx/test_sphinx_builds/test_footnotes.xml b/tests/test_sphinx/test_sphinx_builds/test_footnotes.xml
new file mode 100644
index 00000000..e893875a
--- /dev/null
+++ b/tests/test_sphinx/test_sphinx_builds/test_footnotes.xml
@@ -0,0 +1,51 @@
+<document source="footnote_md.md">
+    <section ids="footnotes-with-markdown" names="footnotes\ with\ markdown">
+        <title>
+            Footnotes with Markdown
+        <paragraph>
+            <footnote_reference auto="1" docname="footnote_md" ids="id1" refid="c">
+                1
+        <note>
+            <paragraph>
+                <footnote_reference auto="1" docname="footnote_md" ids="id2" refid="d">
+                    5
+        <paragraph>
+            <footnote_reference auto="1" docname="footnote_md" ids="id3" refid="a">
+                2
+        <paragraph>
+            <footnote_reference auto="1" docname="footnote_md" ids="id4" refid="b">
+                3
+        <paragraph>
+            <footnote_reference auto="1" docname="footnote_md" ids="id5" refid="id7">
+                4
+             
+            <footnote_reference auto="1" docname="footnote_md" ids="id6" refid="id7">
+                4
+        <transition>
+        <footnote auto="1" backrefs="id1" docname="footnote_md" ids="c" names="c">
+            <label>
+                1
+            <paragraph>
+                a footnote referenced first
+        <footnote auto="1" backrefs="id3" docname="footnote_md" ids="a" names="a">
+            <label>
+                2
+            <paragraph>
+                some footnote 
+                <emphasis>
+                    text
+        <footnote auto="1" backrefs="id4" docname="footnote_md" ids="b" names="b">
+            <label>
+                3
+            <paragraph>
+                a footnote before its reference
+        <footnote auto="1" backrefs="id5 id6" docname="footnote_md" ids="id7" names="123">
+            <label>
+                4
+            <paragraph>
+                multiple references footnote
+        <footnote auto="1" backrefs="id2" docname="footnote_md" ids="d" names="d">
+            <label>
+                5
+            <paragraph>
+                a footnote referenced in a directive

From f9e171e5717fd50a8f6bc3026eeda55798eacd35 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Tue, 10 Mar 2020 22:51:01 +0000
Subject: [PATCH 05/18] Update renderers to use parse_context

As implemented in mistletoe-api==0.10.0a2
---
 docs/using/index.md              |   2 +-
 docs/using/sphinx.md             |   2 +-
 docs/using/syntax.md             | 273 ++++++++++++++++++++++++++-----
 docs/using/use_api.md            |   7 +-
 myst_parser/docutils_renderer.py |  36 ++--
 myst_parser/html_renderer.py     |  19 ++-
 setup.py                         |   2 +-
 7 files changed, 274 insertions(+), 67 deletions(-)

diff --git a/docs/using/index.md b/docs/using/index.md
index d8ce3964..fce812f6 100644
--- a/docs/using/index.md
+++ b/docs/using/index.md
@@ -5,8 +5,8 @@ MyST documents.
 
 ```{toctree}
 install.md
-sphinx.md
 syntax.md
+sphinx.md
 benchmark.md
 use_api.md
 ```
diff --git a/docs/using/sphinx.md b/docs/using/sphinx.md
index e8b8bd69..1670257f 100644
--- a/docs/using/sphinx.md
+++ b/docs/using/sphinx.md
@@ -1,4 +1,4 @@
-# Using with the Sphinx Document Generator
+# Parsing MyST with the Sphinx Document Generator
 
 Sphinx is a documentation generator for building a website or book from multiple source documents and assets. To get started with Sphinx, see their [Quickstart Guide](https://www.sphinx-doc.org/en/master/usage/quickstart.html).
 
diff --git a/docs/using/syntax.md b/docs/using/syntax.md
index 5a55257e..3d0892cb 100644
--- a/docs/using/syntax.md
+++ b/docs/using/syntax.md
@@ -1,6 +1,6 @@
 (example_syntax)=
 
-# Example syntax for MyST
+# The MyST Syntax Guide
 
 As a base, MyST adheres to the [CommonMark specification](https://spec.commonmark.org/).
 For this, it uses the {ref}`mistletoe:intro/top-level` parser,
@@ -24,72 +24,241 @@ and further details of a few major extensions from the CommonMark flavor of mark
 
 ## Parsed Token Classes
 
+MyST builds on the tokens defined by mistletoe, to extend the syntax:
+
+- {ref}`Core block tokens <mistletoe:tokens/block>`
+- {ref}`Core span tokens <mistletoe:tokens/span>`
+- {ref}`Extension tokens <mistletoe:tokens/extension>`
+
 Tokens are listed in their order of precedence.
 For more information, also see the [CommonMark Spec](https://spec.commonmark.org/0.28/), which the parser is tested against.
 
+```{seealso}
+{ref}`Token API <api/tokens>`
+```
+
 ### Block Tokens
 
 #### Extended block tokens
 
-- **FrontMatter**: A YAML block at the start of the document enclosed by `---`. See
-  {ref}`syntax/frontmatter` for more information.
-- **Directives**: enclosed in 3 or more backticks followed by the directive name wrapped
+`````{list-table}
+:header-rows: 1
+:widths: 10 20 20
+
+* - Token
+  - Description
+  - Example
+* - FrontMatter
+  - A YAML block at the start of the document enclosed by `---`
+  - ```yaml
+    ---
+    key: value
+    ---
+    ```
+* - Directives
+  - enclosed in 3 or more backticks followed by the directive name wrapped
   in curly brackets `{}`. See {ref}`syntax/directives` for more details.
-- **Math**: Two `$` characters wrapping multi-line math, e.g.
+  - ````md
+    ```{directive}
+    :option: value
 
-  ```latex
-  $$
-  a=1
-  $$
-  ```
-
-- **LineComment**: `% this is a comment`. See {ref}`syntax/comments` for more
-  information.
-- **BlockBreak**: `+++`. See {ref}`syntax/blockbreaks` for more information.
+    content
+    ```
+    ````
+* - Math
+  - Two `$` characters wrapping multi-line math
+  - ```latex
+    $$
+    a=1
+    $$
+    ```
+* - Table
+  - Standard markdown table style, with pipe separation.
+  - ```md
+    | a    | b    |
+    | :--- | ---: |
+    | c    | d    |
+    ```
+* - LineComment
+  - A commented line. See {ref}`syntax/comments` for more information.
+  - ```latex
+    % this is a comment
+    ```
+* - BlockBreak
+  - Define blocks of text. See {ref}`syntax/blockbreaks` for more information.
+  - ```md
+    +++ {"meta": "data"}
+    ```
+* - Footnote
+  - A definition for a referencing footnote, that is placed at the bottom of the document.
+   See {ref}`syntax/footnotes` for more details.
+  - ```md
+    [^ref]: Some footnote text
+    ```
+`````
 
 #### CommonMark tokens
 
-- **HTMLBlock**: Any valid HTML (rendered in HTML output only)
-- **BlockCode**: indented text (4 spaces)
-- **Heading**: `# Heading` (levels 1-6)
-- **SetextHeading**: underlined header (using multiple `=` or `-`)
-- **Quote**: `> this is a quote`
-- **CodeFence**: enclosed in 3 or more backticks with an optional language name. E.g.:
-
-  ````md
-  ```python
-  print('this is python')
-  ```
-  ````
-
-- **ThematicBreak**: `---`
-- **List**: bullet points or enumerated.
-- **Table**: Standard markdown table styles.
-- **LinkDefinition**: A substitution for an inline link (e.g. `[key][name]`), which can have a reference target (no spaces), and an optional title (in `"`), e.g. `[key]: https://www.google.com "a title"`
-- **Paragraph**: General inline text
+`````{list-table}
+:header-rows: 1
+:widths: 10 20 20
+
+* - Token
+  - Description
+  - Example
+* - HTMLBlock
+  - Any valid HTML (rendered in HTML output only)
+  - ```html
+    <p>some text</p>
+    ```
+* - BlockCode
+  - indented text (4 spaces or a tab)
+  - ```md
+        included as literal *text*
+    ```
+* - Heading
+  - Level 1-6 headings, denoted by number of `#`
+  - ```md
+    ### Heading level 3
+    ```
+* - SetextHeading
+  - Underlined header (using multiple `=` or `-`)
+  - ```md
+    Header
+    ======
+    ```
+* - Quote
+  - quoted text
+  - ```md
+    > this is a quote
+    ```
+* - CodeFence
+  - enclosed in 3 or more backticks with an optional language name
+  - ````md
+    ```python
+    print('this is python')
+    ```
+    ````
+* - ThematicBreak
+  - Creates a horizontal line in the output
+  - ```md
+    ---
+    ```
+* - List
+  - bullet points or enumerated.
+  - ```md
+    - item
+      - nested item
+    1. numbered item
+    ```
+* - LinkDefinition
+  -  A substitution for an inline link, which can have a reference target (no spaces), and an optional title (in `"`)
+  - ```md
+    [key]: https://www.google.com "a title"
+    ```
+* - Paragraph
+  - General inline text
+  - ```md
+    any *text*
+    ```
+
+`````
 
 ### Span (Inline) Tokens
 
 #### Extended inline tokens
 
-- **Role**: `` `{rolename}`interpreted text` ``. See {ref}`syntax/roles` for more
+`````{list-table}
+:header-rows: 1
+:widths: 10 20 20
+
+* - Token
+  - Description
+  - Example
+* - Role
+  - See {ref}`syntax/roles` for more
   information.
-- **Target**: `(target)=` (precedes element to target, e.g. header). See
+  - ```md
+    `{rolename}`interpreted text`
+    ```
+* - Target
+  - Precedes element to target, e.g. header. See
   {ref}`syntax/targets` for more information.
-- **Math**: `$a=1$` or `$$a=1$$`
+  - ```md
+    (target)=
+    ```
+* - Math
+  - dollar enclosed math
+  - ```latex
+    $a=1$ or $$a=1$$
+    ```
+* - FootReference
+  - Reference a footnote. See {ref}`syntax/footnotes` for more details.
+  - ```md
+    [^abc]
+    ```
+`````
 
 #### CommonMark inline tokens
 
-- **HTMLSpan**: any valid HTML (rendered in HTML output only)
-- **EscapeSequence**: `\*`
-- **AutoLink**: `<http://www.google.com>`
-- **InlineCode**: `` `a=1` ``
-- **LineBreak**: Soft or hard (ends with spaces or `\`)
-- **Image**: `![alt](src "title")`
-- **Link**: `[text](target "title")` or `[text][key]` (key from `Footnote`)
-- **Strong**: `**strong**`
-- **Emphasis**: `*emphasis*`
-- **RawText**
+`````{list-table}
+:header-rows: 1
+:widths: 10 20 20
+
+* - Token
+  - Description
+  - Example
+* - HTMLSpan
+  - any valid HTML (rendered in HTML output only)
+  - ```html
+    <p>some text</p>
+    ```
+* - EscapeSequence
+  - escaped symbols (to avoid them being interpreted as other syntax elements)
+  - ```md
+    \*
+    ```
+* - AutoLink
+  - link that is shown in final output
+  - ```md
+    <http://www.google.com>
+    ```
+* - InlineCode
+  - literal text
+  - ```md
+    `a=1`
+    ```
+* - LineBreak
+  - Soft or hard (ends with spaces or \)
+  - ```md
+    A hard break\
+    ```
+* - Image
+  - link to an image
+  - ```md
+    ![alt](src "title")
+    ```
+* - Link
+  - Reference `LinkDefinitions`
+  - ```md
+    [text](target "title") or [text][key]
+    ```
+* - Strong
+  - bold text
+  - ```md
+    **strong**
+    ```
+* - Emphasis
+  - italic text
+  - ```md
+    *emphasis*
+    ```
+* - RawText
+  - any text
+  - ```md
+    any text
+    ```
+`````
 
 (syntax/directives)=
 
@@ -519,3 +688,17 @@ is synonymous with using the [any inline role](https://www.sphinx-doc.org/en/mas
 
 Using the same example, see this ref: [](syntax/targets), and here's a ref back to the top of
 this page: [my text](example_syntax).
+
+(syntax/footnotes)=
+
+### Footnotes
+
+```md
+This is a footnote reference.[^myref]
+
+[^myref]: This is the footnote text
+```
+
+This is a footnote reference.[^myref]
+
+[^myref]: This is the footnote text.
diff --git a/docs/using/use_api.md b/docs/using/use_api.md
index 0aec5d4e..1f27ac55 100644
--- a/docs/using/use_api.md
+++ b/docs/using/use_api.md
@@ -2,7 +2,12 @@
 
 % TODO eventually this should be wrote as a notebook (with MyST-NB)!
 
-MyST-Parser may be used as an API *via* the `myst_parser` package, see {ref}`api/main` for full details.
+MyST-Parser may be used as an API *via* the `myst_parser` package.
+
+```{seealso}
+- {ref}`Programmatic Use of Mistletoe <mistletoe:intro/api_use>`
+- {ref}`The MyST-Parser API <api/main>`
+```
 
 The raw text is first parsed to syntax 'tokens',
 then these are converted to other formats using 'renderers'.
diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
index 1e2de183..d39ec942 100644
--- a/myst_parser/docutils_renderer.py
+++ b/myst_parser/docutils_renderer.py
@@ -20,7 +20,7 @@
 
 from mistletoe import block_tokens, block_tokens_ext, span_tokens, span_tokens_ext
 from mistletoe.renderers.base import BaseRenderer
-from mistletoe.parse_context import get_parse_context
+from mistletoe.parse_context import get_parse_context, ParseContext
 
 from myst_parser import block_tokens as myst_block_tokens
 from myst_parser import span_tokens as myst_span_tokens
@@ -71,8 +71,7 @@ def __init__(
         document: Optional[nodes.document] = None,
         current_node: Optional[nodes.Element] = None,
         config: Optional[dict] = None,
-        find_blocks=None,
-        find_spans=None,
+        parse_context: Optional[ParseContext] = None,
     ):
         """Initialise the renderer.
 
@@ -80,8 +79,13 @@ def __init__(
         :param current_node: The root node from which to begin populating
             (default is document, or should be an ancestor of document)
         :param config: contains configuration specific to the rendering process
-        :param find_blocks: override the default block tokens (classes or class paths)
-        :param find_spans: override the default span tokens (classes or class paths)
+        :param parse_context: the parse context stores global parsing variables,
+            such as the block/span tokens to search for,
+            and link/footnote definitions that have been collected.
+            If None, a new context will be instatiated, with the default
+            block/span tokens for this renderer.
+            These will be re-instatiated on ``__enter__``.
+        :type parse_context: mistletoe.parse_context.ParseContext
         """
         self.config = config or {}
         self.document = document or self.new_document()  # type: nodes.document
@@ -91,7 +95,7 @@ def __init__(
         get_language(self.language_module)
         self._level_to_elem = {0: self.document}
 
-        super().__init__(find_blocks=find_blocks, find_spans=find_spans)
+        super().__init__(parse_context=parse_context)
 
     def new_document(self, source_path="notset") -> nodes.document:
         """Create a new docutils document."""
@@ -133,7 +137,6 @@ def current_node_context(self, node, append: bool = False):
         self.current_node = current_node
 
     def render_document(self, token: block_tokens.Document):
-        self.link_definitions.update(token.link_definitions)
         if token.front_matter:
             self.render_front_matter(token.front_matter)
         self.render_children(token)
@@ -141,12 +144,12 @@ def render_document(self, token: block_tokens.Document):
         if getattr(token, "is_nested", False):
             # if the document is nested in another, we don't want to output footnotes
             return self.document
-
         # we use the footnotes stored in the global context,
         # rather than those stored on the document,
         # since additional references may have been made in nested parses
         foot_refs = get_parse_context().foot_references
         footnotes = get_parse_context().foot_definitions
+
         if foot_refs:
             self.current_node.append(nodes.transition())
         for footref in foot_refs:
@@ -589,7 +592,7 @@ def render_directive(self, token):
             )
         else:
             state_machine = MockStateMachine(self, token.position[0])
-            state = MockState(self, state_machine, token.position[0])
+            state = MockState(self, state_machine, token.position[0], token=token)
             directive_instance = directive_class(
                 name=name,
                 # the list of positional arguments
@@ -846,10 +849,15 @@ class MockState:
     """
 
     def __init__(
-        self, renderer: DocutilsRenderer, state_machine: "MockStateMachine", lineno: int
+        self,
+        renderer: DocutilsRenderer,
+        state_machine: "MockStateMachine",
+        lineno: int,
+        token=None,
     ):
         self._renderer = renderer
         self._lineno = lineno
+        self._token = token
         self.document = renderer.document
         self.state_machine = state_machine
 
@@ -883,8 +891,14 @@ def inline_text(self, text: str, lineno: int):
         # TODO return messages?
         messages = []
         paragraph = nodes.paragraph("")
+        # here we instatiate a new renderer,
+        # so that the nested parse does not effect the current renderer,
+        # but we use the same global parse context, so that link references, etc
+        # are added to the global parse.
         renderer = self._renderer.__class__(
-            document=self.document, current_node=paragraph
+            document=self.document,
+            current_node=paragraph,
+            parse_context=get_parse_context(),
         )
         doc_token = myst_block_tokens.Document.read(
             text, start_line=self._lineno, front_matter=False, reset_definitions=False
diff --git a/myst_parser/html_renderer.py b/myst_parser/html_renderer.py
index f9f368c1..8214493f 100644
--- a/myst_parser/html_renderer.py
+++ b/myst_parser/html_renderer.py
@@ -1,4 +1,7 @@
+from typing import Optional
+
 from mistletoe import block_tokens, block_tokens_ext, span_tokens, span_tokens_ext
+from mistletoe.parse_context import ParseContext
 from mistletoe.renderers import html as html_renderer
 
 from myst_parser.block_tokens import LineComment, BlockBreak, Quote, Paragraph, List
@@ -47,23 +50,25 @@ class HTMLRenderer(html_renderer.HTMLRenderer):
 
     def __init__(
         self,
-        find_blocks=None,
-        find_spans=None,
+        parse_context: Optional[ParseContext] = None,
         add_mathjax=False,
         as_standalone=False,
         add_css=None,
     ):
         """Intitalise HTML renderer
 
-        :param find_blocks: override the default block tokens (classes or class paths)
-        :param find_spans: override the default span tokens (classes or class paths)
+        :param parse_context: the parse context stores global parsing variables,
+            such as the block/span tokens to search for,
+            and link/footnote definitions that have been collected.
+            If None, a new context will be instatiated, with the default
+            block/span tokens for this renderer.
+            These will be re-instatiated on ``__enter__``.
+        :type parse_context: mistletoe.parse_context.ParseContext
         :param add_mathjax: add the mathjax CDN
         :param as_standalone: return the HTML body within a minmal HTML page
         :param add_css: if as_standalone=True, CSS to add to the header
         """
-        super().__init__(
-            find_blocks=find_blocks, find_spans=find_spans, as_standalone=False
-        )
+        super().__init__(parse_context=parse_context, as_standalone=False)
 
         self.mathjax_src = ""
         if add_mathjax:
diff --git a/setup.py b/setup.py
index 28a57d4c..b479ec88 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@
     ],
     keywords="markdown lexer parser development docutils sphinx",
     python_requires=">=3.6",
-    install_requires=["mistletoe-ebp~=0.10.0a1"],
+    install_requires=["mistletoe-ebp~=0.10.0a2"],
     extras_require={
         "sphinx": ["pyyaml", "docutils>=0.15", "sphinx>=2,<3"],
         "code_style": ["flake8<3.8.0,>=3.7.0", "black", "pre-commit==1.17.0"],

From 7d4f16280e8866df104e399b53fbdb8ab52742e2 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 00:21:59 +0000
Subject: [PATCH 06/18] Add user documentation

This also add a note to address #116
---
 docs/using/syntax.md                  | 46 +++++++++++++++++++++++++--
 docs/using/use_api.md                 |  3 +-
 tests/test_renderers/test_docutils.py |  6 +++-
 3 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/docs/using/syntax.md b/docs/using/syntax.md
index 3d0892cb..2ea646c8 100644
--- a/docs/using/syntax.md
+++ b/docs/using/syntax.md
@@ -625,6 +625,22 @@ Is below, but it won't be parsed into the document.
 
 % my comment
 
+````{important}
+Since comments are a block level entity, they will terminate the previous block.
+In practical terms, this means that the following lines
+will be broken up into two paragraphs, resulting in a new line between them:
+
+```
+a line
+% a comment
+another line
+```
+
+a line
+% a comment
+another line
+````
+
 (syntax/blockbreaks)=
 
 ### Block Breaks
@@ -696,9 +712,35 @@ this page: [my text](example_syntax).
 ```md
 This is a footnote reference.[^myref]
 
-[^myref]: This is the footnote text
+[^myref]: This is the footnote definition.
 ```
 
 This is a footnote reference.[^myref]
 
-[^myref]: This is the footnote text.
+[^myref]: This is the footnote definition.
+
+```{important}
+Although footnote references can be used just fine within directives, e.g.[^myref],
+it it recommended that footnote definitions are not set here.
+
+This is because, in the current implementation, they may not be available to
+reference in text above that particular directive.
+```
+
+````{note}
+Currently, footnote definitions may only be on a single line.
+However, it is intended in an update to come, that any preceding text which is
+indented by four or more spaces, will also be included in the footnote definition, e.g.
+
+```md
+[^myref]: This is the footnote definition.
+
+    That continues for all indented lines
+
+    Plus any precding unindented lines,
+that are not separated by a blank line
+
+This is not part of the footnote.
+```
+
+````
diff --git a/docs/using/use_api.md b/docs/using/use_api.md
index 1f27ac55..a0801f49 100644
--- a/docs/using/use_api.md
+++ b/docs/using/use_api.md
@@ -5,7 +5,8 @@
 MyST-Parser may be used as an API *via* the `myst_parser` package.
 
 ```{seealso}
-- {ref}`Programmatic Use of Mistletoe <mistletoe:intro/api_use>`
+% - {ref}`Programmatic Use of Mistletoe <mistletoe:intro/api_use>`
+- {ref}`Programmatic Use of Mistletoe <mistletoe:intro/usage>`
 - {ref}`The MyST-Parser API <api/main>`
 ```
 
diff --git a/tests/test_renderers/test_docutils.py b/tests/test_renderers/test_docutils.py
index 9510e1b0..b7ad1ec2 100644
--- a/tests/test_renderers/test_docutils.py
+++ b/tests/test_renderers/test_docutils.py
@@ -269,12 +269,16 @@ def test_cross_referencing(sphinx_renderer, file_regression):
 
 
 def test_comment(renderer_mock):
-    renderer_mock.render(tokenize_main([r"% a comment"])[0])
+    renderer_mock.render(Document.read(["line 1", r"% a comment", "line 2"]))
     assert renderer_mock.document.pformat() == dedent(
         """\
     <document source="notset">
+        <paragraph>
+            line 1
         <comment xml:space="preserve">
             a comment
+        <paragraph>
+            line 2
     """
     )
 

From 1ce1ceebb6a6acd2b76c61e8d714d73cadcb5dc2 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 01:03:30 +0000
Subject: [PATCH 07/18] More documentation

---
 docs/using/sphinx.md |  2 +-
 docs/using/syntax.md | 24 +++++++++++++++++++++---
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/docs/using/sphinx.md b/docs/using/sphinx.md
index 1670257f..1600c902 100644
--- a/docs/using/sphinx.md
+++ b/docs/using/sphinx.md
@@ -1,4 +1,4 @@
-# Parsing MyST with the Sphinx Document Generator
+# Parsing MyST with Sphinx
 
 Sphinx is a documentation generator for building a website or book from multiple source documents and assets. To get started with Sphinx, see their [Quickstart Guide](https://www.sphinx-doc.org/en/master/usage/quickstart.html).
 
diff --git a/docs/using/syntax.md b/docs/using/syntax.md
index 2ea646c8..6e0db6b6 100644
--- a/docs/using/syntax.md
+++ b/docs/using/syntax.md
@@ -709,6 +709,13 @@ this page: [my text](example_syntax).
 
 ### Footnotes
 
+Footnote labels can be any alpha-numeric string (no spaces), and are case-insensitive.
+The actual label is not displayed in the rendered text; instead they are numbered,
+in the order which they are referenced.
+All footnote definitions are collected, and displayed at the bottom of the page
+(ordered by number).
+Note that un-referenced footnote definitions will not be displayed.
+
 ```md
 This is a footnote reference.[^myref]
 
@@ -719,13 +726,24 @@ This is a footnote reference.[^myref]
 
 [^myref]: This is the footnote definition.
 
-```{important}
+````{important}
 Although footnote references can be used just fine within directives, e.g.[^myref],
-it it recommended that footnote definitions are not set here.
+it it recommended that footnote definitions are not set within directives,
+unless they will only be referenced within that same directive:
+
+```md
+[^other]
+
+[^other]: A definition within a directive
+```
+
+[^other]
+
+[^other]: A definition within a directive
 
 This is because, in the current implementation, they may not be available to
 reference in text above that particular directive.
-```
+````
 
 ````{note}
 Currently, footnote definitions may only be on a single line.

From ddc9c17e56d8270beae6caaaaf96160e6fbc4e39 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 01:49:53 +0000
Subject: [PATCH 08/18] Fix numbering order

---
 myst_parser/docutils_renderer.py              | 13 +++++-
 .../test_sphinx_builds/test_footnotes.html    | 40 +++++++++----------
 .../test_sphinx_builds/test_footnotes.xml     | 26 ++++++------
 3 files changed, 45 insertions(+), 34 deletions(-)

diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
index d39ec942..bef14944 100644
--- a/myst_parser/docutils_renderer.py
+++ b/myst_parser/docutils_renderer.py
@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from contextlib import contextmanager
 import copy
 from os.path import splitext
@@ -144,12 +145,22 @@ def render_document(self, token: block_tokens.Document):
         if getattr(token, "is_nested", False):
             # if the document is nested in another, we don't want to output footnotes
             return self.document
+
         # we use the footnotes stored in the global context,
         # rather than those stored on the document,
         # since additional references may have been made in nested parses
-        foot_refs = get_parse_context().foot_references
         footnotes = get_parse_context().foot_definitions
 
+        # we don't use the foot_references stored on the global context,
+        # since references within directives/roles will have been added after
+        # those from the initial markdown parse
+        # instead we gather them from a walk of the created document
+        # foot_refs = get_parse_context().foot_references
+        foot_refs = OrderedDict()
+        for refnode in self.document.traverse(nodes.footnote_reference):
+            if refnode["refname"] not in foot_refs:
+                foot_refs[refnode["refname"]] = True
+
         if foot_refs:
             self.current_node.append(nodes.transition())
         for footref in foot_refs:
diff --git a/tests/test_sphinx/test_sphinx_builds/test_footnotes.html b/tests/test_sphinx/test_sphinx_builds/test_footnotes.html
index 66acbda1..c0e443f6 100644
--- a/tests/test_sphinx/test_sphinx_builds/test_footnotes.html
+++ b/tests/test_sphinx/test_sphinx_builds/test_footnotes.html
@@ -19,26 +19,26 @@ <h1>
      </p>
      <p>
       <a class="footnote-reference brackets" href="#d" id="id2">
-       5
+       2
       </a>
      </p>
     </div>
     <p>
      <a class="footnote-reference brackets" href="#a" id="id3">
-      2
+      3
      </a>
     </p>
     <p>
      <a class="footnote-reference brackets" href="#b" id="id4">
-      3
+      4
      </a>
     </p>
     <p>
      <a class="footnote-reference brackets" href="#id7" id="id5">
-      4
+      5
      </a>
      <a class="footnote-reference brackets" href="#id7" id="id6">
-      4
+      5
      </a>
     </p>
     <hr class="docutils"/>
@@ -55,10 +55,22 @@ <h1>
        a footnote referenced first
       </p>
      </dd>
+     <dt class="label" id="d">
+      <span class="brackets">
+       <a class="fn-backref" href="#id2">
+        2
+       </a>
+      </span>
+     </dt>
+     <dd>
+      <p>
+       a footnote referenced in a directive
+      </p>
+     </dd>
      <dt class="label" id="a">
       <span class="brackets">
        <a class="fn-backref" href="#id3">
-        2
+        3
        </a>
       </span>
      </dt>
@@ -73,7 +85,7 @@ <h1>
      <dt class="label" id="b">
       <span class="brackets">
        <a class="fn-backref" href="#id4">
-        3
+        4
        </a>
       </span>
      </dt>
@@ -84,7 +96,7 @@ <h1>
      </dd>
      <dt class="label" id="id7">
       <span class="brackets">
-       4
+       5
       </span>
       <span class="fn-backref">
        (
@@ -103,18 +115,6 @@ <h1>
        multiple references footnote
       </p>
      </dd>
-     <dt class="label" id="d">
-      <span class="brackets">
-       <a class="fn-backref" href="#id2">
-        5
-       </a>
-      </span>
-     </dt>
-     <dd>
-      <p>
-       a footnote referenced in a directive
-      </p>
-     </dd>
     </dl>
    </div>
   </div>
diff --git a/tests/test_sphinx/test_sphinx_builds/test_footnotes.xml b/tests/test_sphinx/test_sphinx_builds/test_footnotes.xml
index e893875a..fed8df0c 100644
--- a/tests/test_sphinx/test_sphinx_builds/test_footnotes.xml
+++ b/tests/test_sphinx/test_sphinx_builds/test_footnotes.xml
@@ -8,44 +8,44 @@
         <note>
             <paragraph>
                 <footnote_reference auto="1" docname="footnote_md" ids="id2" refid="d">
-                    5
+                    2
         <paragraph>
             <footnote_reference auto="1" docname="footnote_md" ids="id3" refid="a">
-                2
+                3
         <paragraph>
             <footnote_reference auto="1" docname="footnote_md" ids="id4" refid="b">
-                3
+                4
         <paragraph>
             <footnote_reference auto="1" docname="footnote_md" ids="id5" refid="id7">
-                4
+                5
              
             <footnote_reference auto="1" docname="footnote_md" ids="id6" refid="id7">
-                4
+                5
         <transition>
         <footnote auto="1" backrefs="id1" docname="footnote_md" ids="c" names="c">
             <label>
                 1
             <paragraph>
                 a footnote referenced first
-        <footnote auto="1" backrefs="id3" docname="footnote_md" ids="a" names="a">
+        <footnote auto="1" backrefs="id2" docname="footnote_md" ids="d" names="d">
             <label>
                 2
+            <paragraph>
+                a footnote referenced in a directive
+        <footnote auto="1" backrefs="id3" docname="footnote_md" ids="a" names="a">
+            <label>
+                3
             <paragraph>
                 some footnote 
                 <emphasis>
                     text
         <footnote auto="1" backrefs="id4" docname="footnote_md" ids="b" names="b">
             <label>
-                3
+                4
             <paragraph>
                 a footnote before its reference
         <footnote auto="1" backrefs="id5 id6" docname="footnote_md" ids="id7" names="123">
-            <label>
-                4
-            <paragraph>
-                multiple references footnote
-        <footnote auto="1" backrefs="id2" docname="footnote_md" ids="d" names="d">
             <label>
                 5
             <paragraph>
-                a footnote referenced in a directive
+                multiple references footnote

From 59402b4cbdbce4ccc561e30a3536dac1d82ffea6 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 02:08:02 +0000
Subject: [PATCH 09/18] Minor doc fix

---
 docs/using/syntax.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/using/syntax.md b/docs/using/syntax.md
index 6e0db6b6..988f0929 100644
--- a/docs/using/syntax.md
+++ b/docs/using/syntax.md
@@ -709,7 +709,8 @@ this page: [my text](example_syntax).
 
 ### Footnotes
 
-Footnote labels can be any alpha-numeric string (no spaces), and are case-insensitive.
+Footnote labels **start with `^`** and can then be any alpha-numeric string (no spaces),
+which is case-insensitive.
 The actual label is not displayed in the rendered text; instead they are numbered,
 in the order which they are referenced.
 All footnote definitions are collected, and displayed at the bottom of the page

From 636fa8812395886de2a4d64dc99d78470db3562b Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 02:16:30 +0000
Subject: [PATCH 10/18] Add conda install info and trim README

fixes #85
---
 README.md             | 91 +++++--------------------------------------
 docs/using/install.md |  9 ++++-
 2 files changed, 18 insertions(+), 82 deletions(-)

diff --git a/README.md b/README.md
index a057cc18..ca1e9a2b 100644
--- a/README.md
+++ b/README.md
@@ -5,12 +5,21 @@
 [![Documentation Status][rtd-badge]][rtd-link]
 [![Code style: black][black-badge]][black-link]
 [![PyPI][pypi-badge]][pypi-link]
-<!-- [![Conda][conda-badge]][conda-link] -->
+[![Conda][conda-badge]][conda-link]
 
 An extended commonmark compliant parser, with bridges to docutils & sphinx.
 
 ## Usage
 
+To install the MyST parser, run the following in a
+[Conda environment](https://docs.conda.io) (recommended):
+
+```bash
+conda install -c conda-forge myst-parser
+```
+
+or
+
 ```bash
 pip install myst-parser[sphinx]
 ```
@@ -26,86 +35,6 @@ pip install -e .[sphinx,code_style,testing,rtd]
 
 To use the MyST parser in Sphinx, simply add: `extensions = ["myst_parser"]` to your `conf.py`.
 
-## Parsed Token Classes
-
-For more information, also see the [CommonMark Spec](https://spec.commonmark.org/0.28/).
-
-### Block Tokens
-
-- **FrontMatter**: A YAML block at the start of the document enclosed by `---`
-- **HTMLBlock**: Any valid HTML (rendered in HTML output only)
-- **LineComment**: `% this is a comment`
-- **BlockCode**: indented text (4 spaces)
-- **Heading**: `# Heading` (levels 1-6)
-- **SetextHeading**: underlined header
-- **Quote**: `> this is a quote`
-- **CodeFence**: enclosed in 3 or more backticks.
-  If it starts with a `{name}`, then treated as directive.
-- **ThematicBreak**: `---`
-- **List**: bullet points or enumerated.
-- **Table**: Standard markdown table styles.
-- **Footnote**: A substitution for an inline link (e.g. `[key][name]`), which can have a reference target (no spaces), and an optional title (in `"`), e.g. `[key]: https://www.google.com "a title"`
-- **Paragraph**: General inline text
-
-### Span (Inline) Tokens
-
-- **Role**: `` `{name}`interpreted text` ``
-- **Math**: `$a=1$` or `$$a=1$$`
-- **HTMLSpan**: any valid HTML (rendered in HTML output only)
-- **EscapeSequence**: `\*`
-- **AutoLink**: `<http://www.google.com>`
-- **Target**: `(target)=` (precedes element to target, e.g. header)
-- **InlineCode**: `` `a=1` ``
-- **LineBreak**: Soft or hard (ends with spaces or `\`)
-- **Image**: `![alt](src "title")`
-- **Link**: `[text](target "title")` or `[text][key]` (key from `Footnote`)
-- **Strong**: `**strong**`
-- **Emphasis**: `*emphasis*`
-- **RawText**
-
-## Contributing
-
-### Code Style
-
-Code style is tested using [flake8](http://flake8.pycqa.org),
-with the configuration set in `.flake8`,
-and code formatted with [black](https://github.com/ambv/black).
-
-Installing with `myst-parser[code_style]` makes the [pre-commit](https://pre-commit.com/)
-package available, which will ensure this style is met before commits are submitted, by reformatting the code
-and testing for lint errors.
-It can be setup by:
-
-```shell
->> cd MyST-Parser
->> pre-commit install
-```
-
-Optionally you can run `black` and `flake8` separately:
-
-```shell
->> black .
->> flake8 .
-```
-
-Editors like VS Code also have automatic code reformat utilities, which can adhere to this standard.
-
-### Pull Requests
-
-To contribute, make Pull Requests to the `master` branch (this is the default branch). A PR can consist of one or multiple commits. Before you open a PR, make sure to clean up your commit history and create the commits that you think best divide up the total work as outlined above (use `git rebase` and `git commit --amend`). Ensure all commit messages clearly summarise the changes in the header and the problem that this commit is solving in the body.
-
-Merging pull requests: There are three ways of 'merging' pull requests on GitHub:
-
-- Squash and merge: take all commits, squash them into a single one and put it on top of the base branch.
-    Choose this for pull requests that address a single issue and are well represented by a single commit.
-    Make sure to clean the commit message (title & body)
-- Rebase and merge: take all commits and 'recreate' them on top of the base branch. All commits will be recreated with new hashes.
-    Choose this for pull requests that require more than a single commit.
-    Examples: PRs that contain multiple commits with individually significant changes; PRs that have commits from different authors (squashing commits would remove attribution)
-- Merge with merge commit: put all commits as they are on the base branch, with a merge commit on top
-    Choose for collaborative PRs with many commits. Here, the merge commit provides actual benefits.
-
-
 [travis-badge]: https://travis-ci.org/ExecutableBookProject/MyST-Parser.svg?branch=master
 [travis-link]: https://travis-ci.org/ExecutableBookProject/MyST-Parser
 [coveralls-badge]: https://coveralls.io/repos/github/ExecutableBookProject/MyST-Parser/badge.svg?branch=master
diff --git a/docs/using/install.md b/docs/using/install.md
index 99265bf3..89f56119 100644
--- a/docs/using/install.md
+++ b/docs/using/install.md
@@ -5,7 +5,14 @@ Installing the MyST parser provides access to two tools:
 * A MyST-to-docutils parser and renderer.
 * A Sphinx parser that utilizes the above tool in building your documenation.
 
-To install the MyST parser, run the following:
+To install the MyST parser, run the following in a
+[Conda environment](https://docs.conda.io) (recommended):
+
+```bash
+conda install -c conda-forge myst-parser
+```
+
+or
 
 ```bash
 pip install myst-parser[sphinx]

From bba2827c516aa744ff4bcf7d70272bb58881bb56 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 02:22:36 +0000
Subject: [PATCH 11/18] minor doc fix

---
 docs/using/syntax.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/using/syntax.md b/docs/using/syntax.md
index 988f0929..66d79169 100644
--- a/docs/using/syntax.md
+++ b/docs/using/syntax.md
@@ -229,7 +229,7 @@ For more information, also see the [CommonMark Spec](https://spec.commonmark.org
     `a=1`
     ```
 * - LineBreak
-  - Soft or hard (ends with spaces or \)
+  - Soft or hard (ends with spaces or backslash)
   - ```md
     A hard break\
     ```

From 75e7280f6be79afcb3e47ec50cb8e1875c6a2ad7 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 02:39:00 +0000
Subject: [PATCH 12/18] minor doc add

---
 docs/using/syntax.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/using/syntax.md b/docs/using/syntax.md
index 66d79169..2a473780 100644
--- a/docs/using/syntax.md
+++ b/docs/using/syntax.md
@@ -720,12 +720,12 @@ Note that un-referenced footnote definitions will not be displayed.
 ```md
 This is a footnote reference.[^myref]
 
-[^myref]: This is the footnote definition.
+[^myref]: This **is** the footnote definition.
 ```
 
 This is a footnote reference.[^myref]
 
-[^myref]: This is the footnote definition.
+[^myref]: This **is** the footnote definition.
 
 ````{important}
 Although footnote references can be used just fine within directives, e.g.[^myref],

From 33e22e1b2ab1f62bb300db9941cc805501255de4 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 02:55:34 +0000
Subject: [PATCH 13/18] Add badges to documentation

---
 docs/develop/contributing.md | 17 +++++++++++++++++
 docs/using/install.md        |  8 ++++++++
 2 files changed, 25 insertions(+)

diff --git a/docs/develop/contributing.md b/docs/develop/contributing.md
index f06af930..5575046f 100644
--- a/docs/develop/contributing.md
+++ b/docs/develop/contributing.md
@@ -1,5 +1,11 @@
 # Contributing
 
+[![CI Status][travis-badge]][travis-link]
+[![Coverage][coveralls-badge]][coveralls-link]
+[![CircleCI][circleci-badge]][circleci-link]
+[![Documentation Status][rtd-badge]][rtd-link]
+[![Code style: black][black-badge]][black-link]
+
 ## Code Style
 
 Code style is tested using [flake8](http://flake8.pycqa.org),
@@ -63,3 +69,14 @@ Merging pull requests: There are three ways of 'merging' pull requests on GitHub
     Examples: PRs that contain multiple commits with individually significant changes; PRs that have commits from different authors (squashing commits would remove attribution)
 - Merge with merge commit: put all commits as they are on the base branch, with a merge commit on top
     Choose for collaborative PRs with many commits. Here, the merge commit provides actual benefits.
+
+[travis-badge]: https://travis-ci.org/ExecutableBookProject/MyST-Parser.svg?branch=master
+[travis-link]: https://travis-ci.org/ExecutableBookProject/MyST-Parser
+[coveralls-badge]: https://coveralls.io/repos/github/ExecutableBookProject/MyST-Parser/badge.svg?branch=master
+[coveralls-link]: https://coveralls.io/github/ExecutableBookProject/MyST-Parser?branch=master
+[circleci-badge]: https://circleci.com/gh/ExecutableBookProject/MyST-Parser.svg?style=shield
+[circleci-link]: https://circleci.com/gh/ExecutableBookProject/MyST-Parser
+[rtd-badge]: https://readthedocs.org/projects/myst-parser/badge/?version=latest
+[rtd-link]: https://myst-parser.readthedocs.io/en/latest/?badge=latest
+[black-badge]: https://img.shields.io/badge/code%20style-black-000000.svg
+[black-link]: https://github.com/ambv/black
diff --git a/docs/using/install.md b/docs/using/install.md
index 89f56119..c19c4f2c 100644
--- a/docs/using/install.md
+++ b/docs/using/install.md
@@ -1,5 +1,8 @@
 # Installing the MyST Parser
 
+[![PyPI][pypi-badge]][pypi-link]
+[![Conda][conda-badge]][conda-link]
+
 Installing the MyST parser provides access to two tools:
 
 * A MyST-to-docutils parser and renderer.
@@ -26,3 +29,8 @@ cd MyST-Parser
 git checkout master
 pip install -e .[sphinx,code_style,testing,rtd]
 ```
+
+[pypi-badge]: https://img.shields.io/pypi/v/myst-parser.svg
+[pypi-link]: https://pypi.org/project/myst-parser
+[conda-badge]: https://anaconda.org/conda-forge/myst-parser/badges/version.svg
+[conda-link]: https://anaconda.org/conda-forge/myst-parser

From 48b8e93ee23dce790c5457e92db21f5ba78583d3 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 03:08:35 +0000
Subject: [PATCH 14/18] bump version

---
 myst_parser/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/myst_parser/__init__.py b/myst_parser/__init__.py
index 763b4f64..ce11c1e7 100644
--- a/myst_parser/__init__.py
+++ b/myst_parser/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "0.5.1"
+__version__ = "0.6.0a1"
 
 
 def text_to_tokens(text: str):

From 99dabde9bd0b8f2807adfc2cd02e7e39389c86f6 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Wed, 11 Mar 2020 03:30:22 +0000
Subject: [PATCH 15/18] Update doc reference link

---
 docs/using/use_api.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/using/use_api.md b/docs/using/use_api.md
index a0801f49..1f27ac55 100644
--- a/docs/using/use_api.md
+++ b/docs/using/use_api.md
@@ -5,8 +5,7 @@
 MyST-Parser may be used as an API *via* the `myst_parser` package.
 
 ```{seealso}
-% - {ref}`Programmatic Use of Mistletoe <mistletoe:intro/api_use>`
-- {ref}`Programmatic Use of Mistletoe <mistletoe:intro/usage>`
+- {ref}`Programmatic Use of Mistletoe <mistletoe:intro/api_use>`
 - {ref}`The MyST-Parser API <api/main>`
 ```
 

From e090d0792c19bb194e4c10a8a3e9609f01eb02a6 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Thu, 12 Mar 2020 01:17:38 +0000
Subject: [PATCH 16/18] Bump mistletoe version

This allows markdown parsing errors to be logged to sphinx.
---
 myst_parser/docutils_renderer.py                   | 11 ++++++++---
 myst_parser/sphinx_parser.py                       | 14 ++++++++++++--
 setup.py                                           |  2 +-
 .../test_ast/test_front_matter_basic_strings0_.yml |  2 +-
 4 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
index bef14944..61bc000d 100644
--- a/myst_parser/docutils_renderer.py
+++ b/myst_parser/docutils_renderer.py
@@ -20,6 +20,7 @@
 import yaml
 
 from mistletoe import block_tokens, block_tokens_ext, span_tokens, span_tokens_ext
+from mistletoe.base_elements import SourceLines
 from mistletoe.renderers.base import BaseRenderer
 from mistletoe.parse_context import get_parse_context, ParseContext
 
@@ -113,8 +114,10 @@ def add_line_and_source_path(self, node, token):
 
     def nested_render_text(self, text: str, lineno: int):
         """Render unparsed text."""
+        # TODO propogate SourceLines metadata from parent document
+        lines = SourceLines(text, start_line=lineno, standardize_ends=True)
         doc_token = myst_block_tokens.Document.read(
-            text, start_line=lineno, front_matter=True, reset_definitions=False
+            lines, front_matter=True, reset_definitions=False
         )
         # TODO think if this is the best way: here we consume front matter,
         # but then remove it. this is for example if includes have front matter
@@ -184,7 +187,7 @@ def render_front_matter(self, token):
 
         """
         try:
-            data = yaml.safe_load(token.content) or {}
+            data = token.get_data()
         except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error:
             msg_node = self.reporter.error(
                 "Front matter block:\n" + str(error), line=token.position[0]
@@ -911,8 +914,10 @@ def inline_text(self, text: str, lineno: int):
             current_node=paragraph,
             parse_context=get_parse_context(),
         )
+        # TODO propogate SourceLines metadata from parent document
+        lines = SourceLines(text, start_line=self._lineno, standardize_ends=True)
         doc_token = myst_block_tokens.Document.read(
-            text, start_line=self._lineno, front_matter=False, reset_definitions=False
+            lines, front_matter=False, reset_definitions=False
         )
         # we mark the token as nested so that footnotes etc aren't rendered
         doc_token.is_nested = True
diff --git a/myst_parser/sphinx_parser.py b/myst_parser/sphinx_parser.py
index 44d63337..b0bac94a 100644
--- a/myst_parser/sphinx_parser.py
+++ b/myst_parser/sphinx_parser.py
@@ -1,9 +1,13 @@
 from docutils import frontend, nodes
 from sphinx.parsers import Parser
+from sphinx.util import logging
 
+from mistletoe.base_elements import SourceLines
 from myst_parser.docutils_renderer import SphinxRenderer
 from myst_parser.block_tokens import Document
 
+SPHINX_LOGGER = logging.getLogger(__name__)
+
 
 class MystParser(Parser):
     """Docutils parser for CommonMark + Math + Tables + RST Extensions """
@@ -175,6 +179,12 @@ def parse(self, inputstring: str, document: nodes.document):
             pass
         renderer = SphinxRenderer(document=document)
         with renderer:
-            # TODO capture parsing errors and report via docutils/sphinx
-            doc = Document.read(inputstring)
+            # Log to sphinx (e.g. to warn of duplicate link/footnote definitions)
+            renderer.parse_context.logger = SPHINX_LOGGER
+            lines = SourceLines(
+                inputstring,
+                metadata={"docname": self.env.docname},
+                standardize_ends=True,
+            )
+            doc = Document.read(lines)
             renderer.render(doc)
diff --git a/setup.py b/setup.py
index b479ec88..9b7e89f7 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@
     ],
     keywords="markdown lexer parser development docutils sphinx",
     python_requires=">=3.6",
-    install_requires=["mistletoe-ebp~=0.10.0a2"],
+    install_requires=["mistletoe-ebp~=0.10.0a3"],
     extras_require={
         "sphinx": ["pyyaml", "docutils>=0.15", "sphinx>=2,<3"],
         "code_style": ["flake8<3.8.0,>=3.7.0", "black", "pre-commit==1.17.0"],
diff --git a/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml b/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml
index d8bcc8a7..13d886ae 100644
--- a/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml
@@ -6,7 +6,7 @@ front_matter:
 
     '
   position:
-  - 0
+  - 1
   - 3
   type: FrontMatter
 link_definitions: {}

From ea60b9a6b108429983dd45d33ef1d02215f59aaf Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Thu, 12 Mar 2020 01:22:11 +0000
Subject: [PATCH 17/18] bump version

---
 myst_parser/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/myst_parser/__init__.py b/myst_parser/__init__.py
index ce11c1e7..e7104cbc 100644
--- a/myst_parser/__init__.py
+++ b/myst_parser/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "0.6.0a1"
+__version__ = "0.6.0a2"
 
 
 def text_to_tokens(text: str):

From 9ef11340c5d41bcb6a92d1fe59454476c8577130 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Thu, 12 Mar 2020 04:48:26 +0000
Subject: [PATCH 18/18] bump mistletoe version (introduces Position class)

---
 myst_parser/docutils_renderer.py              | 61 +++++++-----
 myst_parser/sphinx_parser.py                  |  4 +-
 setup.py                                      |  2 +-
 tests/test_syntax/test_ast.py                 | 24 +----
 .../test_block_break_escaped_strings3_.yml    | 24 +++--
 ...est_block_break_follows_list_strings7_.yml | 18 ++--
 .../test_block_break_indent_4_strings2_.yml   | 12 ++-
 .../test_block_break_inline_strings4_.yml     | 12 ++-
 .../test_comment_escaped_strings3_.yml        | 24 +++--
 .../test_comment_follows_list_strings5_.yml   | 18 ++--
 .../test_comment_indent_4_strings2_.yml       | 12 ++-
 .../test_comment_inline_strings4_.yml         | 12 ++-
 .../test_front_matter_basic_strings0_.yml     |  6 +-
 ...t_link_references_ref_escape_strings3_.yml | 36 ++++---
 ...st_link_references_ref_first_strings0_.yml | 18 ++--
 ...est_link_references_ref_last_strings1_.yml | 18 ++--
 ...t_link_references_ref_syntax_strings2_.yml | 24 +++--
 .../test_ast/test_role_basic_strings0_.yml    | 18 ++--
 .../test_ast/test_role_escaped_strings3_.yml  | 36 ++++---
 .../test_role_external_code_strings11_.yml    | 18 ++--
 .../test_role_external_emphasis_strings7_.yml | 24 +++--
 .../test_role_external_math_strings9_.yml     | 12 ++-
 .../test_ast/test_role_indent_2_strings1_.yml | 18 ++--
 .../test_ast/test_role_indent_4_strings2_.yml | 12 ++-
 .../test_ast/test_role_inline_strings4_.yml   | 24 +++--
 .../test_role_internal_code_strings10_.yml    | 18 ++--
 .../test_role_internal_emphasis_strings6_.yml | 18 ++--
 .../test_role_internal_math_strings8_.yml     | 18 ++--
 .../test_ast/test_role_multiple_strings5_.yml | 36 ++++---
 tests/test_syntax/test_ast/test_table.yml     | 96 ++++++++++++-------
 .../test_ast/test_target_basic_strings0_.yml  | 18 ++--
 .../test_target_escaped_strings3_.yml         | 24 +++--
 ...est_target_external_emphasis_strings6_.yml | 24 +++--
 .../test_target_indent_2_strings1_.yml        | 18 ++--
 .../test_target_indent_4_strings2_.yml        | 12 ++-
 .../test_ast/test_target_inline_strings4_.yml | 24 +++--
 ...est_target_internal_emphasis_strings5_.yml | 18 ++--
 tests/test_syntax/test_ast/test_walk.yml      | 27 ++++++
 38 files changed, 549 insertions(+), 289 deletions(-)
 create mode 100644 tests/test_syntax/test_ast/test_walk.yml

diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
index 61bc000d..1fe3f45f 100644
--- a/myst_parser/docutils_renderer.py
+++ b/myst_parser/docutils_renderer.py
@@ -107,15 +107,20 @@ def new_document(self, source_path="notset") -> nodes.document:
     def add_line_and_source_path(self, node, token):
         """Copy the line number and document source path to the docutils node."""
         try:
-            node.line = token.position[0] + 1
+            node.line = token.position.line_start + 1
         except (AttributeError, TypeError):
             pass
         node.source = self.document["source"]
 
-    def nested_render_text(self, text: str, lineno: int):
+    def nested_render_text(self, text: str, lineno: int, token):
         """Render unparsed text."""
-        # TODO propogate SourceLines metadata from parent document
-        lines = SourceLines(text, start_line=lineno, standardize_ends=True)
+        lines = SourceLines(
+            text,
+            start_line=lineno,
+            uri=self.document["source"],
+            metadata=token.position.data,
+            standardize_ends=True,
+        )
         doc_token = myst_block_tokens.Document.read(
             lines, front_matter=True, reset_definitions=False
         )
@@ -190,7 +195,7 @@ def render_front_matter(self, token):
             data = token.get_data()
         except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error:
             msg_node = self.reporter.error(
-                "Front matter block:\n" + str(error), line=token.position[0]
+                "Front matter block:\n" + str(error), line=token.position.line_start
             )
             msg_node += nodes.literal_block(token.content, token.content)
             self.current_node += [msg_node]
@@ -538,7 +543,7 @@ def render_role(self, token):
         name = token.role_name
         # TODO role name white/black lists
         try:
-            lineno = token.position[0]
+            lineno = token.position.line_start
         except (AttributeError, TypeError):
             lineno = 0
         inliner = MockInliner(self, lineno)
@@ -565,7 +570,7 @@ def render_directive(self, token):
         name = token.language[1:-1]
         # TODO directive name white/black lists
         content = token.children[0].content
-        self.document.current_line = token.position[0]
+        self.document.current_line = token.position.line_start
 
         # get directive class
         directive_class, messages = directives.directive(
@@ -575,7 +580,7 @@ def render_directive(self, token):
             error = self.reporter.error(
                 "Unknown directive type '{}'\n".format(name),
                 # nodes.literal_block(content, content),
-                line=token.position[0],
+                line=token.position.line_start,
             )
             self.current_node += [error] + messages
             return
@@ -588,7 +593,7 @@ def render_directive(self, token):
             error = self.reporter.error(
                 "Directive '{}':\n{}".format(name, error),
                 nodes.literal_block(content, content),
-                line=token.position[0],
+                line=token.position.line_start,
             )
             self.current_node += [error]
             return
@@ -602,11 +607,13 @@ def render_directive(self, token):
                 arguments=arguments,
                 options=options,
                 body=body_lines,
-                lineno=token.position[0],
+                token=token,
             )
         else:
-            state_machine = MockStateMachine(self, token.position[0])
-            state = MockState(self, state_machine, token.position[0], token=token)
+            state_machine = MockStateMachine(self, token.position.line_start)
+            state = MockState(
+                self, state_machine, token.position.line_start, token=token
+            )
             directive_instance = directive_class(
                 name=name,
                 # the list of positional arguments
@@ -616,7 +623,7 @@ def render_directive(self, token):
                 # the directive content line by line
                 content=StringList(body_lines, self.document["source"]),
                 # the absolute line number of the first line of the directive
-                lineno=token.position[0],
+                lineno=token.position.line_start,
                 # the line offset of the first line of the content
                 content_offset=0,  # TODO get content offset from `parse_directive_text`
                 # a string containing the entire directive
@@ -630,7 +637,7 @@ def render_directive(self, token):
             result = directive_instance.run()
         except DirectiveError as error:
             msg_node = self.reporter.system_message(
-                error.level, error.msg, line=token.position[0]
+                error.level, error.msg, line=token.position.line_start
             )
             msg_node += nodes.literal_block(content, content)
             result = [msg_node]
@@ -640,7 +647,7 @@ def render_directive(self, token):
                     name, exc.__class__.__name__, exc
                 ),
                 nodes.literal_block(content, content),
-                line=token.position[0],
+                line=token.position.line_start,
             )
             self.current_node += [error]
             return
@@ -867,7 +874,7 @@ def __init__(
         renderer: DocutilsRenderer,
         state_machine: "MockStateMachine",
         lineno: int,
-        token=None,
+        token,
     ):
         self._renderer = renderer
         self._lineno = lineno
@@ -898,7 +905,9 @@ def nested_parse(
         current_match_titles = self.state_machine.match_titles
         self.state_machine.match_titles = match_titles
         with self._renderer.current_node_context(node):
-            self._renderer.nested_render_text(block, self._lineno + input_offset)
+            self._renderer.nested_render_text(
+                block, self._lineno + input_offset, token=self._token
+            )
         self.state_machine.match_titles = current_match_titles
 
     def inline_text(self, text: str, lineno: int):
@@ -914,8 +923,13 @@ def inline_text(self, text: str, lineno: int):
             current_node=paragraph,
             parse_context=get_parse_context(),
         )
-        # TODO propogate SourceLines metadata from parent document
-        lines = SourceLines(text, start_line=self._lineno, standardize_ends=True)
+        lines = SourceLines(
+            text,
+            start_line=self._lineno,
+            uri=self.document["source"],
+            metadata=self._token.position.data,
+            standardize_ends=True,
+        )
         doc_token = myst_block_tokens.Document.read(
             lines, front_matter=False, reset_definitions=False
         )
@@ -1062,7 +1076,7 @@ def __init__(
         arguments: list,
         options: dict,
         body: List[str],
-        lineno: int,
+        token,
     ):
         self.renderer = renderer
         self.document = renderer.document
@@ -1071,7 +1085,8 @@ def __init__(
         self.arguments = arguments
         self.options = options
         self.body = body
-        self.lineno = lineno
+        self.lineno = token.position.line_start
+        self.token = token
 
     def run(self):
 
@@ -1163,7 +1178,7 @@ def run(self):
         if "code" in self.options:
             self.options["source"] = str(path)
             state_machine = MockStateMachine(self.renderer, self.lineno)
-            state = MockState(self.renderer, state_machine, self.lineno)
+            state = MockState(self.renderer, state_machine, self.lineno, self.token)
             codeblock = CodeBlock(
                 name=self.name,
                 arguments=[self.options.pop("code")],
@@ -1186,7 +1201,7 @@ def run(self):
             self.renderer.document["source"] = str(path)
             self.renderer.reporter.source = str(path)
             self.renderer.reporter.get_source_and_line = lambda l: (str(path), l)
-            self.renderer.nested_render_text(file_content, startline)
+            self.renderer.nested_render_text(file_content, startline, token=self.token)
         finally:
             self.renderer.document["source"] = source
             self.renderer.reporter.source = rsource
diff --git a/myst_parser/sphinx_parser.py b/myst_parser/sphinx_parser.py
index b0bac94a..c34ff064 100644
--- a/myst_parser/sphinx_parser.py
+++ b/myst_parser/sphinx_parser.py
@@ -182,9 +182,7 @@ def parse(self, inputstring: str, document: nodes.document):
             # Log to sphinx (e.g. to warn of duplicate link/footnote definitions)
             renderer.parse_context.logger = SPHINX_LOGGER
             lines = SourceLines(
-                inputstring,
-                metadata={"docname": self.env.docname},
-                standardize_ends=True,
+                inputstring, uri=document["source"], standardize_ends=True
             )
             doc = Document.read(lines)
             renderer.render(doc)
diff --git a/setup.py b/setup.py
index 9b7e89f7..37a5def2 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@
     ],
     keywords="markdown lexer parser development docutils sphinx",
     python_requires=">=3.6",
-    install_requires=["mistletoe-ebp~=0.10.0a3"],
+    install_requires=["mistletoe-ebp~=0.10.0a4"],
     extras_require={
         "sphinx": ["pyyaml", "docutils>=0.15", "sphinx>=2,<3"],
         "code_style": ["flake8<3.8.0,>=3.7.0", "black", "pre-commit==1.17.0"],
diff --git a/tests/test_syntax/test_ast.py b/tests/test_syntax/test_ast.py
index 22aa9cb8..fdaac273 100644
--- a/tests/test_syntax/test_ast.py
+++ b/tests/test_syntax/test_ast.py
@@ -20,7 +20,7 @@ def test_render_tokens():
     assert root.children, root.children
 
 
-def test_walk(json_renderer):
+def test_walk(json_renderer, data_regression):
     doc = Document.read(
         dedent(
             """\
@@ -31,27 +31,7 @@ def test_walk(json_renderer):
         )
     )
     tree = [(repr(t.node), repr(t.parent), t.depth) for t in doc.walk()]
-    assert tree == [
-        (
-            "Paragraph(children=2, position=(1, 1))",
-            "Document(children=2, link_definitions=0, footnotes=0, "
-            "footref_order=0, front_matter=None)",
-            1,
-        ),
-        (
-            "Paragraph(children=2, position=(3, 3))",
-            "Document(children=2, link_definitions=0, footnotes=0, "
-            "footref_order=0, front_matter=None)",
-            1,
-        ),
-        ("RawText()", "Paragraph(children=2, position=(1, 1))", 2),
-        ("Strong(children=1)", "Paragraph(children=2, position=(1, 1))", 2),
-        ("RawText()", "Paragraph(children=2, position=(3, 3))", 2),
-        ("Link(target='link', title='')", "Paragraph(children=2, position=(3, 3))", 2),
-        ("RawText()", "Strong(children=1)", 3),
-        ("Emphasis(children=1)", "Link(target='link', title='')", 3),
-        ("RawText()", "Emphasis(children=1)", 4),
-    ]
+    data_regression.check(tree)
 
 
 @pytest.mark.parametrize(
diff --git a/tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml b/tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml
index d5d61fc4..e493e520 100644
--- a/tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_escaped_strings3_.yml
@@ -3,21 +3,29 @@ children:
   - children:
     - content: +
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: EscapeSequence
   - content: ++
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml b/tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml
index a590a643..b0956887 100644
--- a/tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_follows_list_strings7_.yml
@@ -4,19 +4,25 @@ children:
     - children:
       - content: item
         position:
-        - 1
-        - 1
+          data: {}
+          line_end: 1
+          line_start: 1
+          uri: null
         type: RawText
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: Paragraph
     leader: '-'
     loose: false
     next_marker: null
     position:
-    - 0
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 0
+      uri: null
     prepend: 2
     type: ListItem
   loose: false
diff --git a/tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml b/tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml
index 98523da9..0aa3fdf1 100644
--- a/tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_indent_4_strings2_.yml
@@ -4,13 +4,17 @@ children:
 
       '
     position:
-    - 0
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 0
+      uri: null
     type: RawText
   language: ''
   position:
-  - 0
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 0
+    uri: null
   type: BlockCode
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml b/tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml
index df82821f..29fa57e0 100644
--- a/tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml
+++ b/tests/test_syntax/test_ast/test_block_break_inline_strings4_.yml
@@ -2,12 +2,16 @@ children:
 - children:
   - content: a +++
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml b/tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml
index 6a7046ec..538f708e 100644
--- a/tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_comment_escaped_strings3_.yml
@@ -3,21 +3,29 @@ children:
   - children:
     - content: '%'
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: EscapeSequence
   - content: ' comment'
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_comment_follows_list_strings5_.yml b/tests/test_syntax/test_ast/test_comment_follows_list_strings5_.yml
index b4a71b04..f685d7a4 100644
--- a/tests/test_syntax/test_ast/test_comment_follows_list_strings5_.yml
+++ b/tests/test_syntax/test_ast/test_comment_follows_list_strings5_.yml
@@ -4,19 +4,25 @@ children:
     - children:
       - content: item
         position:
-        - 1
-        - 1
+          data: {}
+          line_end: 1
+          line_start: 1
+          uri: null
         type: RawText
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: Paragraph
     leader: '-'
     loose: false
     next_marker: null
     position:
-    - 0
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 0
+      uri: null
     prepend: 2
     type: ListItem
   loose: false
diff --git a/tests/test_syntax/test_ast/test_comment_indent_4_strings2_.yml b/tests/test_syntax/test_ast/test_comment_indent_4_strings2_.yml
index d9f99897..3475a006 100644
--- a/tests/test_syntax/test_ast/test_comment_indent_4_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_comment_indent_4_strings2_.yml
@@ -4,13 +4,17 @@ children:
 
       '
     position:
-    - 0
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 0
+      uri: null
     type: RawText
   language: ''
   position:
-  - 0
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 0
+    uri: null
   type: BlockCode
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_comment_inline_strings4_.yml b/tests/test_syntax/test_ast/test_comment_inline_strings4_.yml
index 900d8192..231113cc 100644
--- a/tests/test_syntax/test_ast/test_comment_inline_strings4_.yml
+++ b/tests/test_syntax/test_ast/test_comment_inline_strings4_.yml
@@ -2,12 +2,16 @@ children:
 - children:
   - content: a % comment
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml b/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml
index 13d886ae..0482f95e 100644
--- a/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_front_matter_basic_strings0_.yml
@@ -6,8 +6,10 @@ front_matter:
 
     '
   position:
-  - 1
-  - 3
+    data: {}
+    line_end: 3
+    line_start: 1
+    uri: null
   type: FrontMatter
 link_definitions: {}
 type: Document
diff --git a/tests/test_syntax/test_ast/test_link_references_ref_escape_strings3_.yml b/tests/test_syntax/test_ast/test_link_references_ref_escape_strings3_.yml
index 84d2901a..e9d4e56d 100644
--- a/tests/test_syntax/test_ast/test_link_references_ref_escape_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_link_references_ref_escape_strings3_.yml
@@ -2,32 +2,44 @@ children:
 - children:
   - content: '[ref]'
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 - children:
   - children:
     - content: '['
       position:
-      - 3
-      - 3
+        data: {}
+        line_end: 3
+        line_start: 3
+        uri: null
       type: RawText
     position:
-    - 3
-    - 3
+      data: {}
+      line_end: 3
+      line_start: 3
+      uri: null
     type: EscapeSequence
   - content: 'ref]: https://google.com "title"'
     position:
-    - 3
-    - 3
+      data: {}
+      line_end: 3
+      line_start: 3
+      uri: null
     type: RawText
   position:
-  - 3
-  - 3
+    data: {}
+    line_end: 3
+    line_start: 3
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_link_references_ref_first_strings0_.yml b/tests/test_syntax/test_ast/test_link_references_ref_first_strings0_.yml
index 5574254a..9aaa1f95 100644
--- a/tests/test_syntax/test_ast/test_link_references_ref_first_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_link_references_ref_first_strings0_.yml
@@ -3,18 +3,24 @@ children:
   - children:
     - content: ref
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     target: https://google.com
     title: title
     type: Link
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_link_references_ref_last_strings1_.yml b/tests/test_syntax/test_ast/test_link_references_ref_last_strings1_.yml
index 2d6c7a22..81da5144 100644
--- a/tests/test_syntax/test_ast/test_link_references_ref_last_strings1_.yml
+++ b/tests/test_syntax/test_ast/test_link_references_ref_last_strings1_.yml
@@ -3,18 +3,24 @@ children:
   - children:
     - content: ref
       position:
-      - 3
-      - 3
+        data: {}
+        line_end: 3
+        line_start: 3
+        uri: null
       type: RawText
     position:
-    - 3
-    - 3
+      data: {}
+      line_end: 3
+      line_start: 3
+      uri: null
     target: https://google.com
     title: title
     type: Link
   position:
-  - 3
-  - 3
+    data: {}
+    line_end: 3
+    line_start: 3
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_link_references_ref_syntax_strings2_.yml b/tests/test_syntax/test_ast/test_link_references_ref_syntax_strings2_.yml
index 149193b4..4e743581 100644
--- a/tests/test_syntax/test_ast/test_link_references_ref_syntax_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_link_references_ref_syntax_strings2_.yml
@@ -4,22 +4,30 @@ children:
     - children:
       - content: syntax
         position:
-        - 1
-        - 1
+          data: {}
+          line_end: 1
+          line_start: 1
+          uri: null
         type: RawText
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: Emphasis
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     target: https://google.com
     title: title
     type: Link
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_basic_strings0_.yml b/tests/test_syntax/test_ast/test_role_basic_strings0_.yml
index a0c64bb6..fd2922fd 100644
--- a/tests/test_syntax/test_ast/test_role_basic_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_role_basic_strings0_.yml
@@ -3,17 +3,23 @@ children:
   - children:
     - content: some content
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     role_name: name
     type: Role
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_escaped_strings3_.yml b/tests/test_syntax/test_ast/test_role_escaped_strings3_.yml
index bedd3106..b01dad0a 100644
--- a/tests/test_syntax/test_ast/test_role_escaped_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_role_escaped_strings3_.yml
@@ -3,31 +3,43 @@ children:
   - children:
     - content: '{'
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: EscapeSequence
   - content: name}
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   - children:
     - content: some content
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: InlineCode
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_external_code_strings11_.yml b/tests/test_syntax/test_ast/test_role_external_code_strings11_.yml
index a96ef5ec..cb97c850 100644
--- a/tests/test_syntax/test_ast/test_role_external_code_strings11_.yml
+++ b/tests/test_syntax/test_ast/test_role_external_code_strings11_.yml
@@ -3,16 +3,22 @@ children:
   - children:
     - content: '{name}`some content`'
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: InlineCode
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_external_emphasis_strings7_.yml b/tests/test_syntax/test_ast/test_role_external_emphasis_strings7_.yml
index 098aab85..9a8fa557 100644
--- a/tests/test_syntax/test_ast/test_role_external_emphasis_strings7_.yml
+++ b/tests/test_syntax/test_ast/test_role_external_emphasis_strings7_.yml
@@ -4,21 +4,29 @@ children:
     - children:
       - content: content
         position:
-        - 1
-        - 1
+          data: {}
+          line_end: 1
+          line_start: 1
+          uri: null
         type: RawText
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       role_name: name
       type: Role
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: Emphasis
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_external_math_strings9_.yml b/tests/test_syntax/test_ast/test_role_external_math_strings9_.yml
index 480c75a3..17f269be 100644
--- a/tests/test_syntax/test_ast/test_role_external_math_strings9_.yml
+++ b/tests/test_syntax/test_ast/test_role_external_math_strings9_.yml
@@ -2,12 +2,16 @@ children:
 - children:
   - content: ${name}`some content`$
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: Math
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_indent_2_strings1_.yml b/tests/test_syntax/test_ast/test_role_indent_2_strings1_.yml
index a0c64bb6..fd2922fd 100644
--- a/tests/test_syntax/test_ast/test_role_indent_2_strings1_.yml
+++ b/tests/test_syntax/test_ast/test_role_indent_2_strings1_.yml
@@ -3,17 +3,23 @@ children:
   - children:
     - content: some content
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     role_name: name
     type: Role
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_indent_4_strings2_.yml b/tests/test_syntax/test_ast/test_role_indent_4_strings2_.yml
index 01fea79d..d50312df 100644
--- a/tests/test_syntax/test_ast/test_role_indent_4_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_role_indent_4_strings2_.yml
@@ -4,13 +4,17 @@ children:
 
       '
     position:
-    - 0
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 0
+      uri: null
     type: RawText
   language: ''
   position:
-  - 0
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 0
+    uri: null
   type: BlockCode
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_inline_strings4_.yml b/tests/test_syntax/test_ast/test_role_inline_strings4_.yml
index 6b379e82..41dc8aaa 100644
--- a/tests/test_syntax/test_ast/test_role_inline_strings4_.yml
+++ b/tests/test_syntax/test_ast/test_role_inline_strings4_.yml
@@ -2,23 +2,31 @@ children:
 - children:
   - content: 'a '
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   - children:
     - content: some content
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     role_name: name
     type: Role
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_internal_code_strings10_.yml b/tests/test_syntax/test_ast/test_role_internal_code_strings10_.yml
index ae266cbe..b489e65e 100644
--- a/tests/test_syntax/test_ast/test_role_internal_code_strings10_.yml
+++ b/tests/test_syntax/test_ast/test_role_internal_code_strings10_.yml
@@ -3,17 +3,23 @@ children:
   - children:
     - content: '``some content``'
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     role_name: name
     type: Role
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_internal_emphasis_strings6_.yml b/tests/test_syntax/test_ast/test_role_internal_emphasis_strings6_.yml
index e08dc4a7..9c209b9c 100644
--- a/tests/test_syntax/test_ast/test_role_internal_emphasis_strings6_.yml
+++ b/tests/test_syntax/test_ast/test_role_internal_emphasis_strings6_.yml
@@ -3,17 +3,23 @@ children:
   - children:
     - content: '*content*'
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     role_name: name
     type: Role
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_internal_math_strings8_.yml b/tests/test_syntax/test_ast/test_role_internal_math_strings8_.yml
index 0f246ade..06337578 100644
--- a/tests/test_syntax/test_ast/test_role_internal_math_strings8_.yml
+++ b/tests/test_syntax/test_ast/test_role_internal_math_strings8_.yml
@@ -3,17 +3,23 @@ children:
   - children:
     - content: some $content$
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     role_name: name
     type: Role
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_role_multiple_strings5_.yml b/tests/test_syntax/test_ast/test_role_multiple_strings5_.yml
index 7d2de519..f612ce0a 100644
--- a/tests/test_syntax/test_ast/test_role_multiple_strings5_.yml
+++ b/tests/test_syntax/test_ast/test_role_multiple_strings5_.yml
@@ -3,33 +3,45 @@ children:
   - children:
     - content: some content
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     role_name: name
     type: Role
   - content: '  '
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   - children:
     - content: other
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     role_name: name2
     type: Role
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_table.yml b/tests/test_syntax/test_ast/test_table.yml
index eaeb2910..4a13e891 100644
--- a/tests/test_syntax/test_ast/test_table.yml
+++ b/tests/test_syntax/test_ast/test_table.yml
@@ -5,43 +5,59 @@ children:
       children:
       - content: hjk
         position:
-        - 3
-        - 3
+          data: {}
+          line_end: null
+          line_start: 3
+          uri: null
         type: RawText
       position:
-      - 3
-      - 3
+        data: {}
+        line_end: null
+        line_start: 3
+        uri: null
       type: TableCell
     - align: null
       children:
       - children:
         - content: y
           position:
-          - 3
-          - 3
+            data: {}
+            line_end: null
+            line_start: 3
+            uri: null
           type: RawText
         position:
-        - 3
-        - 3
+          data: {}
+          line_end: null
+          line_start: 3
+          uri: null
         type: Emphasis
       position:
-      - 3
-      - 3
+        data: {}
+        line_end: null
+        line_start: 3
+        uri: null
       type: TableCell
     - align: 0
       children:
       - content: z
         position:
-        - 3
-        - 3
+          data: {}
+          line_end: null
+          line_start: 3
+          uri: null
         type: RawText
       position:
-      - 3
-      - 3
+        data: {}
+        line_end: null
+        line_start: 3
+        uri: null
       type: TableCell
     position:
-    - 3
-    - 3
+      data: {}
+      line_end: null
+      line_start: 3
+      uri: null
     row_align:
     - null
     - null
@@ -57,46 +73,62 @@ children:
       children:
       - content: abc
         position:
-        - 1
-        - 1
+          data: {}
+          line_end: null
+          line_start: 1
+          uri: null
         type: RawText
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: null
+        line_start: 1
+        uri: null
       type: TableCell
     - align: null
       children:
       - content: d
         position:
-        - 1
-        - 1
+          data: {}
+          line_end: null
+          line_start: 1
+          uri: null
         type: RawText
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: null
+        line_start: 1
+        uri: null
       type: TableCell
     - align: 0
       children:
       - content: e
         position:
-        - 1
-        - 1
+          data: {}
+          line_end: null
+          line_start: 1
+          uri: null
         type: RawText
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: null
+        line_start: 1
+        uri: null
       type: TableCell
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: null
+      line_start: 1
+      uri: null
     row_align:
     - null
     - null
     - 0
     type: TableRow
   position:
-  - 1
-  - 3
+    data: {}
+    line_end: 3
+    line_start: 1
+    uri: null
   type: Table
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_target_basic_strings0_.yml b/tests/test_syntax/test_ast/test_target_basic_strings0_.yml
index 9a56ae97..153fd150 100644
--- a/tests/test_syntax/test_ast/test_target_basic_strings0_.yml
+++ b/tests/test_syntax/test_ast/test_target_basic_strings0_.yml
@@ -3,17 +3,23 @@ children:
   - children:
     - content: target
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     target: target
     type: Target
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_target_escaped_strings3_.yml b/tests/test_syntax/test_ast/test_target_escaped_strings3_.yml
index a2a6ba9f..8feecfd9 100644
--- a/tests/test_syntax/test_ast/test_target_escaped_strings3_.yml
+++ b/tests/test_syntax/test_ast/test_target_escaped_strings3_.yml
@@ -3,21 +3,29 @@ children:
   - children:
     - content: (
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: EscapeSequence
   - content: target)=
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_target_external_emphasis_strings6_.yml b/tests/test_syntax/test_ast/test_target_external_emphasis_strings6_.yml
index dc9e8216..2118b871 100644
--- a/tests/test_syntax/test_ast/test_target_external_emphasis_strings6_.yml
+++ b/tests/test_syntax/test_ast/test_target_external_emphasis_strings6_.yml
@@ -4,21 +4,29 @@ children:
     - children:
       - content: target
         position:
-        - 1
-        - 1
+          data: {}
+          line_end: 1
+          line_start: 1
+          uri: null
         type: RawText
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       target: target
       type: Target
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: Emphasis
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_target_indent_2_strings1_.yml b/tests/test_syntax/test_ast/test_target_indent_2_strings1_.yml
index 9a56ae97..153fd150 100644
--- a/tests/test_syntax/test_ast/test_target_indent_2_strings1_.yml
+++ b/tests/test_syntax/test_ast/test_target_indent_2_strings1_.yml
@@ -3,17 +3,23 @@ children:
   - children:
     - content: target
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     target: target
     type: Target
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_target_indent_4_strings2_.yml b/tests/test_syntax/test_ast/test_target_indent_4_strings2_.yml
index 380cb9a5..27e96e42 100644
--- a/tests/test_syntax/test_ast/test_target_indent_4_strings2_.yml
+++ b/tests/test_syntax/test_ast/test_target_indent_4_strings2_.yml
@@ -4,13 +4,17 @@ children:
 
       '
     position:
-    - 0
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 0
+      uri: null
     type: RawText
   language: ''
   position:
-  - 0
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 0
+    uri: null
   type: BlockCode
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_target_inline_strings4_.yml b/tests/test_syntax/test_ast/test_target_inline_strings4_.yml
index 6bf8f917..e53306b1 100644
--- a/tests/test_syntax/test_ast/test_target_inline_strings4_.yml
+++ b/tests/test_syntax/test_ast/test_target_inline_strings4_.yml
@@ -2,23 +2,31 @@ children:
 - children:
   - content: 'a '
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     type: RawText
   - children:
     - content: target
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     target: target
     type: Target
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_target_internal_emphasis_strings5_.yml b/tests/test_syntax/test_ast/test_target_internal_emphasis_strings5_.yml
index fdb332ce..3121a7c9 100644
--- a/tests/test_syntax/test_ast/test_target_internal_emphasis_strings5_.yml
+++ b/tests/test_syntax/test_ast/test_target_internal_emphasis_strings5_.yml
@@ -3,17 +3,23 @@ children:
   - children:
     - content: '*target*'
       position:
-      - 1
-      - 1
+        data: {}
+        line_end: 1
+        line_start: 1
+        uri: null
       type: RawText
     position:
-    - 1
-    - 1
+      data: {}
+      line_end: 1
+      line_start: 1
+      uri: null
     target: '*target*'
     type: Target
   position:
-  - 1
-  - 1
+    data: {}
+    line_end: 1
+    line_start: 1
+    uri: null
   type: Paragraph
 footnotes: {}
 footref_order: []
diff --git a/tests/test_syntax/test_ast/test_walk.yml b/tests/test_syntax/test_ast/test_walk.yml
new file mode 100644
index 00000000..8c3fd0e7
--- /dev/null
+++ b/tests/test_syntax/test_ast/test_walk.yml
@@ -0,0 +1,27 @@
+- - Paragraph(children=2, position=Position(lines=[1:1]))
+  - Document(children=2, link_definitions=0, footnotes=0, footref_order=0, front_matter=None)
+  - 1
+- - Paragraph(children=2, position=Position(lines=[3:3]))
+  - Document(children=2, link_definitions=0, footnotes=0, footref_order=0, front_matter=None)
+  - 1
+- - RawText()
+  - Paragraph(children=2, position=Position(lines=[1:1]))
+  - 2
+- - Strong(children=1)
+  - Paragraph(children=2, position=Position(lines=[1:1]))
+  - 2
+- - RawText()
+  - Paragraph(children=2, position=Position(lines=[3:3]))
+  - 2
+- - Link(target='link', title='')
+  - Paragraph(children=2, position=Position(lines=[3:3]))
+  - 2
+- - RawText()
+  - Strong(children=1)
+  - 3
+- - Emphasis(children=1)
+  - Link(target='link', title='')
+  - 3
+- - RawText()
+  - Emphasis(children=1)
+  - 4