From bcc50e39cd4e83432e576ed903c69702c56224d3 Mon Sep 17 00:00:00 2001
From: Chris Sewell <chrisj_sewell@hotmail.com>
Date: Sun, 8 Mar 2020 21:52:21 +0000
Subject: [PATCH] Update API documentation (#111)

---
 docs/api/directive.rst                        |  7 ++
 docs/api/index.md                             |  1 +
 docs/api/renderers.rst                        | 18 +++--
 docs/api/tokens.rst                           | 10 +--
 docs/conf.py                                  | 16 ++---
 myst_parser/__init__.py                       |  2 +-
 myst_parser/docutils_renderer.py              |  5 +-
 myst_parser/parse_directives.py               | 17 +++--
 setup.py                                      |  2 +-
 tests/test_syntax/test_ast.py                 | 29 ---------
 .../test_ast/test_math_basic_strings0_.yml    | 14 ----
 ..._math_contains_special_chars_strings1_.yml | 14 ----
 .../test_math_escaped_opening_strings4_.yml   | 29 ---------
 .../test_math_external_emphasis_strings7_.yml | 19 ------
 .../test_math_in_image_strings12_.yml         | 21 ------
 .../test_math_in_link_content_strings10_.yml  | 21 ------
 .../test_math_in_link_target_strings11_.yml   | 21 ------
 .../test_math_internal_emphasis_strings6_.yml | 14 ----
 .../test_ast/test_math_issue_51_strings9_.yml | 65 -------------------
 .../test_math_multiline_strings8_.yml         | 18 -----
 .../test_ast/test_math_multiple_strings3_.yml | 24 -------
 .../test_math_no_closing_strings5_.yml        | 14 ----
 ...math_preceding_special_chars_strings2_.yml | 19 ------
 23 files changed, 52 insertions(+), 348 deletions(-)
 create mode 100644 docs/api/directive.rst
 delete mode 100644 tests/test_syntax/test_ast/test_math_basic_strings0_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_contains_special_chars_strings1_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_escaped_opening_strings4_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_external_emphasis_strings7_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_in_image_strings12_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_in_link_content_strings10_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_in_link_target_strings11_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_internal_emphasis_strings6_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_issue_51_strings9_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_multiline_strings8_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_multiple_strings3_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_no_closing_strings5_.yml
 delete mode 100644 tests/test_syntax/test_ast/test_math_preceding_special_chars_strings2_.yml

diff --git a/docs/api/directive.rst b/docs/api/directive.rst
new file mode 100644
index 00000000..7f3b2d51
--- /dev/null
+++ b/docs/api/directive.rst
@@ -0,0 +1,7 @@
+.. _api/directive:
+
+Directive Parsing
+-----------------
+
+.. automodule:: myst_parser.parse_directives
+    :members:
diff --git a/docs/api/index.md b/docs/api/index.md
index 9fb64385..7505f1fe 100644
--- a/docs/api/index.md
+++ b/docs/api/index.md
@@ -6,6 +6,7 @@
 :maxdepth: 2
 
 tokens.rst
+directive.rst
 renderers.rst
 sphinx_parser.rst
 ```
diff --git a/docs/api/renderers.rst b/docs/api/renderers.rst
index a7d96ca4..7017f0e4 100644
--- a/docs/api/renderers.rst
+++ b/docs/api/renderers.rst
@@ -12,8 +12,10 @@ HTML
 ....
 
 .. autoclass:: myst_parser.html_renderer.HTMLRenderer
-    :members:
+    :special-members: __init__, __enter__, __exit__
+    :members: default_block_tokens, default_span_tokens
     :undoc-members:
+    :member-order: alphabetical
     :show-inheritance:
 
 
@@ -21,16 +23,20 @@ JSON
 ....
 
 .. autoclass:: myst_parser.json_renderer.JsonRenderer
-    :members:
+    :special-members: __init__, __enter__, __exit__
+    :members: default_block_tokens, default_span_tokens
     :undoc-members:
+    :member-order: alphabetical
     :show-inheritance:
 
 Docutils
 ........
 
 .. autoclass:: myst_parser.docutils_renderer.DocutilsRenderer
-    :members:
+    :special-members: __init__, __enter__, __exit__
+    :members: default_block_tokens, default_span_tokens, new_document
     :undoc-members:
+    :member-order: alphabetical
     :show-inheritance:
 
 
@@ -58,6 +64,10 @@ Sphinx
 ......
 
 .. autoclass:: myst_parser.docutils_renderer.SphinxRenderer
-    :members:
+    :special-members: __init__, __enter__, __exit__
+    :members: default_block_tokens, default_span_tokens, mock_sphinx_env
     :undoc-members:
+    :member-order: alphabetical
     :show-inheritance:
+
+.. autofunction:: myst_parser.docutils_renderer.dict_to_docinfo
diff --git a/docs/api/tokens.rst b/docs/api/tokens.rst
index 926bec26..dede5c83 100644
--- a/docs/api/tokens.rst
+++ b/docs/api/tokens.rst
@@ -3,10 +3,12 @@
 Extended AST Tokens
 -------------------
 
-MyST builds on the mistletoe
-:ref:`core block tokens <mistletoe:tokens/block>` and
-:ref:`core span tokens <mistletoe:tokens/span>`
-to extend the syntax (as discussed in :ref:`example_syntax`).
+MyST builds on the mistletoe tokens, to extend the syntax:
+
+- :ref:`Core block tokens <mistletoe:tokens/block>`
+- :ref:`Core span tokens <mistletoe:tokens/span>`
+- :ref:`Extension tokens <mistletoe:tokens/extension>`
+
 
 .. seealso::
 
diff --git a/docs/conf.py b/docs/conf.py
index 49f5563d..359543d7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -99,14 +99,14 @@ def run_apidoc(app):
     "sphinx": ("https://www.sphinx-doc.org/en/master", None),
 }
 
-autodoc_default_options = {
-    "show-inheritance": True,
-    "special-members": "__init__, __enter__, __exit__",
-    "members": True,
-    # 'exclude-members': '',
-    "undoc-members": True,
-    # 'inherited-members': True
-}
+# autodoc_default_options = {
+#     "show-inheritance": True,
+#     "special-members": "__init__, __enter__, __exit__",
+#     "members": True,
+#     # 'exclude-members': '',
+#     "undoc-members": True,
+#     # 'inherited-members': True
+# }
 autodoc_member_order = "bysource"
 
 nitpick_ignore = [
diff --git a/myst_parser/__init__.py b/myst_parser/__init__.py
index 311b240f..13a9f9f7 100644
--- a/myst_parser/__init__.py
+++ b/myst_parser/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "0.5.0a1"
+__version__ = "0.5.0"
 
 
 def text_to_tokens(text: str):
diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
index bf2ab97e..50e11e1f 100644
--- a/myst_parser/docutils_renderer.py
+++ b/myst_parser/docutils_renderer.py
@@ -91,6 +91,7 @@ def __init__(
         super().__init__(find_blocks=find_blocks, find_spans=find_spans)
 
     def new_document(self, source_path="notset") -> nodes.document:
+        """Create a new docutils document."""
         settings = OptionParser(components=(RSTParser,)).get_default_values()
         return new_document(source_path, settings=settings)
 
@@ -634,7 +635,9 @@ def handle_cross_reference(self, token, destination):
             self.render_children(token)
 
     def mock_sphinx_env(self, configuration=None, sourcedir=None):
-        """Load sphinx roles, directives, etc."""
+        """Create a minimimal Sphinx environment;
+        loading sphinx roles, directives, etc.
+        """
         from sphinx.application import builtin_extensions, Sphinx
         from sphinx.config import Config
         from sphinx.environment import BuildEnvironment
diff --git a/myst_parser/parse_directives.py b/myst_parser/parse_directives.py
index ebc5a0bb..1d8d107d 100644
--- a/myst_parser/parse_directives.py
+++ b/myst_parser/parse_directives.py
@@ -1,10 +1,9 @@
-"""Module to parse fenced code blocks as directives.
-
-Such a fenced code block starts with `{directive_name}`,
+"""Fenced code blocks are parsed as directives,
+if the block starts with ``{directive_name}``,
 followed by arguments on the same line.
 
 Directive options are read from a YAML block,
-if the first content line starts with `---`, e.g.
+if the first content line starts with ``---``, e.g.
 
 ::
 
@@ -17,8 +16,8 @@
     content...
     ```
 
-Or the option block will be parsed if the first content line starts with `:`,
-as a YAML block consisting of every line that starts with a `:`, e.g.
+Or the option block will be parsed if the first content line starts with ``:``,
+as a YAML block consisting of every line that starts with a ``:``, e.g.
 
 ::
 
@@ -45,13 +44,15 @@
 
 
 class DirectiveParsingError(Exception):
+    """Raise on parsing/validation error."""
+
     pass
 
 
 def parse_directive_text(
     directive_class: Type[Directive], argument_str: str, content: str
 ):
-    """See module docstring."""
+    """Parse (and validate) the full directive text."""
     if directive_class.option_spec:
         body, options = parse_directive_options(content, directive_class)
     else:
@@ -87,6 +88,7 @@ def parse_directive_text(
 
 
 def parse_directive_options(content: str, directive_class: Type[Directive]):
+    """Parse (and validate) the directive option section."""
     options = {}
     if content.startswith("---"):
         content = "\n".join(content.splitlines()[1:])
@@ -141,6 +143,7 @@ def parse_directive_options(content: str, directive_class: Type[Directive]):
 
 
 def parse_directive_arguments(directive, arg_text):
+    """Parse (and validate) the directive argument section."""
     required = directive.required_arguments
     optional = directive.optional_arguments
     arguments = arg_text.split()
diff --git a/setup.py b/setup.py
index c03e6bb9..32febce5 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@
     ],
     keywords="markdown lexer parser development docutils sphinx",
     python_requires=">=3.6",
-    install_requires=["mistletoe-ebp==0.9.4a2"],
+    install_requires=["mistletoe-ebp~=0.9.4"],
     extras_require={
         "sphinx": ["pyyaml", "docutils>=0.15", "sphinx>=2,<3"],
         "code_style": ["flake8<3.8.0,>=3.7.0", "black", "pre-commit==1.17.0"],
diff --git a/tests/test_syntax/test_ast.py b/tests/test_syntax/test_ast.py
index 3c5355a4..841c2bec 100644
--- a/tests/test_syntax/test_ast.py
+++ b/tests/test_syntax/test_ast.py
@@ -74,35 +74,6 @@ def test_role(name, json_renderer, data_regression, strings):
     data_regression.check(json_renderer.render(document, as_string=False))
 
 
-@pytest.mark.parametrize(
-    "name,strings",
-    [
-        ("basic", ["$a$"]),
-        ("contains_special_chars", ["$a`{_*-%$"]),
-        ("preceding_special_chars", ["{_*-%`$a$"]),
-        ("multiple", ["$a$ $b$"]),
-        ("escaped_opening", ["\\$a $b$"]),
-        ("no_closing", ["$a"]),
-        ("internal_emphasis", ["$*a*$"]),
-        ("external_emphasis", ["*$a$*"]),
-        ("multiline", ["$$a", "c", "b$$"]),
-        (
-            "issue_51",
-            [
-                "Math can be called in-line with single `$` characters around math.",
-                "For example, `$x_{hey}=it+is^{math}$` renders $x_{hey}=it+is^{math}$.",
-            ],
-        ),
-        ("in_link_content", ["[$a$](link)"]),
-        ("in_link_target", ["[a]($b$)"]),
-        ("in_image", ["![$a$]($b$)"]),
-    ],
-)
-def test_math(name, json_renderer, data_regression, strings):
-    document = Document.read(strings)
-    data_regression.check(json_renderer.render(document, as_string=False))
-
-
 @pytest.mark.parametrize(
     "name,strings",
     [
diff --git a/tests/test_syntax/test_ast/test_math_basic_strings0_.yml b/tests/test_syntax/test_ast/test_math_basic_strings0_.yml
deleted file mode 100644
index 8da4c68b..00000000
--- a/tests/test_syntax/test_ast/test_math_basic_strings0_.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-children:
-- children:
-  - content: $a$
-    position:
-    - 1
-    - 1
-    type: Math
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_contains_special_chars_strings1_.yml b/tests/test_syntax/test_ast/test_math_contains_special_chars_strings1_.yml
deleted file mode 100644
index b4202aa8..00000000
--- a/tests/test_syntax/test_ast/test_math_contains_special_chars_strings1_.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-children:
-- children:
-  - content: $a`{_*-%$
-    position:
-    - 1
-    - 1
-    type: Math
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_escaped_opening_strings4_.yml b/tests/test_syntax/test_ast/test_math_escaped_opening_strings4_.yml
deleted file mode 100644
index 8a2f0e09..00000000
--- a/tests/test_syntax/test_ast/test_math_escaped_opening_strings4_.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-children:
-- children:
-  - children:
-    - content: $
-      position:
-      - 1
-      - 1
-      type: RawText
-    position:
-    - 1
-    - 1
-    type: EscapeSequence
-  - content: 'a '
-    position:
-    - 1
-    - 1
-    type: RawText
-  - content: $b$
-    position:
-    - 1
-    - 1
-    type: Math
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_external_emphasis_strings7_.yml b/tests/test_syntax/test_ast/test_math_external_emphasis_strings7_.yml
deleted file mode 100644
index 252a4dd7..00000000
--- a/tests/test_syntax/test_ast/test_math_external_emphasis_strings7_.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-children:
-- children:
-  - children:
-    - content: $a$
-      position:
-      - 1
-      - 1
-      type: Math
-    position:
-    - 1
-    - 1
-    type: Emphasis
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_in_image_strings12_.yml b/tests/test_syntax/test_ast/test_math_in_image_strings12_.yml
deleted file mode 100644
index 8df89ed6..00000000
--- a/tests/test_syntax/test_ast/test_math_in_image_strings12_.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-children:
-- children:
-  - children:
-    - content: $a$
-      position:
-      - 1
-      - 1
-      type: Math
-    position:
-    - 1
-    - 1
-    src: $b$
-    title: ''
-    type: Image
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_in_link_content_strings10_.yml b/tests/test_syntax/test_ast/test_math_in_link_content_strings10_.yml
deleted file mode 100644
index 5f9b43bf..00000000
--- a/tests/test_syntax/test_ast/test_math_in_link_content_strings10_.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-children:
-- children:
-  - children:
-    - content: $a$
-      position:
-      - 1
-      - 1
-      type: Math
-    position:
-    - 1
-    - 1
-    target: link
-    title: ''
-    type: Link
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_in_link_target_strings11_.yml b/tests/test_syntax/test_ast/test_math_in_link_target_strings11_.yml
deleted file mode 100644
index 8ca57cef..00000000
--- a/tests/test_syntax/test_ast/test_math_in_link_target_strings11_.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-children:
-- children:
-  - children:
-    - content: a
-      position:
-      - 1
-      - 1
-      type: RawText
-    position:
-    - 1
-    - 1
-    target: $b$
-    title: ''
-    type: Link
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_internal_emphasis_strings6_.yml b/tests/test_syntax/test_ast/test_math_internal_emphasis_strings6_.yml
deleted file mode 100644
index 30616cba..00000000
--- a/tests/test_syntax/test_ast/test_math_internal_emphasis_strings6_.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-children:
-- children:
-  - content: $*a*$
-    position:
-    - 1
-    - 1
-    type: Math
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_issue_51_strings9_.yml b/tests/test_syntax/test_ast/test_math_issue_51_strings9_.yml
deleted file mode 100644
index d3731ddd..00000000
--- a/tests/test_syntax/test_ast/test_math_issue_51_strings9_.yml
+++ /dev/null
@@ -1,65 +0,0 @@
-children:
-- children:
-  - content: 'Math can be called in-line with single '
-    position:
-    - 1
-    - 2
-    type: RawText
-  - children:
-    - content: $
-      position:
-      - 1
-      - 2
-      type: RawText
-    position:
-    - 1
-    - 2
-    type: InlineCode
-  - content: ' characters around math.'
-    position:
-    - 1
-    - 2
-    type: RawText
-  - content: ''
-    position:
-    - 1
-    - 2
-    soft: true
-    type: LineBreak
-  - content: 'For example, '
-    position:
-    - 1
-    - 2
-    type: RawText
-  - children:
-    - content: $x_{hey}=it+is^{math}$
-      position:
-      - 1
-      - 2
-      type: RawText
-    position:
-    - 1
-    - 2
-    type: InlineCode
-  - content: ' renders '
-    position:
-    - 1
-    - 2
-    type: RawText
-  - content: $x_{hey}=it+is^{math}$
-    position:
-    - 1
-    - 2
-    type: Math
-  - content: .
-    position:
-    - 1
-    - 2
-    type: RawText
-  position:
-  - 1
-  - 2
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_multiline_strings8_.yml b/tests/test_syntax/test_ast/test_math_multiline_strings8_.yml
deleted file mode 100644
index a0cb94fc..00000000
--- a/tests/test_syntax/test_ast/test_math_multiline_strings8_.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-children:
-- children:
-  - content: '$$a
-
-      c
-
-      b$$'
-    position:
-    - 1
-    - 3
-    type: Math
-  position:
-  - 1
-  - 3
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_multiple_strings3_.yml b/tests/test_syntax/test_ast/test_math_multiple_strings3_.yml
deleted file mode 100644
index d07aed74..00000000
--- a/tests/test_syntax/test_ast/test_math_multiple_strings3_.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-children:
-- children:
-  - content: $a$
-    position:
-    - 1
-    - 1
-    type: Math
-  - content: ' '
-    position:
-    - 1
-    - 1
-    type: RawText
-  - content: $b$
-    position:
-    - 1
-    - 1
-    type: Math
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_no_closing_strings5_.yml b/tests/test_syntax/test_ast/test_math_no_closing_strings5_.yml
deleted file mode 100644
index 37d35401..00000000
--- a/tests/test_syntax/test_ast/test_math_no_closing_strings5_.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-children:
-- children:
-  - content: $a
-    position:
-    - 1
-    - 1
-    type: RawText
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document
diff --git a/tests/test_syntax/test_ast/test_math_preceding_special_chars_strings2_.yml b/tests/test_syntax/test_ast/test_math_preceding_special_chars_strings2_.yml
deleted file mode 100644
index 1a56247a..00000000
--- a/tests/test_syntax/test_ast/test_math_preceding_special_chars_strings2_.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-children:
-- children:
-  - content: '{_*-%`'
-    position:
-    - 1
-    - 1
-    type: RawText
-  - content: $a$
-    position:
-    - 1
-    - 1
-    type: Math
-  position:
-  - 1
-  - 1
-  type: Paragraph
-front_matter: null
-link_definitions: {}
-type: Document