Merge ab3feae into 786d99b

docs/.gitignore

@@ -1 +1,2 @@
-api/
+_build/
+_api/

docs/api/index.md

@@ -0,0 +1,11 @@
+(api/main)=
+
+# MyST-Parser API
+
+```{toctree}
+:maxdepth: 2
+
+tokens.rst
+renderers.rst
+sphinx_parser.rst
+```

docs/api/renderers.rst

@@ -0,0 +1,63 @@
+.. _api/renderers:
+
+MyST Renderers
+--------------
+
+MyST-Parser builds on the mistletoe
+:ref:`core renderers <mistletoe:renderers/core>`
+by including the extended tokens, listed in :ref:`api/tokens`,
+and adding bridges to docutils/sphinx:
+
+HTML
+....
+
+.. autoclass:: myst_parser.html_renderer.HTMLRenderer
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+JSON
+....
+
+.. autoclass:: myst_parser.json_renderer.JsonRenderer
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Docutils
+........
+
+.. autoclass:: myst_parser.docutils_renderer.DocutilsRenderer
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+.. autoclass:: myst_parser.docutils_renderer.MockInliner
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. autoclass:: myst_parser.docutils_renderer.MockState
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. autoclass:: myst_parser.docutils_renderer.MockStateMachine
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. autoclass:: myst_parser.docutils_renderer.MockIncludeDirective
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Sphinx
+......
+
+.. autoclass:: myst_parser.docutils_renderer.SphinxRenderer
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/api/sphinx_parser.rst

@@ -0,0 +1,13 @@
+.. _api/sphinx_parser:
+
+Sphinx Parser
+-------------
+
+This class builds on the :py:class:`~myst_parser.docutils_renderer.SphinxRenderer`
+to generate a parser for Sphinx, using the :ref:`Sphinx parser API <sphinx:parser-api>`:
+
+.. autoclass:: myst_parser.sphinx_parser.MystParser
+    :members:
+    :no-undoc-members:
+    :show-inheritance:
+    :exclude-members: __init__

docs/api/tokens.rst

@@ -0,0 +1,63 @@
+.. _api/tokens:
+
+Extended AST Tokens
+-------------------
+
+MyST builds on the mistletoe
+:ref:`core block tokens <mistletoe:tokens/block>` and
+:ref:`core span tokens <mistletoe:tokens/span>`
+to extend the syntax (as discussed in :ref:`example_syntax`).
+
+.. seealso::
+
+    :ref:`example_syntax`
+
+
+LineComment
+...........
+
+.. autoclass:: myst_parser.block_tokens.LineComment
+    :members:
+    :no-undoc-members:
+    :show-inheritance:
+    :exclude-members: __init__
+
+
+BlockBreak
+..........
+
+.. autoclass:: myst_parser.block_tokens.BlockBreak
+    :members:
+    :no-undoc-members:
+    :show-inheritance:
+    :exclude-members: __init__
+
+
+Role
+....
+
+.. autoclass:: myst_parser.span_tokens.Role
+    :members:
+    :no-undoc-members:
+    :show-inheritance:
+    :exclude-members: __init__
+
+
+Math
+....
+
+.. autoclass:: myst_parser.span_tokens.Math
+    :members:
+    :no-undoc-members:
+    :show-inheritance:
+    :exclude-members: __init__
+
+
+Target
+......
+
+.. autoclass:: myst_parser.span_tokens.Target
+    :members:
+    :no-undoc-members:
+    :show-inheritance:
+    :exclude-members: __init__

docs/conf.py

@@ -32,6 +32,7 @@
     "myst_parser",
     "sphinx.ext.autodoc",
     "sphinx.ext.intersphinx",
+    "sphinx.ext.viewcode",
     "sphinxcontrib.bibtex",
 ]
 
@@ -72,7 +73,7 @@ def run_apidoc(app):
     logger.info("running apidoc")
     # get correct paths
     this_folder = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
-    api_folder = os.path.join(this_folder, "api")
+    api_folder = os.path.join(this_folder, "_api")
     module_path = os.path.normpath(os.path.join(this_folder, "../"))
     ignore_paths = ["../setup.py", "../conftest.py", "../tests"]
     ignore_paths = [
@@ -92,7 +93,11 @@ def run_apidoc(app):
         os.remove(os.path.join(api_folder, "modules.rst"))
 
 
-intersphinx_mapping = {"python": ("https://docs.python.org/3.7", None)}
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3.7", None),
+    "mistletoe": ("https://mistletoe-ebp.readthedocs.io/en/latest", None),
+    "sphinx": ("https://www.sphinx-doc.org/en/master", None),
+}
 
 autodoc_default_options = {
     "show-inheritance": True,
@@ -105,27 +110,6 @@ def run_apidoc(app):
 autodoc_member_order = "bysource"
 
 nitpick_ignore = [
-    ("py:class", "mistletoe.base_elements.Token"),
-    ("py:class", "mistletoe.base_elements.BlockToken"),
-    ("py:class", "mistletoe.block_tokens.BlockCode"),
-    ("py:class", "mistletoe.block_tokens.Heading"),
-    ("py:class", "mistletoe.block_tokens.Quote"),
-    ("py:class", "mistletoe.block_tokens.CodeFence"),
-    ("py:class", "mistletoe.block_tokens.Document"),
-    ("py:class", "mistletoe.block_tokens.List"),
-    ("py:class", "mistletoe.block_tokens.ListItem"),
-    ("py:class", "mistletoe.block_tokens.Table"),
-    ("py:class", "mistletoe.block_tokens.TableRow"),
-    ("py:class", "mistletoe.block_tokens.Footnote"),
-    ("py:class", "mistletoe.block_tokens.Paragraph"),
-    ("py:class", "mistletoe.block_tokens.ThematicBreak"),
-    ("py:class", "mistletoe.block_tokens.HTMLBlock"),
-    ("py:class", "mistletoe.renderers.json.JsonRenderer"),
-    ("py:class", "mistletoe.renderers.base.BaseRenderer"),
-    ("py:class", "mistletoe.renderers.html.HTMLRenderer"),
-    ("py:class", "mistletoe.base_elements.SpanToken"),
-    ("py:class", "mistletoe.span_tokens.InlineCode"),
-    ("py:class", "docutils.parsers.Parser"),
     ("py:class", "Tuple"),
     ("py:class", "ForwardRef"),
     ("py:class", "NoneType"),
@@ -135,4 +119,4 @@ def run_apidoc(app):
 
 def setup(app):
     """Add functions to the Sphinx setup."""
-    app.connect("builder-inited", run_apidoc)
+    # app.connect("builder-inited", run_apidoc)

docs/index.md

@@ -5,7 +5,7 @@ MyST is a markdown flavor that implements the best parts of reStructuredText.
 It is a *slight* extension of CommonMark markdown.
 
 This project provides a parser for this flavor of markdown, as well as a bridge between
-MyST syntax and Sphinx. This allows for native markdown support for roles and
+MyST syntax and {doc}`Sphinx <sphinx:intro>`. This allows for native markdown support for roles and
 directives.
 
 ```{warning}
@@ -24,7 +24,7 @@ but there is no community standard around various syntactic choices for these fe
 Sphinx is a documentation generation framework written in Python. It heavily-utilizes
 reStructuredText syntax, which is another markup language for writing documents. In
 particular, Sphinx defines two extension points that are extremely useful:
-in-line **roles** and block-level **directives**.
+**{ref}`in-line roles<sphinx:rst-roles-alt>`** and **{ref}`block-level directives <sphinx:rst-directives>`**.
 
 **This project is an attempt at combining the simplicity and readability of Markdown
 with the power and flexibility of reStructuredText and the Sphinx platform.** It
@@ -54,5 +54,5 @@ caption: Contents
 using/index.md
 examples/index.md
 develop/index.md
-package_api.md
+api/index.md
 ```

docs/using/syntax.md

@@ -3,7 +3,7 @@
 # Example syntax for MyST
 
 As a base, MyST adheres to the [CommonMark specification](https://spec.commonmark.org/).
-For this, it uses the [Mistletoe project](https://github.com/miyuchina/mistletoe),
+For this, it uses the {ref}`mistletoe:intro/top-level` parser,
 which is a well-structured markdown parser for Python that is CommonMark-compliant
 and also extensible.
 
@@ -16,7 +16,11 @@ This project is an attempt to have the best of both worlds: the flexibility
 and extensibility of Sphinx with the simplicity and readability of Markdown.
 
 Below is a summary of the syntax 'tokens' parsed,
-and further details of a few major extensions from the CommonMark flavor of markdown
+and further details of a few major extensions from the CommonMark flavor of markdown.
+
+```{seealso}
+{ref}`MyST Extended AST Tokens API <api/tokens>`
+```
 
 ## Parsed Token Classes
 

docs/using/use_api.md

@@ -2,7 +2,8 @@
 
 % TODO eventually this should be wrote as a notebook (with MyST-NB)!
 
-MyST-Parser may be used as an API *via* the `myst_parser` package.
+MyST-Parser may be used as an API *via* the `myst_parser` package, see {ref}`api/main` for full details.
+
 The raw text is first parsed to syntax 'tokens',
 then these are converted to other formats using 'renderers'.
 

myst_parser/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.4.0"
+__version__ = "0.4.1"
 
 
 def text_to_tokens(text: str):

myst_parser/block_tokens.py

@@ -15,6 +15,7 @@
     BlockCode,
     CodeFence,
 )
+from mistletoe.attr_doc import autodoc
 
 """
 Tokens to be included in the parsing process, in the order specified.
@@ -36,6 +37,7 @@
 ]
 
 
+@autodoc
 @attr.s(slots=True, kw_only=True)
 class Document(block_tokens.Document):
     """Document token."""
@@ -75,6 +77,7 @@ def read(
         return doc
 
 
+@autodoc
 @attr.s(slots=True, kw_only=True)
 class LineComment(block_tokens.BlockToken):
     """Line comment start with % """
@@ -107,6 +110,7 @@ def read(cls, lines):
         )
 
 
+@autodoc
 @attr.s(slots=True, kw_only=True)
 class BlockBreak(block_tokens.BlockToken):
     """Block break token ``+++``.
@@ -150,6 +154,7 @@ def read(cls, lines):
         )
 
 
+@autodoc
 @attr.s(slots=True, kw_only=True)
 class Quote(block_tokens.Quote):
     """Quote token. (`["> # heading\\n", "> paragraph\\n"]`)."""
@@ -168,6 +173,7 @@ def transition(cls, next_line):
         )
 
 
+@autodoc
 @attr.s(slots=True, kw_only=True)
 class Paragraph(block_tokens.Paragraph):
     """Paragraph token. (`["some\\n", "continuous\\n", "lines\\n"]`)
@@ -188,6 +194,7 @@ def transition(cls, next_line):
         )
 
 
+@autodoc
 @attr.s(slots=True, kw_only=True)
 class List(block_tokens.List):
     """List token (unordered or ordered)"""
@@ -231,6 +238,7 @@ def read(cls, lines):
         )
 
 
+@autodoc
 @attr.s(slots=True, kw_only=True)
 class ListItem(block_tokens.ListItem):
     """List items.

myst_parser/sphinx_parser.py

@@ -1,10 +1,11 @@
-from docutils import frontend, parsers
+from docutils import frontend, nodes
+from sphinx.parsers import Parser
 
 from myst_parser.docutils_renderer import SphinxRenderer
 from myst_parser.block_tokens import Document
 
 
-class MystParser(parsers.Parser):
+class MystParser(Parser):
     """Docutils parser for CommonMark + Math + Tables + RST Extensions """
 
     supported = ("md", "markdown")
@@ -159,7 +160,12 @@ class MystParser(parsers.Parser):
     config_section = "myst parser"
     config_section_dependencies = ("parsers",)
 
-    def parse(self, inputstring, document):
+    def parse(self, inputstring: str, document: nodes.document):
+        """Parse source text.
+
+        :param inputstring: The source string to parse
+        :param document: The root docutils node to add AST elements to
+        """
         # TODO add conf.py configurable settings
         self.config = self.default_config.copy()
         try:

setup.py

@@ -36,7 +36,7 @@
     ],
     keywords="markdown lexer parser development docutils sphinx",
     python_requires=">=3.6",
-    install_requires=["mistletoe-ebp~=0.9.2"],
+    install_requires=["mistletoe-ebp~=0.9.3"],
     extras_require={
         "sphinx": ["pyyaml", "docutils>=0.15", "sphinx>=2,<3"],
         "code_style": ["flake8<3.8.0,>=3.7.0", "black", "pre-commit==1.17.0"],