Improvements

README.md

@@ -15,9 +15,45 @@ pip install git+https://github.com/ExecutableBookProject/mistletoe.git@myst
 
 To use the Myst parser in sphinx, simply add: `extensions = ["myst_parser"]` to your `conf.py`.
 
+## Parsed Token Classes
+
+For more information, also see the [CommonMark Spec](https://spec.commonmark.org/0.28/).
+
+### Block Tokens
+
+- **HTMLBlock**: Any valid HTML (rendered in HTML output only)
+- **LineComment**: `% this is a comment`
+- **BlockCode**: indented text (4 spaces)
+- **Heading**: `# Heading` (levels 1-6)
+- **SetextHeading**: underlined header
+- **Quote**: `> this is a quote`
+- **CodeFence**: enclosed in 3 or more backticks.
+  If it starts with a `{name}`, then treated as directive.
+- **ThematicBreak**: `---`
+- **List**: bullet points or enumerated.
+- **Table**: Standard markdown table styles.
+- **Footnote**: A substitution for an inline link (e.g. `[key][name]`), which can have a reference target (no spaces), and an optional title (in `"`), e.g. `[key]: https://www.google.com "a title"`
+- **Paragraph**: General inline text
+
+### Span (Inline) Tokens
+
+- **Role**: `` `{name}`interpreted text` ``
+- **Math**: `$a=1$` or `$$a=1$$`
+- **HTMLSpan**: any valid HTML (rendered in HTML output only)
+- **EscapeSequence**: `\*`
+- **AutoLink**: `<http://www.google.com>`
+- **Target**: `(target)=` (precedes element to target, e.g. header)
+- **InlineCode**: `` `a=1` ``
+- **LineBreak**: Soft or hard (ends with spaces or `\`)
+- **Image**: `![alt](src "title")`
+- **Link**: `[text](target "title")` or `[text][key]` (key from `Footnote`)
+- **Strong**: `**strong**`
+- **Emphasis**: `*emphasis*`
+- **RawText**
+
 ## Contributing
 
-To contribute, make Pull Requests to the `develop` branch (this is the default branch).
+### Code Style
 
 Code style is tested using [flake8](http://flake8.pycqa.org),
 with the configuration set in `.flake8`,
@@ -41,3 +77,18 @@ Optionally you can run `black` and `flake8` separately:
 ```
 
 Editors like VS Code also have automatic code reformat utilities, which can adhere to this standard.
+
+### Pull Requests
+
+To contribute, make Pull Requests to the `develop` branch (this is the default branch). A PR can consist of one or multiple commits. Before you open a PR, make sure to clean up your commit history and create the commits that you think best divide up the total work as outlined above (use `git rebase` and `git commit --amend`). Ensure all commit messages clearly summarise the changes in the header and the problem that this commit is solving in the body.
+
+Merging pull requests: There are three ways of 'merging' pull requests on GitHub:
+
+- Squash and merge: take all commits, squash them into a single one and put it on top of the base branch.
+    Choose this for pull requests that address a single issue and are well represented by a single commit.
+    Make sure to clean the commit message (title & body)
+- Rebase and merge: take all commits and 'recreate' them on top of the base branch. All commits will be recreated with new hashes.
+    Choose this for pull requests that require more than a single commit.
+    Examples: PRs that contain multiple commits with individually significant changes; PRs that have commits from different authors (squashing commits would remove attribution)
+- Merge with merge commit: put all commits as they are on the base branch, with a merge commit on top
+    Choose for collaborative PRs with many commits. Here, the merge commit provides actual benefits.

myst_parser/docutils_renderer.py

@@ -19,6 +19,7 @@
 
 from myst_parser import span_tokens as myst_span_tokens
 from myst_parser import block_tokens as myst_block_tokens
+from myst_parser.utils import escape_url
 
 
 class DocutilsRenderer(BaseRenderer):
@@ -280,15 +281,15 @@ def render_image(self, token):
         img_node = nodes.image()
         img_node["uri"] = token.src
 
-        # TODO how should image alt children be stored?
         img_node["alt"] = ""
-        # if token.children and isinstance(token.children[0], block_token.RawText):
-        #     img_node["alt"] = token.children[0].content
-        #     token.children[0].content = ""
+        if token.children and isinstance(token.children[0], myst_span_tokens.RawText):
+            img_node["alt"] = token.children[0].content
+            token.children[0].content = ""
 
         self.current_node.append(img_node)
-        with self.set_current_node(img_node):
-            self.render_children(token)
+        # TODO how should non-raw alternative text be handled?
+        # with self.set_current_node(img_node):
+        #     self.render_children(token)
 
     def render_list(self, token):
         list_node = None
@@ -312,7 +313,7 @@ def render_list(self, token):
         with self.set_current_node(list_node):
             self.render_children(token)
 
-    def render_list_item(self, token):
+    def render_list_item(self, token: myst_block_tokens.ListItem):
         item_node = nodes.list_item()
         # TODO list item range
         # node.line = token.range[0]
@@ -358,16 +359,18 @@ def render_table_cell(self, token):
             self.render_children(token)
 
     def render_auto_link(self, token):
-        # TODO render_auto_link
-        raise NotImplementedError
+        if token.mailto:
+            refuri = "mailto:{}".format(token.target)
+        else:
+            refuri = escape_url(token.target)
+        ref_node = nodes.reference(token.target, token.target, refuri=refuri)
+        self.current_node.append(ref_node)
 
     def render_html_span(self, token):
-        # TODO render_html_span
-        raise NotImplementedError
+        self.current_node.append(nodes.raw("", token.content, format="html"))
 
     def render_html_block(self, token):
-        # TODO render_html_block
-        raise NotImplementedError
+        self.current_node.append(nodes.raw("", token.content, format="html"))
 
     def render_role(self, token):
         content = token.children[0].content

myst_parser/utils.py

@@ -0,0 +1,9 @@
+import html
+from urllib.parse import quote
+
+
+def escape_url(raw):
+    """
+    Escape urls to prevent code injection craziness. (Hopefully.)
+    """
+    return html.escape(quote(html.unescape(raw), safe="/#:()*?=%@+,&"))

tests/sphinx/sourcedirs/.gitignore

@@ -0,0 +1 @@
+_build/

tests/sphinx/sourcedirs/basic/content.md

@@ -5,21 +5,27 @@
 % comment
 
 ````{note}
-abcd *abc* [google](http://www.google.com)
+abcd *abc* [google](https://www.google.com)
 
   ```{warning}
   xyz
   ```
 
 ````
 
+(target2)=
+
 ```{figure} example.jpg
 ---
 height: 40px
 ---
 Caption
 ```
 
+![*alternative text*](example.jpg)
+
+<https://www.google.com>
+
 **{code}`` a=1{`} ``**
 
 {math}`sdfds`
@@ -47,4 +53,8 @@ this is a second paragraph
 % a comment 3
 - new list?
 
-{ref}`target`
+{ref}`target`  {ref}`target2`
+
+[name][key]
+
+[key]: https://www.google.com "a title"

tests/sphinx/test_sphinx_builds.py

@@ -83,7 +83,9 @@ def read(
     return read
 
 
-@pytest.mark.sphinx(buildername="html", srcdir=os.path.join(SOURCE_DIR, "basic"))
+@pytest.mark.sphinx(
+    buildername="html", srcdir=os.path.join(SOURCE_DIR, "basic"), freshenv=True
+)
 def test_basic(app, status, warning, get_sphinx_app_output, remove_sphinx_builds):
     """basic test."""
     app.build()

tests/sphinx/test_sphinx_builds/test_basic.html

@@ -19,7 +19,7 @@ <h1>
       <em>
        abc
       </em>
-      <a class="reference external" href="http://www.google.com">
+      <a class="reference external" href="https://www.google.com">
        google
       </a>
      </p>
@@ -33,6 +33,8 @@ <h1>
      </div>
     </div>
     <div class="figure align-default" id="id1">
+     <span id="target2">
+     </span>
      <a class="reference internal image-reference" href="_images/example.jpg">
       <img alt="_images/example.jpg" src="_images/example.jpg" style="height: 40px;"/>
      </a>
@@ -45,6 +47,14 @@ <h1>
       </a>
      </p>
     </div>
+    <p>
+     <img alt="" src="_images/example.jpg"/>
+    </p>
+    <p>
+     <a class="reference external" href="https://www.google.com">
+      https://www.google.com
+     </a>
+    </p>
     <p>
      <strong>
       <code class="code docutils literal notranslate">
@@ -136,6 +146,16 @@ <h1>
        Header
       </span>
      </a>
+     <a class="reference internal" href="#target2">
+      <span class="std std-ref">
+       Caption
+      </span>
+     </a>
+    </p>
+    <p>
+     <a class="reference external" href="https://www.google.com">
+      name
+     </a>
     </p>
    </div>
   </div>

tests/test_docutils_renderer.py

@@ -145,6 +145,17 @@ def test_image(renderer_mock):
     )
 
 
+def test_image_with_alt(renderer_mock):
+    renderer_mock.render(tokenize([r"![alt](path/to/image.jpeg)"])[0])
+    assert renderer_mock.document.pformat() == dedent(
+        """\
+    <document source="">
+        <paragraph>
+            <image alt="alt" uri="path/to/image.jpeg">
+    """
+    )
+
+
 def test_quote(renderer_mock):
     render_token(renderer_mock, "Quote", range=(0, 0))
     assert renderer_mock.document.pformat() == dedent(
@@ -251,9 +262,24 @@ def test_comment(renderer_mock):
     )
 
 
-def test_full_run(renderer, file_regression):
+def test_footnote(renderer):
+    renderer.render(
+        Document(["[name][key]", "", '[key]: https://www.google.com "a title"', ""])
+    )
+    assert renderer.document.pformat() == dedent(
+        """\
+    <document source="">
+        <paragraph>
+            <reference refuri="https://www.google.com" title="a title">
+                name
+    """
+    )
+
+
+def test_full_run(sphinx_renderer, file_regression):
     string = dedent(
         """\
+        (target)=
         # header 1
         ## sub header 1
 
@@ -281,11 +307,13 @@ def test_full_run(renderer, file_regression):
         2. b
             1. c
 
+        {ref}`target`
+
         """
     )
 
-    renderer.render(Document(string))
-    file_regression.check(renderer.document.pformat(), extension=".xml")
+    sphinx_renderer.render(Document(string))
+    file_regression.check(sphinx_renderer.document.pformat(), extension=".xml")
 
 
 with open(os.path.join(os.path.dirname(__file__), "sphinx_roles.json"), "r") as fin:

tests/test_docutils_renderer/test_full_run.xml

@@ -1,4 +1,5 @@
 <document source="">
+    <target ids="target" names="target">
     <section ids="header-1" names="header\ 1">
         <title>
             header 1
@@ -56,3 +57,7 @@
                     <list_item>
                         <paragraph>
                             c
+        <paragraph>
+            <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="False" reftarget="target" reftype="ref" refwarn="True">
+                <inline classes="xref std std-ref">
+                    target