Switch to MyST-Parser from recommonmark

.github/workflows/lint.yml

@@ -23,8 +23,6 @@ jobs:
         . $GITHUB_WORKSPACE/../venv/bin/activate
         curl -s -S --retry 3 $BASEDIR/tests/install.sh | bash -
         pip install .[markdown]
-        pip install -e git+https://github.com/readthedocs/commonmark.py.git@dafae75015cc342f3fddb499674bab97ac4a6a96#egg=commonmark
-        pip install -e git+https://github.com/jpmckinney/recommonmark.git@hotfix#egg=recommonmark
     - name: Lint
       run: |
         . $GITHUB_WORKSPACE/../venv/bin/activate

docs/changelog.rst

@@ -1,6 +1,11 @@
 Changelog
 =========
 
+0.3.0 (2021-02-12)
+~~~~~~~~~~~~~~~~~~
+
+-  Switch to MyST-Parser from recommonmark.
+
 0.2.2 (2020-04-06)
 ~~~~~~~~~~~~~~~~~~
 

ocds_babel/markdown_translator.py

@@ -4,23 +4,23 @@
     'visit_document',
     'depart_colspec',
     'depart_list_item',
-    'depart_section',
     # Handled instead via `colspec` nodes.
     'depart_tgroup',
     'visit_tgroup',
     # Inline node
     'depart_emphasis',
     'depart_image',
+    'depart_inline',
     'depart_literal',
     'depart_pending_xref',
     'depart_raw',
     'depart_reference',
     'depart_strong',
     'visit_emphasis',
     'visit_image',
+    'visit_inline',
     'visit_literal',
     'visit_pending_xref',
-    'visit_raw',
     'visit_reference',
     'visit_strong',
     # Text node
@@ -36,6 +36,7 @@ def __init__(self, document, translator):
         # Whether we are writing to the output.
         self.writing = True
 
+        self.heading_level = 0
         # The writing context: 'block-quote', 'td' or 'th'.
         self.contexts = [None]
         # List item markers: '*' or '1.'.
@@ -116,7 +117,7 @@ def depart_paragraph(self, node):
                 self.append('\n')
 
     def visit_literal_block(self, node):
-        self.append('```{}\n'.format(node.attributes.get('language', '')))
+        self.append('```{}\n'.format(node.get('language', '')))
         # Markdown code blocks (indented paragraphs) have no `rawsource`, but fenced code blocks do.
         if node.rawsource:
             text = node.rawsource
@@ -130,14 +131,22 @@ def depart_literal_block(self, node):
         self.append('```\n\n')
 
     def visit_section(self, node):
-        self.append('#' * node.attributes['level'] + ' ')
+        self.heading_level += 1
+        self.append('#' * self.heading_level + ' ')
+
+    def depart_section(self, node):
+        self.heading_level -= 1
 
     def visit_title(self, node):
         self.append(self.gettext(node.rawsource))
 
     def depart_title(self, node):
         self.append('\n\n')
 
+    def visit_raw(self, node):
+        if isinstance(node.parent, nodes.section):
+            self.append(self.gettext(node.astext()) + '\n\n')
+
     # Lists
 
     def visit_bullet_list(self, node):

ocds_babel/translate.py

@@ -38,25 +38,11 @@ def setup(app):
 Install requirements for Markdown translation
 ---------------------------------------------
 
-To translate Markdown files, you must install Sphinx>=1.5 and (for now) forks of `CommonMark <https://commonmarkpy.readthedocs.io/en/latest/>`__ and `recommonmark <https://recommonmark.readthedocs.io/en/latest/>`__, until these bugs are fixed: `#225 <https://github.com/readthedocs/commonmark.py/pull/225>`__, `#181 <https://github.com/readthedocs/recommonmark/pull/181>`__, `#186 <https://github.com/readthedocs/recommonmark/pull/186>`__, `#187 <https://github.com/readthedocs/recommonmark/pull/187>`__.
-
-Install a recent version of Sphinx with:
+To translate Markdown files, you must install:
 
 .. code-block:: bash
 
     pip install ocds-babel[markdown]
-
-or install a specific version like:
-
-.. code-block:: bash
-
-    pip install 'Sphinx==2.2.1'
-
-Then, install the forks of CommonMark and recommonmark:
-
-.. code-block:: bash
-
-    pip install -e git+https://github.com/readthedocs/commonmark.py.git@dafae75015cc342f3fddb499674bab97ac4a6a96#egg=commonmark -e git+https://github.com/jpmckinney/recommonmark.git@hotfix#egg=recommonmark
 """  # noqa: E501
 
 import csv

ocds_babel/translate_markdown.py

@@ -1,27 +1,11 @@
 import logging
-from contextlib import contextmanager
 
-import sphinx
-from docutils.frontend import OptionParser
-from docutils.io import InputError
-from docutils.parsers.rst import Parser, directives
-from docutils.utils import SystemMessage, new_document
-from recommonmark.parser import CommonMarkParser
-from recommonmark.transform import AutoStructify
-from sphinx.application import Sphinx
-from sphinx.util.docutils import docutils_namespace
+from docutils.parsers.rst import directives
+from myst_parser.main import to_docutils
 
 from ocds_babel.directives import NullDirective
 from ocds_babel.markdown_translator import MarkdownTranslator
 
-# patch_docutils is added in Sphinx 1.6.
-if sphinx.version_info >= (1, 6):
-    from sphinx.util.docutils import patch_docutils
-else:
-    @contextmanager
-    def patch_docutils(confdir=None):
-        yield
-
 logger = logging.getLogger('ocds_babel')
 
 
@@ -43,31 +27,9 @@ def translate_markdown_data(name, text, translator, **kwargs):
     for directive_name in ('csv-table-no-translate', 'extensiontable'):
         directives.register_directive(directive_name, NullDirective)
 
-    with patch_docutils(), docutils_namespace():
-        # sphinx-build -b html -q -E …
-        # See build_main() in sphinx/cmd/build.py
-        app = Sphinx('.', None, 'outdir', '.', 'html', status=None, freshenv=True)
-        app.add_config_value('recommonmark_config', {
-            'enable_auto_toc_tree': False,
-        }, True)
-
-        # From code comment in `new_document`.
-        settings = OptionParser(components=(Parser,)).get_default_values()
-        # Get minimal settings for `AutoStructify` to be applied.
-        settings.env = app.builder.env
-
-        document = new_document(name, settings)
-        CommonMarkParser().parse(text, document)
-
-        # To translate messages in `.. list-table`.
-        try:
-            AutoStructify(document).apply()
-        except SystemMessage as e:
-            context = e.__context__
-            if isinstance(context, InputError) and context.strerror == 'No such file or directory':  # csv-table
-                logger.warning(e)
+    document = to_docutils(text, in_sphinx_env=True)
 
-        visitor = MarkdownTranslator(document, translator)
-        document.walkabout(visitor)
+    visitor = MarkdownTranslator(document, translator)
+    document.walkabout(visitor)
 
-        return visitor.astext()
+    return visitor.astext()

setup.py

@@ -16,8 +16,7 @@
     extras_require={
         'markdown': [
             'docutils',
-            # See https://ocds-babel.readthedocs.io/en/latest/api/translate.html
-            # 'recommonmark',
+            'MyST-Parser',
             'Sphinx>=1.5',
         ],
         'test': [

tests/test_translate.py

@@ -107,7 +107,7 @@
 1. Enumerated list item 1
 2. Enumerated list item 2
 
-```eval_rst
+```{eval-rst}
 .. list-table::
     :header-rows: 1
 
@@ -315,6 +315,7 @@ def gettext(self, *args, **kwargs):
                 '[Link list item 2](http://example.com/en/2.html)': '[Élément de liste de liens 2](http://example.com/fr/2.html)',  # noqa: E501
                 # docutils ... optparse
                 '%prog [options]': '%prog [options]',
+                '': '',
             }[args[0]]
 
     monkeypatch.setattr(gettext, 'translation', Translation)
@@ -333,7 +334,7 @@ def gettext(self, *args, **kwargs):
             with open(os.path.join(builddir, 'README.md')) as f:
                 text = f.read()
 
-    assert text == """##### Entête à sauter
+    assert text == """# Entête à sauter
 
 # Titre 1
 
@@ -347,11 +348,11 @@ def gettext(self, *args, **kwargs):
 
 > Texte de citation
 
-```
+```none
 Raw paragraph text
 ```
 
-```
+```default
 Literal block
 ```
 

tox.ini

@@ -5,14 +5,13 @@ envlist = clean,py-sphinx{15,16,17,18,20,21,22,23,24,30,31,32,33},report
 [testenv]
 # https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#github-actions
 passenv = COVERALLS_REPO_TOKEN GITHUB_*
-extras = test
+extras =
+    markdown
+    test
 commands =
     coverage run --append --source=ocds_babel -m pytest
 deps =
     coverage
-    # See https://ocds-babel.readthedocs.io/en/latest/api/translate.html#install-requirements-for-markdown-translation
-    -e git+https://github.com/readthedocs/commonmark.py.git@dafae75015cc342f3fddb499674bab97ac4a6a96#egg=commonmark
-    -e git+https://github.com/jpmckinney/recommonmark.git@hotfix#egg=recommonmark
     sphinx15: Sphinx>=1.5,<1.6
     sphinx16: Sphinx>=1.6,<1.7
     sphinx17: Sphinx>=1.7,<1.8