Update to nbconvert 7 & mistune 3

#36

.github/workflows/main.yaml

@@ -13,12 +13,12 @@ jobs:
     - uses: actions/checkout@v2
     - name: Run Tox tests
       id: test
-      uses: fedora-python/tox-github-action@master
+      uses: fedora-python/tox-github-action@main
       with:
         tox_env: ${{ matrix.tox_env }}
     strategy:
       matrix:
-        tox_env: [py37, py38, py39, py310, py311]
+        tox_env: [py38, py39, py310, py311, py312]
 
     # Use GitHub's Linux Docker host
     runs-on: ubuntu-latest

README.md

@@ -89,6 +89,14 @@ licensed under the same license.
 
 ## Changelog
 
+### naucse_render 2.0
+
+* Update to mistune 3.x & nbconvert 7.x. This changes parsing & formatting
+  for Markdown, syntax highlighting, and Notebooks. In most cases the
+  differences should be superficial.
+
+* Tested with Python 3.8-3.12
+
 ### naucse_render 1.10
 
 * `naucse_render compile` now checks for links to missing lessons and

naucse_render/markdown.py

@@ -12,57 +12,62 @@
 import pygments.formatters.html
 
 
-ansi_convertor = Ansi2HTMLConverter(inline=True)
-
-pygments_formatter = pygments.formatters.html.HtmlFormatter(
-    cssclass='highlight'
-)
-
-_admonition_leading_pattern = re.compile(r'^ *> ?', flags=re.M)
-
-
-class BlockGrammar(mistune.BlockGrammar):
-    admonition = re.compile(r'^> *\[(\S+)\]([^\n]*)\n((>[^\n]*[\n]{0,1})*)')
-    deflist = re.compile(r'^(([^\n: ][^\n]*\n)+)((:( {0,3})[^\n]*\n)( \5[^\n]*\n|\n)+)')
-
-
-class BlockLexer(mistune.BlockLexer):
-    grammar_class = BlockGrammar
+def naucse_admonition_plugin(md):
+    """Parse blockquote-based admonitions
 
-    default_rules = [
-        'admonition',
-        'deflist',
-    ] + mistune.BlockLexer.default_rules
+    Like this:
 
-    def parse_admonition(self, m):
-        self.tokens.append({
-            'type': 'admonition_start',
-            'name': m.group(1),
-            'title': m.group(2).strip(),
-        })
+    > [note] Note Title
+    > rest of note goes here
+    """
+    # Based on Mistune's "spoiler" plugin (the documentation says:
+    # "take a look at the source code in mistune/plugins to find
+    # out how to write a plugin")
+
+    ADMONITION_NAME_PATTERN = re.compile(r' *\[(\S+)\]([^\n]*)\n')
+
+    def parse_naucse_admonition(block, m, state):
+
+        text, end_pos = block.extract_block_quote(m, state)
+        name_match = ADMONITION_NAME_PATTERN.match(text)
+        if name_match:
+            # It's an admonition
+            token = {
+                'type': 'naucse_admonition',
+                'attrs': {
+                    'name': name_match[1].strip(),
+                    'title': name_match[2].strip(),
+                },
+            }
+            text = text[name_match.end():]
+        else:
+            token = {
+                'type': 'block_quote',
+            }
+
+        child = state.child_state(text)
+        rules = block.block_quote_rules
+        block.parse(child, rules)
+        token['children'] = child.tokens
+        if end_pos:
+            state.prepend_token(token)
+            return end_pos
+        state.append_token(token)
+        return state.cursor
+
+    md.block.register(
+        'block_quote',
+        None,
+        parse_naucse_admonition,
+        before='block_quote',
+    )
 
-        text = _admonition_leading_pattern.sub('', m.group(3))
 
-        self.parse(dedent(text))
-        self.tokens.append({
-            'type': 'admonition_end',
-        })
+ansi_convertor = Ansi2HTMLConverter(inline=True)
 
-    def parse_deflist(self, m):
-        self.tokens.append({
-            'type': 'deflist_term_start',
-        })
-        self.parse(dedent(m.group(1)))
-        self.tokens.append({
-            'type': 'deflist_term_end',
-        })
-        self.tokens.append({
-            'type': 'deflist_def_start',
-        })
-        self.parse(dedent(' ' + m.group(3)[1:]))
-        self.tokens.append({
-            'type': 'deflist_def_end',
-        })
+pygments_formatter = pygments.formatters.html.HtmlFormatter(
+    cssclass='highlight'
+)
 
 
 def ansi_convert(code):
@@ -76,11 +81,6 @@ def style_space_after_prompt(html):
                   html)
 
 
-def matrix_multiplication_operator(html):
-    return html.replace('<span class="err">@</span>',
-                        '<span class="o">@</span>')
-
-
 class MSDOSSessionVenvLexer(RegexLexer):
     """Lexer for simplistic MSDOS sessions with optional venvs.
 
@@ -129,23 +129,26 @@ def text_to_id(text):
     return text
 
 
-class Renderer(mistune.Renderer):
+class NaucseRenderer(mistune.HTMLRenderer):
     code_tmpl = '<div class="highlight"><pre><code>{}</code></pre></div>'
 
-    def __init__(self, convert_url, *args, **kwargs):
+    def __init__(self, convert_url, *args, escape=False, **kwargs):
         self._convert_url = convert_url
-        super().__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs, escape=False)
 
-    def admonition(self, name, content):
-        return '<div class="admonition {}">{}</div>'.format(name, content)
+    def naucse_admonition(self, text, title, name):
+        if title:
+            text = f'<p class="admonition-title">{title}</p>\n{text}'
+        return '<div class="admonition {}">{}</div>'.format(name, text)
 
-    def header(self, text, level, raw=None):
+    def heading(self, text, level, raw=None):
         header_id = text_to_id(text)
         return f'''<h{level:d} id="{header_id}">{text}
 <a href="#{header_id}" class="header-link">#</a>
 </h{level:d}>\n'''
 
-    def block_code(self, code, lang):
+    def block_code(self, code, info=None):
+        lang = info
         if lang is not None:
             lang = lang.strip()
         if not lang or lang == 'plain':
@@ -157,67 +160,23 @@ def block_code(self, code, lang):
         lexer = get_lexer_by_name(lang)
         html = pygments.highlight(code, lexer, pygments_formatter).strip()
         html = style_space_after_prompt(html)
-        if lang in ('python', 'pycon'):
-            html = matrix_multiplication_operator(html)
         return html
 
-    def deflist(self, items):
-        tags = {'term': 'dt', 'def': 'dd'}
-        return '<dl>\n{}</dl>'.format(''.join(
-            '<{tag}>{text}</{tag}>'.format(tag=tags[type], text=text)
-            for type, text in items
-        ))
-
-    def link(self, link, title, text):
-        return super().link(self._convert_url(link), title, text)
-
-    def image(self, src, title, text):
-        return super().image(self._convert_url(src), title, text)
-
-
-class Markdown(mistune.Markdown):
-    def output_admonition(self):
-        name = self.token['name']
-        body = self.renderer.placeholder()
-        if self.token['title']:
-            template = '<p class="admonition-title">{}</p>\n'
-            body += template.format(self.token['title'])
-        while self.pop()['type'] != 'admonition_end':
-            body += self.tok()
-        return self.renderer.admonition(name, body)
-
-    def output_deflist_term(self):
-        items = [['term', self.renderer.placeholder()]]
-        while True:
-            end_token = 'deflist_{}_end'.format(items[-1][0])
-            while self.pop()['type'] not in (end_token, 'paragraph'):
-                items[-1][1] += self.tok()
-            if self.token['type'] == 'paragraph':
-                if items[-1][0] == 'term':
-                    items.append(['term', self.renderer.placeholder()])
-                    items[-1][1] += self.token['text']
-                else:
-                    items[-1][1] += self.output_paragraph()
-            elif self.peek()['type'] == 'deflist_term_start':
-                self.pop()
-                items.append(['term', self.renderer.placeholder()])
-            elif self.peek()['type'] == 'deflist_def_start':
-                self.pop()
-                items.append(['def', self.renderer.placeholder()])
-            else:
-                break
-        return self.renderer.deflist(items)
+    def link(self, text, url, title=None):
+        return super().link(text, self._convert_url(url), title)
+
+    def image(self, alt, url, title=None):
+        return super().image(alt, self._convert_url(url), title)
 
 
 def convert_markdown(text, convert_url=None, *, inline=False):
     convert_url = convert_url if convert_url else lambda x: x
 
     text = dedent(text)
 
-    markdown = Markdown(
-        escape=False,
-        block=BlockLexer(),
-        renderer=Renderer(convert_url),
+    markdown = mistune.create_markdown(
+        plugins=['def_list', naucse_admonition_plugin],
+        renderer=NaucseRenderer(convert_url),
     )
     result = markdown(text).strip()
 

naucse_render/templates.py

@@ -67,6 +67,9 @@ def solution(ctx, text):
     solution = ctx['$markdown'](text)
     solutions.append(solution)
 
+    # make sure there are no empty lines, which exit Markdown's raw-HTML mode
+    solution = solution.replace('\n', Markup('\n<span></span>'))
+
     return Markup(textwrap.dedent("""
         <div class="solution" id="solution-{}">
             <h3>Řešení</h3>

pyproject.toml

@@ -4,18 +4,18 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "naucse_render"
-version = '1.10'
+version = '2.0'
 description = 'Converts course material to naucse.python.cz API'
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 classifiers = [
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3",
 ]
 dependencies = [
     'ansi2html',
-    'mistune',
-    'nbconvert>=6.0.7,<7.0',
+    'mistune>=2.0.3,<4',
+    'nbconvert>=7.0,<8.0',
     'traitlets',
     'click',
     'PyYAML',

test_naucse_render/fixtures/expected-compiled/courses/extra-lessons/tasks/index.html

@@ -1,19 +1,15 @@
 <ol>
-
     <li>
         <p>A task</p>
     </li>
-
     <li>
         <p>Task with list:</p>
 <ul>
 <li>a</li>
 <li>b</li>
 </ul>
     </li>
-
     <li>
-        <p>Static files handled with a <img src="naucse:static?filename=smile.png" alt="smile">.</p>
+        <p>Static files handled with a <img src="naucse:static?filename=smile.png" alt="smile" />.</p>
     </li>
-
 </ol>

test_naucse_render/fixtures/expected-compiled/default/course.json

@@ -69,6 +69,7 @@
                         },
                         "ids": [
                             "co_programatorsky_editor_umi",
+                            "solution-0",
                             "volba_a_nastaveni_editoru"
                         ],
                         "license": "cc-by-sa-40",
@@ -77,10 +78,15 @@
                             "#volba_a_nastaveni_editoru",
                             "naucse:page?lesson=beginners/install-editor&page=atom",
                             "naucse:page?lesson=beginners/install-editor&page=gedit",
-                            "naucse:page?lesson=beginners/install-editor&page=gedit#nacvik_odsazovani"
+                            "naucse:page?lesson=beginners/install-editor&page=gedit#nacvik_odsazovani",
+                            "naucse:solution?solution=0"
                         ],
                         "slug": "index",
-                        "solutions": [],
+                        "solutions": [
+                            {
+                                "content": "<div class=\"highlight\"><pre><span></span><span class=\"c1\"># Třikrát:</span>\n<span class=\"k\">for</span> <span class=\"n\">i</span> <span class=\"ow\">in</span> <span class=\"nb\">range</span><span class=\"p\">(</span><span class=\"mi\">3</span><span class=\"p\">):</span>\n\n    <span class=\"c1\"># Nakresli čtverec (kód zkopírovaný z předchozí úlohy a odsazený)</span>\n    <span class=\"k\">for</span> <span class=\"n\">j</span> <span class=\"ow\">in</span> <span class=\"nb\">range</span><span class=\"p\">(</span><span class=\"mi\">4</span><span class=\"p\">):</span>\n        <span class=\"n\">forward</span><span class=\"p\">(</span><span class=\"mi\">50</span><span class=\"p\">)</span>\n        <span class=\"n\">left</span><span class=\"p\">(</span><span class=\"mi\">90</span><span class=\"p\">)</span>\n\n    <span class=\"c1\"># Otoč se o 20°</span>\n    <span class=\"n\">left</span><span class=\"p\">(</span><span class=\"mi\">20</span><span class=\"p\">)</span>\n</pre></div>"
+                            }
+                        ],
                         "source_file": "lessons/beginners/install-editor/index.md",
                         "title": "Instalace editoru",
                         "vars": {}

test_naucse_render/fixtures/expected-compiled/default/install-editor/gedit.html

@@ -3,19 +3,27 @@ <h1 id="instalace_geditu">Instalace  Geditu
 </h1>
 <p>Na Linuxu se Gedit instaluje jako ostatní programy:</p>
 <dl>
-<dt></dt><dt>Fedora</dt><dd><div class="highlight"><pre><span></span><span class="gp">$ </span>sudo dnf install gedit
-</pre></div></dd><dt></dt><dt>Ubuntu</dt><dd><div class="highlight"><pre><span></span><span class="gp">$ </span>sudo apt-get install gedit
-</pre></div></dd></dl><p>Používáš-li jiný Linux, předpokládám že programy instalovat umíš. :)</p>
+<dt>Fedora</dt>
+<dd><div class="highlight"><pre><span></span><span class="gp">$ </span>sudo<span class="w"> </span>dnf<span class="w"> </span>install<span class="w"> </span>gedit
+</pre></div></dd>
+<dt>Ubuntu</dt>
+<dd><div class="highlight"><pre><span></span><span class="gp">$ </span>sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>gedit
+</pre></div></dd>
+</dl>
+<p>Používáš-li jiný Linux, předpokládám že programy instalovat umíš. :)</p>
 <p>Pro Windows a macOS se Gedit dá stáhnout z <a href="https://wiki.gnome.org/Apps/Gedit">domovské stránky</a>.</p>
 <h2 id="nastaveni">Nastavení
 <a href="#nastaveni" class="header-link">#</a>
 </h2>
 <p>...</p>
 <dl>
-<dt></dt><dt>Číslování řádků</dt><dd><p>V sekci Zobrazit/<span class="en">View</span> vyber
+<dt>Číslování řádků</dt>
+<dd><p>V sekci Zobrazit/<span class="en">View</span> vyber
 Zobrazovat čísla řádků/<span class="en">Display Line Numbers</span>.</p>
 <p><span class="figure"><a href="naucse:static?filename=gedit_linenums.png"><img src="naucse:static?filename=gedit_linenums.png" alt=""></a></span></p>
-</dd></dl><p>...</p>
+</dd>
+</dl>
+<p>...</p>
 <h2 id="nacvik_odsazovani">Nácvik odsazování
 <a href="#nacvik_odsazovani" class="header-link">#</a>
 </h2>