attemp to upgrade mistune

TODO.md

@@ -0,0 +1,33 @@
+issues to fix
+
+- fix mistune breaking multiline list entries
+
+```
+-<li>Using virtual machines or operating system containers to manage the runtime
+-(except as described in <a href="CommandLineTool.html#DockerRequirement">DockerRequirement</a>).</li>
++<li>Using virtual machines or operating system containers to manage the runtime</li>
++</ul>
++<p>(except as described in <a href="CommandLineTool.html#DockerRequirement">DockerRequirement</a>).</p>
++<ul>
+ <li>Using remote or distributed file systems to manage input and output files.</li>
+ <li>Transforming file paths.</li>
+-<li>Determining if a process has previously been executed, skipping it and
+-reusing previous results.</li>
++<li>Determining if a process has previously been executed, skipping it and</li>
++</ul>
++<p>reusing previous results.</p>
++<ul>
+```
+
+- fix remaining overquoting: `"` → `&quot;`
+
+new tests
+
+- mistune escaping raw HTML test (table of contents token, for example)
+- mistune plain url hyperlink test
+- mistune heading hyperlink target test (header→heading rename)
+- mistune table test (when the table plugin isn't loaded)
+
+
+
+

mypy.ini

@@ -3,22 +3,7 @@ show_error_context = true
 show_column_numbers = true
 show_error_codes = true
 pretty = true
-
-# --strict options as of mypy 0.910
-warn_unused_configs = True
-disallow_any_generics = True
-disallow_subclassing_any = True
-disallow_untyped_calls = True
-disallow_untyped_defs = True
-disallow_incomplete_defs = True
-check_untyped_defs = True
-disallow_untyped_decorators = True
-no_implicit_optional = True
-warn_redundant_casts = True
-warn_unused_ignores = True
-warn_return_any = True
-implicit_reexport = False
-strict_equality = True
+strict = True
 
 [mypy-ruamel.*]
 ignore_errors = True

requirements.txt

@@ -1,6 +1,6 @@
 ruamel.yaml>= 0.12.4, != 0.16.6, < 0.18
 rdflib>= 4.2.2, < 7.0.0
-mistune>=0.8.1,<0.9
+mistune>=0.8.1,<2.1
 CacheControl==0.12.10
 lockfile==0.12.2  # needed for CacheControl's FileCache
 black

schema_salad/codegen.py

@@ -13,15 +13,14 @@
 )
 from urllib.parse import urlsplit
 
-
 from . import schema
 from .codegen_base import CodeGenBase
 from .exceptions import SchemaSaladException
 from .java_codegen import JavaCodeGen
 from .python_codegen import PythonCodeGen
-from .typescript_codegen import TypeScriptCodeGen
 from .ref_resolver import Loader
 from .schema import shortname
+from .typescript_codegen import TypeScriptCodeGen
 from .utils import aslist
 
 FIELD_SORT_ORDER = ["id", "class", "name"]

schema_salad/makedoc.py

@@ -26,9 +26,30 @@
 from .utils import add_dictlist, aslist
 from .validate import avro_type_name
 
+try:
+    from mistune import Renderer as HTMLRenderer
+
+    mistune_0_renderer = mistune.markdown
+    mistune_2_renderer = None
+except ImportError:  # mistune 2.x
+    from mistune import HTMLRenderer  # type: ignore[attr-defined,no-redef]
+
+    mistune_0_renderer = None  # type: ignore[assignment]
+    mistune_2_renderer = mistune.create_markdown  # type: ignore[attr-defined]
+
+
 _logger = logging.getLogger("salad")
 
 
+def mistune_render(doc: str, renderer: Optional[HTMLRenderer] = None) -> str:
+    """Cope with both mistune 0.8.x and 2.x."""
+    if mistune_0_renderer:
+        return mistune_0_renderer(doc, escape=False, renderer=renderer)
+    return mistune_2_renderer(
+        escape=False, renderer=renderer, plugins=["url", "table"]
+    )(doc)
+
+
 def vocab_type_name(url: str) -> str:
     """Remove the avro namespace, if any."""
     return avro_type_name(url).split(".")[-1]
@@ -57,23 +78,29 @@ def linkto(item: str) -> str:
     return f"[{frg}](#{to_id(frg)})"
 
 
-class MyRenderer(mistune.Renderer):
+class MyRenderer(HTMLRenderer):
+    """Custom Mistune HTMLRenderer."""
+
     def __init__(self) -> None:
-        super().__init__()
-        self.options = {}
+        """Never escape input."""
+        super().__init__(escape=False)
 
     def header(self, text: str, level: int, raw: Optional[Any] = None) -> str:
+        """Mistune 0.8.x custom headers."""
+        return self.heading(text, level)
+
+    def heading(self, text: str, level: int) -> str:
+        """Mistune 2.x custom headers."""
         return (
-            """<h{} id="{}" class="section">{} <a href="#{}">&sect;</a></h{}>""".format(
-                level, to_id(text), text, to_id(text), level
-            )
+            f'<h{level} id="{to_id(text)}" class="section">{text} '
+            f'<a href="#{to_id(text)}">&sect;</a></h{level}>'
         )
 
     def table(self, header: str, body: str) -> str:
-        return (
-            '<table class="table table-striped">\n<thead>{}</thead>\n'
-            "<tbody>\n{}</tbody>\n</table>\n"
-        ).format(header, body)
+        """Add our custom CSS classes to <table>."""
+        table = f'<table class="table table-striped">\n<thead>{header}</thead>\n'
+        table += f"<tbody>\n{body}</tbody>\n</table>\n"
+        return table
 
 
 def to_id(text: str) -> str:
@@ -423,7 +450,7 @@ def extendsfrom(item: Dict[str, Any], ex: List[Dict[str, Any]]) -> None:
 
         doc = doc + "\n\n" + f["doc"]
 
-        doc = mistune.markdown(doc, renderer=MyRenderer())
+        doc = mistune_render(doc, renderer=MyRenderer())
 
         if f["type"] == "record":
             doc += "<h3>Fields</h3>"
@@ -463,7 +490,7 @@ def extendsfrom(item: Dict[str, Any], ex: List[Dict[str, Any]]) -> None:
                     self.typefmt(
                         tp, self.redirects, jsonldPredicate=i.get("jsonldPredicate")
                     ),
-                    mistune.markdown(desc),
+                    mistune_render(desc),
                 )
                 if opt:
                     required.append(tr)

schema_salad/typescript_codegen.py

@@ -17,11 +17,11 @@
 
 import pkg_resources
 
-from . import schema, _logger
+from . import _logger, schema
 from .codegen_base import CodeGenBase, TypeDef
 from .exceptions import SchemaException
-from .schema import shortname
 from .java_codegen import _ensure_directory_and_write, _safe_makedirs
+from .schema import shortname
 
 
 def doc_to_doc_string(doc: Optional[str], indent_level: int = 0) -> str:

setup.py

@@ -78,7 +78,7 @@
     # once the minimum version for ruamel.yaml >= 0.15.99
     # then please update the mypy targets in the Makefile
     "rdflib >= 4.2.2, < 7.0.0",
-    "mistune >= 0.8.1, < 0.9",
+    "mistune >= 0.8.1, < 2.1",
     "CacheControl >= 0.11.7, < 0.13",
     "lockfile >= 0.9",  # needed for CacheControl's FileCache
 ]