🔀 MERGE: Synchronize with markdown-it v12.0.4

.pre-commit-config.yaml

@@ -28,17 +28,17 @@ repos:
     hooks:
     - id: check-manifest
 
+  - repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+    - id: black
+
   - repo: https://gitlab.com/pycqa/flake8
     rev: 3.8.4
     hooks:
     - id: flake8
       additional_dependencies: [flake8-bugbear==21.3.1]
 
-  - repo: https://github.com/psf/black
-    rev: 20.8b1
-    hooks:
-    - id: black
-
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.790
     hooks:

markdown_it/common/html_blocks.py

@@ -45,7 +45,6 @@
     "main",
     "menu",
     "menuitem",
-    "meta",
     "nav",
     "noframes",
     "ol",

markdown_it/common/html_re.py

@@ -17,7 +17,7 @@
 
 close_tag = "<\\/[A-Za-z][A-Za-z0-9\\-]*\\s*>"
 comment = "<!---->|<!--(?:-?[^>-])(?:-?[^-])*-->"
-processing = "<[?].*?[?]>"
+processing = "<[?][\\s\\S]*?[?]>"
 declaration = "<![A-Z]+\\s+[^>]*>"
 cdata = "<!\\[CDATA\\[[\\s\\S]*?\\]\\]>"
 

markdown_it/helpers/parse_link_destination.py

@@ -26,6 +26,8 @@ def parseLinkDestination(string: str, pos: int, maximum: int) -> _Result:
             code = charCodeAt(string, pos)
             if code == 0x0A:  # /* \n */)
                 return result
+            if code == 0x3C:  # / * < * /
+                return result
             if code == 0x3E:  # /* > */) {
                 result.pos = pos + 1
                 result.str = unescapeAll(string[start + 1 : pos])
@@ -55,11 +57,15 @@ def parseLinkDestination(string: str, pos: int, maximum: int) -> _Result:
             break
 
         if code == 0x5C and pos + 1 < maximum:
+            if charCodeAt(string, pos + 1) == 0x20:
+                break
             pos += 2
             continue
 
         if code == 0x28:  # /* ( */)
             level += 1
+            if level > 32:
+                return result
 
         if code == 0x29:  # /* ) */)
             if level == 0:

markdown_it/helpers/parse_link_title.py

@@ -46,6 +46,8 @@ def parseLinkTitle(string: str, pos: int, maximum: int) -> _Result:
             result.str = title
             result.ok = True
             return result
+        elif code == 0x28 and marker == 0x29:  # /* ( */  /* ) */
+            return result
         elif code == 0x0A:
             lines += 1
         elif code == 0x5C and pos + 1 < maximum:  # /* \ */

markdown_it/port.yaml

@@ -1,6 +1,7 @@
 - package: markdown-it/markdown-it
-  commit: 331ae117e09115366db12b517ca526b1b7fee1e8
-  date: Sep 14, 2020
+  version: 12.0.4
+  commit: 7b8969ce5cb2edc54f2c1aa39a85a3a08076337d
+  date: Dec 20, 2020
   notes:
     - Rename variables that use python built-in names, e.g.
       - `max` -> `maximum`
@@ -39,3 +40,4 @@
       rather than `MarkdownIt.renderer.rules[name] = function`
       and renderers should declare a class property `__output__ = "html"`.
       This allows for extensability to more than just HTML renderers
+    - inline tokens in tables are assigned a map (this is helpful for propagation to children)

markdown_it/renderer.py

@@ -219,22 +219,27 @@ def fence(self, tokens: Sequence[Token], idx: int, options, env) -> str:
         token = tokens[idx]
         info = unescapeAll(token.info).strip() if token.info else ""
         langName = ""
+        langAttrs = ""
 
         if info:
             langName = info.split()[0]
+            arr = info.split(maxsplit=1)
+            langName = arr[0]
+            if len(arr) == 2:
+                langAttrs = arr[1]
 
         if options.highlight:
-            highlighted = options.highlight(token.content, langName) or escapeHtml(
-                token.content
-            )
+            highlighted = options.highlight(
+                token.content, langName, langAttrs
+            ) or escapeHtml(token.content)
         else:
             highlighted = escapeHtml(token.content)
 
         if highlighted.startswith("<pre"):
             return highlighted + "\n"
 
         # If language exists, inject class gently, without modifying original token.
-        # May be, one day we will add .clone() for token and simplify this part, but
+        # May be, one day we will add .deepClone() for token and simplify this part, but
         # now we prefer to keep things local.
         if info:
             i = token.attrIndex("class")
@@ -243,6 +248,7 @@ def fence(self, tokens: Sequence[Token], idx: int, options, env) -> str:
             if i < 0:
                 tmpAttrs.append(["class", options.langPrefix + langName])
             else:
+                tmpAttrs[i] = tmpAttrs[i][:]
                 tmpAttrs[i][1] += " " + options.langPrefix + langName
 
             # Fake token just to render attributes

markdown_it/rules_block/table.py

@@ -1,4 +1,4 @@
-# GFM table, non-standard
+# GFM table, https://github.github.com/gfm/#tables-extension-
 import re
 
 from .state_block import StateBlock
@@ -10,7 +10,7 @@
 
 
 def getLine(state: StateBlock, line: int):
-    pos = state.bMarks[line] + state.blkIndent
+    pos = state.bMarks[line] + state.tShift[line]
     maximum = state.eMarks[line]
 
     # return state.src.substr(pos, max - pos)
@@ -21,48 +21,35 @@ def escapedSplit(string):
     result = []
     pos = 0
     max = len(string)
-    escapes = 0
+    isEscaped = False
     lastPos = 0
-    backTicked = False
-    lastBackTick = 0
+    current = ""
     ch = charCodeAt(string, pos)
 
     while pos < max:
-        if ch == 0x60:  # /* ` */
-            if backTicked:
-                # make \` close code sequence, but not open it;
-                # the reason is: `\` is correct code block
-                backTicked = False
-                lastBackTick = pos
-            elif escapes % 2 == 0:
-                backTicked = True
-                lastBackTick = pos
-        # /* | */
-        elif ch == 0x7C and (escapes % 2 == 0) and not backTicked:
-            result.append(string[lastPos:pos])
-            lastPos = pos + 1
-
-        if ch == 0x5C:  # /* \ */
-            escapes += 1
-        else:
-            escapes = 0
+        if ch == 0x7C:  # /* | */
+            if not isEscaped:
+                # pipe separating cells, '|'
+                result.append(current + string[lastPos:pos])
+                current = ""
+                lastPos = pos + 1
+            else:
+                # escaped pipe, '\|'
+                current += string[lastPos : pos - 1]
+                lastPos = pos
 
+        isEscaped = ch == 0x5C  # /* \ */
         pos += 1
 
-        # If there was an un-closed backtick, go back to just after
-        # the last backtick, but as if it was a normal character
-        if pos == max and backTicked:
-            backTicked = False
-            pos = lastBackTick + 1
-
         ch = charCodeAt(string, pos)
 
-    result.append(string[lastPos:])
+    result.append(current + string[lastPos:])
 
     return result
 
 
 def table(state: StateBlock, startLine: int, endLine: int, silent: bool):
+    tbodyLines = None
 
     # should have at least two lines
     if startLine + 2 > endLine:
@@ -129,17 +116,28 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool):
         return False
     if state.sCount[startLine] - state.blkIndent >= 4:
         return False
-    columns = escapedSplit(enclosingPipesRe.sub("", lineText))
+    columns = escapedSplit(lineText)
+    if columns and columns[0] == "":
+        columns.pop(0)
+    if columns and columns[-1] == "":
+        columns.pop()
 
     # header row will define an amount of columns in the entire table,
-    # and align row shouldn't be smaller than that (the rest of the rows can)
+    # and align row should be exactly the same (the rest of the rows can differ)
     columnCount = len(columns)
-    if columnCount > len(aligns):
+    if columnCount == 0 or columnCount != len(aligns):
         return False
 
     if silent:
         return True
 
+    oldParentType = state.parentType
+    state.parentType = "table"
+
+    # use 'blockquote' lists for termination because it's
+    # the most similar to tables
+    terminatorRules = state.md.block.ruler.getRules("blockquote")
+
     token = state.push("table_open", "table", 1)
     token.map = tableLines = [startLine, 0]
 
@@ -151,43 +149,60 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool):
 
     for i in range(len(columns)):
         token = state.push("th_open", "th", 1)
-        token.map = [startLine, startLine + 1]
         if aligns[i]:
             token.attrs = [["style", "text-align:" + aligns[i]]]
 
         token = state.push("inline", "", 0)
-        token.content = columns[i].strip()
+        # note in markdown-it this map was removed in v12.0.0 however, we keep it,
+        # since it is helpful to propagate to children tokens
         token.map = [startLine, startLine + 1]
+        token.content = columns[i].strip()
         token.children = []
 
         token = state.push("th_close", "th", -1)
 
     token = state.push("tr_close", "tr", -1)
     token = state.push("thead_close", "thead", -1)
 
-    token = state.push("tbody_open", "tbody", 1)
-    token.map = tbodyLines = [startLine + 2, 0]
-
     nextLine = startLine + 2
     while nextLine < endLine:
         if state.sCount[nextLine] < state.blkIndent:
             break
 
+        terminate = False
+        for i in range(len(terminatorRules)):
+            if terminatorRules[i](state, nextLine, endLine, True):
+                terminate = True
+                break
+
+        if terminate:
+            break
         lineText = getLine(state, nextLine).strip()
-        if "|" not in lineText:
+        if not lineText:
             break
         if state.sCount[nextLine] - state.blkIndent >= 4:
             break
-        columns = escapedSplit(enclosingPipesRe.sub("", lineText))
+        columns = escapedSplit(lineText)
+        if columns and columns[0] == "":
+            columns.pop(0)
+        if columns and columns[-1] == "":
+            columns.pop()
+
+        if nextLine == startLine + 2:
+            token = state.push("tbody_open", "tbody", 1)
+            token.map = tbodyLines = [startLine + 2, 0]
 
         token = state.push("tr_open", "tr", 1)
+        token.map = [nextLine, nextLine + 1]
+
         for i in range(columnCount):
             token = state.push("td_open", "td", 1)
-            token.map = [nextLine, nextLine + 1]
             if aligns[i]:
                 token.attrs = [["style", "text-align:" + aligns[i]]]
 
             token = state.push("inline", "", 0)
+            # note in markdown-it this map was removed in v12.0.0 however, we keep it,
+            # since it is helpful to propagate to children tokens
             token.map = [nextLine, nextLine + 1]
             try:
                 token.content = columns[i].strip() if columns[i] else ""
@@ -201,9 +216,13 @@ def table(state: StateBlock, startLine: int, endLine: int, silent: bool):
 
         nextLine += 1
 
-    token = state.push("tbody_close", "tbody", -1)
+    if tbodyLines:
+        token = state.push("tbody_close", "tbody", -1)
+        tbodyLines[1] = nextLine
+
     token = state.push("table_close", "table", -1)
 
-    tableLines[1] = tbodyLines[1] = nextLine
+    tableLines[1] = nextLine
+    state.parentType = oldParentType
     state.line = nextLine
     return True