Merge branch 'master' into custom-extras

Merge master to resolve PR conflicts
trentm · Dec 1, 2023 · 4353751 · 4353751
2 parents bf2bec3 + 182b7f5
commit 4353751
Show file tree

Hide file tree

Showing 8 changed files with 66 additions and 8 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -6,6 +6,8 @@
 - [pull #527] Fix base64 images being corrupted in safe mode (issue #526)
 - [pull #529] Add `breaks` extra with ability to hard break on backslashes (issue #525)
 - [pull #532] Fix #493 persisting when `code-friendly` extra enabled
+- [pull #535] Update `_slugify` to use utf-8 encoding (issue #534)
+- [pull #536] Maintain order of appearance in footnotes
 - [pull #519] Add support for custom extras
 - [pull #519] Drop Python 3.5 support
 

diff --git a/lib/markdown2.py b/lib/markdown2.py
@@ -115,7 +115,7 @@
 import logging
 import re
 import sys
-from collections import defaultdict
+from collections import defaultdict, OrderedDict
 from abc import ABC, abstractmethod
 import functools
 from hashlib import sha256
@@ -377,7 +377,9 @@ def reset(self):
 
     def _setup_extras(self):
         if "footnotes" in self.extras:
-            self.footnotes = {}
+            # order of insertion matters for footnotes. Use ordered dict for Python < 3.7
+            # https://docs.python.org/3/whatsnew/3.7.html#summary-release-highlights
+            self.footnotes = OrderedDict()
             self.footnote_ids = []
         if "header-ids" in self.extras:
             self._count_from_header_id = defaultdict(int)
@@ -2116,6 +2118,10 @@ def _add_footnotes(self, text):
             if not self.footnote_return_symbol:
                 self.footnote_return_symbol = "&#8617;"
 
+            # self.footnotes is generated in _strip_footnote_definitions, which runs re.sub on the whole
+            # text. This means that the dict keys are inserted in order of appearance. Use the dict to
+            # sort footnote ids by that same order
+            self.footnote_ids.sort(key=lambda a: list(self.footnotes.keys()).index(a))
             for i, id in enumerate(self.footnote_ids):
                 if i != 0:
                     footer.append('')
@@ -3268,7 +3274,7 @@ def _slugify(value):
     From Django's "django/template/defaultfilters.py".
     """
     import unicodedata
-    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode()
+    value = unicodedata.normalize('NFKD', value).encode('utf-8', 'ignore').decode()
     value = _slugify_strip_re.sub('', value).strip().lower()
     return _slugify_hyphenate_re.sub('-', value)
 ## end of http://code.activestate.com/recipes/577257/ }}}

diff --git a/test/tm-cases/footnotes_order_of_appearance.html b/test/tm-cases/footnotes_order_of_appearance.html
@@ -0,0 +1,34 @@
+<hr />
+
+<h2>title: testing the footnotes bug</h2>
+
+<p>Lorem ipsum dolor sit amet. Aliqua cillum, eu velit.<sup class="footnote-ref" id="fnref-note1"><a href="#fn-note1">3</a></sup> Velit deserunt adipiscing adipiscing ullamco exercitation.</p>
+
+<ul>
+<li>this is a list item</li>
+<li>this is a list item<sup class="footnote-ref" id="fnref-note2"><a href="#fn-note2">1</a></sup></li>
+<li>this is a list item<sup class="footnote-ref" id="fnref-note3"><a href="#fn-note3">2</a></sup></li>
+</ul>
+
+<p>Lorem ipsum dolor sit amet. Adipiscing<sup class="footnote-ref" id="fnref-note4"><a href="#fn-note4">4</a></sup> adipiscing ullamco, exercitation sint. Exercitation sint, fugiat exercitation voluptate amet.</p>
+
+<div class="footnotes">
+<hr />
+<ol>
+<li id="fn-note1">
+<p>this should be the first note&#160;<a href="#fnref-note1" class="footnoteBackLink" title="Jump back to footnote 1 in the text.">&#8617;</a></p>
+</li>
+
+<li id="fn-note2">
+<p>this should be the second note&#160;<a href="#fnref-note2" class="footnoteBackLink" title="Jump back to footnote 2 in the text.">&#8617;</a></p>
+</li>
+
+<li id="fn-note3">
+<p>this should be the third note&#160;<a href="#fnref-note3" class="footnoteBackLink" title="Jump back to footnote 3 in the text.">&#8617;</a></p>
+</li>
+
+<li id="fn-note4">
+<p>this should be the fourth note&#160;<a href="#fnref-note4" class="footnoteBackLink" title="Jump back to footnote 4 in the text.">&#8617;</a></p>
+</li>
+</ol>
+</div>
diff --git a/test/tm-cases/footnotes_order_of_appearance.opts b/test/tm-cases/footnotes_order_of_appearance.opts
@@ -0,0 +1 @@
+{"extras": ["footnotes"]}
diff --git a/test/tm-cases/footnotes_order_of_appearance.text b/test/tm-cases/footnotes_order_of_appearance.text
@@ -0,0 +1,15 @@
+---
+title: testing the footnotes bug
+---
+Lorem ipsum dolor sit amet. Aliqua cillum, eu velit.[^note1] Velit deserunt adipiscing adipiscing ullamco exercitation.
+
+- this is a list item
+- this is a list item[^note2]
+- this is a list item[^note3]
+
+Lorem ipsum dolor sit amet. Adipiscing[^note4] adipiscing ullamco, exercitation sint. Exercitation sint, fugiat exercitation voluptate amet.
+
+[^note1]: this should be the first note
+[^note2]: this should be the second note
+[^note3]: this should be the third note
+[^note4]: this should be the fourth note
diff --git a/test/tm-cases/header_ids_4.html b/test/tm-cases/header_ids_4.html
@@ -1,6 +1,6 @@
 <!-- -*- coding: utf-8 -*- -->
 
-<h1 id="fruit-really-likes">Fruit заголовок <em>really</em> likes</h1>
+<h1 id="fruit-заголовок-really-likes">Fruit заголовок <em>really</em> likes</h1>
 
 <ul>
 <li>apples</li>

diff --git a/test/tm-cases/toc_4.html b/test/tm-cases/toc_4.html
@@ -1,6 +1,6 @@
 <h1 id="python">Python</h1>
 
-<h2 id="-1">蟒蛇</h2>
+<h2 id="蟒蛇">蟒蛇</h2>
 
 <ul>
 <li>外形特性</li>
@@ -9,7 +9,7 @@ <h2 id="-1">蟒蛇</h2>
 
 <h2 id="markdown">Markdown</h2>
 
-<h2 id="-2">标记语言</h2>
+<h2 id="标记语言">标记语言</h2>
 
 <ul>
 <li>类型</li>

diff --git a/test/tm-cases/toc_4.toc_html b/test/tm-cases/toc_4.toc_html
@@ -1,8 +1,8 @@
 <ul>
   <li><a href="#python">Python</a>
   <ul>
-    <li><a href="#-1">蟒蛇</a></li>
+    <li><a href="#蟒蛇">蟒蛇</a></li>
     <li><a href="#markdown">Markdown</a></li>
-    <li><a href="#-2">标记语言</a></li>
+    <li><a href="#标记语言">标记语言</a></li>
   </ul></li>
 </ul>