Update for CM 0.31

generate/src/main.rs

@@ -14,7 +14,7 @@ async fn main() {
 }
 
 async fn commonmark() {
-    let url = "https://raw.githubusercontent.com/commonmark/commonmark-spec/0.30/spec.txt";
+    let url = "https://raw.githubusercontent.com/commonmark/commonmark-spec/0.31.2/spec.txt";
     let data_url = "commonmark-data.txt";
     let code_url = "tests/commonmark.rs";
 
@@ -121,6 +121,10 @@ async fn punctuation() {
         "Pi", // Punctuation, InitialQuote
         "Po", // Punctuation, Other
         "Ps", // Punctuation, Open
+        "Sc", // Symbol, Currency
+        "Sk", // Symbol, Modifier
+        "Sm", // Symbol, Math
+        "So", // Symbol, Other
     ];
 
     let found = value
@@ -148,7 +152,7 @@ async fn punctuation() {
 ///
 /// ## References
 ///
-/// *   [*§ 2.1 Characters and lines* in `CommonMark`](https://spec.commonmark.org/0.30/#unicode-punctuation-character)
+/// *   [*§ 2.1 Characters and lines* in `CommonMark`](https://spec.commonmark.org/0.31.2/#unicode-punctuation-character)
 pub const PUNCTUATION: [char; {}] = [
 {}
 ];

src/construct/attention.rs

@@ -68,7 +68,7 @@
 //!
 //! *   [`attention.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/attention.js)
 //! *   [`micromark-extension-gfm-strikethrough`](https://github.com/micromark/micromark-extension-gfm-strikethrough)
-//! *   [*§ 6.2 Emphasis and strong emphasis* in `CommonMark`](https://spec.commonmark.org/0.30/#emphasis-and-strong-emphasis)
+//! *   [*§ 6.2 Emphasis and strong emphasis* in `CommonMark`](https://spec.commonmark.org/0.31/#emphasis-and-strong-emphasis)
 //! *   [*§ 6.5 Strikethrough (extension)* in `GFM`](https://github.github.com/gfm/#strikethrough-extension-)
 //!
 //! [text]: crate::construct::text

src/construct/autolink.rs

@@ -110,7 +110,7 @@
 //! ## References
 //!
 //! *   [`autolink.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/autolink.js)
-//! *   [*§ 6.4 Autolinks* in `CommonMark`](https://spec.commonmark.org/0.30/#autolinks)
+//! *   [*§ 6.4 Autolinks* in `CommonMark`](https://spec.commonmark.org/0.31/#autolinks)
 //!
 //! [text]: crate::construct::text
 //! [definition]: crate::construct::definition

src/construct/blank_line.rs

@@ -42,7 +42,7 @@
 //! ## References
 //!
 //! *   [`blank-line.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/blank-line.js)
-//! *   [*§ 4.9 Blank lines* in `CommonMark`](https://spec.commonmark.org/0.30/#blank-lines)
+//! *   [*§ 4.9 Blank lines* in `CommonMark`](https://spec.commonmark.org/0.31/#blank-lines)
 //!
 //! [heading_atx]: crate::construct::heading_atx
 //! [list_item]: crate::construct::list_item

src/construct/block_quote.rs

@@ -40,11 +40,11 @@
 //! ## References
 //!
 //! *   [`block-quote.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/block-quote.js)
-//! *   [*§ 5.1 Block quotes* in `CommonMark`](https://spec.commonmark.org/0.30/#block-quotes)
+//! *   [*§ 5.1 Block quotes* in `CommonMark`](https://spec.commonmark.org/0.31/#block-quotes)
 //!
 //! [document]: crate::construct::document
 //! [html-blockquote]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-blockquote-element
-//! [commonmark-block]: https://spec.commonmark.org/0.30/#phase-1-block-structure
+//! [commonmark-block]: https://spec.commonmark.org/0.31/#phase-1-block-structure
 
 use crate::construct::partial_space_or_tab::space_or_tab_min_max;
 use crate::event::Name;

src/construct/character_escape.rs

@@ -34,7 +34,7 @@
 //! ## References
 //!
 //! *   [`character-escape.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/character-escape.js)
-//! *   [*§ 2.4 Backslash escapes* in `CommonMark`](https://spec.commonmark.org/0.30/#backslash-escapes)
+//! *   [*§ 2.4 Backslash escapes* in `CommonMark`](https://spec.commonmark.org/0.31/#backslash-escapes)
 //!
 //! [string]: crate::construct::string
 //! [text]: crate::construct::text

src/construct/character_reference.rs

@@ -59,7 +59,7 @@
 //! ## References
 //!
 //! *   [`character-reference.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/character-reference.js)
-//! *   [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.30/#entity-and-numeric-character-references)
+//! *   [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.31/#entity-and-numeric-character-references)
 //!
 //! [string]: crate::construct::string
 //! [text]: crate::construct::text

src/construct/code_indented.rs

@@ -44,7 +44,7 @@
 //! ## References
 //!
 //! *   [`code-indented.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-indented.js)
-//! *   [*§ 4.4 Indented code blocks* in `CommonMark`](https://spec.commonmark.org/0.30/#indented-code-blocks)
+//! *   [*§ 4.4 Indented code blocks* in `CommonMark`](https://spec.commonmark.org/0.31/#indented-code-blocks)
 //!
 //! [flow]: crate::construct::flow
 //! [text]: crate::construct::text

src/construct/definition.rs

@@ -86,7 +86,7 @@
 //! ## References
 //!
 //! *   [`definition.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/definition.js)
-//! *   [*§ 4.7 Link reference definitions* in `CommonMark`](https://spec.commonmark.org/0.30/#link-reference-definitions)
+//! *   [*§ 4.7 Link reference definitions* in `CommonMark`](https://spec.commonmark.org/0.31/#link-reference-definitions)
 //!
 //! [content]: crate::construct::content
 //! [string]: crate::construct::string

src/construct/gfm_footnote_definition.rs

@@ -156,7 +156,7 @@
 //! [label]: crate::construct::partial_label
 //! [label_end]: crate::construct::label_end
 //! [gfm_label_start_footnote]: crate::construct::gfm_label_start_footnote
-//! [commonmark_block]: https://spec.commonmark.org/0.30/#phase-1-block-structure
+//! [commonmark_block]: https://spec.commonmark.org/0.31/#phase-1-block-structure
 //! [html_a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
 //! [html_h]: https://html.spec.whatwg.org/multipage/sections.html#the-h1,-h2,-h3,-h4,-h5,-and-h6-elements
 //! [html_li]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-li-element

src/construct/hard_break_escape.rs

@@ -36,7 +36,7 @@
 //! ## References
 //!
 //! *   [`hard-break-escape.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/hard-break-escape.js)
-//! *   [*§ 6.7 Hard line breaks* in `CommonMark`](https://spec.commonmark.org/0.30/#hard-line-breaks)
+//! *   [*§ 6.7 Hard line breaks* in `CommonMark`](https://spec.commonmark.org/0.31/#hard-line-breaks)
 //!
 //! [text]: crate::construct::text
 //! [character_escape]: crate::construct::character_escape

src/construct/heading_atx.rs

@@ -53,7 +53,7 @@
 //! ## References
 //!
 //! *   [`heading-atx.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/heading-atx.js)
-//! *   [*§ 4.2 ATX headings* in `CommonMark`](https://spec.commonmark.org/0.30/#atx-headings)
+//! *   [*§ 4.2 ATX headings* in `CommonMark`](https://spec.commonmark.org/0.31/#atx-headings)
 //!
 //! [flow]: crate::construct::flow
 //! [heading_setext]: crate::construct::heading_setext

src/construct/heading_setext.rs

@@ -59,7 +59,7 @@
 //! ## References
 //!
 //! *   [`setext-underline.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/setext-underline.js)
-//! *   [*§ 4.3 Setext headings* in `CommonMark`](https://spec.commonmark.org/0.30/#setext-headings)
+//! *   [*§ 4.3 Setext headings* in `CommonMark`](https://spec.commonmark.org/0.31/#setext-headings)
 //!
 //! [flow]: crate::construct::flow
 //! [paragraph]: crate::construct::paragraph

src/construct/html_flow.rs

@@ -89,7 +89,7 @@
 //! ## References
 //!
 //! *   [`html-flow.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/html-flow.js)
-//! *   [*§ 4.6 HTML blocks* in `CommonMark`](https://spec.commonmark.org/0.30/#html-blocks)
+//! *   [*§ 4.6 HTML blocks* in `CommonMark`](https://spec.commonmark.org/0.31/#html-blocks)
 //!
 //! [flow]: crate::construct::flow
 //! [html_text]: crate::construct::html_text

src/construct/html_text.rs

@@ -45,7 +45,7 @@
 //! ## References
 //!
 //! *   [`html-text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/html-text.js)
-//! *   [*§ 6.6 Raw HTML* in `CommonMark`](https://spec.commonmark.org/0.30/#raw-html)
+//! *   [*§ 6.6 Raw HTML* in `CommonMark`](https://spec.commonmark.org/0.31/#raw-html)
 //!
 //! [text]: crate::construct::text
 //! [html_flow]: crate::construct::html_flow
@@ -146,56 +146,12 @@ pub fn comment_open_inside(tokenizer: &mut Tokenizer) -> State {
     match tokenizer.current {
         Some(b'-') => {
             tokenizer.consume();
-            State::Next(StateName::HtmlTextCommentStart)
+            State::Next(StateName::HtmlTextCommentEnd)
         }
         _ => State::Nok,
     }
 }
 
-/// After `<!--`, in a comment.
-///
-/// > 👉 **Note**: [html (flow)][html_flow] does allow `<!-->` or `<!--->` as
-/// > empty comments.
-/// > This is prohibited in html (text).
-/// > See: <https://github.com/commonmark/commonmark-spec/issues/712>.
-///
-/// ```markdown
-/// > | a <!--b--> c
-///           ^
-/// ```
-///
-/// [html_flow]: crate::construct::html_flow
-pub fn comment_start(tokenizer: &mut Tokenizer) -> State {
-    match tokenizer.current {
-        Some(b'>') => State::Nok,
-        Some(b'-') => {
-            tokenizer.consume();
-            State::Next(StateName::HtmlTextCommentStartDash)
-        }
-        _ => State::Retry(StateName::HtmlTextComment),
-    }
-}
-
-/// After `<!---`, in a comment.
-///
-/// > 👉 **Note**: [html (flow)][html_flow] does allow `<!-->` or `<!--->` as
-/// > empty comments.
-/// > This is prohibited in html (text).
-/// > See: <https://github.com/commonmark/commonmark-spec/issues/712>.
-///
-/// ```markdown
-/// > | a <!---b--> c
-///            ^
-/// ```
-///
-/// [html_flow]: crate::construct::html_flow
-pub fn comment_start_dash(tokenizer: &mut Tokenizer) -> State {
-    match tokenizer.current {
-        Some(b'>') => State::Nok,
-        _ => State::Retry(StateName::HtmlTextComment),
-    }
-}
-
 /// In comment.
 ///
 /// ```markdown
@@ -230,11 +186,24 @@ pub fn comment_close(tokenizer: &mut Tokenizer) -> State {
     match tokenizer.current {
         Some(b'-') => {
             tokenizer.consume();
-            State::Next(StateName::HtmlTextEnd)
+            State::Next(StateName::HtmlTextCommentEnd)
         }
         _ => State::Retry(StateName::HtmlTextComment),
     }
 }
+/// In comment, after `-`.
+///
+/// ```markdown
+/// > | a <!--b--> c
+///             ^
+/// ```
+pub fn comment_end(tokenizer: &mut Tokenizer) -> State {
+    match tokenizer.current {
+        Some(b'>') => State::Retry(StateName::HtmlTextEnd),
+        Some(b'-') => State::Retry(StateName::HtmlTextCommentClose),
+        _ => State::Retry(StateName::HtmlTextComment),
+    }
+}
 
 /// After `<![`, in CDATA, expecting `CDATA[`.
 ///

src/construct/label_end.rs

@@ -155,9 +155,9 @@
 //!
 //! *   [`label-end.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/label-end.js)
 //! *   [`micromark-extension-gfm-task-list-item`](https://github.com/micromark/micromark-extension-gfm-footnote)
-//! *   [*§ 4.7 Link reference definitions* in `CommonMark`](https://spec.commonmark.org/0.30/#link-reference-definitions)
-//! *   [*§ 6.3 Links* in `CommonMark`](https://spec.commonmark.org/0.30/#links)
-//! *   [*§ 6.4 Images* in `CommonMark`](https://spec.commonmark.org/0.30/#images)
+//! *   [*§ 4.7 Link reference definitions* in `CommonMark`](https://spec.commonmark.org/0.31/#link-reference-definitions)
+//! *   [*§ 6.3 Links* in `CommonMark`](https://spec.commonmark.org/0.31/#links)
+//! *   [*§ 6.4 Images* in `CommonMark`](https://spec.commonmark.org/0.31/#images)
 //!
 //! > 👉 **Note**: Footnotes are not specified in GFM yet.
 //! > See [`github/cmark-gfm#270`](https://github.com/github/cmark-gfm/issues/270)

src/construct/label_start_image.rs

@@ -26,7 +26,7 @@
 //! ## References
 //!
 //! *   [`label-start-image.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/label-start-image.js)
-//! *   [*§ 6.4 Images* in `CommonMark`](https://spec.commonmark.org/0.30/#images)
+//! *   [*§ 6.4 Images* in `CommonMark`](https://spec.commonmark.org/0.31/#images)
 //!
 //! [text]: crate::construct::text
 //! [label_end]: crate::construct::label_end

src/construct/label_start_link.rs

@@ -25,7 +25,7 @@
 //! ## References
 //!
 //! *   [`label-start-link.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/label-start-link.js)
-//! *   [*§ 6.3 Links* in `CommonMark`](https://spec.commonmark.org/0.30/#links)
+//! *   [*§ 6.3 Links* in `CommonMark`](https://spec.commonmark.org/0.31/#links)
 //!
 //! [text]: crate::construct::text
 //! [label_end]: crate::construct::label_end

src/construct/list_item.rs

@@ -49,14 +49,14 @@
 //! ## References
 //!
 //! *   [`list.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/list.js)
-//! *   [*§ 5.2 List items* in `CommonMark`](https://spec.commonmark.org/0.30/#list-items)
-//! *   [*§ 5.3 Lists* in `CommonMark`](https://spec.commonmark.org/0.30/#lists)
+//! *   [*§ 5.2 List items* in `CommonMark`](https://spec.commonmark.org/0.31/#list-items)
+//! *   [*§ 5.3 Lists* in `CommonMark`](https://spec.commonmark.org/0.31/#lists)
 //!
 //! [document]: crate::construct::document
 //! [html_li]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-li-element
 //! [html_ol]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-ol-element
 //! [html_ul]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-ul-element
-//! [commonmark_block]: https://spec.commonmark.org/0.30/#phase-1-block-structure
+//! [commonmark_block]: https://spec.commonmark.org/0.31/#phase-1-block-structure
 
 use crate::construct::partial_space_or_tab::space_or_tab_min_max;
 use crate::event::{Kind, Name};

src/construct/paragraph.rs

@@ -33,7 +33,7 @@
 //! ## References
 //!
 //! *   [`content.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/content.js)
-//! *   [*§ 4.8 Paragraphs* in `CommonMark`](https://spec.commonmark.org/0.30/#paragraphs)
+//! *   [*§ 4.8 Paragraphs* in `CommonMark`](https://spec.commonmark.org/0.31/#paragraphs)
 //!
 //! [content]: crate::construct::content
 //! [text]: crate::construct::text

src/construct/partial_whitespace.rs

@@ -48,7 +48,7 @@
 //! ## References
 //!
 //! *   [`initialize/text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark/dev/lib/initialize/text.js)
-//! *   [*§ 6.7 Hard line breaks* in `CommonMark`](https://spec.commonmark.org/0.30/#hard-line-breaks)
+//! *   [*§ 6.7 Hard line breaks* in `CommonMark`](https://spec.commonmark.org/0.31/#hard-line-breaks)
 //!
 //! [string]: crate::construct::string
 //! [text]: crate::construct::text

src/construct/raw_flow.rs

@@ -124,7 +124,7 @@
 //!
 //! *   [`code-fenced.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-fenced.js)
 //! *   [`micromark-extension-math`](https://github.com/micromark/micromark-extension-math)
-//! *   [*§ 4.5 Fenced code blocks* in `CommonMark`](https://spec.commonmark.org/0.30/#fenced-code-blocks)
+//! *   [*§ 4.5 Fenced code blocks* in `CommonMark`](https://spec.commonmark.org/0.31/#fenced-code-blocks)
 //!
 //! > 👉 **Note**: math is not specified anywhere.
 //!

src/construct/raw_text.rs

@@ -104,7 +104,7 @@
 //!
 //! *   [`code-text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-text.js)
 //! *   [`micromark-extension-math`](https://github.com/micromark/micromark-extension-math)
-//! *   [*§ 6.1 Code spans* in `CommonMark`](https://spec.commonmark.org/0.30/#code-spans)
+//! *   [*§ 6.1 Code spans* in `CommonMark`](https://spec.commonmark.org/0.31/#code-spans)
 //!
 //! > 👉 **Note**: math is not specified anywhere.
 //!

src/construct/thematic_break.rs

@@ -49,7 +49,7 @@
 //! ## References
 //!
 //! *   [`thematic-break.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/thematic-break.js)
-//! *   [*§ 4.1 Thematic breaks* in `CommonMark`](https://spec.commonmark.org/0.30/#thematic-breaks)
+//! *   [*§ 4.1 Thematic breaks* in `CommonMark`](https://spec.commonmark.org/0.31/#thematic-breaks)
 //!
 //! [flow]: crate::construct::flow
 //! [heading_setext]: crate::construct::heading_setext

src/state.rs

@@ -283,10 +283,9 @@ pub enum Name {
     HtmlTextCdataClose,
     HtmlTextCdataEnd,
     HtmlTextCommentOpenInside,
-    HtmlTextCommentStart,
-    HtmlTextCommentStartDash,
     HtmlTextComment,
     HtmlTextCommentClose,
+    HtmlTextCommentEnd,
     HtmlTextDeclaration,
     HtmlTextEnd,
     HtmlTextInstruction,
@@ -767,10 +766,9 @@ pub fn call(tokenizer: &mut Tokenizer, name: Name) -> State {
         Name::HtmlTextCdataClose => construct::html_text::cdata_close,
         Name::HtmlTextCdataEnd => construct::html_text::cdata_end,
         Name::HtmlTextCommentOpenInside => construct::html_text::comment_open_inside,
-        Name::HtmlTextCommentStart => construct::html_text::comment_start,
-        Name::HtmlTextCommentStartDash => construct::html_text::comment_start_dash,
         Name::HtmlTextComment => construct::html_text::comment,
         Name::HtmlTextCommentClose => construct::html_text::comment_close,
+        Name::HtmlTextCommentEnd => construct::html_text::comment_end,
         Name::HtmlTextDeclaration => construct::html_text::declaration,
         Name::HtmlTextEnd => construct::html_text::end,
         Name::HtmlTextInstruction => construct::html_text::instruction,

src/util/character_reference.rs

@@ -35,7 +35,7 @@ use core::str;
 /// ## References
 ///
 /// *   [`wooorm/decode-named-character-reference`](https://github.com/wooorm/decode-named-character-reference)
-/// *   [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.30/#entity-and-numeric-character-references)
+/// *   [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.31/#entity-and-numeric-character-references)
 pub fn decode_named(value: &str, html5: bool) -> Option<String> {
     let mut iter = if html5 {
         CHARACTER_REFERENCES.iter()
@@ -78,7 +78,7 @@ pub fn decode_named(value: &str, html5: bool) -> Option<String> {
 /// ## References
 ///
 /// *   [`micromark-util-decode-numeric-character-reference` in `micromark`](https://github.com/micromark/micromark/tree/main/packages/micromark-util-decode-numeric-character-reference)
-/// *   [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.30/#entity-and-numeric-character-references)
+/// *   [*§ 2.5 Entity and numeric character references* in `CommonMark`](https://spec.commonmark.org/0.31/#entity-and-numeric-character-references)
 pub fn decode_numeric(value: &str, radix: u32) -> String {
     if let Some(char) = char::from_u32(u32::from_str_radix(value, radix).unwrap()) {
         if !matches!(char,