Add support for flow JSX, expressions after JSX, expressions

Related-to: micromark/micromark-extension-mdx-jsx@f68f69d
Related-to: micromark/micromark-extension-mdx-expression@d5d92b9

src/configuration.rs

@@ -1068,6 +1068,8 @@ pub struct ParseOptions {
     /// However, single dollars can interfere with “normal” dollars in text.
     /// Pass `false`, to only allow math (text) to form when two or more
     /// dollars are used.
+    /// If you pass `false`, you can still use two or more dollars for text
+    /// math.
     ///
     /// ## Examples
     ///

src/construct/mdx_expression_flow.rs

@@ -107,12 +107,83 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
 ///               ^
 /// ```
 pub fn end(tokenizer: &mut Tokenizer) -> State {
-    tokenizer.concrete = false;
-    tokenizer.tokenize_state.token_1 = Name::Data;
+    // We want to allow tags directly after expressions.
+    //
+    // This case is useful:
+    //
+    // ```mdx
+    // <a>{b}</a>
+    // ```
+    //
+    // This case is not (very?) useful:
+    //
+    // ```mdx
+    // {a}<b/>
+    // ```
+    //
+    // …but it would be tougher than needed to disallow.
+    //
+    // To allow that, here we call the MDX JSX flow construct, and there we
+    // call this one.
+    //
+    // It would introduce a cyclical interdependency if we test JSX and
+    // expressions here.
+    // Because the JSX extension already uses parts of this monorepo, we
+    // instead test it there.
+    //
+    // Note: in the JS version of micromark, arbitrary extensions could be
+    // loaded.
+    // Here we know that only our own construct `mdx_expression_flow` can be
+    // enabled.
 
-    if matches!(tokenizer.current, None | Some(b'\n')) {
-        State::Ok
-    } else {
-        State::Nok
+    // if matches!(tokenizer.current, None | Some(b'\n')) {
+    //     State::Ok
+    // } else {
+    //     State::Nok
+    // }
+    match tokenizer.current {
+        None | Some(b'\n') => {
+            reset(tokenizer);
+            State::Ok
+        }
+        // Tag.
+        Some(b'<') if tokenizer.parse_state.options.constructs.mdx_jsx_flow => {
+            // We can’t just say: fine.
+            // Lines of blocks have to be parsed until an eol/eof.
+            tokenizer.attempt(
+                State::Next(StateName::MdxExpressionFlowAfter),
+                State::Next(StateName::MdxExpressionFlowNok),
+            );
+            State::Retry(StateName::MdxJsxStart)
+        }
+        // // An expression.
+        // Some(b'{') if tokenizer.parse_state.options.constructs.mdx_expression_flow => {
+        //     tokenizer.attempt(
+        //         State::Next(StateName::MdxExpressionFlowAfter),
+        //         State::Next(StateName::MdxExpressionFlowNok),
+        //     );
+        //     State::Retry(StateName::MdxExpressionFlowStart)
+        // }
+        _ => {
+            reset(tokenizer);
+            State::Nok
+        }
     }
 }
+
+/// At something that wasn’t an MDX expression (flow).
+///
+/// ```markdown
+/// > | {A} x
+///     ^
+/// ```
+pub fn nok(tokenizer: &mut Tokenizer) -> State {
+    reset(tokenizer);
+    State::Nok
+}
+
+/// Reset state.
+fn reset(tokenizer: &mut Tokenizer) {
+    tokenizer.concrete = false;
+    tokenizer.tokenize_state.token_1 = Name::Data;
+}

src/construct/mdx_jsx_flow.rs

@@ -112,19 +112,37 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
 ///         ^
 /// ```
 pub fn end(tokenizer: &mut Tokenizer) -> State {
+    // We want to allow expressions directly after tags.
+    // See <https://github.com/micromark/micromark-extension-mdx-expression/blob/d5d92b9/packages/micromark-extension-mdx-expression/dev/lib/syntax.js#L183>
+    // for more info.
+    //
+    // Note: in the JS version of micromark, arbitrary extensions could be
+    // loaded.
+    // Here we know that only our own construct `mdx_expression_flow` can be
+    // enabled.
     match tokenizer.current {
         None | Some(b'\n') => {
             reset(tokenizer);
             State::Ok
         }
-        // Another?
+        // Another tag.
         Some(b'<') => {
+            // We can’t just say: fine.
+            // Lines of blocks have to be parsed until an eol/eof.
             tokenizer.attempt(
                 State::Next(StateName::MdxJsxFlowAfter),
                 State::Next(StateName::MdxJsxFlowNok),
             );
             State::Retry(StateName::MdxJsxStart)
         }
+        // An expression.
+        Some(b'{') if tokenizer.parse_state.options.constructs.mdx_expression_flow => {
+            tokenizer.attempt(
+                State::Next(StateName::MdxJsxFlowAfter),
+                State::Next(StateName::MdxJsxFlowNok),
+            );
+            State::Retry(StateName::MdxExpressionFlowStart)
+        }
         _ => {
             reset(tokenizer);
             State::Nok

src/state.rs

@@ -357,6 +357,7 @@ pub enum Name {
     MdxExpressionFlowBefore,
     MdxExpressionFlowAfter,
     MdxExpressionFlowEnd,
+    MdxExpressionFlowNok,
 
     MdxExpressionStart,
     MdxExpressionBefore,
@@ -843,6 +844,7 @@ pub fn call(tokenizer: &mut Tokenizer, name: Name) -> State {
         Name::MdxExpressionFlowBefore => construct::mdx_expression_flow::before,
         Name::MdxExpressionFlowAfter => construct::mdx_expression_flow::after,
         Name::MdxExpressionFlowEnd => construct::mdx_expression_flow::end,
+        Name::MdxExpressionFlowNok => construct::mdx_expression_flow::nok,
 
         Name::MdxExpressionTextStart => construct::mdx_expression_text::start,
         Name::MdxExpressionTextAfter => construct::mdx_expression_text::after,

tests/mdx_jsx_flow.rs

@@ -1,10 +1,12 @@
+mod test_utils;
 use markdown::{
     mdast::{List, ListItem, MdxJsxFlowElement, Node, Paragraph, Root, Text},
     to_html_with_options, to_mdast,
     unist::Position,
     Constructs, Options, ParseOptions,
 };
 use pretty_assertions::assert_eq;
+use test_utils::swc::{parse_esm, parse_expression};
 
 #[test]
 fn mdx_jsx_flow_agnostic() -> Result<(), String> {
@@ -226,3 +228,116 @@ fn mdx_jsx_flow_essence() -> Result<(), String> {
 
     Ok(())
 }
+
+// Flow is mostly the same as `text`, so we only test the relevant
+// differences.
+#[test]
+fn mdx_jsx_flow_interleaving_with_expressions() -> Result<(), String> {
+    let mdx = Options {
+        parse: ParseOptions::mdx(),
+        ..Default::default()
+    };
+    let swc = Options {
+        parse: ParseOptions {
+            constructs: Constructs::mdx(),
+            mdx_esm_parse: Some(Box::new(parse_esm)),
+            mdx_expression_parse: Some(Box::new(parse_expression)),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+
+    assert_eq!(
+        to_html_with_options("<div>\n{1}\n</div>", &mdx)?,
+        "",
+        "should support tags and expressions (unaware)"
+    );
+
+    assert_eq!(
+        to_html_with_options("<div>\n{'}'}\n</div>", &swc)?,
+        "",
+        "should support tags and expressions (aware)"
+    );
+
+    assert_eq!(
+        to_html_with_options("x<em>{1}</em>", &swc)?,
+        "<p>x</p>",
+        "should support tags and expressions with text before (text)"
+    );
+
+    assert_eq!(
+        to_html_with_options("<em>x{1}</em>", &swc)?,
+        "<p>x</p>",
+        "should support tags and expressions with text between, early (text)"
+    );
+
+    assert_eq!(
+        to_html_with_options("<em>{1}x</em>", &swc)?,
+        "<p>x</p>",
+        "should support tags and expressions with text between, late (text)"
+    );
+
+    assert_eq!(
+        to_html_with_options("<em>{1}</em>x", &swc)?,
+        "<p>x</p>",
+        "should support tags and expressions with text after (text)"
+    );
+
+    assert_eq!(
+        to_html_with_options("<x/>{1}", &swc)?,
+        "",
+        "should support a tag and then an expression (flow)"
+    );
+
+    assert_eq!(
+        to_html_with_options("<x/>{1}x", &swc)?,
+        "<p>x</p>",
+        "should support a tag, an expression, then text (text)"
+    );
+
+    assert_eq!(
+        to_html_with_options("x<x/>{1}", &swc)?,
+        "<p>x</p>",
+        "should support text, a tag, then an expression (text)"
+    );
+
+    assert_eq!(
+        to_html_with_options("{1}<x/>", &swc)?,
+        "",
+        "should support an expression and then a tag (flow)"
+    );
+
+    assert_eq!(
+        to_html_with_options("{1}<x/>x", &swc)?,
+        "<p>x</p>",
+        "should support an expression, a tag, then text (text)"
+    );
+
+    assert_eq!(
+        to_html_with_options("x{1}<x/>", &swc)?,
+        "<p>x</p>",
+        "should support text, an expression, then a tag (text)"
+    );
+
+    assert_eq!(
+        to_html_with_options("<x>{[\n'',\n{c:''}\n]}</x>", &swc)?,
+        "",
+        "should nicely interleaf (micromark/micromark-extension-mdx-jsx#9)"
+    );
+
+    assert_eq!(
+        to_html_with_options(
+            "
+<style>{`
+  .foo {}
+  .bar {}
+`}</style>
+    ",
+            &swc
+        )?,
+        "",
+        "should nicely interleaf (mdx-js/mdx#1945)"
+    );
+
+    Ok(())
+}