feat(md): try to writer markdown parser

quake_core/src/lib.rs

@@ -1,3 +1,4 @@
+#![feature(in_band_lifetimes)]
 extern crate pest;
 #[macro_use]
 extern crate pest_derive;

quake_core/src/markdown/md_processor.rs

@@ -32,31 +32,39 @@ use pulldown_cmark::{CodeBlockKind, CowStr, Event, Options, Parser, Tag};
 use pulldown_cmark_to_cmark::cmark_with_options;
 
 use crate::markdown::references::{NoteReference, RefParser, RefParserState, RefType};
+use crate::markdown::tokenizer::QuakeDown;
 
 pub type MarkdownEvents<'a> = Vec<Event<'a>>;
 
-pub struct QuakeDown {
+pub struct MdProcessor {
     pub pieces: Vec<MdStruct>,
 }
 
-impl Default for QuakeDown {
+impl Default for MdProcessor {
     fn default() -> Self {
-        QuakeDown { pieces: vec![] }
+        MdProcessor { pieces: vec![] }
     }
 }
 
-impl QuakeDown {
+impl MdProcessor {
     pub fn transform(content: &str) -> Result<String, Box<dyn Error>> {
-        let mut down = QuakeDown::default();
-        let events = down.tokenizer(&content)?;
+        let mut down = MdProcessor::default();
+        let events = down.add_custom_syntax(&content)?;
 
         let mapping = events.into_iter().map(event_to_owned).collect();
 
         Ok(events_to_text(mapping))
     }
 
+    pub fn token_it(content: &str) {
+        QuakeDown::from(content);
+    }
+
     // based on https://github.com/zoni/obsidian-export/blob/main/src/lib.rs
-    fn tokenizer<'a>(&mut self, content: &'a str) -> Result<Vec<Event<'a>>, Box<dyn Error>> {
+    fn add_custom_syntax<'a>(
+        &mut self,
+        content: &'a str,
+    ) -> Result<Vec<Event<'a>>, Box<dyn Error>> {
         let mut parser_options = Options::empty();
         parser_options.insert(Options::ENABLE_TABLES);
         parser_options.insert(Options::ENABLE_FOOTNOTES);
@@ -275,7 +283,7 @@ fn codeblock_kind_to_owned<'a>(codeblock_kind: CodeBlockKind) -> CodeBlockKind<'
 
 #[cfg(test)]
 mod tests {
-    use crate::markdown::md_processor::QuakeDown;
+    use crate::markdown::md_processor::MdProcessor;
     use std::fs;
     use std::path::PathBuf;
 
@@ -289,26 +297,35 @@ mod tests {
         let target = base.join("new.md");
         let expect = fs::read_to_string(target).unwrap();
 
-        let actual = QuakeDown::transform(src.as_str()).unwrap();
+        let actual = MdProcessor::transform(src.as_str()).unwrap();
 
         assert_eq!(actual, expect);
     }
 
     #[test]
     fn br_tag_in_html() {
-        let string = QuakeDown::transform("demo `<br />` demo").unwrap();
+        let string = MdProcessor::transform("demo `<br />` demo").unwrap();
         assert_eq!("demo `<br />` demo", string);
     }
 
     #[test]
     fn transform_page_link() {
-        let string = QuakeDown::transform("[[note::SourceCode]]").unwrap();
+        let string = MdProcessor::transform("[[note::SourceCode]]").unwrap();
         assert_eq!("[note::SourceCode](note::SourceCode)", string);
     }
 
     #[test]
     fn transform_page_file() {
-        let string = QuakeDown::transform("![[note::SourceCode]]").unwrap();
+        let string = MdProcessor::transform("![[note::SourceCode]]").unwrap();
         assert_eq!("[note::SourceCode](note::SourceCode)", string);
     }
+
+    #[test]
+    fn down_code_token() {
+        let code = "```
+console.log('hello,world')
+```";
+
+        MdProcessor::token_it(code);
+    }
 }

quake_core/src/markdown/mod.rs

@@ -1,3 +1,4 @@
 pub mod md_processor;
 pub mod md_struct;
 pub mod references;
+pub mod tokenizer;

quake_core/src/markdown/tokenizer.rs

@@ -0,0 +1,95 @@
+use crate::markdown::md_struct::MdStruct;
+use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag};
+
+pub struct QuakeDown {
+    pub pieces: Vec<MdStruct>,
+    pub end_newline: bool,
+    pub is_in_code_block: bool,
+}
+
+impl Default for QuakeDown {
+    fn default() -> Self {
+        QuakeDown {
+            pieces: vec![],
+            end_newline: false,
+            is_in_code_block: false,
+        }
+    }
+}
+
+impl QuakeDown {
+    pub fn from(content: &str) {
+        let mut parser_options = Options::empty();
+        parser_options.insert(Options::ENABLE_TABLES);
+        parser_options.insert(Options::ENABLE_FOOTNOTES);
+        parser_options.insert(Options::ENABLE_STRIKETHROUGH);
+        parser_options.insert(Options::ENABLE_TASKLISTS);
+        parser_options.insert(Options::ENABLE_SMART_PUNCTUATION);
+        let mut parser = Parser::new_ext(&content, parser_options);
+        let mut down = QuakeDown::default();
+        down.transform(&mut parser);
+    }
+
+    pub fn transform(&mut self, parser: &mut Parser) {
+        while let Some(event) = parser.into_iter().next() {
+            match event {
+                Event::Start(tag) => {
+                    match tag {
+                        Tag::Paragraph => {}
+                        Tag::Heading(_) => {}
+                        Tag::BlockQuote => {}
+                        Tag::CodeBlock(info) => match info {
+                            CodeBlockKind::Indented => {
+                                self.is_in_code_block = true;
+                            }
+                            CodeBlockKind::Fenced(_info) => {
+                                self.is_in_code_block = true;
+                            }
+                        },
+                        Tag::List(_) => {}
+                        Tag::Item => {}
+                        Tag::FootnoteDefinition(_) => {}
+                        Tag::Table(_) => {}
+                        Tag::TableHead => {}
+                        Tag::TableRow => {}
+                        Tag::TableCell => {}
+                        Tag::Emphasis => {}
+                        Tag::Strong => {}
+                        Tag::Strikethrough => {}
+                        Tag::Link(_, _, _) => {}
+                        Tag::Image(_, _, _) => {}
+                    };
+                }
+                Event::End(tag) => {
+                    // something here
+                    match tag {
+                        Tag::Paragraph => {}
+                        Tag::Heading(_) => {}
+                        Tag::BlockQuote => {}
+                        Tag::CodeBlock(_) => {}
+                        Tag::List(_) => {}
+                        Tag::Item => {}
+                        Tag::FootnoteDefinition(_) => {}
+                        Tag::Table(_) => {}
+                        Tag::TableHead => {}
+                        Tag::TableRow => {}
+                        Tag::TableCell => {}
+                        Tag::Emphasis => {}
+                        Tag::Strong => {}
+                        Tag::Strikethrough => {}
+                        Tag::Link(_, _, _) => {}
+                        Tag::Image(_, _, _) => {}
+                    }
+                }
+                Event::Text(_) => {}
+                Event::Code(_) => {}
+                Event::Html(_) => {}
+                Event::FootnoteReference(_) => {}
+                Event::SoftBreak => {}
+                Event::HardBreak => {}
+                Event::Rule => {}
+                Event::TaskListMarker(_) => {}
+            }
+        }
+    }
+}