Add to support /// and //! syntax for add doc comment for rules.

Resolve #748

For example:

```rust
//! A parser for JSON file.

/// Matches object, e.g.: `{ "foo": "bar" }`
object = { "{" ~ pair ~ ("," ~ pair)* ~ "}" | "{" ~ "}" }
```

should generate:

```rust
/// A parser for JSON file.
enum Rule {
    /// Matches object, e.g.: `{ "foo": "bar" }`
    object,
}
```

generator/src/generator.rs

@@ -23,6 +23,7 @@ pub fn generate(
     path: Option<PathBuf>,
     rules: Vec<OptimizedRule>,
     defaults: Vec<&str>,
+    grammar_docs: Vec<&str>,
     include_grammar: bool,
 ) -> TokenStream {
     let uses_eoi = defaults.iter().any(|name| *name == "EOI");
@@ -36,7 +37,7 @@ pub fn generate(
     } else {
         quote!()
     };
-    let rule_enum = generate_enum(&rules, uses_eoi);
+    let rule_enum = generate_enum(&rules, grammar_docs, uses_eoi);
     let patterns = generate_patterns(&rules, uses_eoi);
     let skip = generate_skip(&rules);
 
@@ -181,10 +182,26 @@ fn generate_include(name: &Ident, path: &str) -> TokenStream {
     }
 }
 
-fn generate_enum(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream {
-    let rules = rules.iter().map(|rule| format_ident!("r#{}", rule.name));
+fn generate_enum(rules: &[OptimizedRule], grammar_docs: Vec<&str>, uses_eoi: bool) -> TokenStream {
+    let rules = rules.iter().map(|rule| {
+        let rule_name = format_ident!("r#{}", rule.name);
+        if rule.comments.is_empty() {
+            quote! {
+                #rule_name
+            }
+        } else {
+            let comments = rule.comments.join("\n");
+            quote! {
+                #[doc = #comments]
+                #rule_name
+            }
+        }
+    });
+
+    let grammar_docs = grammar_docs.join("\n");
     if uses_eoi {
         quote! {
+            #[doc = #grammar_docs]
             #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
             #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
             pub enum Rule {
@@ -194,6 +211,7 @@ fn generate_enum(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream {
         }
     } else {
         quote! {
+            #[doc = #grammar_docs]
             #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
             #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
             pub enum Rule {
@@ -208,6 +226,7 @@ fn generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream {
         .iter()
         .map(|rule| {
             let rule = format_ident!("r#{}", rule.name);
+
             quote! {
                 Rule::#rule => rules::#rule(state)
             }
@@ -484,6 +503,11 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream {
                 state.restore_on_err(|state| #expr)
             }
         }
+        OptimizedExpr::LineDoc(comment) => {
+            quote! {
+                state.skip(#comment.len())
+            }
+        }
     }
 }
 
@@ -616,6 +640,11 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
                 state.restore_on_err(|state| #expr)
             }
         }
+        OptimizedExpr::LineDoc(comment) => {
+            quote! {
+                state.skip(#comment.len())
+            }
+        }
     }
 }
 
@@ -667,14 +696,17 @@ mod tests {
             name: "f".to_owned(),
             ty: RuleType::Normal,
             expr: OptimizedExpr::Ident("g".to_owned()),
+            comments: vec!["This is rule comment".to_owned()],
         }];
 
         assert_eq!(
-            generate_enum(&rules, false).to_string(),
+            generate_enum(&rules, vec!["Rule doc", "hello"], false).to_string(),
             quote! {
+                #[doc = "Rule doc\nhello"]
                 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
                 #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
                 pub enum Rule {
+                    #[doc = "This is rule comment"]
                     r#f
                 }
             }
@@ -966,11 +998,13 @@ mod tests {
                 name: "a".to_owned(),
                 ty: RuleType::Silent,
                 expr: OptimizedExpr::Str("b".to_owned()),
+                comments: vec![],
             },
             OptimizedRule {
                 name: "if".to_owned(),
                 ty: RuleType::Silent,
                 expr: OptimizedExpr::Ident("a".to_owned()),
+                comments: vec!["If statement".to_owned()],
             },
         ];
 
@@ -981,15 +1015,17 @@ mod tests {
         current_dir.push("test.pest");
         let test_path = current_dir.to_str().expect("path contains invalid unicode");
         assert_eq!(
-            generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, true).to_string(),
+            generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, vec!["This is Rule doc", "This is second line"], true).to_string(),
             quote! {
                 #[allow(non_upper_case_globals)]
                 const _PEST_GRAMMAR_MyParser: &'static str = include_str!(#test_path);
 
+                #[doc = "This is Rule doc\nThis is second line"]
                 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
                 #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
                 pub enum Rule {
                     r#a,
+                    #[doc = "If statement"]
                     r#if
                 }
 

generator/src/lib.rs

@@ -26,6 +26,7 @@ use std::fs::File;
 use std::io::{self, Read};
 use std::path::Path;
 
+use pest::iterators::Pairs;
 use proc_macro2::TokenStream;
 use syn::{Attribute, DeriveInput, Generics, Ident, Lit, Meta};
 
@@ -90,11 +91,32 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream {
         Err(error) => panic!("error parsing \n{}", error.renamed_rules(rename_meta_rule)),
     };
 
+    let grammar_docs = consume_grammar_doc(pairs.clone());
+
     let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone()));
     let ast = unwrap_or_report(parser::consume_rules(pairs));
     let optimized = optimizer::optimize(ast);
 
-    generator::generate(name, &generics, path, optimized, defaults, include_grammar)
+    generator::generate(
+        name,
+        &generics,
+        path,
+        optimized,
+        defaults,
+        grammar_docs,
+        include_grammar,
+    )
+}
+
+fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> Vec<&'_ str> {
+    let mut docs = vec![];
+    for pair in pairs {
+        if pair.as_rule() == Rule::grammar_doc {
+            docs.push(pair.as_str()[3..pair.as_str().len()].trim());
+        }
+    }
+
+    docs
 }
 
 fn read_file<P: AsRef<Path>>(path: P) -> io::Result<String> {

grammars/src/grammars/json.pest

@@ -7,8 +7,11 @@
 // option. All files in the project carrying such notice may not be copied,
 // modified, or distributed except according to those terms.
 
+//! A parser for JSON file.
+//! And this is a example for JSON parser.
 json = { SOI ~ (object | array) ~ EOI }
 
+/// Matches object, e.g.: `{ "foo": "bar" }`
 object = { "{" ~ pair ~ ("," ~ pair)* ~ "}" | "{" ~ "}" }
 pair   = { string ~ ":" ~ value }
 

meta/src/ast.rs

@@ -18,6 +18,8 @@ pub struct Rule {
     pub ty: RuleType,
     /// The rule's expression
     pub expr: Expr,
+    /// Doc comments of the rule
+    pub comments: Vec<String>,
 }
 
 /// All possible rule types
@@ -87,6 +89,8 @@ pub enum Expr {
     Skip(Vec<String>),
     /// Matches an expression and pushes it to the stack, e.g. `push(e)`
     Push(Box<Expr>),
+    /// Matches a doc comment, e.g. `/// comment`
+    LineDoc(String),
 }
 
 impl Expr {

meta/src/grammar.pest

@@ -7,11 +7,12 @@
 // option. All files in the project carrying such notice may not be copied,
 // modified, or distributed except according to those terms.
 
-grammar_rules = _{ SOI ~ grammar_rule+ ~ EOI }
+grammar_rules = _{ SOI ~ grammar_doc* ~ (grammar_rule)+ ~ EOI }
 
 grammar_rule = {
     identifier ~ assignment_operator ~ modifier? ~
-    opening_brace ~ expression ~ closing_brace
+    opening_brace ~ expression ~ closing_brace |
+    line_doc
 }
 
 assignment_operator = { "=" }
@@ -92,7 +93,12 @@ quote          = { "\"" }
 single_quote   = { "'" }
 range_operator = { ".." }
 
-newline    = _{ "\n" | "\r\n" }
-WHITESPACE = _{ " " | "\t" | newline }
-block_comment = _{ "/*" ~ (block_comment | !"*/" ~ ANY)* ~ "*/" }
-COMMENT    = _{ block_comment | ("//" ~ (!newline ~ ANY)*) }
+newline        = _{ "\n" | "\r\n" }
+WHITESPACE     = _{ " " | "\t" | newline }
+line_comment   = _{ ("//" ~ !("/" | "!") ~ (!newline ~ ANY)*) }
+block_comment  = _{ "/*" ~ (block_comment | !"*/" ~ ANY)* ~ "*/" }
+COMMENT        = _{ block_comment | line_comment }
+
+// ref: https://doc.rust-lang.org/reference/comments.html
+grammar_doc = ${ "//!" ~ (!newline ~ ANY)* }
+line_doc    = ${ "///" ~ !"/" ~ (!newline ~ ANY)* }

meta/src/optimizer/concatenator.rs

@@ -10,7 +10,12 @@
 use crate::ast::*;
 
 pub fn concatenate(rule: Rule) -> Rule {
-    let Rule { name, ty, expr } = rule;
+    let Rule {
+        name,
+        ty,
+        expr,
+        comments,
+    } = rule;
     Rule {
         name,
         ty,
@@ -29,5 +34,6 @@ pub fn concatenate(rule: Rule) -> Rule {
                 expr
             }
         }),
+        comments,
     }
 }

meta/src/optimizer/factorizer.rs

@@ -10,7 +10,12 @@
 use crate::ast::*;
 
 pub fn factor(rule: Rule) -> Rule {
-    let Rule { name, ty, expr } = rule;
+    let Rule {
+        name,
+        ty,
+        expr,
+        comments,
+    } = rule;
     Rule {
         name,
         ty,
@@ -47,5 +52,6 @@ pub fn factor(rule: Rule) -> Rule {
                 expr => expr,
             }
         }),
+        comments,
     }
 }

meta/src/optimizer/lister.rs

@@ -10,7 +10,12 @@
 use crate::ast::*;
 
 pub fn list(rule: Rule) -> Rule {
-    let Rule { name, ty, expr } = rule;
+    let Rule {
+        name,
+        ty,
+        expr,
+        comments,
+    } = rule;
     Rule {
         name,
         ty,
@@ -38,5 +43,6 @@ pub fn list(rule: Rule) -> Rule {
                 expr => expr,
             }
         }),
+        comments,
     }
 }