Merge pull request #248 from kevinmehall/closure

Wrap user-supplied code in immediately-invoked closure

peg-macros/ast.rs

@@ -1,4 +1,4 @@
-use proc_macro2::{Ident, Literal, TokenStream};
+use proc_macro2::{Ident, Literal, Group, TokenStream};
 
 #[derive(Debug)]
 pub struct Grammar {
@@ -58,19 +58,15 @@ pub struct TaggedExpr {
 #[derive(Debug, Clone)]
 pub enum Expr {
     LiteralExpr(Literal),
-    PatternExpr(TokenStream),
+    PatternExpr(Group),
     RuleExpr(Ident, Vec<RuleArg>),
     MethodExpr(Ident, TokenStream),
     ChoiceExpr(Vec<Expr>),
     OptionalExpr(Box<Expr>),
     Repeat(Box<Expr>, BoundedRepeat, /*sep*/ Option<Box<Expr>>),
     PosAssertExpr(Box<Expr>),
     NegAssertExpr(Box<Expr>),
-    ActionExpr(
-        Vec<TaggedExpr>,
-        /*action*/ Option<TokenStream>,
-        /*cond*/ bool,
-    ),
+    ActionExpr(Vec<TaggedExpr>, Option<Group>),
     MatchStrExpr(Box<Expr>),
     PositionExpr,
     QuietExpr(Box<Expr>),
@@ -95,7 +91,7 @@ pub struct PrecedenceLevel {
 #[derive(Debug, Clone)]
 pub struct PrecedenceOperator {
     pub elements: Vec<TaggedExpr>,
-    pub action: TokenStream,
+    pub action: Group,
 }
 
 #[derive(Debug, Clone)]

peg-macros/grammar.rustpeg

@@ -3,7 +3,7 @@ pub grammar peg() for FlatTokenStream {
 use crate::ast::*;
 use crate::ast::Expr::*;
 use crate::tokens::FlatTokenStream;
-use proc_macro2::{ TokenStream, Ident, Literal, Delimiter };
+use proc_macro2::{ TokenStream, Ident, Group, Literal, Delimiter };
 
 pub rule peg_grammar() -> Grammar
     = doc:rust_doc_comment() visibility:rust_visibility() "grammar" name:IDENT() args:grammar_args() "for" input_type:$(rust_type()) "{" items:item()* "}"
@@ -67,19 +67,16 @@ rule choice() -> Expr = s:sequence() ++ "/" {
 }
 
 rule sequence() -> Expr
-     = elements:labeled()* code:code_block()? {
+     = elements:labeled()* code:BRACE_GROUP()? {
          if let Some(code) = code {
-            ActionExpr(elements, Some(code.1), code.0)
+            ActionExpr(elements, Some(code))
          } else if elements.len() != 1 {
-             ActionExpr(elements, None, false)
+             ActionExpr(elements, None)
          } else {
              elements.into_iter().next().unwrap().expr
          }
     }
 
-    rule code_block() -> (bool, TokenStream)
-        = "{" is_cond:"?"? code:##group_body() "}" { (is_cond.is_some(), code) }
-
 rule labeled() -> TaggedExpr
     = label:(l:IDENT() ":" {l})? expression:suffixed()
         { TaggedExpr{ name: label, expr: expression } }
@@ -117,7 +114,7 @@ rule primary() -> Expr
   / p:BRACKET_GROUP() { PatternExpr(p) }
   / "(" "@" ")" { MarkerExpr(true) }
   / "@" { MarkerExpr(false) }
-  / "##" method:IDENT() args:PAREN_GROUP() { MethodExpr(method, args) }
+  / "##" method:IDENT() args:PAREN_GROUP() { MethodExpr(method, args.stream()) }
   / "(" expression:expression() ")" { expression }
 
     rule rule_arg() -> RuleArg
@@ -135,9 +132,9 @@ rule precedence_op() -> PrecedenceOperator
 rule KEYWORD() = "pub" / "crate" / "rule" / "use" / "type"
 rule IDENT() -> Ident = !KEYWORD() i:##ident() {i}
 rule LITERAL() -> Literal = ##literal()
-rule PAREN_GROUP()   -> TokenStream = ##group(Delimiter::Parenthesis)
-rule BRACE_GROUP()   -> TokenStream = ##group(Delimiter::Brace)
-rule BRACKET_GROUP() -> TokenStream = ##group(Delimiter::Bracket)
+rule PAREN_GROUP()   -> Group = ##group(Delimiter::Parenthesis)
+rule BRACE_GROUP()   -> Group = ##group(Delimiter::Brace)
+rule BRACKET_GROUP() -> Group = ##group(Delimiter::Bracket)
 rule LIFETIME() = "'" IDENT()
 rule INTEGER() = LITERAL()
 

peg-macros/tokens.rs

@@ -64,33 +64,14 @@ impl FlatTokenStream {
         }
     }
 
-    pub fn group(&self, pos: usize, delim: Delimiter) -> RuleResult<TokenStream> {
+    pub fn group(&self, pos: usize, delim: Delimiter) -> RuleResult<Group> {
         match self.tokens.get(pos) {
             Some(Token::Begin(g, n)) if g.delimiter() == delim => {
-                RuleResult::Matched(*n, g.stream())
+                RuleResult::Matched(*n, g.clone())
             }
             _ => RuleResult::Failed,
         }
     }
-
-    pub fn group_body(&self, mut pos: usize) -> RuleResult<TokenStream> {
-        let mut r: Vec<TokenTree> = Vec::new();
-        while let Some(t) = self.tokens.get(pos) {
-            match t {
-                Token::Ident(i) => r.push(i.clone().into()),
-                Token::Literal(i) => r.push(i.clone().into()),
-                Token::Punct(i) => r.push(i.clone().into()),
-                Token::Begin(g, n) => {
-                    r.push(g.clone().into());
-                    pos = *n;
-                    continue;
-                }
-                Token::End(..) => break,
-            }
-            pos += 1;
-        }
-        RuleResult::Matched(pos, r.into_iter().collect())
-    }
 }
 
 #[derive(Debug, Clone)]

peg-macros/translate.rs

@@ -1,4 +1,4 @@
-use proc_macro2::{Ident, Span, TokenStream};
+use proc_macro2::{Group, Ident, Span, TokenStream, TokenTree};
 use std::collections::{HashMap, HashSet};
 
 use quote::{format_ident, quote, quote_spanned};
@@ -16,6 +16,17 @@ pub fn report_error_expr(span: Span, msg: String) -> TokenStream {
     quote_spanned!(span=> { compile_error!(#msg); panic!() })
 }
 
+/// Test if the group begins with a specific marker character, and if so, return the remaining tokens.
+fn group_check_prefix(group: &Group, prefix: char) -> Option<TokenStream> {
+    let mut iter = group.stream().into_iter();
+    match iter.next() {
+        Some(TokenTree::Punct(p)) if p.as_char() == prefix => {
+            Some(iter.collect())
+        }
+        _ => None
+    }
+}
+
 fn extra_args_def(grammar: &Grammar) -> TokenStream {
     let args: Vec<TokenStream> = grammar
         .args
@@ -351,8 +362,9 @@ fn compile_expr(context: &Context, e: &Expr, result_used: bool) -> TokenStream {
             }}
         }
 
-        PatternExpr(ref pattern) => {
+        PatternExpr(ref pattern_group) => {
             let invert = false;
+            let pattern = pattern_group.stream();
             let pat_str = pattern.to_string();
 
             let (in_set, not_in_set) = cond_swap(
@@ -610,19 +622,24 @@ fn compile_expr(context: &Context, e: &Expr, result_used: bool) -> TokenStream {
             }}
         }
 
-        ActionExpr(ref exprs, ref code, is_cond) => labeled_seq(context, &exprs, {
-            if *is_cond {
-                quote! {
-                    match { #code } {
-                        Ok(res) => ::peg::RuleResult::Matched(__pos, res),
-                        Err(expected) => {
-                            __err_state.mark_failure(__pos, expected);
-                            ::peg::RuleResult::Failed
-                        },
+        ActionExpr(ref exprs, ref code) => labeled_seq(context, &exprs, {
+            if let Some(code) = code {
+                // Peek and see if the first token in the block is '?'. If so, it's a conditional block
+                if let Some(body) = group_check_prefix(&code, '?') {
+                    quote_spanned!{code.span() =>
+                        match (||{ #body })() {
+                            Ok(res) => ::peg::RuleResult::Matched(__pos, res),
+                            Err(expected) => {
+                                __err_state.mark_failure(__pos, expected);
+                                ::peg::RuleResult::Failed
+                            },
+                        }
                     }
+                } else {
+                    quote_spanned!{code.span() => ::peg::RuleResult::Matched(__pos, (||#code)()) }
                 }
             } else {
-                quote! { ::peg::RuleResult::Matched(__pos, { #code }) }
+                quote!(::peg::RuleResult::Matched(__pos, ()))
             }
         }),
         MatchStrExpr(ref expr) => {
@@ -654,7 +671,7 @@ fn compile_expr(context: &Context, e: &Expr, result_used: bool) -> TokenStream {
         PrecedenceExpr { ref levels } => {
             let mut pre_rules = Vec::new();
             let mut level_code = Vec::new();
-            let mut span_capture = None;
+            let mut span_capture: Option<(TokenStream, TokenStream, TokenStream, &Group)> = None;
 
             for (prec, level) in levels.iter().enumerate() {
                 let prec = prec as i32;
@@ -673,12 +690,14 @@ fn compile_expr(context: &Context, e: &Expr, result_used: bool) -> TokenStream {
                     let r_arg = name_or_ignore(right_arg.name.as_ref());
 
                     let action = &op.action;
+                    let action = quote_spanned!(op.action.span()=>(||#action)());
+
                     let action = if let Some((lpos_name, val_name, rpos_name, wrap_action)) =
                         &span_capture
                     {
-                        quote!((|#lpos_name, #val_name, #rpos_name| {#wrap_action})(__lpos, #action, __pos))
+                        quote_spanned!(wrap_action.span()=> (|#lpos_name, #val_name, #rpos_name|#wrap_action)(__lpos, #action, __pos))
                     } else {
-                        quote!({#action})
+                        action
                     };
 
                     match (&left_arg.expr, &right_arg.expr) {

tests/compile-fail/rust_action_type_error.stderr

@@ -1,5 +1,5 @@
 error[E0308]: mismatched types
- --> $DIR/rust_action_type_error.rs:7:29
+ --> $DIR/rust_action_type_error.rs:7:27
   |
 7 |     rule foo() -> X = "a" { Y } //~ ERROR
-  |                             ^ expected struct `X`, found struct `Y`
+  |                           ^^^^^ expected struct `X`, found struct `Y`

tests/run-pass/conditional_block.rs

@@ -28,6 +28,13 @@ peg::parser!( grammar parse() for str {
             }
         }
 
+    pub rule return_early() -> i32 = vs:$([_]+) {?
+        let v = vs.parse::<i32>().map_err(|_| "number")?;
+        if v > 100 {
+            return Err("smaller number");
+        }
+        Ok(v)
+    }
 });
 
 fn main() {
@@ -41,4 +48,8 @@ fn main() {
     assert!(parse::xml("<a><b></b><c></c></a>").is_ok());
     assert!(parse::xml("<a><b><c></b></c></a>").is_err());
     assert!(parse::xml("<a><b></c><c></b></a>").is_err());
+
+    assert!(parse::return_early("a").unwrap_err().expected.tokens().any(|e| e == "number"));
+    assert!(parse::return_early("123").unwrap_err().expected.tokens().any(|e| e == "smaller number"));
+    assert_eq!(parse::return_early("99").unwrap(), 99);
 }