Skip to content

Commit

Permalink
feature: unary prefix/suffix operator support via PrattParser (#710)
Browse files Browse the repository at this point in the history
Closes #461

based on #344

Co-authored-by: Tomas Tauber <me@tomtau.be>
Co-authored-by: Klas Segeljakt <klasseg@kth.se>
  • Loading branch information
3 people committed Oct 1, 2022
1 parent 4a4a398 commit 9f7e4f7
Show file tree
Hide file tree
Showing 13 changed files with 592 additions and 43 deletions.
6 changes: 3 additions & 3 deletions derive/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "pest_derive"
description = "pest's derive macro"
version = "2.3.1"
version = "2.4.0"
edition = "2018"
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
homepage = "https://pest-parser.github.io/"
Expand All @@ -23,5 +23,5 @@ std = ["pest/std", "pest_generator/std"]

[dependencies]
# for tests, included transitively anyway
pest = { path = "../pest", version = "2.3.1", default-features = false }
pest_generator = { path = "../generator", version = "2.3.1", default-features = false }
pest = { path = "../pest", version = "2.4.0", default-features = false }
pest_generator = { path = "../generator", version = "2.4.0", default-features = false }
16 changes: 16 additions & 0 deletions derive/examples/calc.pest
@@ -0,0 +1,16 @@
WHITESPACE = _{ " " | "\t" | NEWLINE }

program = { SOI ~ expr ~ EOI }
expr = { prefix* ~ primary ~ postfix* ~ (infix ~ prefix* ~ primary ~ postfix* )* }
infix = _{ add | sub | mul | div | pow }
add = { "+" } // Addition
sub = { "-" } // Subtraction
mul = { "*" } // Multiplication
div = { "/" } // Division
pow = { "^" } // Exponentiation
prefix = _{ neg }
neg = { "-" } // Negation
postfix = _{ fac }
fac = { "!" } // Factorial
primary = _{ int | "(" ~ expr ~ ")" }
int = @{ (ASCII_NONZERO_DIGIT ~ ASCII_DIGIT+ | ASCII_DIGIT) }
109 changes: 109 additions & 0 deletions derive/examples/calc.rs
@@ -0,0 +1,109 @@
mod parser {
use pest_derive::Parser;

#[derive(Parser)]
#[grammar = "../examples/calc.pest"]
pub struct Parser;
}

use parser::Rule;
use pest::{
iterators::Pairs,
pratt_parser::{Assoc::*, Op, PrattParser},
Parser,
};
use std::io::{stdin, stdout, Write};

fn parse_to_str(pairs: Pairs<Rule>, pratt: &PrattParser<Rule>) -> String {
pratt
.map_primary(|primary| match primary.as_rule() {
Rule::int => primary.as_str().to_owned(),
Rule::expr => parse_to_str(primary.into_inner(), pratt),
_ => unreachable!(),
})
.map_prefix(|op, rhs| match op.as_rule() {
Rule::neg => format!("(-{})", rhs),
_ => unreachable!(),
})
.map_postfix(|lhs, op| match op.as_rule() {
Rule::fac => format!("({}!)", lhs),
_ => unreachable!(),
})
.map_infix(|lhs, op, rhs| match op.as_rule() {
Rule::add => format!("({}+{})", lhs, rhs),
Rule::sub => format!("({}-{})", lhs, rhs),
Rule::mul => format!("({}*{})", lhs, rhs),
Rule::div => format!("({}/{})", lhs, rhs),
Rule::pow => format!("({}^{})", lhs, rhs),
_ => unreachable!(),
})
.parse(pairs)
}

fn parse_to_i32(pairs: Pairs<Rule>, pratt: &PrattParser<Rule>) -> i128 {
pratt
.map_primary(|primary| match primary.as_rule() {
Rule::int => primary.as_str().parse().unwrap(),
Rule::expr => parse_to_i32(primary.into_inner(), pratt),
_ => unreachable!(),
})
.map_prefix(|op, rhs| match op.as_rule() {
Rule::neg => -rhs,
_ => unreachable!(),
})
.map_postfix(|lhs, op| match op.as_rule() {
Rule::fac => (1..lhs + 1).product(),
_ => unreachable!(),
})
.map_infix(|lhs, op, rhs| match op.as_rule() {
Rule::add => lhs + rhs,
Rule::sub => lhs - rhs,
Rule::mul => lhs * rhs,
Rule::div => lhs / rhs,
Rule::pow => (1..rhs + 1).map(|_| lhs).product(),
_ => unreachable!(),
})
.parse(pairs)
}

fn main() {
let pratt = PrattParser::new()
.op(Op::infix(Rule::add, Left) | Op::infix(Rule::sub, Left))
.op(Op::infix(Rule::mul, Left) | Op::infix(Rule::div, Left))
.op(Op::infix(Rule::pow, Right))
.op(Op::postfix(Rule::fac))
.op(Op::prefix(Rule::neg));

let stdin = stdin();
let mut stdout = stdout();

loop {
let source = {
print!("> ");
let _ = stdout.flush();
let mut input = String::new();
let _ = stdin.read_line(&mut input);
input.trim().to_string()
};

let pairs = match parser::Parser::parse(Rule::program, &source) {
Ok(mut parse_tree) => {
parse_tree
.next()
.unwrap()
.into_inner() // inner of program
.next()
.unwrap()
.into_inner() // inner of expr
}
Err(err) => {
println!("Failed parsing input: {:}", err);
continue;
}
};

print!("{} => ", source);
print!("{} => ", parse_to_str(pairs.clone(), &pratt));
println!("{}", parse_to_i32(pairs.clone(), &pratt));
}
}
6 changes: 3 additions & 3 deletions generator/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "pest_generator"
description = "pest code generator"
version = "2.3.1"
version = "2.4.0"
edition = "2018"
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
homepage = "https://pest-parser.github.io/"
Expand All @@ -18,8 +18,8 @@ default = ["std"]
std = ["pest/std"]

[dependencies]
pest = { path = "../pest", version = "2.3.1", default-features = false }
pest_meta = { path = "../meta", version = "2.3.1" }
pest = { path = "../pest", version = "2.4.0", default-features = false }
pest_meta = { path = "../meta", version = "2.4.0" }
proc-macro2 = "1.0"
quote = "1.0"
syn = "1.0"
6 changes: 3 additions & 3 deletions grammars/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "pest_grammars"
description = "pest popular grammar implementations"
version = "2.3.1"
version = "2.4.0"
edition = "2018"
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
homepage = "https://pest-parser.github.io/"
Expand All @@ -14,8 +14,8 @@ readme = "_README.md"
rust-version = "1.56"

[dependencies]
pest = { path = "../pest", version = "2.3.1" }
pest_derive = { path = "../derive", version = "2.3.1" }
pest = { path = "../pest", version = "2.4.0" }
pest_derive = { path = "../derive", version = "2.4.0" }

[dev-dependencies]
criterion = "0.3"
Expand Down
4 changes: 2 additions & 2 deletions meta/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "pest_meta"
description = "pest meta language parser and validator"
version = "2.3.1"
version = "2.4.0"
edition = "2018"
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
homepage = "https://pest-parser.github.io/"
Expand All @@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
rust-version = "1.56"

[dependencies]
pest = { path = "../pest", version = "2.3.1" }
pest = { path = "../pest", version = "2.4.0" }
once_cell = "1.8.0"

[build-dependencies]
Expand Down
29 changes: 14 additions & 15 deletions meta/src/parser.rs
Expand Up @@ -12,7 +12,7 @@ use std::iter::Peekable;

use pest::error::{Error, ErrorVariant};
use pest::iterators::{Pair, Pairs};
use pest::prec_climber::{Assoc, Operator, PrecClimber};
use pest::pratt_parser::{Assoc, Op, PrattParser};
use pest::{Parser, Span};

use crate::ast::{Expr, Rule as AstRule, RuleType};
Expand Down Expand Up @@ -175,10 +175,9 @@ pub fn consume_rules(pairs: Pairs<Rule>) -> Result<Vec<AstRule>, Vec<Error<Rule>
}

fn consume_rules_with_spans(pairs: Pairs<Rule>) -> Result<Vec<ParserRule>, Vec<Error<Rule>>> {
let climber = PrecClimber::new(vec![
Operator::new(Rule::choice_operator, Assoc::Left),
Operator::new(Rule::sequence_operator, Assoc::Left),
]);
let pratt = PrattParser::new()
.op(Op::infix(Rule::choice_operator, Assoc::Left))
.op(Op::infix(Rule::sequence_operator, Assoc::Left));

pairs
.filter(|pair| pair.as_rule() == Rule::grammar_rule)
Expand Down Expand Up @@ -210,7 +209,7 @@ fn consume_rules_with_spans(pairs: Pairs<Rule>) -> Result<Vec<ParserRule>, Vec<E
inner_nodes.next().unwrap();
}

let node = consume_expr(inner_nodes, &climber)?;
let node = consume_expr(inner_nodes, &pratt)?;

Ok(ParserRule {
name,
Expand All @@ -224,17 +223,17 @@ fn consume_rules_with_spans(pairs: Pairs<Rule>) -> Result<Vec<ParserRule>, Vec<E

fn consume_expr<'i>(
pairs: Peekable<Pairs<'i, Rule>>,
climber: &PrecClimber<Rule>,
pratt: &PrattParser<Rule>,
) -> Result<ParserNode<'i>, Vec<Error<Rule>>> {
fn unaries<'i>(
mut pairs: Peekable<Pairs<'i, Rule>>,
climber: &PrecClimber<Rule>,
pratt: &PrattParser<Rule>,
) -> Result<ParserNode<'i>, Vec<Error<Rule>>> {
let pair = pairs.next().unwrap();

let node = match pair.as_rule() {
Rule::opening_paren => {
let node = unaries(pairs, climber)?;
let node = unaries(pairs, pratt)?;
let end = node.span.end_pos();

ParserNode {
Expand All @@ -243,7 +242,7 @@ fn consume_expr<'i>(
}
}
Rule::positive_predicate_operator => {
let node = unaries(pairs, climber)?;
let node = unaries(pairs, pratt)?;
let end = node.span.end_pos();

ParserNode {
Expand All @@ -252,7 +251,7 @@ fn consume_expr<'i>(
}
}
Rule::negative_predicate_operator => {
let node = unaries(pairs, climber)?;
let node = unaries(pairs, pratt)?;
let end = node.span.end_pos();

ParserNode {
Expand All @@ -262,14 +261,14 @@ fn consume_expr<'i>(
}
other_rule => {
let node = match other_rule {
Rule::expression => consume_expr(pair.into_inner().peekable(), climber)?,
Rule::expression => consume_expr(pair.into_inner().peekable(), pratt)?,
Rule::_push => {
let start = pair.clone().as_span().start_pos();
let mut pairs = pair.into_inner();
pairs.next().unwrap(); // opening_paren
let pair = pairs.next().unwrap();

let node = consume_expr(pair.into_inner().peekable(), climber)?;
let node = consume_expr(pair.into_inner().peekable(), pratt)?;
let end = node.span.end_pos();

ParserNode {
Expand Down Expand Up @@ -529,7 +528,7 @@ fn consume_expr<'i>(
Ok(node)
}

let term = |pair: Pair<'i, Rule>| unaries(pair.into_inner().peekable(), climber);
let term = |pair: Pair<'i, Rule>| unaries(pair.into_inner().peekable(), pratt);
let infix = |lhs: Result<ParserNode<'i>, Vec<Error<Rule>>>,
op: Pair<'i, Rule>,
rhs: Result<ParserNode<'i>, Vec<Error<Rule>>>| match op.as_rule() {
Expand Down Expand Up @@ -560,7 +559,7 @@ fn consume_expr<'i>(
_ => unreachable!(),
};

climber.climb(pairs, term, infix)
pratt.map_primary(term).map_infix(infix).parse(pairs)
}

fn unescape(string: &str) -> Option<String> {
Expand Down
12 changes: 6 additions & 6 deletions meta/src/validator.rs
Expand Up @@ -423,7 +423,7 @@ fn left_recursion<'a, 'i: 'a>(rules: HashMap<String, &'a ParserNode<'i>>) -> Vec
return Some(Error::new_from_span(
ErrorVariant::CustomError {
message: format!(
"rule {} is left-recursive ({}); pest::prec_climber might be useful \
"rule {} is left-recursive ({}); pest::pratt_parser might be useful \
in this case",
node.span.as_str(),
chain
Expand Down Expand Up @@ -677,7 +677,7 @@ mod tests {
1 | a = { a }
| ^
|
= rule a is left-recursive (a -> a); pest::prec_climber might be useful in this case")]
= rule a is left-recursive (a -> a); pest::pratt_parser might be useful in this case")]
fn simple_left_recursion() {
let input = "a = { a }";
unwrap_or_report(consume_rules(
Expand All @@ -693,14 +693,14 @@ mod tests {
1 | a = { b } b = { a }
| ^
|
= rule b is left-recursive (b -> a -> b); pest::prec_climber might be useful in this case
= rule b is left-recursive (b -> a -> b); pest::pratt_parser might be useful in this case
--> 1:17
|
1 | a = { b } b = { a }
| ^
|
= rule a is left-recursive (a -> b -> a); pest::prec_climber might be useful in this case")]
= rule a is left-recursive (a -> b -> a); pest::pratt_parser might be useful in this case")]
fn indirect_left_recursion() {
let input = "a = { b } b = { a }";
unwrap_or_report(consume_rules(
Expand All @@ -716,7 +716,7 @@ mod tests {
1 | a = { \"\" ~ \"a\"? ~ \"a\"* ~ (\"a\" | \"\") ~ a }
| ^
|
= rule a is left-recursive (a -> a); pest::prec_climber might be useful in this case")]
= rule a is left-recursive (a -> a); pest::pratt_parser might be useful in this case")]
fn non_failing_left_recursion() {
let input = "a = { \"\" ~ \"a\"? ~ \"a\"* ~ (\"a\" | \"\") ~ a }";
unwrap_or_report(consume_rules(
Expand All @@ -732,7 +732,7 @@ mod tests {
1 | a = { \"a\" | a }
| ^
|
= rule a is left-recursive (a -> a); pest::prec_climber might be useful in this case")]
= rule a is left-recursive (a -> a); pest::pratt_parser might be useful in this case")]
fn non_primary_choice_left_recursion() {
let input = "a = { \"a\" | a }";
unwrap_or_report(consume_rules(
Expand Down
2 changes: 1 addition & 1 deletion pest/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "pest"
description = "The Elegant Parser"
version = "2.3.1"
version = "2.4.0"
edition = "2018"
authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
homepage = "https://pest-parser.github.io/"
Expand Down
6 changes: 6 additions & 0 deletions pest/src/lib.rs
Expand Up @@ -90,6 +90,12 @@ mod macros;
mod parser;
mod parser_state;
mod position;
pub mod pratt_parser;
#[deprecated(
since = "2.4.0",
note = "Use `pest::pratt_parser` instead (it is an equivalent which also supports unary prefix/suffix operators).
While prec_climber is going to be kept in 2.x minor and patch releases, it may be removed in a future major release."
)]
pub mod prec_climber;
mod span;
mod stack;
Expand Down

0 comments on commit 9f7e4f7

Please sign in to comment.