feature-guard the new grammar features with "grammar-extras"

this is to address potential semver issues due to Cargo dependency intermixing, see: pest-parser#849

So, for existing users who depend on 2.X in the dependency tree, 2.7.0 pest-meta should work with 2.5.6 and below pest-generator etc.
People who need the new grammar features can do so by specifying the `grammar-extras` feature in Cargo.toml:
```
...
pest_derive = {version = "2.7", features = ["grammar-extras"]}
```

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
           kind: check
           toolchain: 1.64.0
       - name: cargo check
-        run: cargo check --all --features pretty-print,const_prec_climber,memchr --all-targets
+        run: cargo check --all --features pretty-print,const_prec_climber,memchr,grammar-extras --all-targets
 
   testing:
     name: Unit, Style, and Lint Testing
@@ -44,9 +44,9 @@ jobs:
       - name: cargo fmt
         run: cargo fmt --all -- --check
       - name: cargo clippy
-        run: cargo clippy --all --features pretty-print,const_prec_climber,memchr --all-targets -- -Dwarnings
+        run: cargo clippy --all --features pretty-print,const_prec_climber,memchr,grammar-extras --all-targets -- -Dwarnings
       - name: cargo test
-        run: cargo test --all --features pretty-print,const_prec_climber,memchr --release
+        run: cargo test --all --features pretty-print,const_prec_climber,memchr,grammar-extras --release
       - name: cargo test (ignored)
         run: cargo test -p pest_grammars --lib --verbose --release -- --ignored tests::toml_handles_deep_nesting_unstable
 
@@ -64,7 +64,7 @@ jobs:
           kind: check
           toolchain: 1.64.0
       - name: cargo doc
-        run: cargo doc --all --features pretty-print,const_prec_climber,memchr
+        run: cargo doc --all --features pretty-print,const_prec_climber,memchr,grammar-extras
 
   dependency:
     name: Minimal Versions Testing
@@ -101,7 +101,7 @@ jobs:
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
       - name: Generate code coverage
-        run: cargo llvm-cov --features pretty-print,const_prec_climber,memchr --workspace --lcov --output-path lcov.info
+        run: cargo llvm-cov --features pretty-print,const_prec_climber,memchr,grammar-extras --workspace --lcov --output-path lcov.info
       - name: Upload Results to Codecov
         uses: codecov/codecov-action@v3
         with:

debugger/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>", "Tomas Tauber <me@tomtau.be>"]
 homepage = "https://pest.rs/"
@@ -14,9 +14,9 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.1" }
-pest_meta = { path = "../meta", version = "2.6.1" }
-pest_vm = { path = "../vm", version = "2.6.1" }
+pest = { path = "../pest", version = "2.7.0" }
+pest_meta = { path = "../meta", version = "2.7.0" }
+pest_vm = { path = "../vm", version = "2.7.0" }
 reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] }
 rustyline = "10"
 serde_json = "1"

derive/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -21,8 +21,9 @@ proc-macro = true
 default = ["std"]
 std = ["pest/std", "pest_generator/std"]
 not-bootstrap-in-src = ["pest_generator/not-bootstrap-in-src"]
+grammar-extras = ["pest_generator/grammar-extras"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.6.1", default-features = false }
-pest_generator = { path = "../generator", version = "2.6.1", default-features = false }
+pest = { path = "../pest", version = "2.7.0", default-features = false }
+pest_generator = { path = "../generator", version = "2.7.0", default-features = false }

derive/tests/implicit.rs

@@ -9,6 +9,7 @@ extern crate alloc;
 extern crate pest;
 extern crate pest_derive;
 
+#[cfg(feature = "grammar-extras")]
 use pest::Parser;
 use pest_derive::Parser;
 
@@ -17,6 +18,7 @@ use pest_derive::Parser;
 struct TestImplicitParser;
 
 #[test]
+#[cfg(feature = "grammar-extras")]
 fn test_implicit_whitespace() {
     // this failed to parse due to a bug in the optimizer
     // see: https://github.com/pest-parser/pest/issues/762#issuecomment-1375374868

generator/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -17,10 +17,11 @@ rust-version = "1.60"
 default = ["std"]
 std = ["pest/std"]
 not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"]
+grammar-extras = ["pest_meta/grammar-extras"]
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.1", default-features = false }
-pest_meta = { path = "../meta", version = "2.6.1" }
+pest = { path = "../pest", version = "2.7.0", default-features = false }
+pest_meta = { path = "../meta", version = "2.7.0" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "2.0"

generator/src/generator.rs

@@ -517,6 +517,7 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream {
                 state.restore_on_err(|state| #expr)
             }
         }
+        #[cfg(feature = "grammar-extras")]
         OptimizedExpr::NodeTag(expr, tag) => {
             let expr = generate_expr(*expr);
             quote! {
@@ -655,6 +656,7 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
                 state.restore_on_err(|state| #expr)
             }
         }
+        #[cfg(feature = "grammar-extras")]
         OptimizedExpr::NodeTag(expr, tag) => {
             let expr = generate_expr_atomic(*expr);
             quote! {

grammars/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.1" }
-pest_derive = { path = "../derive", version = "2.6.1" }
+pest = { path = "../pest", version = "2.7.0" }
+pest_derive = { path = "../derive", version = "2.7.0" }
 
 [dev-dependencies]
 criterion = "0.5"

meta/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.1" }
+pest = { path = "../pest", version = "2.7.0" }
 once_cell = "1.8.0"
 
 [build-dependencies]
@@ -25,4 +25,5 @@ cargo = { version = "0.70", optional = true }
 
 [features]
 default = []
-not-bootstrap-in-src = ["dep:cargo"]
+not-bootstrap-in-src = ["dep:cargo"]
+grammar-extras = []

meta/src/ast.rs

@@ -95,6 +95,7 @@ pub enum Expr {
     /// Matches an expression and pushes it to the stack, e.g. `push(e)`
     Push(Box<Expr>),
     /// Matches an expression and assigns a label to it, e.g. #label = exp
+    #[cfg(feature = "grammar-extras")]
     NodeTag(Box<Expr>, String),
 }
 
@@ -166,6 +167,7 @@ impl Expr {
                     let mapped = Box::new(map_internal(*expr, f));
                     Expr::Push(mapped)
                 }
+                #[cfg(feature = "grammar-extras")]
                 Expr::NodeTag(expr, tag) => {
                     let mapped = Box::new(map_internal(*expr, f));
                     Expr::NodeTag(mapped, tag)
@@ -237,6 +239,7 @@ impl Expr {
                     let mapped = Box::new(map_internal(*expr, f));
                     Expr::Push(mapped)
                 }
+                #[cfg(feature = "grammar-extras")]
                 Expr::NodeTag(expr, tag) => {
                     let mapped = Box::new(map_internal(*expr, f));
                     Expr::NodeTag(mapped, tag)
@@ -290,8 +293,11 @@ impl ExprTopDownIterator {
             | Expr::RepMax(expr, _)
             | Expr::RepMinMax(expr, ..)
             | Expr::Opt(expr)
-            | Expr::Push(expr)
-            | Expr::NodeTag(expr, _) => {
+            | Expr::Push(expr) => {
+                self.next = Some(*expr);
+            }
+            #[cfg(feature = "grammar-extras")]
+            Expr::NodeTag(expr, _) => {
                 self.next = Some(*expr);
             }
             _ => {

meta/src/optimizer/mod.rs

@@ -68,6 +68,7 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule {
             Expr::Rep(expr) => OptimizedExpr::Rep(Box::new(to_optimized(*expr))),
             Expr::Skip(strings) => OptimizedExpr::Skip(strings),
             Expr::Push(expr) => OptimizedExpr::Push(Box::new(to_optimized(*expr))),
+            #[cfg(feature = "grammar-extras")]
             Expr::NodeTag(expr, tag) => OptimizedExpr::NodeTag(Box::new(to_optimized(*expr)), tag),
             Expr::RepOnce(_)
             | Expr::RepExact(..)
@@ -139,6 +140,7 @@ pub enum OptimizedExpr {
     /// Matches an expression and pushes it to the stack, e.g. `push(e)`
     Push(Box<OptimizedExpr>),
     /// Matches an expression and assigns a label to it, e.g. #label = exp
+    #[cfg(feature = "grammar-extras")]
     NodeTag(Box<OptimizedExpr>, String),
     /// Restores an expression's checkpoint
     RestoreOnErr(Box<OptimizedExpr>),

meta/src/parser.rs

@@ -20,7 +20,6 @@ use pest::{Parser, Position, Span};
 use crate::ast::{Expr, Rule as AstRule, RuleType};
 use crate::validator;
 
-/// TODO: fix the generator to at least add explicit lifetimes
 #[allow(missing_docs, unused_qualifications)]
 mod grammar {
     #[cfg(not(feature = "not-bootstrap-in-src"))]
@@ -163,6 +162,7 @@ pub enum ParserExpr<'i> {
     /// Matches an expression and pushes it to the stack, e.g. `push(e)`
     Push(Box<ParserNode<'i>>),
     /// Matches an expression and assigns a label to it, e.g. #label = exp
+    #[cfg(feature = "grammar-extras")]
     NodeTag(Box<ParserNode<'i>>, String),
 }
 
@@ -199,6 +199,7 @@ fn convert_node(node: ParserNode<'_>) -> Expr {
             Expr::RepMinMax(Box::new(convert_node(*node)), min, max)
         }
         ParserExpr::Push(node) => Expr::Push(Box::new(convert_node(*node))),
+        #[cfg(feature = "grammar-extras")]
         ParserExpr::NodeTag(node, tag) => Expr::NodeTag(Box::new(convert_node(*node)), tag),
     }
 }
@@ -337,7 +338,10 @@ fn consume_expr<'i>(
         mut pairs: Peekable<Pairs<'i, Rule>>,
         pratt: &PrattParser<Rule>,
     ) -> Result<ParserNode<'i>, Vec<Error<Rule>>> {
+        #[cfg(feature = "grammar-extras")]
         let (pair, tag_start) = get_node_tag(&mut pairs);
+        #[cfg(not(feature = "grammar-extras"))]
+        let (pair, _tag_start) = get_node_tag(&mut pairs);
 
         let node = match pair.as_rule() {
             Rule::opening_paren => {
@@ -632,6 +636,7 @@ fn consume_expr<'i>(
                 )?
             }
         };
+        #[cfg(feature = "grammar-extras")]
         if let Some((tag, start)) = tag_start {
             let span = start.span(&node.span.end_pos());
             Ok(ParserNode {
@@ -641,6 +646,8 @@ fn consume_expr<'i>(
         } else {
             Ok(node)
         }
+        #[cfg(not(feature = "grammar-extras"))]
+        Ok(node)
     }
 
     let term = |pair: Pair<'i, Rule>| unaries(pair.into_inner().peekable(), pratt);

meta/src/validator.rs

@@ -326,9 +326,9 @@ fn is_non_progressing<'i>(
             min == 0 || is_non_progressing(&inner.expr, rules, trace)
         }
         ParserExpr::Push(ref inner) => is_non_progressing(&inner.expr, rules, trace),
-        ParserExpr::RepOnce(ref inner) | ParserExpr::NodeTag(ref inner, _) => {
-            is_non_progressing(&inner.expr, rules, trace)
-        }
+        ParserExpr::RepOnce(ref inner) => is_non_progressing(&inner.expr, rules, trace),
+        #[cfg(feature = "grammar-extras")]
+        ParserExpr::NodeTag(ref inner, _) => is_non_progressing(&inner.expr, rules, trace),
     }
 }
 
@@ -417,9 +417,11 @@ fn is_non_failing<'i>(
         //     @{EOI ~ ANY | ANY ~ SOI | &("A") ~ &("B") | 'z'..'a'}
         ParserExpr::NegPred(_) => false,
         ParserExpr::RepOnce(ref inner) => is_non_failing(&inner.expr, rules, trace),
-        ParserExpr::Push(ref inner)
-        | ParserExpr::NodeTag(ref inner, _)
-        | ParserExpr::PosPred(ref inner) => is_non_failing(&inner.expr, rules, trace),
+        ParserExpr::Push(ref inner) | ParserExpr::PosPred(ref inner) => {
+            is_non_failing(&inner.expr, rules, trace)
+        }
+        #[cfg(feature = "grammar-extras")]
+        ParserExpr::NodeTag(ref inner, _) => is_non_failing(&inner.expr, rules, trace),
     }
 }
 
@@ -933,19 +935,22 @@ mod tests {
             &HashMap::new(),
             &mut Vec::new()
         ));
-        let progressing = ParserExpr::NodeTag(progressing_node.clone(), "TAG".into());
-        let non_progressing = ParserExpr::NodeTag(non_progressing_node.clone(), "TAG".into());
+        #[cfg(feature = "grammar-extras")]
+        {
+            let progressing = ParserExpr::NodeTag(progressing_node.clone(), "TAG".into());
+            let non_progressing = ParserExpr::NodeTag(non_progressing_node.clone(), "TAG".into());
 
-        assert!(!is_non_progressing(
-            &progressing,
-            &HashMap::new(),
-            &mut Vec::new()
-        ));
-        assert!(is_non_progressing(
-            &non_progressing,
-            &HashMap::new(),
-            &mut Vec::new()
-        ));
+            assert!(!is_non_progressing(
+                &progressing,
+                &HashMap::new(),
+                &mut Vec::new()
+            ));
+            assert!(is_non_progressing(
+                &non_progressing,
+                &HashMap::new(),
+                &mut Vec::new()
+            ));
+        }
     }
 
     #[test]
@@ -1290,6 +1295,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature = "grammar-extras")]
     fn failing_non_zero_repetitions() {
         let failing = Box::new(ParserNode {
             expr: ParserExpr::NodeTag(
@@ -1540,16 +1546,19 @@ mod tests {
             &mut Vec::new()
         ));
 
-        assert!(!is_non_failing(
-            &ParserExpr::NodeTag(failing_node.clone(), "TAG".into()),
-            &HashMap::new(),
-            &mut Vec::new()
-        ));
-        assert!(is_non_failing(
-            &ParserExpr::NodeTag(non_failing_node.clone(), "TAG".into()),
-            &HashMap::new(),
-            &mut Vec::new()
-        ));
+        #[cfg(feature = "grammar-extras")]
+        {
+            assert!(!is_non_failing(
+                &ParserExpr::NodeTag(failing_node.clone(), "TAG".into()),
+                &HashMap::new(),
+                &mut Vec::new()
+            ));
+            assert!(is_non_failing(
+                &ParserExpr::NodeTag(non_failing_node.clone(), "TAG".into()),
+                &HashMap::new(),
+                &mut Vec::new()
+            ));
+        }
 
         assert!(!is_non_failing(
             &ParserExpr::Push(failing_node.clone()),

pest/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"