Conditional source-tree bootstrapping (#866)

Based on #495 Use cargo to run the bootstrapper. The default feature was flipped compared to #495, so that the existing CI workflows continue working as before. The projects that want to depend on git-based pest can do as follows: `pest_derive = { git = "...", features = ["not-bootstrap-in-src"] }` Co-authored-by: Michael Zhang <mail@mzhang.io>
pest-parser · Jun 17, 2023 · 867da2e · 867da2e
1 parent 5ce7b43
commit 867da2e
Show file tree

Hide file tree

Showing 13 changed files with 130 additions and 43 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
           kind: check
           toolchain: 1.62.0
       - name: cargo check
-        run: cargo check --all --all-features --all-targets
+        run: cargo check --all --features pretty-print,const_prec_climber,memchr --all-targets
 
   testing:
     name: Unit, Style, and Lint Testing
@@ -44,9 +44,9 @@ jobs:
       - name: cargo fmt
         run: cargo fmt --all -- --check
       - name: cargo clippy
-        run: cargo clippy --all --all-features --all-targets -- -Dwarnings
+        run: cargo clippy --all --features pretty-print,const_prec_climber,memchr --all-targets -- -Dwarnings
       - name: cargo test
-        run: cargo test --all --all-features --release
+        run: cargo test --all --features pretty-print,const_prec_climber,memchr --release
       - name: cargo test (ignored)
         run: cargo test -p pest_grammars --lib --verbose --release -- --ignored tests::toml_handles_deep_nesting_unstable
 
@@ -64,7 +64,7 @@ jobs:
           kind: check
           toolchain: 1.62.0
       - name: cargo doc
-        run: cargo doc --all --all-features
+        run: cargo doc --all --features pretty-print,const_prec_climber,memchr
 
   dependency:
     name: Minimal Versions Testing
@@ -101,7 +101,7 @@ jobs:
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
       - name: Generate code coverage
-        run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info
+        run: cargo llvm-cov --features pretty-print,const_prec_climber,memchr --workspace --lcov --output-path lcov.info
       - name: Upload Results to Codecov
         uses: codecov/codecov-action@v3
         with:
@@ -125,7 +125,7 @@ jobs:
           kind: check
           toolchain: 1.62.0
       - name: Check feature powerset
-        run: cargo hack check --feature-powerset --optional-deps --exclude-all-features --keep-going --lib --tests --ignore-private
+        run: cargo hack check --feature-powerset --optional-deps --exclude-all-features --skip not-bootstrap-in-src,cargo --keep-going --lib --tests --ignore-private
 
   no_std:
     name: check for no_std compatibility

diff --git a/bootstrap/Cargo.toml b/bootstrap/Cargo.toml
@@ -14,3 +14,9 @@ rust-version = "1.56"
 [dependencies]
 pest_generator = "2.1.1" # Use the crates-io version, which (should be) known-good
 quote = "1.0"
+
+[features]
+default = []
+# Whether or not the bootstrapper should put the generated .rs file in the
+# source tree or in the output tree
+not-bootstrap-in-src = []
diff --git a/bootstrap/src/main.rs b/bootstrap/src/main.rs
@@ -3,17 +3,29 @@ extern crate quote;
 extern crate pest_generator;
 
 use pest_generator::derive_parser;
-use std::{fs::File, io::prelude::*, path::Path};
+use std::{
+    env,
+    fs::File,
+    io::prelude::*,
+    path::{Path, PathBuf},
+};
 
 fn main() {
     let pest = Path::new(concat!(
         env!("CARGO_MANIFEST_DIR"),
         "/../meta/src/grammar.pest"
     ));
-    let rs = Path::new(concat!(
-        env!("CARGO_MANIFEST_DIR"),
-        "/../meta/src/grammar.rs"
-    ));
+    let rs: PathBuf = if should_bootstrap_in_src() {
+        Path::new(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/../meta/src/grammar.rs"
+        ))
+        .to_owned()
+    } else {
+        // the path is passed via command-line arguments
+        let path = env::args().nth(1).expect("path to grammar.rs");
+        PathBuf::from(path)
+    };
 
     let derived = {
         let path = pest.to_string_lossy();
@@ -28,3 +40,13 @@ fn main() {
 
     writeln!(file, "pub struct PestParser;\n{}", derived,).unwrap();
 }
+
+#[cfg(not(feature = "not-bootstrap-in-src"))]
+fn should_bootstrap_in_src() -> bool {
+    true
+}
+
+#[cfg(feature = "not-bootstrap-in-src")]
+fn should_bootstrap_in_src() -> bool {
+    false
+}
diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.6.0"
+version = "2.6.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>", "Tomas Tauber <me@tomtau.be>"]
 homepage = "https://pest.rs/"
@@ -14,9 +14,9 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.0" }
-pest_meta = { path = "../meta", version = "2.6.0" }
-pest_vm = { path = "../vm", version = "2.6.0" }
+pest = { path = "../pest", version = "2.6.1" }
+pest_meta = { path = "../meta", version = "2.6.1" }
+pest_vm = { path = "../vm", version = "2.6.1" }
 reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] }
 rustyline = "10"
 serde_json = "1"

diff --git a/derive/Cargo.toml b/derive/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.6.0"
+version = "2.6.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -20,8 +20,9 @@ proc-macro = true
 [features]
 default = ["std"]
 std = ["pest/std", "pest_generator/std"]
+not-bootstrap-in-src = ["pest_generator/not-bootstrap-in-src"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.6.0", default-features = false }
-pest_generator = { path = "../generator", version = "2.6.0", default-features = false }
+pest = { path = "../pest", version = "2.6.1", default-features = false }
+pest_generator = { path = "../generator", version = "2.6.1", default-features = false }
diff --git a/generator/Cargo.toml b/generator/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.6.0"
+version = "2.6.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -16,10 +16,11 @@ rust-version = "1.56"
 [features]
 default = ["std"]
 std = ["pest/std"]
+not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"]
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.0", default-features = false }
-pest_meta = { path = "../meta", version = "2.6.0" }
+pest = { path = "../pest", version = "2.6.1", default-features = false }
+pest_meta = { path = "../meta", version = "2.6.1" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "2.0"
diff --git a/grammars/Cargo.toml b/grammars/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.6.0"
+version = "2.6.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.0" }
-pest_derive = { path = "../derive", version = "2.6.0" }
+pest = { path = "../pest", version = "2.6.1" }
+pest_derive = { path = "../derive", version = "2.6.1" }
 
 [dev-dependencies]
 criterion = "0.3"

diff --git a/grammars/src/lib.rs b/grammars/src/lib.rs
@@ -96,7 +96,7 @@ mod tests {
             "/resources/test/jsonfuzzsample2.json"
         ));
         const ERROR: &str = "call limit reached";
-        pest::set_call_limit(Some(8_000usize.try_into().unwrap()));
+        pest::set_call_limit(Some(5_000usize.try_into().unwrap()));
         let s1 = json::JsonParser::parse(json::Rule::json, sample1);
         assert!(s1.is_err());
         assert_eq!(s1.unwrap_err().variant.message(), ERROR);

diff --git a/meta/Cargo.toml b/meta/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.6.0"
+version = "2.6.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -16,8 +16,13 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.0" }
+pest = { path = "../pest", version = "2.6.1" }
 once_cell = "1.8.0"
 
 [build-dependencies]
 sha2 = { version = "0.10", default-features = false }
+cargo = { version = "0.62", optional = true }
+
+[features]
+default = []
+not-bootstrap-in-src = ["cargo"]
diff --git a/meta/build.rs b/meta/build.rs
@@ -1,8 +1,19 @@
+#[cfg(feature = "not-bootstrap-in-src")]
+use cargo::{
+    core::{resolver::CliFeatures, Workspace},
+    ops,
+    ops::{CompileOptions, Packages},
+    util::command_prelude::CompileMode,
+    Config,
+};
 use sha2::{Digest, Sha256};
 use std::env;
+#[cfg(feature = "not-bootstrap-in-src")]
+use std::ffi::OsString;
 use std::fs::{self, File};
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
+#[cfg(not(feature = "not-bootstrap-in-src"))]
 use std::process::Command;
 
 fn display_digest(digest: &[u8]) -> String {
@@ -43,19 +54,56 @@ fn main() {
             let mut hash_file = File::create(hash_path).unwrap();
             writeln!(hash_file, "{}", current_hash).unwrap();
 
-            // This "dynamic linking" is probably so fragile I don't even want to hear it
-            let status = Command::new(manifest_dir.join("../target/debug/pest_bootstrap"))
-                .spawn()
-                .unwrap_or_else(|_| {
+            #[cfg(not(feature = "not-bootstrap-in-src"))]
+            {
+                // This "dynamic linking" is probably so fragile I don't even want to hear it
+                let status = Command::new(manifest_dir.join("../target/debug/pest_bootstrap"))
+                    .spawn()
+                    .unwrap_or_else(|_| {
+                        panic!(
+                            "Bootstrap failed because no bootstrap executable was found. \
+                        Please run `cargo build --package pest_bootstrap` or `cargo bootstrap` \
+                        and then try again.",
+                        )
+                    })
+                    .wait()
+                    .unwrap();
+                assert!(status.success(), "Bootstrap failed");
+            }
+
+            #[cfg(feature = "not-bootstrap-in-src")]
+            {
+                let config = Config::default().expect("cargo config");
+                let workspace_manifest = manifest_dir
+                    .join("../Cargo.toml")
+                    .canonicalize()
+                    .expect("workspace manifest");
+                let workspace = Workspace::new(&workspace_manifest, &config).expect("workspace");
+
+                let mut opts =
+                    CompileOptions::new(&config, CompileMode::Build).expect("compile options");
+                opts.spec = Packages::Packages(vec!["pest_bootstrap".to_owned()]);
+                opts.cli_features = CliFeatures::from_command_line(
+                    &["not-bootstrap-in-src".to_owned()],
+                    false,
+                    true,
+                )
+                .expect("cli features");
+
+                let path = format!(
+                    "{}/__pest_grammar.rs",
+                    env::var("OUT_DIR").expect("OUT_DIR env var")
+                )
+                .parse::<OsString>()
+                .expect("grammar path");
+                if let Err(e) = ops::run(&workspace, &opts, &[path]) {
                     panic!(
-                        "Bootstrap failed because no bootstrap executable was found. \
-                         Please run `cargo build --package pest_bootstrap` or `cargo bootstrap` \
-                         and then try again.",
-                    )
-                })
-                .wait()
-                .unwrap();
-            assert!(status.success(), "Bootstrap failed");
+                        "Bootstrap failed: {e}. \
+                        Please run `cargo build --package pest_bootstrap` or `cargo bootstrap` \
+                        and then try again."
+                    );
+                }
+            }
         } else {
             println!("       Fresh `meta/src/grammar.rs`");
         }

diff --git a/meta/src/parser.rs b/meta/src/parser.rs
@@ -23,7 +23,11 @@ use crate::validator;
 /// TODO: fix the generator to at least add explicit lifetimes
 #[allow(missing_docs, unused_qualifications)]
 mod grammar {
+    #[cfg(not(feature = "not-bootstrap-in-src"))]
     include!("grammar.rs");
+
+    #[cfg(feature = "not-bootstrap-in-src")]
+    include!(concat!(env!("OUT_DIR"), "/__pest_grammar.rs"));
 }
 
 pub use self::grammar::*;

diff --git a/pest/Cargo.toml b/pest/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.6.0"
+version = "2.6.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"

diff --git a/vm/Cargo.toml b/vm/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_vm"
 description = "pest grammar virtual machine"
-version = "2.6.0"
+version = "2.6.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,5 +14,5 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.0" }
-pest_meta = { path = "../meta", version = "2.6.0" }
+pest = { path = "../pest", version = "2.6.1" }
+pest_meta = { path = "../meta", version = "2.6.1" }