diff --git a/Cargo.lock b/Cargo.lock index c42860b50..5cef54b81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1144,6 +1144,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys 0.52.0", +] + [[package]] name = "flate2" version = "1.0.28" @@ -1219,6 +1231,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futures" version = "0.3.30" @@ -1751,6 +1772,26 @@ dependencies = [ "log", ] +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "insta" version = "1.36.1" @@ -1871,6 +1912,26 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kqueue" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2102,6 +2163,8 @@ dependencies = [ "marzano-test-utils", "marzano-util", "marzano_messenger", + "notify", + "notify-debouncer-mini", "openssl", "opentelemetry", "opentelemetry-otlp", @@ -2405,6 +2468,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.48.0", ] @@ -2498,6 +2562,36 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.4.2", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "walkdir", + "windows-sys 0.48.0", +] + +[[package]] +name = "notify-debouncer-mini" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d40b221972a1fc5ef4d858a2f671fb34c75983eb385463dff3780eeff6a9d43" +dependencies = [ + "crossbeam-channel", + "log", + "notify", +] + [[package]] name = "ntest" version = "0.9.1" diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 7b833e4cf..06c996425 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -59,7 +59,7 @@ flate2 = { version = "1.0.17", features = [ "rust_backend", ], default-features = false } axoupdater = { version = "0.6.4", default-features = false, features = [ - "github_releases" + "github_releases", ] } opentelemetry-otlp = { version = "0.14.0", optional = true, features = [ @@ -73,6 +73,8 @@ opentelemetry_sdk = { version = "0.21.1", optional = true, features = [ tracing-opentelemetry = { version = "0.22.0", optional = true, default-features = false } tracing = { version = "0.1.40", default-features = false, features = [] } +notify = "6.1.1" +notify-debouncer-mini = "0.4.1" tracing-subscriber = { version = "0.3", default-features = false, optional = true } fs-err = { version = "2.11.0" } @@ -106,12 +108,8 @@ workflows_v2 = [] bundled_workflows = [] remote_redis = [] remote_pubsub = [] -remote_workflows = [ - "dep:grit_cloud_client", -] -workflow_server = [ - "dep:grit_cloud_client", -] +remote_workflows = ["dep:grit_cloud_client"] +workflow_server = ["dep:grit_cloud_client"] server = [ "workflows_v2", "external_functions", @@ -150,7 +148,7 @@ grit_beta = [ "ai_builtins", "workflows_v2", "remote_workflows", - "workflow_server" + "workflow_server", # "grit_timing", ] grit_timing = [] diff --git a/crates/cli/src/commands/patterns.rs b/crates/cli/src/commands/patterns.rs index 837fcd69b..c2339dc4e 100644 --- a/crates/cli/src/commands/patterns.rs +++ b/crates/cli/src/commands/patterns.rs @@ -36,7 +36,7 @@ pub enum PatternCommands { Describe(PatternsDescribeArgs), } -#[derive(Args, Debug, Serialize)] +#[derive(Args, Debug, Serialize, Clone)] pub struct PatternsTestArgs { /// Regex of a specific pattern to test #[clap(long = "filter")] @@ -53,6 +53,9 @@ pub struct PatternsTestArgs { /// Update expected test outputs #[clap(long = "update")] pub update: bool, + /// Enable watch mode on .grit dir + #[clap(long = "watch")] + pub watch: bool, } #[derive(Args, Debug, Serialize)] diff --git a/crates/cli/src/commands/patterns_test.rs b/crates/cli/src/commands/patterns_test.rs index 404065c23..11b8c866e 100644 --- a/crates/cli/src/commands/patterns_test.rs +++ b/crates/cli/src/commands/patterns_test.rs @@ -8,11 +8,11 @@ use marzano_gritmodule::formatting::format_rich_files; use marzano_gritmodule::markdown::replace_sample_in_md_file; use marzano_gritmodule::patterns_directory::PatternsDirectory; use marzano_gritmodule::testing::{ - collect_testable_patterns, get_sample_name, has_output_mismatch, test_pattern_sample, - GritTestResultState, MismatchInfo, SampleTestResult, + collect_testable_patterns, get_grit_pattern_test_info, get_sample_name, has_output_mismatch, + test_pattern_sample, GritTestResultState, MismatchInfo, SampleTestResult, }; -use marzano_language::target_language::PatternLanguage; +use marzano_language::{grit_parser::MarzanoGritParser, target_language::PatternLanguage}; use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator}; use serde::Serialize; @@ -29,13 +29,25 @@ use marzano_messenger::emit::{get_visibility, VisibilityLevels}; use super::patterns::PatternsTestArgs; use anyhow::{anyhow, bail, Context as _, Result}; +use std::collections::HashMap; +use std::{path::Path, time::Duration}; + +use marzano_core::pattern_compiler::compiler::get_dependents_of_target_patterns_by_traversal_from_src; +use marzano_gritmodule::searcher::collect_from_file; +use notify::{self, RecursiveMode}; +use notify_debouncer_mini::{new_debouncer_opt, Config}; pub async fn get_marzano_pattern_test_results( patterns: Vec, libs: &PatternsDirectory, - args: PatternsTestArgs, + args: &PatternsTestArgs, output: OutputFormat, ) -> Result<()> { + if patterns.is_empty() { + bail!("No testable patterns found. To test a pattern, make sure it is defined in .grit/grit.yaml or a .md file in your .grit/patterns directory."); + } + info!("Found {} testable patterns.", patterns.len()); + let resolver = GritModuleResolver::new(); let final_results: DashMap> = DashMap::new(); @@ -242,7 +254,6 @@ pub async fn get_marzano_pattern_test_results( bail!("Output format not supported for this command"); } } - Ok(()) } @@ -273,11 +284,199 @@ pub(crate) async fn run_patterns_test( let testable_patterns = collect_testable_patterns(patterns); - if testable_patterns.is_empty() { - bail!("No testable patterns found. To test a pattern, make sure it is defined in .grit/grit.yaml or a .md file in your .grit/patterns directory."); + get_marzano_pattern_test_results(testable_patterns.clone(), &libs, &arg, flags.clone().into()) + .await?; + + if arg.watch { + let _ = enable_watch_mode(testable_patterns, &libs, &arg, flags.into()).await; } - info!("Found {} testable patterns.", testable_patterns.len(),); - get_marzano_pattern_test_results(testable_patterns, &libs, arg, flags.into()).await + Ok(()) +} + +async fn enable_watch_mode( + testable_patterns: Vec, + libs: &PatternsDirectory, + args: &PatternsTestArgs, + output: OutputFormat, +) -> Result<()> { + let path = Path::new(".grit"); + let ignore_path = [".grit/.gritmodules", ".grit/.gitignore", ".log"]; + // setup debouncer + let (tx, rx) = std::sync::mpsc::channel(); + // notify backend configuration + let backend_config = notify::Config::default().with_poll_interval(Duration::from_millis(10)); + // debouncer configuration + let debouncer_config = Config::default() + .with_timeout(Duration::from_millis(10)) + .with_notify_config(backend_config); + // select backend via fish operator, here PollWatcher backend + let mut debouncer = new_debouncer_opt::<_, notify::PollWatcher>(debouncer_config, tx)?; + + debouncer.watcher().watch(path, RecursiveMode::Recursive)?; + log::info!("\n[Watch Mode] Enabled on path: {}", path.display()); + + let testable_patterns_map = testable_patterns + .iter() + .map(|p| (p.local_name.as_ref().unwrap(), p)) + .collect::>(); + + // event processing + for result in rx { + match result { + Ok(event) => { + let modified_file_path = &event.first().unwrap().path; + + if !modified_file_path.is_file() { + continue; + } + let modified_file_path = modified_file_path + .clone() + .into_os_string() + .into_string() + .unwrap(); + + //temporary fix, until notify crate adds support for ignoring paths + for path in &ignore_path { + if modified_file_path.contains(path) { + continue; + } + } + log::info!("\n[Watch Mode] File modified: {:?}", modified_file_path); + let (modified_patterns, deleted_patterns) = + get_modified_and_deleted_patterns(&modified_file_path, &testable_patterns) + .await?; + + if modified_patterns.is_empty() && deleted_patterns.is_empty() { + log::info!("[Watch Mode] No patterns changed.\n"); + continue; + } + + let deleted_patterns_names = deleted_patterns + .iter() + .map(|p| p.local_name.as_ref().unwrap()) + .collect::>(); + + let mut patterns_to_test = modified_patterns.clone(); + let mut patterns_to_test_names = patterns_to_test + .iter() + .map(|p| p.local_name.clone().unwrap()) + .collect::>(); + + if !modified_patterns.is_empty() { + let modified_patterns_dependents_names = get_dependents_of_target_patterns( + libs, + &testable_patterns, + &modified_patterns, + )?; + for name in &modified_patterns_dependents_names { + if !deleted_patterns_names.contains(&name) + && !patterns_to_test_names.contains(name) + { + patterns_to_test + .push((*testable_patterns_map.get(name).unwrap()).clone()); + patterns_to_test_names.push(name.to_owned()); + } + } + } + + if !deleted_patterns.is_empty() { + let deleted_patterns_dependents_names = get_dependents_of_target_patterns( + libs, + &testable_patterns, + &deleted_patterns, + )?; + for name in &deleted_patterns_dependents_names { + if !deleted_patterns_names.contains(&name) + && !patterns_to_test_names.contains(name) + { + patterns_to_test + .push((*testable_patterns_map.get(name).unwrap()).clone()); + patterns_to_test_names.push(name.to_owned()); + } + } + } + + log::info!( + "[Watch Mode] Pattern(s) to test: {:?}", + patterns_to_test_names + ); + if patterns_to_test_names.is_empty() { + continue; + } + + let _ = + get_marzano_pattern_test_results(patterns_to_test, libs, args, output.clone()) + .await; + } + Err(error) => { + log::error!("[Watch Mode] Error: {error:?}") + } + } + } + Ok(()) +} + +fn get_dependents_of_target_patterns( + libs: &PatternsDirectory, + testable_patterns: &Vec, + target_patterns: &Vec, +) -> Result> { + let mut target_patterns_names = Vec::new(); + for p in target_patterns { + target_patterns_names.push(p.local_name.as_ref().unwrap()); + } + let mut dependents_names = >::new(); + + let resolver = GritModuleResolver::new(); + + for p in testable_patterns { + let body = format!("{}()", p.local_name.as_ref().unwrap()); + let lang = PatternLanguage::get_language(&p.body); + let libs = libs.get_language_directory_or_default(lang)?; + let rich_pattern = resolver.make_pattern(&body, p.local_name.to_owned())?; + let src = rich_pattern.body; + let mut parser = MarzanoGritParser::new()?; + + let dependents = get_dependents_of_target_patterns_by_traversal_from_src( + &libs, + src, + &mut parser, + &target_patterns_names, + )?; + + for d in dependents { + if !dependents_names.contains(&d) { + dependents_names.push(d); + } + } + } + Ok(dependents_names) +} + +async fn get_modified_and_deleted_patterns( + modified_path: &String, + testable_patterns: &Vec, +) -> Result<(Vec, Vec)> { + let path = Path::new(modified_path); + let file_patterns = collect_from_file(path, &None).await.unwrap_or(vec![]); + let modified_patterns = get_grit_pattern_test_info(file_patterns); + + let mut modified_pattern_names = >::new(); + for pattern in &modified_patterns { + modified_pattern_names.push(pattern.local_name.as_ref().unwrap()); + } + //modified_patterns = patterns which are updated/edited or newly created. + //deleted_patterns = patterns which are deleted. Only remaining dependents of deleted_patterns should gets tested. + let mut deleted_patterns = >::new(); + for pattern in testable_patterns { + if pattern.config.path.as_ref().unwrap() == modified_path + && !modified_pattern_names.contains(&pattern.local_name.as_ref().unwrap()) + { + deleted_patterns.push(pattern.clone()); + } + } + + Ok((modified_patterns, deleted_patterns)) } #[derive(Debug, Serialize)] diff --git a/crates/cli/src/commands/plumbing.rs b/crates/cli/src/commands/plumbing.rs index d06fc5e96..ea697a08c 100644 --- a/crates/cli/src/commands/plumbing.rs +++ b/crates/cli/src/commands/plumbing.rs @@ -264,9 +264,10 @@ pub(crate) async fn run_plumbing( get_marzano_pattern_test_results( patterns, &libs, - PatternsTestArgs { + &PatternsTestArgs { update: false, verbose: false, + watch: false, filter: None, exclude: vec![], }, diff --git a/crates/cli/src/flags.rs b/crates/cli/src/flags.rs index ea6d1f95b..716683060 100644 --- a/crates/cli/src/flags.rs +++ b/crates/cli/src/flags.rs @@ -1,4 +1,4 @@ -#[derive(Debug, Default, clap::Args)] +#[derive(Debug, Default, clap::Args, Clone)] pub struct GlobalFormatFlags { /// Enable JSON output, only supported on some commands #[arg(long, global = true, conflicts_with = "jsonl")] diff --git a/crates/cli_bin/fixtures/.grit/.gitignore b/crates/cli_bin/fixtures/.grit/.gitignore new file mode 100644 index 000000000..e4fdfb17c --- /dev/null +++ b/crates/cli_bin/fixtures/.grit/.gitignore @@ -0,0 +1,2 @@ +.gritmodules* +*.log diff --git a/crates/cli_bin/fixtures/.grit/grit.yaml b/crates/cli_bin/fixtures/.grit/grit.yaml new file mode 100644 index 000000000..758e168a6 --- /dev/null +++ b/crates/cli_bin/fixtures/.grit/grit.yaml @@ -0,0 +1,81 @@ +version: 0.0.1 +patterns: + - name: github.com/getgrit/stdlib#* + - name: our_cargo_use_long_dependency + level: error + body: | + language toml + + cargo_use_long_dependency() where $filename <: not includes or { + "language-submodules", + "language-metavariables" + } + - name: cargo_use_long_dependency + level: error + body: | + language toml + + `[dependencies] + $deps` where { + $filename <: or { includes "Cargo.toml", includes "cargo.toml" }, + $deps <: some bubble `$name = $version` where { + $version <: string(), + $version => `{ version = $version }`, + } + } + - name: no_treesitter_in_grit_crates + description: | + The `grit-pattern-matcher` and `grit-util` crates should remain free of + TreeSitter dependencies. This also implies they cannot have dependencies on any + of the `marzano-*` crates, since those *can* have TreeSitter dependencies. + level: error + body: | + language toml + + `[dependencies] + $deps` where { + $filename <: or { includes "Cargo.toml", includes "cargo.toml" }, + $absolute_filename <: or { + includes "crates/grit-pattern-matcher", + includes "crates/grit-util" + }, + $deps <: some bubble `$name = $specifier` where $name <: or { + includes "tree_sitter", + includes "tree-sitter", + includes "marzano" + } + } + - name: no_println_in_lsp + description: Don't use println!() in LSP code, it breaks the LSP stdio protocol. + level: error + body: | + engine marzano(0.1) + language rust + + `println!($_)` => . where { + $filename <: not includes "test.rs", + $absolute_filename <: includes "lsp", + } + - name: no_println_in_core + description: Don't use println or other debugging macros in core code. + level: error + body: | + engine marzano(0.1) + language rust + + `println!($args)` as $print where { + $outcome = ., + or { + $absolute_filename <: includes "crates/core", + $absolute_filename <: includes "crates/gritmodule", + $absolute_filename <: includes "crates/util", + and { $absolute_filename <: includes "crates/cli/", $outcome = `log::info!($args)` } + }, + // Allow tests and build utils + $absolute_filename <: not includes or { + "tests/", + "build.rs", + "test" + }, + $print <: not within `mod tests { $_ }` + } => $outcome diff --git a/crates/cli_bin/tests/common/mod.rs b/crates/cli_bin/tests/common/mod.rs index 8e8e94022..3c79929f3 100644 --- a/crates/cli_bin/tests/common/mod.rs +++ b/crates/cli_bin/tests/common/mod.rs @@ -1,12 +1,13 @@ -use std::{env, path}; +use std::{env, path, process}; use anyhow::Result; +use assert_cmd::cargo::CommandCargoExt; use assert_cmd::Command; use marzano_gritmodule::config::GRIT_GLOBAL_DIR_ENV; use tempfile::tempdir; -const BIN_NAME: &str = "marzano"; +pub const BIN_NAME: &str = "marzano"; #[allow(dead_code)] pub fn get_test_cmd() -> Result { @@ -15,6 +16,13 @@ pub fn get_test_cmd() -> Result { Ok(cmd) } +#[allow(dead_code)] +pub fn get_test_process_cmd() -> Result { + let mut cmd = process::Command::cargo_bin(BIN_NAME)?; + cmd.env("GRIT_TELEMETRY_DISABLED", "true"); + Ok(cmd) +} + // This is used in tests #[allow(dead_code)] pub fn get_fixtures_root() -> Result { diff --git a/crates/cli_bin/tests/patterns_test.rs b/crates/cli_bin/tests/patterns_test.rs index 76da62522..bbca569d4 100644 --- a/crates/cli_bin/tests/patterns_test.rs +++ b/crates/cli_bin/tests/patterns_test.rs @@ -1,7 +1,16 @@ +use std::{ + env, fs, + io::{BufRead, BufReader}, + process::Stdio, + sync::mpsc, + thread, + time::Duration, +}; + use anyhow::Result; use insta::assert_snapshot; -use crate::common::{get_fixture, get_test_cmd}; +use crate::common::{get_fixture, get_test_cmd, get_test_process_cmd}; mod common; @@ -309,3 +318,106 @@ fn tests_python_pattern_with_file_name() -> Result<()> { Ok(()) } + +#[test] +fn patterns_test_watch_mode_case_patterns_changed() -> Result<()> { + let (tx, rx) = mpsc::channel(); + + let (temp_dir, temp_grit_dir) = get_fixture(".grit", false)?; + let test_yaml_path = temp_grit_dir.join("grit.yaml"); + let temp_dir_path = temp_dir.path().to_owned(); + + let _cmd_handle = thread::spawn(move || { + let mut cmd = get_test_process_cmd() + .unwrap() + .args(&["patterns", "test", "--watch"]) + .current_dir(&temp_dir_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("Failed to start command"); + + let stdout = BufReader::new(cmd.stdout.take().unwrap()); + let stderr = BufReader::new(cmd.stderr.take().unwrap()); + for line in stdout.lines().chain(stderr.lines()) { + if let Ok(line) = line { + tx.send(line).unwrap(); + } + } + }); + thread::sleep(Duration::from_secs(1)); + + let content = fs::read_to_string(&test_yaml_path).expect("Unable to read the file"); + fs::write(&test_yaml_path, content)?; + thread::sleep(Duration::from_secs(1)); + + let mut output = Vec::new(); + while let Ok(line) = rx.try_recv() { + output.push(line); + } + let expected_output = vec![ + "[Watch Mode] Enabled on path: .grit", + "[Watch Mode] File modified: \".grit/grit.yaml\"", + "[Watch Mode] Pattern(s) to test: [\"our_cargo_use_long_dependency\", \"cargo_use_long_dependency\", \"no_treesitter_in_grit_crates\", \"no_println_in_lsp\", \"no_println_in_core\"]", + "Found 5 testable patterns.", + ]; + for expected_line in expected_output { + assert!( + output.iter().any(|line| line.contains(expected_line)), + "Expected output not found: {}", + expected_line + ); + } + Ok(()) +} + +#[test] +fn patterns_test_watch_mode_case_no_pattern_to_test() -> Result<()> { + let (tx, rx) = mpsc::channel(); + + let (temp_dir, temp_grit_dir) = get_fixture(".grit", false)?; + let test_yaml_path = temp_grit_dir.join("grit.yaml"); + let temp_dir_path = temp_dir.path().to_owned(); + + let _cmd_handle = thread::spawn(move || { + let mut cmd = get_test_process_cmd() + .unwrap() + .args(&["patterns", "test", "--watch"]) + .current_dir(&temp_dir_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("Failed to start command"); + + let stdout = BufReader::new(cmd.stdout.take().unwrap()); + let stderr = BufReader::new(cmd.stderr.take().unwrap()); + for line in stdout.lines().chain(stderr.lines()) { + if let Ok(line) = line { + tx.send(line).unwrap(); + } + } + }); + thread::sleep(Duration::from_secs(1)); + + fs::write(&test_yaml_path, "")?; + thread::sleep(Duration::from_secs(1)); + + let mut output = Vec::new(); + while let Ok(line) = rx.try_recv() { + output.push(line); + } + + let expected_output = vec![ + "[Watch Mode] Enabled on path: .grit", + "[Watch Mode] File modified: \".grit/grit.yaml\"", + "[Watch Mode] Pattern(s) to test: []", + ]; + for expected_line in expected_output { + assert!( + output.iter().any(|line| line.contains(expected_line)), + "Expected output not found: {}", + expected_line + ); + } + Ok(()) +} diff --git a/crates/core/src/pattern_compiler/compiler.rs b/crates/core/src/pattern_compiler/compiler.rs index 1e5d5e70e..8a48401c1 100644 --- a/crates/core/src/pattern_compiler/compiler.rs +++ b/crates/core/src/pattern_compiler/compiler.rs @@ -567,6 +567,85 @@ fn find_definition_if_exists( Ok(None) } +pub fn get_dependents_of_target_patterns_by_traversal_from_src( + libs: &BTreeMap, + src: &str, + parser: &mut MarzanoGritParser, + target_patterns: &[&String], +) -> Result> { + let mut dependents = >::new(); + let node_like = "nodeLike"; + let predicate_call = "predicateCall"; + + let tree = parser.parse_file(src, Some(Path::new(DEFAULT_FILE_NAME)))?; + + let DefsToFilenames { + patterns: pattern_file, + predicates: predicate_file, + functions: function_file, + foreign_functions: foreign_file, + } = defs_to_filenames(libs, parser, tree.root_node())?; + + let name_to_filename: BTreeMap<&String, &String> = pattern_file + .iter() + .chain(predicate_file.iter()) + .chain(function_file.iter()) + .chain(foreign_file.iter()) + .collect(); + + let mut traversed_stack = >::new(); + let mut stack: Vec = vec![tree]; + while let Some(tree) = stack.pop() { + let root = tree.root_node(); + let cursor = root.walk(); + + for n in traverse(cursor, Order::Pre).filter(|n| { + n.node.is_named() && (n.node.kind() == node_like || n.node.kind() == predicate_call) + }) { + let name = n + .child_by_field_name("name") + .ok_or_else(|| anyhow!("missing name of nodeLike"))?; + let name = name.text()?; + let name = name.trim().to_string(); + + if target_patterns.contains(&&name) { + while let Some(e) = traversed_stack.pop() { + dependents.push(e); + } + } + if let Some(file_name) = name_to_filename.get(&name) { + if let Some(tree) = find_child_tree_definition( + file_name, + parser, + libs, + &mut traversed_stack, + &name, + )? { + stack.push(tree); + } + } + } + } + Ok(dependents) +} + +fn find_child_tree_definition( + file_name: &str, + parser: &mut MarzanoGritParser, + libs: &BTreeMap, + traversed_stack: &mut Vec, + name: &str, +) -> Result> { + if !traversed_stack.contains(&name.to_string()) { + if let Some(file_body) = libs.get(file_name) { + traversed_stack.push(name.to_owned()); + let tree = parser.parse_file(file_body, Some(Path::new(file_name)))?; + return Ok(Some(tree)); + } + } + Ok(None) +} + pub struct CompilationResult { pub compilation_warnings: AnalysisLogs, pub problem: Problem, diff --git a/crates/gritmodule/src/config.rs b/crates/gritmodule/src/config.rs index 03735b8c7..2b8979ca1 100644 --- a/crates/gritmodule/src/config.rs +++ b/crates/gritmodule/src/config.rs @@ -132,19 +132,24 @@ pub struct GritSerializedDefinitionConfig { pub samples: Option>, } -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Clone, Debug)] pub struct GritPatternTestConfig { pub path: Option, pub samples: Option>, } -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Clone, Debug)] #[serde(rename_all = "camelCase")] pub struct GritPatternTestInfo { pub body: String, pub config: GritPatternTestConfig, pub local_name: Option, } +impl AsRef for GritPatternTestInfo { + fn as_ref(&self) -> &GritPatternTestInfo { + self + } +} #[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq)] pub struct ModuleGritPattern { @@ -293,7 +298,7 @@ impl Ord for ResolvedGritDefinition { pub fn pattern_config_to_model( pattern: GritDefinitionConfig, - source: &ModuleRepo, + source: &Option, ) -> Result { let mut split_name = pattern.name.split('#'); let repo = split_name.next(); @@ -315,7 +320,7 @@ pub fn pattern_config_to_model( Some(split_repo.collect::>().join("/")) }; if defined_local_name.is_none() { - Some(source.clone()) + source.clone() } else if host.is_none() || full_name.is_none() { None } else { diff --git a/crates/gritmodule/src/parser.rs b/crates/gritmodule/src/parser.rs index 61e98e371..37eb8b878 100644 --- a/crates/gritmodule/src/parser.rs +++ b/crates/gritmodule/src/parser.rs @@ -9,6 +9,7 @@ use crate::{ fetcher::ModuleRepo, markdown::get_patterns_from_md, searcher::find_repo_root_from, + yaml::get_patterns_from_yaml, }; use anyhow::{Context, Result}; use marzano_util::rich_path::RichFile; @@ -22,6 +23,8 @@ pub enum PatternFileExt { Grit, #[serde(rename = "markdown")] Md, + #[serde(rename = "yaml")] + Yaml, } impl fmt::Display for PatternFileExt { @@ -29,6 +32,7 @@ impl fmt::Display for PatternFileExt { match self { PatternFileExt::Grit => write!(f, "grit"), PatternFileExt::Md => write!(f, "md"), + PatternFileExt::Yaml => write!(f, "yaml"), } } } @@ -39,11 +43,12 @@ impl PatternFileExt { match ext { "grit" => Some(PatternFileExt::Grit), "md" => Some(PatternFileExt::Md), + "yaml" => Some(PatternFileExt::Yaml), _ => None, } } - fn get_patterns( + async fn get_patterns( &self, file: &mut RichFile, source_module: &Option, @@ -66,6 +71,14 @@ impl PatternFileExt { ) }) } + PatternFileExt::Yaml => get_patterns_from_yaml(file, source_module, root, "") + .await + .with_context(|| { + format!( + "Failed to parse yaml pattern {}", + extract_relative_file_path(file, root) + ) + }), } } @@ -73,6 +86,7 @@ impl PatternFileExt { match self { PatternFileExt::Grit => "grit", PatternFileExt::Md => "md", + PatternFileExt::Yaml => "yaml", } } } @@ -89,6 +103,7 @@ pub async fn get_patterns_from_file( content, }; ext.get_patterns(&mut file, &source_module, &repo_root) + .await } pub fn extract_relative_file_path(file: &RichFile, root: &Option) -> String { diff --git a/crates/gritmodule/src/resolver.rs b/crates/gritmodule/src/resolver.rs index 3c8abaa86..f4ba854dd 100644 --- a/crates/gritmodule/src/resolver.rs +++ b/crates/gritmodule/src/resolver.rs @@ -535,7 +535,8 @@ async fn get_grit_files_for_module( Some(config) => { if let Some(module) = module { let repo_root = find_repo_root_from(repo_path).await?; - get_patterns_from_yaml(&config, module, &repo_root, repo_dir).await? + get_patterns_from_yaml(&config, &Some(module.to_owned()), &repo_root, repo_dir) + .await? } else { vec![] } @@ -586,7 +587,8 @@ async fn resolve_patterns_for_module( Some(config) => { if let Some(module) = module { let repo_root = find_repo_root_from(repo_path).await?; - get_patterns_from_yaml(&config, module, &repo_root, repo_dir).await? + get_patterns_from_yaml(&config, &Some(module.to_owned()), &repo_root, repo_dir) + .await? } else { vec![] } diff --git a/crates/gritmodule/src/snapshots/marzano_gritmodule__yaml__tests__gets_module_patterns.snap b/crates/gritmodule/src/snapshots/marzano_gritmodule__yaml__tests__gets_module_patterns.snap index 635ad6701..74c35f3d4 100644 --- a/crates/gritmodule/src/snapshots/marzano_gritmodule__yaml__tests__gets_module_patterns.snap +++ b/crates/gritmodule/src/snapshots/marzano_gritmodule__yaml__tests__gets_module_patterns.snap @@ -72,9 +72,5 @@ expression: patterns line: 9 column: 11 raw: ~ - module: - host: "" - fullName: "" - remote: "" - providerName: "" + module: ~ local_name: remove_console_error diff --git a/crates/gritmodule/src/testing.rs b/crates/gritmodule/src/testing.rs index b9cd44521..69a2665b4 100644 --- a/crates/gritmodule/src/testing.rs +++ b/crates/gritmodule/src/testing.rs @@ -10,26 +10,48 @@ use marzano_util::rich_path::RichFile; use serde::{Deserialize, Serialize}; use crate::config::{ - GritPatternSample, GritPatternTestConfig, GritPatternTestInfo, ResolvedGritDefinition, + GritPatternSample, GritPatternTestConfig, GritPatternTestInfo, ModuleGritPattern, + ResolvedGritDefinition, }; -fn map_pattern_to_test_info(pattern: &mut ResolvedGritDefinition) -> GritPatternTestInfo { - let samples = pattern.config.samples.take(); +fn map_pattern_to_test_info(pattern: ResolvedGritDefinition) -> GritPatternTestInfo { + let samples = pattern.config.samples; GritPatternTestInfo { - body: pattern.body.clone(), + body: pattern.body, config: GritPatternTestConfig { - path: Some(pattern.config.path.clone()), + path: Some(pattern.config.path), samples, }, - local_name: Some(pattern.local_name.clone()), + local_name: Some(pattern.local_name), } } pub fn collect_testable_patterns( - mut patterns: Vec, + patterns: Vec, ) -> Vec { let testable_patterns: Vec = - patterns.iter_mut().map(map_pattern_to_test_info).collect(); + patterns.into_iter().map(map_pattern_to_test_info).collect(); + testable_patterns +} + +fn map_file_pattern_to_test_info(pattern: ModuleGritPattern) -> GritPatternTestInfo { + let samples = pattern.config.samples; + GritPatternTestInfo { + body: pattern.config.body.unwrap(), + config: GritPatternTestConfig { + path: Some(pattern.config.path), + samples, + }, + local_name: Some(pattern.local_name), + } +} + +pub fn get_grit_pattern_test_info(patterns: Vec) -> Vec { + let testable_patterns: Vec = patterns + .into_iter() + .filter(|p| p.config.body.is_some()) + .map(|pattern: ModuleGritPattern| map_file_pattern_to_test_info(pattern)) + .collect(); testable_patterns } diff --git a/crates/gritmodule/src/yaml.rs b/crates/gritmodule/src/yaml.rs index eddf0dcdf..beea33572 100644 --- a/crates/gritmodule/src/yaml.rs +++ b/crates/gritmodule/src/yaml.rs @@ -5,7 +5,7 @@ use std::{ collections::HashSet, path::{Path, PathBuf}, }; -use tokio::fs; +use tokio::{fs, task}; use crate::{ config::{ @@ -60,7 +60,7 @@ pub fn get_grit_config(source: &str, source_path: &str) -> Result { pub async fn get_patterns_from_yaml( file: &RichFile, - source_module: &ModuleRepo, + source_module: &Option, root: &Option, repo_dir: &str, ) -> Result> { @@ -95,11 +95,12 @@ pub async fn get_patterns_from_yaml( continue; } let extension = extension.unwrap(); - file_readers.push(tokio::spawn(get_patterns_from_file( - pattern_file, - Some(source_module.clone()), - extension, - ))); + let source_module = source_module.clone(); + file_readers.push(task::spawn_blocking(move || { + tokio::runtime::Runtime::new().unwrap().block_on(async { + get_patterns_from_file(pattern_file, source_module, extension).await + }) + })); } for file_reader in file_readers {