diff --git a/Cargo.lock b/Cargo.lock index 2129aecdabd5e..c2945f8469730 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3339,6 +3339,7 @@ dependencies = [ "itertools 0.10.5", "path-clean 1.0.1", "path-slash", + "rayon", "regex", "tempdir", "test-case", diff --git a/crates/turborepo-globwalk/Cargo.toml b/crates/turborepo-globwalk/Cargo.toml index 3bb1fdf105d4a..48b68a33b38aa 100644 --- a/crates/turborepo-globwalk/Cargo.toml +++ b/crates/turborepo-globwalk/Cargo.toml @@ -14,6 +14,7 @@ camino = { workspace = true } itertools.workspace = true path-clean = "1.0.1" path-slash = "0.2.1" +rayon = "1" regex.workspace = true thiserror.workspace = true tracing = "0.1.37" diff --git a/crates/turborepo-globwalk/src/lib.rs b/crates/turborepo-globwalk/src/lib.rs index 0c9b1169ddc2d..b25a6574c2def 100644 --- a/crates/turborepo-globwalk/src/lib.rs +++ b/crates/turborepo-globwalk/src/lib.rs @@ -13,6 +13,7 @@ use camino::Utf8PathBuf; use itertools::Itertools; use path_clean::PathClean; use path_slash::PathExt; +use rayon::prelude::*; use regex::Regex; use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError}; use wax::{walk::FileIterator, BuildError, Glob}; @@ -308,37 +309,38 @@ pub fn globwalk_internal( let (base_path_new, include_paths, exclude_paths) = preprocess_paths_and_globs(base_path, include, exclude)?; - let ex_patterns = exclude_paths + let ex_patterns: Vec<_> = exclude_paths .into_iter() .map(glob_with_contextual_error) .collect::>()?; - include_paths - .into_iter() + let include_patterns = include_paths + .into_par_iter() .map(glob_with_contextual_error) - .map_ok(|glob| walk_glob(walk_type, &base_path_new, &ex_patterns, glob)) - // flat map to bring the results in the vec to the same level as the potential outer err - // this is the same as a flat_map_ok - .flat_map(|s| s.unwrap_or_else(|e| vec![Err(e)])) + .collect::, _>>()?; + + include_patterns + .into_par_iter() + // Use flat_map_iter as we only want parallelism for walking the globs and not iterating + // over the results. + // See https://docs.rs/rayon/latest/rayon/iter/trait.ParallelIterator.html#method.flat_map_iter + .flat_map_iter(|glob| walk_glob(walk_type, &base_path_new, ex_patterns.clone(), glob)) .collect() } #[tracing::instrument(skip(ex_patterns), fields(glob=glob.to_string().as_str()))] fn walk_glob( walk_type: WalkType, - base_path_new: &PathBuf, - ex_patterns: &Vec, + base_path_new: &Path, + ex_patterns: Vec, glob: Glob, ) -> Vec> { - glob.walk(&base_path_new) - .not(ex_patterns.clone()) + glob.walk(base_path_new) + .not(ex_patterns) .unwrap_or_else(|e| { // Per docs, only fails if exclusion list is too large, since we're using // pre-compiled globs - panic!( - "Failed to compile exclusion globs: {:?}: {}", - ex_patterns, e, - ) + panic!("Failed to compile exclusion globs: {}", e,) }) .filter_map(|entry| visit_file(walk_type, entry)) .collect::>()