feat(hashing): support $TURBO_DEFAULT in inputs

crates/turborepo-lib/src/run/summary/task.rs

@@ -1,6 +1,5 @@
 use std::collections::BTreeMap;
 
-use itertools::Itertools;
 use serde::Serialize;
 use turbopath::{AnchoredSystemPathBuf, RelativeUnixPathBuf};
 use turborepo_cache::CacheHitMetadata;

crates/turborepo-scm/src/manual.rs

@@ -49,13 +49,16 @@ pub(crate) fn hash_files(
     Ok(hashes)
 }
 
-pub(crate) fn get_package_file_hashes_from_processing_gitignore<S: AsRef<str>>(
+pub(crate) fn get_package_file_hashes_without_git<S: AsRef<str>>(
     turbo_root: &AbsoluteSystemPath,
     package_path: &AnchoredSystemPath,
     inputs: &[S],
+    include_default_files: bool,
 ) -> Result<GitHashes, Error> {
     let full_package_path = turbo_root.resolve(package_path);
     let mut hashes = GitHashes::new();
+    let mut default_file_hashes = GitHashes::new();
+    let mut excluded_file_hashes = GitHashes::new();
 
     let mut walker_builder = WalkBuilder::new(&full_package_path);
     let mut includes = Vec::new();
@@ -82,6 +85,7 @@ pub(crate) fn get_package_file_hashes_from_processing_gitignore<S: AsRef<str>>(
     } else {
         Some(any(excludes)?)
     };
+
     let walker = walker_builder
         .follow_links(false)
         // if inputs have been provided manually, we shouldn't skip ignored files to mimic the
@@ -90,6 +94,7 @@ pub(crate) fn get_package_file_hashes_from_processing_gitignore<S: AsRef<str>>(
         .require_git(false)
         .hidden(false) // this results in yielding hidden files (e.g. .gitignore)
         .build();
+
     for dirent in walker {
         let dirent = dirent?;
         let metadata = dirent.metadata()?;
@@ -98,26 +103,80 @@ pub(crate) fn get_package_file_hashes_from_processing_gitignore<S: AsRef<str>>(
         if metadata.is_dir() {
             continue;
         }
+
         let path = AbsoluteSystemPath::from_std_path(dirent.path())?;
         let relative_path = full_package_path.anchor(path)?;
         let relative_path = relative_path.to_unix();
+
+        // if we have includes and this path doesn't match any of them, skip it
         if let Some(include_pattern) = include_pattern.as_ref() {
             if !include_pattern.is_match(relative_path.as_str()) {
                 continue;
             }
         }
+
+        // if we have excludes and this path matches any of them, skip it
         if let Some(exclude_pattern) = exclude_pattern.as_ref() {
             if exclude_pattern.is_match(relative_path.as_str()) {
                 continue;
             }
         }
+
         // FIXME: we don't hash symlinks...
         if metadata.is_symlink() {
             continue;
         }
         let hash = git_like_hash_file(path)?;
         hashes.insert(relative_path, hash);
     }
+
+    // If we're including default files, we need to walk again, but this time with
+    // git_ignore enabled
+    if include_default_files {
+        let walker = walker_builder
+            .follow_links(false)
+            .git_ignore(true)
+            .require_git(false)
+            .hidden(false) // this results in yielding hidden files (e.g. .gitignore)
+            .build();
+
+        for dirent in walker {
+            let dirent = dirent?;
+            let metadata = dirent.metadata()?;
+            // We need to do this here, rather than as a filter, because the root
+            // directory is always yielded and not subject to the supplied filter.
+            if metadata.is_dir() {
+                continue;
+            }
+
+            let path = AbsoluteSystemPath::from_std_path(dirent.path())?;
+            let relative_path = full_package_path.anchor(path)?;
+            let relative_path = relative_path.to_unix();
+
+            if let Some(exclude_pattern) = exclude_pattern.as_ref() {
+                if exclude_pattern.is_match(relative_path.as_str()) {
+                    // track excludes so we can exclude them to the hash map later
+                    if !metadata.is_symlink() {
+                        let hash = git_like_hash_file(path)?;
+                        excluded_file_hashes.insert(relative_path.clone(), hash);
+                    }
+                }
+            }
+
+            // FIXME: we don't hash symlinks...
+            if metadata.is_symlink() {
+                continue;
+            }
+            let hash = git_like_hash_file(path)?;
+            default_file_hashes.insert(relative_path, hash);
+        }
+    }
+
+    // merge default with all hashes
+    hashes.extend(default_file_hashes);
+    // remove excluded files
+    hashes.retain(|key, _| !excluded_file_hashes.contains_key(key));
+
     Ok(hashes)
 }
 
@@ -321,7 +380,7 @@ mod tests {
         );
 
         let hashes =
-            get_package_file_hashes_from_processing_gitignore::<&str>(&turbo_root, &pkg_path, &[])
+            get_package_file_hashes_without_git::<&str>(&turbo_root, &pkg_path, &[], false)
                 .unwrap();
         assert_eq!(hashes, expected);
 
@@ -349,10 +408,11 @@ mod tests {
             }
         }
 
-        let hashes = get_package_file_hashes_from_processing_gitignore(
+        let hashes = get_package_file_hashes_without_git(
             &turbo_root,
             &pkg_path,
             &["**/*file", "!some-dir/excluded-file"],
+            false,
         )
         .unwrap();
 

crates/turborepo-scm/src/package_deps.rs

@@ -9,6 +9,8 @@ use crate::{hash_object::hash_objects, Error, Git, SCM};
 
 pub type GitHashes = HashMap<RelativeUnixPathBuf, String>;
 
+const INPUT_INCLUDE_DEFAULT_FILES: &str = "$TURBO_DEFAULT";
+
 impl SCM {
     pub fn get_hashes_for_files(
         &self,
@@ -31,19 +33,33 @@ impl SCM {
         inputs: &[S],
         telemetry: Option<PackageTaskEventBuilder>,
     ) -> Result<GitHashes, Error> {
+        // If the inputs contain "$TURBO_DEFAULT", we need to include the "default" file
+        // hashes as well. NOTE: we intentionally don't remove "$TURBO_DEFAULT"
+        // from the inputs if it exists in the off chance that the user has a
+        // file named "$TURBO_DEFAULT" in their package (pls no).
+        let include_default_files = inputs
+            .iter()
+            .any(|input| input.as_ref() == INPUT_INCLUDE_DEFAULT_FILES);
+
         match self {
             SCM::Manual => {
                 if let Some(telemetry) = telemetry {
                     telemetry.track_file_hash_method(FileHashMethod::Manual);
                 }
-                crate::manual::get_package_file_hashes_from_processing_gitignore(
+                crate::manual::get_package_file_hashes_without_git(
                     turbo_root,
                     package_path,
-                    inputs,
+                    &inputs,
+                    include_default_files,
                 )
             }
             SCM::Git(git) => {
-                let result = git.get_package_file_hashes(turbo_root, package_path, inputs);
+                let result = git.get_package_file_hashes(
+                    turbo_root,
+                    package_path,
+                    &inputs,
+                    include_default_files,
+                );
                 match result {
                     Ok(hashes) => {
                         if let Some(telemetry) = telemetry {
@@ -59,10 +75,11 @@ impl SCM {
                         if let Some(telemetry) = telemetry {
                             telemetry.track_file_hash_method(FileHashMethod::Manual);
                         }
-                        crate::manual::get_package_file_hashes_from_processing_gitignore(
+                        crate::manual::get_package_file_hashes_without_git(
                             turbo_root,
                             package_path,
-                            inputs,
+                            &inputs,
+                            include_default_files,
                         )
                     }
                 }
@@ -99,7 +116,40 @@ impl Git {
         turbo_root: &AbsoluteSystemPath,
         package_path: &AnchoredSystemPath,
         inputs: &[S],
+        include_default_files: bool,
     ) -> Result<GitHashes, Error> {
+        if include_default_files && !inputs.is_empty() {
+            // collect the default files and the inputs
+            let default_file_hashes =
+                self.get_package_file_hashes_from_index(turbo_root, package_path)?;
+
+            // we need to get hashes for excludes separately so we can remove them from the
+            // defaults later on
+            let mut includes = Vec::new();
+            let mut excludes = Vec::new();
+            for input in inputs {
+                let input_str = input.as_ref();
+                if let Some(exclude) = input_str.strip_prefix('!') {
+                    excludes.push(exclude);
+                } else {
+                    includes.push(input_str);
+                }
+            }
+            let manual_includes_hashes =
+                self.get_package_file_hashes_from_inputs(turbo_root, package_path, &includes)?;
+            let manual_excludes_hashes =
+                self.get_package_file_hashes_from_inputs(turbo_root, package_path, &excludes)?;
+
+            // merge the two includes
+            let mut hashes = default_file_hashes;
+            hashes.extend(manual_includes_hashes);
+
+            // remove the excludes
+            hashes.retain(|key, _| !manual_excludes_hashes.contains_key(key));
+
+            return Ok(hashes);
+        }
+
         if inputs.is_empty() {
             self.get_package_file_hashes_from_index(turbo_root, package_path)
         } else {
@@ -216,7 +266,7 @@ mod tests {
     use turbopath::{AbsoluteSystemPathBuf, AnchoredSystemPathBuf, RelativeUnixPathBuf};
 
     use super::*;
-    use crate::{manual::get_package_file_hashes_from_processing_gitignore, SCM};
+    use crate::{manual::get_package_file_hashes_without_git, SCM};
 
     fn tmp_dir() -> (tempfile::TempDir, AbsoluteSystemPathBuf) {
         let tmp_dir = tempfile::tempdir().unwrap();
@@ -269,8 +319,7 @@ mod tests {
 
         let pkg_path = git_root.anchor(&git_root).unwrap();
         let manual_hashes =
-            get_package_file_hashes_from_processing_gitignore(&git_root, &pkg_path, &["l*"])
-                .unwrap();
+            get_package_file_hashes_without_git(&git_root, &pkg_path, &["l*"], false).unwrap();
         assert!(manual_hashes.is_empty());
     }
 
@@ -373,7 +422,7 @@ mod tests {
                 "bfe53d766e64d78f80050b73cd1c88095bc70abb",
             ),
         ]);
-        let hashes = git.get_package_file_hashes::<&str>(&repo_root, &package_path, &[])?;
+        let hashes = git.get_package_file_hashes::<&str>(&repo_root, &package_path, &[], false)?;
         assert_eq!(hashes, all_expected);
 
         // add the new root file as an option
@@ -425,7 +474,7 @@ mod tests {
                 (key, value)
             }));
             let hashes = git
-                .get_package_file_hashes(&repo_root, &package_path, inputs)
+                .get_package_file_hashes(&repo_root, &package_path, inputs, false)
                 .unwrap();
             assert_eq!(hashes, expected);
         }