feat(core): use rust utilities for caching operations

packages/nx/Cargo.toml

@@ -4,32 +4,31 @@ version = '0.1.0'
 edition = '2021'
 
 [dependencies]
-xxhash-rust = { version = '0.8.5', features = ['xxh3', 'xxh64'] }
-napi = { version = '2.12.6', default-features = false, features = ['anyhow', 'napi4', 'tokio_rt'] }
-napi-derive = '2.9.3'
-ignore = '0.4'
+anyhow = "1.0.71"
+colored = "2"
 crossbeam-channel = '0.5'
-
+fs_extra = "1.3.0"
 globset = "0.4.10"
+hashbrown = { version = "0.14.0", features = ["rayon"] }
+ignore = '0.4'
 ignore-files = "1.3.0"
-watchexec = "2.3.0"
-watchexec-filterer-ignore = "1.2.1"
-watchexec-events = "1.0.0"
-watchexec-signals = "1.0.0"
-
-tracing = "0.1.37"
-tracing-subscriber = { version = "0.3.17", features = ["env-filter"]}
-anyhow = "1.0.71"
-thiserror = "1.0.40"
 itertools = "0.10.5"
+jsonc-parser = { version = "0.21.1", features = ["serde"] }
+napi = { version = '2.12.6', default-features = false, features = ['anyhow', 'napi4', 'tokio_rt'] }
+napi-derive = '2.9.3'
 rayon = "1.7.0"
-hashbrown = {version = "0.14.0", features = ["rayon"]}
-
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-jsonc-parser = {version =  "0.21.1", features = ["serde"] }
-
-colored = "2"
+thiserror = "1.0.40"
+tokio = { version = "1.28.2", features = ["fs"] }
+tracing = "0.1.37"
+tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
+walkdir = '2.3.3'
+watchexec = "2.3.0"
+watchexec-events = "1.0.0"
+watchexec-filterer-ignore = "1.2.1"
+watchexec-signals = "1.0.0"
+xxhash-rust = { version = '0.8.5', features = ['xxh3', 'xxh64'] }
 
 [lib]
 crate-type = ['cdylib']

packages/nx/src/native/cache/expand_outputs.rs

@@ -0,0 +1,83 @@
+use std::path::PathBuf;
+
+use crate::native::utils::glob::build_glob_set;
+use crate::native::utils::path::Normalize;
+use crate::native::walker::nx_walker_sync;
+
+#[napi]
+/// Expands the given entries into a list of existing files.
+/// First checks if the entry exists, if not, it will glob the working directory to find the file.
+pub fn expand_outputs(directory: String, entries: Vec<String>) -> anyhow::Result<Vec<String>> {
+    let directory: PathBuf = directory.into();
+
+    let (existing_paths, not_found): (Vec<_>, Vec<_>) = entries.into_iter().partition(|entry| {
+        let path = directory.join(entry);
+        path.exists()
+    });
+
+    if not_found.is_empty() {
+        return Ok(existing_paths);
+    }
+
+    let glob_set = build_glob_set(not_found)?;
+    let found_paths = nx_walker_sync(directory)
+        .filter_map(|path| {
+            if glob_set.is_match(&path) {
+                Some(path.to_normalized_string())
+            } else {
+                None
+            }
+        })
+        .chain(existing_paths);
+
+    Ok(found_paths.collect())
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use assert_fs::prelude::*;
+    use assert_fs::TempDir;
+    use std::{assert_eq, vec};
+
+    fn setup_fs() -> TempDir {
+        let temp = TempDir::new().unwrap();
+        temp.child("test.txt").touch().unwrap();
+        temp.child("foo.txt").touch().unwrap();
+        temp.child("bar.txt").touch().unwrap();
+        temp.child("baz").child("qux.txt").touch().unwrap();
+        temp.child("nested")
+            .child("deeply")
+            .child("nx.darwin-arm64.node")
+            .touch()
+            .unwrap();
+        temp.child("folder").child("nested-folder").touch().unwrap();
+        temp.child("packages")
+            .child("nx")
+            .child("src")
+            .child("native")
+            .child("nx.darwin-arm64.node")
+            .touch()
+            .unwrap();
+        temp
+    }
+    #[test]
+    fn should_expand_outputs() {
+        let temp = setup_fs();
+        let entries = vec![
+            "packages/nx/src/native/*.node".to_string(),
+            "folder/nested-folder".to_string(),
+            "test.txt".to_string(),
+        ];
+        let mut result = expand_outputs(temp.display().to_string(), entries).unwrap();
+        result.sort();
+        assert_eq!(
+            result,
+            vec![
+                "folder/nested-folder",
+                "packages/nx/src/native/nx.darwin-arm64.node",
+                "test.txt"
+            ]
+        );
+    }
+}

packages/nx/src/native/cache/file_ops.rs

@@ -0,0 +1,65 @@
+use std::fs;
+use std::path::PathBuf;
+
+#[napi]
+pub fn remove(src: String) -> anyhow::Result<()> {
+    fs_extra::remove_items(&[src]).map_err(anyhow::Error::from)
+}
+
+#[napi]
+pub fn copy(src: String, dest: String) -> anyhow::Result<()> {
+    let copy_options = fs_extra::dir::CopyOptions::new()
+        .overwrite(true)
+        .skip_exist(false);
+
+    let dest: PathBuf = dest.into();
+    let dest_parent = dest.parent().unwrap_or(&dest);
+
+    if !dest_parent.exists() {
+        fs::create_dir_all(dest_parent)?;
+    }
+
+    fs_extra::copy_items(&[src], dest_parent, &copy_options)?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use assert_fs::prelude::*;
+    use assert_fs::TempDir;
+
+    #[test]
+    fn should_copy_directories() {
+        let temp = TempDir::new().unwrap();
+        temp.child("parent")
+            .child("child")
+            .child("grand-child")
+            .child(".config")
+            .child("file.txt")
+            .touch()
+            .unwrap();
+        let src = temp.join("parent/child/grand-child/.config");
+        let dest = temp.join("new-parent/child/grand-child/.config");
+        copy(src.to_string_lossy().into(), dest.to_string_lossy().into()).unwrap();
+
+        assert!(temp
+            .child("new-parent/child/grand-child/.config/file.txt")
+            .exists());
+    }
+
+    #[test]
+    fn should_copy_single_files() {
+        let temp = TempDir::new().unwrap();
+        temp.child("parent")
+            .child("file.txt")
+            .write_str("content")
+            .unwrap();
+
+        let src = temp.join("parent/file.txt");
+        let dest = temp.join("new-parent/file.txt");
+        copy(src.to_string_lossy().into(), dest.to_string_lossy().into()).unwrap();
+
+        assert!(temp.child("new-parent/file.txt").exists());
+    }
+}

packages/nx/src/native/cache/mod.rs

@@ -0,0 +1,2 @@
+pub mod expand_outputs;
+pub mod file_ops;

packages/nx/src/native/hasher.rs

@@ -1,31 +1,20 @@
-#![allow(unused)]
-
-use crate::native::parallel_walker::nx_walker;
 use crate::native::types::FileData;
 use crate::native::utils::glob::build_glob_set;
 use crate::native::utils::path::Normalize;
-use anyhow::anyhow;
-use crossbeam_channel::unbounded;
-use globset::{Glob, GlobSetBuilder};
-use ignore::WalkBuilder;
-use itertools::Itertools;
-use std::cmp::Ordering;
+use crate::native::walker::nx_walker;
+use rayon::prelude::*;
 use std::collections::HashMap;
-use std::path::Path;
-use std::thread::available_parallelism;
 use xxhash_rust::xxh3;
 
-type FileHashes = HashMap<String, String>;
-
 #[napi]
-fn hash_array(input: Vec<String>) -> String {
+pub fn hash_array(input: Vec<String>) -> String {
     let joined = input.join(",");
     let content = joined.as_bytes();
     xxh3::xxh3_64(content).to_string()
 }
 
 #[napi]
-fn hash_file(file: String) -> Option<FileData> {
+pub fn hash_file(file: String) -> Option<FileData> {
     let Ok(content) = std::fs::read(&file) else {
         return None;
     };
@@ -36,7 +25,7 @@ fn hash_file(file: String) -> Option<FileData> {
 }
 
 #[napi]
-fn hash_files(workspace_root: String) -> HashMap<String, String> {
+pub fn hash_files(workspace_root: String) -> HashMap<String, String> {
     nx_walker(workspace_root, |rec| {
         let mut collection: HashMap<String, String> = HashMap::new();
         for (path, content) in rec {
@@ -74,7 +63,7 @@ fn hash_files_matching_globs(
     }
 
     // Sort the file data so that its in deterministically ordered by file path
-    hashes.sort();
+    hashes.par_sort();
 
     let sorted_file_hashes: Vec<String> =
         hashes.into_iter().map(|file_data| file_data.hash).collect();

packages/nx/src/native/index.d.ts

@@ -3,6 +3,13 @@
 
 /* auto-generated by NAPI-RS */
 
+/**
+ * Expands the given entries into a list of existing files.
+ * First checks if the entry exists, if not, it will glob the working directory to find the file.
+ */
+export function expandOutputs(directory: string, entries: Array<string>): Array<string>
+export function remove(src: string): void
+export function copy(src: string, dest: string): void
 export function hashArray(input: Array<string>): string
 export function hashFile(file: string): FileData | null
 export function hashFiles(workspaceRoot: string): Record<string, string>

packages/nx/src/native/index.js

@@ -246,8 +246,11 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { hashArray, hashFile, hashFiles, hashFilesMatchingGlobs, EventType, Watcher, WorkspaceErrors, getConfigFiles, getWorkspaceFilesNative } = nativeBinding
+const { expandOutputs, remove, copy, hashArray, hashFile, hashFiles, hashFilesMatchingGlobs, EventType, Watcher, WorkspaceErrors, getConfigFiles, getWorkspaceFilesNative } = nativeBinding
 
+module.exports.expandOutputs = expandOutputs
+module.exports.remove = remove
+module.exports.copy = copy
 module.exports.hashArray = hashArray
 module.exports.hashFile = hashFile
 module.exports.hashFiles = hashFiles

packages/nx/src/native/mod.rs

@@ -1,7 +1,8 @@
+pub mod cache;
 pub mod hasher;
 mod logger;
-mod parallel_walker;
 mod types;
 mod utils;
+mod walker;
 pub mod watch;
 pub mod workspace;

packages/nx/src/native/utils/path.rs

@@ -1,14 +1,29 @@
+use std::path::Path;
+
 pub trait Normalize {
     fn to_normalized_string(&self) -> String;
 }
 
 impl Normalize for std::path::Path {
     fn to_normalized_string(&self) -> String {
-        // convert back-slashes in Windows paths, since the js expects only forward-slash path separators
-        if cfg!(windows) {
-            self.display().to_string().replace('\\', "/")
-        } else {
-            self.display().to_string()
-        }
+        normalize_path(self)
+    }
+}
+
+impl Normalize for std::path::PathBuf {
+    fn to_normalized_string(&self) -> String {
+        normalize_path(self)
+    }
+}
+
+fn normalize_path<P>(path: P) -> String
+where
+    P: AsRef<Path>,
+{
+    // convert back-slashes in Windows paths, since the js expects only forward-slash path separators
+    if cfg!(windows) {
+        path.as_ref().display().to_string().replace('\\', "/")
+    } else {
+        path.as_ref().display().to_string()
     }
 }