feat(extraction): add support for faster, cached extraction (#191)

orogene · Mar 5, 2023 · 5bf0425 · 5bf0425
1 parent 922e5eb
commit 5bf0425
Show file tree

Hide file tree

Showing 13 changed files with 474 additions and 183 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -52,9 +52,10 @@ async-process = "1.0.1"
 async-std = "1.12.0"
 async-tar-wasm = "0.4.2-wasm.1"
 async-trait = "0.1.64"
+backon = "0.4.0"
 bincode = "1.3.1"
 bytecount = "0.6.0"
-cacache = "11.1.0"
+cacache = "11.3.0"
 chrono = "0.4.23"
 chrono-humanize = "0.0.11"
 clap = "4.1.4"
@@ -68,6 +69,7 @@ directories = "4.0.1"
 flate2 = "1.0.25"
 futures = "0.3.26"
 indicatif = "0.17.3"
+io_tee = "0.1.1"
 http-cache-reqwest = "0.6.0"
 humansize = "1.1.0"
 js-sys = "0.3.61"
@@ -80,7 +82,6 @@ percent-encoding = "2.1.0"
 pretty_assertions = "1.3.0"
 proc-macro2 = "1.0.18"
 quote = "1.0.7"
-reflink = "0.1.3"
 reqwest = "0.11.14"
 reqwest-middleware = "0.2.0"
 serde = "1.0.152"

diff --git a/crates/nassun/Cargo.toml b/crates/nassun/Cargo.toml
@@ -33,9 +33,10 @@ url = { workspace = true }
 async-tar-wasm = { workspace = true }
 async-process = { workspace = true }
 async-std = { workspace = true, features = ["attributes", "std"] }
+backon = { workspace = true }
 cacache = { workspace = true }
 flate2 = { workspace = true }
-reflink = { workspace = true }
+io_tee = { workspace = true }
 tar = { workspace = true }
 tempfile = { workspace = true }
 which = { workspace = true }

diff --git a/crates/nassun/src/client.rs b/crates/nassun/src/client.rs
@@ -65,12 +65,16 @@ impl NassunOpts {
             .cloned()
             .unwrap_or_else(|| "https://registry.npmjs.org/".parse().unwrap());
         let mut client_builder = OroClient::builder().registry(registry);
-        if let Some(cache) = self.cache {
-            client_builder = client_builder.cache(cache);
-        }
+        let cache = if let Some(cache) = self.cache {
+            client_builder = client_builder.cache(cache.clone());
+            Arc::new(Some(cache))
+        } else {
+            Arc::new(None)
+        };
         let client = client_builder.build();
         Nassun {
-            // cache: self.cache,
+            #[cfg(not(target_arch = "wasm32"))]
+            cache,
             resolver: PackageResolver {
                 #[cfg(target_arch = "wasm32")]
                 base_dir: PathBuf::from("."),
@@ -96,7 +100,7 @@ impl NassunOpts {
 /// Toplevel client for making package requests.
 #[derive(Clone)]
 pub struct Nassun {
-    // cache: Option<PathBuf>,
+    cache: Arc<Option<PathBuf>>,
     resolver: PackageResolver,
     npm_fetcher: Arc<dyn PackageFetcher>,
     #[cfg(not(target_arch = "wasm32"))]
@@ -194,7 +198,9 @@ impl Nassun {
         let spec = spec.as_ref().parse()?;
         let fetcher = self.pick_fetcher(&spec);
         let name = fetcher.name(&spec, &self.resolver.base_dir).await?;
-        self.resolver.resolve(name, spec, fetcher).await
+        self.resolver
+            .resolve(name, spec, fetcher, self.cache.clone())
+            .await
     }
 
     /// Resolves a package directly from a previously-calculated
@@ -208,14 +214,16 @@ impl Nassun {
         resolved: PackageResolution,
     ) -> Package {
         let fetcher = self.pick_fetcher(&from);
-        self.resolver.resolve_from(name, from, resolved, fetcher)
+        self.resolver
+            .resolve_from(name, from, resolved, fetcher, self.cache.clone())
     }
 
     /// Creates a "resolved" package from a plain [`oro_common::Manifest`].
     /// This is useful for, say, creating dummy packages for top-level
     /// projects.
     pub fn dummy_from_manifest(manifest: CorgiManifest) -> Package {
         Package {
+            cache: Arc::new(None),
             from: PackageSpec::Dir {
                 path: PathBuf::from("."),
             },