Merge branch 'main' into newtype-it

Author: rklaehn

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "iroh-blobs"
-version = "0.93.0"
+version = "0.94.0"
 edition = "2021"
 description = "content-addressed blobs for iroh"
 license = "MIT OR Apache-2.0"
@@ -19,7 +19,7 @@ derive_more = { version = "2.0.1", features = ["from", "try_from", "into", "debu
 futures-lite = "2.6.0"
 quinn = { package = "iroh-quinn", version = "0.14.0" }
 n0-future = "0.2.0"
-n0-snafu = "0.2.0"
+n0-snafu = "0.2.2"
 range-collections = { version = "0.4.6", features = ["serde"] }
 smallvec = { version = "1", features = ["serde", "const_new"] }
 snafu = "0.8.5"
@@ -36,11 +36,11 @@ chrono = "0.4.39"
 nested_enum_utils = "0.2.1"
 ref-cast = "1.0.24"
 arrayvec = "0.7.6"
-iroh = "0.91.1"
+iroh = "0.92"
 self_cell = "1.1.0"
 genawaiter = { version = "0.99.1", features = ["futures03"] }
-iroh-base = "0.91.1"
-irpc = { version = "0.7.0", features = ["rpc", "quinn_endpoint_setup", "spans", "stream", "derive"], default-features = false }
+iroh-base = "0.92"
+irpc = { version = "0.8.0", features = ["rpc", "quinn_endpoint_setup", "spans", "stream", "derive"], default-features = false }
 iroh-metrics = { version = "0.35" }
 redb = { version = "=2.4", optional = true }
 reflink-copy = { version = "0.1.24", optional = true }
@@ -55,11 +55,11 @@ serde_test = "1.0.177"
 tempfile = "3.17.1"
 test-strategy = "0.4.0"
 testresult = "0.4.1"
-tracing-subscriber = { version = "0.3.19", features = ["fmt"] }
+tracing-subscriber = { version = "0.3.20", features = ["fmt"] }
 tracing-test = "0.2.5"
 walkdir = "2.5.0"
 atomic_refcell = "0.1.13"
-iroh = { version = "0.91.1", features = ["discovery-local-network"]}
+iroh = { version = "0.92", features = ["discovery-local-network"]}
 async-compression = { version = "0.4.30", features = ["lz4", "tokio"] }
 concat_const = "0.2.0"
 

Cargo.toml

@@ -39,8 +39,3 @@ name = "ring"
 [[licenses.clarify.license-files]]
 hash = 3171872035
 path = "LICENSE"
-
-[sources]
-allow-git = [
-    "https://github.com/n0-computer/iroh",
-]

deny.toml

@@ -152,6 +152,9 @@ use crate::{
     HashAndFormat,
 };
 
+/// Maximum number of external paths we track per blob.
+const MAX_EXTERNAL_PATHS: usize = 8;
+
 /// Create a 16 byte unique ID.
 fn new_uuid() -> [u8; 16] {
     use rand::RngCore;
@@ -1239,18 +1242,21 @@ async fn export_path_impl(
         }
     };
     trace!("exporting {} to {}", cmd.hash.to_hex(), target.display());
-    let data = match data_location {
-        DataLocation::Inline(data) => MemOrFile::Mem(data),
-        DataLocation::Owned(size) => {
-            MemOrFile::File((ctx.options().path.data_path(&cmd.hash), size))
-        }
-        DataLocation::External(paths, size) => MemOrFile::File((
-            paths
-                .into_iter()
-                .next()
-                .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "no external data path"))?,
-            size,
-        )),
+    let (data, mut external) = match data_location {
+        DataLocation::Inline(data) => (MemOrFile::Mem(data), vec![]),
+        DataLocation::Owned(size) => (
+            MemOrFile::File((ctx.options().path.data_path(&cmd.hash), size)),
+            vec![],
+        ),
+        DataLocation::External(paths, size) => (
+            MemOrFile::File((
+                paths.first().cloned().ok_or_else(|| {
+                    io::Error::new(io::ErrorKind::NotFound, "no external data path")
+                })?,
+                size,
+            )),
+            paths,
+        ),
     };
     let size = match &data {
         MemOrFile::Mem(data) => data.len() as u64,
@@ -1274,21 +1280,40 @@ async fn export_path_impl(
                 );
             }
             ExportMode::TryReference => {
-                match std::fs::rename(&source_path, &target) {
-                    Ok(()) => {}
-                    Err(cause) => {
-                        const ERR_CROSS: i32 = 18;
-                        if cause.raw_os_error() == Some(ERR_CROSS) {
-                            let source = fs::File::open(&source_path)?;
-                            let mut target = fs::File::create(&target)?;
-                            copy_with_progress(&source, size, &mut target, tx).await?;
-                        } else {
-                            return Err(cause.into());
+                if !external.is_empty() {
+                    // the file already exists externally, so we need to copy it.
+                    // if the OS supports reflink, we might as well use that.
+                    let res =
+                        reflink_or_copy_with_progress(&source_path, &target, size, tx).await?;
+                    trace!(
+                        "exported {} also to {}, {res:?}",
+                        source_path.display(),
+                        target.display()
+                    );
+                    external.push(target);
+                    external.sort();
+                    external.dedup();
+                    external.truncate(MAX_EXTERNAL_PATHS);
+                } else {
+                    // the file was previously owned, so we can just move it.
+                    // if that fails with ERR_CROSS, we fall back to copy.
+                    match std::fs::rename(&source_path, &target) {
+                        Ok(()) => {}
+                        Err(cause) => {
+                            const ERR_CROSS: i32 = 18;
+                            if cause.raw_os_error() == Some(ERR_CROSS) {
+                                reflink_or_copy_with_progress(&source_path, &target, size, tx)
+                                    .await?;
+                            } else {
+                                return Err(cause.into());
+                            }
                         }
                     }
-                }
+                    external.push(target);
+                };
+                // setting the new entry state will also take care of deleting the owned data file!
                 ctx.set(EntryState::Complete {
-                    data_location: DataLocation::External(vec![target], size),
+                    data_location: DataLocation::External(external, size),
                     outboard_location,
                 })
                 .await?;
@@ -1421,6 +1446,12 @@ pub struct FsStore {
     db: tokio::sync::mpsc::Sender<InternalCommand>,
 }
 
+impl From<FsStore> for Store {
+    fn from(value: FsStore) -> Self {
+        Store::from_sender(value.sender)
+    }
+}
+
 impl Deref for FsStore {
     type Target = Store;
 

src/store/fs.rs

@@ -74,6 +74,12 @@ pub struct MemStore {
     client: ApiClient,
 }
 
+impl From<MemStore> for crate::api::Store {
+    fn from(value: MemStore) -> Self {
+        crate::api::Store::from_sender(value.client)
+    }
+}
+
 impl AsRef<crate::api::Store> for MemStore {
     fn as_ref(&self) -> &crate::api::Store {
         crate::api::Store::ref_from_sender(&self.client)

src/store/mem.rs

@@ -59,6 +59,18 @@ impl Deref for ReadonlyMemStore {
     }
 }
 
+impl From<ReadonlyMemStore> for crate::api::Store {
+    fn from(value: ReadonlyMemStore) -> Self {
+        crate::api::Store::from_sender(value.client)
+    }
+}
+
+impl AsRef<crate::api::Store> for ReadonlyMemStore {
+    fn as_ref(&self) -> &crate::api::Store {
+        crate::api::Store::ref_from_sender(&self.client)
+    }
+}
+
 struct Actor {
     commands: tokio::sync::mpsc::Receiver<proto::Command>,
     tasks: JoinSet<()>,