From 7a1f14b8ec6beac885818bf4175b241b9351b0e7 Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 2 Oct 2025 17:40:15 -0400 Subject: [PATCH 01/10] refactor(gc): move store::fs::gc -> store::gc --- src/store/fs.rs | 3 +-- src/store/fs/options.rs | 2 +- src/store/{fs => }/gc.rs | 0 src/store/mod.rs | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) rename src/store/{fs => }/gc.rs (100%) diff --git a/src/store/fs.rs b/src/store/fs.rs index 48946abd6..9aa1aa56c 100644 --- a/src/store/fs.rs +++ b/src/store/fs.rs @@ -91,7 +91,6 @@ use bytes::Bytes; use delete_set::{BaoFilePart, ProtectHandle}; use entity_manager::{EntityManagerState, SpawnArg}; use entry_state::{DataLocation, OutboardLocation}; -use gc::run_gc; use import::{ImportEntry, ImportSource}; use irpc::{channel::mpsc, RpcMessage}; use meta::list_blobs; @@ -120,6 +119,7 @@ use crate::{ }, util::entity_manager::{self, ActiveEntityState}, }, + gc::run_gc, util::{BaoTreeSender, FixedSize, MemOrFile, ValueOrPoisioned}, IROH_BLOCK_SIZE, }, @@ -141,7 +141,6 @@ use entry_state::EntryState; use import::{import_byte_stream, import_bytes, import_path, ImportEntryMsg}; use options::Options; use tracing::Instrument; -mod gc; use crate::{ api::{ diff --git a/src/store/fs/options.rs b/src/store/fs/options.rs index f7dfa82f6..afd723c5b 100644 --- a/src/store/fs/options.rs +++ b/src/store/fs/options.rs @@ -4,8 +4,8 @@ use std::{ time::Duration, }; -pub use super::gc::{GcConfig, ProtectCb, ProtectOutcome}; use super::{meta::raw_outboard_size, temp_name}; +pub use crate::store::gc::{GcConfig, ProtectCb, ProtectOutcome}; use crate::Hash; /// Options for directories used by the file store. diff --git a/src/store/fs/gc.rs b/src/store/gc.rs similarity index 100% rename from src/store/fs/gc.rs rename to src/store/gc.rs diff --git a/src/store/mod.rs b/src/store/mod.rs index 4fdb30606..9d7290da5 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -7,6 +7,7 @@ use bao_tree::BlockSize; #[cfg(feature = "fs-store")] pub mod fs; +mod gc; pub mod mem; pub mod readonly_mem; mod test; From 9a9dc2b582fd99cfc2d656e524457686b30f319e Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 2 Oct 2025 17:52:23 -0400 Subject: [PATCH 02/10] feat(MemStore): add GC option, impl new_with_opts --- src/store/mem.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/store/mem.rs b/src/store/mem.rs index e5529e7fa..b5a3a8ce6 100644 --- a/src/store/mem.rs +++ b/src/store/mem.rs @@ -58,6 +58,7 @@ use crate::{ }, protocol::ChunkRangesExt, store::{ + gc::{run_gc, GcConfig}, util::{SizeInfo, SparseMemFile, Tag}, IROH_BLOCK_SIZE, }, @@ -66,7 +67,9 @@ use crate::{ }; #[derive(Debug, Default)] -pub struct Options {} +pub struct Options { + pub gc_config: Option, +} #[derive(Debug, Clone)] #[repr(transparent)] @@ -113,6 +116,10 @@ impl MemStore { } pub fn new() -> Self { + Self::new_with_opts(Options::default()) + } + + pub fn new_with_opts(opts: Options) -> Self { let (sender, receiver) = tokio::sync::mpsc::channel(32); tokio::spawn( Actor { @@ -130,7 +137,13 @@ impl MemStore { } .run(), ); - Self::from_sender(sender.into()) + + let store = Self::from_sender(sender.into()); + if let Some(gc_config) = opts.gc_config { + tokio::spawn(run_gc(store.deref().clone(), gc_config)); + } + + store } } From f0276f16362ba4b24805be1a834da96ea86fff77 Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 2 Oct 2025 18:17:35 -0400 Subject: [PATCH 03/10] refactor(store): move create_n0_bao test util into util::tests::create_n0_bao --- src/api/blobs/reader.rs | 3 ++- src/api/remote.rs | 3 ++- src/store/fs.rs | 18 ++---------------- src/store/gc.rs | 2 +- src/store/util.rs | 18 ++++++++++++++++++ src/tests.rs | 4 ++-- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/api/blobs/reader.rs b/src/api/blobs/reader.rs index 5077c2632..294d916ef 100644 --- a/src/api/blobs/reader.rs +++ b/src/api/blobs/reader.rs @@ -225,10 +225,11 @@ mod tests { protocol::ChunkRangesExt, store::{ fs::{ - tests::{create_n0_bao, test_data, INTERESTING_SIZES}, + tests::{test_data, INTERESTING_SIZES}, FsStore, }, mem::MemStore, + util::tests::create_n0_bao, }, }; diff --git a/src/api/remote.rs b/src/api/remote.rs index dcfbc4fb4..09754d566 100644 --- a/src/api/remote.rs +++ b/src/api/remote.rs @@ -1073,10 +1073,11 @@ mod tests { protocol::{ChunkRangesExt, ChunkRangesSeq, GetRequest}, store::{ fs::{ - tests::{create_n0_bao, test_data, INTERESTING_SIZES}, + tests::{test_data, INTERESTING_SIZES}, FsStore, }, mem::MemStore, + util::tests::create_n0_bao, }, tests::{add_test_hash_seq, add_test_hash_seq_incomplete}, }; diff --git a/src/store/fs.rs b/src/store/fs.rs index 9aa1aa56c..46d391178 100644 --- a/src/store/fs.rs +++ b/src/store/fs.rs @@ -1497,10 +1497,7 @@ pub mod tests { use core::panic; use std::collections::{HashMap, HashSet}; - use bao_tree::{ - io::{outboard::PreOrderMemOutboard, round_up_to_chunks_groups}, - ChunkRanges, - }; + use bao_tree::{io::round_up_to_chunks_groups, ChunkRanges}; use n0_future::{stream, Stream, StreamExt}; use testresult::TestResult; use walkdir::WalkDir; @@ -1509,7 +1506,7 @@ pub mod tests { use crate::{ api::blobs::Bitfield, store::{ - util::{read_checksummed, SliceInfoExt, Tag}, + util::{read_checksummed, tests::create_n0_bao, SliceInfoExt, Tag}, IROH_BLOCK_SIZE, }, }; @@ -1526,17 +1523,6 @@ pub mod tests { 1024 * 1024 * 8, // data file, outboard file ]; - /// Create n0 flavoured bao. Note that this can be used to request ranges below a chunk group size, - /// which can not be exported via bao because we don't store hashes below the chunk group level. - pub fn create_n0_bao(data: &[u8], ranges: &ChunkRanges) -> anyhow::Result<(Hash, Vec)> { - let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE); - let mut encoded = Vec::new(); - let size = data.len() as u64; - encoded.extend_from_slice(&size.to_le_bytes()); - bao_tree::io::sync::encode_ranges_validated(data, &outboard, ranges, &mut encoded)?; - Ok((outboard.root.into(), encoded)) - } - pub fn round_up_request(size: u64, ranges: &ChunkRanges) -> ChunkRanges { let last_chunk = ChunkNum::chunks(size); let data_range = ChunkRanges::from(..last_chunk); diff --git a/src/store/gc.rs b/src/store/gc.rs index da7836e76..06d5e4df1 100644 --- a/src/store/gc.rs +++ b/src/store/gc.rs @@ -253,7 +253,7 @@ mod tests { use crate::{ api::{blobs::AddBytesOptions, ExportBaoError, RequestError, Store}, hashseq::HashSeq, - store::fs::{options::PathOptions, tests::create_n0_bao}, + store::{fs::options::PathOptions, util::tests::create_n0_bao}, BlobFormat, }; diff --git a/src/store/util.rs b/src/store/util.rs index 7bc3a3227..1acc0c4da 100644 --- a/src/store/util.rs +++ b/src/store/util.rs @@ -404,3 +404,21 @@ impl bao_tree::io::mixed::Sender for BaoTreeSender { self.0.send(item).await } } + +#[cfg(test)] +pub mod tests { + use crate::hash::Hash; + use crate::store::IROH_BLOCK_SIZE; + use bao_tree::{io::outboard::PreOrderMemOutboard, ChunkRanges}; + + /// Create n0 flavoured bao. Note that this can be used to request ranges below a chunk group size, + /// which can not be exported via bao because we don't store hashes below the chunk group level. + pub fn create_n0_bao(data: &[u8], ranges: &ChunkRanges) -> anyhow::Result<(Hash, Vec)> { + let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE); + let mut encoded = Vec::new(); + let size = data.len() as u64; + encoded.extend_from_slice(&size.to_le_bytes()); + bao_tree::io::sync::encode_ranges_validated(data, &outboard, ranges, &mut encoded)?; + Ok((outboard.root.into(), encoded)) + } +} diff --git a/src/tests.rs b/src/tests.rs index 09b2e5b33..c280f0004 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -19,11 +19,11 @@ use crate::{ provider::events::{AbortReason, EventMask, EventSender, ProviderMessage, RequestUpdate}, store::{ fs::{ - tests::{create_n0_bao, test_data, INTERESTING_SIZES}, + tests::{test_data, INTERESTING_SIZES}, FsStore, }, mem::MemStore, - util::observer::Combine, + util::{observer::Combine, tests::create_n0_bao}, }, util::sink::Drain, BlobFormat, Hash, HashAndFormat, From f5d808007a80d13177103259e9677b2335cad5f0 Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 2 Oct 2025 23:27:28 -0400 Subject: [PATCH 04/10] test(gc): ensure adding and deleting a tag doesn't affect gc --- src/store/gc.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/store/gc.rs b/src/store/gc.rs index 06d5e4df1..8fdef4fe2 100644 --- a/src/store/gc.rs +++ b/src/store/gc.rs @@ -266,6 +266,7 @@ mod tests { let et = blobs.add_slice("e").temp_tag().await?; let ft = blobs.add_slice("f").temp_tag().await?; let gt = blobs.add_slice("g").temp_tag().await?; + let ht = blobs.add_slice("h").with_named_tag("h").await?; let a = *at.hash(); let b = *bt.hash(); let c = *ct.hash(); @@ -273,6 +274,7 @@ mod tests { let e = *et.hash(); let f = *ft.hash(); let g = *gt.hash(); + let h = ht.hash; store.tags().set("c", *ct.hash_and_format()).await?; let dehs = [d, e].into_iter().collect::(); let hehs = blobs @@ -292,6 +294,7 @@ mod tests { store.tags().set("fg", *fghs.hash_and_format()).await?; drop(fghs); drop(bt); + store.tags().delete(ht.name).await?; let mut live = HashSet::new(); gc_run_once(store, &mut live).await?; // a is protected because we keep the temp tag @@ -313,6 +316,9 @@ mod tests { assert!(store.has(f).await?); assert!(live.contains(&g)); assert!(store.has(g).await?); + // h is not protected because we deleted the tag before gc ran + assert!(!live.contains(&h)); + assert!(!store.has(h).await?); drop(at); drop(hehs); Ok(()) From 19ea6eaa153088a4ec65d2360ca09dcb56ef76a5 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 6 Oct 2025 17:17:28 +0300 Subject: [PATCH 05/10] fmt --- src/store/util.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/store/util.rs b/src/store/util.rs index 1acc0c4da..567a6cf82 100644 --- a/src/store/util.rs +++ b/src/store/util.rs @@ -407,10 +407,10 @@ impl bao_tree::io::mixed::Sender for BaoTreeSender { #[cfg(test)] pub mod tests { - use crate::hash::Hash; - use crate::store::IROH_BLOCK_SIZE; use bao_tree::{io::outboard::PreOrderMemOutboard, ChunkRanges}; + use crate::{hash::Hash, store::IROH_BLOCK_SIZE}; + /// Create n0 flavoured bao. Note that this can be used to request ranges below a chunk group size, /// which can not be exported via bao because we don't store hashes below the chunk group level. pub fn create_n0_bao(data: &[u8], ranges: &ChunkRanges) -> anyhow::Result<(Hash, Vec)> { From 3668422e4ecde970c0f37d2a83f753f0b31eadf7 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 6 Oct 2025 17:34:18 +0300 Subject: [PATCH 06/10] proper feature gating --- src/api/blobs.rs | 2 -- src/store/gc.rs | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/blobs.rs b/src/api/blobs.rs index 897e0371c..4f8384cca 100644 --- a/src/api/blobs.rs +++ b/src/api/blobs.rs @@ -144,7 +144,6 @@ impl Blobs { /// clears the protections before. /// /// Users should rely only on garbage collection for blob deletion. - #[cfg(feature = "fs-store")] pub(crate) async fn delete_with_opts(&self, options: DeleteOptions) -> RequestResult<()> { trace!("{options:?}"); self.client.rpc(options).await??; @@ -152,7 +151,6 @@ impl Blobs { } /// See [`Self::delete_with_opts`]. - #[cfg(feature = "fs-store")] pub(crate) async fn delete( &self, hashes: impl IntoIterator>, diff --git a/src/store/gc.rs b/src/store/gc.rs index 8fdef4fe2..f8288453d 100644 --- a/src/store/gc.rs +++ b/src/store/gc.rs @@ -294,7 +294,7 @@ mod tests { store.tags().set("fg", *fghs.hash_and_format()).await?; drop(fghs); drop(bt); - store.tags().delete(ht.name).await?; + store.tags().delete("h").await?; let mut live = HashSet::new(); gc_run_once(store, &mut live).await?; // a is protected because we keep the temp tag @@ -372,6 +372,7 @@ mod tests { } #[tokio::test] + #[cfg(feature = "fs-store")] async fn gc_smoke_fs() -> TestResult { tracing_subscriber::fmt::try_init().ok(); let testdir = tempfile::tempdir()?; @@ -391,6 +392,7 @@ mod tests { } #[tokio::test] + #[cfg(feature = "fs-store")] async fn gc_check_deletion_fs() -> TestResult { tracing_subscriber::fmt::try_init().ok(); let testdir = tempfile::tempdir()?; From e96d78f3f006fcf2bf768e972f5bc2b59929661c Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 6 Oct 2025 17:42:03 +0300 Subject: [PATCH 07/10] allow iroh main dep --- deny.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deny.toml b/deny.toml index bb2a4118f..85be20882 100644 --- a/deny.toml +++ b/deny.toml @@ -39,3 +39,8 @@ name = "ring" [[licenses.clarify.license-files]] hash = 3171872035 path = "LICENSE" + +[sources] +allow-git = [ + "https://github.com/n0-computer/iroh", +] \ No newline at end of file From d112bae9269d8c6ea2f9c92fd801b8a1b563b97f Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 6 Oct 2025 17:45:27 +0300 Subject: [PATCH 08/10] more feature gating --- src/store/gc.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/store/gc.rs b/src/store/gc.rs index f8288453d..abb9903e4 100644 --- a/src/store/gc.rs +++ b/src/store/gc.rs @@ -240,12 +240,9 @@ pub async fn run_gc(store: Store, config: GcConfig) { #[cfg(test)] mod tests { - use std::{ - io::{self}, - path::Path, - }; + use std::io::{self}; - use bao_tree::{io::EncodeError, ChunkNum}; + use bao_tree::io::EncodeError; use range_collections::RangeSet2; use testresult::TestResult; @@ -253,7 +250,6 @@ mod tests { use crate::{ api::{blobs::AddBytesOptions, ExportBaoError, RequestError, Store}, hashseq::HashSeq, - store::{fs::options::PathOptions, util::tests::create_n0_bao}, BlobFormat, }; @@ -324,7 +320,11 @@ mod tests { Ok(()) } - async fn gc_file_delete(path: &Path, store: &Store) -> TestResult<()> { + #[cfg(feature = "fs-store")] + async fn gc_file_delete(path: &std::path::Path, store: &Store) -> TestResult<()> { + use bao_tree::ChunkNum; + + use crate::store::{fs::options::PathOptions, util::tests::create_n0_bao}; let mut live = HashSet::new(); let options = PathOptions::new(&path.join("db")); // create a large complete file and check that the data and outboard files are deleted by gc From c02685b30f7323617df37157cbb721fa57d652a7 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 6 Oct 2025 18:13:51 +0300 Subject: [PATCH 09/10] one more --- src/store/util.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/store/util.rs b/src/store/util.rs index 567a6cf82..7ae8610af 100644 --- a/src/store/util.rs +++ b/src/store/util.rs @@ -413,6 +413,7 @@ pub mod tests { /// Create n0 flavoured bao. Note that this can be used to request ranges below a chunk group size, /// which can not be exported via bao because we don't store hashes below the chunk group level. + #[cfg(feature = "fs-store")] pub fn create_n0_bao(data: &[u8], ranges: &ChunkRanges) -> anyhow::Result<(Hash, Vec)> { let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE); let mut encoded = Vec::new(); From 4bfab0e9cd47dcbba229cd82f14fb1615b55b6cd Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 6 Oct 2025 18:17:44 +0300 Subject: [PATCH 10/10] aaargh! --- src/store/util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/store/util.rs b/src/store/util.rs index 7ae8610af..03be152bb 100644 --- a/src/store/util.rs +++ b/src/store/util.rs @@ -406,6 +406,7 @@ impl bao_tree::io::mixed::Sender for BaoTreeSender { } #[cfg(test)] +#[cfg(feature = "fs-store")] pub mod tests { use bao_tree::{io::outboard::PreOrderMemOutboard, ChunkRanges}; @@ -413,7 +414,6 @@ pub mod tests { /// Create n0 flavoured bao. Note that this can be used to request ranges below a chunk group size, /// which can not be exported via bao because we don't store hashes below the chunk group level. - #[cfg(feature = "fs-store")] pub fn create_n0_bao(data: &[u8], ranges: &ChunkRanges) -> anyhow::Result<(Hash, Vec)> { let outboard = PreOrderMemOutboard::create(data, IROH_BLOCK_SIZE); let mut encoded = Vec::new();