Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
260 changes: 168 additions & 92 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ tokio = { version = "1.43.0", features = ["full"] }
tokio-util = { version = "0.7.13", features = ["full"] }
tracing = "0.1.41"
iroh-io = "0.6.1"
rand = "0.8.5"
rand = "0.9.2"
hex = "0.4.3"
serde = "1.0.217"
postcard = { version = "1.1.1", features = ["experimental-derive", "use-std"] }
Expand Down
14 changes: 9 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ Here is a basic example of how to set up `iroh-blobs` with `iroh`:

```rust,no_run
use iroh::{protocol::Router, Endpoint};
use iroh_blobs::{store::mem::MemStore, BlobsProtocol};
use iroh_blobs::{store::mem::MemStore, BlobsProtocol, ticket::BlobTicket};

#[tokio::main]
async fn main() -> anyhow::Result<()> {
Expand All @@ -44,15 +44,19 @@ async fn main() -> anyhow::Result<()> {

// create a protocol handler using an in-memory blob store.
let store = MemStore::new();
let blobs = BlobsProtocol::new(&store, endpoint.clone(), None);
let tag = store.add_slice(b"Hello world").await?;

let _ = endpoint.online().await;
let addr = endpoint.node_addr();
let ticket = BlobTicket::new(addr, tag.hash, tag.format);

// build the router
let blobs = BlobsProtocol::new(&store, None);
let router = Router::builder(endpoint)
.accept(iroh_blobs::ALPN, blobs.clone())
.accept(iroh_blobs::ALPN, blobs)
.spawn();

let tag = blobs.add_slice(b"Hello world").await?;
println!("We are now serving {}", blobs.ticket(tag).await?);
println!("We are now serving {}", ticket);

// wait for control-c
tokio::signal::ctrl_c().await;
Expand Down
3 changes: 1 addition & 2 deletions examples/common/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,12 @@ pub fn get_or_generate_secret_key() -> Result<SecretKey> {
use std::{env, str::FromStr};

use anyhow::Context;
use rand::thread_rng;
if let Ok(secret) = env::var("IROH_SECRET") {
// Parse the secret key from string
SecretKey::from_str(&secret).context("Invalid secret key format")
} else {
// Generate a new random key
let secret_key = SecretKey::generate(&mut thread_rng());
let secret_key = SecretKey::generate(&mut rand::rng());
println!(
"Generated new secret key: {}",
hex::encode(secret_key.to_bytes())
Expand Down
2 changes: 1 addition & 1 deletion examples/custom-protocol.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ async fn listen(text: Vec<String>) -> Result<()> {
proto.insert_and_index(text).await?;
}
// Build the iroh-blobs protocol handler, which is used to download blobs.
let blobs = BlobsProtocol::new(&store, endpoint.clone(), None);
let blobs = BlobsProtocol::new(&store, None);

// create a router that handles both our custom protocol and the iroh-blobs protocol.
let node = Router::builder(endpoint)
Expand Down
4 changes: 2 additions & 2 deletions examples/expiring-tags.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,14 +162,14 @@ async fn main() -> anyhow::Result<()> {
let expires_at = SystemTime::now()
.checked_add(Duration::from_secs(10))
.unwrap();
create_expiring_tag(&store, &[*a.hash(), *b.hash()], "expiring", expires_at).await?;
create_expiring_tag(&store, &[a.hash(), b.hash()], "expiring", expires_at).await?;

// add a single blob and tag it with an expiry date 60 seconds in the future
let c = batch.add_bytes("blob 3".as_bytes()).await?;
let expires_at = SystemTime::now()
.checked_add(Duration::from_secs(60))
.unwrap();
create_expiring_tag(&store, &[*c.hash()], "expiring", expires_at).await?;
create_expiring_tag(&store, &[c.hash()], "expiring", expires_at).await?;
// batch goes out of scope, so data is only protected by the tags we created
}

Expand Down
12 changes: 6 additions & 6 deletions examples/limit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use std::{
use anyhow::Result;
use clap::Parser;
use common::setup_logging;
use iroh::{protocol::Router, NodeAddr, NodeId, SecretKey, Watcher};
use iroh::{protocol::Router, NodeAddr, NodeId, SecretKey};
use iroh_blobs::{
provider::events::{
AbortReason, ConnectMode, EventMask, EventSender, ProviderMessage, RequestMode,
Expand All @@ -31,7 +31,7 @@ use iroh_blobs::{
ticket::BlobTicket,
BlobFormat, BlobsProtocol, Hash,
};
use rand::thread_rng;
use rand::rng;

use crate::common::get_or_generate_secret_key;

Expand Down Expand Up @@ -255,7 +255,7 @@ async fn main() -> Result<()> {
let mut allowed_nodes = allowed_nodes.into_iter().collect::<HashSet<_>>();
if secrets > 0 {
println!("Generating {secrets} new secret keys for allowed nodes:");
let mut rand = thread_rng();
let mut rand = rng();
for _ in 0..secrets {
let secret = SecretKey::generate(&mut rand);
let public = secret.public();
Expand Down Expand Up @@ -357,9 +357,9 @@ async fn setup(store: MemStore, events: EventSender) -> Result<(Router, NodeAddr
.secret_key(secret)
.bind()
.await?;
let _ = endpoint.home_relay().initialized().await;
let addr = endpoint.node_addr().initialized().await;
let blobs = BlobsProtocol::new(&store, endpoint.clone(), Some(events));
endpoint.online().await;
let addr = endpoint.node_addr();
let blobs = BlobsProtocol::new(&store, Some(events));
let router = Router::builder(endpoint)
.accept(iroh_blobs::ALPN, blobs)
.spawn();
Expand Down
4 changes: 2 additions & 2 deletions examples/mdns-discovery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ async fn accept(path: &Path) -> Result<()> {
.await?;
let builder = Router::builder(endpoint.clone());
let store = MemStore::new();
let blobs = BlobsProtocol::new(&store, endpoint.clone(), None);
let blobs = BlobsProtocol::new(&store, None);
let builder = builder.accept(iroh_blobs::ALPN, blobs.clone());
let node = builder.spawn();

Expand All @@ -87,7 +87,7 @@ async fn accept(path: &Path) -> Result<()> {
}

async fn connect(node_id: PublicKey, hash: Hash, out: Option<PathBuf>) -> Result<()> {
let key = SecretKey::generate(rand::rngs::OsRng);
let key = SecretKey::generate(&mut rand::rng());
// todo: disable discovery publishing once https://github.com/n0-computer/iroh/issues/3401 is implemented
let discovery = MdnsDiscovery::builder();

Expand Down
22 changes: 13 additions & 9 deletions examples/random_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use std::{env, path::PathBuf, str::FromStr};

use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use iroh::{SecretKey, Watcher};
use iroh::{discovery::static_provider::StaticProvider, SecretKey};
use iroh_base::ticket::NodeTicket;
use iroh_blobs::{
api::downloader::Shuffled,
Expand Down Expand Up @@ -93,7 +93,7 @@ pub fn get_or_generate_secret_key() -> Result<SecretKey> {
SecretKey::from_str(&secret).context("Invalid secret key format")
} else {
// Generate a new random key
let secret_key = SecretKey::generate(&mut rand::thread_rng());
let secret_key = SecretKey::generate(&mut rand::rng());
let secret_key_str = hex::encode(secret_key.to_bytes());
println!("Generated new random secret key");
println!("To reuse this key, set the IROH_SECRET={secret_key_str}");
Expand Down Expand Up @@ -204,20 +204,20 @@ async fn provide(args: ProvideArgs) -> anyhow::Result<()> {
println!("Using store at: {}", path.display());
let mut rng = match args.common.seed {
Some(seed) => StdRng::seed_from_u64(seed),
None => StdRng::from_entropy(),
None => StdRng::from_rng(&mut rand::rng()),
};
let blobs = create_random_blobs(
&store,
args.num_blobs,
|_, rand| rand.gen_range(1..=args.blob_size),
|_, rand| rand.random_range(1..=args.blob_size),
&mut rng,
)
.await?;
let hs = add_hash_sequences(
&store,
&blobs,
args.hash_seqs,
|_, rand| rand.gen_range(1..=args.hash_seq_size),
|_, rand| rand.random_range(1..=args.hash_seq_size),
&mut rng,
)
.await?;
Expand All @@ -238,11 +238,11 @@ async fn provide(args: ProvideArgs) -> anyhow::Result<()> {
.bind()
.await?;
let (dump_task, events_tx) = dump_provider_events(args.allow_push);
let blobs = iroh_blobs::BlobsProtocol::new(&store, endpoint.clone(), Some(events_tx));
let blobs = iroh_blobs::BlobsProtocol::new(&store, Some(events_tx));
let router = iroh::protocol::Router::builder(endpoint.clone())
.accept(iroh_blobs::ALPN, blobs)
.spawn();
let addr = router.endpoint().node_addr().initialized().await;
let addr = router.endpoint().node_addr();
let ticket = NodeTicket::from(addr.clone());
println!("Node address: {addr:?}");
println!("ticket:\n{ticket}");
Expand All @@ -265,10 +265,14 @@ async fn request(args: RequestArgs) -> anyhow::Result<()> {
.unwrap_or_else(|| tempdir.as_ref().unwrap().path().to_path_buf());
let store = FsStore::load(&path).await?;
println!("Using store at: {}", path.display());
let endpoint = iroh::Endpoint::builder().bind().await?;
let sp = StaticProvider::new();
let endpoint = iroh::Endpoint::builder()
.discovery(sp.clone())
.bind()
.await?;
let downloader = store.downloader(&endpoint);
for ticket in &args.nodes {
endpoint.add_node_addr(ticket.node_addr().clone())?;
sp.add_node_info(ticket.node_addr().clone());
}
let nodes = args
.nodes
Expand Down
15 changes: 7 additions & 8 deletions examples/transfer-collection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,7 @@
use std::collections::HashMap;

use anyhow::{Context, Result};
use iroh::{
discovery::static_provider::StaticProvider, protocol::Router, Endpoint, NodeAddr, Watcher,
};
use iroh::{discovery::static_provider::StaticProvider, protocol::Router, Endpoint, NodeAddr};
use iroh_blobs::{
api::{downloader::Shuffled, Store, TempTag},
format::collection::Collection,
Expand Down Expand Up @@ -38,7 +36,7 @@ impl Node {

// this BlobsProtocol accepts connections from other nodes and serves blobs from the store
// we pass None to skip subscribing to request events
let blobs = BlobsProtocol::new(&store, endpoint.clone(), None);
let blobs = BlobsProtocol::new(&store, None);
// Routers group one or more protocols together to accept connections from other nodes,
// here we're only using one, but could add more in a real world use case as needed
let router = Router::builder(endpoint)
Expand All @@ -54,7 +52,8 @@ impl Node {
// get address of this node. Has the side effect of waiting for the node
// to be online & ready to accept connections
async fn node_addr(&self) -> Result<NodeAddr> {
let addr = self.router.endpoint().node_addr().initialized().await;
self.router.endpoint().online().await;
let addr = self.router.endpoint().node_addr();
Ok(addr)
}

Expand All @@ -80,14 +79,14 @@ impl Node {

let collection_items = collection_items
.iter()
.map(|(name, tag)| (name.to_string(), *tag.hash()))
.map(|(name, tag)| (name.to_string(), tag.hash()))
.collect::<Vec<_>>();

let collection = Collection::from_iter(collection_items);

let tt = collection.store(&self.store).await?;
self.store.tags().create(*tt.hash_and_format()).await?;
Ok(*tt.hash())
self.store.tags().create(tt.hash_and_format()).await?;
Ok(tt.hash())
}

/// retrieve an entire collection from a given hash and provider
Expand Down
2 changes: 1 addition & 1 deletion examples/transfer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ async fn main() -> anyhow::Result<()> {
// We initialize an in-memory backing store for iroh-blobs
let store = MemStore::new();
// Then we initialize a struct that can accept blobs requests over iroh connections
let blobs = BlobsProtocol::new(&store, endpoint.clone(), None);
let blobs = BlobsProtocol::new(&store, None);

// Grab all passed in arguments, the first one is the binary itself, so we skip it.
let args: Vec<String> = std::env::args().skip(1).collect();
Expand Down
1 change: 1 addition & 0 deletions proptest-regressions/store/fs/util/entity_manager.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc 0f2ebc49ab2f84e112f08407bb94654fbcb1f19050a4a8a6196383557696438a # shrinks to input = _TestCountersManagerProptestFsArgs { entries: [(15313427648878534792, 264348813928009031854006459208395772047), (1642534478798447378, 15989109311941500072752977306696275871), (8755041673862065815, 172763711808688570294350362332402629716), (4993597758667891804, 114145440157220458287429360639759690928), (15031383154962489250, 63217081714858286463391060323168548783), (17668469631267503333, 11878544422669770587175118199598836678), (10507570291819955314, 126584081645379643144412921692654648228), (3979008599365278329, 283717221942996985486273080647433218905), (8316838360288996639, 334043288511621783152802090833905919408), (15673798930962474157, 77551315511802713260542200115027244708), (12058791254144360414, 56638044274259821850511200885092637649), (8191628769638031337, 314181956273420400069887649110740549194), (6290369460137232066, 255779791286732775990301011955519176773), (11919824746661852269, 319400891587146831511371932480749645441), (12491631698789073154, 271279849791970841069522263758329847554), (53891048909263304, 12061234604041487609497959407391945555), (9486366498650667097, 311383186592430597410801882015456718030), (15696332331789302593, 306911490707714340526403119780178604150), (8699088947997536151, 312272624973367009520183311568498652066), (1144772544750976199, 200591877747619565555594857038887015), (5907208586200645081, 299942008952473970881666769409865744975), (3384528743842518913, 26230956866762934113564101494944411446), (13877357832690956494, 229457597607752760006918374695475345151), (2965687966026226090, 306489188264741716662410004273408761623), (13624286905717143613, 232801392956394366686194314010536008033), (3622356130274722018, 162030840677521022192355139208505458492), (17807768575470996347, 264107246314713159406963697924105744409), (5103434150074147746, 331686166459964582006209321975587627262), (5962771466034321974, 300961804728115777587520888809168362574), (2930645694242691907, 127752709774252686733969795258447263979), (16197574560597474644, 245410120683069493317132088266217906749), (12478835478062365617, 103838791113879912161511798836229961653), (5503595333662805357, 92368472243854403026472376408708548349), (18122734335129614364, 288955542597300001147753560885976966029), (12688080215989274550, 85237436689682348751672119832134138932), (4148468277722853958, 297778117327421209654837771300216669574), (8749445804640085302, 79595866493078234154562014325793780126), (12442730869682574563, 196176786402808588883611974143577417817), (6110644747049355904, 26592587989877021920275416199052685135), (5851164380497779369, 158876888501825038083692899057819261957), (9497384378514985275, 15279835675313542048650599472403150097), (10661092311826161857, 250089949043892591422587928179995867509), (10046856000675345423, 231369150063141386398059701278066296663)] }
cc 76888f93675aca856046821142e0f8dd6171ecbca2b2fb2612e2ccf8fb642b67 # shrinks to input = _TestCountersManagerProptestFsArgs { entries: [(4306300120905349883, 44028232064888275756989554345798606606), (13419562989696853297, 297225061196384743010175600480992461777), (4600545388725048575, 319024777944692442173521074338932622027), (11924469201417769946, 290126334103578499810346516670302802842), (2150076364877215359, 213957508179788124392023233632127334025), (2513497990495955776, 7425952384271563468605443743299630055), (14784519504379667574, 209102176380410663068514976101053847121), (3589018664409806533, 143539073128281654988615675279132949539), (12163255676316221910, 68261431317828245529088264283730310447), (15953238975034584216, 120566915371382433441278003421157478859), (6293912069208757821, 54376221216199661139416453798278484358), (18408187014091379100, 160227239986709222921681152272167766516), (18224691851384849998, 230951397761410506492316028434133464542), (17218108759165771012, 230831401271946284847544140042531898300), (15156861699203125197, 274419864858876512298091294679889505416), (13197866550741263112, 317569618673855709115791823801131083319), (5457536710317675425, 264100465594513117047187960359952352601), (6419381816113193473, 97830434597410923324208428511886405696), (5509774606527762921, 51377792339839665748346223023626770993), (3302884055341784375, 260024947302198645578544387819129813215), (7918740211035003255, 281378863798916751001154282897883115117), (2107923747770684554, 4222310695795814822585776810386837522), (1988849030739458584, 97044202427348897203209230721452399078), (17000851872752693509, 154967569583821344066124364203881263442), (7204118357407989275, 293489743217018103289756063378018736213), (8379490247240411923, 91291993991616380545421710143276496062), (6067846780114877285, 117378294706679402333724324394932467070), (6559069473214523762, 330533491611532325905048043451453362184), (1066716766275783661, 14900329515024496203681878322771717089), (3969544049792556621, 299925942970250984690757497097936404520), (1871651009149288279, 269532663769476813929854896620535419927), (9885923542173402939, 332347180744841318697161540487151553089), (8743551960605987234, 82895354018256482956918848969653357161), (18444906840677790884, 140951189435890586485485914583535891710), (13186290687428042898, 156781959554744750775008814037900689629), (11253344694094324994, 173003087909699540403477415680185472166), (15359595929118467798, 334133929399407497923349560480857143925), (450753775453578376, 185062409187456936422223327885008555109), (5812669297982828223, 304450764862712727874277633964000192257), (5446431204912329700, 63591795618582560687940570634859474113), (12639950240321649272, 229465965587199764990249271930115998317), (8666241046976392242, 127169189810538544860066577390902103071), (15875344269296451901, 59314152116324788008302123296358029667), (17554612189790211905, 271354287586940637417955997246049015908), (2654666284440384247, 236192749343056755001648024964710799784), (3653085434641832523, 222611620216912476618464093834705618103), (2117280733558696133, 160273208193736809842040581629127362879), (15052687776534295171, 145937384428000340885721647247111254565), (14037243134892329831, 48648195516567212103580801887048711483), (9467080097152043608, 266945396762492281384357764614500138375), (2706297963598729254, 301505662334146630272416432816290497813), (7293916663622670946, 238683745638275436602208159421396156156), (9244966065396610028, 33307016963739390689548576588029894837), (1752320522681001931, 67331614351445449534791948958610485134), (13095820849418318043, 167220720368084276476264354546008346754), (2689852485877961108, 295988764749889891843145129746265206397), (16677044930197861079, 238123490797857333537723337779861037465), (1921976638111110551, 198905043115016585827638257647548833710), (78362912300221566, 97081461393166374265589962390002181072), (3959569947932321574, 224306094090967444142819090846108416832), (11193248764198058671, 209017727259932159026175830711818202266), (6959892815010617835, 209133472960436703368896187256879102139), (10121904169365490638, 120711360828413383714152810706442997143), (15460955954420808897, 303801388017089859688481259123309944609)] }
8 changes: 4 additions & 4 deletions src/api/blobs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -656,20 +656,20 @@ impl<'a> AddProgress<'a> {
pub async fn with_named_tag(self, name: impl AsRef<[u8]>) -> RequestResult<HashAndFormat> {
let blobs = self.blobs.clone();
let tt = self.temp_tag().await?;
let haf = *tt.hash_and_format();
let haf = tt.hash_and_format();
let tags = Tags::ref_from_sender(&blobs.client);
tags.set(name, *tt.hash_and_format()).await?;
tags.set(name, haf).await?;
drop(tt);
Ok(haf)
}

pub async fn with_tag(self) -> RequestResult<TagInfo> {
let blobs = self.blobs.clone();
let tt = self.temp_tag().await?;
let hash = *tt.hash();
let hash = tt.hash();
let format = tt.format();
let tags = Tags::ref_from_sender(&blobs.client);
let name = tags.create(*tt.hash_and_format()).await?;
let name = tags.create(tt.hash_and_format()).await?;
drop(tt);
Ok(TagInfo { name, hash, format })
}
Expand Down
Loading
Loading