diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b957da7..7f072457 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ env: RUSTFLAGS: -Dwarnings RUSTDOCFLAGS: -Dwarnings MSRV: "1.72" - RS_EXAMPLES_LIST: "content-tracker,iroh-ipfs,dumbpipe-web,iroh-pkarr-node-discovery" + RS_EXAMPLES_LIST: "iroh-mainline-content-discovery,iroh-ipfs,dumbpipe-web,iroh-pkarr-node-discovery" GO_EXAMPLES_LIST: "dall_e_worker" jobs: diff --git a/.gitignore b/.gitignore index 74f8bc23..44f1444b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ target/ *iroh-data -*/.sendme* \ No newline at end of file +*/.sendme* + +# Added by cargo + +/target diff --git a/content-tracker/src/discovery.rs b/content-tracker/src/discovery.rs deleted file mode 100644 index b52d2239..00000000 --- a/content-tracker/src/discovery.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! Discovery services -//! -//! Originally copied and adapted from -use std::{ - collections::BTreeSet, - net::{IpAddr, SocketAddr}, -}; - -use anyhow::Result; -use futures::{future, future::BoxFuture, FutureExt}; -use iroh::net::{AddrInfo, NodeAddr}; -use pkarr::{ - dns::rdata::{RData, A, AAAA, TXT}, - dns::{Name, Packet, ResourceRecord, CLASS}, - url::Url, - Keypair, PkarrClient, SignedPacket, -}; -use tracing::{info, warn}; - -use crate::NodeId; - -const DERP_REGION_KEY: &str = "_derp_region.iroh."; - -#[allow(dead_code)] -const PKARR_RELAY_URL: &str = "https://iroh-discovery.rklaehn.workers.dev/"; - -#[allow(unused)] -fn filter_ipaddr(rr: &ResourceRecord) -> Option { - if rr.class != CLASS::IN { - return None; - } - let addr: IpAddr = match rr.rdata { - RData::A(A { address }) => IpAddr::V4(address.into()), - RData::AAAA(AAAA { address }) => IpAddr::V6(address.into()), - _ => return None, - }; - Some(addr) -} - -fn filter_txt(rr: &ResourceRecord) -> Option { - if rr.class != CLASS::IN { - return None; - } - if let RData::TXT(txt) = &rr.rdata { - String::try_from(txt.clone()).ok() - } else { - None - } -} - -fn filter_u16(rr: &ResourceRecord) -> Option { - if rr.class != CLASS::IN { - return None; - } - if let RData::A(A { address }) = rr.rdata { - Some(address as _) - } else { - None - } -} - -fn packet_to_node_addr(node_id: &NodeId, packet: &SignedPacket) -> NodeAddr { - let direct_addresses = packet - .resource_records("@") - .filter_map(filter_txt) - .filter_map(|addr| addr.parse().ok()) - .collect::>(); - let derp_region = packet - .resource_records(DERP_REGION_KEY) - .find_map(filter_u16); - NodeAddr { - node_id: *node_id, - info: AddrInfo { - derp_region, - direct_addresses, - }, - } -} - -fn node_addr_to_packet(keypair: &Keypair, info: &AddrInfo, ttl: u32) -> Result { - let mut packet = Packet::new_reply(0); - for addr in &info.direct_addresses { - let addr = addr.to_string(); - packet.answers.push(ResourceRecord::new( - Name::new("@").unwrap(), - CLASS::IN, - ttl, - RData::TXT(TXT::try_from(addr.as_str())?.into_owned()), - )); - } - if let Some(derp_region) = info.derp_region { - packet.answers.push(ResourceRecord::new( - Name::new(DERP_REGION_KEY).unwrap(), - CLASS::IN, - ttl, - RData::A(A { - address: derp_region as _, - }), - )); - } - Ok(SignedPacket::from_packet(keypair, &packet)?) -} - -/// A discovery method that uses the pkarr DNS protocol. See pkarr.org for more -/// information. -/// -/// This is using pkarr via a simple http relay or self-contained server. -#[derive(Debug)] -pub struct PkarrRelayDiscovery { - keypair: pkarr::Keypair, - relay: Url, - client: PkarrClient, -} - -impl PkarrRelayDiscovery { - #[allow(dead_code)] - pub fn new(secret_key: iroh::net::key::SecretKey, relay: Url) -> Self { - let keypair = pkarr::Keypair::from_secret_key(&secret_key.to_bytes()); - Self { - keypair, - relay, - client: PkarrClient::new(), - } - } -} - -impl iroh::net::magicsock::Discovery for PkarrRelayDiscovery { - fn publish(&self, info: &AddrInfo) { - info!("publishing {:?} via {}", info, self.relay); - let signed_packet = node_addr_to_packet(&self.keypair, info, 0).unwrap(); - let client = self.client.clone(); - let relay = self.relay.clone(); - tokio::spawn(async move { - let res = client.relay_put(&relay, signed_packet).await; - if let Err(e) = res { - warn!("error publishing: {}", e); - } else { - info!("done publishing"); - } - }); - } - - fn resolve<'a>( - &'a self, - node_id: &'a NodeId, - ) -> futures::future::BoxFuture<'a, Result> { - async move { - info!("resolving {} via {}", node_id, self.relay); - let pkarr_public_key = pkarr::PublicKey::try_from(*node_id.as_bytes()).unwrap(); - let packet = self.client.relay_get(&self.relay, pkarr_public_key).await?; - let addr = packet_to_node_addr(node_id, &packet); - info!("resolved: {} to {:?}", node_id, addr); - Ok(addr.info) - } - .boxed() - } -} - -/// A discovery method that just uses a hardcoded region. -#[derive(Debug)] -pub struct HardcodedRegionDiscovery { - region: u16, -} - -impl HardcodedRegionDiscovery { - /// Create a new discovery method that always returns the given region. - pub fn new(region: u16) -> Self { - Self { region } - } -} - -impl iroh::net::magicsock::Discovery for HardcodedRegionDiscovery { - fn publish(&self, _info: &AddrInfo) {} - - fn resolve<'a>(&'a self, _node_id: &'a NodeId) -> BoxFuture<'a, Result> { - future::ok(AddrInfo { - derp_region: Some(self.region), - direct_addresses: Default::default(), - }) - .boxed() - } -} diff --git a/content-tracker/src/main.rs b/content-tracker/src/main.rs deleted file mode 100644 index 0dff38f8..00000000 --- a/content-tracker/src/main.rs +++ /dev/null @@ -1,260 +0,0 @@ -pub mod args; -pub mod discovery; -pub mod io; -pub mod iroh_bytes_util; -pub mod options; -pub mod protocol; -pub mod tracker; - -use std::{ - collections::BTreeSet, - sync::atomic::{AtomicBool, Ordering}, - time::{Duration, Instant}, -}; - -use clap::Parser; -use io::CONFIG_DEFAULTS_FILE; -use iroh::net::{ - magic_endpoint::{get_alpn, get_peer_id}, - AddrInfo, MagicEndpoint, NodeAddr, -}; -use iroh::util::fs::load_secret_key; -use tokio_util::task::LocalPoolHandle; - -use crate::{ - args::{AnnounceArgs, Args, Commands, QueryArgs, ServerArgs}, - io::{load_from_file, setup_logging, tracker_home, tracker_path, CONFIG_FILE, SERVER_KEY_FILE}, - options::Options, - protocol::{ - Announce, AnnounceKind, Query, QueryFlags, Request, Response, REQUEST_SIZE_LIMIT, - TRACKER_ALPN, - }, - tracker::Tracker, -}; - -pub type NodeId = iroh::net::key::PublicKey; - -static VERBOSE: AtomicBool = AtomicBool::new(false); - -fn set_verbose(verbose: bool) { - VERBOSE.store(verbose, Ordering::Relaxed); -} - -pub fn verbose() -> bool { - VERBOSE.load(Ordering::Relaxed) -} - -#[macro_export] -macro_rules! log { - ($($arg:tt)*) => { - if $crate::verbose() { - println!($($arg)*); - } - }; -} - -/// Wait until the endpoint has figured out it's own DERP region. -async fn await_derp_region(endpoint: &MagicEndpoint) -> anyhow::Result<()> { - let t0 = Instant::now(); - loop { - let addr = endpoint.my_addr().await?; - if addr.derp_region().is_some() { - break; - } - if t0.elapsed() > Duration::from_secs(10) { - anyhow::bail!("timeout waiting for DERP region"); - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - Ok(()) -} - -async fn create_endpoint( - key: iroh::net::key::SecretKey, - port: u16, -) -> anyhow::Result { - // let pkarr_relay_discovery = discovery::PkarrRelayDiscovery::new(key.clone(), PKARR_RELAY_URL.parse().unwrap()); - let region_discover = discovery::HardcodedRegionDiscovery::new(2); - iroh::net::MagicEndpoint::builder() - .secret_key(key) - .discovery(Box::new(region_discover)) - .alpns(vec![TRACKER_ALPN.to_vec()]) - .bind(port) - .await -} - -/// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. -pub async fn accept_conn( - mut conn: quinn::Connecting, -) -> anyhow::Result<(NodeId, String, quinn::Connection)> { - let alpn = get_alpn(&mut conn).await?; - let conn = conn.await?; - let peer_id = get_peer_id(&conn)?; - Ok((peer_id, alpn, conn)) -} - -/// Write default options to a sample config file. -fn write_defaults() -> anyhow::Result<()> { - let default_path = tracker_path(CONFIG_DEFAULTS_FILE)?; - crate::io::save_to_file(Options::default(), &default_path)?; - Ok(()) -} - -async fn server(args: ServerArgs) -> anyhow::Result<()> { - set_verbose(!args.quiet); - let rt = tokio::runtime::Handle::current(); - let tpc = LocalPoolHandle::new(2); - let rt = iroh::bytes::util::runtime::Handle::new(rt, tpc); - let home = tracker_home()?; - log!("tracker starting using {}", home.display()); - let key_path = tracker_path(SERVER_KEY_FILE)?; - let key = load_secret_key(key_path).await?; - let endpoint = create_endpoint(key, args.port).await?; - let config_path = tracker_path(CONFIG_FILE)?; - write_defaults()?; - let mut options = load_from_file::(&config_path)?; - options.make_paths_relative(&home); - let db = Tracker::new(options)?; - await_derp_region(&endpoint).await?; - let addr = endpoint.my_addr().await?; - log!("listening on {:?}", addr); - log!("tracker addr: {}\n", addr.node_id); - log!("usage:"); - log!("tracker announce --tracker {} ", addr.node_id); - log!( - "tracker query --tracker {} or ", - addr.node_id - ); - log!(); - let db2 = db.clone(); - let endpoint2 = endpoint.clone(); - let _task = rt - .local_pool() - .spawn_pinned(move || db2.probe_loop(endpoint2)); - while let Some(connecting) = endpoint.accept().await { - tracing::info!("got connecting"); - let db = db.clone(); - tokio::spawn(async move { - let Ok((remote_node_id, alpn, conn)) = accept_conn(connecting).await else { - tracing::error!("error accepting connection"); - return; - }; - // if we were supporting multiple protocols, we'd need to check the ALPN here. - tracing::info!("got connection from {} {}", remote_node_id, alpn); - if let Err(cause) = db.handle_connection(conn).await { - tracing::error!("error handling connection: {}", cause); - } - }); - } - Ok(()) -} - -async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { - set_verbose(true); - // todo: uncomment once the connection problems are fixed - // for now, a random node id is more reliable. - // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; - let key = iroh::net::key::SecretKey::generate(); - let endpoint = create_endpoint(key, 11112).await?; - log!("announce {:?}", args); - log!("trying to connect to {:?}", args.tracker); - let info = NodeAddr { - node_id: args.tracker, - info: AddrInfo { - derp_region: Some(2), - direct_addresses: Default::default(), - }, - }; - let connection = endpoint.connect(info, TRACKER_ALPN).await?; - log!("connected to {:?}", connection.remote_address()); - let (mut send, mut recv) = connection.open_bi().await?; - log!("opened bi stream"); - let kind = if args.partial { - AnnounceKind::Partial - } else { - AnnounceKind::Complete - }; - let host = if let Some(host) = args.host { - host - } else { - let hosts = args - .content - .iter() - .filter_map(|x| x.host()) - .collect::>(); - if hosts.len() != 1 { - anyhow::bail!( - "content for all tickets must be from the same host, unless a host is specified" - ); - } - *hosts.iter().next().unwrap() - }; - let content = args.content.iter().map(|x| x.hash_and_format()).collect(); - let announce = Announce { - host, - kind, - content, - }; - let request = Request::Announce(announce); - let request = postcard::to_stdvec(&request)?; - log!("sending announce"); - send.write_all(&request).await?; - send.finish().await?; - let _response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; - Ok(()) -} - -async fn query(args: QueryArgs) -> anyhow::Result<()> { - set_verbose(true); - // todo: uncomment once the connection problems are fixed - // for now, a random node id is more reliable. - // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; - let key = iroh::net::key::SecretKey::generate(); - let endpoint = create_endpoint(key, args.port.unwrap_or_default()).await?; - let query = Query { - content: args.content.hash_and_format(), - flags: QueryFlags { - complete: !args.partial, - verified: args.verified, - }, - }; - let info = NodeAddr { - node_id: args.tracker, - info: AddrInfo { - derp_region: Some(2), - direct_addresses: Default::default(), - }, - }; - log!("trying to connect to tracker at {:?}", args.tracker); - let connection = endpoint.connect(info, TRACKER_ALPN).await?; - log!("connected to {:?}", connection.remote_address()); - let (mut send, mut recv) = connection.open_bi().await?; - log!("opened bi stream"); - let request = Request::Query(query); - let request = postcard::to_stdvec(&request)?; - log!("sending query"); - send.write_all(&request).await?; - send.finish().await?; - let response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; - let response = postcard::from_bytes::(&response)?; - match response { - Response::QueryResponse(response) => { - log!("content {}", response.content); - for peer in response.hosts { - log!("- peer {}", peer); - } - } - } - Ok(()) -} - -#[tokio::main(flavor = "multi_thread")] -async fn main() -> anyhow::Result<()> { - setup_logging(); - let args = Args::parse(); - match args.command { - Commands::Server(args) => server(args).await, - Commands::Announce(args) => announce(args).await, - Commands::Query(args) => query(args).await, - } -} diff --git a/iroh-gateway/letsencrypt-staging/cached_account_oRa99sDBdwbXeqQw1VDnwOYt0pA7aWeEZ-I6s6zRNLk b/iroh-gateway/letsencrypt-staging/cached_account_oRa99sDBdwbXeqQw1VDnwOYt0pA7aWeEZ-I6s6zRNLk new file mode 100644 index 00000000..c68e377f Binary files /dev/null and b/iroh-gateway/letsencrypt-staging/cached_account_oRa99sDBdwbXeqQw1VDnwOYt0pA7aWeEZ-I6s6zRNLk differ diff --git a/content-tracker/.gitignore b/iroh-mainline-content-discovery/.gitignore similarity index 100% rename from content-tracker/.gitignore rename to iroh-mainline-content-discovery/.gitignore diff --git a/content-tracker/Cargo.lock b/iroh-mainline-content-discovery/Cargo.lock similarity index 85% rename from content-tracker/Cargo.lock rename to iroh-mainline-content-discovery/Cargo.lock index d512553a..75f6ec7b 100644 --- a/content-tracker/Cargo.lock +++ b/iroh-mainline-content-discovery/Cargo.lock @@ -49,12 +49,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aliasable" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" - [[package]] name = "allocator-api2" version = "0.2.16" @@ -111,7 +105,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -121,14 +115,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" dependencies = [ "backtrace", ] @@ -201,7 +195,7 @@ version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" dependencies = [ - "http", + "http 0.2.9", "log", "url", "wildmatch", @@ -268,12 +262,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.5" @@ -292,15 +280,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597bb81c80a54b6a4381b23faba8d7774b144c94cbd1d6fe3f1329bd776554ab" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -441,17 +420,6 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" -[[package]] -name = "clipboard-win" -version = "4.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" -dependencies = [ - "error-code", - "str-buf", - "winapi", -] - [[package]] name = "cobs" version = "0.2.3" @@ -465,56 +433,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] -name = "colored" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" -dependencies = [ - "is-terminal", - "lazy_static", - "windows-sys 0.48.0", -] - -[[package]] -name = "comfy-table" -version = "7.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" -dependencies = [ - "crossterm", - "strum", - "strum_macros", - "unicode-width", -] - -[[package]] -name = "config" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7" -dependencies = [ - "async-trait", - "indexmap 1.9.3", - "lazy_static", - "nom", - "pathdiff", - "ron", - "serde", - "serde_json", - "toml 0.5.11", -] - -[[package]] -name = "console" -version = "0.15.7" +name = "concurrent-queue" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys 0.45.0", + "crossbeam-utils", ] [[package]] @@ -598,28 +522,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossterm" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" -dependencies = [ - "bitflags 2.4.1", - "crossterm_winapi", - "libc", - "parking_lot", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - [[package]] name = "crypto-bigint" version = "0.5.3" @@ -744,9 +646,9 @@ dependencies = [ [[package]] name = "default-net" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ee644f45fd960bc570bfe62a43485a5378eb44364ab423b8edbf874955038" +checksum = "7ba429d84a27fa854c66fd2e29eb1cdf6d38bbfd4495bd9f522f12a7f21e05bf" dependencies = [ "dlopen2", "libc", @@ -802,6 +704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -825,17 +728,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "dialoguer" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" -dependencies = [ - "console", - "shell-words", - "thiserror", -] - [[package]] name = "digest" version = "0.10.7" @@ -848,15 +740,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys", -] - [[package]] name = "dirs-next" version = "2.0.0" @@ -867,18 +750,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -957,36 +828,19 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" +checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" dependencies = [ "curve25519-dalek", "ed25519", "rand_core", "serde", "sha2", + "subtle", "zeroize", ] -[[package]] -name = "educe" -version = "0.4.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0042ff8246a363dbe77d2ceedb073339e85a804b9a47636c6e016a9a32c05f" -dependencies = [ - "enum-ordinalize", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "either" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" - [[package]] name = "elliptic-curve" version = "0.13.6" @@ -1012,12 +866,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - [[package]] name = "encoding_rs" version = "0.8.33" @@ -1045,19 +893,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "enum-ordinalize" -version = "3.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf1fa3f06bbff1ea5b1a9c7b14aa992a39657db60a2759457328d7e058f49ee" -dependencies = [ - "num-bigint", - "num-traits", - "proc-macro2", - "quote", - "syn 2.0.39", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -1086,17 +921,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] -name = "error-code" -version = "2.3.1" +name = "event-listener" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" dependencies = [ - "libc", - "str-buf", + "concurrent-queue", + "parking", + "pin-project-lite", ] [[package]] @@ -1111,17 +947,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" -[[package]] -name = "fd-lock" -version = "3.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" -dependencies = [ - "cfg-if", - "rustix", - "windows-sys 0.48.0", -] - [[package]] name = "ff" version = "0.13.0" @@ -1138,19 +963,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a481586acf778f1b1455424c343f71124b048ffa5f4fc3f8f6ae9dc432dcb3c7" -[[package]] -name = "flume" -version = "0.10.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" -dependencies = [ - "futures-core", - "futures-sink", - "nanorand", - "pin-project", - "spin 0.9.8", -] - [[package]] name = "flume" version = "0.11.0" @@ -1171,18 +983,18 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1195,9 +1007,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1205,15 +1017,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1222,15 +1034,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -1239,15 +1051,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -1257,9 +1069,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1297,7 +1109,7 @@ version = "0.99.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738" dependencies = [ - "proc-macro-error 0.4.12", + "proc-macro-error", "proc-macro-hack", "proc-macro2", "quote", @@ -1380,7 +1192,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.9", "indexmap 1.9.3", "slab", "tokio", @@ -1446,15 +1258,6 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3688e69b38018fec1557254f64c8dc2cc8ec502890182f395dbb0aa997aa5735" -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - [[package]] name = "hostname" version = "0.3.1" @@ -1483,6 +1286,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -1490,40 +1304,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", - "http", + "http 0.2.9", "pin-project-lite", ] [[package]] -name = "httparse" -version = "1.8.0" +name = "http-body" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] [[package]] -name = "httpdate" -version = "1.0.3" +name = "http-body-util" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "pin-project-lite", +] [[package]] -name = "human-time" -version = "0.1.6" +name = "httparse" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "259822ea527bd0d5ebf3108d84e98ac4f20769e2e8b2f3ab76e1dd6e21de7f3c" -dependencies = [ - "human-time-macros", -] +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] -name = "human-time-macros" -version = "0.1.8" +name = "httpdate" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a04129f85bfd960234ed3fa53212a4904881e024322e804ac5a48da239e44a09" -dependencies = [ - "quote", - "syn 1.0.109", -] +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -1542,8 +1360,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.9", + "http-body 0.4.5", "httparse", "httpdate", "itoa", @@ -1555,6 +1373,25 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1562,13 +1399,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.9", + "hyper 0.14.27", "rustls", "tokio", "tokio-rustls", ] +[[package]] +name = "hyper-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.1.0", + "pin-project-lite", + "socket2 0.5.5", + "tokio", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -1580,7 +1435,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.51.1", ] [[package]] @@ -1602,6 +1457,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "igd" version = "0.12.1" @@ -1611,8 +1476,8 @@ dependencies = [ "attohttpc", "bytes", "futures", - "http", - "hyper", + "http 0.2.9", + "hyper 0.14.27", "log", "rand", "tokio", @@ -1628,7 +1493,6 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", - "serde", ] [[package]] @@ -1641,20 +1505,6 @@ dependencies = [ "hashbrown 0.14.2", ] -[[package]] -name = "indicatif" -version = "0.17.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" -dependencies = [ - "console", - "instant", - "number_prefix", - "portable-atomic", - "tokio", - "unicode-width", -] - [[package]] name = "inout" version = "0.1.3" @@ -1690,7 +1540,7 @@ checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ "socket2 0.5.5", "widestring", - "windows-sys 0.48.0", + "windows-sys", "winreg", ] @@ -1701,61 +1551,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] -name = "iroh" -version = "0.10.0" +name = "iroh-base" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1291ca5df5580a718d5f4bdc1a1016a10e3d5cfe7e49513de7e020ab50ab47a4" +checksum = "e1b42ef43639a86a49132998f066810b702798818993e34565dd4eea835f626e" dependencies = [ "anyhow", "bao-tree", - "bytes", - "clap", - "colored", - "comfy-table", - "config", - "console", "data-encoding", - "derive_more", - "dialoguer", - "dirs-next", - "flume 0.11.0", - "futures", - "genawaiter", "hex", - "human-time", - "indicatif", - "iroh-bytes", - "iroh-gossip", - "iroh-io", - "iroh-metrics", - "iroh-net", - "iroh-sync", "multibase", - "num_cpus", - "once_cell", - "parking_lot", - "portable-atomic", "postcard", - "quic-rpc", - "quinn", - "rand", - "range-collections", - "rustyline", "serde", - "shell-words", - "shellexpand", - "strum", - "tempfile", + "serde-error", "thiserror", - "time", - "tokio", - "tokio-stream", - "tokio-util", - "toml 0.8.8", - "tracing", - "tracing-subscriber", - "url", - "walkdir", ] [[package]] @@ -1773,9 +1582,9 @@ dependencies = [ [[package]] name = "iroh-bytes" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "680d97457001823a3378d03abba93741f5a82c0de45b52d0c7ff03d8697859f5" +checksum = "38213865ec542f82fc4d8a9748b5cdae92da1707c134428e9f60b1cd46ccbe39" dependencies = [ "anyhow", "bao-tree", @@ -1783,12 +1592,12 @@ dependencies = [ "chrono", "data-encoding", "derive_more", - "flume 0.11.0", + "flume", "futures", "genawaiter", "hex", + "iroh-base", "iroh-io", - "multibase", "num_cpus", "once_cell", "postcard", @@ -1808,95 +1617,111 @@ dependencies = [ ] [[package]] -name = "iroh-content-tracker" -version = "0.1.0" +name = "iroh-io" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ace4f69567bfeb726672bab901d3e81be0c01119d860b2a10b7efb1553b880" dependencies = [ - "anyhow", - "bao-tree", "bytes", + "futures", + "pin-project", + "smallvec", + "tokio", +] + +[[package]] +name = "iroh-mainline-content-discovery" +version = "0.3.0" +dependencies = [ + "anyhow", "clap", "derive_more", - "dirs-next", + "ed25519-dalek", "futures", - "hex", - "humantime", - "iroh", + "genawaiter", + "iroh-bytes", + "iroh-net", + "iroh-pkarr-node-discovery", + "mainline", "pkarr", "postcard", "quinn", - "rand", + "rcgen 0.12.0", + "rustls", "serde", - "serde_json", - "simple-mdns", "tempfile", "tokio", - "tokio-util", - "toml 0.7.8", "tracing", "tracing-subscriber", - "ttl_cache", ] [[package]] -name = "iroh-gossip" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee5612d03bae65e891dbe9436a1b623c63eea5093e6f874f96410591f424e4fa" +name = "iroh-mainline-tracker" +version = "0.3.0" dependencies = [ "anyhow", + "bao-tree", "bytes", - "data-encoding", + "clap", "derive_more", + "dirs-next", "ed25519-dalek", + "flume", "futures", "genawaiter", - "indexmap 2.1.0", - "iroh-blake3", - "iroh-metrics", + "hex", + "humantime", + "iroh-bytes", + "iroh-mainline-content-discovery", "iroh-net", - "once_cell", + "iroh-pkarr-node-discovery", + "mainline", + "pkarr", "postcard", "quinn", "rand", - "rand_core", + "rcgen 0.12.0", + "redb", + "rustls", "serde", + "serde_json", + "simple-mdns", + "tempfile", "tokio", "tokio-util", + "toml", "tracing", -] - -[[package]] -name = "iroh-io" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ace4f69567bfeb726672bab901d3e81be0c01119d860b2a10b7efb1553b880" -dependencies = [ - "bytes", - "futures", - "pin-project", - "smallvec", - "tokio", + "tracing-subscriber", + "ttl_cache", + "url", ] [[package]] name = "iroh-metrics" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99471ac28a915891586cd23c09810ff4162e1e5756a48bd544b89e2294784f36" +checksum = "92a3271df85ec2a18a358d36723039eeacb464d8764531c8fd9f822dac39410d" dependencies = [ + "anyhow", "erased_set", - "hyper", + "http-body-util", + "hyper 1.1.0", + "hyper-util", "once_cell", "prometheus-client", + "reqwest", + "serde", "struct_iterable", + "time", + "tokio", "tracing", ] [[package]] name = "iroh-net" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9aa3e09b825b124117b6e710f9dfc97f7dadaee00d6501a4156953afa0a2ec" +checksum = "0477847a7fe225f71fbdff7b0baccea8fc9c9b33e7755f4b28e19f075bd64499" dependencies = [ "aead", "anyhow", @@ -1910,14 +1735,17 @@ dependencies = [ "derive_more", "duct", "ed25519-dalek", - "flume 0.11.0", + "flume", "futures", "governor", "hex", "hostname", - "http", - "hyper", + "http 1.0.0", + "http-body-util", + "hyper 1.1.0", + "hyper-util", "igd", + "iroh-base", "iroh-metrics", "libc", "netlink-packet-core", @@ -1932,9 +1760,9 @@ dependencies = [ "quinn-udp", "rand", "rand_core", - "rcgen", + "rcgen 0.11.3", "reqwest", - "ring 0.16.20", + "ring 0.17.5", "rtnetlink", "rustls", "rustls-webpki", @@ -1944,6 +1772,7 @@ dependencies = [ "smallvec", "socket2 0.5.5", "ssh-key", + "strum", "stun-rs", "surge-ping", "thiserror", @@ -1956,6 +1785,7 @@ dependencies = [ "trust-dns-resolver", "ttl_cache", "url", + "watchable", "webpki-roots", "windows 0.51.1", "wmi", @@ -1964,61 +1794,15 @@ dependencies = [ ] [[package]] -name = "iroh-sync" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5b874bd84584be4a8eaf1c9765e27ce150b55e299df084e25215455db2a04e" +name = "iroh-pkarr-node-discovery" +version = "0.1.1" dependencies = [ "anyhow", - "bytes", - "data-encoding", - "derive_more", - "ed25519-dalek", - "flume 0.11.0", "futures", - "hex", - "iroh-blake3", - "iroh-bytes", - "iroh-metrics", "iroh-net", - "lru", - "num_enum", - "once_cell", - "ouroboros", - "parking_lot", - "postcard", - "quinn", - "rand", - "rand_core", - "redb", - "serde", - "strum", - "thiserror", + "pkarr", "tokio", - "tokio-stream", - "tokio-util", "tracing", - "url", -] - -[[package]] -name = "is-terminal" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" -dependencies = [ - "hermit-abi", - "rustix", - "windows-sys 0.48.0", -] - -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", ] [[package]] @@ -2096,15 +1880,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "lru" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" -dependencies = [ - "hashbrown 0.14.2", -] - [[package]] name = "lru-cache" version = "0.1.2" @@ -2123,6 +1898,24 @@ dependencies = [ "libc", ] +[[package]] +name = "mainline" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2c7580f473d39e96a18d5fee049d9d20d65b5b62c7d107f9ccf82ef0cc46d1b" +dependencies = [ + "bytes", + "crc", + "ed25519-dalek", + "flume", + "rand", + "serde", + "serde_bencode", + "serde_bytes", + "sha1_smol", + "thiserror", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -2185,7 +1978,7 @@ checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -2421,12 +2214,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "object" version = "0.32.1" @@ -2463,12 +2250,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "os_pipe" version = "1.1.4" @@ -2476,32 +2257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ae859aa07428ca9a929b936690f8b12dc5f11dd8c6992a18ca93919f28bc177" dependencies = [ "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "ouroboros" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c86de06555b970aec45229b27291b53154f21a5743a163419f4e4c0b065dcde" -dependencies = [ - "aliasable", - "ouroboros_macro", - "static_assertions", -] - -[[package]] -name = "ouroboros_macro" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cad0c4b129e9696e37cb712b243777b90ef489a0bfaa0ac34e7d9b860e4f134" -dependencies = [ - "heck", - "itertools", - "proc-macro-error 1.0.4", - "proc-macro2", - "quote", - "syn 2.0.39", + "windows-sys", ] [[package]] @@ -2534,6 +2290,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2558,16 +2320,10 @@ dependencies = [ ] [[package]] -name = "paste" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" - -[[package]] -name = "pathdiff" -version = "0.2.1" +name = "paste" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "pem" @@ -2575,7 +2331,7 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b13fe415cdf3c8e44518e18a7c95a13431d9bdf6d15367d82b23c377fdd441a" dependencies = [ - "base64 0.21.5", + "base64", "serde", ] @@ -2585,7 +2341,7 @@ version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923" dependencies = [ - "base64 0.21.5", + "base64", "serde", ] @@ -2600,9 +2356,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" @@ -2683,14 +2439,14 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkarr" -version = "0.3.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "178fff5a1cce6c245ec8f9d78c06cc137dda0fd5d41f415a1f10c43d565f163b" +checksum = "52fb91b6f9777a6cd25ea754efcdba5c90d9decf151290439816b3d2827063ff" dependencies = [ "bytes", "ed25519-dalek", + "mainline", "rand", - "reqwest", "self_cell", "simple-dns", "thiserror", @@ -2778,12 +2534,6 @@ dependencies = [ "universal-hash", ] -[[package]] -name = "portable-atomic" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" - [[package]] name = "positioned-io" version = "0.3.3" @@ -2888,20 +2638,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" dependencies = [ - "proc-macro-error-attr 0.4.12", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr 1.0.4", + "proc-macro-error-attr", "proc-macro2", "quote", "syn 1.0.109", @@ -2921,17 +2658,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -2949,9 +2675,9 @@ dependencies = [ [[package]] name = "prometheus-client" -version = "0.21.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c99afa9a01501019ac3a14d71d9f94050346f55ca471ce90c799a15c58f61e2" +checksum = "510c4f1c9d81d556458f94c98f857748130ea9737bbd6053da497503b26ea63c" dependencies = [ "dtoa", "itoa", @@ -2970,16 +2696,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "pyo3-build-config" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" -dependencies = [ - "once_cell", - "target-lexicon", -] - [[package]] name = "quanta" version = "0.11.1" @@ -2996,25 +2712,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "quic-rpc" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d60c2fc2390baad4b9d41ae9957ae88c3095496f88e252ef50722df8b5b78d7" -dependencies = [ - "bincode", - "educe", - "flume 0.10.14", - "futures", - "pin-project", - "quinn", - "serde", - "tokio", - "tokio-serde", - "tokio-util", - "tracing", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -3066,7 +2763,7 @@ dependencies = [ "libc", "socket2 0.5.5", "tracing", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -3161,14 +2858,25 @@ dependencies = [ "yasna", ] +[[package]] +name = "rcgen" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d918c80c5a4c7560db726763020bd16db179e4d5b828078842274a443addb5d" +dependencies = [ + "pem 3.0.2", + "ring 0.17.5", + "time", + "yasna", +] + [[package]] name = "redb" -version = "1.3.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58f6da33e3b54de2ef82201ce2b465e67f337deb15d45f54355e0f77202bb4" +checksum = "72623e6275cd430215b741f41ebda34db93a13ebde253f908b70871c46afc5ba" dependencies = [ "libc", - "pyo3-build-config", ] [[package]] @@ -3213,13 +2921,13 @@ dependencies = [ [[package]] name = "reflink-copy" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f97f7665e51f23760e9e4949d454a4782c76ef954acaeec9d1b0f48a58e4529e" +checksum = "767be24c0da52e7448d495b8d162506a9aa125426651d547d545d6c2b4b65b62" dependencies = [ "cfg-if", "rustix", - "windows 0.51.1", + "windows 0.52.0", ] [[package]] @@ -3278,15 +2986,15 @@ version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ - "base64 0.21.5", + "base64", "bytes", "encoding_rs", "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", + "http 0.2.9", + "http-body 0.4.5", + "hyper 0.14.27", "hyper-rustls", "ipnet", "js-sys", @@ -3358,19 +3066,7 @@ dependencies = [ "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", -] - -[[package]] -name = "ron" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" -dependencies = [ - "base64 0.13.1", - "bitflags 1.3.2", - "indexmap 1.9.3", - "serde", + "windows-sys", ] [[package]] @@ -3452,7 +3148,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -3485,7 +3181,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.5", + "base64", ] [[package]] @@ -3504,29 +3200,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" -[[package]] -name = "rustyline" -version = "12.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9" -dependencies = [ - "bitflags 2.4.1", - "cfg-if", - "clipboard-win", - "fd-lock", - "home", - "libc", - "log", - "memchr", - "nix", - "radix_trie", - "scopeguard", - "unicode-segmentation", - "unicode-width", - "utf8parse", - "winapi", -] - [[package]] name = "ryu" version = "1.0.15" @@ -3542,22 +3215,13 @@ dependencies = [ "cipher", ] -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schannel" version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -3615,9 +3279,9 @@ dependencies = [ [[package]] name = "self_cell" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c309e515543e67811222dbc9e3dd7e1056279b782e1dacffe4242b718734fb6" +checksum = "58bf37232d3bb9a2c4e641ca2a11d83b5062066f88df7fed36c28772046d65ba" [[package]] name = "semver" @@ -3643,6 +3307,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_bencode" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70dfc7b7438b99896e7f8992363ab8e2c4ba26aa5ec675d32d1c3c2c33d413e" +dependencies = [ + "serde", + "serde_bytes", +] + [[package]] name = "serde_bytes" version = "0.11.12" @@ -3669,7 +3343,6 @@ version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ - "indexmap 2.1.0", "itoa", "ryu", "serde", @@ -3712,6 +3385,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc30b1e1e8c40c121ca33b86c23308a090d19974ef001b4bf6e61fd1a0fb095c" +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.8" @@ -3742,21 +3421,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "shell-words" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" - -[[package]] -name = "shellexpand" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" -dependencies = [ - "dirs", -] - [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -3834,7 +3498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -3903,18 +3567,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "str-buf" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" - [[package]] name = "strsim" version = "0.10.0" @@ -4082,12 +3734,6 @@ dependencies = [ "libc", ] -[[package]] -name = "target-lexicon" -version = "0.12.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" - [[package]] name = "tempfile" version = "3.8.1" @@ -4098,7 +3744,7 @@ dependencies = [ "fastrand", "redox_syscall", "rustix", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -4177,9 +3823,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -4190,14 +3836,14 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.5", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", @@ -4221,12 +3867,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb6f50b5523d014ba161512c37457acb16fd8218c883c7152e0a67ab763f2d4" dependencies = [ "async-trait", - "base64 0.21.5", + "base64", "chrono", "futures", "log", "pem 2.0.1", - "rcgen", + "rcgen 0.11.3", "reqwest", "ring 0.16.20", "rustls", @@ -4240,33 +3886,6 @@ dependencies = [ "x509-parser", ] -[[package]] -name = "tokio-serde" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911a61637386b789af998ee23f50aa30d5fd7edcec8d6d3dedae5e5815205466" -dependencies = [ - "bincode", - "bytes", - "educe", - "futures-core", - "futures-sink", - "pin-project", - "serde", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util", -] - [[package]] name = "tokio-util" version = "0.7.10" @@ -4279,21 +3898,10 @@ dependencies = [ "futures-util", "hashbrown 0.14.2", "pin-project-lite", - "slab", "tokio", "tracing", ] -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "indexmap 1.9.3", - "serde", -] - [[package]] name = "toml" version = "0.7.8" @@ -4306,18 +3914,6 @@ dependencies = [ "toml_edit 0.19.15", ] -[[package]] -name = "toml" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.21.0", -] - [[package]] name = "toml_datetime" version = "0.6.5" @@ -4351,19 +3947,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "toml_edit" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" -dependencies = [ - "indexmap 2.1.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "tower-service" version = "0.3.2" @@ -4455,7 +4038,7 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna", + "idna 0.4.0", "ipnet", "once_cell", "rand", @@ -4545,18 +4128,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - -[[package]] -name = "unicode-width" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" - [[package]] name = "unicode-xid" version = "0.2.4" @@ -4587,12 +4158,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna", + "idna 0.5.0", "percent-encoding", "serde", ] @@ -4615,16 +4186,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "walkdir" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "want" version = "0.3.1" @@ -4706,6 +4267,18 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" +[[package]] +name = "watchable" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45b42a2f611916b5965120a9cde2b60f2db4454826dd9ad5e6f47c24a5b3b259" +dependencies = [ + "event-listener", + "futures-util", + "parking_lot", + "thiserror", +] + [[package]] name = "web-sys" version = "0.3.65" @@ -4750,15 +4323,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -4782,10 +4346,20 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" dependencies = [ - "windows-core", + "windows-core 0.51.1", "windows-targets 0.48.5", ] +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core 0.52.0", + "windows-targets 0.52.0", +] + [[package]] name = "windows-core" version = "0.51.1" @@ -4795,6 +4369,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-implement" version = "0.48.0" @@ -4817,15 +4400,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -4835,21 +4409,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -4866,10 +4425,19 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" +name = "windows-targets" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] [[package]] name = "windows_aarch64_gnullvm" @@ -4878,10 +4446,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" +name = "windows_aarch64_gnullvm" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" [[package]] name = "windows_aarch64_msvc" @@ -4890,10 +4458,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "windows_i686_gnu" -version = "0.42.2" +name = "windows_aarch64_msvc" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" [[package]] name = "windows_i686_gnu" @@ -4902,10 +4470,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] -name = "windows_i686_msvc" -version = "0.42.2" +name = "windows_i686_gnu" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" [[package]] name = "windows_i686_msvc" @@ -4914,10 +4482,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +name = "windows_i686_msvc" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" [[package]] name = "windows_x86_64_gnu" @@ -4926,10 +4494,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" +name = "windows_x86_64_gnu" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" [[package]] name = "windows_x86_64_gnullvm" @@ -4938,10 +4506,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +name = "windows_x86_64_gnullvm" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" [[package]] name = "windows_x86_64_msvc" @@ -4949,6 +4517,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" version = "0.5.19" @@ -4965,7 +4539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] diff --git a/iroh-mainline-content-discovery/Cargo.toml b/iroh-mainline-content-discovery/Cargo.toml new file mode 100644 index 00000000..6dfd668b --- /dev/null +++ b/iroh-mainline-content-discovery/Cargo.toml @@ -0,0 +1,25 @@ +[workspace] +members = [ + "iroh-mainline-content-discovery", + "iroh-mainline-tracker", +] +resolver = "2" + +[profile.release] +debug = true + +[profile.optimized-release] +inherits = 'release' +debug = false +lto = true +debug-assertions = false +opt-level = 3 +panic = 'abort' +incremental = false + + +[workspace.lints.rust] +missing_debug_implementations = "warn" + +[workspace.lints.clippy] +unused-async = "warn" diff --git a/content-tracker/README.md b/iroh-mainline-content-discovery/README.md similarity index 83% rename from content-tracker/README.md rename to iroh-mainline-content-discovery/README.md index 5778b957..c3be4796 100644 --- a/content-tracker/README.md +++ b/iroh-mainline-content-discovery/README.md @@ -1,11 +1,6 @@ -# Iroh content tracker +# Iroh content discovery -This is an example how to write a simple service using mostly iroh-net. -The purpose of this service is to track providers of iroh content. - -You can *announce* to a tracker that you believe some host has some content. -The tracker will then periodically *verify* that the content is present. -Finally, you can *query* a tracker for hosts for a content. +This library provides global content discovery for iroh. ## Building from source diff --git a/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml new file mode 100644 index 00000000..2f548969 --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "iroh-mainline-content-discovery" +version = "0.3.0" +edition = "2021" +description = "Content discovery for iroh, using the bittorrent mainline DHT" +license = "MIT OR Apache-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +# Required features for the protocol types. +# +# The protocol is using postcard, but we don't need a postcard dependency for just the type definitions +iroh-net = "0.12.0" +iroh-bytes = "0.12.0" +serde = { version = "1", features = ["derive"] } + +# Optional features for the client functionality +tracing = { version = "0.1", optional = true } +quinn = { version = "0.10", optional = true } +iroh-pkarr-node-discovery = { path = "../../iroh-pkarr-node-discovery", optional = true } +mainline = { version = "1.0.0", optional = true } +pkarr = { version = "1.0.1", features = ["async"], optional = true } +anyhow = { version = "1", features = ["backtrace"], optional = true } +postcard = { version = "1", default-features = false, features = ["alloc", "use-std"], optional = true } +ed25519-dalek = { version = "2.1.0", optional = true } +futures = { version = "0.3.25", optional = true } +rcgen = { version = "0.12.0", optional = true } +rustls = { version = "0.21", optional = true } +genawaiter = { version = "0.99.1", features = ["futures03"], optional = true } + +# Optional features for the cli +clap = { version = "4", features = ["derive"], optional = true } +tempfile = { version = "3.4", optional = true } +derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"], optional = true } +tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } +tokio = { version = "1", features = ["io-util", "rt"], optional = true } + +[features] +client = ["iroh-pkarr-node-discovery", "mainline", "pkarr", "quinn", "tracing", "anyhow", "rcgen", "genawaiter", "rustls", "futures", "postcard"] +cli = ["client", "clap", "tempfile", "derive_more", "tracing-subscriber", "tokio"] +default = [] + +[[bin]] +name = "iroh-mainline-content-discovery" +required-features = ["cli"] diff --git a/content-tracker/src/args.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/args.rs similarity index 88% rename from content-tracker/src/args.rs rename to iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/args.rs index 893c6a91..665368f3 100644 --- a/content-tracker/src/args.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/args.rs @@ -1,11 +1,10 @@ //! Command line arguments. use clap::{Parser, Subcommand}; -use iroh::bytes::{Hash, HashAndFormat}; -use iroh::ticket::blob::Ticket; +use iroh_bytes::{Hash, HashAndFormat}; +use iroh_mainline_content_discovery::TrackerId; +use iroh_net::{ticket::BlobTicket, NodeId}; use std::{fmt::Display, str::FromStr}; -use crate::NodeId; - #[derive(Parser, Debug)] pub struct Args { #[clap(subcommand)] @@ -14,27 +13,16 @@ pub struct Args { #[derive(Subcommand, Debug)] pub enum Commands { - Server(ServerArgs), Announce(AnnounceArgs), Query(QueryArgs), } -#[derive(Parser, Debug)] -pub struct ServerArgs { - /// The port to listen on. - #[clap(long, default_value_t = 0xacacu16)] - pub port: u16, - - #[clap(long)] - pub quiet: bool, -} - /// Various ways to specify content. #[derive(Debug, Clone, derive_more::From)] pub enum ContentArg { Hash(Hash), HashAndFormat(HashAndFormat), - Ticket(Ticket), + Ticket(BlobTicket), } impl ContentArg { @@ -78,7 +66,7 @@ impl FromStr for ContentArg { Ok(hash.into()) } else if let Ok(haf) = HashAndFormat::from_str(s) { Ok(haf.into()) - } else if let Ok(ticket) = Ticket::from_str(s) { + } else if let Ok(ticket) = BlobTicket::from_str(s) { Ok(ticket.into()) } else { anyhow::bail!("invalid hash and format") @@ -115,7 +103,7 @@ pub struct AnnounceArgs { #[derive(Parser, Debug)] pub struct QueryArgs { #[clap(long)] - pub tracker: NodeId, + pub tracker: TrackerId, /// the port to use for querying #[clap(long)] diff --git a/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/client.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/client.rs new file mode 100644 index 00000000..ee3717fe --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/client.rs @@ -0,0 +1,282 @@ +use genawaiter::sync::Gen; +use std::{ + collections::{BTreeSet, HashSet}, + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::Arc, + time::Duration, +}; + +use futures::{future::BoxFuture, FutureExt, Stream, StreamExt}; +use iroh_bytes::HashAndFormat; +use iroh_net::{MagicEndpoint, NodeId}; +use iroh_pkarr_node_discovery::PkarrNodeDiscovery; +use mainline::common::{GetPeerResponse, StoreQueryMetdata}; +use pkarr::PkarrClient; + +use crate::protocol::{ + Announce, Query, QueryResponse, Request, Response, ALPN, REQUEST_SIZE_LIMIT, +}; + +/// Announce to a tracker. +/// +/// You can only announce content you yourself claim to have, to avoid spamming other nodes. +/// +/// `endpoint` is the magic endpoint to use for announcing. +/// `tracker` is the node id of the tracker to announce to. It must understand the [TRACKER_ALPN] protocol. +/// `content` is the content to announce. +/// `kind` is the kind of the announcement. We can claim to have the complete data or only some of it. +pub async fn announce(connection: quinn::Connection, args: Announce) -> anyhow::Result<()> { + let (mut send, mut recv) = connection.open_bi().await?; + tracing::debug!("opened bi stream"); + let request = Request::Announce(args); + let request = postcard::to_stdvec(&request)?; + tracing::debug!("sending announce"); + send.write_all(&request).await?; + send.finish().await?; + let _response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; + Ok(()) +} + +/// The mapping from an iroh [HashAndFormat] to a bittorrent infohash, aka [mainline::Id]. +/// +/// Since an infohash is just 20 bytes, this can not be a bidirectional mapping. +pub fn to_infohash(haf: HashAndFormat) -> mainline::Id { + let mut data = [0u8; 20]; + data.copy_from_slice(&haf.hash.as_bytes()[..20]); + mainline::Id::from_bytes(data).unwrap() +} + +fn unique_tracker_addrs( + mut response: mainline::common::Response, +) -> impl Stream { + Gen::new(|co| async move { + let mut found = HashSet::new(); + while let Some(response) = response.next_async().await { + println!("got get_peers response: {:?}", response); + let tracker = response.peer; + if !found.insert(tracker) { + continue; + } + co.yield_(tracker).await; + } + }) +} + +async fn query_socket_one( + endpoint: impl QuinnConnectionProvider, + addr: SocketAddr, + args: Query, +) -> anyhow::Result> { + let connection = endpoint.connect(addr).await?; + let result = query(connection, args).await?; + Ok(result.hosts) +} + +async fn query_magic_one( + endpoint: MagicEndpoint, + node_id: &NodeId, + args: Query, +) -> anyhow::Result> { + let connection = endpoint.connect_by_node_id(node_id, ALPN).await?; + let result = query(connection, args).await?; + Ok(result.hosts) +} + +/// A connection provider that can be used to connect to a tracker. +/// +/// This can either be a [`quinn::Endpoint`] where connections are created on demand, +/// or some sort of connection pool. +pub trait QuinnConnectionProvider: Clone { + fn connect(&self, addr: Addr) -> BoxFuture>; +} + +impl QuinnConnectionProvider for quinn::Endpoint { + fn connect(&self, addr: SocketAddr) -> BoxFuture> { + async move { Ok(self.connect(addr, "localhost")?.await?) }.boxed() + } +} + +/// Query multiple trackers in parallel and merge the results. +pub fn query_trackers( + endpoint: MagicEndpoint, + trackers: impl IntoIterator, + args: Query, + query_parallelism: usize, +) -> impl Stream> { + futures::stream::iter(trackers) + .map(move |tracker| { + let endpoint = endpoint.clone(); + async move { + let hosts = match query_magic_one(endpoint, &tracker, args).await { + Ok(hosts) => hosts.into_iter().map(anyhow::Ok).collect(), + Err(cause) => vec![Err(cause)], + }; + futures::stream::iter(hosts) + } + }) + .buffer_unordered(query_parallelism) + .flatten() +} + +/// Query the mainline DHT for trackers for the given content, then query each tracker for peers. +pub fn query_dht( + endpoint: impl QuinnConnectionProvider, + dht: mainline::dht::Dht, + args: Query, + query_parallelism: usize, +) -> impl Stream> { + let dht = dht.as_async(); + let info_hash = to_infohash(args.content); + let response: mainline::common::Response = dht.get_peers(info_hash); + let unique_tracker_addrs = unique_tracker_addrs(response); + unique_tracker_addrs + .map(move |addr| { + let endpoint = endpoint.clone(); + async move { + let hosts = match query_socket_one(endpoint, addr, args).await { + Ok(hosts) => hosts.into_iter().map(anyhow::Ok).collect(), + Err(cause) => vec![Err(cause)], + }; + futures::stream::iter(hosts) + } + }) + .buffer_unordered(query_parallelism) + .flatten() +} + +/// Announce to the mainline DHT in parallel. +/// +/// Note that this should only be called from a publicly reachable node, where port is the port +/// on which the tracker protocol is reachable. +pub fn announce_dht( + dht: mainline::dht::Dht, + content: BTreeSet, + port: u16, + announce_parallelism: usize, +) -> impl Stream)> { + let dht = dht.as_async(); + futures::stream::iter(content) + .map(move |content| { + let dht = dht.clone(); + async move { + let info_hash = to_infohash(content); + let res = dht.announce_peer(info_hash, Some(port)).await; + (content, res) + } + }) + .buffer_unordered(announce_parallelism) +} + +/// Assume an existing connection to a tracker and query it for peers for some content. +pub async fn query(connection: quinn::Connection, args: Query) -> anyhow::Result { + tracing::info!("connected to {:?}", connection.remote_address()); + let (mut send, mut recv) = connection.open_bi().await?; + tracing::info!("opened bi stream"); + let request = Request::Query(args); + let request = postcard::to_stdvec(&request)?; + tracing::info!("sending query"); + send.write_all(&request).await?; + send.finish().await?; + let response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; + let response = postcard::from_bytes::(&response)?; + Ok(match response { + Response::QueryResponse(response) => response, + }) +} + +/// Create a quinn client endpoint. +fn create_quinn_client( + bind_addr: SocketAddr, + alpn_protocols: Vec>, + keylog: bool, +) -> anyhow::Result { + let secret_key = iroh_net::key::SecretKey::generate(); + let tls_client_config = + iroh_net::tls::make_client_config(&secret_key, None, alpn_protocols, keylog)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); + let mut endpoint = quinn::Endpoint::client(bind_addr)?; + let mut transport_config = quinn::TransportConfig::default(); + transport_config.keep_alive_interval(Some(Duration::from_secs(1))); + client_config.transport_config(Arc::new(transport_config)); + endpoint.set_default_client_config(client_config); + Ok(endpoint) +} + +async fn create_endpoint( + key: iroh_net::key::SecretKey, + port: u16, + publish: bool, +) -> anyhow::Result { + let pkarr = PkarrClient::new(); + let discovery_key = if publish { Some(&key) } else { None }; + let mainline_discovery = PkarrNodeDiscovery::new(pkarr, discovery_key); + iroh_net::MagicEndpoint::builder() + .secret_key(key) + .discovery(Box::new(mainline_discovery)) + .alpns(vec![ALPN.to_vec()]) + .bind(port) + .await +} + +/// A tracker id for queries - either a node id or an address. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum TrackerId { + NodeId(NodeId), + Addr(std::net::SocketAddr), +} + +impl std::fmt::Display for TrackerId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TrackerId::NodeId(node_id) => write!(f, "{}", node_id), + TrackerId::Addr(addr) => write!(f, "{}", addr), + } + } +} + +impl std::str::FromStr for TrackerId { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + if let Ok(node_id) = s.parse() { + return Ok(TrackerId::NodeId(node_id)); + } + if let Ok(addr) = s.parse() { + return Ok(TrackerId::Addr(addr)); + } + anyhow::bail!("invalid tracker id") + } +} + +/// Connect to a tracker using the [crate::protocol::ALPN] protocol, using either +/// a node id or an address. +/// +/// Note that this is less efficient than using an existing endpoint when doing multiple requests. +/// It is provided as a convenience function for short lived utilities. +pub async fn connect(tracker: &TrackerId, local_port: u16) -> anyhow::Result { + match tracker { + TrackerId::Addr(tracker) => connect_socket(*tracker, local_port).await, + TrackerId::NodeId(tracker) => connect_magic(&tracker, local_port).await, + } +} + +/// Create a magic endpoint and connect to a tracker using the [crate::protocol::ALPN] protocol. +async fn connect_magic(tracker: &NodeId, local_port: u16) -> anyhow::Result { + // todo: uncomment once the connection problems are fixed + // for now, a random node id is more reliable. + // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; + let key = iroh_net::key::SecretKey::generate(); + let endpoint = create_endpoint(key, local_port, false).await?; + tracing::info!("trying to connect to tracker at {:?}", tracker); + let connection = endpoint.connect_by_node_id(tracker, ALPN).await?; + Ok(connection) +} + +/// Create a quinn endpoint and connect to a tracker using the [crate::protocol::ALPN] protocol. +async fn connect_socket(tracker: SocketAddr, local_port: u16) -> anyhow::Result { + let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, local_port)); + let endpoint = create_quinn_client(bind_addr, vec![ALPN.to_vec()], false)?; + tracing::info!("trying to connect to tracker at {:?}", tracker); + let connection = endpoint.connect(tracker, "localhost")?.await?; + Ok(connection) +} diff --git a/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/lib.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/lib.rs new file mode 100644 index 00000000..e0ccaab3 --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/lib.rs @@ -0,0 +1,10 @@ +//! A library for discovering content using the mainline DHT. +//! +//! This library contains the protocol for announcing and querying content, as +//! well as the client side implementation and a few helpers for p2p quinn +//! connections. +#[cfg(feature = "client")] +mod client; +pub mod protocol; +#[cfg(feature = "client")] +pub use client::*; diff --git a/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/main.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/main.rs new file mode 100644 index 00000000..3ec66c16 --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/main.rs @@ -0,0 +1,168 @@ +pub mod args; + +use std::collections::BTreeSet; + +use anyhow::Context; +use clap::Parser; +use iroh_mainline_content_discovery::protocol::{Announce, AnnounceKind, Query, QueryFlags, ALPN}; +use iroh_net::{ + magic_endpoint::{get_alpn, get_remote_node_id}, + MagicEndpoint, NodeId, +}; +use iroh_pkarr_node_discovery::PkarrNodeDiscovery; +use pkarr::PkarrClient; +use tokio::io::AsyncWriteExt; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; + +use crate::args::{AnnounceArgs, Args, Commands, QueryArgs}; + +async fn create_endpoint( + key: iroh_net::key::SecretKey, + port: u16, + publish: bool, +) -> anyhow::Result { + let pkarr = PkarrClient::new(); + let discovery_key = if publish { Some(&key) } else { None }; + let mainline_discovery = PkarrNodeDiscovery::new(pkarr, discovery_key); + iroh_net::MagicEndpoint::builder() + .secret_key(key) + .discovery(Box::new(mainline_discovery)) + .alpns(vec![ALPN.to_vec()]) + .bind(port) + .await +} + +/// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. +pub async fn accept_conn( + mut conn: quinn::Connecting, +) -> anyhow::Result<(NodeId, String, quinn::Connection)> { + let alpn = get_alpn(&mut conn).await?; + let conn = conn.await?; + let peer_id = get_remote_node_id(&conn)?; + Ok((peer_id, alpn, conn)) +} + +async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { + // todo: uncomment once the connection problems are fixed + // for now, a random node id is more reliable. + // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; + let key = iroh_net::key::SecretKey::generate(); + let content = args.content.iter().map(|x| x.hash_and_format()).collect(); + let host = if let Some(host) = args.host { + host + } else { + let hosts = args + .content + .iter() + .filter_map(|x| x.host()) + .collect::>(); + if hosts.len() != 1 { + anyhow::bail!( + "content for all tickets must be from the same host, unless a host is specified" + ); + } + *hosts.iter().next().unwrap() + }; + println!("announcing to {}", args.tracker); + println!("host {} has", host); + for content in &content { + println!(" {}", content); + } + let endpoint = create_endpoint(key, 11112, false).await?; + let connection = endpoint.connect_by_node_id(&args.tracker, ALPN).await?; + println!("connected to {:?}", connection.remote_address()); + let kind = if args.partial { + AnnounceKind::Partial + } else { + AnnounceKind::Complete + }; + let announce = Announce { + host, + kind, + content, + }; + iroh_mainline_content_discovery::announce(connection, announce).await?; + println!("done"); + Ok(()) +} + +async fn query(args: QueryArgs) -> anyhow::Result<()> { + let connection = + iroh_mainline_content_discovery::connect(&args.tracker, args.port.unwrap_or_default()) + .await?; + let q = Query { + content: args.content.hash_and_format(), + flags: QueryFlags { + complete: !args.partial, + verified: args.verified, + }, + }; + let res = iroh_mainline_content_discovery::query(connection, q).await?; + println!( + "querying tracker {} for content {}", + args.tracker, args.content + ); + for peer in res.hosts { + println!("{}", peer); + } + Ok(()) +} + +// set the RUST_LOG env var to one of {debug,info,warn} to see logging info +pub fn setup_logging() { + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) + .with(EnvFilter::from_default_env()) + .try_init() + .ok(); +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> anyhow::Result<()> { + setup_logging(); + let args = Args::parse(); + match args.command { + Commands::Announce(args) => announce(args).await, + Commands::Query(args) => query(args).await, + } +} + +/// Loads a [`SecretKey`] from the provided file. +pub async fn load_secret_key( + key_path: std::path::PathBuf, +) -> anyhow::Result { + if key_path.exists() { + let keystr = tokio::fs::read(key_path).await?; + let secret_key = + iroh_net::key::SecretKey::try_from_openssh(keystr).context("invalid keyfile")?; + Ok(secret_key) + } else { + let secret_key = iroh_net::key::SecretKey::generate(); + let ser_key = secret_key.to_openssh()?; + + // Try to canoncialize if possible + let key_path = key_path.canonicalize().unwrap_or(key_path); + let key_path_parent = key_path.parent().ok_or_else(|| { + anyhow::anyhow!("no parent directory found for '{}'", key_path.display()) + })?; + tokio::fs::create_dir_all(&key_path_parent).await?; + + // write to tempfile + let (file, temp_file_path) = tempfile::NamedTempFile::new_in(key_path_parent) + .context("unable to create tempfile")? + .into_parts(); + let mut file = tokio::fs::File::from_std(file); + file.write_all(ser_key.as_bytes()) + .await + .context("unable to write keyfile")?; + file.flush().await?; + drop(file); + + // move file + tokio::fs::rename(temp_file_path, key_path) + .await + .context("failed to rename keyfile")?; + + Ok(secret_key) + } +} diff --git a/content-tracker/src/protocol.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/protocol.rs similarity index 91% rename from content-tracker/src/protocol.rs rename to iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/protocol.rs index 0968d481..a54c44ba 100644 --- a/content-tracker/src/protocol.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/protocol.rs @@ -1,13 +1,11 @@ //! The protocol for communicating with the tracker. -use std::collections::BTreeSet; - -use iroh::bytes::HashAndFormat; +use iroh_bytes::HashAndFormat; +use iroh_net::NodeId; use serde::{Deserialize, Serialize}; - -use crate::NodeId; +use std::collections::BTreeSet; /// The ALPN string for this protocol -pub const TRACKER_ALPN: &[u8] = b"n0/tracker/1"; +pub const ALPN: &[u8] = b"n0/tracker/1"; /// Maximum size of a request pub const REQUEST_SIZE_LIMIT: usize = 1024 * 16; @@ -37,6 +35,8 @@ impl AnnounceKind { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Announce { /// The peer that supposedly has the data. + /// + /// Should we get this from the connection? pub host: NodeId, /// The blobs or sets that the peer claims to have. pub content: BTreeSet, @@ -45,7 +45,7 @@ pub struct Announce { } /// -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct QueryFlags { /// Only return peers that supposedly have the complete data. /// @@ -63,7 +63,7 @@ pub struct QueryFlags { } /// Query a peer for a blob or set of blobs. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct Query { /// The content we want to find. /// diff --git a/content-tracker/Cargo.toml b/iroh-mainline-content-discovery/iroh-mainline-tracker/Cargo.toml similarity index 57% rename from content-tracker/Cargo.toml rename to iroh-mainline-content-discovery/iroh-mainline-tracker/Cargo.toml index 223e300c..124e2607 100644 --- a/content-tracker/Cargo.toml +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "iroh-content-tracker" -version = "0.1.0" +name = "iroh-mainline-tracker" +version = "0.3.0" edition = "2021" -description = "Simple content tracker server for iroh" +description = "Content discovery for iroh, using the bittorrent mainline DHT" license = "MIT OR Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -11,17 +11,24 @@ license = "MIT OR Apache-2.0" anyhow = { version = "1", features = ["backtrace"] } bao-tree = { version = "0.9.1", features = ["tokio_fsm"], default-features = false } bytes = "1" -clap = { version = "4", features = ["derive"] } +clap = { version = "4", features = ["derive"], optional = true } derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } dirs-next = "2" +ed25519-dalek = "2.1.0" futures = "0.3.25" hex = "0.4.3" humantime = "2.1.0" -iroh = "0.10.0" -pkarr = "0.3.0" +iroh-net = "0.12.0" +iroh-bytes = "0.12.0" +iroh-pkarr-node-discovery = { path = "../../iroh-pkarr-node-discovery" } +mainline = "1.0.0" +pkarr = { version = "1.0.1", features = ["async"] } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quinn = "0.10" rand = "0.8" +rcgen = "0.12.0" +redb = "1.5.0" +rustls = "0.21" serde = { version = "1", features = ["derive"] } serde_json = "1.0.107" simple-mdns = { version = "0.5.0", features = ["async-tokio"] } @@ -32,3 +39,15 @@ toml = "0.7.3" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } ttl_cache = "0.5.1" +url = "2.5.0" +flume = "0.11.0" +genawaiter = { version = "0.99.1", features = ["futures03"] } +iroh-mainline-content-discovery = { path = "../iroh-mainline-content-discovery", features = ["client"] } + +[features] +cli = ["clap"] +default = ["cli"] + +[[bin]] +name = "iroh-mainline-tracker" +required-features = ["cli"] diff --git a/iroh-mainline-content-discovery/iroh-mainline-tracker/src/args.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/args.rs new file mode 100644 index 00000000..72a18c8e --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/args.rs @@ -0,0 +1,30 @@ +//! Command line arguments. +use clap::{Parser, Subcommand}; + +#[derive(Parser, Debug)] +pub struct Args { + #[clap(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand, Debug)] +pub enum Commands { + Server(ServerArgs), +} + +#[derive(Parser, Debug)] +pub struct ServerArgs { + /// The port to listen on. + #[clap(long, default_value_t = 0xacacu16)] + pub magic_port: u16, + + /// The quinn port to listen on. + /// + /// The server must be reachable under this port from the internet, via + /// UDP for QUIC connections. + #[clap(long, default_value_t = 0xbcbcu16)] + pub quinn_port: u16, + + #[clap(long)] + pub quiet: bool, +} diff --git a/content-tracker/src/io.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/io.rs similarity index 97% rename from content-tracker/src/io.rs rename to iroh-mainline-content-discovery/iroh-mainline-tracker/src/io.rs index 86977387..54188cb3 100644 --- a/content-tracker/src/io.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/io.rs @@ -8,11 +8,13 @@ use std::{ }; use anyhow::Context; -use iroh::bytes::{get::Stats, HashAndFormat}; +use iroh_bytes::{get::Stats, HashAndFormat}; +use iroh_mainline_content_discovery::protocol::AnnounceKind; +use iroh_net::NodeId; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tracing_subscriber::{prelude::*, EnvFilter}; -use crate::{protocol::AnnounceKind, tracker::ProbeKind, NodeId}; +use crate::tracker::ProbeKind; pub const CONFIG_DEFAULTS_FILE: &str = "config.defaults.toml"; pub const CONFIG_FILE: &str = "config.toml"; diff --git a/content-tracker/src/iroh_bytes_util.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/iroh_bytes_util.rs similarity index 84% rename from content-tracker/src/iroh_bytes_util.rs rename to iroh-mainline-content-discovery/iroh-mainline-tracker/src/iroh_bytes_util.rs index 7846826a..17d8048b 100644 --- a/content-tracker/src/iroh_bytes_util.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/iroh_bytes_util.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use bao_tree::{ByteNum, ChunkNum, ChunkRanges}; use bytes::Bytes; -use iroh::bytes::{ +use iroh_bytes::{ get::{ fsm::{BlobContentNext, EndBlobNext}, Stats, @@ -14,8 +14,6 @@ use iroh::bytes::{ }; use rand::Rng; -use crate::log; - /// Get the claimed size of a blob from a peer. /// /// This is just reading the size header and then immediately closing the connection. @@ -24,13 +22,13 @@ pub async fn unverified_size( connection: &quinn::Connection, hash: &Hash, ) -> anyhow::Result<(u64, Stats)> { - let request = iroh::bytes::protocol::GetRequest::new( + let request = iroh_bytes::protocol::GetRequest::new( *hash, RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), ); - let request = iroh::bytes::get::fsm::start(connection.clone(), request); + let request = iroh_bytes::get::fsm::start(connection.clone(), request); let connected = request.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { + let iroh_bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { unreachable!("expected start root"); }; let at_blob_header = start.next(); @@ -47,14 +45,14 @@ pub async fn verified_size( connection: &quinn::Connection, hash: &Hash, ) -> anyhow::Result<(u64, Stats)> { - log!("Getting verified size of {}", hash.to_hex()); - let request = iroh::bytes::protocol::GetRequest::new( + tracing::debug!("Getting verified size of {}", hash.to_hex()); + let request = iroh_bytes::protocol::GetRequest::new( *hash, RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), ); - let request = iroh::bytes::get::fsm::start(connection.clone(), request); + let request = iroh_bytes::get::fsm::start(connection.clone(), request); let connected = request.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { + let iroh_bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { unreachable!("expected start root"); }; let header = start.next(); @@ -74,7 +72,7 @@ pub async fn verified_size( unreachable!("expected closing"); }; let stats = closing.next().await?; - log!( + tracing::debug!( "Got verified size of {}, {:.6}s", hash.to_hex(), stats.elapsed.as_secs_f64() @@ -88,17 +86,17 @@ pub async fn get_hash_seq_and_sizes( max_size: u64, ) -> anyhow::Result<(HashSeq, Arc<[u64]>)> { let content = HashAndFormat::hash_seq(*hash); - log!("Getting hash seq and children sizes of {}", content); - let request = iroh::bytes::protocol::GetRequest::new( + tracing::debug!("Getting hash seq and children sizes of {}", content); + let request = iroh_bytes::protocol::GetRequest::new( *hash, RangeSpecSeq::from_ranges_infinite([ ChunkRanges::all(), ChunkRanges::from(ChunkNum(u64::MAX)..), ]), ); - let at_start = iroh::bytes::get::fsm::start(connection.clone(), request); + let at_start = iroh_bytes::get::fsm::start(connection.clone(), request); let at_connected = at_start.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartRoot(start) = at_connected.next().await? else { + let iroh_bytes::get::fsm::ConnectedNext::StartRoot(start) = at_connected.next().await? else { unreachable!("query includes root"); }; let at_start_root = start.next(); @@ -127,7 +125,7 @@ pub async fn get_hash_seq_and_sizes( } }; let _stats = closing.next().await?; - log!( + tracing::debug!( "Got hash seq and children sizes of {}: {:?}", content, sizes @@ -144,9 +142,9 @@ pub async fn chunk_probe( let ranges = ChunkRanges::from(chunk..chunk + 1); let ranges = RangeSpecSeq::from_ranges([ranges]); let request = GetRequest::new(*hash, ranges); - let request = iroh::bytes::get::fsm::start(connection.clone(), request); + let request = iroh_bytes::get::fsm::start(connection.clone(), request); let connected = request.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { + let iroh_bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { unreachable!("query includes root"); }; let header = start.next(); diff --git a/iroh-mainline-content-discovery/iroh-mainline-tracker/src/lib.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/lib.rs new file mode 100644 index 00000000..ee588b6c --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/lib.rs @@ -0,0 +1,4 @@ +pub mod io; +pub mod iroh_bytes_util; +pub mod options; +pub mod tracker; diff --git a/iroh-mainline-content-discovery/iroh-mainline-tracker/src/main.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/main.rs new file mode 100644 index 00000000..13141ecb --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/main.rs @@ -0,0 +1,214 @@ +pub mod args; + +use std::{ + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; + +use anyhow::Context; +use clap::Parser; +use iroh_mainline_content_discovery::protocol::ALPN; +use iroh_mainline_tracker::{ + io::{ + self, load_from_file, setup_logging, tracker_home, tracker_path, CONFIG_DEFAULTS_FILE, + CONFIG_FILE, SERVER_KEY_FILE, + }, + options::Options, + tracker::Tracker, +}; +use iroh_net::{ + magic_endpoint::{get_alpn, get_remote_node_id}, + MagicEndpoint, NodeId, +}; +use iroh_pkarr_node_discovery::PkarrNodeDiscovery; +use pkarr::PkarrClient; +use tokio::io::AsyncWriteExt; +use tokio_util::task::LocalPoolHandle; + +use crate::args::{Args, Commands, ServerArgs}; + +static VERBOSE: AtomicBool = AtomicBool::new(false); + +fn set_verbose(verbose: bool) { + VERBOSE.store(verbose, Ordering::Relaxed); +} + +pub fn verbose() -> bool { + VERBOSE.load(Ordering::Relaxed) +} + +#[macro_export] +macro_rules! log { + ($($arg:tt)*) => { + if $crate::verbose() { + println!($($arg)*); + } + }; +} + +/// Wait until the endpoint has figured out it's own DERP region. +async fn await_derp_region(endpoint: &MagicEndpoint) -> anyhow::Result<()> { + let t0 = Instant::now(); + loop { + let addr = endpoint.my_addr().await?; + if addr.derp_url().is_some() { + break; + } + if t0.elapsed() > Duration::from_secs(10) { + anyhow::bail!("timeout waiting for DERP region"); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + Ok(()) +} + +async fn create_endpoint( + key: iroh_net::key::SecretKey, + port: u16, + publish: bool, +) -> anyhow::Result { + let pkarr = PkarrClient::new(); + let discovery_key = if publish { Some(&key) } else { None }; + let mainline_discovery = PkarrNodeDiscovery::new(pkarr, discovery_key); + iroh_net::MagicEndpoint::builder() + .secret_key(key) + .discovery(Box::new(mainline_discovery)) + .alpns(vec![ALPN.to_vec()]) + .bind(port) + .await +} + +/// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. +pub async fn accept_conn( + mut conn: quinn::Connecting, +) -> anyhow::Result<(NodeId, String, quinn::Connection)> { + let alpn = get_alpn(&mut conn).await?; + let conn = conn.await?; + let peer_id = get_remote_node_id(&conn)?; + Ok((peer_id, alpn, conn)) +} + +/// Write default options to a sample config file. +fn write_defaults() -> anyhow::Result<()> { + let default_path = tracker_path(CONFIG_DEFAULTS_FILE)?; + io::save_to_file(Options::default(), &default_path)?; + Ok(()) +} + +async fn server(args: ServerArgs) -> anyhow::Result<()> { + set_verbose(!args.quiet); + let tpc = LocalPoolHandle::new(2); + let home = tracker_home()?; + log!("tracker starting using {}", home.display()); + let key_path = tracker_path(SERVER_KEY_FILE)?; + let key = load_secret_key(key_path).await?; + let endpoint = create_endpoint(key.clone(), args.magic_port, true).await?; + let config_path = tracker_path(CONFIG_FILE)?; + write_defaults()?; + let mut options = load_from_file::(&config_path)?; + options.make_paths_relative(&home); + let db = Tracker::new(options)?; + await_derp_region(&endpoint).await?; + let addr = endpoint.my_addr().await?; + log!("listening on {:?}", addr); + log!("tracker addr: {}\n", addr.node_id); + log!("usage:"); + log!("tracker announce --tracker {} ", addr.node_id); + log!( + "tracker query --tracker {} or ", + addr.node_id + ); + let db2 = db.clone(); + let db3 = db.clone(); + let db4 = db.clone(); + let endpoint2 = endpoint.clone(); + let _probe_task = tpc.spawn_pinned(move || db2.probe_loop(endpoint2)); + let _announce_task = tpc.spawn_pinned(move || db3.dht_announce_loop(args.quinn_port)); + let magic_accept_task = tokio::spawn(db.magic_accept_loop(endpoint)); + let server_config = configure_server(&key)?; + let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, args.quinn_port)); + let quinn_endpoint = quinn::Endpoint::server(server_config, bind_addr)?; + let quinn_accept_task = tokio::spawn(db4.quinn_accept_loop(quinn_endpoint)); + magic_accept_task.await??; + quinn_accept_task.await??; + Ok(()) +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> anyhow::Result<()> { + setup_logging(); + let args = Args::parse(); + match args.command { + Commands::Server(args) => server(args).await, + } +} + +/// Returns default server configuration along with its certificate. +#[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 +fn configure_server(secret_key: &iroh_net::key::SecretKey) -> anyhow::Result { + make_server_config(secret_key, 8, 1024, vec![ALPN.to_vec()]) +} + +/// Create a [`quinn::ServerConfig`] with the given secret key and limits. +pub fn make_server_config( + secret_key: &iroh_net::key::SecretKey, + max_streams: u64, + max_connections: u32, + alpn_protocols: Vec>, +) -> anyhow::Result { + let tls_server_config = iroh_net::tls::make_server_config(secret_key, alpn_protocols, false)?; + let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(tls_server_config)); + let mut transport_config = quinn::TransportConfig::default(); + transport_config + .max_concurrent_bidi_streams(max_streams.try_into()?) + .max_concurrent_uni_streams(0u32.into()); + + server_config + .transport_config(Arc::new(transport_config)) + .concurrent_connections(max_connections); + Ok(server_config) +} + +/// Loads a [`SecretKey`] from the provided file. +pub async fn load_secret_key( + key_path: std::path::PathBuf, +) -> anyhow::Result { + if key_path.exists() { + let keystr = tokio::fs::read(key_path).await?; + let secret_key = + iroh_net::key::SecretKey::try_from_openssh(keystr).context("invalid keyfile")?; + Ok(secret_key) + } else { + let secret_key = iroh_net::key::SecretKey::generate(); + let ser_key = secret_key.to_openssh()?; + + // Try to canoncialize if possible + let key_path = key_path.canonicalize().unwrap_or(key_path); + let key_path_parent = key_path.parent().ok_or_else(|| { + anyhow::anyhow!("no parent directory found for '{}'", key_path.display()) + })?; + tokio::fs::create_dir_all(&key_path_parent).await?; + + // write to tempfile + let (file, temp_file_path) = tempfile::NamedTempFile::new_in(key_path_parent) + .context("unable to create tempfile")? + .into_parts(); + let mut file = tokio::fs::File::from_std(file); + file.write_all(ser_key.as_bytes()) + .await + .context("unable to write keyfile")?; + file.flush().await?; + drop(file); + + // move file + tokio::fs::rename(temp_file_path, key_path) + .await + .context("failed to rename keyfile")?; + + Ok(secret_key) + } +} diff --git a/content-tracker/src/options.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/options.rs similarity index 93% rename from content-tracker/src/options.rs rename to iroh-mainline-content-discovery/iroh-mainline-tracker/src/options.rs index e5a80dcb..9d28a9d3 100644 --- a/content-tracker/src/options.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/options.rs @@ -29,6 +29,10 @@ pub struct Options { pub announce_data_path: Option, // number of peers to probe in parallel pub probe_parallelism: usize, + + pub dht_announce_parallelism: usize, + + pub dht_announce_interval: Duration, } impl Default for Options { @@ -43,6 +47,8 @@ impl Default for Options { probe_log: Some("probe.log".into()), announce_data_path: Some("announce.data.toml".into()), probe_parallelism: 4, + dht_announce_parallelism: 4, + dht_announce_interval: Duration::from_secs(10), } } } diff --git a/content-tracker/src/tracker.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/tracker.rs similarity index 77% rename from content-tracker/src/tracker.rs rename to iroh-mainline-content-discovery/iroh-mainline-tracker/src/tracker.rs index 37501745..d51d6827 100644 --- a/content-tracker/src/tracker.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/tracker.rs @@ -7,13 +7,22 @@ use std::{ use bao_tree::{ByteNum, ChunkNum}; use futures::StreamExt; -use iroh::bytes::{ +use iroh_bytes::{ get::{fsm::EndBlobNext, Stats}, hashseq::HashSeq, protocol::GetRequest, BlobFormat, Hash, HashAndFormat, }; -use iroh::net::MagicEndpoint; +use iroh_mainline_content_discovery::{ + announce_dht, + protocol::{ + Announce, AnnounceKind, Query, QueryResponse, Request, Response, REQUEST_SIZE_LIMIT, + }, +}; +use iroh_net::{ + magic_endpoint::{get_alpn, get_remote_node_id}, + MagicEndpoint, NodeId, +}; use rand::Rng; use crate::{ @@ -21,12 +30,7 @@ use crate::{ iroh_bytes_util::{ chunk_probe, get_hash_seq_and_sizes, random_hash_seq_ranges, unverified_size, verified_size, }, - log, options::Options, - protocol::{ - Announce, AnnounceKind, Query, QueryResponse, Request, Response, REQUEST_SIZE_LIMIT, - }, - NodeId, }; /// The tracker server. @@ -158,22 +162,98 @@ impl Tracker { } } + pub async fn dht_announce_loop(self, port: u16) -> anyhow::Result<()> { + let dht = mainline::Dht::default(); + loop { + let state = self.0.state.read().unwrap(); + let content: BTreeSet = + state.announce_data.iter().map(|(haf, _)| *haf).collect(); + drop(state); + let mut announce = announce_dht( + dht.clone(), + content, + port, + self.0.options.dht_announce_parallelism, + ); + while let Some((content, res)) = announce.next().await { + match res { + Ok(sqm) => { + let stored_at = sqm.stored_at(); + tracing::info!( + "announced {} as {} on {} nodes", + content, + sqm.target(), + stored_at.len() + ); + for item in stored_at { + tracing::debug!("stored at {} {}", item.id, item.address); + } + } + Err(cause) => { + tracing::warn!("error announcing: {}", cause); + } + } + } + tokio::time::sleep(self.0.options.dht_announce_interval).await; + } + } + + pub async fn magic_accept_loop(self, endpoint: MagicEndpoint) -> std::io::Result<()> { + while let Some(connecting) = endpoint.accept().await { + tracing::info!("got connecting"); + let db = self.clone(); + tokio::spawn(async move { + let Ok((remote_node_id, alpn, conn)) = accept_conn(connecting).await else { + tracing::error!("error accepting connection"); + return; + }; + // if we were supporting multiple protocols, we'd need to check the ALPN here. + tracing::info!("got connection from {} {}", remote_node_id, alpn); + if let Err(cause) = db.handle_connection(conn).await { + tracing::error!("error handling connection: {}", cause); + } + }); + } + Ok(()) + } + + pub async fn quinn_accept_loop(self, endpoint: quinn::Endpoint) -> std::io::Result<()> { + let local_addr = endpoint.local_addr()?; + println!("quinn listening on {}", local_addr); + while let Some(connecting) = endpoint.accept().await { + tracing::info!("got connecting"); + let db = self.clone(); + tokio::spawn(async move { + let Ok((remote_node_id, alpn, conn)) = accept_conn(connecting).await else { + tracing::error!("error accepting connection"); + return; + }; + // if we were supporting multiple protocols, we'd need to check the ALPN here. + tracing::info!("got connection from {} {}", remote_node_id, alpn); + if let Err(cause) = db.handle_connection(conn).await { + tracing::error!("error handling connection: {}", cause); + } + }); + } + Ok(()) + } + /// Handle a single incoming connection on the tracker ALPN. pub async fn handle_connection(&self, connection: quinn::Connection) -> anyhow::Result<()> { - log!("calling accept_bi"); + tracing::debug!("calling accept_bi"); let (mut send, mut recv) = connection.accept_bi().await?; - log!("got bi stream"); + tracing::debug!("got bi stream"); let request = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; let request = postcard::from_bytes::(&request)?; match request { Request::Announce(announce) => { - log!("got announce: {:?}", announce); + tracing::debug!("got announce: {:?}", announce); self.handle_announce(announce)?; send.finish().await?; } Request::Query(query) => { - log!("handle query: {:?}", query); + tracing::debug!("handle query: {:?}", query); let response = self.handle_query(query)?; let response = Response::QueryResponse(response); let response = postcard::to_stdvec(&response)?; @@ -232,9 +312,9 @@ impl Tracker { let HashAndFormat { hash, format } = content; let mut rng = rand::thread_rng(); let stats = if probe_kind == ProbeKind::Incomplete { - log!("Size probing {}...", cap); + tracing::debug!("Size probing {}...", cap); let (size, stats) = unverified_size(connection, hash).await?; - log!( + tracing::debug!( "Size probed {}, got unverified size {}, {:.6}s", cap, size, @@ -246,9 +326,9 @@ impl Tracker { BlobFormat::Raw => { let size = self.get_or_insert_size(connection, hash).await?; let random_chunk = rng.gen_range(0..ByteNum(size).chunks().0); - log!("Chunk probing {}, chunk {}", cap, random_chunk); + tracing::debug!("Chunk probing {}, chunk {}", cap, random_chunk); let stats = chunk_probe(connection, hash, ChunkNum(random_chunk)).await?; - log!( + tracing::debug!( "Chunk probed {}, chunk {}, {:.6}s", cap, random_chunk, @@ -266,11 +346,11 @@ impl Tracker { }) .collect::>() .join(", "); - log!("Seq probing {} using {}", cap, text); + tracing::debug!("Seq probing {} using {}", cap, text); let request = GetRequest::new(*hash, ranges); - let request = iroh::bytes::get::fsm::start(connection.clone(), request); + let request = iroh_bytes::get::fsm::start(connection.clone(), request); let connected = request.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartChild(child) = + let iroh_bytes::get::fsm::ConnectedNext::StartChild(child) = connected.next().await? else { unreachable!("request does not include root"); @@ -284,7 +364,7 @@ impl Tracker { unreachable!("request contains only one blob"); }; let stats = closing.next().await?; - log!( + tracing::debug!( "Seq probed {} using {}, {:.6}s", cap, text, @@ -409,7 +489,7 @@ impl Tracker { )> { let t0 = Instant::now(); let res = endpoint - .connect_by_node_id(&host, &iroh::bytes::protocol::ALPN) + .connect_by_node_id(&host, &iroh_bytes::protocol::ALPN) .await; log_connection_attempt(&self.0.options.dial_log, &host, t0, &res)?; let connection = match res { @@ -440,3 +520,13 @@ impl Tracker { anyhow::Ok((host, results)) } } + +/// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. +async fn accept_conn( + mut conn: quinn::Connecting, +) -> anyhow::Result<(NodeId, String, quinn::Connection)> { + let alpn = get_alpn(&mut conn).await?; + let conn = conn.await?; + let peer_id = get_remote_node_id(&conn)?; + Ok((peer_id, alpn, conn)) +} diff --git a/iroh-pkarr-node-discovery/LICENSE-APACHE b/iroh-pkarr-node-discovery/LICENSE-APACHE new file mode 100644 index 00000000..4c83a284 --- /dev/null +++ b/iroh-pkarr-node-discovery/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. \ No newline at end of file diff --git a/iroh-pkarr-node-discovery/LICENSE-MIT b/iroh-pkarr-node-discovery/LICENSE-MIT new file mode 100644 index 00000000..749aa1ec --- /dev/null +++ b/iroh-pkarr-node-discovery/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file