From 28137080b079a992e3f52e0a4fc29969fb914f0a Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Wed, 10 Jan 2024 16:22:47 +0100 Subject: [PATCH 01/13] WIP Content tracker to iroh 0.12 (derp URLs) --- content-tracker/Cargo.lock | 317 ++++++++++++++++++++++++------- content-tracker/Cargo.toml | 8 +- content-tracker/src/args.rs | 6 +- content-tracker/src/discovery.rs | 196 ++++++------------- content-tracker/src/main.rs | 39 ++-- 5 files changed, 324 insertions(+), 242 deletions(-) diff --git a/content-tracker/Cargo.lock b/content-tracker/Cargo.lock index d512553a..89952611 100644 --- a/content-tracker/Cargo.lock +++ b/content-tracker/Cargo.lock @@ -201,7 +201,7 @@ version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" dependencies = [ - "http", + "http 0.2.9", "log", "url", "wildmatch", @@ -487,6 +487,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "config" version = "0.13.3" @@ -744,9 +753,9 @@ dependencies = [ [[package]] name = "default-net" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ee644f45fd960bc570bfe62a43485a5378eb44364ab423b8edbf874955038" +checksum = "7ba429d84a27fa854c66fd2e29eb1cdf6d38bbfd4495bd9f522f12a7f21e05bf" dependencies = [ "dlopen2", "libc", @@ -802,6 +811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -957,15 +967,16 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" +checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" dependencies = [ "curve25519-dalek", "ed25519", "rand_core", "serde", "sha2", + "subtle", "zeroize", ] @@ -1099,6 +1110,17 @@ dependencies = [ "str-buf", ] +[[package]] +name = "event-listener" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -1171,9 +1193,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -1380,7 +1402,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.9", "indexmap 1.9.3", "slab", "tokio", @@ -1404,6 +1426,15 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.2", +] + [[package]] name = "heck" version = "0.4.1" @@ -1483,6 +1514,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -1490,7 +1532,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", - "http", + "http 0.2.9", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -1542,8 +1607,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.9", + "http-body 0.4.5", "httparse", "httpdate", "itoa", @@ -1555,6 +1620,25 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1562,13 +1646,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.9", + "hyper 0.14.27", "rustls", "tokio", "tokio-rustls", ] +[[package]] +name = "hyper-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.1.0", + "pin-project-lite", + "socket2 0.5.5", + "tokio", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -1602,6 +1704,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "igd" version = "0.12.1" @@ -1611,8 +1723,8 @@ dependencies = [ "attohttpc", "bytes", "futures", - "http", - "hyper", + "http 0.2.9", + "hyper 0.14.27", "log", "rand", "tokio", @@ -1702,9 +1814,9 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "iroh" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1291ca5df5580a718d5f4bdc1a1016a10e3d5cfe7e49513de7e020ab50ab47a4" +checksum = "d3aea70a8acc645bc78c388f9412b4ae5f94ddeeea13c946b29af4c2db097c23" dependencies = [ "anyhow", "bao-tree", @@ -1721,9 +1833,11 @@ dependencies = [ "flume 0.11.0", "futures", "genawaiter", + "hashlink", "hex", "human-time", "indicatif", + "iroh-base", "iroh-bytes", "iroh-gossip", "iroh-io", @@ -1758,6 +1872,23 @@ dependencies = [ "walkdir", ] +[[package]] +name = "iroh-base" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b42ef43639a86a49132998f066810b702798818993e34565dd4eea835f626e" +dependencies = [ + "anyhow", + "bao-tree", + "data-encoding", + "hex", + "multibase", + "postcard", + "serde", + "serde-error", + "thiserror", +] + [[package]] name = "iroh-blake3" version = "1.4.3" @@ -1773,9 +1904,9 @@ dependencies = [ [[package]] name = "iroh-bytes" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "680d97457001823a3378d03abba93741f5a82c0de45b52d0c7ff03d8697859f5" +checksum = "38213865ec542f82fc4d8a9748b5cdae92da1707c134428e9f60b1cd46ccbe39" dependencies = [ "anyhow", "bao-tree", @@ -1787,8 +1918,8 @@ dependencies = [ "futures", "genawaiter", "hex", + "iroh-base", "iroh-io", - "multibase", "num_cpus", "once_cell", "postcard", @@ -1809,7 +1940,7 @@ dependencies = [ [[package]] name = "iroh-content-tracker" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "bao-tree", @@ -1817,11 +1948,12 @@ dependencies = [ "clap", "derive_more", "dirs-next", + "ed25519-dalek", "futures", "hex", "humantime", "iroh", - "pkarr", + "mainline", "postcard", "quinn", "rand", @@ -1835,13 +1967,14 @@ dependencies = [ "tracing", "tracing-subscriber", "ttl_cache", + "url", ] [[package]] name = "iroh-gossip" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee5612d03bae65e891dbe9436a1b623c63eea5093e6f874f96410591f424e4fa" +checksum = "74fcad132a6ff61ad83fc63203948919d2d77d58deac806bd08580f11e968a24" dependencies = [ "anyhow", "bytes", @@ -1851,6 +1984,7 @@ dependencies = [ "futures", "genawaiter", "indexmap 2.1.0", + "iroh-base", "iroh-blake3", "iroh-metrics", "iroh-net", @@ -1880,23 +2014,30 @@ dependencies = [ [[package]] name = "iroh-metrics" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99471ac28a915891586cd23c09810ff4162e1e5756a48bd544b89e2294784f36" +checksum = "92a3271df85ec2a18a358d36723039eeacb464d8764531c8fd9f822dac39410d" dependencies = [ + "anyhow", "erased_set", - "hyper", + "http-body-util", + "hyper 1.1.0", + "hyper-util", "once_cell", "prometheus-client", + "reqwest", + "serde", "struct_iterable", + "time", + "tokio", "tracing", ] [[package]] name = "iroh-net" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9aa3e09b825b124117b6e710f9dfc97f7dadaee00d6501a4156953afa0a2ec" +checksum = "0477847a7fe225f71fbdff7b0baccea8fc9c9b33e7755f4b28e19f075bd64499" dependencies = [ "aead", "anyhow", @@ -1915,9 +2056,12 @@ dependencies = [ "governor", "hex", "hostname", - "http", - "hyper", + "http 1.0.0", + "http-body-util", + "hyper 1.1.0", + "hyper-util", "igd", + "iroh-base", "iroh-metrics", "libc", "netlink-packet-core", @@ -1934,7 +2078,7 @@ dependencies = [ "rand_core", "rcgen", "reqwest", - "ring 0.16.20", + "ring 0.17.5", "rtnetlink", "rustls", "rustls-webpki", @@ -1944,6 +2088,7 @@ dependencies = [ "smallvec", "socket2 0.5.5", "ssh-key", + "strum", "stun-rs", "surge-ping", "thiserror", @@ -1956,6 +2101,7 @@ dependencies = [ "trust-dns-resolver", "ttl_cache", "url", + "watchable", "webpki-roots", "windows 0.51.1", "wmi", @@ -1965,9 +2111,9 @@ dependencies = [ [[package]] name = "iroh-sync" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5b874bd84584be4a8eaf1c9765e27ce150b55e299df084e25215455db2a04e" +checksum = "3e27f9c5bf0576ad2786c1e32ee046518144b8f221eef6578c882633b6e27985" dependencies = [ "anyhow", "bytes", @@ -1977,8 +2123,8 @@ dependencies = [ "flume 0.11.0", "futures", "hex", + "iroh-base", "iroh-blake3", - "iroh-bytes", "iroh-metrics", "iroh-net", "lru", @@ -2123,6 +2269,24 @@ dependencies = [ "libc", ] +[[package]] +name = "mainline" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2c7580f473d39e96a18d5fee049d9d20d65b5b62c7d107f9ccf82ef0cc46d1b" +dependencies = [ + "bytes", + "crc", + "ed25519-dalek", + "flume 0.11.0", + "rand", + "serde", + "serde_bencode", + "serde_bytes", + "sha1_smol", + "thiserror", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -2534,6 +2698,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2600,9 +2770,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" @@ -2681,23 +2851,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkarr" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "178fff5a1cce6c245ec8f9d78c06cc137dda0fd5d41f415a1f10c43d565f163b" -dependencies = [ - "bytes", - "ed25519-dalek", - "rand", - "reqwest", - "self_cell", - "simple-dns", - "thiserror", - "url", - "z32", -] - [[package]] name = "pkcs1" version = "0.7.5" @@ -2949,9 +3102,9 @@ dependencies = [ [[package]] name = "prometheus-client" -version = "0.21.2" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c99afa9a01501019ac3a14d71d9f94050346f55ca471ce90c799a15c58f61e2" +checksum = "510c4f1c9d81d556458f94c98f857748130ea9737bbd6053da497503b26ea63c" dependencies = [ "dtoa", "itoa", @@ -3284,9 +3437,9 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", + "http 0.2.9", + "http-body 0.4.5", + "hyper 0.14.27", "hyper-rustls", "ipnet", "js-sys", @@ -3615,9 +3768,9 @@ dependencies = [ [[package]] name = "self_cell" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c309e515543e67811222dbc9e3dd7e1056279b782e1dacffe4242b718734fb6" +checksum = "58bf37232d3bb9a2c4e641ca2a11d83b5062066f88df7fed36c28772046d65ba" [[package]] name = "semver" @@ -3643,6 +3796,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_bencode" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70dfc7b7438b99896e7f8992363ab8e2c4ba26aa5ec675d32d1c3c2c33d413e" +dependencies = [ + "serde", + "serde_bytes", +] + [[package]] name = "serde_bytes" version = "0.11.12" @@ -3712,6 +3875,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc30b1e1e8c40c121ca33b86c23308a090d19974ef001b4bf6e61fd1a0fb095c" +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.8" @@ -4455,7 +4624,7 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna", + "idna 0.4.0", "ipnet", "once_cell", "rand", @@ -4587,12 +4756,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna", + "idna 0.5.0", "percent-encoding", "serde", ] @@ -4706,6 +4875,18 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" +[[package]] +name = "watchable" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45b42a2f611916b5965120a9cde2b60f2db4454826dd9ad5e6f47c24a5b3b259" +dependencies = [ + "event-listener", + "futures-util", + "parking_lot", + "thiserror", +] + [[package]] name = "web-sys" version = "0.3.65" @@ -5023,12 +5204,6 @@ dependencies = [ "time", ] -[[package]] -name = "z32" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e62ec361f7259399a83b7983270dd0249067d94600c50475e65dca64e24083" - [[package]] name = "zerocopy" version = "0.7.25" diff --git a/content-tracker/Cargo.toml b/content-tracker/Cargo.toml index 223e300c..4b72b854 100644 --- a/content-tracker/Cargo.toml +++ b/content-tracker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-content-tracker" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Simple content tracker server for iroh" license = "MIT OR Apache-2.0" @@ -14,11 +14,12 @@ bytes = "1" clap = { version = "4", features = ["derive"] } derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } dirs-next = "2" +ed25519-dalek = "2.1.0" futures = "0.3.25" hex = "0.4.3" humantime = "2.1.0" -iroh = "0.10.0" -pkarr = "0.3.0" +iroh = "0.12.0" +mainline = "1.0.0" postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quinn = "0.10" rand = "0.8" @@ -32,3 +33,4 @@ toml = "0.7.3" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } ttl_cache = "0.5.1" +url = "2.5.0" diff --git a/content-tracker/src/args.rs b/content-tracker/src/args.rs index 893c6a91..2f2cca6f 100644 --- a/content-tracker/src/args.rs +++ b/content-tracker/src/args.rs @@ -1,7 +1,7 @@ //! Command line arguments. use clap::{Parser, Subcommand}; use iroh::bytes::{Hash, HashAndFormat}; -use iroh::ticket::blob::Ticket; +use iroh::ticket::BlobTicket; use std::{fmt::Display, str::FromStr}; use crate::NodeId; @@ -34,7 +34,7 @@ pub struct ServerArgs { pub enum ContentArg { Hash(Hash), HashAndFormat(HashAndFormat), - Ticket(Ticket), + Ticket(BlobTicket), } impl ContentArg { @@ -78,7 +78,7 @@ impl FromStr for ContentArg { Ok(hash.into()) } else if let Ok(haf) = HashAndFormat::from_str(s) { Ok(haf.into()) - } else if let Ok(ticket) = Ticket::from_str(s) { + } else if let Ok(ticket) = BlobTicket::from_str(s) { Ok(ticket.into()) } else { anyhow::bail!("invalid hash and format") diff --git a/content-tracker/src/discovery.rs b/content-tracker/src/discovery.rs index b52d2239..0da186db 100644 --- a/content-tracker/src/discovery.rs +++ b/content-tracker/src/discovery.rs @@ -1,180 +1,100 @@ //! Discovery services //! //! Originally copied and adapted from -use std::{ - collections::BTreeSet, - net::{IpAddr, SocketAddr}, -}; - use anyhow::Result; +use ed25519_dalek::SigningKey; use futures::{future, future::BoxFuture, FutureExt}; -use iroh::net::{AddrInfo, NodeAddr}; -use pkarr::{ - dns::rdata::{RData, A, AAAA, TXT}, - dns::{Name, Packet, ResourceRecord, CLASS}, - url::Url, - Keypair, PkarrClient, SignedPacket, +use iroh::{ + base::ticket::Ticket, + net::{key::SecretKey, magicsock::Discovery, AddrInfo, NodeAddr}, }; -use tracing::{info, warn}; +use mainline::{common::MutableItem, dht::DhtSettings}; +use url::Url; use crate::NodeId; -const DERP_REGION_KEY: &str = "_derp_region.iroh."; - -#[allow(dead_code)] -const PKARR_RELAY_URL: &str = "https://iroh-discovery.rklaehn.workers.dev/"; - -#[allow(unused)] -fn filter_ipaddr(rr: &ResourceRecord) -> Option { - if rr.class != CLASS::IN { - return None; - } - let addr: IpAddr = match rr.rdata { - RData::A(A { address }) => IpAddr::V4(address.into()), - RData::AAAA(AAAA { address }) => IpAddr::V6(address.into()), - _ => return None, - }; - Some(addr) -} - -fn filter_txt(rr: &ResourceRecord) -> Option { - if rr.class != CLASS::IN { - return None; - } - if let RData::TXT(txt) = &rr.rdata { - String::try_from(txt.clone()).ok() - } else { - None - } -} - -fn filter_u16(rr: &ResourceRecord) -> Option { - if rr.class != CLASS::IN { - return None; - } - if let RData::A(A { address }) = rr.rdata { - Some(address as _) - } else { - None - } -} - -fn packet_to_node_addr(node_id: &NodeId, packet: &SignedPacket) -> NodeAddr { - let direct_addresses = packet - .resource_records("@") - .filter_map(filter_txt) - .filter_map(|addr| addr.parse().ok()) - .collect::>(); - let derp_region = packet - .resource_records(DERP_REGION_KEY) - .find_map(filter_u16); - NodeAddr { - node_id: *node_id, - info: AddrInfo { - derp_region, - direct_addresses, - }, - } -} - -fn node_addr_to_packet(keypair: &Keypair, info: &AddrInfo, ttl: u32) -> Result { - let mut packet = Packet::new_reply(0); - for addr in &info.direct_addresses { - let addr = addr.to_string(); - packet.answers.push(ResourceRecord::new( - Name::new("@").unwrap(), - CLASS::IN, - ttl, - RData::TXT(TXT::try_from(addr.as_str())?.into_owned()), - )); - } - if let Some(derp_region) = info.derp_region { - packet.answers.push(ResourceRecord::new( - Name::new(DERP_REGION_KEY).unwrap(), - CLASS::IN, - ttl, - RData::A(A { - address: derp_region as _, - }), - )); - } - Ok(SignedPacket::from_packet(keypair, &packet)?) -} - -/// A discovery method that uses the pkarr DNS protocol. See pkarr.org for more -/// information. -/// -/// This is using pkarr via a simple http relay or self-contained server. -#[derive(Debug)] -pub struct PkarrRelayDiscovery { - keypair: pkarr::Keypair, - relay: Url, - client: PkarrClient, +#[derive(Debug, Clone)] +pub struct MainlineDiscovery { + mainline_client: mainline::async_dht::AsyncDht, + keypair: iroh::net::key::SecretKey, } -impl PkarrRelayDiscovery { - #[allow(dead_code)] - pub fn new(secret_key: iroh::net::key::SecretKey, relay: Url) -> Self { - let keypair = pkarr::Keypair::from_secret_key(&secret_key.to_bytes()); +impl MainlineDiscovery { + pub fn new(keypair: SecretKey) -> Self { + let mainline_client = mainline::Dht::new(DhtSettings::default()).as_async(); Self { + mainline_client, keypair, - relay, - client: PkarrClient::new(), } } } -impl iroh::net::magicsock::Discovery for PkarrRelayDiscovery { +impl Discovery for MainlineDiscovery { fn publish(&self, info: &AddrInfo) { - info!("publishing {:?} via {}", info, self.relay); - let signed_packet = node_addr_to_packet(&self.keypair, info, 0).unwrap(); - let client = self.client.clone(); - let relay = self.relay.clone(); + let Ok(ticket) = iroh::net::ticket::NodeTicket::new(NodeAddr { + node_id: self.keypair.public(), + info: info.clone(), + }) else { + return; + }; + let ticket = ticket.to_bytes().into(); + let signing_key = SigningKey::from(self.keypair.to_bytes()); + let item = MutableItem::new(signing_key, ticket, 0, None); + let mainline_client = self.mainline_client.clone(); tokio::spawn(async move { - let res = client.relay_put(&relay, signed_packet).await; - if let Err(e) = res { - warn!("error publishing: {}", e); - } else { - info!("done publishing"); + let res = mainline_client.put_mutable(item).await; + match res { + Ok(x) => { + println!("published to mainline: {:?}", x); + } + Err(cause) => { + tracing::warn!("error publishing to mainline: {}", cause); + } } }); } - fn resolve<'a>( - &'a self, - node_id: &'a NodeId, - ) -> futures::future::BoxFuture<'a, Result> { - async move { - info!("resolving {} via {}", node_id, self.relay); - let pkarr_public_key = pkarr::PublicKey::try_from(*node_id.as_bytes()).unwrap(); - let packet = self.client.relay_get(&self.relay, pkarr_public_key).await?; - let addr = packet_to_node_addr(node_id, &packet); - info!("resolved: {} to {:?}", node_id, addr); - Ok(addr.info) - } + fn resolve<'a>(&'a self, node_id: &'a iroh::net::NodeId) -> BoxFuture<'a, Result> { + let public_key = *node_id.clone().as_bytes(); + let mainline_client = self.mainline_client.clone(); + tokio::spawn(async move { + let mut response = mainline_client.get_mutable(&public_key, None).await; + while let Some(item) = response.next_async().await { + let Ok(ticket) = iroh::net::ticket::NodeTicket::from_bytes(item.item.value()) + else { + continue; + }; + return Ok(ticket.node_addr().info.clone()); + } + Err(anyhow::anyhow!("not found")) + }) + .map(|res| match res { + Ok(res) => res, + Err(cause) => Err(cause.into()), + }) .boxed() } } /// A discovery method that just uses a hardcoded region. #[derive(Debug)] -pub struct HardcodedRegionDiscovery { - region: u16, +pub struct HardcodedDerperDiscovery { + derp_url: Url, } -impl HardcodedRegionDiscovery { +impl HardcodedDerperDiscovery { /// Create a new discovery method that always returns the given region. - pub fn new(region: u16) -> Self { - Self { region } + pub fn new(derp_url: Url) -> Self { + Self { derp_url } } } -impl iroh::net::magicsock::Discovery for HardcodedRegionDiscovery { +impl iroh::net::magicsock::Discovery for HardcodedDerperDiscovery { fn publish(&self, _info: &AddrInfo) {} fn resolve<'a>(&'a self, _node_id: &'a NodeId) -> BoxFuture<'a, Result> { future::ok(AddrInfo { - derp_region: Some(self.region), + derp_url: Some(self.derp_url.clone()), direct_addresses: Default::default(), }) .boxed() diff --git a/content-tracker/src/main.rs b/content-tracker/src/main.rs index 0dff38f8..c657f75f 100644 --- a/content-tracker/src/main.rs +++ b/content-tracker/src/main.rs @@ -15,7 +15,7 @@ use std::{ use clap::Parser; use io::CONFIG_DEFAULTS_FILE; use iroh::net::{ - magic_endpoint::{get_alpn, get_peer_id}, + magic_endpoint::{get_alpn, get_remote_node_id}, AddrInfo, MagicEndpoint, NodeAddr, }; use iroh::util::fs::load_secret_key; @@ -58,7 +58,7 @@ async fn await_derp_region(endpoint: &MagicEndpoint) -> anyhow::Result<()> { let t0 = Instant::now(); loop { let addr = endpoint.my_addr().await?; - if addr.derp_region().is_some() { + if addr.derp_url().is_some() { break; } if t0.elapsed() > Duration::from_secs(10) { @@ -73,11 +73,10 @@ async fn create_endpoint( key: iroh::net::key::SecretKey, port: u16, ) -> anyhow::Result { - // let pkarr_relay_discovery = discovery::PkarrRelayDiscovery::new(key.clone(), PKARR_RELAY_URL.parse().unwrap()); - let region_discover = discovery::HardcodedRegionDiscovery::new(2); + let mainline_discovery = discovery::MainlineDiscovery::new(key.clone()); iroh::net::MagicEndpoint::builder() .secret_key(key) - .discovery(Box::new(region_discover)) + .discovery(Box::new(mainline_discovery)) .alpns(vec![TRACKER_ALPN.to_vec()]) .bind(port) .await @@ -89,7 +88,7 @@ pub async fn accept_conn( ) -> anyhow::Result<(NodeId, String, quinn::Connection)> { let alpn = get_alpn(&mut conn).await?; let conn = conn.await?; - let peer_id = get_peer_id(&conn)?; + let peer_id = get_remote_node_id(&conn)?; Ok((peer_id, alpn, conn)) } @@ -102,9 +101,7 @@ fn write_defaults() -> anyhow::Result<()> { async fn server(args: ServerArgs) -> anyhow::Result<()> { set_verbose(!args.quiet); - let rt = tokio::runtime::Handle::current(); let tpc = LocalPoolHandle::new(2); - let rt = iroh::bytes::util::runtime::Handle::new(rt, tpc); let home = tracker_home()?; log!("tracker starting using {}", home.display()); let key_path = tracker_path(SERVER_KEY_FILE)?; @@ -128,9 +125,7 @@ async fn server(args: ServerArgs) -> anyhow::Result<()> { log!(); let db2 = db.clone(); let endpoint2 = endpoint.clone(); - let _task = rt - .local_pool() - .spawn_pinned(move || db2.probe_loop(endpoint2)); + let _task = tpc.spawn_pinned(move || db2.probe_loop(endpoint2)); while let Some(connecting) = endpoint.accept().await { tracing::info!("got connecting"); let db = db.clone(); @@ -158,14 +153,9 @@ async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { let endpoint = create_endpoint(key, 11112).await?; log!("announce {:?}", args); log!("trying to connect to {:?}", args.tracker); - let info = NodeAddr { - node_id: args.tracker, - info: AddrInfo { - derp_region: Some(2), - direct_addresses: Default::default(), - }, - }; - let connection = endpoint.connect(info, TRACKER_ALPN).await?; + let connection = endpoint + .connect_by_node_id(&args.tracker, TRACKER_ALPN) + .await?; log!("connected to {:?}", connection.remote_address()); let (mut send, mut recv) = connection.open_bi().await?; log!("opened bi stream"); @@ -218,15 +208,10 @@ async fn query(args: QueryArgs) -> anyhow::Result<()> { verified: args.verified, }, }; - let info = NodeAddr { - node_id: args.tracker, - info: AddrInfo { - derp_region: Some(2), - direct_addresses: Default::default(), - }, - }; log!("trying to connect to tracker at {:?}", args.tracker); - let connection = endpoint.connect(info, TRACKER_ALPN).await?; + let connection = endpoint + .connect_by_node_id(&args.tracker, TRACKER_ALPN) + .await?; log!("connected to {:?}", connection.remote_address()); let (mut send, mut recv) = connection.open_bi().await?; log!("opened bi stream"); From 407e0ba03530634f56fe7aa725b3216f11d56cce Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sat, 20 Jan 2024 11:17:11 +0200 Subject: [PATCH 02/13] WIP --- content-tracker/Cargo.lock | 46 ++++++----- content-tracker/Cargo.toml | 2 + content-tracker/src/args.rs | 6 +- content-tracker/src/discovery.rs | 130 ++++++++++++++++++++++++++++++- content-tracker/src/main.rs | 24 ++---- content-tracker/src/tracker.rs | 40 +++++++++- 6 files changed, 207 insertions(+), 41 deletions(-) diff --git a/content-tracker/Cargo.lock b/content-tracker/Cargo.lock index 89952611..17f4136b 100644 --- a/content-tracker/Cargo.lock +++ b/content-tracker/Cargo.lock @@ -1954,9 +1954,11 @@ dependencies = [ "humantime", "iroh", "mainline", + "pkarr", "postcard", "quinn", "rand", + "redb", "serde", "serde_json", "simple-mdns", @@ -2851,6 +2853,23 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkarr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52fb91b6f9777a6cd25ea754efcdba5c90d9decf151290439816b3d2827063ff" +dependencies = [ + "bytes", + "ed25519-dalek", + "mainline", + "rand", + "self_cell", + "simple-dns", + "thiserror", + "url", + "z32", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -3123,16 +3142,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "pyo3-build-config" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" -dependencies = [ - "once_cell", - "target-lexicon", -] - [[package]] name = "quanta" version = "0.11.1" @@ -3316,12 +3325,11 @@ dependencies = [ [[package]] name = "redb" -version = "1.3.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58f6da33e3b54de2ef82201ce2b465e67f337deb15d45f54355e0f77202bb4" +checksum = "72623e6275cd430215b741f41ebda34db93a13ebde253f908b70871c46afc5ba" dependencies = [ "libc", - "pyo3-build-config", ] [[package]] @@ -4251,12 +4259,6 @@ dependencies = [ "libc", ] -[[package]] -name = "target-lexicon" -version = "0.12.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" - [[package]] name = "tempfile" version = "3.8.1" @@ -5204,6 +5206,12 @@ dependencies = [ "time", ] +[[package]] +name = "z32" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e62ec361f7259399a83b7983270dd0249067d94600c50475e65dca64e24083" + [[package]] name = "zerocopy" version = "0.7.25" diff --git a/content-tracker/Cargo.toml b/content-tracker/Cargo.toml index 4b72b854..7c1fa053 100644 --- a/content-tracker/Cargo.toml +++ b/content-tracker/Cargo.toml @@ -20,9 +20,11 @@ hex = "0.4.3" humantime = "2.1.0" iroh = "0.12.0" mainline = "1.0.0" +pkarr = { version = "1.0.1", features = ["async"] } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quinn = "0.10" rand = "0.8" +redb = "1.5.0" serde = { version = "1", features = ["derive"] } serde_json = "1.0.107" simple-mdns = { version = "0.5.0", features = ["async-tokio"] } diff --git a/content-tracker/src/args.rs b/content-tracker/src/args.rs index 2f2cca6f..6342e493 100644 --- a/content-tracker/src/args.rs +++ b/content-tracker/src/args.rs @@ -23,7 +23,11 @@ pub enum Commands { pub struct ServerArgs { /// The port to listen on. #[clap(long, default_value_t = 0xacacu16)] - pub port: u16, + pub magic_port: u16, + + /// The quinn port to listen on. + #[clap(long, default_value_t = 0xbcbcu16)] + pub quinn_port: u16, #[clap(long)] pub quiet: bool, diff --git a/content-tracker/src/discovery.rs b/content-tracker/src/discovery.rs index 0da186db..3c57f1eb 100644 --- a/content-tracker/src/discovery.rs +++ b/content-tracker/src/discovery.rs @@ -1,22 +1,148 @@ //! Discovery services //! //! Originally copied and adapted from -use anyhow::Result; +use std::{net::IpAddr, str::FromStr}; + +use anyhow::{Context, Result}; use ed25519_dalek::SigningKey; use futures::{future, future::BoxFuture, FutureExt}; use iroh::{ base::ticket::Ticket, net::{key::SecretKey, magicsock::Discovery, AddrInfo, NodeAddr}, + ticket::NodeTicket, }; use mainline::{common::MutableItem, dht::DhtSettings}; +use pkarr::{ + dns::{ + rdata::{RData, A, AAAA, TXT}, + Name, Packet, ResourceRecord, CLASS, + }, + Keypair, SignedPacket, +}; use url::Url; use crate::NodeId; +const IROH_TICKET_KEY: &str = "_ticket.iroh."; +#[derive(Debug, Clone)] +pub struct PkarrDiscovery { + keypair: SecretKey, + pkarr_client: pkarr::PkarrClient, +} + +impl PkarrDiscovery { + pub fn new(keypair: SecretKey) -> Self { + let pkarr_client = pkarr::PkarrClient::new(); + Self { + keypair, + pkarr_client, + } + } +} + +fn node_addr_to_packet(secret: SecretKey, addr: NodeAddr, ttl: u32) -> Result { + let mut packet = Packet::new_reply(0); + let ticket = NodeTicket::new(addr)?; + let ticket = ticket.to_string(); + packet.answers.push(ResourceRecord::new( + Name::new(IROH_TICKET_KEY).unwrap(), + CLASS::IN, + ttl, + RData::TXT(TXT::new().with_string(&ticket)?), + )); + let keypair = Keypair::from_secret_key(&secret.to_bytes()); + let signed_packet = SignedPacket::from_packet(&keypair, &packet)?; + Ok(signed_packet) +} + +#[allow(unused)] +fn filter_ipaddr(rr: &ResourceRecord) -> Option { + if rr.class != CLASS::IN { + return None; + } + let addr: IpAddr = match rr.rdata { + RData::A(A { address }) => IpAddr::V4(address.into()), + RData::AAAA(AAAA { address }) => IpAddr::V6(address.into()), + _ => return None, + }; + Some(addr) +} + +fn filter_txt(rr: &ResourceRecord) -> Option { + if rr.class != CLASS::IN { + return None; + } + if let RData::TXT(txt) = &rr.rdata { + String::try_from(txt.clone()).ok() + } else { + None + } +} + +fn filter_ticket(rr: &ResourceRecord) -> Option { + let txt = filter_txt(rr)?; + NodeTicket::from_str(&txt).ok() +} + +fn packet_to_node_addr(packet: &SignedPacket) -> anyhow::Result { + packet + .resource_records(IROH_TICKET_KEY) + .find_map(filter_ticket) + .map(|ticket| ticket.node_addr().clone()) + .context("No ticket found in packet") +} + +impl Discovery for PkarrDiscovery { + fn publish(&self, info: &AddrInfo) { + let Ok(signed_packet) = node_addr_to_packet( + self.keypair.clone(), + NodeAddr { + node_id: self.keypair.public(), + info: info.clone(), + }, + 60, + ) else { + return; + }; + let pkarr_client = self.pkarr_client.clone(); + + tokio::spawn(async move { + let res = pkarr_client.publish(&signed_packet).await; + match res { + Ok(x) => { + println!("published to pkarr: {:?}", x); + } + Err(cause) => { + tracing::warn!("error publishing to pkarr: {}", cause); + } + } + }); + } + + fn resolve<'a>(&'a self, node_id: &'a iroh::net::NodeId) -> BoxFuture<'a, Result> { + let pkarr_client = self.pkarr_client.clone(); + let node_id_bytes = *node_id.as_bytes(); + tokio::spawn(async move { + let public_key = pkarr::PublicKey::try_from(node_id_bytes)?; + let signed_packet = pkarr_client + .resolve(public_key) + .await + .context("not found")?; + let addr: NodeAddr = packet_to_node_addr(&signed_packet)?; + Ok(addr.info) + }) + .map(|res| match res { + Ok(res) => res, + Err(cause) => Err(cause.into()), + }) + .boxed() + } +} + #[derive(Debug, Clone)] pub struct MainlineDiscovery { mainline_client: mainline::async_dht::AsyncDht, - keypair: iroh::net::key::SecretKey, + keypair: SecretKey, } impl MainlineDiscovery { diff --git a/content-tracker/src/main.rs b/content-tracker/src/main.rs index c657f75f..f1e853de 100644 --- a/content-tracker/src/main.rs +++ b/content-tracker/src/main.rs @@ -73,7 +73,7 @@ async fn create_endpoint( key: iroh::net::key::SecretKey, port: u16, ) -> anyhow::Result { - let mainline_discovery = discovery::MainlineDiscovery::new(key.clone()); + let mainline_discovery = discovery::PkarrDiscovery::new(key.clone()); iroh::net::MagicEndpoint::builder() .secret_key(key) .discovery(Box::new(mainline_discovery)) @@ -106,7 +106,7 @@ async fn server(args: ServerArgs) -> anyhow::Result<()> { log!("tracker starting using {}", home.display()); let key_path = tracker_path(SERVER_KEY_FILE)?; let key = load_secret_key(key_path).await?; - let endpoint = create_endpoint(key, args.port).await?; + let endpoint = create_endpoint(key, args.magic_port).await?; let config_path = tracker_path(CONFIG_FILE)?; write_defaults()?; let mut options = load_from_file::(&config_path)?; @@ -125,22 +125,10 @@ async fn server(args: ServerArgs) -> anyhow::Result<()> { log!(); let db2 = db.clone(); let endpoint2 = endpoint.clone(); - let _task = tpc.spawn_pinned(move || db2.probe_loop(endpoint2)); - while let Some(connecting) = endpoint.accept().await { - tracing::info!("got connecting"); - let db = db.clone(); - tokio::spawn(async move { - let Ok((remote_node_id, alpn, conn)) = accept_conn(connecting).await else { - tracing::error!("error accepting connection"); - return; - }; - // if we were supporting multiple protocols, we'd need to check the ALPN here. - tracing::info!("got connection from {} {}", remote_node_id, alpn); - if let Err(cause) = db.handle_connection(conn).await { - tracing::error!("error handling connection: {}", cause); - } - }); - } + let _probe_task = tpc.spawn_pinned(move || db2.probe_loop(endpoint2)); + let magic_accept_task = tokio::spawn(db.magic_accept_loop(endpoint)); + // let _quinn_accept_task = tokio::spawn(db.quinn_accept_loop(endpoint)); + magic_accept_task.await??; Ok(()) } diff --git a/content-tracker/src/tracker.rs b/content-tracker/src/tracker.rs index 37501745..3de58914 100644 --- a/content-tracker/src/tracker.rs +++ b/content-tracker/src/tracker.rs @@ -26,7 +26,7 @@ use crate::{ protocol::{ Announce, AnnounceKind, Query, QueryResponse, Request, Response, REQUEST_SIZE_LIMIT, }, - NodeId, + NodeId, accept_conn, }; /// The tracker server. @@ -158,6 +158,44 @@ impl Tracker { } } + pub async fn magic_accept_loop(self, endpoint: MagicEndpoint) -> std::io::Result<()> { + while let Some(connecting) = endpoint.accept().await { + tracing::info!("got connecting"); + let db = self.clone(); + tokio::spawn(async move { + let Ok((remote_node_id, alpn, conn)) = accept_conn(connecting).await else { + tracing::error!("error accepting connection"); + return; + }; + // if we were supporting multiple protocols, we'd need to check the ALPN here. + tracing::info!("got connection from {} {}", remote_node_id, alpn); + if let Err(cause) = db.handle_connection(conn).await { + tracing::error!("error handling connection: {}", cause); + } + }); + } + Ok(()) + } + + pub async fn quinn_accept_loop(self, endpoint: quinn::Endpoint) -> std::io::Result<()> { + while let Some(connecting) = endpoint.accept().await { + tracing::info!("got connecting"); + let db = self.clone(); + tokio::spawn(async move { + let Ok((remote_node_id, alpn, conn)) = accept_conn(connecting).await else { + tracing::error!("error accepting connection"); + return; + }; + // if we were supporting multiple protocols, we'd need to check the ALPN here. + tracing::info!("got connection from {} {}", remote_node_id, alpn); + if let Err(cause) = db.handle_connection(conn).await { + tracing::error!("error handling connection: {}", cause); + } + }); + } + Ok(()) + } + /// Handle a single incoming connection on the tracker ALPN. pub async fn handle_connection(&self, connection: quinn::Connection) -> anyhow::Result<()> { log!("calling accept_bi"); From 5413618ba9d97b7c32dbc4a5aad87198398b0f1b Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sat, 20 Jan 2024 13:47:59 +0200 Subject: [PATCH 03/13] Use iroh-pkarr-node-discovery --- content-tracker/Cargo.lock | 63 +++++---- content-tracker/Cargo.toml | 1 + content-tracker/src/discovery.rs | 228 ------------------------------- content-tracker/src/main.rs | 8 +- content-tracker/src/tracker.rs | 3 +- 5 files changed, 47 insertions(+), 256 deletions(-) delete mode 100644 content-tracker/src/discovery.rs diff --git a/content-tracker/Cargo.lock b/content-tracker/Cargo.lock index 17f4136b..fe6f944e 100644 --- a/content-tracker/Cargo.lock +++ b/content-tracker/Cargo.lock @@ -126,9 +126,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" dependencies = [ "backtrace", ] @@ -1202,9 +1202,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1217,9 +1217,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1227,15 +1227,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1244,15 +1244,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -1261,15 +1261,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -1279,9 +1279,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1953,6 +1953,7 @@ dependencies = [ "hex", "humantime", "iroh", + "iroh-pkarr-node-discovery", "mainline", "pkarr", "postcard", @@ -2111,6 +2112,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "iroh-pkarr-node-discovery" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80dd8cd00337e660ec7155fb7911932ac3f85abdc1742b52b9257da6fb2945e2" +dependencies = [ + "anyhow", + "futures", + "iroh-net", + "pkarr", + "tokio", + "tracing", +] + [[package]] name = "iroh-sync" version = "0.12.0" @@ -4348,9 +4363,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -4366,9 +4381,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", diff --git a/content-tracker/Cargo.toml b/content-tracker/Cargo.toml index 7c1fa053..ca7c1453 100644 --- a/content-tracker/Cargo.toml +++ b/content-tracker/Cargo.toml @@ -19,6 +19,7 @@ futures = "0.3.25" hex = "0.4.3" humantime = "2.1.0" iroh = "0.12.0" +iroh-pkarr-node-discovery = "0.1.0" mainline = "1.0.0" pkarr = { version = "1.0.1", features = ["async"] } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } diff --git a/content-tracker/src/discovery.rs b/content-tracker/src/discovery.rs deleted file mode 100644 index 3c57f1eb..00000000 --- a/content-tracker/src/discovery.rs +++ /dev/null @@ -1,228 +0,0 @@ -//! Discovery services -//! -//! Originally copied and adapted from -use std::{net::IpAddr, str::FromStr}; - -use anyhow::{Context, Result}; -use ed25519_dalek::SigningKey; -use futures::{future, future::BoxFuture, FutureExt}; -use iroh::{ - base::ticket::Ticket, - net::{key::SecretKey, magicsock::Discovery, AddrInfo, NodeAddr}, - ticket::NodeTicket, -}; -use mainline::{common::MutableItem, dht::DhtSettings}; -use pkarr::{ - dns::{ - rdata::{RData, A, AAAA, TXT}, - Name, Packet, ResourceRecord, CLASS, - }, - Keypair, SignedPacket, -}; -use url::Url; - -use crate::NodeId; - -const IROH_TICKET_KEY: &str = "_ticket.iroh."; -#[derive(Debug, Clone)] -pub struct PkarrDiscovery { - keypair: SecretKey, - pkarr_client: pkarr::PkarrClient, -} - -impl PkarrDiscovery { - pub fn new(keypair: SecretKey) -> Self { - let pkarr_client = pkarr::PkarrClient::new(); - Self { - keypair, - pkarr_client, - } - } -} - -fn node_addr_to_packet(secret: SecretKey, addr: NodeAddr, ttl: u32) -> Result { - let mut packet = Packet::new_reply(0); - let ticket = NodeTicket::new(addr)?; - let ticket = ticket.to_string(); - packet.answers.push(ResourceRecord::new( - Name::new(IROH_TICKET_KEY).unwrap(), - CLASS::IN, - ttl, - RData::TXT(TXT::new().with_string(&ticket)?), - )); - let keypair = Keypair::from_secret_key(&secret.to_bytes()); - let signed_packet = SignedPacket::from_packet(&keypair, &packet)?; - Ok(signed_packet) -} - -#[allow(unused)] -fn filter_ipaddr(rr: &ResourceRecord) -> Option { - if rr.class != CLASS::IN { - return None; - } - let addr: IpAddr = match rr.rdata { - RData::A(A { address }) => IpAddr::V4(address.into()), - RData::AAAA(AAAA { address }) => IpAddr::V6(address.into()), - _ => return None, - }; - Some(addr) -} - -fn filter_txt(rr: &ResourceRecord) -> Option { - if rr.class != CLASS::IN { - return None; - } - if let RData::TXT(txt) = &rr.rdata { - String::try_from(txt.clone()).ok() - } else { - None - } -} - -fn filter_ticket(rr: &ResourceRecord) -> Option { - let txt = filter_txt(rr)?; - NodeTicket::from_str(&txt).ok() -} - -fn packet_to_node_addr(packet: &SignedPacket) -> anyhow::Result { - packet - .resource_records(IROH_TICKET_KEY) - .find_map(filter_ticket) - .map(|ticket| ticket.node_addr().clone()) - .context("No ticket found in packet") -} - -impl Discovery for PkarrDiscovery { - fn publish(&self, info: &AddrInfo) { - let Ok(signed_packet) = node_addr_to_packet( - self.keypair.clone(), - NodeAddr { - node_id: self.keypair.public(), - info: info.clone(), - }, - 60, - ) else { - return; - }; - let pkarr_client = self.pkarr_client.clone(); - - tokio::spawn(async move { - let res = pkarr_client.publish(&signed_packet).await; - match res { - Ok(x) => { - println!("published to pkarr: {:?}", x); - } - Err(cause) => { - tracing::warn!("error publishing to pkarr: {}", cause); - } - } - }); - } - - fn resolve<'a>(&'a self, node_id: &'a iroh::net::NodeId) -> BoxFuture<'a, Result> { - let pkarr_client = self.pkarr_client.clone(); - let node_id_bytes = *node_id.as_bytes(); - tokio::spawn(async move { - let public_key = pkarr::PublicKey::try_from(node_id_bytes)?; - let signed_packet = pkarr_client - .resolve(public_key) - .await - .context("not found")?; - let addr: NodeAddr = packet_to_node_addr(&signed_packet)?; - Ok(addr.info) - }) - .map(|res| match res { - Ok(res) => res, - Err(cause) => Err(cause.into()), - }) - .boxed() - } -} - -#[derive(Debug, Clone)] -pub struct MainlineDiscovery { - mainline_client: mainline::async_dht::AsyncDht, - keypair: SecretKey, -} - -impl MainlineDiscovery { - pub fn new(keypair: SecretKey) -> Self { - let mainline_client = mainline::Dht::new(DhtSettings::default()).as_async(); - Self { - mainline_client, - keypair, - } - } -} - -impl Discovery for MainlineDiscovery { - fn publish(&self, info: &AddrInfo) { - let Ok(ticket) = iroh::net::ticket::NodeTicket::new(NodeAddr { - node_id: self.keypair.public(), - info: info.clone(), - }) else { - return; - }; - let ticket = ticket.to_bytes().into(); - let signing_key = SigningKey::from(self.keypair.to_bytes()); - let item = MutableItem::new(signing_key, ticket, 0, None); - let mainline_client = self.mainline_client.clone(); - tokio::spawn(async move { - let res = mainline_client.put_mutable(item).await; - match res { - Ok(x) => { - println!("published to mainline: {:?}", x); - } - Err(cause) => { - tracing::warn!("error publishing to mainline: {}", cause); - } - } - }); - } - - fn resolve<'a>(&'a self, node_id: &'a iroh::net::NodeId) -> BoxFuture<'a, Result> { - let public_key = *node_id.clone().as_bytes(); - let mainline_client = self.mainline_client.clone(); - tokio::spawn(async move { - let mut response = mainline_client.get_mutable(&public_key, None).await; - while let Some(item) = response.next_async().await { - let Ok(ticket) = iroh::net::ticket::NodeTicket::from_bytes(item.item.value()) - else { - continue; - }; - return Ok(ticket.node_addr().info.clone()); - } - Err(anyhow::anyhow!("not found")) - }) - .map(|res| match res { - Ok(res) => res, - Err(cause) => Err(cause.into()), - }) - .boxed() - } -} - -/// A discovery method that just uses a hardcoded region. -#[derive(Debug)] -pub struct HardcodedDerperDiscovery { - derp_url: Url, -} - -impl HardcodedDerperDiscovery { - /// Create a new discovery method that always returns the given region. - pub fn new(derp_url: Url) -> Self { - Self { derp_url } - } -} - -impl iroh::net::magicsock::Discovery for HardcodedDerperDiscovery { - fn publish(&self, _info: &AddrInfo) {} - - fn resolve<'a>(&'a self, _node_id: &'a NodeId) -> BoxFuture<'a, Result> { - future::ok(AddrInfo { - derp_url: Some(self.derp_url.clone()), - direct_addresses: Default::default(), - }) - .boxed() - } -} diff --git a/content-tracker/src/main.rs b/content-tracker/src/main.rs index f1e853de..330eb981 100644 --- a/content-tracker/src/main.rs +++ b/content-tracker/src/main.rs @@ -1,5 +1,4 @@ pub mod args; -pub mod discovery; pub mod io; pub mod iroh_bytes_util; pub mod options; @@ -16,9 +15,11 @@ use clap::Parser; use io::CONFIG_DEFAULTS_FILE; use iroh::net::{ magic_endpoint::{get_alpn, get_remote_node_id}, - AddrInfo, MagicEndpoint, NodeAddr, + MagicEndpoint, }; use iroh::util::fs::load_secret_key; +use iroh_pkarr_node_discovery::PkarrNodeDiscovery; +use pkarr::PkarrClient; use tokio_util::task::LocalPoolHandle; use crate::{ @@ -73,7 +74,8 @@ async fn create_endpoint( key: iroh::net::key::SecretKey, port: u16, ) -> anyhow::Result { - let mainline_discovery = discovery::PkarrDiscovery::new(key.clone()); + let pkarr = PkarrClient::new(); + let mainline_discovery = PkarrNodeDiscovery::new(pkarr, Some(&key)); iroh::net::MagicEndpoint::builder() .secret_key(key) .discovery(Box::new(mainline_discovery)) diff --git a/content-tracker/src/tracker.rs b/content-tracker/src/tracker.rs index 3de58914..11e6e528 100644 --- a/content-tracker/src/tracker.rs +++ b/content-tracker/src/tracker.rs @@ -17,6 +17,7 @@ use iroh::net::MagicEndpoint; use rand::Rng; use crate::{ + accept_conn, io::{log_connection_attempt, log_probe_attempt, AnnounceData}, iroh_bytes_util::{ chunk_probe, get_hash_seq_and_sizes, random_hash_seq_ranges, unverified_size, verified_size, @@ -26,7 +27,7 @@ use crate::{ protocol::{ Announce, AnnounceKind, Query, QueryResponse, Request, Response, REQUEST_SIZE_LIMIT, }, - NodeId, accept_conn, + NodeId, }; /// The tracker server. From 2d49a8017ce20b0ab2ba21d4a2d4ed6f23087466 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sat, 20 Jan 2024 14:22:28 +0200 Subject: [PATCH 04/13] Allow tracker id to be either node id or direct socket addr --- content-tracker/Cargo.lock | 2 -- content-tracker/Cargo.toml | 2 +- content-tracker/src/args.rs | 4 +-- content-tracker/src/main.rs | 50 +++++++++++++++++++++++++++++++------ 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/content-tracker/Cargo.lock b/content-tracker/Cargo.lock index fe6f944e..7b8d0ba1 100644 --- a/content-tracker/Cargo.lock +++ b/content-tracker/Cargo.lock @@ -2115,8 +2115,6 @@ dependencies = [ [[package]] name = "iroh-pkarr-node-discovery" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80dd8cd00337e660ec7155fb7911932ac3f85abdc1742b52b9257da6fb2945e2" dependencies = [ "anyhow", "futures", diff --git a/content-tracker/Cargo.toml b/content-tracker/Cargo.toml index ca7c1453..7f67d8d4 100644 --- a/content-tracker/Cargo.toml +++ b/content-tracker/Cargo.toml @@ -19,7 +19,7 @@ futures = "0.3.25" hex = "0.4.3" humantime = "2.1.0" iroh = "0.12.0" -iroh-pkarr-node-discovery = "0.1.0" +iroh-pkarr-node-discovery = { path = "../iroh-pkarr-node-discovery" } mainline = "1.0.0" pkarr = { version = "1.0.1", features = ["async"] } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } diff --git a/content-tracker/src/args.rs b/content-tracker/src/args.rs index 6342e493..445a921f 100644 --- a/content-tracker/src/args.rs +++ b/content-tracker/src/args.rs @@ -4,7 +4,7 @@ use iroh::bytes::{Hash, HashAndFormat}; use iroh::ticket::BlobTicket; use std::{fmt::Display, str::FromStr}; -use crate::NodeId; +use crate::{NodeId, TrackerId}; #[derive(Parser, Debug)] pub struct Args { @@ -119,7 +119,7 @@ pub struct AnnounceArgs { #[derive(Parser, Debug)] pub struct QueryArgs { #[clap(long)] - pub tracker: NodeId, + pub tracker: TrackerId, /// the port to use for querying #[clap(long)] diff --git a/content-tracker/src/main.rs b/content-tracker/src/main.rs index 330eb981..d14ee3a3 100644 --- a/content-tracker/src/main.rs +++ b/content-tracker/src/main.rs @@ -7,6 +7,8 @@ pub mod tracker; use std::{ collections::BTreeSet, + net::SocketAddr, + str::FromStr, sync::atomic::{AtomicBool, Ordering}, time::{Duration, Instant}, }; @@ -35,6 +37,35 @@ use crate::{ pub type NodeId = iroh::net::key::PublicKey; +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum TrackerId { + NodeId(NodeId), + Addr(SocketAddr), +} + +impl std::fmt::Display for TrackerId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TrackerId::NodeId(node_id) => write!(f, "{}", node_id), + TrackerId::Addr(addr) => write!(f, "{}", addr), + } + } +} + +impl FromStr for TrackerId { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + if let Ok(node_id) = s.parse() { + return Ok(TrackerId::NodeId(node_id)); + } + if let Ok(addr) = s.parse() { + return Ok(TrackerId::Addr(addr)); + } + anyhow::bail!("invalid tracker id") + } +} + static VERBOSE: AtomicBool = AtomicBool::new(false); fn set_verbose(verbose: bool) { @@ -73,9 +104,11 @@ async fn await_derp_region(endpoint: &MagicEndpoint) -> anyhow::Result<()> { async fn create_endpoint( key: iroh::net::key::SecretKey, port: u16, + publish: bool, ) -> anyhow::Result { let pkarr = PkarrClient::new(); - let mainline_discovery = PkarrNodeDiscovery::new(pkarr, Some(&key)); + let discovery_key = if publish { Some(&key) } else { None }; + let mainline_discovery = PkarrNodeDiscovery::new(pkarr, discovery_key); iroh::net::MagicEndpoint::builder() .secret_key(key) .discovery(Box::new(mainline_discovery)) @@ -108,7 +141,7 @@ async fn server(args: ServerArgs) -> anyhow::Result<()> { log!("tracker starting using {}", home.display()); let key_path = tracker_path(SERVER_KEY_FILE)?; let key = load_secret_key(key_path).await?; - let endpoint = create_endpoint(key, args.magic_port).await?; + let endpoint = create_endpoint(key, args.magic_port, true).await?; let config_path = tracker_path(CONFIG_FILE)?; write_defaults()?; let mut options = load_from_file::(&config_path)?; @@ -140,7 +173,7 @@ async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { // for now, a random node id is more reliable. // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; let key = iroh::net::key::SecretKey::generate(); - let endpoint = create_endpoint(key, 11112).await?; + let endpoint = create_endpoint(key, 11112, false).await?; log!("announce {:?}", args); log!("trying to connect to {:?}", args.tracker); let connection = endpoint @@ -190,7 +223,7 @@ async fn query(args: QueryArgs) -> anyhow::Result<()> { // for now, a random node id is more reliable. // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; let key = iroh::net::key::SecretKey::generate(); - let endpoint = create_endpoint(key, args.port.unwrap_or_default()).await?; + let endpoint = create_endpoint(key, args.port.unwrap_or_default(), false).await?; let query = Query { content: args.content.hash_and_format(), flags: QueryFlags { @@ -198,10 +231,11 @@ async fn query(args: QueryArgs) -> anyhow::Result<()> { verified: args.verified, }, }; - log!("trying to connect to tracker at {:?}", args.tracker); - let connection = endpoint - .connect_by_node_id(&args.tracker, TRACKER_ALPN) - .await?; + let TrackerId::NodeId(tracker) = args.tracker else { + anyhow::bail!("tracker must be specified as a node id"); + }; + log!("trying to connect to tracker at {:?}", tracker); + let connection = endpoint.connect_by_node_id(&tracker, TRACKER_ALPN).await?; log!("connected to {:?}", connection.remote_address()); let (mut send, mut recv) = connection.open_bi().await?; log!("opened bi stream"); From 4347d3ae24ff62c8620ccfb73deb6f12347d7852 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sat, 20 Jan 2024 15:18:13 +0200 Subject: [PATCH 05/13] Hack in listening on raw sockets --- content-tracker/Cargo.lock | 18 +++++- content-tracker/Cargo.toml | 2 + content-tracker/src/main.rs | 105 ++++++++++++++++++++++++++++++--- content-tracker/src/tracker.rs | 2 + 4 files changed, 118 insertions(+), 9 deletions(-) diff --git a/content-tracker/Cargo.lock b/content-tracker/Cargo.lock index 7b8d0ba1..157c491b 100644 --- a/content-tracker/Cargo.lock +++ b/content-tracker/Cargo.lock @@ -1959,7 +1959,9 @@ dependencies = [ "postcard", "quinn", "rand", + "rcgen 0.12.0", "redb", + "rustls", "serde", "serde_json", "simple-mdns", @@ -2079,7 +2081,7 @@ dependencies = [ "quinn-udp", "rand", "rand_core", - "rcgen", + "rcgen 0.11.3", "reqwest", "ring 0.17.5", "rtnetlink", @@ -3336,6 +3338,18 @@ dependencies = [ "yasna", ] +[[package]] +name = "rcgen" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d918c80c5a4c7560db726763020bd16db179e4d5b828078842274a443addb5d" +dependencies = [ + "pem 3.0.2", + "ring 0.17.5", + "time", + "yasna", +] + [[package]] name = "redb" version = "1.5.0" @@ -4410,7 +4424,7 @@ dependencies = [ "futures", "log", "pem 2.0.1", - "rcgen", + "rcgen 0.11.3", "reqwest", "ring 0.16.20", "rustls", diff --git a/content-tracker/Cargo.toml b/content-tracker/Cargo.toml index 7f67d8d4..10bae9e8 100644 --- a/content-tracker/Cargo.toml +++ b/content-tracker/Cargo.toml @@ -25,7 +25,9 @@ pkarr = { version = "1.0.1", features = ["async"] } postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } quinn = "0.10" rand = "0.8" +rcgen = "0.12.0" redb = "1.5.0" +rustls = "0.21" serde = { version = "1", features = ["derive"] } serde_json = "1.0.107" simple-mdns = { version = "0.5.0", features = ["async-tokio"] } diff --git a/content-tracker/src/main.rs b/content-tracker/src/main.rs index d14ee3a3..bc2fe10a 100644 --- a/content-tracker/src/main.rs +++ b/content-tracker/src/main.rs @@ -7,9 +7,12 @@ pub mod tracker; use std::{ collections::BTreeSet, - net::SocketAddr, + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, str::FromStr, - sync::atomic::{AtomicBool, Ordering}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, time::{Duration, Instant}, }; @@ -141,7 +144,7 @@ async fn server(args: ServerArgs) -> anyhow::Result<()> { log!("tracker starting using {}", home.display()); let key_path = tracker_path(SERVER_KEY_FILE)?; let key = load_secret_key(key_path).await?; - let endpoint = create_endpoint(key, args.magic_port, true).await?; + let endpoint = create_endpoint(key.clone(), args.magic_port, true).await?; let config_path = tracker_path(CONFIG_FILE)?; write_defaults()?; let mut options = load_from_file::(&config_path)?; @@ -159,11 +162,16 @@ async fn server(args: ServerArgs) -> anyhow::Result<()> { ); log!(); let db2 = db.clone(); + let db3 = db.clone(); let endpoint2 = endpoint.clone(); let _probe_task = tpc.spawn_pinned(move || db2.probe_loop(endpoint2)); let magic_accept_task = tokio::spawn(db.magic_accept_loop(endpoint)); - // let _quinn_accept_task = tokio::spawn(db.quinn_accept_loop(endpoint)); + let server_config = configure_server(&key)?; + let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, args.quinn_port)); + let quinn_endpoint = quinn::Endpoint::server(server_config, bind_addr)?; + let quinn_accept_task = tokio::spawn(db3.quinn_accept_loop(quinn_endpoint)); magic_accept_task.await??; + quinn_accept_task.await??; Ok(()) } @@ -219,6 +227,13 @@ async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { async fn query(args: QueryArgs) -> anyhow::Result<()> { set_verbose(true); + match args.tracker { + TrackerId::Addr(tracker) => query_quinn(args, tracker).await, + TrackerId::NodeId(tracker) => query_magic(args, tracker).await, + } +} + +async fn query_magic(args: QueryArgs, tracker: NodeId) -> anyhow::Result<()> { // todo: uncomment once the connection problems are fixed // for now, a random node id is more reliable. // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; @@ -231,9 +246,6 @@ async fn query(args: QueryArgs) -> anyhow::Result<()> { verified: args.verified, }, }; - let TrackerId::NodeId(tracker) = args.tracker else { - anyhow::bail!("tracker must be specified as a node id"); - }; log!("trying to connect to tracker at {:?}", tracker); let connection = endpoint.connect_by_node_id(&tracker, TRACKER_ALPN).await?; log!("connected to {:?}", connection.remote_address()); @@ -257,6 +269,42 @@ async fn query(args: QueryArgs) -> anyhow::Result<()> { Ok(()) } +async fn query_quinn(args: QueryArgs, tracker: SocketAddr) -> anyhow::Result<()> { + // todo: uncomment once the connection problems are fixed + // for now, a random node id is more reliable. + // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; + let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)); + let endpoint = create_quinn_client(bind_addr, vec![TRACKER_ALPN.to_vec()], false)?; + let query = Query { + content: args.content.hash_and_format(), + flags: QueryFlags { + complete: !args.partial, + verified: args.verified, + }, + }; + log!("trying to connect to tracker at {:?}", tracker); + let connection = endpoint.connect(tracker, "localhost")?.await?; + log!("connected to {:?}", connection.remote_address()); + let (mut send, mut recv) = connection.open_bi().await?; + log!("opened bi stream"); + let request = Request::Query(query); + let request = postcard::to_stdvec(&request)?; + log!("sending query"); + send.write_all(&request).await?; + send.finish().await?; + let response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; + let response = postcard::from_bytes::(&response)?; + match response { + Response::QueryResponse(response) => { + log!("content {}", response.content); + for peer in response.hosts { + log!("- peer {}", peer); + } + } + } + Ok(()) +} + #[tokio::main(flavor = "multi_thread")] async fn main() -> anyhow::Result<()> { setup_logging(); @@ -267,3 +315,46 @@ async fn main() -> anyhow::Result<()> { Commands::Query(args) => query(args).await, } } + +/// Returns default server configuration along with its certificate. +#[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 +fn configure_server(secret_key: &iroh::net::key::SecretKey) -> anyhow::Result { + make_server_config(secret_key, 8, 1024, vec![TRACKER_ALPN.to_vec()]) +} + +fn create_quinn_client( + bind_addr: SocketAddr, + alpn_protocols: Vec>, + keylog: bool, +) -> anyhow::Result { + let secret_key = iroh::net::key::SecretKey::generate(); + let tls_client_config = + iroh::net::tls::make_client_config(&secret_key, None, alpn_protocols, keylog)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); + let mut endpoint = quinn::Endpoint::client(bind_addr)?; + let mut transport_config = quinn::TransportConfig::default(); + transport_config.keep_alive_interval(Some(Duration::from_secs(1))); + client_config.transport_config(Arc::new(transport_config)); + endpoint.set_default_client_config(client_config); + Ok(endpoint) +} + +/// Create a [`quinn::ServerConfig`] with the given secret key and limits. +pub fn make_server_config( + secret_key: &iroh::net::key::SecretKey, + max_streams: u64, + max_connections: u32, + alpn_protocols: Vec>, +) -> anyhow::Result { + let tls_server_config = iroh::net::tls::make_server_config(secret_key, alpn_protocols, false)?; + let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(tls_server_config)); + let mut transport_config = quinn::TransportConfig::default(); + transport_config + .max_concurrent_bidi_streams(max_streams.try_into()?) + .max_concurrent_uni_streams(0u32.into()); + + server_config + .transport_config(Arc::new(transport_config)) + .concurrent_connections(max_connections); + Ok(server_config) +} diff --git a/content-tracker/src/tracker.rs b/content-tracker/src/tracker.rs index 11e6e528..f5e216c1 100644 --- a/content-tracker/src/tracker.rs +++ b/content-tracker/src/tracker.rs @@ -179,6 +179,8 @@ impl Tracker { } pub async fn quinn_accept_loop(self, endpoint: quinn::Endpoint) -> std::io::Result<()> { + let local_addr = endpoint.local_addr()?; + println!("quinn listening on {}", local_addr); while let Some(connecting) = endpoint.accept().await { tracing::info!("got connecting"); let db = self.clone(); From 2a3bc261b80c9def95f2aeb73e6adfee605ceaa5 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sat, 20 Jan 2024 16:59:07 +0200 Subject: [PATCH 06/13] Finish listening on sockets --- content-tracker/src/main.rs | 43 +++++++++---------------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/content-tracker/src/main.rs b/content-tracker/src/main.rs index bc2fe10a..c54757fb 100644 --- a/content-tracker/src/main.rs +++ b/content-tracker/src/main.rs @@ -40,6 +40,7 @@ use crate::{ pub type NodeId = iroh::net::key::PublicKey; +/// A tracker id for queries - either a node id or an address. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum TrackerId { NodeId(NodeId), @@ -228,7 +229,7 @@ async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { async fn query(args: QueryArgs) -> anyhow::Result<()> { set_verbose(true); match args.tracker { - TrackerId::Addr(tracker) => query_quinn(args, tracker).await, + TrackerId::Addr(tracker) => query_socket(args, tracker).await, TrackerId::NodeId(tracker) => query_magic(args, tracker).await, } } @@ -239,42 +240,20 @@ async fn query_magic(args: QueryArgs, tracker: NodeId) -> anyhow::Result<()> { // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; let key = iroh::net::key::SecretKey::generate(); let endpoint = create_endpoint(key, args.port.unwrap_or_default(), false).await?; - let query = Query { - content: args.content.hash_and_format(), - flags: QueryFlags { - complete: !args.partial, - verified: args.verified, - }, - }; log!("trying to connect to tracker at {:?}", tracker); let connection = endpoint.connect_by_node_id(&tracker, TRACKER_ALPN).await?; - log!("connected to {:?}", connection.remote_address()); - let (mut send, mut recv) = connection.open_bi().await?; - log!("opened bi stream"); - let request = Request::Query(query); - let request = postcard::to_stdvec(&request)?; - log!("sending query"); - send.write_all(&request).await?; - send.finish().await?; - let response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; - let response = postcard::from_bytes::(&response)?; - match response { - Response::QueryResponse(response) => { - log!("content {}", response.content); - for peer in response.hosts { - log!("- peer {}", peer); - } - } - } - Ok(()) + query_connection(args, connection).await } -async fn query_quinn(args: QueryArgs, tracker: SocketAddr) -> anyhow::Result<()> { - // todo: uncomment once the connection problems are fixed - // for now, a random node id is more reliable. - // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; +async fn query_socket(args: QueryArgs, tracker: SocketAddr) -> anyhow::Result<()> { let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)); let endpoint = create_quinn_client(bind_addr, vec![TRACKER_ALPN.to_vec()], false)?; + log!("trying to connect to tracker at {:?}", tracker); + let connection = endpoint.connect(tracker, "localhost")?.await?; + query_connection(args, connection).await +} + +async fn query_connection(args: QueryArgs, connection: quinn::Connection) -> anyhow::Result<()> { let query = Query { content: args.content.hash_and_format(), flags: QueryFlags { @@ -282,8 +261,6 @@ async fn query_quinn(args: QueryArgs, tracker: SocketAddr) -> anyhow::Result<()> verified: args.verified, }, }; - log!("trying to connect to tracker at {:?}", tracker); - let connection = endpoint.connect(tracker, "localhost")?.await?; log!("connected to {:?}", connection.remote_address()); let (mut send, mut recv) = connection.open_bi().await?; log!("opened bi stream"); From 658358b248704b711daac1978a33721b190c81b6 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sat, 20 Jan 2024 17:09:24 +0200 Subject: [PATCH 07/13] rename --- ...nt_oRa99sDBdwbXeqQw1VDnwOYt0pA7aWeEZ-I6s6zRNLk | Bin 0 -> 138 bytes .../.gitignore | 0 .../Cargo.lock | 0 .../Cargo.toml | 6 +++--- .../README.md | 0 .../src/args.rs | 0 .../src/io.rs | 0 .../src/iroh_bytes_util.rs | 0 .../src/main.rs | 0 .../src/options.rs | 0 .../src/protocol.rs | 0 .../src/tracker.rs | 0 12 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 iroh-gateway/letsencrypt-staging/cached_account_oRa99sDBdwbXeqQw1VDnwOYt0pA7aWeEZ-I6s6zRNLk rename {content-tracker => iroh-mainline-content-discovery}/.gitignore (100%) rename {content-tracker => iroh-mainline-content-discovery}/Cargo.lock (100%) rename {content-tracker => iroh-mainline-content-discovery}/Cargo.toml (90%) rename {content-tracker => iroh-mainline-content-discovery}/README.md (100%) rename {content-tracker => iroh-mainline-content-discovery}/src/args.rs (100%) rename {content-tracker => iroh-mainline-content-discovery}/src/io.rs (100%) rename {content-tracker => iroh-mainline-content-discovery}/src/iroh_bytes_util.rs (100%) rename {content-tracker => iroh-mainline-content-discovery}/src/main.rs (100%) rename {content-tracker => iroh-mainline-content-discovery}/src/options.rs (100%) rename {content-tracker => iroh-mainline-content-discovery}/src/protocol.rs (100%) rename {content-tracker => iroh-mainline-content-discovery}/src/tracker.rs (100%) diff --git a/iroh-gateway/letsencrypt-staging/cached_account_oRa99sDBdwbXeqQw1VDnwOYt0pA7aWeEZ-I6s6zRNLk b/iroh-gateway/letsencrypt-staging/cached_account_oRa99sDBdwbXeqQw1VDnwOYt0pA7aWeEZ-I6s6zRNLk new file mode 100644 index 0000000000000000000000000000000000000000..c68e377fb8b2e1fa556fa24421b268c7a8de5c26 GIT binary patch literal 138 zcmXqLY-eI*Fc4;A*J|@PXUoLM#sOw9GqSVf8e}suGO{ShzLa3@I8bw1%vQ-@Szq`J zJ Date: Sat, 20 Jan 2024 18:46:50 +0200 Subject: [PATCH 08/13] Make this also a lib --- iroh-mainline-content-discovery/Cargo.lock | 909 +++--------------- iroh-mainline-content-discovery/Cargo.toml | 13 +- iroh-mainline-content-discovery/src/args.rs | 7 +- iroh-mainline-content-discovery/src/io.rs | 5 +- .../src/iroh_bytes_util.rs | 34 +- iroh-mainline-content-discovery/src/lib.rs | 195 ++++ iroh-mainline-content-discovery/src/main.rs | 236 +++-- .../src/protocol.rs | 10 +- .../src/tracker.rs | 32 +- 9 files changed, 484 insertions(+), 957 deletions(-) create mode 100644 iroh-mainline-content-discovery/src/lib.rs diff --git a/iroh-mainline-content-discovery/Cargo.lock b/iroh-mainline-content-discovery/Cargo.lock index 157c491b..2ba85e16 100644 --- a/iroh-mainline-content-discovery/Cargo.lock +++ b/iroh-mainline-content-discovery/Cargo.lock @@ -49,12 +49,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aliasable" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" - [[package]] name = "allocator-api2" version = "0.2.16" @@ -111,7 +105,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -121,7 +115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -268,12 +262,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.5" @@ -292,15 +280,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597bb81c80a54b6a4381b23faba8d7774b144c94cbd1d6fe3f1329bd776554ab" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -441,17 +420,6 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" -[[package]] -name = "clipboard-win" -version = "4.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" -dependencies = [ - "error-code", - "str-buf", - "winapi", -] - [[package]] name = "cobs" version = "0.2.3" @@ -464,29 +432,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" -[[package]] -name = "colored" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" -dependencies = [ - "is-terminal", - "lazy_static", - "windows-sys 0.48.0", -] - -[[package]] -name = "comfy-table" -version = "7.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" -dependencies = [ - "crossterm", - "strum", - "strum_macros", - "unicode-width", -] - [[package]] name = "concurrent-queue" version = "2.4.0" @@ -496,36 +441,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "config" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7" -dependencies = [ - "async-trait", - "indexmap 1.9.3", - "lazy_static", - "nom", - "pathdiff", - "ron", - "serde", - "serde_json", - "toml 0.5.11", -] - -[[package]] -name = "console" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys 0.45.0", -] - [[package]] name = "const-oid" version = "0.9.5" @@ -607,28 +522,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossterm" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" -dependencies = [ - "bitflags 2.4.1", - "crossterm_winapi", - "libc", - "parking_lot", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - [[package]] name = "crypto-bigint" version = "0.5.3" @@ -835,17 +728,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "dialoguer" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" -dependencies = [ - "console", - "shell-words", - "thiserror", -] - [[package]] name = "digest" version = "0.10.7" @@ -858,15 +740,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys", -] - [[package]] name = "dirs-next" version = "2.0.0" @@ -877,18 +750,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -980,24 +841,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "educe" -version = "0.4.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0042ff8246a363dbe77d2ceedb073339e85a804b9a47636c6e016a9a32c05f" -dependencies = [ - "enum-ordinalize", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "either" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" - [[package]] name = "elliptic-curve" version = "0.13.6" @@ -1023,12 +866,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - [[package]] name = "encoding_rs" version = "0.8.33" @@ -1056,19 +893,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "enum-ordinalize" -version = "3.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf1fa3f06bbff1ea5b1a9c7b14aa992a39657db60a2759457328d7e058f49ee" -dependencies = [ - "num-bigint", - "num-traits", - "proc-macro2", - "quote", - "syn 2.0.39", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -1097,17 +921,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "error-code" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" -dependencies = [ - "libc", - "str-buf", + "windows-sys", ] [[package]] @@ -1133,17 +947,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" -[[package]] -name = "fd-lock" -version = "3.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" -dependencies = [ - "cfg-if", - "rustix", - "windows-sys 0.48.0", -] - [[package]] name = "ff" version = "0.13.0" @@ -1160,19 +963,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a481586acf778f1b1455424c343f71124b048ffa5f4fc3f8f6ae9dc432dcb3c7" -[[package]] -name = "flume" -version = "0.10.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" -dependencies = [ - "futures-core", - "futures-sink", - "nanorand", - "pin-project", - "spin 0.9.8", -] - [[package]] name = "flume" version = "0.11.0" @@ -1319,7 +1109,7 @@ version = "0.99.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738" dependencies = [ - "proc-macro-error 0.4.12", + "proc-macro-error", "proc-macro-hack", "proc-macro2", "quote", @@ -1426,15 +1216,6 @@ dependencies = [ "allocator-api2", ] -[[package]] -name = "hashlink" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" -dependencies = [ - "hashbrown 0.14.2", -] - [[package]] name = "heck" version = "0.4.1" @@ -1477,15 +1258,6 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3688e69b38018fec1557254f64c8dc2cc8ec502890182f395dbb0aa997aa5735" -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - [[package]] name = "hostname" version = "0.3.1" @@ -1571,25 +1343,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "human-time" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "259822ea527bd0d5ebf3108d84e98ac4f20769e2e8b2f3ab76e1dd6e21de7f3c" -dependencies = [ - "human-time-macros", -] - -[[package]] -name = "human-time-macros" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a04129f85bfd960234ed3fa53212a4904881e024322e804ac5a48da239e44a09" -dependencies = [ - "quote", - "syn 1.0.109", -] - [[package]] name = "humantime" version = "2.1.0" @@ -1682,7 +1435,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.51.1", ] [[package]] @@ -1740,7 +1493,6 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", - "serde", ] [[package]] @@ -1753,20 +1505,6 @@ dependencies = [ "hashbrown 0.14.2", ] -[[package]] -name = "indicatif" -version = "0.17.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" -dependencies = [ - "console", - "instant", - "number_prefix", - "portable-atomic", - "tokio", - "unicode-width", -] - [[package]] name = "inout" version = "0.1.3" @@ -1802,7 +1540,7 @@ checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ "socket2 0.5.5", "widestring", - "windows-sys 0.48.0", + "windows-sys", "winreg", ] @@ -1812,66 +1550,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" -[[package]] -name = "iroh" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3aea70a8acc645bc78c388f9412b4ae5f94ddeeea13c946b29af4c2db097c23" -dependencies = [ - "anyhow", - "bao-tree", - "bytes", - "clap", - "colored", - "comfy-table", - "config", - "console", - "data-encoding", - "derive_more", - "dialoguer", - "dirs-next", - "flume 0.11.0", - "futures", - "genawaiter", - "hashlink", - "hex", - "human-time", - "indicatif", - "iroh-base", - "iroh-bytes", - "iroh-gossip", - "iroh-io", - "iroh-metrics", - "iroh-net", - "iroh-sync", - "multibase", - "num_cpus", - "once_cell", - "parking_lot", - "portable-atomic", - "postcard", - "quic-rpc", - "quinn", - "rand", - "range-collections", - "rustyline", - "serde", - "shell-words", - "shellexpand", - "strum", - "tempfile", - "thiserror", - "time", - "tokio", - "tokio-stream", - "tokio-util", - "toml 0.8.8", - "tracing", - "tracing-subscriber", - "url", - "walkdir", -] - [[package]] name = "iroh-base" version = "0.12.0" @@ -1914,7 +1592,7 @@ dependencies = [ "chrono", "data-encoding", "derive_more", - "flume 0.11.0", + "flume", "futures", "genawaiter", "hex", @@ -1939,8 +1617,21 @@ dependencies = [ ] [[package]] -name = "iroh-content-tracker" -version = "0.2.0" +name = "iroh-io" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ace4f69567bfeb726672bab901d3e81be0c01119d860b2a10b7efb1553b880" +dependencies = [ + "bytes", + "futures", + "pin-project", + "smallvec", + "tokio", +] + +[[package]] +name = "iroh-mainline-content-discovery" +version = "0.3.0" dependencies = [ "anyhow", "bao-tree", @@ -1952,7 +1643,8 @@ dependencies = [ "futures", "hex", "humantime", - "iroh", + "iroh-bytes", + "iroh-net", "iroh-pkarr-node-discovery", "mainline", "pkarr", @@ -1968,55 +1660,13 @@ dependencies = [ "tempfile", "tokio", "tokio-util", - "toml 0.7.8", + "toml", "tracing", "tracing-subscriber", "ttl_cache", "url", ] -[[package]] -name = "iroh-gossip" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fcad132a6ff61ad83fc63203948919d2d77d58deac806bd08580f11e968a24" -dependencies = [ - "anyhow", - "bytes", - "data-encoding", - "derive_more", - "ed25519-dalek", - "futures", - "genawaiter", - "indexmap 2.1.0", - "iroh-base", - "iroh-blake3", - "iroh-metrics", - "iroh-net", - "once_cell", - "postcard", - "quinn", - "rand", - "rand_core", - "serde", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "iroh-io" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ace4f69567bfeb726672bab901d3e81be0c01119d860b2a10b7efb1553b880" -dependencies = [ - "bytes", - "futures", - "pin-project", - "smallvec", - "tokio", -] - [[package]] name = "iroh-metrics" version = "0.12.0" @@ -2056,7 +1706,7 @@ dependencies = [ "derive_more", "duct", "ed25519-dalek", - "flume 0.11.0", + "flume", "futures", "governor", "hex", @@ -2100,88 +1750,30 @@ dependencies = [ "time", "tokio", "tokio-rustls", - "tokio-rustls-acme", - "tokio-util", - "tracing", - "trust-dns-resolver", - "ttl_cache", - "url", - "watchable", - "webpki-roots", - "windows 0.51.1", - "wmi", - "x509-parser", - "zeroize", -] - -[[package]] -name = "iroh-pkarr-node-discovery" -version = "0.1.0" -dependencies = [ - "anyhow", - "futures", - "iroh-net", - "pkarr", - "tokio", - "tracing", -] - -[[package]] -name = "iroh-sync" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e27f9c5bf0576ad2786c1e32ee046518144b8f221eef6578c882633b6e27985" -dependencies = [ - "anyhow", - "bytes", - "data-encoding", - "derive_more", - "ed25519-dalek", - "flume 0.11.0", - "futures", - "hex", - "iroh-base", - "iroh-blake3", - "iroh-metrics", - "iroh-net", - "lru", - "num_enum", - "once_cell", - "ouroboros", - "parking_lot", - "postcard", - "quinn", - "rand", - "rand_core", - "redb", - "serde", - "strum", - "thiserror", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "url", -] - -[[package]] -name = "is-terminal" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" -dependencies = [ - "hermit-abi", - "rustix", - "windows-sys 0.48.0", + "tokio-rustls-acme", + "tokio-util", + "tracing", + "trust-dns-resolver", + "ttl_cache", + "url", + "watchable", + "webpki-roots", + "windows 0.51.1", + "wmi", + "x509-parser", + "zeroize", ] [[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +name = "iroh-pkarr-node-discovery" +version = "0.1.1" dependencies = [ - "either", + "anyhow", + "futures", + "iroh-net", + "pkarr", + "tokio", + "tracing", ] [[package]] @@ -2259,15 +1851,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "lru" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" -dependencies = [ - "hashbrown 0.14.2", -] - [[package]] name = "lru-cache" version = "0.1.2" @@ -2295,7 +1878,7 @@ dependencies = [ "bytes", "crc", "ed25519-dalek", - "flume 0.11.0", + "flume", "rand", "serde", "serde_bencode", @@ -2366,7 +1949,7 @@ checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -2602,12 +2185,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "object" version = "0.32.1" @@ -2644,12 +2221,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "os_pipe" version = "1.1.4" @@ -2657,32 +2228,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ae859aa07428ca9a929b936690f8b12dc5f11dd8c6992a18ca93919f28bc177" dependencies = [ "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "ouroboros" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c86de06555b970aec45229b27291b53154f21a5743a163419f4e4c0b065dcde" -dependencies = [ - "aliasable", - "ouroboros_macro", - "static_assertions", -] - -[[package]] -name = "ouroboros_macro" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cad0c4b129e9696e37cb712b243777b90ef489a0bfaa0ac34e7d9b860e4f134" -dependencies = [ - "heck", - "itertools", - "proc-macro-error 1.0.4", - "proc-macro2", - "quote", - "syn 2.0.39", + "windows-sys", ] [[package]] @@ -2750,19 +2296,13 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" -[[package]] -name = "pathdiff" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" - [[package]] name = "pem" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b13fe415cdf3c8e44518e18a7c95a13431d9bdf6d15367d82b23c377fdd441a" dependencies = [ - "base64 0.21.5", + "base64", "serde", ] @@ -2772,7 +2312,7 @@ version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923" dependencies = [ - "base64 0.21.5", + "base64", "serde", ] @@ -2965,12 +2505,6 @@ dependencies = [ "universal-hash", ] -[[package]] -name = "portable-atomic" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" - [[package]] name = "positioned-io" version = "0.3.3" @@ -3075,20 +2609,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" dependencies = [ - "proc-macro-error-attr 0.4.12", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr 1.0.4", + "proc-macro-error-attr", "proc-macro2", "quote", "syn 1.0.109", @@ -3108,17 +2629,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -3173,25 +2683,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "quic-rpc" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d60c2fc2390baad4b9d41ae9957ae88c3095496f88e252ef50722df8b5b78d7" -dependencies = [ - "bincode", - "educe", - "flume 0.10.14", - "futures", - "pin-project", - "quinn", - "serde", - "tokio", - "tokio-serde", - "tokio-util", - "tracing", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -3243,7 +2734,7 @@ dependencies = [ "libc", "socket2 0.5.5", "tracing", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -3401,13 +2892,13 @@ dependencies = [ [[package]] name = "reflink-copy" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f97f7665e51f23760e9e4949d454a4782c76ef954acaeec9d1b0f48a58e4529e" +checksum = "767be24c0da52e7448d495b8d162506a9aa125426651d547d545d6c2b4b65b62" dependencies = [ "cfg-if", "rustix", - "windows 0.51.1", + "windows 0.52.0", ] [[package]] @@ -3466,7 +2957,7 @@ version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ - "base64 0.21.5", + "base64", "bytes", "encoding_rs", "futures-core", @@ -3546,19 +3037,7 @@ dependencies = [ "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", -] - -[[package]] -name = "ron" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" -dependencies = [ - "base64 0.13.1", - "bitflags 1.3.2", - "indexmap 1.9.3", - "serde", + "windows-sys", ] [[package]] @@ -3640,7 +3119,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -3673,7 +3152,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.5", + "base64", ] [[package]] @@ -3692,29 +3171,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" -[[package]] -name = "rustyline" -version = "12.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9" -dependencies = [ - "bitflags 2.4.1", - "cfg-if", - "clipboard-win", - "fd-lock", - "home", - "libc", - "log", - "memchr", - "nix", - "radix_trie", - "scopeguard", - "unicode-segmentation", - "unicode-width", - "utf8parse", - "winapi", -] - [[package]] name = "ryu" version = "1.0.15" @@ -3730,22 +3186,13 @@ dependencies = [ "cipher", ] -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schannel" version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -3867,7 +3314,6 @@ version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ - "indexmap 2.1.0", "itoa", "ryu", "serde", @@ -3946,21 +3392,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "shell-words" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" - -[[package]] -name = "shellexpand" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" -dependencies = [ - "dirs", -] - [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -4038,7 +3469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -4107,18 +3538,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "str-buf" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" - [[package]] name = "strsim" version = "0.10.0" @@ -4296,7 +3715,7 @@ dependencies = [ "fastrand", "redox_syscall", "rustix", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -4388,7 +3807,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.5", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -4419,7 +3838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb6f50b5523d014ba161512c37457acb16fd8218c883c7152e0a67ab763f2d4" dependencies = [ "async-trait", - "base64 0.21.5", + "base64", "chrono", "futures", "log", @@ -4438,33 +3857,6 @@ dependencies = [ "x509-parser", ] -[[package]] -name = "tokio-serde" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911a61637386b789af998ee23f50aa30d5fd7edcec8d6d3dedae5e5815205466" -dependencies = [ - "bincode", - "bytes", - "educe", - "futures-core", - "futures-sink", - "pin-project", - "serde", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util", -] - [[package]] name = "tokio-util" version = "0.7.10" @@ -4477,21 +3869,10 @@ dependencies = [ "futures-util", "hashbrown 0.14.2", "pin-project-lite", - "slab", "tokio", "tracing", ] -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "indexmap 1.9.3", - "serde", -] - [[package]] name = "toml" version = "0.7.8" @@ -4504,18 +3885,6 @@ dependencies = [ "toml_edit 0.19.15", ] -[[package]] -name = "toml" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.21.0", -] - [[package]] name = "toml_datetime" version = "0.6.5" @@ -4549,19 +3918,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "toml_edit" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" -dependencies = [ - "indexmap 2.1.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "tower-service" version = "0.3.2" @@ -4743,18 +4099,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - -[[package]] -name = "unicode-width" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" - [[package]] name = "unicode-xid" version = "0.2.4" @@ -4813,16 +4157,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "walkdir" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "want" version = "0.3.1" @@ -4960,15 +4294,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -4992,10 +4317,20 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" dependencies = [ - "windows-core", + "windows-core 0.51.1", "windows-targets 0.48.5", ] +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core 0.52.0", + "windows-targets 0.52.0", +] + [[package]] name = "windows-core" version = "0.51.1" @@ -5005,6 +4340,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-implement" version = "0.48.0" @@ -5027,15 +4371,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -5045,21 +4380,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -5076,10 +4396,19 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" +name = "windows-targets" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] [[package]] name = "windows_aarch64_gnullvm" @@ -5088,10 +4417,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" +name = "windows_aarch64_gnullvm" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" [[package]] name = "windows_aarch64_msvc" @@ -5100,10 +4429,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "windows_i686_gnu" -version = "0.42.2" +name = "windows_aarch64_msvc" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" [[package]] name = "windows_i686_gnu" @@ -5112,10 +4441,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] -name = "windows_i686_msvc" -version = "0.42.2" +name = "windows_i686_gnu" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" [[package]] name = "windows_i686_msvc" @@ -5124,10 +4453,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +name = "windows_i686_msvc" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" [[package]] name = "windows_x86_64_gnu" @@ -5136,10 +4465,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" +name = "windows_x86_64_gnu" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" [[package]] name = "windows_x86_64_gnullvm" @@ -5148,10 +4477,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +name = "windows_x86_64_gnullvm" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" [[package]] name = "windows_x86_64_msvc" @@ -5159,6 +4488,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" version = "0.5.19" @@ -5175,7 +4510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] diff --git a/iroh-mainline-content-discovery/Cargo.toml b/iroh-mainline-content-discovery/Cargo.toml index 23b2a7eb..2aaa733a 100644 --- a/iroh-mainline-content-discovery/Cargo.toml +++ b/iroh-mainline-content-discovery/Cargo.toml @@ -11,14 +11,15 @@ license = "MIT OR Apache-2.0" anyhow = { version = "1", features = ["backtrace"] } bao-tree = { version = "0.9.1", features = ["tokio_fsm"], default-features = false } bytes = "1" -clap = { version = "4", features = ["derive"] } +clap = { version = "4", features = ["derive"], optional = true } derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } dirs-next = "2" ed25519-dalek = "2.1.0" futures = "0.3.25" hex = "0.4.3" humantime = "2.1.0" -iroh = "0.12.0" +iroh-net = "0.12.0" +iroh-bytes = "0.12.0" iroh-pkarr-node-discovery = { path = "../iroh-pkarr-node-discovery" } mainline = "1.0.0" pkarr = { version = "1.0.1", features = ["async"] } @@ -39,3 +40,11 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } ttl_cache = "0.5.1" url = "2.5.0" + +[features] +cli = ["clap"] +default = ["cli"] + +[[bin]] +name = "iroh-mainline-content-discovery" +required-features = ["cli"] diff --git a/iroh-mainline-content-discovery/src/args.rs b/iroh-mainline-content-discovery/src/args.rs index 445a921f..d26fac7d 100644 --- a/iroh-mainline-content-discovery/src/args.rs +++ b/iroh-mainline-content-discovery/src/args.rs @@ -1,11 +1,10 @@ //! Command line arguments. use clap::{Parser, Subcommand}; -use iroh::bytes::{Hash, HashAndFormat}; -use iroh::ticket::BlobTicket; +use iroh_bytes::{Hash, HashAndFormat}; +use iroh_mainline_content_discovery::TrackerId; +use iroh_net::{ticket::BlobTicket, NodeId}; use std::{fmt::Display, str::FromStr}; -use crate::{NodeId, TrackerId}; - #[derive(Parser, Debug)] pub struct Args { #[clap(subcommand)] diff --git a/iroh-mainline-content-discovery/src/io.rs b/iroh-mainline-content-discovery/src/io.rs index 86977387..024851a8 100644 --- a/iroh-mainline-content-discovery/src/io.rs +++ b/iroh-mainline-content-discovery/src/io.rs @@ -8,11 +8,12 @@ use std::{ }; use anyhow::Context; -use iroh::bytes::{get::Stats, HashAndFormat}; +use iroh_bytes::{get::Stats, HashAndFormat}; +use iroh_net::NodeId; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tracing_subscriber::{prelude::*, EnvFilter}; -use crate::{protocol::AnnounceKind, tracker::ProbeKind, NodeId}; +use crate::{protocol::AnnounceKind, tracker::ProbeKind}; pub const CONFIG_DEFAULTS_FILE: &str = "config.defaults.toml"; pub const CONFIG_FILE: &str = "config.toml"; diff --git a/iroh-mainline-content-discovery/src/iroh_bytes_util.rs b/iroh-mainline-content-discovery/src/iroh_bytes_util.rs index 7846826a..17d8048b 100644 --- a/iroh-mainline-content-discovery/src/iroh_bytes_util.rs +++ b/iroh-mainline-content-discovery/src/iroh_bytes_util.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use bao_tree::{ByteNum, ChunkNum, ChunkRanges}; use bytes::Bytes; -use iroh::bytes::{ +use iroh_bytes::{ get::{ fsm::{BlobContentNext, EndBlobNext}, Stats, @@ -14,8 +14,6 @@ use iroh::bytes::{ }; use rand::Rng; -use crate::log; - /// Get the claimed size of a blob from a peer. /// /// This is just reading the size header and then immediately closing the connection. @@ -24,13 +22,13 @@ pub async fn unverified_size( connection: &quinn::Connection, hash: &Hash, ) -> anyhow::Result<(u64, Stats)> { - let request = iroh::bytes::protocol::GetRequest::new( + let request = iroh_bytes::protocol::GetRequest::new( *hash, RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), ); - let request = iroh::bytes::get::fsm::start(connection.clone(), request); + let request = iroh_bytes::get::fsm::start(connection.clone(), request); let connected = request.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { + let iroh_bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { unreachable!("expected start root"); }; let at_blob_header = start.next(); @@ -47,14 +45,14 @@ pub async fn verified_size( connection: &quinn::Connection, hash: &Hash, ) -> anyhow::Result<(u64, Stats)> { - log!("Getting verified size of {}", hash.to_hex()); - let request = iroh::bytes::protocol::GetRequest::new( + tracing::debug!("Getting verified size of {}", hash.to_hex()); + let request = iroh_bytes::protocol::GetRequest::new( *hash, RangeSpecSeq::from_ranges(vec![ChunkRanges::from(ChunkNum(u64::MAX)..)]), ); - let request = iroh::bytes::get::fsm::start(connection.clone(), request); + let request = iroh_bytes::get::fsm::start(connection.clone(), request); let connected = request.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { + let iroh_bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { unreachable!("expected start root"); }; let header = start.next(); @@ -74,7 +72,7 @@ pub async fn verified_size( unreachable!("expected closing"); }; let stats = closing.next().await?; - log!( + tracing::debug!( "Got verified size of {}, {:.6}s", hash.to_hex(), stats.elapsed.as_secs_f64() @@ -88,17 +86,17 @@ pub async fn get_hash_seq_and_sizes( max_size: u64, ) -> anyhow::Result<(HashSeq, Arc<[u64]>)> { let content = HashAndFormat::hash_seq(*hash); - log!("Getting hash seq and children sizes of {}", content); - let request = iroh::bytes::protocol::GetRequest::new( + tracing::debug!("Getting hash seq and children sizes of {}", content); + let request = iroh_bytes::protocol::GetRequest::new( *hash, RangeSpecSeq::from_ranges_infinite([ ChunkRanges::all(), ChunkRanges::from(ChunkNum(u64::MAX)..), ]), ); - let at_start = iroh::bytes::get::fsm::start(connection.clone(), request); + let at_start = iroh_bytes::get::fsm::start(connection.clone(), request); let at_connected = at_start.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartRoot(start) = at_connected.next().await? else { + let iroh_bytes::get::fsm::ConnectedNext::StartRoot(start) = at_connected.next().await? else { unreachable!("query includes root"); }; let at_start_root = start.next(); @@ -127,7 +125,7 @@ pub async fn get_hash_seq_and_sizes( } }; let _stats = closing.next().await?; - log!( + tracing::debug!( "Got hash seq and children sizes of {}: {:?}", content, sizes @@ -144,9 +142,9 @@ pub async fn chunk_probe( let ranges = ChunkRanges::from(chunk..chunk + 1); let ranges = RangeSpecSeq::from_ranges([ranges]); let request = GetRequest::new(*hash, ranges); - let request = iroh::bytes::get::fsm::start(connection.clone(), request); + let request = iroh_bytes::get::fsm::start(connection.clone(), request); let connected = request.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { + let iroh_bytes::get::fsm::ConnectedNext::StartRoot(start) = connected.next().await? else { unreachable!("query includes root"); }; let header = start.next(); diff --git a/iroh-mainline-content-discovery/src/lib.rs b/iroh-mainline-content-discovery/src/lib.rs new file mode 100644 index 00000000..5c341542 --- /dev/null +++ b/iroh-mainline-content-discovery/src/lib.rs @@ -0,0 +1,195 @@ +use std::{net::SocketAddr, sync::Arc, time::Duration}; + +use anyhow::Context; +use iroh_net::{ + magic_endpoint::{get_alpn, get_remote_node_id}, + MagicEndpoint, NodeId, +}; +use iroh_pkarr_node_discovery::PkarrNodeDiscovery; +use pkarr::PkarrClient; +use protocol::QueryResponse; +use tokio::io::AsyncWriteExt; + +use crate::protocol::{Announce, Query, Request, Response, REQUEST_SIZE_LIMIT, TRACKER_ALPN}; + +pub mod io; +pub mod iroh_bytes_util; +pub mod options; +pub mod protocol; +pub mod tracker; + +/// A tracker id for queries - either a node id or an address. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum TrackerId { + NodeId(NodeId), + Addr(std::net::SocketAddr), +} + +impl std::fmt::Display for TrackerId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TrackerId::NodeId(node_id) => write!(f, "{}", node_id), + TrackerId::Addr(addr) => write!(f, "{}", addr), + } + } +} + +impl std::str::FromStr for TrackerId { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + if let Ok(node_id) = s.parse() { + return Ok(TrackerId::NodeId(node_id)); + } + if let Ok(addr) = s.parse() { + return Ok(TrackerId::Addr(addr)); + } + anyhow::bail!("invalid tracker id") + } +} + +/// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. +pub(crate) async fn accept_conn( + mut conn: quinn::Connecting, +) -> anyhow::Result<(NodeId, String, quinn::Connection)> { + let alpn = get_alpn(&mut conn).await?; + let conn = conn.await?; + let peer_id = get_remote_node_id(&conn)?; + Ok((peer_id, alpn, conn)) +} + +/// Announce to a tracker. +/// +/// You can only announce content you yourself claim to have, to avoid spamming other nodes. +/// +/// `endpoint` is the magic endpoint to use for announcing. +/// `tracker` is the node id of the tracker to announce to. It must understand the [TRACKER_ALPN] protocol. +/// `content` is the content to announce. +/// `kind` is the kind of the announcement. We can claim to have the complete data or only some of it. +pub async fn announce(connection: quinn::Connection, args: Announce) -> anyhow::Result<()> { + let (mut send, mut recv) = connection.open_bi().await?; + tracing::debug!("opened bi stream"); + let request = Request::Announce(args); + let request = postcard::to_stdvec(&request)?; + tracing::debug!("sending announce"); + send.write_all(&request).await?; + send.finish().await?; + let _response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; + Ok(()) +} + +/// Assume an existing connection to a tracker and query it for peers for some content. +pub async fn query(connection: quinn::Connection, args: Query) -> anyhow::Result { + tracing::info!("connected to {:?}", connection.remote_address()); + let (mut send, mut recv) = connection.open_bi().await?; + tracing::info!("opened bi stream"); + let request = Request::Query(args); + let request = postcard::to_stdvec(&request)?; + tracing::info!("sending query"); + send.write_all(&request).await?; + send.finish().await?; + let response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; + let response = postcard::from_bytes::(&response)?; + Ok(match response { + Response::QueryResponse(response) => response, + }) +} + +/// Returns default server configuration along with its certificate. +#[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 +fn configure_server(secret_key: &iroh_net::key::SecretKey) -> anyhow::Result { + make_server_config(secret_key, 8, 1024, vec![TRACKER_ALPN.to_vec()]) +} + +fn create_quinn_client( + bind_addr: SocketAddr, + alpn_protocols: Vec>, + keylog: bool, +) -> anyhow::Result { + let secret_key = iroh_net::key::SecretKey::generate(); + let tls_client_config = + iroh_net::tls::make_client_config(&secret_key, None, alpn_protocols, keylog)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); + let mut endpoint = quinn::Endpoint::client(bind_addr)?; + let mut transport_config = quinn::TransportConfig::default(); + transport_config.keep_alive_interval(Some(Duration::from_secs(1))); + client_config.transport_config(Arc::new(transport_config)); + endpoint.set_default_client_config(client_config); + Ok(endpoint) +} + +/// Create a [`quinn::ServerConfig`] with the given secret key and limits. +pub fn make_server_config( + secret_key: &iroh_net::key::SecretKey, + max_streams: u64, + max_connections: u32, + alpn_protocols: Vec>, +) -> anyhow::Result { + let tls_server_config = iroh_net::tls::make_server_config(secret_key, alpn_protocols, false)?; + let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(tls_server_config)); + let mut transport_config = quinn::TransportConfig::default(); + transport_config + .max_concurrent_bidi_streams(max_streams.try_into()?) + .max_concurrent_uni_streams(0u32.into()); + + server_config + .transport_config(Arc::new(transport_config)) + .concurrent_connections(max_connections); + Ok(server_config) +} + +/// Loads a [`SecretKey`] from the provided file. +pub async fn load_secret_key( + key_path: std::path::PathBuf, +) -> anyhow::Result { + if key_path.exists() { + let keystr = tokio::fs::read(key_path).await?; + let secret_key = + iroh_net::key::SecretKey::try_from_openssh(keystr).context("invalid keyfile")?; + Ok(secret_key) + } else { + let secret_key = iroh_net::key::SecretKey::generate(); + let ser_key = secret_key.to_openssh()?; + + // Try to canoncialize if possible + let key_path = key_path.canonicalize().unwrap_or(key_path); + let key_path_parent = key_path.parent().ok_or_else(|| { + anyhow::anyhow!("no parent directory found for '{}'", key_path.display()) + })?; + tokio::fs::create_dir_all(&key_path_parent).await?; + + // write to tempfile + let (file, temp_file_path) = tempfile::NamedTempFile::new_in(key_path_parent) + .context("unable to create tempfile")? + .into_parts(); + let mut file = tokio::fs::File::from_std(file); + file.write_all(ser_key.as_bytes()) + .await + .context("unable to write keyfile")?; + file.flush().await?; + drop(file); + + // move file + tokio::fs::rename(temp_file_path, key_path) + .await + .context("failed to rename keyfile")?; + + Ok(secret_key) + } +} + +async fn create_endpoint( + key: iroh_net::key::SecretKey, + port: u16, + publish: bool, +) -> anyhow::Result { + let pkarr = PkarrClient::new(); + let discovery_key = if publish { Some(&key) } else { None }; + let mainline_discovery = PkarrNodeDiscovery::new(pkarr, discovery_key); + iroh_net::MagicEndpoint::builder() + .secret_key(key) + .discovery(Box::new(mainline_discovery)) + .alpns(vec![TRACKER_ALPN.to_vec()]) + .bind(port) + .await +} diff --git a/iroh-mainline-content-discovery/src/main.rs b/iroh-mainline-content-discovery/src/main.rs index c54757fb..06c496f3 100644 --- a/iroh-mainline-content-discovery/src/main.rs +++ b/iroh-mainline-content-discovery/src/main.rs @@ -1,14 +1,8 @@ pub mod args; -pub mod io; -pub mod iroh_bytes_util; -pub mod options; -pub mod protocol; -pub mod tracker; use std::{ collections::BTreeSet, net::{Ipv4Addr, SocketAddr, SocketAddrV4}, - str::FromStr, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -16,20 +10,14 @@ use std::{ time::{Duration, Instant}, }; +use anyhow::Context; use clap::Parser; -use io::CONFIG_DEFAULTS_FILE; -use iroh::net::{ - magic_endpoint::{get_alpn, get_remote_node_id}, - MagicEndpoint, -}; -use iroh::util::fs::load_secret_key; -use iroh_pkarr_node_discovery::PkarrNodeDiscovery; -use pkarr::PkarrClient; -use tokio_util::task::LocalPoolHandle; - -use crate::{ - args::{AnnounceArgs, Args, Commands, QueryArgs, ServerArgs}, - io::{load_from_file, setup_logging, tracker_home, tracker_path, CONFIG_FILE, SERVER_KEY_FILE}, +use iroh_mainline_content_discovery::TrackerId; +use iroh_mainline_content_discovery::{ + io::{ + self, load_from_file, setup_logging, tracker_home, tracker_path, CONFIG_DEFAULTS_FILE, + CONFIG_FILE, SERVER_KEY_FILE, + }, options::Options, protocol::{ Announce, AnnounceKind, Query, QueryFlags, Request, Response, REQUEST_SIZE_LIMIT, @@ -37,38 +25,16 @@ use crate::{ }, tracker::Tracker, }; +use iroh_net::{ + magic_endpoint::{get_alpn, get_remote_node_id}, + MagicEndpoint, NodeId, +}; +use iroh_pkarr_node_discovery::PkarrNodeDiscovery; +use pkarr::PkarrClient; +use tokio::io::AsyncWriteExt; +use tokio_util::task::LocalPoolHandle; -pub type NodeId = iroh::net::key::PublicKey; - -/// A tracker id for queries - either a node id or an address. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum TrackerId { - NodeId(NodeId), - Addr(SocketAddr), -} - -impl std::fmt::Display for TrackerId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - TrackerId::NodeId(node_id) => write!(f, "{}", node_id), - TrackerId::Addr(addr) => write!(f, "{}", addr), - } - } -} - -impl FromStr for TrackerId { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - if let Ok(node_id) = s.parse() { - return Ok(TrackerId::NodeId(node_id)); - } - if let Ok(addr) = s.parse() { - return Ok(TrackerId::Addr(addr)); - } - anyhow::bail!("invalid tracker id") - } -} +use crate::args::{AnnounceArgs, Args, Commands, QueryArgs, ServerArgs}; static VERBOSE: AtomicBool = AtomicBool::new(false); @@ -106,14 +72,14 @@ async fn await_derp_region(endpoint: &MagicEndpoint) -> anyhow::Result<()> { } async fn create_endpoint( - key: iroh::net::key::SecretKey, + key: iroh_net::key::SecretKey, port: u16, publish: bool, ) -> anyhow::Result { let pkarr = PkarrClient::new(); let discovery_key = if publish { Some(&key) } else { None }; let mainline_discovery = PkarrNodeDiscovery::new(pkarr, discovery_key); - iroh::net::MagicEndpoint::builder() + iroh_net::MagicEndpoint::builder() .secret_key(key) .discovery(Box::new(mainline_discovery)) .alpns(vec![TRACKER_ALPN.to_vec()]) @@ -134,7 +100,7 @@ pub async fn accept_conn( /// Write default options to a sample config file. fn write_defaults() -> anyhow::Result<()> { let default_path = tracker_path(CONFIG_DEFAULTS_FILE)?; - crate::io::save_to_file(Options::default(), &default_path)?; + io::save_to_file(Options::default(), &default_path)?; Ok(()) } @@ -177,25 +143,11 @@ async fn server(args: ServerArgs) -> anyhow::Result<()> { } async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { - set_verbose(true); // todo: uncomment once the connection problems are fixed // for now, a random node id is more reliable. // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; - let key = iroh::net::key::SecretKey::generate(); - let endpoint = create_endpoint(key, 11112, false).await?; - log!("announce {:?}", args); - log!("trying to connect to {:?}", args.tracker); - let connection = endpoint - .connect_by_node_id(&args.tracker, TRACKER_ALPN) - .await?; - log!("connected to {:?}", connection.remote_address()); - let (mut send, mut recv) = connection.open_bi().await?; - log!("opened bi stream"); - let kind = if args.partial { - AnnounceKind::Partial - } else { - AnnounceKind::Complete - }; + let key = iroh_net::key::SecretKey::generate(); + let content = args.content.iter().map(|x| x.hash_and_format()).collect(); let host = if let Some(host) = args.host { host } else { @@ -211,75 +163,75 @@ async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { } *hosts.iter().next().unwrap() }; - let content = args.content.iter().map(|x| x.hash_and_format()).collect(); + println!("announcing to {}", args.tracker); + println!("host {} has", host); + for content in &content { + println!(" {}", content); + } + let endpoint = create_endpoint(key, 11112, false).await?; + let connection = endpoint + .connect_by_node_id(&args.tracker, TRACKER_ALPN) + .await?; + println!("connected to {:?}", connection.remote_address()); + let kind = if args.partial { + AnnounceKind::Partial + } else { + AnnounceKind::Complete + }; let announce = Announce { host, kind, content, }; - let request = Request::Announce(announce); - let request = postcard::to_stdvec(&request)?; - log!("sending announce"); - send.write_all(&request).await?; - send.finish().await?; - let _response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; + iroh_mainline_content_discovery::announce(connection, announce).await?; + println!("done"); Ok(()) } async fn query(args: QueryArgs) -> anyhow::Result<()> { - set_verbose(true); - match args.tracker { - TrackerId::Addr(tracker) => query_socket(args, tracker).await, - TrackerId::NodeId(tracker) => query_magic(args, tracker).await, + let connection = connect(&args.tracker).await?; + let q = Query { + content: args.content.hash_and_format(), + flags: QueryFlags { + complete: !args.partial, + verified: args.verified, + }, + }; + let res = iroh_mainline_content_discovery::query(connection, q).await?; + println!( + "querying tracker {} for content {}", + args.tracker, args.content + ); + for peer in res.hosts { + println!("{}", peer); } + Ok(()) } -async fn query_magic(args: QueryArgs, tracker: NodeId) -> anyhow::Result<()> { +async fn connect(tracker: &TrackerId) -> anyhow::Result { + match tracker { + TrackerId::Addr(tracker) => connect_socket(*tracker).await, + TrackerId::NodeId(tracker) => connect_magic(tracker.clone()).await, + } +} + +async fn connect_magic(tracker: NodeId) -> anyhow::Result { // todo: uncomment once the connection problems are fixed // for now, a random node id is more reliable. // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; - let key = iroh::net::key::SecretKey::generate(); - let endpoint = create_endpoint(key, args.port.unwrap_or_default(), false).await?; - log!("trying to connect to tracker at {:?}", tracker); + let key = iroh_net::key::SecretKey::generate(); + let endpoint = create_endpoint(key, 0, false).await?; + tracing::info!("trying to connect to tracker at {:?}", tracker); let connection = endpoint.connect_by_node_id(&tracker, TRACKER_ALPN).await?; - query_connection(args, connection).await + Ok(connection) } -async fn query_socket(args: QueryArgs, tracker: SocketAddr) -> anyhow::Result<()> { +async fn connect_socket(tracker: SocketAddr) -> anyhow::Result { let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)); let endpoint = create_quinn_client(bind_addr, vec![TRACKER_ALPN.to_vec()], false)?; - log!("trying to connect to tracker at {:?}", tracker); + tracing::info!("trying to connect to tracker at {:?}", tracker); let connection = endpoint.connect(tracker, "localhost")?.await?; - query_connection(args, connection).await -} - -async fn query_connection(args: QueryArgs, connection: quinn::Connection) -> anyhow::Result<()> { - let query = Query { - content: args.content.hash_and_format(), - flags: QueryFlags { - complete: !args.partial, - verified: args.verified, - }, - }; - log!("connected to {:?}", connection.remote_address()); - let (mut send, mut recv) = connection.open_bi().await?; - log!("opened bi stream"); - let request = Request::Query(query); - let request = postcard::to_stdvec(&request)?; - log!("sending query"); - send.write_all(&request).await?; - send.finish().await?; - let response = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; - let response = postcard::from_bytes::(&response)?; - match response { - Response::QueryResponse(response) => { - log!("content {}", response.content); - for peer in response.hosts { - log!("- peer {}", peer); - } - } - } - Ok(()) + Ok(connection) } #[tokio::main(flavor = "multi_thread")] @@ -295,7 +247,7 @@ async fn main() -> anyhow::Result<()> { /// Returns default server configuration along with its certificate. #[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 -fn configure_server(secret_key: &iroh::net::key::SecretKey) -> anyhow::Result { +fn configure_server(secret_key: &iroh_net::key::SecretKey) -> anyhow::Result { make_server_config(secret_key, 8, 1024, vec![TRACKER_ALPN.to_vec()]) } @@ -304,9 +256,9 @@ fn create_quinn_client( alpn_protocols: Vec>, keylog: bool, ) -> anyhow::Result { - let secret_key = iroh::net::key::SecretKey::generate(); + let secret_key = iroh_net::key::SecretKey::generate(); let tls_client_config = - iroh::net::tls::make_client_config(&secret_key, None, alpn_protocols, keylog)?; + iroh_net::tls::make_client_config(&secret_key, None, alpn_protocols, keylog)?; let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); let mut endpoint = quinn::Endpoint::client(bind_addr)?; let mut transport_config = quinn::TransportConfig::default(); @@ -318,12 +270,12 @@ fn create_quinn_client( /// Create a [`quinn::ServerConfig`] with the given secret key and limits. pub fn make_server_config( - secret_key: &iroh::net::key::SecretKey, + secret_key: &iroh_net::key::SecretKey, max_streams: u64, max_connections: u32, alpn_protocols: Vec>, ) -> anyhow::Result { - let tls_server_config = iroh::net::tls::make_server_config(secret_key, alpn_protocols, false)?; + let tls_server_config = iroh_net::tls::make_server_config(secret_key, alpn_protocols, false)?; let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(tls_server_config)); let mut transport_config = quinn::TransportConfig::default(); transport_config @@ -335,3 +287,43 @@ pub fn make_server_config( .concurrent_connections(max_connections); Ok(server_config) } + +/// Loads a [`SecretKey`] from the provided file. +pub async fn load_secret_key( + key_path: std::path::PathBuf, +) -> anyhow::Result { + if key_path.exists() { + let keystr = tokio::fs::read(key_path).await?; + let secret_key = + iroh_net::key::SecretKey::try_from_openssh(keystr).context("invalid keyfile")?; + Ok(secret_key) + } else { + let secret_key = iroh_net::key::SecretKey::generate(); + let ser_key = secret_key.to_openssh()?; + + // Try to canoncialize if possible + let key_path = key_path.canonicalize().unwrap_or(key_path); + let key_path_parent = key_path.parent().ok_or_else(|| { + anyhow::anyhow!("no parent directory found for '{}'", key_path.display()) + })?; + tokio::fs::create_dir_all(&key_path_parent).await?; + + // write to tempfile + let (file, temp_file_path) = tempfile::NamedTempFile::new_in(key_path_parent) + .context("unable to create tempfile")? + .into_parts(); + let mut file = tokio::fs::File::from_std(file); + file.write_all(ser_key.as_bytes()) + .await + .context("unable to write keyfile")?; + file.flush().await?; + drop(file); + + // move file + tokio::fs::rename(temp_file_path, key_path) + .await + .context("failed to rename keyfile")?; + + Ok(secret_key) + } +} diff --git a/iroh-mainline-content-discovery/src/protocol.rs b/iroh-mainline-content-discovery/src/protocol.rs index 0968d481..8669e6d9 100644 --- a/iroh-mainline-content-discovery/src/protocol.rs +++ b/iroh-mainline-content-discovery/src/protocol.rs @@ -1,10 +1,8 @@ //! The protocol for communicating with the tracker. -use std::collections::BTreeSet; - -use iroh::bytes::HashAndFormat; +use iroh_bytes::HashAndFormat; +use iroh_net::NodeId; use serde::{Deserialize, Serialize}; - -use crate::NodeId; +use std::collections::BTreeSet; /// The ALPN string for this protocol pub const TRACKER_ALPN: &[u8] = b"n0/tracker/1"; @@ -37,6 +35,8 @@ impl AnnounceKind { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Announce { /// The peer that supposedly has the data. + /// + /// Should we get this from the connection? pub host: NodeId, /// The blobs or sets that the peer claims to have. pub content: BTreeSet, diff --git a/iroh-mainline-content-discovery/src/tracker.rs b/iroh-mainline-content-discovery/src/tracker.rs index f5e216c1..eb5441e3 100644 --- a/iroh-mainline-content-discovery/src/tracker.rs +++ b/iroh-mainline-content-discovery/src/tracker.rs @@ -7,13 +7,13 @@ use std::{ use bao_tree::{ByteNum, ChunkNum}; use futures::StreamExt; -use iroh::bytes::{ +use iroh_bytes::{ get::{fsm::EndBlobNext, Stats}, hashseq::HashSeq, protocol::GetRequest, BlobFormat, Hash, HashAndFormat, }; -use iroh::net::MagicEndpoint; +use iroh_net::{MagicEndpoint, NodeId}; use rand::Rng; use crate::{ @@ -22,12 +22,10 @@ use crate::{ iroh_bytes_util::{ chunk_probe, get_hash_seq_and_sizes, random_hash_seq_ranges, unverified_size, verified_size, }, - log, options::Options, protocol::{ Announce, AnnounceKind, Query, QueryResponse, Request, Response, REQUEST_SIZE_LIMIT, }, - NodeId, }; /// The tracker server. @@ -201,20 +199,20 @@ impl Tracker { /// Handle a single incoming connection on the tracker ALPN. pub async fn handle_connection(&self, connection: quinn::Connection) -> anyhow::Result<()> { - log!("calling accept_bi"); + tracing::debug!("calling accept_bi"); let (mut send, mut recv) = connection.accept_bi().await?; - log!("got bi stream"); + tracing::debug!("got bi stream"); let request = recv.read_to_end(REQUEST_SIZE_LIMIT).await?; let request = postcard::from_bytes::(&request)?; match request { Request::Announce(announce) => { - log!("got announce: {:?}", announce); + tracing::debug!("got announce: {:?}", announce); self.handle_announce(announce)?; send.finish().await?; } Request::Query(query) => { - log!("handle query: {:?}", query); + tracing::debug!("handle query: {:?}", query); let response = self.handle_query(query)?; let response = Response::QueryResponse(response); let response = postcard::to_stdvec(&response)?; @@ -273,9 +271,9 @@ impl Tracker { let HashAndFormat { hash, format } = content; let mut rng = rand::thread_rng(); let stats = if probe_kind == ProbeKind::Incomplete { - log!("Size probing {}...", cap); + tracing::debug!("Size probing {}...", cap); let (size, stats) = unverified_size(connection, hash).await?; - log!( + tracing::debug!( "Size probed {}, got unverified size {}, {:.6}s", cap, size, @@ -287,9 +285,9 @@ impl Tracker { BlobFormat::Raw => { let size = self.get_or_insert_size(connection, hash).await?; let random_chunk = rng.gen_range(0..ByteNum(size).chunks().0); - log!("Chunk probing {}, chunk {}", cap, random_chunk); + tracing::debug!("Chunk probing {}, chunk {}", cap, random_chunk); let stats = chunk_probe(connection, hash, ChunkNum(random_chunk)).await?; - log!( + tracing::debug!( "Chunk probed {}, chunk {}, {:.6}s", cap, random_chunk, @@ -307,11 +305,11 @@ impl Tracker { }) .collect::>() .join(", "); - log!("Seq probing {} using {}", cap, text); + tracing::debug!("Seq probing {} using {}", cap, text); let request = GetRequest::new(*hash, ranges); - let request = iroh::bytes::get::fsm::start(connection.clone(), request); + let request = iroh_bytes::get::fsm::start(connection.clone(), request); let connected = request.next().await?; - let iroh::bytes::get::fsm::ConnectedNext::StartChild(child) = + let iroh_bytes::get::fsm::ConnectedNext::StartChild(child) = connected.next().await? else { unreachable!("request does not include root"); @@ -325,7 +323,7 @@ impl Tracker { unreachable!("request contains only one blob"); }; let stats = closing.next().await?; - log!( + tracing::debug!( "Seq probed {} using {}, {:.6}s", cap, text, @@ -450,7 +448,7 @@ impl Tracker { )> { let t0 = Instant::now(); let res = endpoint - .connect_by_node_id(&host, &iroh::bytes::protocol::ALPN) + .connect_by_node_id(&host, &iroh_bytes::protocol::ALPN) .await; log_connection_attempt(&self.0.options.dial_log, &host, t0, &res)?; let connection = match res { From a2f6097e5c2b1de43d9cb9b5d34001108af72a3d Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sat, 20 Jan 2024 20:46:20 +0200 Subject: [PATCH 09/13] Implementing DHT publishing --- iroh-mainline-content-discovery/Cargo.lock | 2 + iroh-mainline-content-discovery/Cargo.toml | 2 + iroh-mainline-content-discovery/src/lib.rs | 201 +++++++++++++++--- iroh-mainline-content-discovery/src/main.rs | 58 +---- .../src/options.rs | 6 + .../src/protocol.rs | 4 +- .../src/tracker.rs | 39 +++- 7 files changed, 227 insertions(+), 85 deletions(-) diff --git a/iroh-mainline-content-discovery/Cargo.lock b/iroh-mainline-content-discovery/Cargo.lock index 2ba85e16..21209026 100644 --- a/iroh-mainline-content-discovery/Cargo.lock +++ b/iroh-mainline-content-discovery/Cargo.lock @@ -1640,7 +1640,9 @@ dependencies = [ "derive_more", "dirs-next", "ed25519-dalek", + "flume", "futures", + "genawaiter", "hex", "humantime", "iroh-bytes", diff --git a/iroh-mainline-content-discovery/Cargo.toml b/iroh-mainline-content-discovery/Cargo.toml index 2aaa733a..9e7362d0 100644 --- a/iroh-mainline-content-discovery/Cargo.toml +++ b/iroh-mainline-content-discovery/Cargo.toml @@ -40,6 +40,8 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } ttl_cache = "0.5.1" url = "2.5.0" +flume = "0.11.0" +genawaiter = { version = "0.99.1", features = ["futures03"] } [features] cli = ["clap"] diff --git a/iroh-mainline-content-discovery/src/lib.rs b/iroh-mainline-content-discovery/src/lib.rs index 5c341542..da27eab8 100644 --- a/iroh-mainline-content-discovery/src/lib.rs +++ b/iroh-mainline-content-discovery/src/lib.rs @@ -1,11 +1,20 @@ -use std::{net::SocketAddr, sync::Arc, time::Duration}; +use genawaiter::sync::Gen; +use std::{ + collections::{BTreeSet, HashSet}, + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::Arc, + time::Duration, +}; use anyhow::Context; +use futures::{future::BoxFuture, FutureExt, Stream, StreamExt}; +use iroh_bytes::HashAndFormat; use iroh_net::{ magic_endpoint::{get_alpn, get_remote_node_id}, MagicEndpoint, NodeId, }; use iroh_pkarr_node_discovery::PkarrNodeDiscovery; +use mainline::common::{GetPeerResponse, StoreQueryMetdata}; use pkarr::PkarrClient; use protocol::QueryResponse; use tokio::io::AsyncWriteExt; @@ -18,36 +27,6 @@ pub mod options; pub mod protocol; pub mod tracker; -/// A tracker id for queries - either a node id or an address. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum TrackerId { - NodeId(NodeId), - Addr(std::net::SocketAddr), -} - -impl std::fmt::Display for TrackerId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - TrackerId::NodeId(node_id) => write!(f, "{}", node_id), - TrackerId::Addr(addr) => write!(f, "{}", addr), - } - } -} - -impl std::str::FromStr for TrackerId { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - if let Ok(node_id) = s.parse() { - return Ok(TrackerId::NodeId(node_id)); - } - if let Ok(addr) = s.parse() { - return Ok(TrackerId::Addr(addr)); - } - anyhow::bail!("invalid tracker id") - } -} - /// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. pub(crate) async fn accept_conn( mut conn: quinn::Connecting, @@ -78,6 +57,100 @@ pub async fn announce(connection: quinn::Connection, args: Announce) -> anyhow:: Ok(()) } +fn to_infohash(haf: HashAndFormat) -> mainline::Id { + let mut data = [0u8; 20]; + data.copy_from_slice(&haf.hash.as_bytes()[..20]); + mainline::Id::from_bytes(data).unwrap() +} + +fn unique_tracker_addrs( + mut response: mainline::common::Response, +) -> impl Stream { + Gen::new(|co| async move { + let mut found = HashSet::new(); + while let Some(response) = response.next_async().await { + let tracker = response.peer; + if !found.insert(tracker) { + continue; + } + co.yield_(tracker).await; + } + }) +} + +async fn query_one( + endpoint: impl ConnectionProvider, + addr: SocketAddr, + args: Query, +) -> anyhow::Result> { + let connection = endpoint.connect(addr).await?; + let result = query(connection, args).await?; + Ok(result.hosts) +} + +/// A connection provider that can be used to connect to a tracker. +/// +/// This can either be a [`quinn::Endpoint`] where connections are created on demand, +/// or some sort of connection pool. +pub trait ConnectionProvider: Clone { + fn connect(&self, addr: SocketAddr) -> BoxFuture>; +} + +impl ConnectionProvider for quinn::Endpoint { + fn connect(&self, addr: SocketAddr) -> BoxFuture> { + async move { Ok(self.connect(addr, "localhost")?.await?) }.boxed() + } +} + +/// Query the mainline DHT for trackers for the given content, then query each tracker for peers. +pub async fn query_dht( + endpoint: impl ConnectionProvider, + dht: mainline::dht::Dht, + args: Query, + query_parallelism: usize, +) -> impl Stream> { + let dht = dht.as_async(); + let info_hash = to_infohash(args.content); + let response: mainline::common::Response = dht.get_peers(info_hash); + let unique_tracker_addrs = unique_tracker_addrs(response); + unique_tracker_addrs + .map(move |addr| { + let endpoint = endpoint.clone(); + async move { + let hosts = match query_one(endpoint, addr, args).await { + Ok(hosts) => hosts.into_iter().map(anyhow::Ok).collect(), + Err(cause) => vec![Err(cause)], + }; + futures::stream::iter(hosts) + } + }) + .buffer_unordered(query_parallelism) + .flatten() +} + +/// Announce to the mainline DHT in parallel. +/// +/// Note that this should only be called from a publicly reachable node, where port is the port +/// on which the tracker protocol is reachable. +pub fn announce_dht( + dht: mainline::dht::Dht, + content: BTreeSet, + port: u16, + announce_parallelism: usize, +) -> impl Stream)> { + let dht = dht.as_async(); + futures::stream::iter(content) + .map(move |content| { + let dht = dht.clone(); + async move { + let info_hash = to_infohash(content); + let res = dht.announce_peer(info_hash, Some(port)).await; + (content, res) + } + }) + .buffer_unordered(announce_parallelism) +} + /// Assume an existing connection to a tracker and query it for peers for some content. pub async fn query(connection: quinn::Connection, args: Query) -> anyhow::Result { tracing::info!("connected to {:?}", connection.remote_address()); @@ -193,3 +266,69 @@ async fn create_endpoint( .bind(port) .await } + +/// A tracker id for queries - either a node id or an address. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum TrackerId { + NodeId(NodeId), + Addr(std::net::SocketAddr), +} + +impl std::fmt::Display for TrackerId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TrackerId::NodeId(node_id) => write!(f, "{}", node_id), + TrackerId::Addr(addr) => write!(f, "{}", addr), + } + } +} + +impl std::str::FromStr for TrackerId { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + if let Ok(node_id) = s.parse() { + return Ok(TrackerId::NodeId(node_id)); + } + if let Ok(addr) = s.parse() { + return Ok(TrackerId::Addr(addr)); + } + anyhow::bail!("invalid tracker id") + } +} + +/// Connect to a tracker using the [protocol::TRACKER_ALPN] protocol, using either +/// a node id or an address. +/// +/// Note that this is less efficient than using an existing endpoint when doing multiple requests. +/// It is provided as a convenience function for short lived utilities. +pub async fn connect(tracker: &TrackerId, local_port: u16) -> anyhow::Result { + match tracker { + TrackerId::Addr(tracker) => connect_socket(*tracker, local_port).await, + TrackerId::NodeId(tracker) => connect_magic(&tracker, local_port).await, + } +} + +/// Create a magic endpoint and connect to a tracker using the [protocol::TRACKER_ALPN] protocol. +pub async fn connect_magic(tracker: &NodeId, local_port: u16) -> anyhow::Result { + // todo: uncomment once the connection problems are fixed + // for now, a random node id is more reliable. + // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; + let key = iroh_net::key::SecretKey::generate(); + let endpoint = create_endpoint(key, local_port, false).await?; + tracing::info!("trying to connect to tracker at {:?}", tracker); + let connection = endpoint.connect_by_node_id(tracker, TRACKER_ALPN).await?; + Ok(connection) +} + +/// Create a quinn endpoint and connect to a tracker using the [protocol::TRACKER_ALPN] protocol. +pub async fn connect_socket( + tracker: SocketAddr, + local_port: u16, +) -> anyhow::Result { + let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, local_port)); + let endpoint = create_quinn_client(bind_addr, vec![TRACKER_ALPN.to_vec()], false)?; + tracing::info!("trying to connect to tracker at {:?}", tracker); + let connection = endpoint.connect(tracker, "localhost")?.await?; + Ok(connection) +} diff --git a/iroh-mainline-content-discovery/src/main.rs b/iroh-mainline-content-discovery/src/main.rs index 06c496f3..fd0761e5 100644 --- a/iroh-mainline-content-discovery/src/main.rs +++ b/iroh-mainline-content-discovery/src/main.rs @@ -12,17 +12,13 @@ use std::{ use anyhow::Context; use clap::Parser; -use iroh_mainline_content_discovery::TrackerId; use iroh_mainline_content_discovery::{ io::{ self, load_from_file, setup_logging, tracker_home, tracker_path, CONFIG_DEFAULTS_FILE, CONFIG_FILE, SERVER_KEY_FILE, }, options::Options, - protocol::{ - Announce, AnnounceKind, Query, QueryFlags, Request, Response, REQUEST_SIZE_LIMIT, - TRACKER_ALPN, - }, + protocol::{Announce, AnnounceKind, Query, QueryFlags, TRACKER_ALPN}, tracker::Tracker, }; use iroh_net::{ @@ -127,16 +123,17 @@ async fn server(args: ServerArgs) -> anyhow::Result<()> { "tracker query --tracker {} or ", addr.node_id ); - log!(); let db2 = db.clone(); let db3 = db.clone(); + let db4 = db.clone(); let endpoint2 = endpoint.clone(); let _probe_task = tpc.spawn_pinned(move || db2.probe_loop(endpoint2)); + let _announce_task = tpc.spawn_pinned(move || db3.dht_announce_loop(args.quinn_port)); let magic_accept_task = tokio::spawn(db.magic_accept_loop(endpoint)); let server_config = configure_server(&key)?; let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, args.quinn_port)); let quinn_endpoint = quinn::Endpoint::server(server_config, bind_addr)?; - let quinn_accept_task = tokio::spawn(db3.quinn_accept_loop(quinn_endpoint)); + let quinn_accept_task = tokio::spawn(db4.quinn_accept_loop(quinn_endpoint)); magic_accept_task.await??; quinn_accept_task.await??; Ok(()) @@ -189,7 +186,9 @@ async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { } async fn query(args: QueryArgs) -> anyhow::Result<()> { - let connection = connect(&args.tracker).await?; + let connection = + iroh_mainline_content_discovery::connect(&args.tracker, args.port.unwrap_or_default()) + .await?; let q = Query { content: args.content.hash_and_format(), flags: QueryFlags { @@ -208,32 +207,6 @@ async fn query(args: QueryArgs) -> anyhow::Result<()> { Ok(()) } -async fn connect(tracker: &TrackerId) -> anyhow::Result { - match tracker { - TrackerId::Addr(tracker) => connect_socket(*tracker).await, - TrackerId::NodeId(tracker) => connect_magic(tracker.clone()).await, - } -} - -async fn connect_magic(tracker: NodeId) -> anyhow::Result { - // todo: uncomment once the connection problems are fixed - // for now, a random node id is more reliable. - // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; - let key = iroh_net::key::SecretKey::generate(); - let endpoint = create_endpoint(key, 0, false).await?; - tracing::info!("trying to connect to tracker at {:?}", tracker); - let connection = endpoint.connect_by_node_id(&tracker, TRACKER_ALPN).await?; - Ok(connection) -} - -async fn connect_socket(tracker: SocketAddr) -> anyhow::Result { - let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)); - let endpoint = create_quinn_client(bind_addr, vec![TRACKER_ALPN.to_vec()], false)?; - tracing::info!("trying to connect to tracker at {:?}", tracker); - let connection = endpoint.connect(tracker, "localhost")?.await?; - Ok(connection) -} - #[tokio::main(flavor = "multi_thread")] async fn main() -> anyhow::Result<()> { setup_logging(); @@ -251,23 +224,6 @@ fn configure_server(secret_key: &iroh_net::key::SecretKey) -> anyhow::Result>, - keylog: bool, -) -> anyhow::Result { - let secret_key = iroh_net::key::SecretKey::generate(); - let tls_client_config = - iroh_net::tls::make_client_config(&secret_key, None, alpn_protocols, keylog)?; - let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); - let mut endpoint = quinn::Endpoint::client(bind_addr)?; - let mut transport_config = quinn::TransportConfig::default(); - transport_config.keep_alive_interval(Some(Duration::from_secs(1))); - client_config.transport_config(Arc::new(transport_config)); - endpoint.set_default_client_config(client_config); - Ok(endpoint) -} - /// Create a [`quinn::ServerConfig`] with the given secret key and limits. pub fn make_server_config( secret_key: &iroh_net::key::SecretKey, diff --git a/iroh-mainline-content-discovery/src/options.rs b/iroh-mainline-content-discovery/src/options.rs index e5a80dcb..9d28a9d3 100644 --- a/iroh-mainline-content-discovery/src/options.rs +++ b/iroh-mainline-content-discovery/src/options.rs @@ -29,6 +29,10 @@ pub struct Options { pub announce_data_path: Option, // number of peers to probe in parallel pub probe_parallelism: usize, + + pub dht_announce_parallelism: usize, + + pub dht_announce_interval: Duration, } impl Default for Options { @@ -43,6 +47,8 @@ impl Default for Options { probe_log: Some("probe.log".into()), announce_data_path: Some("announce.data.toml".into()), probe_parallelism: 4, + dht_announce_parallelism: 4, + dht_announce_interval: Duration::from_secs(10), } } } diff --git a/iroh-mainline-content-discovery/src/protocol.rs b/iroh-mainline-content-discovery/src/protocol.rs index 8669e6d9..a4d5d3d5 100644 --- a/iroh-mainline-content-discovery/src/protocol.rs +++ b/iroh-mainline-content-discovery/src/protocol.rs @@ -45,7 +45,7 @@ pub struct Announce { } /// -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct QueryFlags { /// Only return peers that supposedly have the complete data. /// @@ -63,7 +63,7 @@ pub struct QueryFlags { } /// Query a peer for a blob or set of blobs. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct Query { /// The content we want to find. /// diff --git a/iroh-mainline-content-discovery/src/tracker.rs b/iroh-mainline-content-discovery/src/tracker.rs index eb5441e3..86e01b0e 100644 --- a/iroh-mainline-content-discovery/src/tracker.rs +++ b/iroh-mainline-content-discovery/src/tracker.rs @@ -17,7 +17,7 @@ use iroh_net::{MagicEndpoint, NodeId}; use rand::Rng; use crate::{ - accept_conn, + accept_conn, announce_dht, io::{log_connection_attempt, log_probe_attempt, AnnounceData}, iroh_bytes_util::{ chunk_probe, get_hash_seq_and_sizes, random_hash_seq_ranges, unverified_size, verified_size, @@ -26,6 +26,7 @@ use crate::{ protocol::{ Announce, AnnounceKind, Query, QueryResponse, Request, Response, REQUEST_SIZE_LIMIT, }, + to_infohash, }; /// The tracker server. @@ -157,6 +158,42 @@ impl Tracker { } } + pub async fn dht_announce_loop(self, port: u16) -> anyhow::Result<()> { + let dht = mainline::Dht::default(); + loop { + let state = self.0.state.read().unwrap(); + let content: BTreeSet = + state.announce_data.iter().map(|(haf, _)| *haf).collect(); + drop(state); + let mut announce = announce_dht( + dht.clone(), + content, + port, + self.0.options.dht_announce_parallelism, + ); + while let Some((content, res)) = announce.next().await { + match res { + Ok(sqm) => { + let stored_at = sqm.stored_at(); + tracing::info!( + "announced {} as {} on {} nodes", + content, + sqm.target(), + stored_at.len() + ); + for item in stored_at { + tracing::debug!("stored at {} {}", item.id, item.address); + } + } + Err(cause) => { + tracing::warn!("error announcing: {}", cause); + } + } + } + tokio::time::sleep(self.0.options.dht_announce_interval).await; + } + } + pub async fn magic_accept_loop(self, endpoint: MagicEndpoint) -> std::io::Result<()> { while let Some(connecting) = endpoint.accept().await { tracing::info!("got connecting"); From 3dfc9c6e88bc8d046665318f6dfee8ec398411f9 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sun, 21 Jan 2024 13:18:32 +0200 Subject: [PATCH 10/13] Adapt ci and make some more things optional --- .github/workflows/ci.yml | 2 +- iroh-mainline-content-discovery/Cargo.toml | 5 +- iroh-mainline-content-discovery/src/lib.rs | 65 ++++++++++++++++++---- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b957da7..7f072457 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ env: RUSTFLAGS: -Dwarnings RUSTDOCFLAGS: -Dwarnings MSRV: "1.72" - RS_EXAMPLES_LIST: "content-tracker,iroh-ipfs,dumbpipe-web,iroh-pkarr-node-discovery" + RS_EXAMPLES_LIST: "iroh-mainline-content-discovery,iroh-ipfs,dumbpipe-web,iroh-pkarr-node-discovery" GO_EXAMPLES_LIST: "dall_e_worker" jobs: diff --git a/iroh-mainline-content-discovery/Cargo.toml b/iroh-mainline-content-discovery/Cargo.toml index 9e7362d0..143e0584 100644 --- a/iroh-mainline-content-discovery/Cargo.toml +++ b/iroh-mainline-content-discovery/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT OR Apache-2.0" [dependencies] anyhow = { version = "1", features = ["backtrace"] } -bao-tree = { version = "0.9.1", features = ["tokio_fsm"], default-features = false } +bao-tree = { version = "0.9.1", features = ["tokio_fsm"], default-features = false, optional = true } bytes = "1" clap = { version = "4", features = ["derive"], optional = true } derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } @@ -44,7 +44,8 @@ flume = "0.11.0" genawaiter = { version = "0.99.1", features = ["futures03"] } [features] -cli = ["clap"] +cli = ["clap", "tracker"] +tracker = ["bao-tree"] default = ["cli"] [[bin]] diff --git a/iroh-mainline-content-discovery/src/lib.rs b/iroh-mainline-content-discovery/src/lib.rs index da27eab8..40bfada1 100644 --- a/iroh-mainline-content-discovery/src/lib.rs +++ b/iroh-mainline-content-discovery/src/lib.rs @@ -7,7 +7,10 @@ use std::{ }; use anyhow::Context; -use futures::{future::BoxFuture, FutureExt, Stream, StreamExt}; +use futures::{ + future::{self, BoxFuture}, + FutureExt, Stream, StreamExt, +}; use iroh_bytes::HashAndFormat; use iroh_net::{ magic_endpoint::{get_alpn, get_remote_node_id}, @@ -21,10 +24,14 @@ use tokio::io::AsyncWriteExt; use crate::protocol::{Announce, Query, Request, Response, REQUEST_SIZE_LIMIT, TRACKER_ALPN}; +#[cfg(feature = "tracker")] pub mod io; +#[cfg(feature = "tracker")] pub mod iroh_bytes_util; +#[cfg(feature = "tracker")] pub mod options; pub mod protocol; +#[cfg(feature = "tracker")] pub mod tracker; /// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. @@ -57,7 +64,8 @@ pub async fn announce(connection: quinn::Connection, args: Announce) -> anyhow:: Ok(()) } -fn to_infohash(haf: HashAndFormat) -> mainline::Id { +/// The mapping from a [`HashAndFormat`] to a [`mainline::Id`]. +pub fn to_infohash(haf: HashAndFormat) -> mainline::Id { let mut data = [0u8; 20]; data.copy_from_slice(&haf.hash.as_bytes()[..20]); mainline::Id::from_bytes(data).unwrap() @@ -69,6 +77,7 @@ fn unique_tracker_addrs( Gen::new(|co| async move { let mut found = HashSet::new(); while let Some(response) = response.next_async().await { + println!("got get_peers response: {:?}", response); let tracker = response.peer; if !found.insert(tracker) { continue; @@ -78,7 +87,7 @@ fn unique_tracker_addrs( }) } -async fn query_one( +async fn query_socket_one( endpoint: impl ConnectionProvider, addr: SocketAddr, args: Query, @@ -88,6 +97,18 @@ async fn query_one( Ok(result.hosts) } +async fn query_magic_one( + endpoint: MagicEndpoint, + node_id: &NodeId, + args: Query, +) -> anyhow::Result> { + let connection = endpoint + .connect_by_node_id(node_id, protocol::TRACKER_ALPN) + .await?; + let result = query(connection, args).await?; + Ok(result.hosts) +} + /// A connection provider that can be used to connect to a tracker. /// /// This can either be a [`quinn::Endpoint`] where connections are created on demand, @@ -102,8 +123,29 @@ impl ConnectionProvider for quinn::Endpoint { } } +pub fn query_trackers( + endpoint: MagicEndpoint, + trackers: impl IntoIterator, + args: Query, + query_parallelism: usize, +) -> impl Stream> { + futures::stream::iter(trackers) + .map(move |tracker| { + let endpoint = endpoint.clone(); + async move { + let hosts = match query_magic_one(endpoint, &tracker, args).await { + Ok(hosts) => hosts.into_iter().map(anyhow::Ok).collect(), + Err(cause) => vec![Err(cause)], + }; + futures::stream::iter(hosts) + } + }) + .buffer_unordered(query_parallelism) + .flatten() +} + /// Query the mainline DHT for trackers for the given content, then query each tracker for peers. -pub async fn query_dht( +pub fn query_dht( endpoint: impl ConnectionProvider, dht: mainline::dht::Dht, args: Query, @@ -117,7 +159,7 @@ pub async fn query_dht( .map(move |addr| { let endpoint = endpoint.clone(); async move { - let hosts = match query_one(endpoint, addr, args).await { + let hosts = match query_socket_one(endpoint, addr, args).await { Ok(hosts) => hosts.into_iter().map(anyhow::Ok).collect(), Err(cause) => vec![Err(cause)], }; @@ -132,6 +174,7 @@ pub async fn query_dht( /// /// Note that this should only be called from a publicly reachable node, where port is the port /// on which the tracker protocol is reachable. +#[cfg(feature = "tracker")] pub fn announce_dht( dht: mainline::dht::Dht, content: BTreeSet, @@ -174,7 +217,7 @@ fn configure_server(secret_key: &iroh_net::key::SecretKey) -> anyhow::Result>, keylog: bool, @@ -192,7 +235,7 @@ fn create_quinn_client( } /// Create a [`quinn::ServerConfig`] with the given secret key and limits. -pub fn make_server_config( +fn make_server_config( secret_key: &iroh_net::key::SecretKey, max_streams: u64, max_connections: u32, @@ -212,6 +255,7 @@ pub fn make_server_config( } /// Loads a [`SecretKey`] from the provided file. +#[cfg(feature = "tracker")] pub async fn load_secret_key( key_path: std::path::PathBuf, ) -> anyhow::Result { @@ -310,7 +354,7 @@ pub async fn connect(tracker: &TrackerId, local_port: u16) -> anyhow::Result anyhow::Result { +async fn connect_magic(tracker: &NodeId, local_port: u16) -> anyhow::Result { // todo: uncomment once the connection problems are fixed // for now, a random node id is more reliable. // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; @@ -322,10 +366,7 @@ pub async fn connect_magic(tracker: &NodeId, local_port: u16) -> anyhow::Result< } /// Create a quinn endpoint and connect to a tracker using the [protocol::TRACKER_ALPN] protocol. -pub async fn connect_socket( - tracker: SocketAddr, - local_port: u16, -) -> anyhow::Result { +async fn connect_socket(tracker: SocketAddr, local_port: u16) -> anyhow::Result { let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, local_port)); let endpoint = create_quinn_client(bind_addr, vec![TRACKER_ALPN.to_vec()], false)?; tracing::info!("trying to connect to tracker at {:?}", tracker); From f992752f05199594db6dfeb810f2dbd577678213 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sun, 21 Jan 2024 13:20:11 +0200 Subject: [PATCH 11/13] Add license files for iroh-pkarr-node-discovery --- iroh-pkarr-node-discovery/LICENSE-APACHE | 5 +++++ iroh-pkarr-node-discovery/LICENSE-MIT | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 iroh-pkarr-node-discovery/LICENSE-APACHE create mode 100644 iroh-pkarr-node-discovery/LICENSE-MIT diff --git a/iroh-pkarr-node-discovery/LICENSE-APACHE b/iroh-pkarr-node-discovery/LICENSE-APACHE new file mode 100644 index 00000000..4c83a284 --- /dev/null +++ b/iroh-pkarr-node-discovery/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. \ No newline at end of file diff --git a/iroh-pkarr-node-discovery/LICENSE-MIT b/iroh-pkarr-node-discovery/LICENSE-MIT new file mode 100644 index 00000000..749aa1ec --- /dev/null +++ b/iroh-pkarr-node-discovery/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file From 527c73c08362d55d04f47ff5bea58881b9546c03 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sun, 21 Jan 2024 15:15:42 +0200 Subject: [PATCH 12/13] Move stuff around and minimize dependencies --- .gitignore | 6 +- Cargo.toml | 8 + iroh-mainline-content-discovery/Cargo.lock | 27 +++ iroh-mainline-content-discovery/Cargo.toml | 70 +++----- iroh-mainline-content-discovery/README.md | 9 +- .../Cargo.toml | 44 +++++ .../src/args.rs | 15 -- .../src/client.rs} | 139 +++------------ .../src/lib.rs | 10 ++ .../src/main.rs | 168 ++++++++++++++++++ .../src/protocol.rs | 2 +- .../iroh-mainline-tracker/Cargo.toml | 53 ++++++ .../iroh-mainline-tracker/src/args.rs | 30 ++++ .../{ => iroh-mainline-tracker}/src/io.rs | 3 +- .../src/iroh_bytes_util.rs | 0 .../iroh-mainline-tracker/src/lib.rs | 4 + .../{ => iroh-mainline-tracker}/src/main.rs | 81 +-------- .../src/options.rs | 0 .../src/tracker.rs | 26 ++- 19 files changed, 423 insertions(+), 272 deletions(-) create mode 100644 Cargo.toml create mode 100644 iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml rename iroh-mainline-content-discovery/{ => iroh-mainline-content-discovery}/src/args.rs (91%) rename iroh-mainline-content-discovery/{src/lib.rs => iroh-mainline-content-discovery/src/client.rs} (68%) create mode 100644 iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/lib.rs create mode 100644 iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/main.rs rename iroh-mainline-content-discovery/{ => iroh-mainline-content-discovery}/src/protocol.rs (98%) create mode 100644 iroh-mainline-content-discovery/iroh-mainline-tracker/Cargo.toml create mode 100644 iroh-mainline-content-discovery/iroh-mainline-tracker/src/args.rs rename iroh-mainline-content-discovery/{ => iroh-mainline-tracker}/src/io.rs (98%) rename iroh-mainline-content-discovery/{ => iroh-mainline-tracker}/src/iroh_bytes_util.rs (100%) create mode 100644 iroh-mainline-content-discovery/iroh-mainline-tracker/src/lib.rs rename iroh-mainline-content-discovery/{ => iroh-mainline-tracker}/src/main.rs (72%) rename iroh-mainline-content-discovery/{ => iroh-mainline-tracker}/src/options.rs (100%) rename iroh-mainline-content-discovery/{ => iroh-mainline-tracker}/src/tracker.rs (97%) diff --git a/.gitignore b/.gitignore index 74f8bc23..44f1444b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ target/ *iroh-data -*/.sendme* \ No newline at end of file +*/.sendme* + +# Added by cargo + +/target diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..44a7175e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "iroh-examples" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/iroh-mainline-content-discovery/Cargo.lock b/iroh-mainline-content-discovery/Cargo.lock index 21209026..75f6ec7b 100644 --- a/iroh-mainline-content-discovery/Cargo.lock +++ b/iroh-mainline-content-discovery/Cargo.lock @@ -1632,6 +1632,32 @@ dependencies = [ [[package]] name = "iroh-mainline-content-discovery" version = "0.3.0" +dependencies = [ + "anyhow", + "clap", + "derive_more", + "ed25519-dalek", + "futures", + "genawaiter", + "iroh-bytes", + "iroh-net", + "iroh-pkarr-node-discovery", + "mainline", + "pkarr", + "postcard", + "quinn", + "rcgen 0.12.0", + "rustls", + "serde", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "iroh-mainline-tracker" +version = "0.3.0" dependencies = [ "anyhow", "bao-tree", @@ -1646,6 +1672,7 @@ dependencies = [ "hex", "humantime", "iroh-bytes", + "iroh-mainline-content-discovery", "iroh-net", "iroh-pkarr-node-discovery", "mainline", diff --git a/iroh-mainline-content-discovery/Cargo.toml b/iroh-mainline-content-discovery/Cargo.toml index 143e0584..6dfd668b 100644 --- a/iroh-mainline-content-discovery/Cargo.toml +++ b/iroh-mainline-content-discovery/Cargo.toml @@ -1,53 +1,25 @@ -[package] -name = "iroh-mainline-content-discovery" -version = "0.3.0" -edition = "2021" -description = "Content discovery for iroh, using the bittorrent mainline DHT" -license = "MIT OR Apache-2.0" +[workspace] +members = [ + "iroh-mainline-content-discovery", + "iroh-mainline-tracker", +] +resolver = "2" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[profile.release] +debug = true -[dependencies] -anyhow = { version = "1", features = ["backtrace"] } -bao-tree = { version = "0.9.1", features = ["tokio_fsm"], default-features = false, optional = true } -bytes = "1" -clap = { version = "4", features = ["derive"], optional = true } -derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } -dirs-next = "2" -ed25519-dalek = "2.1.0" -futures = "0.3.25" -hex = "0.4.3" -humantime = "2.1.0" -iroh-net = "0.12.0" -iroh-bytes = "0.12.0" -iroh-pkarr-node-discovery = { path = "../iroh-pkarr-node-discovery" } -mainline = "1.0.0" -pkarr = { version = "1.0.1", features = ["async"] } -postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } -quinn = "0.10" -rand = "0.8" -rcgen = "0.12.0" -redb = "1.5.0" -rustls = "0.21" -serde = { version = "1", features = ["derive"] } -serde_json = "1.0.107" -simple-mdns = { version = "0.5.0", features = ["async-tokio"] } -tempfile = "3.4" -tokio = { version = "1", features = ["io-util", "rt"] } -tokio-util = { version = "0.7", features = ["io-util", "io", "rt"] } -toml = "0.7.3" -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -ttl_cache = "0.5.1" -url = "2.5.0" -flume = "0.11.0" -genawaiter = { version = "0.99.1", features = ["futures03"] } +[profile.optimized-release] +inherits = 'release' +debug = false +lto = true +debug-assertions = false +opt-level = 3 +panic = 'abort' +incremental = false -[features] -cli = ["clap", "tracker"] -tracker = ["bao-tree"] -default = ["cli"] -[[bin]] -name = "iroh-mainline-content-discovery" -required-features = ["cli"] +[workspace.lints.rust] +missing_debug_implementations = "warn" + +[workspace.lints.clippy] +unused-async = "warn" diff --git a/iroh-mainline-content-discovery/README.md b/iroh-mainline-content-discovery/README.md index 5778b957..c3be4796 100644 --- a/iroh-mainline-content-discovery/README.md +++ b/iroh-mainline-content-discovery/README.md @@ -1,11 +1,6 @@ -# Iroh content tracker +# Iroh content discovery -This is an example how to write a simple service using mostly iroh-net. -The purpose of this service is to track providers of iroh content. - -You can *announce* to a tracker that you believe some host has some content. -The tracker will then periodically *verify* that the content is present. -Finally, you can *query* a tracker for hosts for a content. +This library provides global content discovery for iroh. ## Building from source diff --git a/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml new file mode 100644 index 00000000..6f9199ae --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "iroh-mainline-content-discovery" +version = "0.3.0" +edition = "2021" +description = "Content discovery for iroh, using the bittorrent mainline DHT" +license = "MIT OR Apache-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +# Required features for the protocol types +iroh-net = "0.12.0" +iroh-bytes = "0.12.0" +serde = { version = "1", features = ["derive"] } + +# Optional features for the client +tracing = { version = "0.1", optional = true } +quinn = { version = "0.10", optional = true } +iroh-pkarr-node-discovery = { path = "../../iroh-pkarr-node-discovery", optional = true } +mainline = { version = "1.0.0", optional = true } +pkarr = { version = "1.0.1", features = ["async"], optional = true } +anyhow = { version = "1", features = ["backtrace"], optional = true } +postcard = { version = "1", default-features = false, features = ["alloc", "use-std"], optional = true } +ed25519-dalek = { version = "2.1.0", optional = true } +futures = { version = "0.3.25", optional = true } +rcgen = { version = "0.12.0", optional = true } +rustls = { version = "0.21", optional = true } +genawaiter = { version = "0.99.1", features = ["futures03"], optional = true } + +# Optional features for the cli +clap = { version = "4", features = ["derive"], optional = true } +tempfile = { version = "3.4", optional = true } +derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"], optional = true } +tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true } +tokio = { version = "1", features = ["io-util", "rt"], optional = true } + +[features] +client = ["iroh-pkarr-node-discovery", "mainline", "pkarr", "quinn", "tracing", "anyhow", "rcgen", "genawaiter", "rustls", "futures", "postcard"] +cli = ["client", "clap", "tempfile", "derive_more", "tracing-subscriber", "tokio"] +default = [] + +[[bin]] +name = "iroh-mainline-content-discovery" +required-features = ["cli"] diff --git a/iroh-mainline-content-discovery/src/args.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/args.rs similarity index 91% rename from iroh-mainline-content-discovery/src/args.rs rename to iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/args.rs index d26fac7d..665368f3 100644 --- a/iroh-mainline-content-discovery/src/args.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/args.rs @@ -13,25 +13,10 @@ pub struct Args { #[derive(Subcommand, Debug)] pub enum Commands { - Server(ServerArgs), Announce(AnnounceArgs), Query(QueryArgs), } -#[derive(Parser, Debug)] -pub struct ServerArgs { - /// The port to listen on. - #[clap(long, default_value_t = 0xacacu16)] - pub magic_port: u16, - - /// The quinn port to listen on. - #[clap(long, default_value_t = 0xbcbcu16)] - pub quinn_port: u16, - - #[clap(long)] - pub quiet: bool, -} - /// Various ways to specify content. #[derive(Debug, Clone, derive_more::From)] pub enum ContentArg { diff --git a/iroh-mainline-content-discovery/src/lib.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/client.rs similarity index 68% rename from iroh-mainline-content-discovery/src/lib.rs rename to iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/client.rs index 40bfada1..ee3717fe 100644 --- a/iroh-mainline-content-discovery/src/lib.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/client.rs @@ -6,43 +6,16 @@ use std::{ time::Duration, }; -use anyhow::Context; -use futures::{ - future::{self, BoxFuture}, - FutureExt, Stream, StreamExt, -}; +use futures::{future::BoxFuture, FutureExt, Stream, StreamExt}; use iroh_bytes::HashAndFormat; -use iroh_net::{ - magic_endpoint::{get_alpn, get_remote_node_id}, - MagicEndpoint, NodeId, -}; +use iroh_net::{MagicEndpoint, NodeId}; use iroh_pkarr_node_discovery::PkarrNodeDiscovery; use mainline::common::{GetPeerResponse, StoreQueryMetdata}; use pkarr::PkarrClient; -use protocol::QueryResponse; -use tokio::io::AsyncWriteExt; - -use crate::protocol::{Announce, Query, Request, Response, REQUEST_SIZE_LIMIT, TRACKER_ALPN}; -#[cfg(feature = "tracker")] -pub mod io; -#[cfg(feature = "tracker")] -pub mod iroh_bytes_util; -#[cfg(feature = "tracker")] -pub mod options; -pub mod protocol; -#[cfg(feature = "tracker")] -pub mod tracker; - -/// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. -pub(crate) async fn accept_conn( - mut conn: quinn::Connecting, -) -> anyhow::Result<(NodeId, String, quinn::Connection)> { - let alpn = get_alpn(&mut conn).await?; - let conn = conn.await?; - let peer_id = get_remote_node_id(&conn)?; - Ok((peer_id, alpn, conn)) -} +use crate::protocol::{ + Announce, Query, QueryResponse, Request, Response, ALPN, REQUEST_SIZE_LIMIT, +}; /// Announce to a tracker. /// @@ -64,7 +37,9 @@ pub async fn announce(connection: quinn::Connection, args: Announce) -> anyhow:: Ok(()) } -/// The mapping from a [`HashAndFormat`] to a [`mainline::Id`]. +/// The mapping from an iroh [HashAndFormat] to a bittorrent infohash, aka [mainline::Id]. +/// +/// Since an infohash is just 20 bytes, this can not be a bidirectional mapping. pub fn to_infohash(haf: HashAndFormat) -> mainline::Id { let mut data = [0u8; 20]; data.copy_from_slice(&haf.hash.as_bytes()[..20]); @@ -88,7 +63,7 @@ fn unique_tracker_addrs( } async fn query_socket_one( - endpoint: impl ConnectionProvider, + endpoint: impl QuinnConnectionProvider, addr: SocketAddr, args: Query, ) -> anyhow::Result> { @@ -102,9 +77,7 @@ async fn query_magic_one( node_id: &NodeId, args: Query, ) -> anyhow::Result> { - let connection = endpoint - .connect_by_node_id(node_id, protocol::TRACKER_ALPN) - .await?; + let connection = endpoint.connect_by_node_id(node_id, ALPN).await?; let result = query(connection, args).await?; Ok(result.hosts) } @@ -113,16 +86,17 @@ async fn query_magic_one( /// /// This can either be a [`quinn::Endpoint`] where connections are created on demand, /// or some sort of connection pool. -pub trait ConnectionProvider: Clone { - fn connect(&self, addr: SocketAddr) -> BoxFuture>; +pub trait QuinnConnectionProvider: Clone { + fn connect(&self, addr: Addr) -> BoxFuture>; } -impl ConnectionProvider for quinn::Endpoint { +impl QuinnConnectionProvider for quinn::Endpoint { fn connect(&self, addr: SocketAddr) -> BoxFuture> { async move { Ok(self.connect(addr, "localhost")?.await?) }.boxed() } } +/// Query multiple trackers in parallel and merge the results. pub fn query_trackers( endpoint: MagicEndpoint, trackers: impl IntoIterator, @@ -146,7 +120,7 @@ pub fn query_trackers( /// Query the mainline DHT for trackers for the given content, then query each tracker for peers. pub fn query_dht( - endpoint: impl ConnectionProvider, + endpoint: impl QuinnConnectionProvider, dht: mainline::dht::Dht, args: Query, query_parallelism: usize, @@ -174,7 +148,6 @@ pub fn query_dht( /// /// Note that this should only be called from a publicly reachable node, where port is the port /// on which the tracker protocol is reachable. -#[cfg(feature = "tracker")] pub fn announce_dht( dht: mainline::dht::Dht, content: BTreeSet, @@ -211,13 +184,8 @@ pub async fn query(connection: quinn::Connection, args: Query) -> anyhow::Result }) } -/// Returns default server configuration along with its certificate. -#[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 -fn configure_server(secret_key: &iroh_net::key::SecretKey) -> anyhow::Result { - make_server_config(secret_key, 8, 1024, vec![TRACKER_ALPN.to_vec()]) -} - -pub fn create_quinn_client( +/// Create a quinn client endpoint. +fn create_quinn_client( bind_addr: SocketAddr, alpn_protocols: Vec>, keylog: bool, @@ -234,67 +202,6 @@ pub fn create_quinn_client( Ok(endpoint) } -/// Create a [`quinn::ServerConfig`] with the given secret key and limits. -fn make_server_config( - secret_key: &iroh_net::key::SecretKey, - max_streams: u64, - max_connections: u32, - alpn_protocols: Vec>, -) -> anyhow::Result { - let tls_server_config = iroh_net::tls::make_server_config(secret_key, alpn_protocols, false)?; - let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(tls_server_config)); - let mut transport_config = quinn::TransportConfig::default(); - transport_config - .max_concurrent_bidi_streams(max_streams.try_into()?) - .max_concurrent_uni_streams(0u32.into()); - - server_config - .transport_config(Arc::new(transport_config)) - .concurrent_connections(max_connections); - Ok(server_config) -} - -/// Loads a [`SecretKey`] from the provided file. -#[cfg(feature = "tracker")] -pub async fn load_secret_key( - key_path: std::path::PathBuf, -) -> anyhow::Result { - if key_path.exists() { - let keystr = tokio::fs::read(key_path).await?; - let secret_key = - iroh_net::key::SecretKey::try_from_openssh(keystr).context("invalid keyfile")?; - Ok(secret_key) - } else { - let secret_key = iroh_net::key::SecretKey::generate(); - let ser_key = secret_key.to_openssh()?; - - // Try to canoncialize if possible - let key_path = key_path.canonicalize().unwrap_or(key_path); - let key_path_parent = key_path.parent().ok_or_else(|| { - anyhow::anyhow!("no parent directory found for '{}'", key_path.display()) - })?; - tokio::fs::create_dir_all(&key_path_parent).await?; - - // write to tempfile - let (file, temp_file_path) = tempfile::NamedTempFile::new_in(key_path_parent) - .context("unable to create tempfile")? - .into_parts(); - let mut file = tokio::fs::File::from_std(file); - file.write_all(ser_key.as_bytes()) - .await - .context("unable to write keyfile")?; - file.flush().await?; - drop(file); - - // move file - tokio::fs::rename(temp_file_path, key_path) - .await - .context("failed to rename keyfile")?; - - Ok(secret_key) - } -} - async fn create_endpoint( key: iroh_net::key::SecretKey, port: u16, @@ -306,7 +213,7 @@ async fn create_endpoint( iroh_net::MagicEndpoint::builder() .secret_key(key) .discovery(Box::new(mainline_discovery)) - .alpns(vec![TRACKER_ALPN.to_vec()]) + .alpns(vec![ALPN.to_vec()]) .bind(port) .await } @@ -341,7 +248,7 @@ impl std::str::FromStr for TrackerId { } } -/// Connect to a tracker using the [protocol::TRACKER_ALPN] protocol, using either +/// Connect to a tracker using the [crate::protocol::ALPN] protocol, using either /// a node id or an address. /// /// Note that this is less efficient than using an existing endpoint when doing multiple requests. @@ -353,7 +260,7 @@ pub async fn connect(tracker: &TrackerId, local_port: u16) -> anyhow::Result anyhow::Result { // todo: uncomment once the connection problems are fixed // for now, a random node id is more reliable. @@ -361,14 +268,14 @@ async fn connect_magic(tracker: &NodeId, local_port: u16) -> anyhow::Result anyhow::Result { let bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, local_port)); - let endpoint = create_quinn_client(bind_addr, vec![TRACKER_ALPN.to_vec()], false)?; + let endpoint = create_quinn_client(bind_addr, vec![ALPN.to_vec()], false)?; tracing::info!("trying to connect to tracker at {:?}", tracker); let connection = endpoint.connect(tracker, "localhost")?.await?; Ok(connection) diff --git a/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/lib.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/lib.rs new file mode 100644 index 00000000..e0ccaab3 --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/lib.rs @@ -0,0 +1,10 @@ +//! A library for discovering content using the mainline DHT. +//! +//! This library contains the protocol for announcing and querying content, as +//! well as the client side implementation and a few helpers for p2p quinn +//! connections. +#[cfg(feature = "client")] +mod client; +pub mod protocol; +#[cfg(feature = "client")] +pub use client::*; diff --git a/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/main.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/main.rs new file mode 100644 index 00000000..3ec66c16 --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/main.rs @@ -0,0 +1,168 @@ +pub mod args; + +use std::collections::BTreeSet; + +use anyhow::Context; +use clap::Parser; +use iroh_mainline_content_discovery::protocol::{Announce, AnnounceKind, Query, QueryFlags, ALPN}; +use iroh_net::{ + magic_endpoint::{get_alpn, get_remote_node_id}, + MagicEndpoint, NodeId, +}; +use iroh_pkarr_node_discovery::PkarrNodeDiscovery; +use pkarr::PkarrClient; +use tokio::io::AsyncWriteExt; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; + +use crate::args::{AnnounceArgs, Args, Commands, QueryArgs}; + +async fn create_endpoint( + key: iroh_net::key::SecretKey, + port: u16, + publish: bool, +) -> anyhow::Result { + let pkarr = PkarrClient::new(); + let discovery_key = if publish { Some(&key) } else { None }; + let mainline_discovery = PkarrNodeDiscovery::new(pkarr, discovery_key); + iroh_net::MagicEndpoint::builder() + .secret_key(key) + .discovery(Box::new(mainline_discovery)) + .alpns(vec![ALPN.to_vec()]) + .bind(port) + .await +} + +/// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. +pub async fn accept_conn( + mut conn: quinn::Connecting, +) -> anyhow::Result<(NodeId, String, quinn::Connection)> { + let alpn = get_alpn(&mut conn).await?; + let conn = conn.await?; + let peer_id = get_remote_node_id(&conn)?; + Ok((peer_id, alpn, conn)) +} + +async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { + // todo: uncomment once the connection problems are fixed + // for now, a random node id is more reliable. + // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; + let key = iroh_net::key::SecretKey::generate(); + let content = args.content.iter().map(|x| x.hash_and_format()).collect(); + let host = if let Some(host) = args.host { + host + } else { + let hosts = args + .content + .iter() + .filter_map(|x| x.host()) + .collect::>(); + if hosts.len() != 1 { + anyhow::bail!( + "content for all tickets must be from the same host, unless a host is specified" + ); + } + *hosts.iter().next().unwrap() + }; + println!("announcing to {}", args.tracker); + println!("host {} has", host); + for content in &content { + println!(" {}", content); + } + let endpoint = create_endpoint(key, 11112, false).await?; + let connection = endpoint.connect_by_node_id(&args.tracker, ALPN).await?; + println!("connected to {:?}", connection.remote_address()); + let kind = if args.partial { + AnnounceKind::Partial + } else { + AnnounceKind::Complete + }; + let announce = Announce { + host, + kind, + content, + }; + iroh_mainline_content_discovery::announce(connection, announce).await?; + println!("done"); + Ok(()) +} + +async fn query(args: QueryArgs) -> anyhow::Result<()> { + let connection = + iroh_mainline_content_discovery::connect(&args.tracker, args.port.unwrap_or_default()) + .await?; + let q = Query { + content: args.content.hash_and_format(), + flags: QueryFlags { + complete: !args.partial, + verified: args.verified, + }, + }; + let res = iroh_mainline_content_discovery::query(connection, q).await?; + println!( + "querying tracker {} for content {}", + args.tracker, args.content + ); + for peer in res.hosts { + println!("{}", peer); + } + Ok(()) +} + +// set the RUST_LOG env var to one of {debug,info,warn} to see logging info +pub fn setup_logging() { + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) + .with(EnvFilter::from_default_env()) + .try_init() + .ok(); +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> anyhow::Result<()> { + setup_logging(); + let args = Args::parse(); + match args.command { + Commands::Announce(args) => announce(args).await, + Commands::Query(args) => query(args).await, + } +} + +/// Loads a [`SecretKey`] from the provided file. +pub async fn load_secret_key( + key_path: std::path::PathBuf, +) -> anyhow::Result { + if key_path.exists() { + let keystr = tokio::fs::read(key_path).await?; + let secret_key = + iroh_net::key::SecretKey::try_from_openssh(keystr).context("invalid keyfile")?; + Ok(secret_key) + } else { + let secret_key = iroh_net::key::SecretKey::generate(); + let ser_key = secret_key.to_openssh()?; + + // Try to canoncialize if possible + let key_path = key_path.canonicalize().unwrap_or(key_path); + let key_path_parent = key_path.parent().ok_or_else(|| { + anyhow::anyhow!("no parent directory found for '{}'", key_path.display()) + })?; + tokio::fs::create_dir_all(&key_path_parent).await?; + + // write to tempfile + let (file, temp_file_path) = tempfile::NamedTempFile::new_in(key_path_parent) + .context("unable to create tempfile")? + .into_parts(); + let mut file = tokio::fs::File::from_std(file); + file.write_all(ser_key.as_bytes()) + .await + .context("unable to write keyfile")?; + file.flush().await?; + drop(file); + + // move file + tokio::fs::rename(temp_file_path, key_path) + .await + .context("failed to rename keyfile")?; + + Ok(secret_key) + } +} diff --git a/iroh-mainline-content-discovery/src/protocol.rs b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/protocol.rs similarity index 98% rename from iroh-mainline-content-discovery/src/protocol.rs rename to iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/protocol.rs index a4d5d3d5..a54c44ba 100644 --- a/iroh-mainline-content-discovery/src/protocol.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/src/protocol.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; /// The ALPN string for this protocol -pub const TRACKER_ALPN: &[u8] = b"n0/tracker/1"; +pub const ALPN: &[u8] = b"n0/tracker/1"; /// Maximum size of a request pub const REQUEST_SIZE_LIMIT: usize = 1024 * 16; diff --git a/iroh-mainline-content-discovery/iroh-mainline-tracker/Cargo.toml b/iroh-mainline-content-discovery/iroh-mainline-tracker/Cargo.toml new file mode 100644 index 00000000..124e2607 --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "iroh-mainline-tracker" +version = "0.3.0" +edition = "2021" +description = "Content discovery for iroh, using the bittorrent mainline DHT" +license = "MIT OR Apache-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = { version = "1", features = ["backtrace"] } +bao-tree = { version = "0.9.1", features = ["tokio_fsm"], default-features = false } +bytes = "1" +clap = { version = "4", features = ["derive"], optional = true } +derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } +dirs-next = "2" +ed25519-dalek = "2.1.0" +futures = "0.3.25" +hex = "0.4.3" +humantime = "2.1.0" +iroh-net = "0.12.0" +iroh-bytes = "0.12.0" +iroh-pkarr-node-discovery = { path = "../../iroh-pkarr-node-discovery" } +mainline = "1.0.0" +pkarr = { version = "1.0.1", features = ["async"] } +postcard = { version = "1", default-features = false, features = ["alloc", "use-std", "experimental-derive"] } +quinn = "0.10" +rand = "0.8" +rcgen = "0.12.0" +redb = "1.5.0" +rustls = "0.21" +serde = { version = "1", features = ["derive"] } +serde_json = "1.0.107" +simple-mdns = { version = "0.5.0", features = ["async-tokio"] } +tempfile = "3.4" +tokio = { version = "1", features = ["io-util", "rt"] } +tokio-util = { version = "0.7", features = ["io-util", "io", "rt"] } +toml = "0.7.3" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +ttl_cache = "0.5.1" +url = "2.5.0" +flume = "0.11.0" +genawaiter = { version = "0.99.1", features = ["futures03"] } +iroh-mainline-content-discovery = { path = "../iroh-mainline-content-discovery", features = ["client"] } + +[features] +cli = ["clap"] +default = ["cli"] + +[[bin]] +name = "iroh-mainline-tracker" +required-features = ["cli"] diff --git a/iroh-mainline-content-discovery/iroh-mainline-tracker/src/args.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/args.rs new file mode 100644 index 00000000..72a18c8e --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/args.rs @@ -0,0 +1,30 @@ +//! Command line arguments. +use clap::{Parser, Subcommand}; + +#[derive(Parser, Debug)] +pub struct Args { + #[clap(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand, Debug)] +pub enum Commands { + Server(ServerArgs), +} + +#[derive(Parser, Debug)] +pub struct ServerArgs { + /// The port to listen on. + #[clap(long, default_value_t = 0xacacu16)] + pub magic_port: u16, + + /// The quinn port to listen on. + /// + /// The server must be reachable under this port from the internet, via + /// UDP for QUIC connections. + #[clap(long, default_value_t = 0xbcbcu16)] + pub quinn_port: u16, + + #[clap(long)] + pub quiet: bool, +} diff --git a/iroh-mainline-content-discovery/src/io.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/io.rs similarity index 98% rename from iroh-mainline-content-discovery/src/io.rs rename to iroh-mainline-content-discovery/iroh-mainline-tracker/src/io.rs index 024851a8..54188cb3 100644 --- a/iroh-mainline-content-discovery/src/io.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/io.rs @@ -9,11 +9,12 @@ use std::{ use anyhow::Context; use iroh_bytes::{get::Stats, HashAndFormat}; +use iroh_mainline_content_discovery::protocol::AnnounceKind; use iroh_net::NodeId; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tracing_subscriber::{prelude::*, EnvFilter}; -use crate::{protocol::AnnounceKind, tracker::ProbeKind}; +use crate::tracker::ProbeKind; pub const CONFIG_DEFAULTS_FILE: &str = "config.defaults.toml"; pub const CONFIG_FILE: &str = "config.toml"; diff --git a/iroh-mainline-content-discovery/src/iroh_bytes_util.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/iroh_bytes_util.rs similarity index 100% rename from iroh-mainline-content-discovery/src/iroh_bytes_util.rs rename to iroh-mainline-content-discovery/iroh-mainline-tracker/src/iroh_bytes_util.rs diff --git a/iroh-mainline-content-discovery/iroh-mainline-tracker/src/lib.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/lib.rs new file mode 100644 index 00000000..ee588b6c --- /dev/null +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/lib.rs @@ -0,0 +1,4 @@ +pub mod io; +pub mod iroh_bytes_util; +pub mod options; +pub mod tracker; diff --git a/iroh-mainline-content-discovery/src/main.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/main.rs similarity index 72% rename from iroh-mainline-content-discovery/src/main.rs rename to iroh-mainline-content-discovery/iroh-mainline-tracker/src/main.rs index fd0761e5..13141ecb 100644 --- a/iroh-mainline-content-discovery/src/main.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/main.rs @@ -1,7 +1,6 @@ pub mod args; use std::{ - collections::BTreeSet, net::{Ipv4Addr, SocketAddr, SocketAddrV4}, sync::{ atomic::{AtomicBool, Ordering}, @@ -12,13 +11,13 @@ use std::{ use anyhow::Context; use clap::Parser; -use iroh_mainline_content_discovery::{ +use iroh_mainline_content_discovery::protocol::ALPN; +use iroh_mainline_tracker::{ io::{ self, load_from_file, setup_logging, tracker_home, tracker_path, CONFIG_DEFAULTS_FILE, CONFIG_FILE, SERVER_KEY_FILE, }, options::Options, - protocol::{Announce, AnnounceKind, Query, QueryFlags, TRACKER_ALPN}, tracker::Tracker, }; use iroh_net::{ @@ -30,7 +29,7 @@ use pkarr::PkarrClient; use tokio::io::AsyncWriteExt; use tokio_util::task::LocalPoolHandle; -use crate::args::{AnnounceArgs, Args, Commands, QueryArgs, ServerArgs}; +use crate::args::{Args, Commands, ServerArgs}; static VERBOSE: AtomicBool = AtomicBool::new(false); @@ -78,7 +77,7 @@ async fn create_endpoint( iroh_net::MagicEndpoint::builder() .secret_key(key) .discovery(Box::new(mainline_discovery)) - .alpns(vec![TRACKER_ALPN.to_vec()]) + .alpns(vec![ALPN.to_vec()]) .bind(port) .await } @@ -139,89 +138,19 @@ async fn server(args: ServerArgs) -> anyhow::Result<()> { Ok(()) } -async fn announce(args: AnnounceArgs) -> anyhow::Result<()> { - // todo: uncomment once the connection problems are fixed - // for now, a random node id is more reliable. - // let key = load_secret_key(tracker_path(CLIENT_KEY)?).await?; - let key = iroh_net::key::SecretKey::generate(); - let content = args.content.iter().map(|x| x.hash_and_format()).collect(); - let host = if let Some(host) = args.host { - host - } else { - let hosts = args - .content - .iter() - .filter_map(|x| x.host()) - .collect::>(); - if hosts.len() != 1 { - anyhow::bail!( - "content for all tickets must be from the same host, unless a host is specified" - ); - } - *hosts.iter().next().unwrap() - }; - println!("announcing to {}", args.tracker); - println!("host {} has", host); - for content in &content { - println!(" {}", content); - } - let endpoint = create_endpoint(key, 11112, false).await?; - let connection = endpoint - .connect_by_node_id(&args.tracker, TRACKER_ALPN) - .await?; - println!("connected to {:?}", connection.remote_address()); - let kind = if args.partial { - AnnounceKind::Partial - } else { - AnnounceKind::Complete - }; - let announce = Announce { - host, - kind, - content, - }; - iroh_mainline_content_discovery::announce(connection, announce).await?; - println!("done"); - Ok(()) -} - -async fn query(args: QueryArgs) -> anyhow::Result<()> { - let connection = - iroh_mainline_content_discovery::connect(&args.tracker, args.port.unwrap_or_default()) - .await?; - let q = Query { - content: args.content.hash_and_format(), - flags: QueryFlags { - complete: !args.partial, - verified: args.verified, - }, - }; - let res = iroh_mainline_content_discovery::query(connection, q).await?; - println!( - "querying tracker {} for content {}", - args.tracker, args.content - ); - for peer in res.hosts { - println!("{}", peer); - } - Ok(()) -} - #[tokio::main(flavor = "multi_thread")] async fn main() -> anyhow::Result<()> { setup_logging(); let args = Args::parse(); match args.command { Commands::Server(args) => server(args).await, - Commands::Announce(args) => announce(args).await, - Commands::Query(args) => query(args).await, } } /// Returns default server configuration along with its certificate. #[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 fn configure_server(secret_key: &iroh_net::key::SecretKey) -> anyhow::Result { - make_server_config(secret_key, 8, 1024, vec![TRACKER_ALPN.to_vec()]) + make_server_config(secret_key, 8, 1024, vec![ALPN.to_vec()]) } /// Create a [`quinn::ServerConfig`] with the given secret key and limits. diff --git a/iroh-mainline-content-discovery/src/options.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/options.rs similarity index 100% rename from iroh-mainline-content-discovery/src/options.rs rename to iroh-mainline-content-discovery/iroh-mainline-tracker/src/options.rs diff --git a/iroh-mainline-content-discovery/src/tracker.rs b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/tracker.rs similarity index 97% rename from iroh-mainline-content-discovery/src/tracker.rs rename to iroh-mainline-content-discovery/iroh-mainline-tracker/src/tracker.rs index 86e01b0e..d51d6827 100644 --- a/iroh-mainline-content-discovery/src/tracker.rs +++ b/iroh-mainline-content-discovery/iroh-mainline-tracker/src/tracker.rs @@ -13,20 +13,24 @@ use iroh_bytes::{ protocol::GetRequest, BlobFormat, Hash, HashAndFormat, }; -use iroh_net::{MagicEndpoint, NodeId}; +use iroh_mainline_content_discovery::{ + announce_dht, + protocol::{ + Announce, AnnounceKind, Query, QueryResponse, Request, Response, REQUEST_SIZE_LIMIT, + }, +}; +use iroh_net::{ + magic_endpoint::{get_alpn, get_remote_node_id}, + MagicEndpoint, NodeId, +}; use rand::Rng; use crate::{ - accept_conn, announce_dht, io::{log_connection_attempt, log_probe_attempt, AnnounceData}, iroh_bytes_util::{ chunk_probe, get_hash_seq_and_sizes, random_hash_seq_ranges, unverified_size, verified_size, }, options::Options, - protocol::{ - Announce, AnnounceKind, Query, QueryResponse, Request, Response, REQUEST_SIZE_LIMIT, - }, - to_infohash, }; /// The tracker server. @@ -516,3 +520,13 @@ impl Tracker { anyhow::Ok((host, results)) } } + +/// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. +async fn accept_conn( + mut conn: quinn::Connecting, +) -> anyhow::Result<(NodeId, String, quinn::Connection)> { + let alpn = get_alpn(&mut conn).await?; + let conn = conn.await?; + let peer_id = get_remote_node_id(&conn)?; + Ok((peer_id, alpn, conn)) +} From 8e12a9474cb2cf4dcad044c4f625b0dd22f131d7 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Sun, 21 Jan 2024 15:17:56 +0200 Subject: [PATCH 13/13] Remove wrong Cargo.toml --- Cargo.toml | 8 -------- .../iroh-mainline-content-discovery/Cargo.toml | 6 ++++-- 2 files changed, 4 insertions(+), 10 deletions(-) delete mode 100644 Cargo.toml diff --git a/Cargo.toml b/Cargo.toml deleted file mode 100644 index 44a7175e..00000000 --- a/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "iroh-examples" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] diff --git a/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml index 6f9199ae..2f548969 100644 --- a/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml +++ b/iroh-mainline-content-discovery/iroh-mainline-content-discovery/Cargo.toml @@ -8,12 +8,14 @@ license = "MIT OR Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -# Required features for the protocol types +# Required features for the protocol types. +# +# The protocol is using postcard, but we don't need a postcard dependency for just the type definitions iroh-net = "0.12.0" iroh-bytes = "0.12.0" serde = { version = "1", features = ["derive"] } -# Optional features for the client +# Optional features for the client functionality tracing = { version = "0.1", optional = true } quinn = { version = "0.10", optional = true } iroh-pkarr-node-discovery = { path = "../../iroh-pkarr-node-discovery", optional = true }