From 7b6d0c836aaf7cdc29f54b8f58c4bb69156ac8ab Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 15 May 2024 23:46:49 +0200 Subject: [PATCH 01/24] feat: node-wide default author for documents --- iroh/src/client/authors.rs | 12 +++++-- iroh/src/docs_engine.rs | 4 +++ iroh/src/docs_engine/rpc.rs | 14 ++++++-- iroh/src/node.rs | 70 +++++++++++++++++++++++++++++++++++++ iroh/src/node/builder.rs | 17 +++++++-- iroh/src/node/rpc.rs | 6 ++++ iroh/src/rpc_protocol.rs | 17 +++++++++ iroh/src/util/fs.rs | 31 ++++++++++++++++ iroh/src/util/path.rs | 3 ++ 9 files changed, 168 insertions(+), 6 deletions(-) diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index 690ae228da..9597001db6 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -6,8 +6,8 @@ use iroh_docs::{Author, AuthorId}; use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ - AuthorCreateRequest, AuthorDeleteRequest, AuthorExportRequest, AuthorImportRequest, - AuthorListRequest, RpcService, + AuthorCreateRequest, AuthorDefaultRequest, AuthorDeleteRequest, AuthorExportRequest, + AuthorImportRequest, AuthorListRequest, RpcService, }; use super::flatten; @@ -28,6 +28,12 @@ where Ok(res.author_id) } + /// Get the default document author of this node. + pub async fn default(&self) -> Result { + let res = self.rpc.rpc(AuthorDefaultRequest).await?; + Ok(res.author_id) + } + /// List document authors for which we have a secret key. pub async fn list(&self) -> Result>> { let stream = self.rpc.server_streaming(AuthorListRequest {}).await?; @@ -53,6 +59,8 @@ where /// Deletes the given author by id. /// /// Warning: This permanently removes this author. + /// + /// Deleting the default author is not supported. pub async fn delete(&self, author: AuthorId) -> Result<()> { self.rpc.rpc(AuthorDeleteRequest { author }).await??; Ok(()) diff --git a/iroh/src/docs_engine.rs b/iroh/src/docs_engine.rs index 018d894f16..b42cd8933f 100644 --- a/iroh/src/docs_engine.rs +++ b/iroh/src/docs_engine.rs @@ -8,6 +8,7 @@ use anyhow::Result; use futures_lite::{Stream, StreamExt}; use iroh_blobs::downloader::Downloader; use iroh_blobs::{store::EntryStatus, Hash}; +use iroh_docs::AuthorId; use iroh_docs::{actor::SyncHandle, ContentStatus, ContentStatusCallback, Entry, NamespaceId}; use iroh_gossip::net::Gossip; use iroh_net::util::SharedAbortingJoinHandle; @@ -46,6 +47,7 @@ pub struct Engine { actor_handle: SharedAbortingJoinHandle<()>, #[debug("ContentStatusCallback")] content_status_cb: ContentStatusCallback, + default_author: AuthorId, } impl Engine { @@ -59,6 +61,7 @@ impl Engine { replica_store: iroh_docs::store::Store, bao_store: B, downloader: Downloader, + default_author: AuthorId, ) -> Self { let (live_actor_tx, to_live_actor_recv) = mpsc::channel(ACTOR_CHANNEL_CAP); let (to_gossip_actor, to_gossip_actor_recv) = mpsc::channel(ACTOR_CHANNEL_CAP); @@ -101,6 +104,7 @@ impl Engine { to_live_actor: live_actor_tx, actor_handle: actor_handle.into(), content_status_cb, + default_author, } } diff --git a/iroh/src/docs_engine/rpc.rs b/iroh/src/docs_engine/rpc.rs index 51739a92c6..ea1153890c 100644 --- a/iroh/src/docs_engine/rpc.rs +++ b/iroh/src/docs_engine/rpc.rs @@ -8,8 +8,9 @@ use tokio_stream::StreamExt; use crate::client::docs::ShareMode; use crate::rpc_protocol::{ - AuthorDeleteRequest, AuthorDeleteResponse, AuthorExportRequest, AuthorExportResponse, - AuthorImportRequest, AuthorImportResponse, DocGetSyncPeersRequest, DocGetSyncPeersResponse, + AuthorDefaultRequest, AuthorDefaultResponse, AuthorDeleteRequest, AuthorDeleteResponse, + AuthorExportRequest, AuthorExportResponse, AuthorImportRequest, AuthorImportResponse, + DocGetSyncPeersRequest, DocGetSyncPeersResponse, }; use crate::{ docs_engine::Engine, @@ -44,6 +45,12 @@ impl Engine { }) } + pub fn author_default(&self, _req: AuthorDefaultRequest) -> AuthorDefaultResponse { + AuthorDefaultResponse { + author_id: self.default_author, + } + } + pub fn author_list( &self, _req: AuthorListRequest, @@ -76,6 +83,9 @@ impl Engine { } pub async fn author_delete(&self, req: AuthorDeleteRequest) -> RpcResult { + if req.author == self.default_author { + return Err(anyhow!("Deleting the default author is not supported").into()); + } self.sync.delete_author(req.author).await?; Ok(AuthorDeleteResponse) } diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 88665a8c72..1430cb6853 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -387,4 +387,74 @@ mod tests { ); Ok(()) } + + #[tokio::test] + async fn test_default_author_memory() -> Result<()> { + let iroh = Node::memory().spawn().await?; + let author = iroh.authors.default().await?; + assert!(iroh.authors.export(author).await?.is_some()); + assert!(iroh.authors.delete(author).await.is_err()); + Ok(()) + } + + #[cfg(feature = "fs-store")] + #[tokio::test] + async fn test_default_author_persist() -> Result<()> { + use crate::util::path::IrohPaths; + + let _guard = iroh_test::logging::setup(); + + let iroh_root_dir = tempfile::TempDir::new()?; + let iroh_root = iroh_root_dir.path(); + + // check that the default author exists and cannot be deleted. + let default_author = { + let iroh = Node::persistent(iroh_root).await?.spawn().await?; + let author = iroh.authors.default().await?; + assert!(iroh.authors.export(author).await?.is_some()); + assert!(iroh.authors.delete(author).await.is_err()); + iroh.shutdown().await?; + author + }; + + // check that the default author is persisted across restarts. + { + let iroh = Node::persistent(iroh_root).await?.spawn().await?; + let author = iroh.authors.default().await?; + assert_eq!(author, default_author); + assert!(iroh.authors.export(author).await?.is_some()); + assert!(iroh.authors.delete(author).await.is_err()); + iroh.shutdown().await?; + }; + + // check that a new default author is created if the default author file is deleted + // manually. + let default_author = { + tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(&iroh_root)).await?; + let iroh = Node::persistent(iroh_root).await?.spawn().await?; + let author = iroh.authors.default().await?; + assert!(author != default_author); + assert!(iroh.authors.export(author).await?.is_some()); + assert!(iroh.authors.delete(author).await.is_err()); + iroh.shutdown().await?; + author + }; + + // check that the node fails to start if the default author is missing from the docs store. + { + let mut docs_store = iroh_docs::store::fs::Store::persistent( + IrohPaths::DocsDatabase.with_root(&iroh_root), + )?; + docs_store.delete_author(default_author)?; + docs_store.flush()?; + drop(docs_store); + let iroh = Node::persistent(iroh_root).await?.spawn().await; + assert!(iroh.is_err()); + tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(&iroh_root)).await?; + let iroh = Node::persistent(iroh_root).await?.spawn().await; + assert!(iroh.is_ok()); + } + + Ok(()) + } } diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index 61a53f2828..1b6fa2ff36 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -35,7 +35,10 @@ use crate::{ docs_engine::Engine, node::NodeInner, rpc_protocol::{Request, Response, RpcService}, - util::{fs::load_secret_key, path::IrohPaths}, + util::{ + fs::{load_default_author, load_secret_key}, + path::IrohPaths, + }, }; use super::{rpc, rpc_status::RpcStatus, Node}; @@ -429,14 +432,24 @@ where // initialize the gossip protocol let gossip = Gossip::from_endpoint(endpoint.clone(), Default::default(), &addr.info); - // spawn the sync engine + // initialize the downloader let downloader = Downloader::new(self.blobs_store.clone(), endpoint.clone(), lp.clone()); + + let default_author = match self.storage { + StorageConfig::Persistent(ref root) => { + let path = IrohPaths::DefaultAuthor.with_root(root); + load_default_author(path, &mut self.docs_store).await? + } + StorageConfig::Mem => self.docs_store.new_author(&mut rand::thread_rng())?.id(), + }; + let sync = Engine::spawn( endpoint.clone(), gossip.clone(), self.docs_store, self.blobs_store.clone(), downloader.clone(), + default_author, ); let sync_db = sync.sync.clone(); diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index 0c50f7ed33..b14f6dea32 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -161,6 +161,12 @@ impl Handler { }) .await } + AuthorDefault(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.author_default(req) + }) + .await + } DocOpen(msg) => { chan.rpc(msg, handler, |handler, req| async move { handler.inner.sync.doc_open(req).await diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 90b2637a4c..576f972822 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -438,6 +438,21 @@ pub struct AuthorCreateResponse { pub author_id: AuthorId, } +/// Get the default author. +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorDefaultRequest; + +impl RpcMsg for AuthorDefaultRequest { + type Response = AuthorDefaultResponse; +} + +/// Response for [`AuthorDefaultRequest`] +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorDefaultResponse { + /// The id of the author + pub author_id: AuthorId, +} + /// Delete an author #[derive(Serialize, Deserialize, Debug)] pub struct AuthorDeleteRequest { @@ -1070,6 +1085,7 @@ pub enum Request { AuthorList(AuthorListRequest), AuthorCreate(AuthorCreateRequest), + AuthorDefault(AuthorDefaultRequest), AuthorImport(AuthorImportRequest), AuthorExport(AuthorExportRequest), AuthorDelete(AuthorDeleteRequest), @@ -1130,6 +1146,7 @@ pub enum Response { AuthorList(RpcResult), AuthorCreate(RpcResult), + AuthorDefault(AuthorDefaultResponse), AuthorImport(RpcResult), AuthorExport(RpcResult), AuthorDelete(RpcResult), diff --git a/iroh/src/util/fs.rs b/iroh/src/util/fs.rs index d1af9650b0..f945a4f837 100644 --- a/iroh/src/util/fs.rs +++ b/iroh/src/util/fs.rs @@ -3,10 +3,12 @@ use std::{ borrow::Cow, fs::read_dir, path::{Component, Path, PathBuf}, + str::FromStr, }; use anyhow::{bail, Context}; use bytes::Bytes; +use iroh_docs::AuthorId; use iroh_net::key::SecretKey; use tokio::io::AsyncWriteExt; use walkdir::WalkDir; @@ -119,6 +121,35 @@ pub fn relative_canonicalized_path_to_string(path: impl AsRef) -> anyhow:: canonicalized_path_to_string(path, true) } +/// Load the default author public key from a path, and check that it is present in the `docs_store`. +/// +/// If `path` does not exist, a new author keypair is created and persisted in the docs store, and +/// the public key is written to `path`, in base32 encoding. +/// +/// If `path` does exist, but does not contain an ed25519 public key in base32 encoding, an error +/// is returned. +/// +/// If `path` exists and is a valid author public key, but its secret key does not exist in the +/// docs store, an error is returned. +pub async fn load_default_author( + path: PathBuf, + docs_store: &mut iroh_docs::store::fs::Store, +) -> anyhow::Result { + if path.exists() { + let data = tokio::fs::read_to_string(&path).await?; + let author_id = AuthorId::from_str(&data)?; + if !docs_store.get_author(&author_id)?.is_some() { + bail!("The default author is missing from the docs store. To recover, delete the file `{}`. Then iroh will create a new default author.", path.to_string_lossy()) + } + Ok(author_id) + } else { + let author_id = docs_store.new_author(&mut rand::thread_rng())?.id(); + docs_store.flush()?; + tokio::fs::write(path, author_id.to_string()).await?; + Ok(author_id) + } +} + /// Loads a [`SecretKey`] from the provided file, or stores a newly generated one /// at the given location. pub async fn load_secret_key(key_path: PathBuf) -> anyhow::Result { diff --git a/iroh/src/util/path.rs b/iroh/src/util/path.rs index 0240e11de5..531c295608 100644 --- a/iroh/src/util/path.rs +++ b/iroh/src/util/path.rs @@ -24,6 +24,9 @@ pub enum IrohPaths { #[strum(serialize = "rpc.lock")] /// Path to RPC lock file, containing the RPC port if running. RpcLock, + /// Path to the [`AuthorId`] of the node's default author + #[strum(serialize = "default-author")] + DefaultAuthor, } impl AsRef for IrohPaths { From 04735a0e9effef37bee94a04d6a0cfb4d697d2dc Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 15 May 2024 23:59:43 +0200 Subject: [PATCH 02/24] adjust example --- iroh/examples/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/examples/client.rs b/iroh/examples/client.rs index c0eeb952f6..3e4d018aed 100644 --- a/iroh/examples/client.rs +++ b/iroh/examples/client.rs @@ -17,7 +17,7 @@ async fn main() -> anyhow::Result<()> { let client = node.client(); let doc = client.docs.create().await?; - let author = client.authors.create().await?; + let author = client.authors.default().await?; doc.set_bytes(author, "hello", "world").await?; From c5d224bb05b20178d8ddf8eff0cea8b65966a52d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 16 May 2024 00:02:20 +0200 Subject: [PATCH 03/24] fix comments --- iroh/src/node/builder.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index 1b6fa2ff36..2969733580 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -435,6 +435,7 @@ where // initialize the downloader let downloader = Downloader::new(self.blobs_store.clone(), endpoint.clone(), lp.clone()); + // load or create the default author for documents let default_author = match self.storage { StorageConfig::Persistent(ref root) => { let path = IrohPaths::DefaultAuthor.with_root(root); @@ -443,6 +444,7 @@ where StorageConfig::Mem => self.docs_store.new_author(&mut rand::thread_rng())?.id(), }; + // spawn the docs engine let sync = Engine::spawn( endpoint.clone(), gossip.clone(), From d078ae3d29b4be8c9a03d473a6aa5bf97977c151 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 16 May 2024 00:09:03 +0200 Subject: [PATCH 04/24] expand docs --- iroh/src/client/authors.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index 9597001db6..62ec80a300 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -23,12 +23,23 @@ where C: ServiceConnection, { /// Create a new document author. + /// + /// You likely want to save the returned [`AuthorId`] somewhere so that you can use this author + /// again. + /// + /// If you need only a single author, use [`Self::default`]. pub async fn create(&self) -> Result { let res = self.rpc.rpc(AuthorCreateRequest).await??; Ok(res.author_id) } /// Get the default document author of this node. + /// + /// On persistent nodes, a new author is created on first start and its public key is saved + /// in the data directory. For in-memory nodes, the default author is a random, new author. + /// + /// The default author can neither be changed nor deleted. If you need more semantics around + /// authors than a single author per node, use [`Self::create`]. pub async fn default(&self) -> Result { let res = self.rpc.rpc(AuthorDefaultRequest).await?; Ok(res.author_id) From 0915b152e76702b4b1188f879065f9522be8b40f Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 16 May 2024 00:11:39 +0200 Subject: [PATCH 05/24] fixup --- iroh/src/client/authors.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index 62ec80a300..d14d9d22d9 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -33,10 +33,10 @@ where Ok(res.author_id) } - /// Get the default document author of this node. + /// Returns the default document author of this node. /// - /// On persistent nodes, a new author is created on first start and its public key is saved - /// in the data directory. For in-memory nodes, the default author is a random, new author. + /// On persistent nodes, the author is created on first start and its public key is saved + /// in the data directory. /// /// The default author can neither be changed nor deleted. If you need more semantics around /// authors than a single author per node, use [`Self::create`]. @@ -71,7 +71,7 @@ where /// /// Warning: This permanently removes this author. /// - /// Deleting the default author is not supported. + /// Returns an error if attempting to delete the default author. pub async fn delete(&self, author: AuthorId) -> Result<()> { self.rpc.rpc(AuthorDeleteRequest { author }).await??; Ok(()) From 2310cd3f45d2ed2d96fef1e13358aea647bf70bf Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 16 May 2024 00:23:24 +0200 Subject: [PATCH 06/24] feat: default author support in cli and console --- iroh-cli/src/commands.rs | 16 +++++-- iroh-cli/src/commands/author.rs | 17 +++++++ iroh-cli/src/commands/console.rs | 13 +++-- iroh-cli/src/commands/doc.rs | 10 ++-- iroh-cli/src/config.rs | 81 +++++++++++++++++++------------- 5 files changed, 89 insertions(+), 48 deletions(-) diff --git a/iroh-cli/src/commands.rs b/iroh-cli/src/commands.rs index 9fd279be12..92a1042641 100644 --- a/iroh-cli/src/commands.rs +++ b/iroh-cli/src/commands.rs @@ -115,36 +115,44 @@ impl Cli { match self.command { Commands::Console => { - let env = ConsoleEnv::for_console(data_dir)?; + let data_dir_owned = data_dir.to_owned(); if self.start { let config = NodeConfig::load(self.config.as_deref()).await?; start::run_with_command( &config, data_dir, RunType::SingleCommandNoAbort, - |iroh| async move { console::run(&iroh, &env).await }, + |iroh| async move { + let env = ConsoleEnv::for_console(data_dir_owned, &iroh).await?; + console::run(&iroh, &env).await + }, ) .await } else { crate::logging::init_terminal_logging()?; let iroh = QuicIroh::connect(data_dir).await.context("rpc connect")?; + let env = ConsoleEnv::for_console(data_dir_owned, &iroh).await?; console::run(&iroh, &env).await } } Commands::Rpc(command) => { - let env = ConsoleEnv::for_cli(data_dir)?; + let data_dir_owned = data_dir.to_owned(); if self.start { let config = NodeConfig::load(self.config.as_deref()).await?; start::run_with_command( &config, data_dir, RunType::SingleCommandAbortable, - |iroh| async move { command.run(&iroh, &env).await }, + move |iroh| async move { + let env = ConsoleEnv::for_cli(data_dir_owned, &iroh).await?; + command.run(&iroh, &env).await + }, ) .await } else { crate::logging::init_terminal_logging()?; let iroh = QuicIroh::connect(data_dir).await.context("rpc connect")?; + let env = ConsoleEnv::for_cli(data_dir_owned, &iroh).await?; command.run(&iroh, &env).await } } diff --git a/iroh-cli/src/commands/author.rs b/iroh-cli/src/commands/author.rs index 1499a523c7..daf96c72f6 100644 --- a/iroh-cli/src/commands/author.rs +++ b/iroh-cli/src/commands/author.rs @@ -26,6 +26,12 @@ pub enum AuthorCommands { Export { author: AuthorId }, /// Import an author Import { author: String }, + /// Print the default author for this node. + Default { + /// Switch to the default author (only in the Iroh console). + #[clap(long)] + switch: bool, + } /// List authors. #[clap(alias = "ls")] List, @@ -47,6 +53,17 @@ impl AuthorCommands { println!("{}", author_id); } } + Self::Default { switch } => { + if switch && !env.is_console() { + bail!("The --switch flag is only supported within the Iroh console."); + } + let author_id = iroh.authors.default().await?; + println!("{}", author_id); + if switch { + env.set_author(author_id)?; + println!("Active author is now {}", fmt_short(author_id.as_bytes())); + } + } Self::New { switch } => { if switch && !env.is_console() { bail!("The --switch flag is only supported within the Iroh console."); diff --git a/iroh-cli/src/commands/console.rs b/iroh-cli/src/commands/console.rs index f087e6d490..4d8a25555d 100644 --- a/iroh-cli/src/commands/console.rs +++ b/iroh-cli/src/commands/console.rs @@ -87,13 +87,12 @@ impl Repl { pub fn prompt(&self) -> String { let mut pwd = String::new(); - if let Some(author) = &self.env.author(None).ok() { - pwd.push_str(&format!( - "{}{} ", - "author:".blue(), - fmt_short(author.as_bytes()).blue().bold(), - )); - } + let author = self.env.author(); + pwd.push_str(&format!( + "{}{} ", + "author:".blue(), + fmt_short(author.as_bytes()).blue().bold(), + )); if let Some(doc) = &self.env.doc(None).ok() { pwd.push_str(&format!( "{}{} ", diff --git a/iroh-cli/src/commands/doc.rs b/iroh-cli/src/commands/doc.rs index a461685f7b..7c6465b592 100644 --- a/iroh-cli/src/commands/doc.rs +++ b/iroh-cli/src/commands/doc.rs @@ -360,7 +360,7 @@ impl DocCommands { value, } => { let doc = get_doc(iroh, env, doc).await?; - let author = env.author(author)?; + let author = author.unwrap_or(env.author()); let key = key.as_bytes().to_vec(); let value = value.as_bytes().to_vec(); let hash = doc.set_bytes(author, key, value).await?; @@ -372,7 +372,7 @@ impl DocCommands { prefix, } => { let doc = get_doc(iroh, env, doc).await?; - let author = env.author(author)?; + let author = author.unwrap_or(env.author()); let prompt = format!("Deleting all entries whose key starts with {prefix}. Continue?"); if Confirm::new() @@ -453,7 +453,7 @@ impl DocCommands { no_prompt, } => { let doc = get_doc(iroh, env, doc).await?; - let author = env.author(author)?; + let author = author.unwrap_or(env.author()); let mut prefix = prefix.unwrap_or_else(|| String::from("")); if prefix.ends_with('/') { @@ -979,7 +979,9 @@ mod tests { let author = client.authors.create().await.context("author create")?; // set up command, getting iroh node - let cli = ConsoleEnv::for_console(data_dir.path()).context("ConsoleEnv")?; + let cli = ConsoleEnv::for_console(data_dir.path().to_owned(), &node) + .await + .context("ConsoleEnv")?; let iroh = iroh::client::QuicIroh::connect(data_dir.path()) .await .context("rpc connect")?; diff --git a/iroh-cli/src/config.rs b/iroh-cli/src/config.rs index a2bc03fc1d..376500a861 100644 --- a/iroh-cli/src/config.rs +++ b/iroh-cli/src/config.rs @@ -9,13 +9,17 @@ use std::{ }; use anyhow::{anyhow, bail, Context, Result}; -use iroh::docs::{AuthorId, NamespaceId}; use iroh::net::{ defaults::{default_eu_relay_node, default_na_relay_node}, relay::{RelayMap, RelayNode}, }; use iroh::node::GcPolicy; +use iroh::{ + client::{Iroh, RpcService}, + docs::{AuthorId, NamespaceId}, +}; use parking_lot::RwLock; +use quic_rpc::ServiceConnection; use serde::{Deserialize, Serialize}; const ENV_AUTHOR: &str = "IROH_AUTHOR"; @@ -145,7 +149,8 @@ pub(crate) struct ConsoleEnv(Arc>); struct ConsoleEnvInner { /// Active author. Read from IROH_AUTHOR env variable. /// For console also read from/persisted to a file (see [`ConsolePaths::DefaultAuthor`]) - author: Option, + /// Defaults to the node's default author if both are empty. + author: AuthorId, /// Active doc. Read from IROH_DOC env variable. Not persisted. doc: Option, is_console: bool, @@ -154,43 +159,45 @@ struct ConsoleEnvInner { impl ConsoleEnv { /// Read from environment variables and the console config file. - pub(crate) fn for_console(iroh_data_root: &Path) -> Result { - let author = match env_author()? { - Some(author) => Some(author), - None => Self::get_console_default_author(iroh_data_root)?, - }; + pub(crate) async fn for_console>( + iroh_data_dir: PathBuf, + iroh: &Iroh, + ) -> Result { + let configured_author = Self::get_console_default_author(&iroh_data_dir)?; + let author = env_author(configured_author, iroh).await?; let env = ConsoleEnvInner { author, doc: env_doc()?, is_console: true, - iroh_data_dir: iroh_data_root.to_path_buf(), + iroh_data_dir, }; Ok(Self(Arc::new(RwLock::new(env)))) } /// Read only from environment variables. - pub(crate) fn for_cli(iroh_data_root: &Path) -> Result { + pub(crate) async fn for_cli>( + iroh_data_dir: PathBuf, + iroh: &Iroh, + ) -> Result { + let author = env_author(None, iroh).await?; let env = ConsoleEnvInner { - author: env_author()?, + author, doc: env_doc()?, is_console: false, - iroh_data_dir: iroh_data_root.to_path_buf(), + iroh_data_dir, }; Ok(Self(Arc::new(RwLock::new(env)))) } fn get_console_default_author(root: &Path) -> anyhow::Result> { let author_path = ConsolePaths::DefaultAuthor.with_root(root); - if let Ok(s) = std::fs::read(&author_path) { - let author = String::from_utf8(s) - .map_err(Into::into) - .and_then(|s| AuthorId::from_str(&s)) - .with_context(|| { - format!( - "Failed to parse author file at {}", - author_path.to_string_lossy() - ) - })?; + if let Ok(s) = std::fs::read_to_string(&author_path) { + let author = AuthorId::from_str(&s).with_context(|| { + format!( + "Failed to parse author file at {}", + author_path.to_string_lossy() + ) + })?; Ok(Some(author)) } else { Ok(None) @@ -217,7 +224,7 @@ impl ConsoleEnv { if !inner.is_console { bail!("Switching the author is only supported within the Iroh console, not on the command line"); } - inner.author = Some(author); + inner.author = author; std::fs::write(author_path, author.to_string().as_bytes())?; Ok(()) } @@ -248,26 +255,34 @@ impl ConsoleEnv { } /// Get the active author. - pub(crate) fn author(&self, arg: Option) -> anyhow::Result { + /// + /// This is either the node's default author, or in the console optionally the author manually + /// switched to. + pub(crate) fn author( + &self, + ) -> AuthorId { let inner = self.0.read(); - let author_id = arg.or(inner.author).ok_or_else(|| { - anyhow!( - "Missing author id. Set the active author with the `IROH_AUTHOR` environment variable or the `-a` option.\n\ - In the console, you can also set the active author with `author switch`." - ) - })?; - Ok(author_id) + inner.author } } -fn env_author() -> Result> { - env::var(ENV_AUTHOR) +async fn env_author>( + from_config: Option, + iroh: &Iroh, +) -> Result { + if let Some(author) = env::var(ENV_AUTHOR) .ok() .map(|s| { s.parse() .context("Failed to parse IROH_AUTHOR environment variable") }) - .transpose() + .transpose()? + .or(from_config) + { + Ok(author) + } else { + iroh.authors.default().await + } } fn env_doc() -> Result> { From 8d870d731850d7b3a9ace08baa1c1c327430f75d Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 16 May 2024 01:24:28 +0200 Subject: [PATCH 07/24] fixup --- iroh-cli/src/commands/author.rs | 2 +- iroh-cli/src/config.rs | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/iroh-cli/src/commands/author.rs b/iroh-cli/src/commands/author.rs index daf96c72f6..8da797845c 100644 --- a/iroh-cli/src/commands/author.rs +++ b/iroh-cli/src/commands/author.rs @@ -31,7 +31,7 @@ pub enum AuthorCommands { /// Switch to the default author (only in the Iroh console). #[clap(long)] switch: bool, - } + }, /// List authors. #[clap(alias = "ls")] List, diff --git a/iroh-cli/src/config.rs b/iroh-cli/src/config.rs index 376500a861..103aa95c87 100644 --- a/iroh-cli/src/config.rs +++ b/iroh-cli/src/config.rs @@ -258,9 +258,7 @@ impl ConsoleEnv { /// /// This is either the node's default author, or in the console optionally the author manually /// switched to. - pub(crate) fn author( - &self, - ) -> AuthorId { + pub(crate) fn author(&self) -> AuthorId { let inner = self.0.read(); inner.author } From 7691aef55f49b0f1d477916ce7fac6dbfbbcd024 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 16 May 2024 01:33:28 +0200 Subject: [PATCH 08/24] test: fix author tests --- iroh/src/client/authors.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index d14d9d22d9..a528f9c035 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -88,10 +88,16 @@ mod tests { async fn test_authors() -> Result<()> { let node = Node::memory().spawn().await?; + // default author always exists + let authors: Vec<_> = node.authors.list().await?.try_collect().await?; + assert_eq!(authors.len(), 1); + let default_author = node.authors.default().await?; + assert_eq!(authors, vec![default_author]); + let author_id = node.authors.create().await?; let authors: Vec<_> = node.authors.list().await?.try_collect().await?; - assert_eq!(authors.len(), 1); + assert_eq!(authors.len(), 2); let author = node .authors @@ -105,7 +111,7 @@ mod tests { node.authors.import(author).await?; let authors: Vec<_> = node.authors.list().await?.try_collect().await?; - assert_eq!(authors.len(), 1); + assert_eq!(authors.len(), 2); Ok(()) } From b3258569f99da3c8ef7c78813dc46bbe219876e9 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 16 May 2024 11:48:46 +0200 Subject: [PATCH 09/24] fix authors test --- iroh/src/client/authors.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index a528f9c035..b2ba2b606a 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -106,7 +106,7 @@ mod tests { .expect("should have author"); node.authors.delete(author_id).await?; let authors: Vec<_> = node.authors.list().await?.try_collect().await?; - assert!(authors.is_empty()); + assert_eq!(authors.len(), 1); node.authors.import(author).await?; From 0217c009a20aded7d2b2c19eeb95369f908ed72c Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 16 May 2024 12:08:40 +0200 Subject: [PATCH 10/24] chore: clippy & doclinks --- iroh/src/util/fs.rs | 2 +- iroh/src/util/path.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh/src/util/fs.rs b/iroh/src/util/fs.rs index f945a4f837..d95cdf0629 100644 --- a/iroh/src/util/fs.rs +++ b/iroh/src/util/fs.rs @@ -138,7 +138,7 @@ pub async fn load_default_author( if path.exists() { let data = tokio::fs::read_to_string(&path).await?; let author_id = AuthorId::from_str(&data)?; - if !docs_store.get_author(&author_id)?.is_some() { + if docs_store.get_author(&author_id)?.is_none() { bail!("The default author is missing from the docs store. To recover, delete the file `{}`. Then iroh will create a new default author.", path.to_string_lossy()) } Ok(author_id) diff --git a/iroh/src/util/path.rs b/iroh/src/util/path.rs index 531c295608..f7ee91af40 100644 --- a/iroh/src/util/path.rs +++ b/iroh/src/util/path.rs @@ -24,7 +24,7 @@ pub enum IrohPaths { #[strum(serialize = "rpc.lock")] /// Path to RPC lock file, containing the RPC port if running. RpcLock, - /// Path to the [`AuthorId`] of the node's default author + /// Path to the [`iroh_docs::AuthorId`] of the node's default author #[strum(serialize = "default-author")] DefaultAuthor, } From 185fe6ce7848877dc675ef2762b4133ce6a07e75 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 16 May 2024 12:12:48 +0200 Subject: [PATCH 11/24] chore: clippy --- iroh/src/node.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 1430cb6853..bf3b1bb814 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -430,7 +430,7 @@ mod tests { // check that a new default author is created if the default author file is deleted // manually. let default_author = { - tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(&iroh_root)).await?; + tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)).await?; let iroh = Node::persistent(iroh_root).await?.spawn().await?; let author = iroh.authors.default().await?; assert!(author != default_author); @@ -443,14 +443,14 @@ mod tests { // check that the node fails to start if the default author is missing from the docs store. { let mut docs_store = iroh_docs::store::fs::Store::persistent( - IrohPaths::DocsDatabase.with_root(&iroh_root), + IrohPaths::DocsDatabase.with_root(iroh_root), )?; docs_store.delete_author(default_author)?; docs_store.flush()?; drop(docs_store); let iroh = Node::persistent(iroh_root).await?.spawn().await; assert!(iroh.is_err()); - tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(&iroh_root)).await?; + tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)).await?; let iroh = Node::persistent(iroh_root).await?.spawn().await; assert!(iroh.is_ok()); } From 894aaa35d0625a7be8dc314451c6587de53fe79c Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 17 May 2024 11:21:36 +0200 Subject: [PATCH 12/24] refactor: rename --- iroh/src/client/authors.rs | 4 ++-- iroh/src/docs_engine/rpc.rs | 6 +++--- iroh/src/node/rpc.rs | 2 +- iroh/src/rpc_protocol.rs | 14 +++++++------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index b2ba2b606a..24773c3a55 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -6,7 +6,7 @@ use iroh_docs::{Author, AuthorId}; use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ - AuthorCreateRequest, AuthorDefaultRequest, AuthorDeleteRequest, AuthorExportRequest, + AuthorCreateRequest, AuthorGetDefaultRequest, AuthorDeleteRequest, AuthorExportRequest, AuthorImportRequest, AuthorListRequest, RpcService, }; @@ -41,7 +41,7 @@ where /// The default author can neither be changed nor deleted. If you need more semantics around /// authors than a single author per node, use [`Self::create`]. pub async fn default(&self) -> Result { - let res = self.rpc.rpc(AuthorDefaultRequest).await?; + let res = self.rpc.rpc(AuthorGetDefaultRequest).await?; Ok(res.author_id) } diff --git a/iroh/src/docs_engine/rpc.rs b/iroh/src/docs_engine/rpc.rs index ea1153890c..9009bef113 100644 --- a/iroh/src/docs_engine/rpc.rs +++ b/iroh/src/docs_engine/rpc.rs @@ -8,7 +8,7 @@ use tokio_stream::StreamExt; use crate::client::docs::ShareMode; use crate::rpc_protocol::{ - AuthorDefaultRequest, AuthorDefaultResponse, AuthorDeleteRequest, AuthorDeleteResponse, + AuthorGetDefaultRequest, AuthorGetDefaultResponse, AuthorDeleteRequest, AuthorDeleteResponse, AuthorExportRequest, AuthorExportResponse, AuthorImportRequest, AuthorImportResponse, DocGetSyncPeersRequest, DocGetSyncPeersResponse, }; @@ -45,8 +45,8 @@ impl Engine { }) } - pub fn author_default(&self, _req: AuthorDefaultRequest) -> AuthorDefaultResponse { - AuthorDefaultResponse { + pub fn author_default(&self, _req: AuthorGetDefaultRequest) -> AuthorGetDefaultResponse { + AuthorGetDefaultResponse { author_id: self.default_author, } } diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index b14f6dea32..791a28dc75 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -161,7 +161,7 @@ impl Handler { }) .await } - AuthorDefault(msg) => { + AuthorGetDefault(msg) => { chan.rpc(msg, handler, |handler, req| async move { handler.inner.sync.author_default(req) }) diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index 576f972822..aa1f29c7b6 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -440,15 +440,15 @@ pub struct AuthorCreateResponse { /// Get the default author. #[derive(Serialize, Deserialize, Debug)] -pub struct AuthorDefaultRequest; +pub struct AuthorGetDefaultRequest; -impl RpcMsg for AuthorDefaultRequest { - type Response = AuthorDefaultResponse; +impl RpcMsg for AuthorGetDefaultRequest { + type Response = AuthorGetDefaultResponse; } -/// Response for [`AuthorDefaultRequest`] +/// Response for [`AuthorGetDefaultRequest`] #[derive(Serialize, Deserialize, Debug)] -pub struct AuthorDefaultResponse { +pub struct AuthorGetDefaultResponse { /// The id of the author pub author_id: AuthorId, } @@ -1085,7 +1085,7 @@ pub enum Request { AuthorList(AuthorListRequest), AuthorCreate(AuthorCreateRequest), - AuthorDefault(AuthorDefaultRequest), + AuthorGetDefault(AuthorGetDefaultRequest), AuthorImport(AuthorImportRequest), AuthorExport(AuthorExportRequest), AuthorDelete(AuthorDeleteRequest), @@ -1146,7 +1146,7 @@ pub enum Response { AuthorList(RpcResult), AuthorCreate(RpcResult), - AuthorDefault(AuthorDefaultResponse), + AuthorGetDefault(AuthorGetDefaultResponse), AuthorImport(RpcResult), AuthorExport(RpcResult), AuthorDelete(RpcResult), From b361aaeb7707a9f0add62d4f4b38be202b7e1e02 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 17 May 2024 12:28:16 +0200 Subject: [PATCH 13/24] feat: set default author --- iroh/src/client/authors.rs | 23 ++++++++++-- iroh/src/docs_engine.rs | 75 +++++++++++++++++++++++++++++++++---- iroh/src/docs_engine/rpc.rs | 17 +++++++-- iroh/src/node.rs | 15 ++++++++ iroh/src/node/builder.rs | 20 +++++----- iroh/src/node/rpc.rs | 6 +++ iroh/src/rpc_protocol.rs | 16 ++++++++ iroh/src/util/fs.rs | 31 --------------- 8 files changed, 146 insertions(+), 57 deletions(-) diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index 24773c3a55..eac2a55496 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -6,8 +6,8 @@ use iroh_docs::{Author, AuthorId}; use quic_rpc::{RpcClient, ServiceConnection}; use crate::rpc_protocol::{ - AuthorCreateRequest, AuthorGetDefaultRequest, AuthorDeleteRequest, AuthorExportRequest, - AuthorImportRequest, AuthorListRequest, RpcService, + AuthorCreateRequest, AuthorDeleteRequest, AuthorExportRequest, AuthorGetDefaultRequest, + AuthorImportRequest, AuthorListRequest, AuthorSetDefaultRequest, RpcService, }; use super::flatten; @@ -38,13 +38,25 @@ where /// On persistent nodes, the author is created on first start and its public key is saved /// in the data directory. /// - /// The default author can neither be changed nor deleted. If you need more semantics around - /// authors than a single author per node, use [`Self::create`]. + /// The default author can be set with [`Self::set_default`]. pub async fn default(&self) -> Result { let res = self.rpc.rpc(AuthorGetDefaultRequest).await?; Ok(res.author_id) } + /// Set the node-wide default author. + /// + /// If the author does not exist, an error is returned. + /// + /// This is a noop on memory nodes. On peristent node, the author id will be saved to a file in + /// the data directory, and reloaded after a node restart. + pub async fn set_default(&self, author_id: AuthorId) -> Result<()> { + self.rpc + .rpc(AuthorSetDefaultRequest { author_id }) + .await??; + Ok(()) + } + /// List document authors for which we have a secret key. pub async fn list(&self) -> Result>> { let stream = self.rpc.server_streaming(AuthorListRequest {}).await?; @@ -113,6 +125,9 @@ mod tests { let authors: Vec<_> = node.authors.list().await?.try_collect().await?; assert_eq!(authors.len(), 2); + node.authors.set_default(author_id).await?; + assert_eq!(node.authors.default().await?, author_id); + Ok(()) } } diff --git a/iroh/src/docs_engine.rs b/iroh/src/docs_engine.rs index b42cd8933f..1227c8d4bc 100644 --- a/iroh/src/docs_engine.rs +++ b/iroh/src/docs_engine.rs @@ -2,14 +2,15 @@ //! //! [`iroh_docs::Replica`] is also called documents here. -use std::{io, sync::Arc}; +use std::path::PathBuf; +use std::{io, str::FromStr, sync::Arc}; -use anyhow::Result; +use anyhow::{bail, Result}; use futures_lite::{Stream, StreamExt}; use iroh_blobs::downloader::Downloader; use iroh_blobs::{store::EntryStatus, Hash}; -use iroh_docs::AuthorId; use iroh_docs::{actor::SyncHandle, ContentStatus, ContentStatusCallback, Entry, NamespaceId}; +use iroh_docs::{Author, AuthorId}; use iroh_gossip::net::Gossip; use iroh_net::util::SharedAbortingJoinHandle; use iroh_net::{key::PublicKey, Endpoint, NodeAddr}; @@ -48,6 +49,61 @@ pub struct Engine { #[debug("ContentStatusCallback")] content_status_cb: ContentStatusCallback, default_author: AuthorId, + default_author_storage: Arc, +} + +/// Where to persist the default author. +/// +/// If set to `Mem`, a new author will be created in the docs store before spawning the sync +/// engine. Changing the default author will not be persisted. +/// +/// If set to `Persistent`, the default author will be loaded from and persisted to the specified +/// path (as base32 encoded string of the author's public key). +#[derive(Debug)] +pub enum DefaultAuthorStorage { + Mem, + Persistent(PathBuf), +} + +impl DefaultAuthorStorage { + pub async fn load(&self, docs_store: &SyncHandle) -> anyhow::Result { + match self { + Self::Mem => { + let author = Author::new(&mut rand::thread_rng()); + let author_id = author.id(); + docs_store.import_author(author).await?; + Ok(author_id) + } + Self::Persistent(ref path) => { + if path.exists() { + let data = tokio::fs::read_to_string(path).await?; + let author_id = AuthorId::from_str(&data)?; + if docs_store.export_author(author_id).await?.is_none() { + bail!("The default author is missing from the docs store. To recover, delete the file `{}`. Then iroh will create a new default author.", path.to_string_lossy()) + } + Ok(author_id) + } else { + let author = Author::new(&mut rand::thread_rng()); + let author_id = author.id(); + docs_store.import_author(author).await?; + tokio::fs::write(path, author_id.to_string()).await?; + Ok(author_id) + } + } + } + } + pub async fn save(&self, docs_store: &SyncHandle, author_id: AuthorId) -> anyhow::Result<()> { + if docs_store.export_author(author_id).await?.is_none() { + bail!("The author does not exist"); + } + match self { + Self::Mem => {} + Self::Persistent(ref path) => { + tokio::fs::write(path, author_id.to_string()).await?; + } + } + Ok(()) + } } impl Engine { @@ -55,14 +111,14 @@ impl Engine { /// /// This will spawn two tokio tasks for the live sync coordination and gossip actors, and a /// thread for the [`iroh_docs::actor::SyncHandle`]. - pub(crate) fn spawn( + pub(crate) async fn spawn( endpoint: Endpoint, gossip: Gossip, replica_store: iroh_docs::store::Store, bao_store: B, downloader: Downloader, - default_author: AuthorId, - ) -> Self { + default_author_storage: DefaultAuthorStorage, + ) -> anyhow::Result { let (live_actor_tx, to_live_actor_recv) = mpsc::channel(ACTOR_CHANNEL_CAP); let (to_gossip_actor, to_gossip_actor_recv) = mpsc::channel(ACTOR_CHANNEL_CAP); let me = endpoint.node_id().fmt_short(); @@ -98,14 +154,17 @@ impl Engine { .instrument(error_span!("sync", %me)), ); - Self { + let default_author = default_author_storage.load(&sync).await?; + + Ok(Self { endpoint, sync, to_live_actor: live_actor_tx, actor_handle: actor_handle.into(), content_status_cb, default_author, - } + default_author_storage: Arc::new(default_author_storage), + }) } /// Start to sync a document. diff --git a/iroh/src/docs_engine/rpc.rs b/iroh/src/docs_engine/rpc.rs index 9009bef113..944e6b0318 100644 --- a/iroh/src/docs_engine/rpc.rs +++ b/iroh/src/docs_engine/rpc.rs @@ -8,9 +8,10 @@ use tokio_stream::StreamExt; use crate::client::docs::ShareMode; use crate::rpc_protocol::{ - AuthorGetDefaultRequest, AuthorGetDefaultResponse, AuthorDeleteRequest, AuthorDeleteResponse, - AuthorExportRequest, AuthorExportResponse, AuthorImportRequest, AuthorImportResponse, - DocGetSyncPeersRequest, DocGetSyncPeersResponse, + AuthorDeleteRequest, AuthorDeleteResponse, AuthorExportRequest, AuthorExportResponse, + AuthorGetDefaultRequest, AuthorGetDefaultResponse, AuthorImportRequest, AuthorImportResponse, + AuthorSetDefaultRequest, AuthorSetDefaultResponse, DocGetSyncPeersRequest, + DocGetSyncPeersResponse, }; use crate::{ docs_engine::Engine, @@ -51,6 +52,16 @@ impl Engine { } } + pub async fn author_set_default( + &self, + req: AuthorSetDefaultRequest, + ) -> RpcResult { + self.default_author_storage + .save(&self.sync, req.author_id) + .await?; + Ok(AuthorSetDefaultResponse) + } + pub fn author_list( &self, _req: AuthorListRequest, diff --git a/iroh/src/node.rs b/iroh/src/node.rs index bf3b1bb814..9b0517b8cf 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -455,6 +455,21 @@ mod tests { assert!(iroh.is_ok()); } + // check that the default author can be set manually and is persisted. + let default_author = { + let iroh = Node::persistent(iroh_root).await?.spawn().await?; + let author = iroh.authors.create().await?; + iroh.authors.set_default(author).await?; + iroh.shutdown().await?; + author + }; + { + let iroh = Node::persistent(iroh_root).await?.spawn().await?; + let author = iroh.authors.default().await?; + assert_eq!(author, default_author); + iroh.shutdown().await?; + } + Ok(()) } } diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index 2969733580..a9acf58914 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -32,13 +32,10 @@ use tracing::{debug, error, error_span, info, trace, warn, Instrument}; use crate::{ client::RPC_ALPN, - docs_engine::Engine, node::NodeInner, + docs_engine::{DefaultAuthorStorage, Engine}, rpc_protocol::{Request, Response, RpcService}, - util::{ - fs::{load_default_author, load_secret_key}, - path::IrohPaths, - }, + util::{fs::load_secret_key, path::IrohPaths}, }; use super::{rpc, rpc_status::RpcStatus, Node}; @@ -369,7 +366,7 @@ where /// This will create the underlying network server and spawn a tokio task accepting /// connections. The returned [`Node`] can be used to control the task as well as /// get information about it. - pub async fn spawn(mut self) -> Result> { + pub async fn spawn(self) -> Result> { trace!("spawning node"); let lp = LocalPoolHandle::new(num_cpus::get()); @@ -436,12 +433,12 @@ where let downloader = Downloader::new(self.blobs_store.clone(), endpoint.clone(), lp.clone()); // load or create the default author for documents - let default_author = match self.storage { + let default_author_storage = match self.storage { StorageConfig::Persistent(ref root) => { let path = IrohPaths::DefaultAuthor.with_root(root); - load_default_author(path, &mut self.docs_store).await? + DefaultAuthorStorage::Persistent(path) } - StorageConfig::Mem => self.docs_store.new_author(&mut rand::thread_rng())?.id(), + StorageConfig::Mem => DefaultAuthorStorage::Mem, }; // spawn the docs engine @@ -451,8 +448,9 @@ where self.docs_store, self.blobs_store.clone(), downloader.clone(), - default_author, - ); + default_author_storage, + ) + .await?; let sync_db = sync.sync.clone(); let gc_task = if let GcPolicy::Interval(gc_period) = self.gc_policy { diff --git a/iroh/src/node/rpc.rs b/iroh/src/node/rpc.rs index 791a28dc75..ba03e10486 100644 --- a/iroh/src/node/rpc.rs +++ b/iroh/src/node/rpc.rs @@ -167,6 +167,12 @@ impl Handler { }) .await } + AuthorSetDefault(msg) => { + chan.rpc(msg, handler, |handler, req| async move { + handler.inner.sync.author_set_default(req).await + }) + .await + } DocOpen(msg) => { chan.rpc(msg, handler, |handler, req| async move { handler.inner.sync.doc_open(req).await diff --git a/iroh/src/rpc_protocol.rs b/iroh/src/rpc_protocol.rs index aa1f29c7b6..7bfb5d60b3 100644 --- a/iroh/src/rpc_protocol.rs +++ b/iroh/src/rpc_protocol.rs @@ -453,6 +453,20 @@ pub struct AuthorGetDefaultResponse { pub author_id: AuthorId, } +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorSetDefaultRequest { + /// The id of the author + pub author_id: AuthorId, +} + +impl RpcMsg for AuthorSetDefaultRequest { + type Response = RpcResult; +} + +/// Response for [`AuthorGetDefaultRequest`] +#[derive(Serialize, Deserialize, Debug)] +pub struct AuthorSetDefaultResponse; + /// Delete an author #[derive(Serialize, Deserialize, Debug)] pub struct AuthorDeleteRequest { @@ -1086,6 +1100,7 @@ pub enum Request { AuthorList(AuthorListRequest), AuthorCreate(AuthorCreateRequest), AuthorGetDefault(AuthorGetDefaultRequest), + AuthorSetDefault(AuthorSetDefaultRequest), AuthorImport(AuthorImportRequest), AuthorExport(AuthorExportRequest), AuthorDelete(AuthorDeleteRequest), @@ -1147,6 +1162,7 @@ pub enum Response { AuthorList(RpcResult), AuthorCreate(RpcResult), AuthorGetDefault(AuthorGetDefaultResponse), + AuthorSetDefault(RpcResult), AuthorImport(RpcResult), AuthorExport(RpcResult), AuthorDelete(RpcResult), diff --git a/iroh/src/util/fs.rs b/iroh/src/util/fs.rs index d95cdf0629..d1af9650b0 100644 --- a/iroh/src/util/fs.rs +++ b/iroh/src/util/fs.rs @@ -3,12 +3,10 @@ use std::{ borrow::Cow, fs::read_dir, path::{Component, Path, PathBuf}, - str::FromStr, }; use anyhow::{bail, Context}; use bytes::Bytes; -use iroh_docs::AuthorId; use iroh_net::key::SecretKey; use tokio::io::AsyncWriteExt; use walkdir::WalkDir; @@ -121,35 +119,6 @@ pub fn relative_canonicalized_path_to_string(path: impl AsRef) -> anyhow:: canonicalized_path_to_string(path, true) } -/// Load the default author public key from a path, and check that it is present in the `docs_store`. -/// -/// If `path` does not exist, a new author keypair is created and persisted in the docs store, and -/// the public key is written to `path`, in base32 encoding. -/// -/// If `path` does exist, but does not contain an ed25519 public key in base32 encoding, an error -/// is returned. -/// -/// If `path` exists and is a valid author public key, but its secret key does not exist in the -/// docs store, an error is returned. -pub async fn load_default_author( - path: PathBuf, - docs_store: &mut iroh_docs::store::fs::Store, -) -> anyhow::Result { - if path.exists() { - let data = tokio::fs::read_to_string(&path).await?; - let author_id = AuthorId::from_str(&data)?; - if docs_store.get_author(&author_id)?.is_none() { - bail!("The default author is missing from the docs store. To recover, delete the file `{}`. Then iroh will create a new default author.", path.to_string_lossy()) - } - Ok(author_id) - } else { - let author_id = docs_store.new_author(&mut rand::thread_rng())?.id(); - docs_store.flush()?; - tokio::fs::write(path, author_id.to_string()).await?; - Ok(author_id) - } -} - /// Loads a [`SecretKey`] from the provided file, or stores a newly generated one /// at the given location. pub async fn load_secret_key(key_path: PathBuf) -> anyhow::Result { From e54fa90c6b7a1639b339f54f83f34618bf619966 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 17 May 2024 14:56:42 +0200 Subject: [PATCH 14/24] fix: actually use new default author when changing it --- iroh/src/client/authors.rs | 1 + iroh/src/docs_engine.rs | 152 ++++++++++++++++++++++-------------- iroh/src/docs_engine/rpc.rs | 11 +-- 3 files changed, 97 insertions(+), 67 deletions(-) diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index eac2a55496..e794f60b9b 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -125,6 +125,7 @@ mod tests { let authors: Vec<_> = node.authors.list().await?.try_collect().await?; assert_eq!(authors.len(), 2); + assert!(node.authors.default().await? != author_id); node.authors.set_default(author_id).await?; assert_eq!(node.authors.default().await?, author_id); diff --git a/iroh/src/docs_engine.rs b/iroh/src/docs_engine.rs index 1227c8d4bc..212a356c98 100644 --- a/iroh/src/docs_engine.rs +++ b/iroh/src/docs_engine.rs @@ -3,7 +3,11 @@ //! [`iroh_docs::Replica`] is also called documents here. use std::path::PathBuf; -use std::{io, str::FromStr, sync::Arc}; +use std::{ + io, + str::FromStr, + sync::{Arc, RwLock}, +}; use anyhow::{bail, Result}; use futures_lite::{Stream, StreamExt}; @@ -48,62 +52,7 @@ pub struct Engine { actor_handle: SharedAbortingJoinHandle<()>, #[debug("ContentStatusCallback")] content_status_cb: ContentStatusCallback, - default_author: AuthorId, - default_author_storage: Arc, -} - -/// Where to persist the default author. -/// -/// If set to `Mem`, a new author will be created in the docs store before spawning the sync -/// engine. Changing the default author will not be persisted. -/// -/// If set to `Persistent`, the default author will be loaded from and persisted to the specified -/// path (as base32 encoded string of the author's public key). -#[derive(Debug)] -pub enum DefaultAuthorStorage { - Mem, - Persistent(PathBuf), -} - -impl DefaultAuthorStorage { - pub async fn load(&self, docs_store: &SyncHandle) -> anyhow::Result { - match self { - Self::Mem => { - let author = Author::new(&mut rand::thread_rng()); - let author_id = author.id(); - docs_store.import_author(author).await?; - Ok(author_id) - } - Self::Persistent(ref path) => { - if path.exists() { - let data = tokio::fs::read_to_string(path).await?; - let author_id = AuthorId::from_str(&data)?; - if docs_store.export_author(author_id).await?.is_none() { - bail!("The default author is missing from the docs store. To recover, delete the file `{}`. Then iroh will create a new default author.", path.to_string_lossy()) - } - Ok(author_id) - } else { - let author = Author::new(&mut rand::thread_rng()); - let author_id = author.id(); - docs_store.import_author(author).await?; - tokio::fs::write(path, author_id.to_string()).await?; - Ok(author_id) - } - } - } - } - pub async fn save(&self, docs_store: &SyncHandle, author_id: AuthorId) -> anyhow::Result<()> { - if docs_store.export_author(author_id).await?.is_none() { - bail!("The author does not exist"); - } - match self { - Self::Mem => {} - Self::Persistent(ref path) => { - tokio::fs::write(path, author_id.to_string()).await?; - } - } - Ok(()) - } + default_author: Arc, } impl Engine { @@ -154,7 +103,7 @@ impl Engine { .instrument(error_span!("sync", %me)), ); - let default_author = default_author_storage.load(&sync).await?; + let default_author = DefaultAuthor::load(default_author_storage, &sync).await?; Ok(Self { endpoint, @@ -162,8 +111,7 @@ impl Engine { to_live_actor: live_actor_tx, actor_handle: actor_handle.into(), content_status_cb, - default_author, - default_author_storage: Arc::new(default_author_storage), + default_author: Arc::new(default_author), }) } @@ -336,3 +284,87 @@ impl LiveEvent { }) } } + +/// Where to persist the default author. +/// +/// If set to `Mem`, a new author will be created in the docs store before spawning the sync +/// engine. Changing the default author will not be persisted. +/// +/// If set to `Persistent`, the default author will be loaded from and persisted to the specified +/// path (as base32 encoded string of the author's public key). +#[derive(Debug)] +pub enum DefaultAuthorStorage { + Mem, + Persistent(PathBuf), +} + +impl DefaultAuthorStorage { + pub async fn load(&self, docs_store: &SyncHandle) -> anyhow::Result { + match self { + Self::Mem => { + let author = Author::new(&mut rand::thread_rng()); + let author_id = author.id(); + docs_store.import_author(author).await?; + Ok(author_id) + } + Self::Persistent(ref path) => { + if path.exists() { + let data = tokio::fs::read_to_string(path).await?; + let author_id = AuthorId::from_str(&data)?; + if docs_store.export_author(author_id).await?.is_none() { + bail!("The default author is missing from the docs store. To recover, delete the file `{}`. Then iroh will create a new default author.", path.to_string_lossy()) + } + Ok(author_id) + } else { + let author = Author::new(&mut rand::thread_rng()); + let author_id = author.id(); + docs_store.import_author(author).await?; + tokio::fs::write(path, author_id.to_string()).await?; + Ok(author_id) + } + } + } + } + pub async fn persist( + &self, + docs_store: &SyncHandle, + author_id: AuthorId, + ) -> anyhow::Result<()> { + if docs_store.export_author(author_id).await?.is_none() { + bail!("The author does not exist"); + } + match self { + Self::Mem => { + // persistence is not possible for the mem storage so this is a noop. + } + Self::Persistent(ref path) => { + tokio::fs::write(path, author_id.to_string()).await?; + } + } + Ok(()) + } +} + +#[derive(Debug)] +struct DefaultAuthor { + value: RwLock, + storage: DefaultAuthorStorage, +} + +impl DefaultAuthor { + async fn load(storage: DefaultAuthorStorage, docs_store: &SyncHandle) -> Result { + let value = storage.load(docs_store).await?; + Ok(Self { + value: RwLock::new(value), + storage, + }) + } + fn get(&self) -> AuthorId { + *self.value.read().unwrap() + } + async fn set(&self, author_id: AuthorId, docs_store: &SyncHandle) -> Result<()> { + self.storage.persist(&docs_store, author_id).await?; + *self.value.write().unwrap() = author_id; + Ok(()) + } +} diff --git a/iroh/src/docs_engine/rpc.rs b/iroh/src/docs_engine/rpc.rs index 944e6b0318..76f2afd761 100644 --- a/iroh/src/docs_engine/rpc.rs +++ b/iroh/src/docs_engine/rpc.rs @@ -47,18 +47,15 @@ impl Engine { } pub fn author_default(&self, _req: AuthorGetDefaultRequest) -> AuthorGetDefaultResponse { - AuthorGetDefaultResponse { - author_id: self.default_author, - } + let author_id = self.default_author.get(); + AuthorGetDefaultResponse { author_id } } pub async fn author_set_default( &self, req: AuthorSetDefaultRequest, ) -> RpcResult { - self.default_author_storage - .save(&self.sync, req.author_id) - .await?; + self.default_author.set(req.author_id, &self.sync).await?; Ok(AuthorSetDefaultResponse) } @@ -94,7 +91,7 @@ impl Engine { } pub async fn author_delete(&self, req: AuthorDeleteRequest) -> RpcResult { - if req.author == self.default_author { + if req.author == self.default_author.get() { return Err(anyhow!("Deleting the default author is not supported").into()); } self.sync.delete_author(req.author).await?; From f3ec119b91691bcd33bda2b2de150bdd87edcdac Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 17 May 2024 14:58:54 +0200 Subject: [PATCH 15/24] fix test --- iroh/src/node.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 9b0517b8cf..a66de219d2 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -453,6 +453,7 @@ mod tests { tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)).await?; let iroh = Node::persistent(iroh_root).await?.spawn().await; assert!(iroh.is_ok()); + iroh?.shutdown().await?; } // check that the default author can be set manually and is persisted. From 82ad6457849542fa476b10fa62aef21964e01bb3 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Fri, 17 May 2024 23:33:43 +0200 Subject: [PATCH 16/24] fix: shutdown the blobs store if node spawn fails --- iroh/src/docs_engine.rs | 24 ++++++++------- iroh/src/node.rs | 64 ++++++++++++++++++++++------------------ iroh/src/node/builder.rs | 13 ++++++++ 3 files changed, 62 insertions(+), 39 deletions(-) diff --git a/iroh/src/docs_engine.rs b/iroh/src/docs_engine.rs index 212a356c98..389526833f 100644 --- a/iroh/src/docs_engine.rs +++ b/iroh/src/docs_engine.rs @@ -103,7 +103,15 @@ impl Engine { .instrument(error_span!("sync", %me)), ); - let default_author = DefaultAuthor::load(default_author_storage, &sync).await?; + let default_author = match DefaultAuthor::load(default_author_storage, &sync).await { + Ok(author) => author, + Err(err) => { + // If loading the default author failed, make sure to shutdown the sync actor before + // returning. + sync.shutdown().await?; + return Err(err); + } + }; Ok(Self { endpoint, @@ -325,14 +333,7 @@ impl DefaultAuthorStorage { } } } - pub async fn persist( - &self, - docs_store: &SyncHandle, - author_id: AuthorId, - ) -> anyhow::Result<()> { - if docs_store.export_author(author_id).await?.is_none() { - bail!("The author does not exist"); - } + pub async fn persist(&self, author_id: AuthorId) -> anyhow::Result<()> { match self { Self::Mem => { // persistence is not possible for the mem storage so this is a noop. @@ -363,7 +364,10 @@ impl DefaultAuthor { *self.value.read().unwrap() } async fn set(&self, author_id: AuthorId, docs_store: &SyncHandle) -> Result<()> { - self.storage.persist(&docs_store, author_id).await?; + if docs_store.export_author(author_id).await?.is_none() { + bail!("The author does not exist"); + } + self.storage.persist(author_id).await?; *self.value.write().unwrap() = author_id; Ok(()) } diff --git a/iroh/src/node.rs b/iroh/src/node.rs index a66de219d2..d97b02b939 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -404,39 +404,41 @@ mod tests { let _guard = iroh_test::logging::setup(); - let iroh_root_dir = tempfile::TempDir::new()?; + let iroh_root_dir = tempfile::TempDir::new().unwrap(); let iroh_root = iroh_root_dir.path(); // check that the default author exists and cannot be deleted. let default_author = { - let iroh = Node::persistent(iroh_root).await?.spawn().await?; - let author = iroh.authors.default().await?; - assert!(iroh.authors.export(author).await?.is_some()); + let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + let author = iroh.authors.default().await.unwrap(); + assert!(iroh.authors.export(author).await.unwrap().is_some()); assert!(iroh.authors.delete(author).await.is_err()); - iroh.shutdown().await?; + iroh.shutdown().await.unwrap(); author }; // check that the default author is persisted across restarts. { - let iroh = Node::persistent(iroh_root).await?.spawn().await?; - let author = iroh.authors.default().await?; + let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + let author = iroh.authors.default().await.unwrap(); assert_eq!(author, default_author); - assert!(iroh.authors.export(author).await?.is_some()); + assert!(iroh.authors.export(author).await.unwrap().is_some()); assert!(iroh.authors.delete(author).await.is_err()); - iroh.shutdown().await?; + iroh.shutdown().await.unwrap(); }; // check that a new default author is created if the default author file is deleted // manually. let default_author = { - tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)).await?; - let iroh = Node::persistent(iroh_root).await?.spawn().await?; - let author = iroh.authors.default().await?; + tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)) + .await + .unwrap(); + let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + let author = iroh.authors.default().await.unwrap(); assert!(author != default_author); - assert!(iroh.authors.export(author).await?.is_some()); + assert!(iroh.authors.export(author).await.unwrap().is_some()); assert!(iroh.authors.delete(author).await.is_err()); - iroh.shutdown().await?; + iroh.shutdown().await.unwrap(); author }; @@ -444,31 +446,35 @@ mod tests { { let mut docs_store = iroh_docs::store::fs::Store::persistent( IrohPaths::DocsDatabase.with_root(iroh_root), - )?; - docs_store.delete_author(default_author)?; - docs_store.flush()?; + ) + .unwrap(); + docs_store.delete_author(default_author).unwrap(); + docs_store.flush().unwrap(); drop(docs_store); - let iroh = Node::persistent(iroh_root).await?.spawn().await; + let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await; assert!(iroh.is_err()); - tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)).await?; - let iroh = Node::persistent(iroh_root).await?.spawn().await; + tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)) + .await + .unwrap(); + drop(iroh); + let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await; assert!(iroh.is_ok()); - iroh?.shutdown().await?; + iroh.unwrap().shutdown().await.unwrap(); } // check that the default author can be set manually and is persisted. let default_author = { - let iroh = Node::persistent(iroh_root).await?.spawn().await?; - let author = iroh.authors.create().await?; - iroh.authors.set_default(author).await?; - iroh.shutdown().await?; + let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + let author = iroh.authors.create().await.unwrap(); + iroh.authors.set_default(author).await.unwrap(); + assert_eq!(iroh.authors.default().await.unwrap(), author); + iroh.shutdown().await.unwrap(); author }; { - let iroh = Node::persistent(iroh_root).await?.spawn().await?; - let author = iroh.authors.default().await?; - assert_eq!(author, default_author); - iroh.shutdown().await?; + let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + assert_eq!(iroh.authors.default().await.unwrap(), default_author); + iroh.shutdown().await.unwrap(); } Ok(()) diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index a9acf58914..e54182c81a 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -367,6 +367,19 @@ where /// connections. The returned [`Node`] can be used to control the task as well as /// get information about it. pub async fn spawn(self) -> Result> { + // We clone the blob store to shut it down in case the node fails to spawn. + let blobs_store = self.blobs_store.clone(); + let docs_store = self.docs_store.clone(); + match self.spawn_inner().await { + Ok(node) => Ok(node), + Err(err) => { + blobs_store.shutdown().await; + Err(err) + } + } + } + + async fn spawn_inner(self) -> Result> { trace!("spawning node"); let lp = LocalPoolHandle::new(num_cpus::get()); From 004bf5cc3be05cb046101a419d117562684bd6e5 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Sat, 18 May 2024 13:06:53 +0200 Subject: [PATCH 17/24] fixup --- iroh/src/node/builder.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index e54182c81a..4628c2e603 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -369,7 +369,6 @@ where pub async fn spawn(self) -> Result> { // We clone the blob store to shut it down in case the node fails to spawn. let blobs_store = self.blobs_store.clone(); - let docs_store = self.docs_store.clone(); match self.spawn_inner().await { Ok(node) => Ok(node), Err(err) => { From c36326ff0642761a9b6e5a35bc16bf4c7bd56707 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Tue, 21 May 2024 11:02:55 +0200 Subject: [PATCH 18/24] better error messages --- iroh/src/docs_engine.rs | 27 ++++++++++++++++++++++----- iroh/src/node.rs | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/iroh/src/docs_engine.rs b/iroh/src/docs_engine.rs index 389526833f..b64870fda3 100644 --- a/iroh/src/docs_engine.rs +++ b/iroh/src/docs_engine.rs @@ -9,7 +9,7 @@ use std::{ sync::{Arc, RwLock}, }; -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; use futures_lite::{Stream, StreamExt}; use iroh_blobs::downloader::Downloader; use iroh_blobs::{store::EntryStatus, Hash}; @@ -317,8 +317,18 @@ impl DefaultAuthorStorage { } Self::Persistent(ref path) => { if path.exists() { - let data = tokio::fs::read_to_string(path).await?; - let author_id = AuthorId::from_str(&data)?; + let data = tokio::fs::read_to_string(path).await.with_context(|| { + format!( + "Failed to read the default author file at `{}`", + path.to_string_lossy() + ) + })?; + let author_id = AuthorId::from_str(&data).with_context(|| { + format!( + "Failed to parse the default author from `{}`", + path.to_string_lossy() + ) + })?; if docs_store.export_author(author_id).await?.is_none() { bail!("The default author is missing from the docs store. To recover, delete the file `{}`. Then iroh will create a new default author.", path.to_string_lossy()) } @@ -327,7 +337,7 @@ impl DefaultAuthorStorage { let author = Author::new(&mut rand::thread_rng()); let author_id = author.id(); docs_store.import_author(author).await?; - tokio::fs::write(path, author_id.to_string()).await?; + self.persist(author_id).await?; Ok(author_id) } } @@ -339,7 +349,14 @@ impl DefaultAuthorStorage { // persistence is not possible for the mem storage so this is a noop. } Self::Persistent(ref path) => { - tokio::fs::write(path, author_id.to_string()).await?; + tokio::fs::write(path, author_id.to_string()) + .await + .with_context(|| { + format!( + "Failed to write the default author to `{}`", + path.to_string_lossy() + ) + })?; } } Ok(()) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index d97b02b939..1fa827bc61 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -409,7 +409,12 @@ mod tests { // check that the default author exists and cannot be deleted. let default_author = { - let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + let iroh = Node::persistent(iroh_root) + .await + .unwrap() + .spawn() + .await + .unwrap(); let author = iroh.authors.default().await.unwrap(); assert!(iroh.authors.export(author).await.unwrap().is_some()); assert!(iroh.authors.delete(author).await.is_err()); @@ -419,7 +424,12 @@ mod tests { // check that the default author is persisted across restarts. { - let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + let iroh = Node::persistent(iroh_root) + .await + .unwrap() + .spawn() + .await + .unwrap(); let author = iroh.authors.default().await.unwrap(); assert_eq!(author, default_author); assert!(iroh.authors.export(author).await.unwrap().is_some()); @@ -433,7 +443,12 @@ mod tests { tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)) .await .unwrap(); - let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + let iroh = Node::persistent(iroh_root) + .await + .unwrap() + .spawn() + .await + .unwrap(); let author = iroh.authors.default().await.unwrap(); assert!(author != default_author); assert!(iroh.authors.export(author).await.unwrap().is_some()); @@ -464,7 +479,12 @@ mod tests { // check that the default author can be set manually and is persisted. let default_author = { - let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + let iroh = Node::persistent(iroh_root) + .await + .unwrap() + .spawn() + .await + .unwrap(); let author = iroh.authors.create().await.unwrap(); iroh.authors.set_default(author).await.unwrap(); assert_eq!(iroh.authors.default().await.unwrap(), author); @@ -472,7 +492,12 @@ mod tests { author }; { - let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await.unwrap(); + let iroh = Node::persistent(iroh_root) + .await + .unwrap() + .spawn() + .await + .unwrap(); assert_eq!(iroh.authors.default().await.unwrap(), default_author); iroh.shutdown().await.unwrap(); } From 9bd9ef53ca4b326959974068b7d36577dfde2e67 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 22 May 2024 13:36:40 +0200 Subject: [PATCH 19/24] debug --- iroh/src/node.rs | 1 + iroh/src/node/builder.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 1fa827bc61..82a18596c1 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -467,6 +467,7 @@ mod tests { docs_store.flush().unwrap(); drop(docs_store); let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await; + dbg!(&iroh); assert!(iroh.is_err()); tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)) .await diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index 4628c2e603..128bb7348c 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -372,6 +372,7 @@ where match self.spawn_inner().await { Ok(node) => Ok(node), Err(err) => { + debug!("failed to spawn node, shutting down blobs store"); blobs_store.shutdown().await; Err(err) } From 0d56232d4c3ca3c88cafe120e919c348e6590194 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 22 May 2024 13:47:53 +0200 Subject: [PATCH 20/24] maybe gha? --- iroh/src/node/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index 128bb7348c..e29c96377a 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -372,7 +372,7 @@ where match self.spawn_inner().await { Ok(node) => Ok(node), Err(err) => { - debug!("failed to spawn node, shutting down blobs store"); + debug!("failed to spawn node, shutting down"); blobs_store.shutdown().await; Err(err) } From 5ccdf0b1a272ef82bb523eb487149fdfe0bc28cd Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 22 May 2024 13:53:32 +0200 Subject: [PATCH 21/24] fixup rebase --- iroh/src/node/builder.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh/src/node/builder.rs b/iroh/src/node/builder.rs index e29c96377a..fe226d74a5 100644 --- a/iroh/src/node/builder.rs +++ b/iroh/src/node/builder.rs @@ -32,8 +32,8 @@ use tracing::{debug, error, error_span, info, trace, warn, Instrument}; use crate::{ client::RPC_ALPN, - node::NodeInner, docs_engine::{DefaultAuthorStorage, Engine}, + node::NodeInner, rpc_protocol::{Request, Response, RpcService}, util::{fs::load_secret_key, path::IrohPaths}, }; @@ -379,7 +379,7 @@ where } } - async fn spawn_inner(self) -> Result> { + async fn spawn_inner(mut self) -> Result> { trace!("spawning node"); let lp = LocalPoolHandle::new(num_cpus::get()); From a6f014a4db077625df1acc6a624eb80dc6c96dda Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 23 May 2024 01:44:55 +0200 Subject: [PATCH 22/24] try if macos just needs more time --- iroh/src/node.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 82a18596c1..d202c2753f 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -469,6 +469,12 @@ mod tests { let iroh = Node::persistent(iroh_root).await.unwrap().spawn().await; dbg!(&iroh); assert!(iroh.is_err()); + + // somehow the blob store is not shutdown correctly (yet?) on macos. + // so we give it some time until we find a proper fix. + #[cfg(target_os = "macos")] + tokio::time::sleep(Duration::from_secs(1)).await; + tokio::fs::remove_file(IrohPaths::DefaultAuthor.with_root(iroh_root)) .await .unwrap(); From 62423aae1de74cdca18936a119790a035443130a Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Thu, 23 May 2024 01:57:08 +0200 Subject: [PATCH 23/24] fix docs --- iroh/src/client/authors.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iroh/src/client/authors.rs b/iroh/src/client/authors.rs index e794f60b9b..b695b3da7c 100644 --- a/iroh/src/client/authors.rs +++ b/iroh/src/client/authors.rs @@ -48,8 +48,8 @@ where /// /// If the author does not exist, an error is returned. /// - /// This is a noop on memory nodes. On peristent node, the author id will be saved to a file in - /// the data directory, and reloaded after a node restart. + /// On a persistent node, the author id will be saved to a file in the data directory and + /// reloaded after a restart. pub async fn set_default(&self, author_id: AuthorId) -> Result<()> { self.rpc .rpc(AuthorSetDefaultRequest { author_id }) From 0b8188ce0375622fe7e015b6d542559d3b5b6efc Mon Sep 17 00:00:00 2001 From: Franz Heinzmann Date: Thu, 23 May 2024 09:59:30 +0200 Subject: [PATCH 24/24] fix(iroh-cli): store console files in subdirectory of iroh data dir ## Description We used to store the console files (history and current author) in a subdirectory `console` under the iroh data dir. This got lost in some refacorings, and the console files currently are created in the iroh data directory directly. This is bad because the iroh data directory is a single namespace, and we should use it cautiously. This PR goes back to using a `console` subdirectory. Existing files are migrated on first start of the console. Based on #2299 because this already made the environment init functions async. ## Breaking Changes ## Notes & open questions ## Change checklist - [ ] Self-review. - [ ] Documentation updates if relevant. - [ ] Tests if relevant. - [ ] All breaking changes documented. --- iroh-cli/src/commands/console.rs | 2 +- iroh-cli/src/config.rs | 94 ++++++++++++++++++-------------- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/iroh-cli/src/commands/console.rs b/iroh-cli/src/commands/console.rs index 4d8a25555d..df10126d75 100644 --- a/iroh-cli/src/commands/console.rs +++ b/iroh-cli/src/commands/console.rs @@ -53,7 +53,7 @@ impl Repl { pub fn run(self) -> anyhow::Result<()> { let mut rl = DefaultEditor::with_config(Config::builder().check_cursor_position(true).build())?; - let history_path = ConsolePaths::History.with_root(self.env.iroh_data_dir()); + let history_path = ConsolePaths::History.with_iroh_data_dir(self.env.iroh_data_dir()); rl.load_history(&history_path).ok(); loop { // prepare a channel to receive a signal from the main thread when a command completed diff --git a/iroh-cli/src/config.rs b/iroh-cli/src/config.rs index 103aa95c87..861c2ec5ad 100644 --- a/iroh-cli/src/config.rs +++ b/iroh-cli/src/config.rs @@ -1,7 +1,7 @@ //! Configuration for the iroh CLI. use std::{ - env, fmt, + env, net::SocketAddr, path::{Path, PathBuf}, str::FromStr, @@ -21,6 +21,7 @@ use iroh::{ use parking_lot::RwLock; use quic_rpc::ServiceConnection; use serde::{Deserialize, Serialize}; +use tracing::warn; const ENV_AUTHOR: &str = "IROH_AUTHOR"; const ENV_DOC: &str = "IROH_DOC"; @@ -30,47 +31,20 @@ const ENV_FILE_RUST_LOG: &str = "IROH_FILE_RUST_LOG"; /// CONFIG_FILE_NAME is the name of the optional config file located in the iroh home directory pub(crate) const CONFIG_FILE_NAME: &str = "iroh.config.toml"; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, strum::AsRefStr, strum::EnumString, strum::Display)] pub(crate) enum ConsolePaths { - DefaultAuthor, + #[strum(serialize = "current-author")] + CurrentAuthor, + #[strum(serialize = "history")] History, } -impl From<&ConsolePaths> for &'static str { - fn from(value: &ConsolePaths) -> Self { - match value { - ConsolePaths::DefaultAuthor => "default_author.pubkey", - ConsolePaths::History => "history", - } - } -} -impl FromStr for ConsolePaths { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - Ok(match s { - "default_author.pubkey" => Self::DefaultAuthor, - "history" => Self::History, - _ => bail!("unknown file or directory"), - }) - } -} - -impl fmt::Display for ConsolePaths { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s: &str = self.into(); - write!(f, "{s}") - } -} -impl AsRef for ConsolePaths { - fn as_ref(&self) -> &Path { - let s: &str = self.into(); - Path::new(s) - } -} - impl ConsolePaths { - pub fn with_root(self, root: impl AsRef) -> PathBuf { - PathBuf::from(root.as_ref()).join(self) + fn root(iroh_data_dir: impl AsRef) -> PathBuf { + PathBuf::from(iroh_data_dir.as_ref()).join("console") + } + pub fn with_iroh_data_dir(self, iroh_data_dir: impl AsRef) -> PathBuf { + Self::root(iroh_data_dir).join(self.as_ref()) } } @@ -163,6 +137,18 @@ impl ConsoleEnv { iroh_data_dir: PathBuf, iroh: &Iroh, ) -> Result { + let console_data_dir = ConsolePaths::root(&iroh_data_dir); + tokio::fs::create_dir_all(&console_data_dir) + .await + .with_context(|| { + format!( + "failed to create console data directory at `{}`", + console_data_dir.to_string_lossy() + ) + })?; + + Self::migrate_console_files_016_017(&iroh_data_dir).await?; + let configured_author = Self::get_console_default_author(&iroh_data_dir)?; let author = env_author(configured_author, iroh).await?; let env = ConsoleEnvInner { @@ -189,8 +175,8 @@ impl ConsoleEnv { Ok(Self(Arc::new(RwLock::new(env)))) } - fn get_console_default_author(root: &Path) -> anyhow::Result> { - let author_path = ConsolePaths::DefaultAuthor.with_root(root); + fn get_console_default_author(iroh_data_root: &Path) -> anyhow::Result> { + let author_path = ConsolePaths::CurrentAuthor.with_iroh_data_dir(iroh_data_root); if let Ok(s) = std::fs::read_to_string(&author_path) { let author = AuthorId::from_str(&s).with_context(|| { format!( @@ -219,7 +205,7 @@ impl ConsoleEnv { /// Will error if not running in the Iroh console. /// Will persist to a file in the Iroh data dir otherwise. pub(crate) fn set_author(&self, author: AuthorId) -> anyhow::Result<()> { - let author_path = ConsolePaths::DefaultAuthor.with_root(self.iroh_data_dir()); + let author_path = ConsolePaths::CurrentAuthor.with_iroh_data_dir(self.iroh_data_dir()); let mut inner = self.0.write(); if !inner.is_console { bail!("Switching the author is only supported within the Iroh console, not on the command line"); @@ -262,6 +248,34 @@ impl ConsoleEnv { let inner = self.0.read(); inner.author } + + pub(crate) async fn migrate_console_files_016_017(iroh_data_dir: &Path) -> Result<()> { + // In iroh up to 0.16, we stored console settings directly in the data directory. Starting + // from 0.17, they live in a subdirectory and have new paths. + let old_current_author = iroh_data_dir.join("default_author.pubkey"); + if old_current_author.is_file() { + if let Err(err) = tokio::fs::rename( + &old_current_author, + ConsolePaths::CurrentAuthor.with_iroh_data_dir(iroh_data_dir), + ) + .await + { + warn!(path=%old_current_author.to_string_lossy(), "failed to migrate the console's current author file: {err}"); + } + } + let old_history = iroh_data_dir.join("history"); + if old_history.is_file() { + if let Err(err) = tokio::fs::rename( + &old_history, + ConsolePaths::History.with_iroh_data_dir(iroh_data_dir), + ) + .await + { + warn!(path=%old_history.to_string_lossy(), "failed to migrate the console's history file: {err}"); + } + } + Ok(()) + } } async fn env_author>(