fix(examples): Adjust and add examples (#1968)

## Description

Fixes incorrect information in our `iroh/examples`

Also adds an example that shows the most basic way to use iroh to fetch
content.

related to #1929 

## Notes & open questions

Unfortunately, we don't seem to have very simple APIs for fetching (that
isn't just exporting the content to an external path). Working through
this.

Also adds:
- `BlobDownloadProgress` - this takes a stream of `DownloadProgress` and
puts it into a form that will poll the stream by calling `finish`
- adjusts the `blob_download0` function to not eat certain errors

## Change checklist

- [x] Self-review.
- [x] Documentation updates if relevant.

iroh-net/Cargo.toml

@@ -126,3 +126,16 @@ metrics = ["iroh-metrics/metrics"]
 [[bin]]
 name = "derper"
 required-features = ["derper"]
+
+[[example]]
+name = "listen"
+
+[[example]]
+name = "connect"
+
+[[example]]
+name = "listen-unreliable"
+
+[[example]]
+name = "connect-unreliable"
+

iroh-net/examples/connect-unreliable.rs

@@ -0,0 +1,84 @@
+//! The smallest example showing how to use iroh-net and `MagicEndpoint` to connect to a remote node and pass bytes using unreliable datagrams.
+//!
+//! We use the node ID (the PublicKey of the remote node), the direct UDP addresses, and the DERP url to achieve a connection.
+//!
+//! This example uses the default DERP servers to attempt to holepunch, and will use that DERP server to relay packets if the two devices cannot establish a direct UDP connection.
+//!
+//! Run the `listen-unreliable` example first (`iroh-net/examples/listen-unreliable.rs`), which will give you instructions on how to run this example to watch two nodes connect and exchange bytes.
+use std::net::SocketAddr;
+
+use clap::Parser;
+use iroh_base::base32;
+use iroh_net::{derp::DerpMode, key::SecretKey, MagicEndpoint, NodeAddr};
+use tracing::info;
+use url::Url;
+
+// An example ALPN that we are using to communicate over the `MagicEndpoint`
+const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/magic/0";
+
+#[derive(Debug, Parser)]
+struct Cli {
+    /// The id of the remote node.
+    #[clap(long)]
+    node_id: iroh_net::NodeId,
+    /// The list of direct UDP addresses for the remote node.
+    #[clap(long, value_parser, num_args = 1.., value_delimiter = ' ')]
+    addrs: Vec<SocketAddr>,
+    /// The url of the DERP server the remote node can also be reached at.
+    #[clap(long)]
+    derp_url: Url,
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt::init();
+    println!("\nconnect (unreliable) example!\n");
+    let args = Cli::parse();
+    let secret_key = SecretKey::generate();
+    println!("secret key: {}", base32::fmt(secret_key.to_bytes()));
+
+    // Build a `MagicEndpoint`, which uses PublicKeys as node identifiers, uses QUIC for directly connecting to other nodes, and uses the DERP protocol and DERP servers to holepunch direct connections between nodes when there are NATs or firewalls preventing direct connections. If no direct connection can be made, packets are relayed over the DERP servers.
+    let endpoint = MagicEndpoint::builder()
+        // The secret key is used to authenticate with other nodes. The PublicKey portion of this secret key is how we identify nodes, often referred to as the `node_id` in our codebase.
+        .secret_key(secret_key)
+        // Set the ALPN protocols this endpoint will accept on incoming connections
+        .alpns(vec![EXAMPLE_ALPN.to_vec()])
+        // `DerpMode::Default` means that we will use the default DERP servers to holepunch and relay.
+        // Use `DerpMode::Custom` to pass in a `DerpMap` with custom DERP urls.
+        // Use `DerpMode::Disable` to disable holepunching and relaying over HTTPS
+        // If you want to experiment with relaying using your own DERP server, you must pass in the same custom DERP url to both the `listen` code AND the `connect` code
+        .derp_mode(DerpMode::Default)
+        // You can choose a port to bind to, but passing in `0` will bind the socket to a random available port
+        .bind(0)
+        .await?;
+
+    let me = endpoint.node_id();
+    println!("node id: {me}");
+    println!("node listening addresses:");
+    for local_endpoint in endpoint.local_endpoints().await? {
+        println!("\t{}", local_endpoint.addr)
+    }
+
+    let derp_url = endpoint
+        .my_derp()
+        .expect("should be connected to a DERP server, try calling `endpoint.local_endpoints()` or `endpoint.connect()` first, to ensure the endpoint has actually attempted a connection before checking for the connected DERP server");
+    println!("node DERP server url: {derp_url}\n");
+    // Build a `NodeAddr` from the node_id, DERP url, and UDP addresses.
+    let addr = NodeAddr::from_parts(args.node_id, Some(args.derp_url), args.addrs);
+
+    // Attempt to connect, over the given ALPN.
+    // Returns a QUIC connection.
+    let conn = endpoint.connect(addr, EXAMPLE_ALPN).await?;
+    info!("connected");
+
+    // Send a datagram over the connection.
+    let message = format!("{me} is saying 'hello!'");
+    conn.send_datagram(message.as_bytes().to_vec().into())?;
+
+    // Read a datagram over the connection.
+    let message = conn.read_datagram().await?;
+    let message = String::from_utf8(message.into())?;
+    println!("received: {message}");
+
+    Ok(())
+}

iroh-net/examples/connect.rs

@@ -0,0 +1,87 @@
+//! The smallest example showing how to use iroh-net and `MagicEndpoint` to connect to a remote node.
+//!
+//! We use the node ID (the PublicKey of the remote node), the direct UDP addresses, and the DERP url to achieve a connection.
+//!
+//! This example uses the default DERP servers to attempt to holepunch, and will use that DERP server to relay packets if the two devices cannot establish a direct UDP connection.
+//!
+//! Run the `listen` example first (`iroh-net/examples/listen.rs`), which will give you instructions on how to run this example to watch two nodes connect and exchange bytes.
+use std::net::SocketAddr;
+
+use clap::Parser;
+use iroh_base::base32;
+use iroh_net::{derp::DerpMode, key::SecretKey, MagicEndpoint, NodeAddr};
+use tracing::info;
+use url::Url;
+
+// An example ALPN that we are using to communicate over the `MagicEndpoint`
+const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/magic/0";
+
+#[derive(Debug, Parser)]
+struct Cli {
+    /// The id of the remote node.
+    #[clap(long)]
+    node_id: iroh_net::NodeId,
+    /// The list of direct UDP addresses for the remote node.
+    #[clap(long, value_parser, num_args = 1.., value_delimiter = ' ')]
+    addrs: Vec<SocketAddr>,
+    /// The url of the DERP server the remote node can also be reached at.
+    #[clap(long)]
+    derp_url: Url,
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt::init();
+    println!("\nconnect example!\n");
+    let args = Cli::parse();
+    let secret_key = SecretKey::generate();
+    println!("secret key: {}", base32::fmt(secret_key.to_bytes()));
+
+    // Build a `MagicEndpoint`, which uses PublicKeys as node identifiers, uses QUIC for directly connecting to other nodes, and uses the DERP protocol and DERP servers to holepunch direct connections between nodes when there are NATs or firewalls preventing direct connections. If no direct connection can be made, packets are relayed over the DERP servers.
+    let endpoint = MagicEndpoint::builder()
+        // The secret key is used to authenticate with other nodes. The PublicKey portion of this secret key is how we identify nodes, often referred to as the `node_id` in our codebase.
+        .secret_key(secret_key)
+        // Set the ALPN protocols this endpoint will accept on incoming connections
+        .alpns(vec![EXAMPLE_ALPN.to_vec()])
+        // `DerpMode::Default` means that we will use the default DERP servers to holepunch and relay.
+        // Use `DerpMode::Custom` to pass in a `DerpMap` with custom DERP urls.
+        // Use `DerpMode::Disable` to disable holepunching and relaying over HTTPS
+        // If you want to experiment with relaying using your own DERP server, you must pass in the same custom DERP url to both the `listen` code AND the `connect` code
+        .derp_mode(DerpMode::Default)
+        // You can choose a port to bind to, but passing in `0` will bind the socket to a random available port
+        .bind(0)
+        .await?;
+
+    let me = endpoint.node_id();
+    println!("node id: {me}");
+    println!("node listening addresses:");
+    for local_endpoint in endpoint.local_endpoints().await? {
+        println!("\t{}", local_endpoint.addr)
+    }
+
+    let derp_url = endpoint
+        .my_derp()
+        .expect("should be connected to a DERP server, try calling `endpoint.local_endpoints()` or `endpoint.connect()` first, to ensure the endpoint has actually attempted a connection before checking for the connected DERP server");
+    println!("node DERP server url: {derp_url}\n");
+    // Build a `NodeAddr` from the node_id, DERP url, and UDP addresses.
+    let addr = NodeAddr::from_parts(args.node_id, Some(args.derp_url), args.addrs);
+
+    // Attempt to connect, over the given ALPN.
+    // Returns a Quinn connection.
+    let conn = endpoint.connect(addr, EXAMPLE_ALPN).await?;
+    info!("connected");
+
+    // Use the Quinn API to send and recv content.
+    let (mut send, mut recv) = conn.open_bi().await?;
+
+    let message = format!("{me} is saying 'hello!'");
+    send.write_all(message.as_bytes()).await?;
+
+    // Call `finish` to close the send side of the connection gracefully.
+    send.finish().await?;
+    let message = recv.read_to_end(100).await?;
+    let message = String::from_utf8(message)?;
+    println!("received: {message}");
+
+    Ok(())
+}

iroh-net/examples/listen-unreliable.rs

@@ -0,0 +1,86 @@
+//! The smallest example showing how to use iroh-net and `MagicEndpoint` to connect two devices and pass bytes using unreliable datagrams.
+//!
+//! This example uses the default DERP servers to attempt to holepunch, and will use that DERP server to relay packets if the two devices cannot establish a direct UDP connection.
+//! run this example from the project root:
+//!     $ cargo run --example listen-unreliable
+use iroh_base::base32;
+use iroh_net::{derp::DerpMode, key::SecretKey, MagicEndpoint};
+use tracing::info;
+
+// An example ALPN that we are using to communicate over the `MagicEndpoint`
+const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/magic/0";
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt::init();
+    println!("\nlisten (unreliable) example!\n");
+    let secret_key = SecretKey::generate();
+    println!("secret key: {}", base32::fmt(secret_key.to_bytes()));
+
+    // Build a `MagicEndpoint`, which uses PublicKeys as node identifiers, uses QUIC for directly connecting to other nodes, and uses the DERP protocol and DERP servers to holepunch direct connections between nodes when there are NATs or firewalls preventing direct connections. If no direct connection can be made, packets are relayed over the DERP servers.
+    let endpoint = MagicEndpoint::builder()
+        // The secret key is used to authenticate with other nodes. The PublicKey portion of this secret key is how we identify nodes, often referred to as the `node_id` in our codebase.
+        .secret_key(secret_key)
+        // set the ALPN protocols this endpoint will accept on incoming connections
+        .alpns(vec![EXAMPLE_ALPN.to_vec()])
+        // `DerpMode::Default` means that we will use the default DERP servers to holepunch and relay.
+        // Use `DerpMode::Custom` to pass in a `DerpMap` with custom DERP urls.
+        // Use `DerpMode::Disable` to disable holepunching and relaying over HTTPS
+        // If you want to experiment with relaying using your own DERP server, you must pass in the same custom DERP url to both the `listen` code AND the `connect` code
+        .derp_mode(DerpMode::Default)
+        // you can choose a port to bind to, but passing in `0` will bind the socket to a random available port
+        .bind(0)
+        .await?;
+
+    let me = endpoint.node_id();
+    println!("node id: {me}");
+    println!("node listening addresses:");
+
+    let local_addrs = endpoint
+        .local_endpoints()
+        .await?
+        .into_iter()
+        .map(|endpoint| {
+            let addr = endpoint.addr.to_string();
+            println!("\t{addr}");
+            addr
+        })
+        .collect::<Vec<_>>()
+        .join(" ");
+
+    let derp_url = endpoint
+        .my_derp()
+        .expect("should be connected to a DERP server, try calling `endpoint.local_endpoints()` or `endpoint.connect()` first, to ensure the endpoint has actually attempted a connection before checking for the connected DERP server");
+    println!("node DERP server url: {derp_url}");
+    println!("\nin a separate terminal run:");
+
+    println!(
+        "\tcargo run --example connect-unreliable -- --node-id {me} --addrs \"{local_addrs}\" --derp-url {derp_url}\n"
+    );
+    // accept incoming connections, returns a normal QUIC connection
+
+    while let Some(conn) = endpoint.accept().await {
+        // accept the connection and extract the `node_id` and ALPN
+        let (node_id, alpn, conn) = iroh_net::magic_endpoint::accept_conn(conn).await?;
+        info!(
+            "new (unreliable) connection from {node_id} with ALPN {alpn} (coming from {})",
+            conn.remote_address()
+        );
+        // spawn a task to handle reading and writing off of the connection
+        tokio::spawn(async move {
+            // use the `quinn` API to read a datagram off the connection, and send a datagra, in return
+            while let Ok(message) = conn.read_datagram().await {
+                let message = String::from_utf8(message.into())?;
+                println!("received: {message}");
+
+                let message = format!("hi! you connected to {me}. bye bye");
+                conn.send_datagram(message.as_bytes().to_vec().into())?;
+            }
+
+            Ok::<_, anyhow::Error>(())
+        });
+    }
+    // stop with SIGINT (ctrl-c)
+
+    Ok(())
+}

iroh-net/examples/listen.rs

@@ -0,0 +1,90 @@
+//! The smallest example showing how to use iroh-net and `MagicEndpoint` to connect two devices.
+//!
+//! This example uses the default DERP servers to attempt to holepunch, and will use that DERP server to relay packets if the two devices cannot establish a direct UDP connection.
+//! run this example from the project root:
+//!     $ cargo run --example listen
+use iroh_base::base32;
+use iroh_net::{derp::DerpMode, key::SecretKey, MagicEndpoint};
+use tracing::{debug, info};
+
+// An example ALPN that we are using to communicate over the `MagicEndpoint`
+const EXAMPLE_ALPN: &[u8] = b"n0/iroh/examples/magic/0";
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt::init();
+    println!("\nlisten example!\n");
+    let secret_key = SecretKey::generate();
+    println!("secret key: {}", base32::fmt(secret_key.to_bytes()));
+
+    // Build a `MagicEndpoint`, which uses PublicKeys as node identifiers, uses QUIC for directly connecting to other nodes, and uses the DERP protocol and DERP servers to holepunch direct connections between nodes when there are NATs or firewalls preventing direct connections. If no direct connection can be made, packets are relayed over the DERP servers.
+    let endpoint = MagicEndpoint::builder()
+        // The secret key is used to authenticate with other nodes. The PublicKey portion of this secret key is how we identify nodes, often referred to as the `node_id` in our codebase.
+        .secret_key(secret_key)
+        // set the ALPN protocols this endpoint will accept on incoming connections
+        .alpns(vec![EXAMPLE_ALPN.to_vec()])
+        // `DerpMode::Default` means that we will use the default DERP servers to holepunch and relay.
+        // Use `DerpMode::Custom` to pass in a `DerpMap` with custom DERP urls.
+        // Use `DerpMode::Disable` to disable holepunching and relaying over HTTPS
+        // If you want to experiment with relaying using your own DERP server, you must pass in the same custom DERP url to both the `listen` code AND the `connect` code
+        .derp_mode(DerpMode::Default)
+        // you can choose a port to bind to, but passing in `0` will bind the socket to a random available port
+        .bind(0)
+        .await?;
+
+    let me = endpoint.node_id();
+    println!("node id: {me}");
+    println!("node listening addresses:");
+
+    let local_addrs = endpoint
+        .local_endpoints()
+        .await?
+        .into_iter()
+        .map(|endpoint| {
+            let addr = endpoint.addr.to_string();
+            println!("\t{addr}");
+            addr
+        })
+        .collect::<Vec<_>>()
+        .join(" ");
+
+    let derp_url = endpoint
+        .my_derp()
+        .expect("should be connected to a DERP server, try calling `endpoint.local_endpoints()` or `endpoint.connect()` first, to ensure the endpoint has actually attempted a connection before checking for the connected DERP server");
+    println!("node DERP server url: {derp_url}");
+    println!("\nin a separate terminal run:");
+
+    println!(
+        "\tcargo run --example connect -- --node-id {me} --addrs \"{local_addrs}\" --derp-url {derp_url}\n"
+    );
+    // accept incoming connections, returns a normal QUIC connection
+    while let Some(conn) = endpoint.accept().await {
+        // accept the connection and extract the `node_id` and ALPN
+        let (node_id, alpn, conn) = iroh_net::magic_endpoint::accept_conn(conn).await?;
+        info!(
+            "new connection from {node_id} with ALPN {alpn} (coming from {})",
+            conn.remote_address()
+        );
+
+        // spawn a task to handle reading and writing off of the connection
+        tokio::spawn(async move {
+            // accept a bi-directional QUIC connection
+            // use the `quinn` APIs to send and recv content
+            let (mut send, mut recv) = conn.accept_bi().await?;
+            debug!("accepted bi stream, waiting for data...");
+            let message = recv.read_to_end(100).await?;
+            let message = String::from_utf8(message)?;
+            println!("received: {message}");
+
+            let message = format!("hi! you connected to {me}. bye bye");
+            send.write_all(message.as_bytes()).await?;
+            // call `finish` to close the connection gracefully
+            send.finish().await?;
+
+            Ok::<_, anyhow::Error>(())
+        });
+    }
+    // stop with SIGINT (ctrl-c)
+
+    Ok(())
+}