From 0a560243a5e5a1d87fa79c8824b1198f5f61224c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 27 Jan 2022 13:22:40 -0500 Subject: [PATCH 1/4] Use 0.0.0.0 in place of 127.0.0.1 to appease docker --- sled-agent/src/sim/storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 8e3bfbd5b6b..7f0b06192be 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -140,7 +140,7 @@ impl CrucibleServer { fn new(log: &Logger) -> Self { let data = Arc::new(CrucibleData::new()); let config = dropshot::ConfigDropshot { - bind_address: SocketAddr::new("127.0.0.1".parse().unwrap(), 0), + bind_address: SocketAddr::new("0.0.0.0".parse().unwrap(), 0), ..Default::default() }; let dropshot_log = log From 7ee829545b6a49621492d1e8cb11f4ab24be1bff Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 27 Jan 2022 14:55:59 -0500 Subject: [PATCH 2/4] Plumbing through CLI --- README.adoc | 4 ++-- nexus/test-utils/src/lib.rs | 7 +++++-- sled-agent/src/bin/sled-agent-sim.rs | 6 +++++- sled-agent/src/sim/config.rs | 5 +++-- sled-agent/src/sim/server.rs | 3 +-- sled-agent/src/sim/sled_agent.rs | 10 +++++---- sled-agent/src/sim/storage.rs | 21 +++++++++++-------- .../output/cmd-sled-agent-sim-noargs-stderr | 3 ++- 8 files changed, 36 insertions(+), 23 deletions(-) diff --git a/README.adoc b/README.adoc index a921d084cd7..743d579bed1 100644 --- a/README.adoc +++ b/README.adoc @@ -165,11 +165,11 @@ listening: http://127.0.0.1:12220 ---- Nexus can also serve the web console. Instructions for generating the static assets and pointing Nexus to them are https://github.com/oxidecomputer/console/blob/main/docs/serve-from-nexus.md[here]. Without console assets, Nexus will still start and run normally as an API. A few link:./nexus/src/external_api/console_api.rs[console-specific routes] will 404. -. `sled-agent-sim` only accepts configuration on the command line. Run it with a uuid identifying itself (this would be a uuid for the sled it's managing), an IP:port for itself, and the IP:port of `nexus`'s _internal_ interface. Using default config, this might look like this: +. `sled-agent-sim` only accepts configuration on the command line. Run it with a uuid identifying itself (this would be a uuid for the sled it's managing), an IP:port for itself, the IP:port of `nexus`'s _internal_ interface used by Simulated Crucible Agents when creating simulated datsets. Using default config, this might look like this: + [source,text] ---- -$ cargo run --bin=sled-agent-sim -- $(uuidgen) 127.0.0.1:12345 127.0.0.1:12221 +$ cargo run --bin=sled-agent-sim -- $(uuidgen) 127.0.0.1:12345 127.0.0.1:12221 127.0.0.1 ... Jun 02 12:21:50.989 INFO listening, local_addr: 127.0.0.1:12345, component: dropshot ---- diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index dba247c3250..6be1f1e2c36 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -17,7 +17,7 @@ use oximeter_collector::Oximeter; use oximeter_producer::Server as ProducerServer; use slog::o; use slog::Logger; -use std::net::SocketAddr; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path; use std::time::Duration; use uuid::Uuid; @@ -183,7 +183,10 @@ pub async fn start_sled_agent( }, /* TODO-cleanup this is unused */ log: ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug }, - storage: sim::ConfigStorage::default(), + storage: sim::ConfigStorage { + zpools: vec![], + ip: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), + }, }; sim::Server::start(&config, &log).await diff --git a/sled-agent/src/bin/sled-agent-sim.rs b/sled-agent/src/bin/sled-agent-sim.rs index 286f27681ee..4b3196eb5f1 100644 --- a/sled-agent/src/bin/sled-agent-sim.rs +++ b/sled-agent/src/bin/sled-agent-sim.rs @@ -18,7 +18,7 @@ use omicron_common::cmd::CmdError; use omicron_sled_agent::sim::{ run_server, Config, ConfigStorage, ConfigZpool, SimMode, }; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use structopt::StructOpt; use uuid::Uuid; @@ -52,6 +52,9 @@ struct Args { #[structopt(name = "NEXUS_IP:PORT", parse(try_from_str))] nexus_addr: SocketAddr, + + #[structopt(name = "CRUCIBLE_IP", parse(try_from_str))] + crucible_addr: IpAddr, } #[tokio::main] @@ -78,6 +81,7 @@ async fn do_run() -> Result<(), CmdError> { storage: ConfigStorage { // Create 10 "virtual" U.2s, with 1 TB of storage. zpools: vec![ConfigZpool { size: 1 << 40 }; 10], + ip: args.crucible_addr, }, }; diff --git a/sled-agent/src/sim/config.rs b/sled-agent/src/sim/config.rs index 5926791793c..7c3f24266f8 100644 --- a/sled-agent/src/sim/config.rs +++ b/sled-agent/src/sim/config.rs @@ -10,7 +10,7 @@ use dropshot::ConfigDropshot; use dropshot::ConfigLogging; use serde::Deserialize; use serde::Serialize; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use uuid::Uuid; /** @@ -43,9 +43,10 @@ pub struct ConfigZpool { } /// Configuration describing simulated storage. -#[derive(Clone, Default, Debug, PartialEq, Deserialize, Serialize)] +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] pub struct ConfigStorage { pub zpools: Vec, + pub ip: IpAddr, } /** diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index c741cb1f4f3..6187f1ed50e 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -50,8 +50,7 @@ impl Server { "server" => config.id.clone().to_string() )); let sled_agent = Arc::new(SledAgent::new_simulated_with_id( - &config.id, - config.sim_mode, + &config, sa_log, Arc::clone(&nexus_client), )); diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 71e0c692152..400df346ecd 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use uuid::Uuid; use super::collection::SimCollection; -use super::config::SimMode; +use super::config::Config; use super::disk::SimDisk; use super::instance::SimInstance; use super::storage::{CrucibleData, Storage}; @@ -48,11 +48,12 @@ impl SledAgent { */ /** Constructs a simulated SledAgent with the given uuid. */ pub fn new_simulated_with_id( - id: &Uuid, - sim_mode: SimMode, + config: &Config, log: Logger, nexus_client: Arc, ) -> SledAgent { + let id = config.id; + let sim_mode = config.sim_mode; info!(&log, "created simulated sled agent"; "sim_mode" => ?sim_mode); let instance_log = log.new(o!("kind" => "instances")); @@ -71,8 +72,9 @@ impl SledAgent { sim_mode, )), storage: Mutex::new(Storage::new( - *id, + id, Arc::clone(&nexus_client), + config.storage.ip, storage_log, )), } diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 7f0b06192be..31f8b84293e 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -16,7 +16,7 @@ use nexus_client::types::{ use nexus_client::Client as NexusClient; use slog::Logger; use std::collections::HashMap; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use std::str::FromStr; use std::sync::Arc; use uuid::Uuid; @@ -137,10 +137,10 @@ pub struct CrucibleServer { } impl CrucibleServer { - fn new(log: &Logger) -> Self { + fn new(log: &Logger, crucible_ip: IpAddr) -> Self { let data = Arc::new(CrucibleData::new()); let config = dropshot::ConfigDropshot { - bind_address: SocketAddr::new("0.0.0.0".parse().unwrap(), 0), + bind_address: SocketAddr::new(crucible_ip, 0), ..Default::default() }; let dropshot_log = log @@ -163,21 +163,22 @@ impl CrucibleServer { } } -pub struct Zpool { +struct Zpool { datasets: HashMap, } impl Zpool { - pub fn new() -> Self { + fn new() -> Self { Zpool { datasets: HashMap::new() } } - pub fn insert_dataset( + fn insert_dataset( &mut self, log: &Logger, id: Uuid, + crucible_ip: IpAddr, ) -> &CrucibleServer { - self.datasets.insert(id, CrucibleServer::new(log)); + self.datasets.insert(id, CrucibleServer::new(log, crucible_ip)); self.datasets .get(&id) .expect("Failed to get the dataset we just inserted") @@ -190,15 +191,17 @@ pub struct Storage { nexus_client: Arc, log: Logger, zpools: HashMap, + crucible_ip: IpAddr, } impl Storage { pub fn new( sled_id: Uuid, nexus_client: Arc, + crucible_ip: IpAddr, log: Logger, ) -> Self { - Self { sled_id, nexus_client, log, zpools: HashMap::new() } + Self { sled_id, nexus_client, log, zpools: HashMap::new(), crucible_ip } } /// Adds a Zpool to the sled's simulated storage and notifies Nexus. @@ -221,7 +224,7 @@ impl Storage { .zpools .get_mut(&zpool_id) .expect("Zpool does not exist") - .insert_dataset(&self.log, dataset_id); + .insert_dataset(&self.log, dataset_id, self.crucible_ip); // Notify Nexus let request = DatasetPutRequest { diff --git a/sled-agent/tests/output/cmd-sled-agent-sim-noargs-stderr b/sled-agent/tests/output/cmd-sled-agent-sim-noargs-stderr index 0b26275cd74..d72c102d556 100644 --- a/sled-agent/tests/output/cmd-sled-agent-sim-noargs-stderr +++ b/sled-agent/tests/output/cmd-sled-agent-sim-noargs-stderr @@ -2,8 +2,9 @@ sled-agent-sim: parsing arguments: error: The following required arguments were + USAGE: - sled-agent-sim --sim-mode + sled-agent-sim --sim-mode For more information try --help From ec4684b60c4933473927246c84c1d5acdcfac340 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 27 Jan 2022 15:02:09 -0500 Subject: [PATCH 3/4] Log address in Dataset upsert --- nexus/src/nexus.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index ff352301657..0de77a45af8 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -286,7 +286,7 @@ impl Nexus { address: SocketAddr, kind: DatasetKind, ) -> Result<(), Error> { - info!(self.log, "upserting dataset"; "zpool_id" => zpool_id.to_string(), "dataset_id" => id.to_string()); + info!(self.log, "upserting dataset"; "zpool_id" => zpool_id.to_string(), "dataset_id" => id.to_string(), "address" => address.to_string()); let dataset = db::model::Dataset::new(id, zpool_id, address, kind); self.db_datastore.dataset_upsert(dataset).await?; Ok(()) From 10181f6970038dcef7e8e1efffe0f75974d65fcf Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 27 Jan 2022 15:23:13 -0500 Subject: [PATCH 4/4] derive IP addres from SA, fix an upsert bug --- README.adoc | 4 ++-- nexus/src/db/datastore.rs | 2 +- sled-agent/src/bin/sled-agent-sim.rs | 7 ++----- sled-agent/tests/output/cmd-sled-agent-sim-noargs-stderr | 3 +-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/README.adoc b/README.adoc index 743d579bed1..a921d084cd7 100644 --- a/README.adoc +++ b/README.adoc @@ -165,11 +165,11 @@ listening: http://127.0.0.1:12220 ---- Nexus can also serve the web console. Instructions for generating the static assets and pointing Nexus to them are https://github.com/oxidecomputer/console/blob/main/docs/serve-from-nexus.md[here]. Without console assets, Nexus will still start and run normally as an API. A few link:./nexus/src/external_api/console_api.rs[console-specific routes] will 404. -. `sled-agent-sim` only accepts configuration on the command line. Run it with a uuid identifying itself (this would be a uuid for the sled it's managing), an IP:port for itself, the IP:port of `nexus`'s _internal_ interface used by Simulated Crucible Agents when creating simulated datsets. Using default config, this might look like this: +. `sled-agent-sim` only accepts configuration on the command line. Run it with a uuid identifying itself (this would be a uuid for the sled it's managing), an IP:port for itself, and the IP:port of `nexus`'s _internal_ interface. Using default config, this might look like this: + [source,text] ---- -$ cargo run --bin=sled-agent-sim -- $(uuidgen) 127.0.0.1:12345 127.0.0.1:12221 127.0.0.1 +$ cargo run --bin=sled-agent-sim -- $(uuidgen) 127.0.0.1:12345 127.0.0.1:12221 ... Jun 02 12:21:50.989 INFO listening, local_addr: 127.0.0.1:12345, component: dropshot ---- diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index 38c6b43d14f..3313598aeb2 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -239,7 +239,7 @@ impl DataStore { .do_update() .set(( dsl::time_modified.eq(Utc::now()), - dsl::pool_id.eq(excluded(dsl::id)), + dsl::pool_id.eq(excluded(dsl::pool_id)), dsl::ip.eq(excluded(dsl::ip)), dsl::port.eq(excluded(dsl::port)), dsl::kind.eq(excluded(dsl::kind)), diff --git a/sled-agent/src/bin/sled-agent-sim.rs b/sled-agent/src/bin/sled-agent-sim.rs index 4b3196eb5f1..3cd7a84384f 100644 --- a/sled-agent/src/bin/sled-agent-sim.rs +++ b/sled-agent/src/bin/sled-agent-sim.rs @@ -18,7 +18,7 @@ use omicron_common::cmd::CmdError; use omicron_sled_agent::sim::{ run_server, Config, ConfigStorage, ConfigZpool, SimMode, }; -use std::net::{IpAddr, SocketAddr}; +use std::net::SocketAddr; use structopt::StructOpt; use uuid::Uuid; @@ -52,9 +52,6 @@ struct Args { #[structopt(name = "NEXUS_IP:PORT", parse(try_from_str))] nexus_addr: SocketAddr, - - #[structopt(name = "CRUCIBLE_IP", parse(try_from_str))] - crucible_addr: IpAddr, } #[tokio::main] @@ -81,7 +78,7 @@ async fn do_run() -> Result<(), CmdError> { storage: ConfigStorage { // Create 10 "virtual" U.2s, with 1 TB of storage. zpools: vec![ConfigZpool { size: 1 << 40 }; 10], - ip: args.crucible_addr, + ip: args.sled_agent_addr.ip(), }, }; diff --git a/sled-agent/tests/output/cmd-sled-agent-sim-noargs-stderr b/sled-agent/tests/output/cmd-sled-agent-sim-noargs-stderr index d72c102d556..0b26275cd74 100644 --- a/sled-agent/tests/output/cmd-sled-agent-sim-noargs-stderr +++ b/sled-agent/tests/output/cmd-sled-agent-sim-noargs-stderr @@ -2,9 +2,8 @@ sled-agent-sim: parsing arguments: error: The following required arguments were - USAGE: - sled-agent-sim --sim-mode + sled-agent-sim --sim-mode For more information try --help