diff --git a/Cargo.lock b/Cargo.lock index e66d8fe8fe9..d87c8797b53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2524,6 +2524,7 @@ dependencies = [ "futures", "http", "ipnetwork", + "macaddr", "mockall", "nexus-client", "omicron-common", diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 72433768241..8ecbf493db0 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1072,6 +1072,12 @@ impl std::fmt::Display for Ipv6Net { } } +impl From for Ipv6Net { + fn from(n: ipnetwork::Ipv6Network) -> Ipv6Net { + Self(n) + } +} + impl JsonSchema for Ipv6Net { fn schema_name() -> String { "Ipv6Net".to_string() diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 6ce0ac8f174..071611d7ae5 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -113,19 +113,21 @@ we'll assign addresses as per RFD 63 as well as incorporating DNS based service discovery. For the purposes of local development today, we specify some hardcoded IPv6 -unique local addresses in `fd00:1de::/16`: +unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1::/64`: [options="header"] |=================================================================================================== -| Service | Endpoint -| Sled Agent: Bootstrap | `[::]:12346` -| Sled Agent: Dropshot API | `[fd00:1de::]:12345` -| Cockroach DB | `[fd00:1de::5]:32221` -| Oximeter | `[fd00:1de::6]:12223` -| Nexus: External API | `[fd00:1de::7]:12220` -| Nexus: Internal API | `[fd00:1de::7]:12221` -| Clickhouse | `[fd00:1de::8]:8123` -| Crucible Downstairs | `[fd00:1de::9]:32345`, `[fd00:1de::10]:32345`, `[fd00:1de::11]:32345` +| Service | Endpoint +| Sled Agent: Bootstrap | Derived from MAC address of physical data link. +| Sled Agent: Dropshot API | `[fd00:1122:3344:1::1]:12345` +| Cockroach DB | `[fd00:1122:3344:1::2]:32221` +| Nexus: External API | `[fd00:1122:3344:1::3]:12220` +| Nexus: Internal API | `[fd00:1122:3344:1::3]:12221` +| Oximeter | `[fd00:1122:3344:1::4]:12223` +| Clickhouse | `[fd00:1122:3344:1::5]:8123` +| Crucible Downstairs 1 | `[fd00:1122:3344:1::6]:32345` +| Crucible Downstairs 2 | `[fd00:1122:3344:1::7]:32345` +| Crucible Downstairs 3 | `[fd00:1122:3344:1::8]:32345` |=================================================================================================== Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 7e117a3ae17..cf04477f449 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -12,7 +12,7 @@ "paths": { "/request_share": { "get": { - "operationId": "api_request_share", + "operationId": "request_share", "requestBody": { "content": { "application/json": { @@ -42,6 +42,39 @@ } } } + }, + "/start_sled": { + "put": { + "operationId": "start_sled", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledAgentRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledAgentResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } } }, "components": { @@ -77,6 +110,13 @@ "request_id" ] }, + "Ipv6Net": { + "title": "An IPv6 subnet", + "description": "An IPv6 subnet, including prefix and subnet mask", + "type": "string", + "pattern": "^(fd|FD)[0-9a-fA-F]{2}:((([0-9a-fA-F]{1,4}\\:){6}[0-9a-fA-F]{1,4})|(([0-9a-fA-F]{1,4}:){1,6}:))/(6[4-9]|[7-9][0-9]|1[0-1][0-9]|12[0-6])$", + "maxLength": 43 + }, "ShareRequest": { "description": "Identity signed by local RoT and Oxide certificate chain.", "type": "object", @@ -110,6 +150,44 @@ "required": [ "shared_secret" ] + }, + "SledAgentRequest": { + "description": "Configuration information for launching a Sled Agent.", + "type": "object", + "properties": { + "subnet": { + "description": "Portion of the IP space to be managed by the Sled Agent.", + "allOf": [ + { + "$ref": "#/components/schemas/SledSubnet" + } + ] + } + }, + "required": [ + "subnet" + ] + }, + "SledAgentResponse": { + "description": "Describes the Sled Agent running on the device.", + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "id" + ] + }, + "SledSubnet": { + "description": "Represents subnets belonging to Sleds.\n\nThis is a thin wrapper around the [`Ipv6Net`] type - which may be accessed by [`AsRef`] - which adds additional validation that this is a /64 subnet with an expected prefix.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Net" + } + ] } } } diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 7faf6b1866f..379bfdceed1 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -17,6 +17,7 @@ crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } futures = "0.3.21" ipnetwork = "0.18" +macaddr = { version = "1.0.1", features = [ "serde_std" ] } nexus-client = { path = "../nexus-client" } omicron-common = { path = "../common" } p256 = "0.9.0" diff --git a/sled-agent/src/bin/sled-agent.rs b/sled-agent/src/bin/sled-agent.rs index c2989ee0280..82000ceb308 100644 --- a/sled-agent/src/bin/sled-agent.rs +++ b/sled-agent/src/bin/sled-agent.rs @@ -4,8 +4,6 @@ //! Executable program to run the sled agent -#![feature(async_closure)] - use dropshot::ConfigDropshot; use dropshot::ConfigLogging; use dropshot::ConfigLoggingLevel; @@ -13,10 +11,12 @@ use omicron_common::api::external::Error; use omicron_common::cmd::fatal; use omicron_common::cmd::CmdError; use omicron_sled_agent::bootstrap::{ - config::Config as BootstrapConfig, server as bootstrap_server, + agent::bootstrap_address, config::Config as BootstrapConfig, + server as bootstrap_server, }; use omicron_sled_agent::rack_setup::config::SetupServiceConfig as RssConfig; use omicron_sled_agent::{config::Config as SledConfig, server as sled_server}; +use std::net::SocketAddr; use std::path::PathBuf; use structopt::StructOpt; @@ -81,10 +81,8 @@ async fn do_run() -> Result<(), CmdError> { } }, Args::Run { config_path } => { - let mut config = SledConfig::from_file(&config_path) + let config = SledConfig::from_file(&config_path) .map_err(|e| CmdError::Failure(e.to_string()))?; - config.dropshot.request_body_max_bytes = 1024 * 1024; - let config = config; // - Sled agent starts with the normal config file - typically // called "config.toml". @@ -113,11 +111,18 @@ async fn do_run() -> Result<(), CmdError> { None }; + // Derive the bootstrap address from the data link's MAC address. + let link = config + .get_link() + .map_err(|e| CmdError::Failure(e.to_string()))?; + let bootstrap_address = bootstrap_address(link) + .map_err(|e| CmdError::Failure(e.to_string()))?; + // Configure and run the Bootstrap server. let bootstrap_config = BootstrapConfig { id: config.id, dropshot: ConfigDropshot { - bind_address: config.bootstrap_address, + bind_address: SocketAddr::V6(bootstrap_address), request_body_max_bytes: 1024 * 1024, ..Default::default() }, @@ -126,32 +131,21 @@ async fn do_run() -> Result<(), CmdError> { }, rss_config, }; - let run_bootstrap = async move || -> Result<(), CmdError> { - bootstrap_server::Server::start(&bootstrap_config) - .await - .map_err(CmdError::Failure)? - .wait_for_finish() - .await - .map_err(CmdError::Failure) - }; - let run_sled_server = async move || -> Result<(), CmdError> { - sled_server::Server::start(&config) - .await - .map_err(CmdError::Failure)? - .wait_for_finish() - .await - .map_err(CmdError::Failure) - }; + // TODO: It's a little silly to pass the config this way - namely, + // that we construct the bootstrap config from `config`, but then + // pass it separately just so the sled agent can ingest it later on. + bootstrap_server::Server::start( + *bootstrap_address.ip(), + bootstrap_config, + config, + ) + .await + .map_err(CmdError::Failure)? + .wait_for_finish() + .await + .map_err(CmdError::Failure)?; - tokio::select! { - Err(e) = run_bootstrap() => { - eprintln!("Boot server exited unexpectedly: {:?}", e); - }, - Err(e) = run_sled_server() => { - eprintln!("Sled server exited unexpectedly: {:?}", e); - }, - } Ok(()) } } diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index 9f85cf6d1c6..8e3ed304b94 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -4,21 +4,27 @@ //! Bootstrap-related APIs. -use super::config::Config; +use super::config::{Config, BOOTSTRAP_AGENT_PORT}; use super::discovery; +use super::params::SledAgentRequest; use super::trust_quorum::{ self, RackSecret, ShareDistribution, TrustQuorumError, }; -use super::views::ShareResponse; +use super::views::{ShareResponse, SledAgentResponse}; +use crate::config::Config as SledConfig; +use crate::illumos::dladm::{self, Dladm, PhysicalLink}; +use crate::illumos::zone::{self, Zones}; use crate::rack_setup::service::Service as RackSetupService; -use omicron_common::api::external::Error as ExternalError; +use crate::server::Server as SledServer; +use omicron_common::api::external::{Error as ExternalError, MacAddr}; use omicron_common::backoff::{ internal_service_policy, retry_notify, BackoffError, }; use slog::Logger; use std::io; -use std::path::Path; +use std::net::{Ipv6Addr, SocketAddrV6}; +use std::path::{Path, PathBuf}; use thiserror::Error; use tokio::sync::Mutex; @@ -34,8 +40,17 @@ pub enum BootstrapError { #[error("Error modifying SMF service: {0}")] SmfAdm(#[from] smf::AdmError), + #[error("Error starting sled agent: {0}")] + SledError(String), + + #[error(transparent)] + Toml(#[from] toml::de::Error), + #[error(transparent)] TrustQuorum(#[from] TrustQuorumError), + + #[error(transparent)] + Zone(#[from] zone::Error), } impl From for ExternalError { @@ -74,13 +89,74 @@ pub(crate) struct Agent { share: Option, rss: Mutex>, + sled_agent: Mutex>, + sled_config: SledConfig, +} + +fn get_subnet_path() -> PathBuf { + Path::new(omicron_common::OMICRON_CONFIG_PATH).join("subnet.toml") +} + +fn mac_to_socket_addr(mac: MacAddr) -> SocketAddrV6 { + let mac_bytes = mac.into_array(); + assert_eq!(6, mac_bytes.len()); + + let address = Ipv6Addr::new( + 0xfdb0, + ((mac_bytes[0] as u16) << 8) | mac_bytes[1] as u16, + ((mac_bytes[2] as u16) << 8) | mac_bytes[3] as u16, + ((mac_bytes[4] as u16) << 8) | mac_bytes[5] as u16, + 0, + 0, + 0, + 1, + ); + + SocketAddrV6::new(address, BOOTSTRAP_AGENT_PORT, 0, 0) +} + +// TODO(https://github.com/oxidecomputer/omicron/issues/945): This address +// could be randomly generated when it no longer needs to be durable. +pub fn bootstrap_address( + link: PhysicalLink, +) -> Result { + let mac = Dladm::get_mac(link)?; + Ok(mac_to_socket_addr(mac)) } impl Agent { - pub fn new(log: Logger) -> Result { - let peer_monitor = discovery::PeerMonitor::new(&log)?; + pub async fn new( + log: Logger, + sled_config: SledConfig, + address: Ipv6Addr, + ) -> Result { + Zones::ensure_has_global_zone_v6_address( + sled_config.data_link.clone(), + address, + "bootstrap6", + )?; + + let peer_monitor = discovery::PeerMonitor::new(&log, address)?; let share = read_key_share()?; - Ok(Agent { log, peer_monitor, share, rss: Mutex::new(None) }) + let agent = Agent { + log, + peer_monitor, + share, + rss: Mutex::new(None), + sled_agent: Mutex::new(None), + sled_config, + }; + + let subnet_path = get_subnet_path(); + if subnet_path.exists() { + info!(agent.log, "Sled already configured, loading sled agent"); + let sled_request: SledAgentRequest = toml::from_str( + &tokio::fs::read_to_string(&subnet_path).await?, + )?; + agent.request_agent(sled_request).await?; + } + + Ok(agent) } /// Implements the "request share" API. @@ -97,6 +173,55 @@ impl Agent { Ok(ShareResponse { shared_secret: vec![] }) } + /// Initializes the Sled Agent on behalf of the RSS, if one has not already + /// been initialized. + pub async fn request_agent( + &self, + request: SledAgentRequest, + ) -> Result { + info!(&self.log, "Loading Sled Agent: {:?}", request); + + let sled_address = + crate::config::get_sled_address(*request.subnet.as_ref()); + + let mut maybe_agent = self.sled_agent.lock().await; + if let Some(server) = &*maybe_agent { + // Server already exists, return it. + info!(&self.log, "Sled Agent already loaded"); + + if &server.address().ip() != sled_address.ip() { + let err_str = format!( + "Sled Agent already running on address {}, but {} was requested", + server.address().ip(), + sled_address.ip(), + ); + return Err(BootstrapError::SledError(err_str)); + } + + return Ok(SledAgentResponse { id: server.id() }); + } + // Server does not exist, initialize it. + let server = SledServer::start(&self.sled_config, sled_address) + .await + .map_err(|e| BootstrapError::SledError(e))?; + maybe_agent.replace(server); + info!(&self.log, "Sled Agent loaded; recording configuration"); + + // Record the subnet, so the sled agent can be automatically + // initialized on the next boot. + tokio::fs::write( + get_subnet_path(), + &toml::to_string( + &toml::Value::try_from(&request.subnet) + .expect("Cannot serialize IP"), + ) + .expect("Cannot convert toml to string"), + ) + .await?; + + Ok(SledAgentResponse { id: self.sled_config.id }) + } + /// Communicates with peers, sharing secrets, until the rack has been /// sufficiently unlocked. async fn establish_sled_quorum( @@ -105,7 +230,7 @@ impl Agent { let rack_secret = retry_notify( internal_service_policy(), || async { - let other_agents = self.peer_monitor.addrs().await; + let other_agents = self.peer_monitor.peer_addrs().await; info!( &self.log, "Bootstrap: Communicating with peers: {:?}", other_agents @@ -131,8 +256,13 @@ impl Agent { // Retrieve verified rack_secret shares from a quorum of agents let other_agents: Vec = other_agents .into_iter() - .map(|mut addr| { - addr.set_port(trust_quorum::PORT); + .map(|addr| { + let addr = SocketAddrV6::new( + addr, + trust_quorum::PORT, + 0, + 0, + ); trust_quorum::Client::new( &self.log, share.verifier.clone(), @@ -205,6 +335,7 @@ impl Agent { let rss = RackSetupService::new( self.log.new(o!("component" => "RSS")), rss_config.clone(), + self.peer_monitor.observer().await, ); self.rss.lock().await.replace(rss); } @@ -233,3 +364,19 @@ impl Agent { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use macaddr::MacAddr6; + + #[test] + fn test_mac_to_socket_addr() { + let mac = MacAddr("a8:40:25:10:00:01".parse::().unwrap()); + + assert_eq!( + mac_to_socket_addr(mac).ip(), + &"fdb0:a840:2510:1::1".parse::().unwrap(), + ); + } +} diff --git a/sled-agent/src/bootstrap/config.rs b/sled-agent/src/bootstrap/config.rs index 1fec659b5b3..fc8951954ec 100644 --- a/sled-agent/src/bootstrap/config.rs +++ b/sled-agent/src/bootstrap/config.rs @@ -10,6 +10,8 @@ use serde::Deserialize; use serde::Serialize; use uuid::Uuid; +pub const BOOTSTRAP_AGENT_PORT: u16 = 12346; + /// Configuration for a bootstrap agent #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub struct Config { diff --git a/sled-agent/src/bootstrap/discovery.rs b/sled-agent/src/bootstrap/discovery.rs index 3c87833a378..384b8fd027e 100644 --- a/sled-agent/src/bootstrap/discovery.rs +++ b/sled-agent/src/bootstrap/discovery.rs @@ -8,25 +8,44 @@ use super::multicast; use slog::Logger; use std::collections::HashSet; use std::io; -use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; +use std::net::{Ipv6Addr, SocketAddr}; use std::sync::Arc; use tokio::net::UdpSocket; -use tokio::sync::Mutex; +use tokio::sync::{broadcast, Mutex}; use tokio::task::JoinHandle; +// NOTE: This is larger than the expected number of sleds per rack, as +// peers may change as new sleds are swapped in for old ones. +// +// See the "TODO" below about removal of sleds from the HashSet +const PEER_CAPACITY_MAXIMUM: usize = 128; + /// Manages Sled Discovery - both our announcement to other Sleds, /// as well as our discovery of those sleds. pub struct PeerMonitor { - sleds: Arc>>, + // TODO: When can we remove sleds from this HashSet? Presumably, if a sled + // has been detached from the bootstrap network, we should drop it. + // + // Without such removal, the set size will be unbounded (though admittedly, + // growing slowly). + // + // Options: + // - Have some sort of expiration mechanism? This could turn the set of + // sleds here into "the sleds which we know were connected within the past + // hour", for example. + // - Have some other interface to identify the detachment of a peer. + our_address: Ipv6Addr, + sleds: Arc>>, + notification_sender: broadcast::Sender, _worker: JoinHandle<()>, } async fn monitor_worker( log: Logger, - address: SocketAddrV6, sender: UdpSocket, listener: UdpSocket, - sleds: Arc>>, + sleds: Arc>>, + notification_sender: broadcast::Sender, ) { // Let this message be a reminder that this content is *not* // encrypted, authenticated, or otherwise verified. We're just using @@ -37,16 +56,25 @@ async fn monitor_worker( let mut buf = vec![0u8; 128]; tokio::select! { _ = tokio::time::sleep(tokio::time::Duration::from_millis(5000)) => { - trace!(log, "Bootstrap Peer Monitor: Broadcasting our own address: {}", address); - if let Err(e) = sender.try_send_to(message, address.into()) { + if let Err(e) = sender.try_send_to(message, SocketAddr::V6(multicast::multicast_address())) { warn!(log, "PeerMonitor failed to broadcast: {}", e); } } result = listener.recv_from(&mut buf) => { match result { Ok((_, addr)) => { - info!(log, "Bootstrap Peer Monitor: Successfully received an address: {}", addr); - sleds.lock().await.insert(addr); + match addr { + SocketAddr::V6(addr) => { + let mut sleds = sleds.lock().await; + if sleds.insert(*addr.ip()) { + info!(log, "Bootstrap Peer Monitor: Successfully received an address: {}", addr); + // We don't actually care if no one is listening, so + // drop the error if that's the case. + let _ = notification_sender.send(*addr.ip()); + } + } + _ => continue, + } }, Err(e) => warn!(log, "PeerMonitor failed to receive: {}", e), } @@ -57,16 +85,7 @@ async fn monitor_worker( impl PeerMonitor { /// Creates a new [`PeerMonitor`]. - // TODO: Address, port, interface, etc, probably should be - // configuration options. - pub fn new(log: &Logger) -> Result { - let scope = multicast::Ipv6MulticastScope::LinkLocal.first_hextet(); - let address = SocketAddrV6::new( - Ipv6Addr::new(scope, 0, 0, 0, 0, 0, 0, 0x1), - 7645, - 0, - 0, - ); + pub fn new(log: &Logger, address: Ipv6Addr) -> Result { let loopback = false; let interface = 0; let (sender, listener) = @@ -76,18 +95,75 @@ impl PeerMonitor { let sleds_for_worker = sleds.clone(); let log = log.clone(); + let (tx, _) = tokio::sync::broadcast::channel(PEER_CAPACITY_MAXIMUM); + + let notification_sender = tx.clone(); let worker = tokio::task::spawn(async move { - monitor_worker(log, address, sender, listener, sleds_for_worker) - .await + monitor_worker( + log, + sender, + listener, + sleds_for_worker, + notification_sender, + ) + .await }); - Ok(PeerMonitor { sleds, _worker: worker }) + Ok(PeerMonitor { + our_address: address, + sleds, + notification_sender: tx, + _worker: worker, + }) } /// Returns the addresses of connected sleds. /// + /// For an interface that allows monitoring the connected sleds, rather + /// than just sampling at a single point-in-time, consider using + /// [`Self::observer`]. + /// /// Note: These sleds have not yet been verified. - pub async fn addrs(&self) -> Vec { + pub async fn peer_addrs(&self) -> Vec { self.sleds.lock().await.iter().map(|addr| *addr).collect() } + + /// Returns a [`PeerMonitorObserver`] which can be used to view the results + /// of monitoring for peers. + pub async fn observer(&self) -> PeerMonitorObserver { + PeerMonitorObserver { + our_address: self.our_address, + actual_sleds: self.sleds.clone(), + sender: self.notification_sender.clone(), + } + } +} + +/// Provides a read-only view of monitored peers, with a mechanism for +/// observing the incoming queue of new peers. +pub struct PeerMonitorObserver { + our_address: Ipv6Addr, + // A shared reference to the "true" set of sleds. + // + // This is only used to re-synchronize our set of sleds + // if we get out-of-sync due to long notification queues. + actual_sleds: Arc>>, + sender: broadcast::Sender, +} + +impl PeerMonitorObserver { + /// Returns the address of this sled. + pub fn our_address(&self) -> Ipv6Addr { + self.our_address + } + + /// Returns the current set of sleds and a receiver to hear about + /// new ones. + pub async fn subscribe( + &mut self, + ) -> (HashSet, broadcast::Receiver) { + let sleds = self.actual_sleds.lock().await; + let receiver = self.sender.subscribe(); + (sleds.clone(), receiver) + } } diff --git a/sled-agent/src/bootstrap/http_entrypoints.rs b/sled-agent/src/bootstrap/http_entrypoints.rs index c8a6bde01ca..8c2952a8054 100644 --- a/sled-agent/src/bootstrap/http_entrypoints.rs +++ b/sled-agent/src/bootstrap/http_entrypoints.rs @@ -34,14 +34,18 @@ use omicron_common::api::external::Error as ExternalError; use std::sync::Arc; use super::agent::Agent; -use super::{params::ShareRequest, views::ShareResponse}; +use super::{ + params::{ShareRequest, SledAgentRequest}, + views::{ShareResponse, SledAgentResponse}, +}; /// Returns a description of the bootstrap agent API pub(crate) fn ba_api() -> ApiDescription> { fn register_endpoints( api: &mut ApiDescription>, ) -> Result<(), String> { - api.register(api_request_share)?; + api.register(request_share)?; + api.register(start_sled)?; Ok(()) } @@ -56,7 +60,7 @@ pub(crate) fn ba_api() -> ApiDescription> { method = GET, path = "/request_share", }] -async fn api_request_share( +async fn request_share( rqctx: Arc>>, request: TypedBody, ) -> Result, HttpError> { @@ -70,3 +74,22 @@ async fn api_request_share( .map_err(|e| ExternalError::from(e))?, )) } + +#[endpoint { + method = PUT, + path = "/start_sled", +}] +async fn start_sled( + rqctx: Arc>>, + request: TypedBody, +) -> Result, HttpError> { + let bootstrap_agent = rqctx.context(); + + let request = request.into_inner(); + Ok(HttpResponseOk( + bootstrap_agent + .request_agent(request) + .await + .map_err(|e| ExternalError::from(e))?, + )) +} diff --git a/sled-agent/src/bootstrap/mod.rs b/sled-agent/src/bootstrap/mod.rs index 14bf4d8da96..da4df934d77 100644 --- a/sled-agent/src/bootstrap/mod.rs +++ b/sled-agent/src/bootstrap/mod.rs @@ -5,12 +5,12 @@ //! Bootstrap-related utilities pub mod agent; -mod client; +pub mod client; pub mod config; -mod discovery; +pub mod discovery; mod http_entrypoints; -mod multicast; -mod params; +pub mod multicast; +pub(crate) mod params; pub mod server; mod spdm; pub mod trust_quorum; diff --git a/sled-agent/src/bootstrap/multicast.rs b/sled-agent/src/bootstrap/multicast.rs index 78a611ff978..5bf69d3cc10 100644 --- a/sled-agent/src/bootstrap/multicast.rs +++ b/sled-agent/src/bootstrap/multicast.rs @@ -81,13 +81,7 @@ fn new_ipv6_udp_listener( socket.set_reuse_address(true)?; socket.join_multicast_v6(addr.ip(), interface)?; - - // TODO: I tried binding on the input value of "addr.ip()", but doing so - // returns errno 22 ("Invalid Input"). - // - // This may be binding to a larger address range than we want. - let bind_address = - SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, addr.port(), 0, 0); + let bind_address = SocketAddrV6::new(*addr.ip(), addr.port(), 0, 0); socket.bind(&(bind_address).into())?; // Convert from: socket2 -> std -> tokio @@ -96,99 +90,41 @@ fn new_ipv6_udp_listener( /// Create a new sending socket, capable of sending IPv6 multicast traffic. fn new_ipv6_udp_sender( + addr: &Ipv6Addr, loopback: bool, interface: u32, ) -> io::Result { let socket = new_ipv6_udp_socket()?; socket.set_multicast_loop_v6(loopback)?; socket.set_multicast_if_v6(interface)?; - let address = SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, 0, 0, 0); + let address = SocketAddrV6::new(*addr, 0, 0, 0); socket.bind(&address.into())?; UdpSocket::from_std(std::net::UdpSocket::from(socket)) } +pub fn multicast_address() -> SocketAddrV6 { + let scope = Ipv6MulticastScope::LinkLocal.first_hextet(); + SocketAddrV6::new(Ipv6Addr::new(scope, 0, 0, 0, 0, 0, 0, 0x1), 7645, 0, 0) +} + /// Returns the (sender, receiver) sockets of an IPv6 UDP multicast group. /// -/// * `address`: The address to use. Consider a value from: -/// , -/// and the [`Ipv6MulticastScope`] helper to provide the first hextet. +/// * `address`: The address to use for sending. /// * `loopback`: If true, the receiver packet will see multicast packets sent /// on our sender, in addition to those sent by everyone else in the multicast /// group. /// * `interface`: The index of the interface to join (zero indicates "any /// interface"). pub fn new_ipv6_udp_pair( - address: &SocketAddrV6, + address: &Ipv6Addr, loopback: bool, interface: u32, ) -> io::Result<(UdpSocket, UdpSocket)> { - let sender = new_ipv6_udp_sender(loopback, interface)?; - let listener = new_ipv6_udp_listener(address, interface)?; + let sender = new_ipv6_udp_sender(&address, loopback, interface)?; + let listener = new_ipv6_udp_listener(&multicast_address(), interface)?; Ok((sender, listener)) } -#[cfg(test)] -mod test { - use super::*; - - // NOTE: This test is ignored by default - it relies on a networking - // setup that isn't consistent between our automated test infrastructure. - // It can still be run locally with: - // - // $ cargo test -p omicron-sled-agent -- --ignored - #[tokio::test] - #[ignore] - async fn test_multicast_ipv6() { - let message = b"Hello World!"; - let scope = Ipv6MulticastScope::LinkLocal.first_hextet(); - let address = SocketAddrV6::new( - Ipv6Addr::new(scope, 0, 0, 0, 0, 0, 0, 0x1), - 7645, - 0, - 0, - ); - - // For this test, we want to see our own transmission. - // Unlike most usage in the Sled Agent, this means we want - // loopback to be enabled. - let loopback = true; - let interface = 0; - let (sender, listener) = - new_ipv6_udp_pair(&address, loopback, interface).unwrap(); - - // Create a receiver task which reads for messages that have - // been broadcast, verifies the message, and returns the - // calling address. - let receiver_task_handle = tokio::task::spawn(async move { - let mut buf = vec![0u8; 32]; - let (len, addr) = listener.recv_from(&mut buf).await?; - assert_eq!(message.len(), len); - assert_eq!(message, &buf[..message.len()]); - Ok::<_, io::Error>(addr) - }); - - // Send a message repeatedly, and exit successfully if we - // manage to receive the response. - tokio::pin!(receiver_task_handle); - let mut send_count = 0; - loop { - tokio::select! { - result = sender.send_to(message, address) => { - assert_eq!(message.len(), result.unwrap()); - send_count += 1; - if send_count > 10 { - panic!("10 multicast UDP messages sent with no response"); - } - tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; - } - result = &mut receiver_task_handle => { - let addr = result.unwrap().unwrap(); - eprintln!("Receiver received message: {:#?}", addr); - break; - } - } - } - } -} +// Refer to sled-agent/tests/integration_tests/multicast.rs for tests. diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs index ea9112c296f..b6c55bb1479 100644 --- a/sled-agent/src/bootstrap/params.rs +++ b/sled-agent/src/bootstrap/params.rs @@ -4,6 +4,7 @@ //! Request body types for the bootstrap agent +use omicron_common::api::external::Ipv6Net; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -13,3 +14,53 @@ pub struct ShareRequest { // TODO-completeness: format TBD; currently opaque. pub identity: Vec, } + +#[derive(thiserror::Error, Debug)] +pub enum SubnetError { + #[error("Subnet has unexpected prefix length: {0}")] + BadPrefixLength(u8), +} + +/// Represents subnets belonging to Sleds. +/// +/// This is a thin wrapper around the [`Ipv6Net`] type - which may be accessed +/// by [`AsRef`] - which adds additional validation that this is a /64 +/// subnet with an expected prefix. +// Note: The inner field is intentionally non-pub; this makes it +// more difficult to construct a sled subnet which avoids the +// validation performed by the constructor. +#[derive(Clone, Debug, Serialize, JsonSchema, PartialEq)] +pub struct SledSubnet(Ipv6Net); + +impl SledSubnet { + pub fn new(ip: Ipv6Net) -> Result { + let prefix = ip.0.prefix(); + if prefix != 64 { + return Err(SubnetError::BadPrefixLength(prefix)); + } + Ok(SledSubnet(ip)) + } +} + +impl<'de> serde::Deserialize<'de> for SledSubnet { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let net = Ipv6Net::deserialize(deserializer)?; + SledSubnet::new(net).map_err(serde::de::Error::custom) + } +} + +impl AsRef for SledSubnet { + fn as_ref(&self) -> &Ipv6Net { + &self.0 + } +} + +/// Configuration information for launching a Sled Agent. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct SledAgentRequest { + /// Portion of the IP space to be managed by the Sled Agent. + pub subnet: SledSubnet, +} diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index 5c8c668b5df..ab34d13b1dd 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -7,7 +7,9 @@ use super::agent::Agent; use super::config::Config; use super::http_entrypoints::ba_api as http_api; +use crate::config::Config as SledConfig; use slog::Drain; +use std::net::Ipv6Addr; use std::sync::Arc; /// Wraps a [Agent] object, and provides helper methods for exposing it @@ -18,7 +20,11 @@ pub struct Server { } impl Server { - pub async fn start(config: &Config) -> Result { + pub async fn start( + address: Ipv6Addr, + config: Config, + sled_config: SledConfig, + ) -> Result { let (drain, registration) = slog_dtrace::with_drain( config.log.to_logger("bootstrap-agent").map_err(|message| { format!("initializing logger: {}", message) @@ -38,8 +44,11 @@ impl Server { "component" => "BootstrapAgent", "server" => config.id.clone().to_string() )); - let bootstrap_agent = - Arc::new(Agent::new(ba_log).map_err(|e| e.to_string())?); + let bootstrap_agent = Arc::new( + Agent::new(ba_log, sled_config, address) + .await + .map_err(|e| e.to_string())?, + ); let ba = Arc::clone(&bootstrap_agent); let dropshot_log = log.new(o!("component" => "dropshot")); @@ -58,7 +67,7 @@ impl Server { // This ordering allows the bootstrap agent to communicate with // other bootstrap agents on the rack during the initialization // process. - if let Err(e) = server.bootstrap_agent.initialize(config).await { + if let Err(e) = server.bootstrap_agent.initialize(&config).await { let _ = server.close().await; return Err(e.to_string()); } diff --git a/sled-agent/src/bootstrap/trust_quorum/client.rs b/sled-agent/src/bootstrap/trust_quorum/client.rs index 30331bd6238..7eb1ff2808b 100644 --- a/sled-agent/src/bootstrap/trust_quorum/client.rs +++ b/sled-agent/src/bootstrap/trust_quorum/client.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::net::SocketAddr; +use std::net::{SocketAddr, SocketAddrV6}; use slog::Logger; use tokio::net::TcpStream; @@ -16,15 +16,15 @@ use crate::bootstrap::spdm; pub struct Client { log: Logger, verifier: Verifier, - addr: SocketAddr, + addr: SocketAddrV6, } impl Client { - pub fn new(log: &Logger, verifier: Verifier, addr: SocketAddr) -> Client { + pub fn new(log: &Logger, verifier: Verifier, addr: SocketAddrV6) -> Client { Client { log: log.clone(), verifier, addr } } - pub fn addr(&self) -> &SocketAddr { + pub fn addr(&self) -> &SocketAddrV6 { &self.addr } @@ -49,7 +49,7 @@ impl Client { if self.verifier.verify(&share) { Ok(share) } else { - Err(TrustQuorumError::InvalidShare(self.addr)) + Err(TrustQuorumError::InvalidShare(SocketAddr::V6(self.addr))) } } } diff --git a/sled-agent/src/bootstrap/views.rs b/sled-agent/src/bootstrap/views.rs index 5787fe0b8ae..56d3a9b80d9 100644 --- a/sled-agent/src/bootstrap/views.rs +++ b/sled-agent/src/bootstrap/views.rs @@ -6,6 +6,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use uuid::Uuid; /// Sent between bootstrap agents to establish trust quorum. #[derive(Serialize, Deserialize, JsonSchema)] @@ -13,3 +14,9 @@ pub struct ShareResponse { // TODO-completeness: format TBD; currently opaque. pub shared_secret: Vec, } + +/// Describes the Sled Agent running on the device. +#[derive(Serialize, Deserialize, JsonSchema)] +pub struct SledAgentResponse { + pub id: Uuid, +} diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 5812b89e839..b02fbdabcd9 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -5,26 +5,32 @@ //! Interfaces for working with sled agent configuration use crate::common::vlan::VlanID; -use crate::illumos::dladm::PhysicalLink; +use crate::illumos::dladm::{self, Dladm, PhysicalLink}; use crate::illumos::zpool::ZpoolName; -use dropshot::ConfigDropshot; use dropshot::ConfigLogging; +use omicron_common::api::external::Ipv6Net; use serde::Deserialize; -use std::net::SocketAddr; +use std::net::{SocketAddr, SocketAddrV6}; use std::path::Path; use uuid::Uuid; +pub const SLED_AGENT_PORT: u16 = 12345; + +/// Given a subnet, return the sled agent address. +pub(crate) fn get_sled_address(subnet: Ipv6Net) -> SocketAddrV6 { + let mut iter = subnet.iter(); + let _anycast_ip = iter.next().unwrap(); + let sled_agent_ip = iter.next().unwrap(); + SocketAddrV6::new(sled_agent_ip, SLED_AGENT_PORT, 0, 0) +} + /// Configuration for a sled agent #[derive(Clone, Debug, Deserialize)] pub struct Config { /// Unique id for the sled pub id: Uuid, - /// Address of the Bootstrap Agent interface. - pub bootstrap_address: SocketAddr, /// Address of Nexus instance pub nexus_address: SocketAddr, - /// Configuration for the sled agent dropshot server - pub dropshot: ConfigDropshot, /// Configuration for the sled agent debug log pub log: ConfigLogging, /// Optional VLAN ID to be used for tagging guest VNICs. @@ -53,4 +59,13 @@ impl Config { let config = toml::from_str(&contents)?; Ok(config) } + + pub fn get_link(&self) -> Result { + let link = if let Some(link) = self.data_link.clone() { + link + } else { + Dladm::find_physical()? + }; + Ok(link) + } } diff --git a/sled-agent/src/illumos/addrobj.rs b/sled-agent/src/illumos/addrobj.rs index 46ac2a5933c..80f41fd9010 100644 --- a/sled-agent/src/illumos/addrobj.rs +++ b/sled-agent/src/illumos/addrobj.rs @@ -13,7 +13,7 @@ /// ^ ^ /// | | AddrObject name /// | Interface name -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct AddrObject { interface: String, name: String, @@ -50,8 +50,8 @@ impl AddrObject { } } -impl ToString for AddrObject { - fn to_string(&self) -> String { - format!("{}/{}", self.interface, self.name) +impl std::fmt::Display for AddrObject { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}/{}", self.interface, self.name) } } diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs index ed99ad46cbb..92c1ed03916 100644 --- a/sled-agent/src/illumos/dladm.rs +++ b/sled-agent/src/illumos/dladm.rs @@ -8,6 +8,7 @@ use crate::common::vlan::VlanID; use crate::illumos::{execute, ExecutionError, PFEXEC}; use omicron_common::api::external::MacAddr; use serde::{Deserialize, Serialize}; +use std::str::FromStr; pub const VNIC_PREFIX: &str = "ox"; pub const VNIC_PREFIX_CONTROL: &str = "oxControl"; @@ -24,6 +25,9 @@ pub enum Error { #[error("Failed to parse output: {0}")] Parse(#[from] std::string::FromUtf8Error), + + #[error("Failed to parse MAC: {0}")] + ParseMac(#[from] macaddr::ParseError), } /// The name of a physical datalink. @@ -52,6 +56,37 @@ impl Dladm { Ok(PhysicalLink(name)) } + /// Returns the MAC address of a physical link. + pub fn get_mac(link: PhysicalLink) -> Result { + let mut command = std::process::Command::new(PFEXEC); + let cmd = command.args(&[ + DLADM, + "show-phys", + "-m", + "-p", + "-o", + "ADDRESS", + &link.0, + ]); + let output = execute(cmd)?; + let name = String::from_utf8(output.stdout)? + .lines() + .next() + .map(|s| s.trim()) + .ok_or_else(|| Error::NotFound)? + .to_string(); + + // Ensure the MAC address is zero-padded, so it may be parsed as a + // MacAddr. This converts segments like ":a" to ":0a". + let name = name + .split(':') + .map(|segment| format!("{:0>2}", segment)) + .collect::>() + .join(":"); + let mac = MacAddr::from_str(&name)?; + Ok(mac) + } + /// Creates a new VNIC atop a physical device. /// /// * `physical`: The physical link on top of which a device will be diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index 72ec5155d08..ceb7cb5c57c 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -6,7 +6,7 @@ use ipnetwork::IpNetwork; use slog::Logger; -use std::net::IpAddr; +use std::net::{IpAddr, Ipv6Addr}; use crate::illumos::addrobj::AddrObject; use crate::illumos::dladm::{Dladm, PhysicalLink, VNIC_PREFIX_CONTROL}; @@ -66,9 +66,6 @@ pub enum Error { #[error("Error accessing filesystem: {0}")] Filesystem(std::io::Error), - #[error("Unexpected IP address: {0}")] - Ip(IpNetwork), - #[error("Value not found")] NotFound, } @@ -268,7 +265,10 @@ impl Zones { .ok_or(Error::NotFound) } - /// Gets the address if one exists, creates one if one does not exist. + /// Ensures that an IP address on an interface matches the requested value. + /// + /// - If the address exists, ensure it has the desired value. + /// - If the address does not exist, create it. /// /// This address may be optionally within a zone `zone`. /// If `None` is supplied, the address is queried from the Global Zone. @@ -281,8 +281,13 @@ impl Zones { match Self::get_address(zone, addrobj) { Ok(addr) => { if let AddressRequest::Static(expected_addr) = addrtype { + // If the address is static, we need to validate that it + // matches the value we asked for. if addr != expected_addr { - return Err(Error::Ip(addr)); + // If the address doesn't match, try removing the old + // value before using the new one. + Self::delete_address(zone, addrobj)?; + return Self::create_address(zone, addrobj, addrtype); } } Ok(addr) @@ -357,7 +362,6 @@ impl Zones { addrobj: &AddrObject, addrtype: AddressRequest, ) -> Result<(), Error> { - // No link-local address was found, attempt to make one. let mut command = std::process::Command::new(PFEXEC); let mut args = vec![]; if let Some(zone) = zone { @@ -388,6 +392,27 @@ impl Zones { Ok(()) } + #[allow(clippy::needless_lifetimes)] + pub fn delete_address<'a>( + zone: Option<&'a str>, + addrobj: &AddrObject, + ) -> Result<(), Error> { + let mut command = std::process::Command::new(PFEXEC); + let mut args = vec![]; + if let Some(zone) = zone { + args.push(ZLOGIN.to_string()); + args.push(zone.to_string()); + }; + + args.push(IPADM.to_string()); + args.push("delete-addr".to_string()); + args.push(addrobj.to_string()); + + let cmd = command.args(args); + execute(cmd)?; + Ok(()) + } + // Ensures a link local IPv6 exists for the object. // // This is necessary for allocating IPv6 addresses on illumos. @@ -433,15 +458,10 @@ impl Zones { // from RSS. pub fn ensure_has_global_zone_v6_address( physical_link: Option, - address: IpAddr, + address: Ipv6Addr, + name: &str, ) -> Result<(), Error> { - if !address.is_ipv6() { - return Err(Error::Ip(address.into())); - } - - // Ensure that addrconf has been set up in the Global - // Zone. - + // Ensure that addrconf has been set up in the Global Zone. let link = if let Some(link) = physical_link { link } else { @@ -458,8 +478,8 @@ impl Zones { // prefix. Anything else must be routed through Sidecar. Self::ensure_address( None, - &gz_link_local_addrobj.on_same_interface("sled6")?, - AddressRequest::new_static(address, Some(64)), + &gz_link_local_addrobj.on_same_interface(name)?, + AddressRequest::new_static(IpAddr::V6(address), Some(64)), )?; Ok(()) } diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index d6610d69701..80951b04639 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -22,7 +22,7 @@ pub mod common; pub mod bootstrap; pub mod config; mod http_entrypoints; -mod illumos; +pub mod illumos; mod instance; mod instance_manager; mod nexus; diff --git a/sled-agent/src/rack_setup/config.rs b/sled-agent/src/rack_setup/config.rs index 3174777a00e..53545b28984 100644 --- a/sled-agent/src/rack_setup/config.rs +++ b/sled-agent/src/rack_setup/config.rs @@ -6,9 +6,10 @@ use crate::config::ConfigError; use crate::params::{DatasetEnsureBody, ServiceRequest}; +use ipnetwork::Ipv6Network; use serde::Deserialize; use serde::Serialize; -use std::net::SocketAddr; +use std::net::Ipv6Addr; use std::path::Path; /// Configuration for the "rack setup service", which is controlled during @@ -23,6 +24,8 @@ use std::path::Path; /// can act as a stand-in initialization service. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub struct SetupServiceConfig { + pub rack_subnet: Ipv6Addr, + #[serde(default, rename = "request")] pub requests: Vec, } @@ -30,9 +33,6 @@ pub struct SetupServiceConfig { /// A request to initialize a sled. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub struct SledRequest { - /// The Sled Agent address receiving these requests. - pub sled_address: SocketAddr, - /// Datasets to be created. #[serde(default, rename = "dataset")] pub datasets: Vec, @@ -42,6 +42,14 @@ pub struct SledRequest { pub services: Vec, } +fn new_network(addr: Ipv6Addr, prefix: u8) -> Ipv6Network { + let net = Ipv6Network::new(addr, prefix).unwrap(); + + // ipnetwork inputs/outputs the provided IPv6 address, unmodified by the + // prefix. We manually mask `addr` based on `prefix` ourselves. + Ipv6Network::new(net.network(), prefix).unwrap() +} + impl SetupServiceConfig { pub fn from_file>(path: P) -> Result { let path = path.as_ref(); @@ -49,4 +57,64 @@ impl SetupServiceConfig { let config = toml::from_str(&contents)?; Ok(config) } + + pub fn az_subnet(&self) -> Ipv6Network { + new_network(self.rack_subnet, 48) + } + + pub fn rack_subnet(&self) -> Ipv6Network { + new_network(self.rack_subnet, 56) + } + + pub fn sled_subnet(&self, index: u8) -> Ipv6Network { + let mut rack_network = self.rack_subnet().network().octets(); + + // To set bits distinguishing the /64 from the /56, we modify the 7th octet. + rack_network[7] = index; + Ipv6Network::new(Ipv6Addr::from(rack_network), 64).unwrap() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_subnets() { + let cfg = SetupServiceConfig { + rack_subnet: "fd00:1122:3344:0100::".parse().unwrap(), + requests: vec![], + }; + + assert_eq!( + // Masked out in AZ Subnet + // vv + "fd00:1122:3344:0000::/48".parse::().unwrap(), + cfg.az_subnet() + ); + assert_eq!( + // Shows up from Rack Subnet + // vv + "fd00:1122:3344:0100::/56".parse::().unwrap(), + cfg.rack_subnet() + ); + assert_eq!( + // 0th Sled Subnet + // vv + "fd00:1122:3344:0100::/64".parse::().unwrap(), + cfg.sled_subnet(0) + ); + assert_eq!( + // 1st Sled Subnet + // vv + "fd00:1122:3344:0101::/64".parse::().unwrap(), + cfg.sled_subnet(1) + ); + assert_eq!( + // Last Sled Subnet + // vv + "fd00:1122:3344:01ff::/64".parse::().unwrap(), + cfg.sled_subnet(255) + ); + } } diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 84b2b40bd8b..c3de81fd451 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -4,12 +4,22 @@ //! Rack Setup Service implementation -use super::config::SetupServiceConfig as Config; +use super::config::{SetupServiceConfig as Config, SledRequest}; +use crate::bootstrap::{ + client as bootstrap_agent_client, config::BOOTSTRAP_AGENT_PORT, + discovery::PeerMonitorObserver, params::SledAgentRequest, + params::SledSubnet, +}; +use crate::config::get_sled_address; use omicron_common::backoff::{ internal_service_policy, retry_notify, BackoffError, }; +use serde::{Deserialize, Serialize}; use slog::Logger; +use std::collections::{HashMap, HashSet}; +use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use thiserror::Error; +use tokio::sync::Mutex; /// Describes errors which may occur while operating the setup service. #[derive(Error, Debug)] @@ -17,12 +27,21 @@ pub enum SetupServiceError { #[error("Error accessing filesystem: {0}")] Io(#[from] std::io::Error), + #[error("Error making HTTP request to Bootstrap Agent: {0}")] + BootstrapApi( + #[from] + bootstrap_agent_client::Error, + ), + #[error("Error making HTTP request to Sled Agent: {0}")] SledApi(#[from] sled_agent_client::Error), #[error("Cannot deserialize TOML file")] Toml(#[from] toml::de::Error), + #[error("Failed to monitor for peers: {0}")] + PeerMonitor(#[from] tokio::sync::broadcast::error::RecvError), + #[error(transparent)] Http(#[from] reqwest::Error), @@ -30,16 +49,39 @@ pub enum SetupServiceError { Configuration, } +// The workload / information allocated to a single sled. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +struct SledAllocation { + initialization_request: SledAgentRequest, + services_request: SledRequest, +} + /// The interface to the Rack Setup Service. pub struct Service { handle: tokio::task::JoinHandle>, } impl Service { - pub fn new(log: Logger, config: Config) -> Self { + /// Creates a new rack setup service, which runs in a background task. + /// + /// Arguments: + /// - `log`: The logger. + /// - `config`: The config file, which is used to setup the rack. + /// - `peer_monitor`: The mechanism by which the setup service discovers + /// bootstrap agents on nearby sleds. + pub fn new( + log: Logger, + config: Config, + peer_monitor: PeerMonitorObserver, + ) -> Self { let handle = tokio::task::spawn(async move { - let svc = ServiceInner::new(log); - svc.inject_rack_setup_requests(&config).await + let svc = ServiceInner::new(log.clone(), peer_monitor); + if let Err(e) = svc.inject_rack_setup_requests(&config).await { + warn!(log, "RSS injection failed: {}", e); + Err(e) + } else { + Ok(()) + } }); Service { handle } @@ -51,176 +93,452 @@ impl Service { } } +fn rss_plan_path() -> std::path::PathBuf { + std::path::Path::new(omicron_common::OMICRON_CONFIG_PATH) + .join("rss-plan.toml") +} + +fn rss_completed_plan_path() -> std::path::PathBuf { + std::path::Path::new(omicron_common::OMICRON_CONFIG_PATH) + .join("rss-plan-completed.toml") +} + +// Describes the options when awaiting for peers. +enum PeerExpectation { + // Await a set of peers that matches this group of IPv6 addresses exactly. + // + // TODO: We currently don't deal with the case where: + // + // - RSS boots, sees some sleds, comes up with a plan. + // - RSS reboots, sees a *different* set of sleds, and needs + // to adjust the plan. + // + // This case is fairly tricky because some sleds may have + // already received requests to initialize - modifying the + // allocated subnets would be non-trivial. + LoadOldPlan(HashSet), + // Await any peers, as long as there are at least enough to make a new plan. + CreateNewPlan(usize), +} + /// The implementation of the Rack Setup Service. struct ServiceInner { log: Logger, + peer_monitor: Mutex, } impl ServiceInner { - pub fn new(log: Logger) -> Self { - ServiceInner { log } + fn new(log: Logger, peer_monitor: PeerMonitorObserver) -> Self { + ServiceInner { log, peer_monitor: Mutex::new(peer_monitor) } + } + + async fn initialize_sled_agent( + &self, + bootstrap_addr: SocketAddrV6, + request: &SledAgentRequest, + ) -> Result<(), SetupServiceError> { + let dur = std::time::Duration::from_secs(60); + + let client = reqwest::ClientBuilder::new() + .connect_timeout(dur) + .timeout(dur) + .build()?; + + let url = format!("http://{}", bootstrap_addr); + info!(self.log, "Sending request to peer agent: {}", url); + let client = bootstrap_agent_client::Client::new_with_client( + &url, + client, + self.log.new(o!("BootstrapAgentClient" => url.clone())), + ); + + let sled_agent_initialize = || async { + client + .start_sled(&bootstrap_agent_client::types::SledAgentRequest { + subnet: bootstrap_agent_client::types::SledSubnet( + bootstrap_agent_client::types::Ipv6Net( + request.subnet.as_ref().to_string(), + ), + ), + }) + .await + .map_err(BackoffError::transient)?; + + Ok::< + (), + BackoffError< + bootstrap_agent_client::Error< + bootstrap_agent_client::types::Error, + >, + >, + >(()) + }; + + let log_failure = |error, _| { + warn!(self.log, "failed to start sled agent"; "error" => ?error); + }; + retry_notify( + internal_service_policy(), + sled_agent_initialize, + log_failure, + ) + .await?; + info!(self.log, "Peer agent at {} initialized", url); + Ok(()) + } + + async fn initialize_datasets( + &self, + sled_address: SocketAddr, + datasets: &Vec, + ) -> Result<(), SetupServiceError> { + let dur = std::time::Duration::from_secs(60); + + let client = reqwest::ClientBuilder::new() + .connect_timeout(dur) + .timeout(dur) + .build()?; + let client = sled_agent_client::Client::new_with_client( + &format!("http://{}", sled_address), + client, + self.log.new(o!("SledAgentClient" => sled_address)), + ); + + info!(self.log, "sending dataset requests..."); + for dataset in datasets { + let filesystem_put = || async { + info!(self.log, "creating new filesystem: {:?}", dataset); + client + .filesystem_put(&dataset.clone().into()) + .await + .map_err(BackoffError::transient)?; + Ok::< + (), + BackoffError< + sled_agent_client::Error< + sled_agent_client::types::Error, + >, + >, + >(()) + }; + let log_failure = |error, _| { + warn!(self.log, "failed to create filesystem"; "error" => ?error); + }; + retry_notify( + internal_service_policy(), + filesystem_put, + log_failure, + ) + .await?; + } + Ok(()) + } + + async fn initialize_services( + &self, + sled_address: SocketAddr, + services: &Vec, + ) -> Result<(), SetupServiceError> { + let dur = std::time::Duration::from_secs(60); + let client = reqwest::ClientBuilder::new() + .connect_timeout(dur) + .timeout(dur) + .build()?; + let client = sled_agent_client::Client::new_with_client( + &format!("http://{}", sled_address), + client, + self.log.new(o!("SledAgentClient" => sled_address)), + ); + + info!(self.log, "sending service requests..."); + let services_put = || async { + info!(self.log, "initializing sled services: {:?}", services); + client + .services_put(&sled_agent_client::types::ServiceEnsureBody { + services: services + .iter() + .map(|s| s.clone().into()) + .collect(), + }) + .await + .map_err(BackoffError::transient)?; + Ok::< + (), + BackoffError< + sled_agent_client::Error, + >, + >(()) + }; + let log_failure = |error, _| { + warn!(self.log, "failed to initialize services"; "error" => ?error); + }; + retry_notify(internal_service_policy(), services_put, log_failure) + .await?; + Ok(()) + } + + async fn load_plan( + &self, + ) -> Result>, SetupServiceError> + { + // If we already created a plan for this RSS to allocate + // subnets/requests to sleds, re-use that existing plan. + let rss_plan_path = rss_plan_path(); + if rss_plan_path.exists() { + info!(self.log, "RSS plan already created, loading from file"); + + let plan: std::collections::HashMap = + toml::from_str( + &tokio::fs::read_to_string(&rss_plan_path).await?, + )?; + Ok(Some(plan)) + } else { + Ok(None) + } + } + + async fn create_plan( + &self, + config: &Config, + addrs: impl IntoIterator, + ) -> Result, SetupServiceError> { + let addrs = addrs.into_iter().enumerate(); + + // TODO: The use of "zip" here means that if we have more addrs than + // requests, we won't initialize some of them. Maybe that's okay? + // Maybe that's the responsibility of Nexus? + let requests_and_sleds = config.requests.iter().zip(addrs); + + let allocations = requests_and_sleds.map(|(request, sled)| { + let (idx, bootstrap_addr) = sled; + info!( + self.log, + "Creating plan for the sled at {:?}", bootstrap_addr + ); + let bootstrap_addr = + SocketAddrV6::new(bootstrap_addr, BOOTSTRAP_AGENT_PORT, 0, 0); + let sled_subnet_index = + u8::try_from(idx + 1).expect("Too many peers!"); + let subnet = + SledSubnet::new(config.sled_subnet(sled_subnet_index).into()) + .expect("Created Invalid Subnet"); + + ( + bootstrap_addr, + SledAllocation { + initialization_request: SledAgentRequest { subnet }, + services_request: request.clone(), + }, + ) + }); + + info!(self.log, "Serializing plan"); + + let mut plan = std::collections::HashMap::new(); + for (addr, allocation) in allocations { + plan.insert(addr, allocation); + } + + // Once we've constructed a plan, write it down to durable storage. + let serialized_plan = toml::Value::try_from(&plan) + .expect("Cannot serialize configuration"); + let plan_str = toml::to_string(&serialized_plan) + .expect("Cannot turn config to string"); + + info!(self.log, "Plan serialized as: {}", plan_str); + tokio::fs::write(&rss_plan_path(), plan_str).await?; + info!(self.log, "Plan written to storage"); + + Ok(plan) + } + + // Waits for sufficient neighbors to exist so the initial set of requests + // can be send out. + async fn wait_for_peers( + &self, + expectation: PeerExpectation, + ) -> Result, SetupServiceError> { + let mut peer_monitor = self.peer_monitor.lock().await; + let (mut all_addrs, mut peer_rx) = peer_monitor.subscribe().await; + all_addrs.insert(peer_monitor.our_address()); + + loop { + { + match expectation { + PeerExpectation::LoadOldPlan(ref expected) => { + if all_addrs.is_superset(expected) { + return Ok(all_addrs + .into_iter() + .collect::>()); + } + info!(self.log, "Waiting for a LoadOldPlan set of peers; not found yet."); + } + PeerExpectation::CreateNewPlan(wanted_peer_count) => { + if all_addrs.len() >= wanted_peer_count { + return Ok(all_addrs + .into_iter() + .collect::>()); + } + info!( + self.log, + "Waiting for {} peers (currently have {})", + wanted_peer_count, + all_addrs.len(), + ); + } + } + } + + info!(self.log, "Waiting for more peers"); + let new_peer = peer_rx.recv().await?; + all_addrs.insert(new_peer); + } } // In lieu of having an operator send requests to all sleds via an // initialization service, the sled-agent configuration may allow for the // automated injection of setup requests from a sled. + // + // This method has a few distinct phases, identified by files in durable + // storage: + // + // 1. ALLOCATION PLAN CREATION. When the RSS starts up for the first time, + // it creates an allocation plan to provision subnets and services + // to an initial set of sleds. + // + // This plan is stored at "rss_plan_path()". + // + // 2. ALLOCATION PLAN EXECUTION. The RSS then carries out this plan, making + // requests to the sleds enumerated within the "allocation plan". + // + // 3. MARKING SETUP COMPLETE. Once the RSS has successfully initialized the + // rack, the "rss_plan_path()" file is renamed to + // "rss_completed_plan_path()". This indicates that the plan executed + // successfully, and no work remains. async fn inject_rack_setup_requests( &self, config: &Config, ) -> Result<(), SetupServiceError> { info!(self.log, "Injecting RSS configuration: {:#?}", config); - let serialized_config = toml::Value::try_from(&config) - .expect("Cannot serialize configuration"); - let config_str = toml::to_string(&serialized_config) - .expect("Cannot turn config to string"); + // We expect this directory to exist - ensure that it does, before any + // subsequent operations which may write configs here. + tokio::fs::create_dir_all(omicron_common::OMICRON_CONFIG_PATH).await?; - // First, check if this request has previously been made. - // - // Normally, the rack setup service is run with a human-in-the-loop, - // but with this automated injection, we need a way to determine the - // (destructive) initialization has occurred. + // Check if a previous RSS plan has completed successfully. // - // We do this by storing the configuration at "rss_config_path" - // after successfully performing initialization. - let rss_config_path = - std::path::Path::new(omicron_common::OMICRON_CONFIG_PATH) - .join("config-rss.toml"); - if rss_config_path.exists() { + // If it has, the system should be up-and-running. + let rss_completed_plan_path = rss_completed_plan_path(); + if rss_completed_plan_path.exists() { + // TODO(https://github.com/oxidecomputer/omicron/issues/724): If the + // running configuration doesn't match Config, we could try to + // update things. info!( self.log, - "RSS configuration already exists at {}", - rss_config_path.to_string_lossy() + "RSS configuration looks like it has already been applied", ); - let old_config: Config = toml::from_str( - &tokio::fs::read_to_string(&rss_config_path).await?, - )?; - if &old_config == config { - info!( - self.log, - "RSS config already applied from: {}", - rss_config_path.to_string_lossy() - ); - return Ok(()); - } - - // TODO(https://github.com/oxidecomputer/omicron/issues/724): - // We could potentially handle this case by deleting all - // datasets (in preparation for applying the new - // configuration), but at the moment it's an error. - warn!( - self.log, - "Rack Setup Service Config was already applied, but has changed. - This means that you may have datasets set up on this sled, but they - may not match the ones requested by the supplied configuration.\n - To re-initialize this sled: - - Disable all Oxide services - - Delete all datasets within the attached zpool - - Delete the configuration file ({}) - - Restart the sled agent", - rss_config_path.to_string_lossy() - ); - return Err(SetupServiceError::Configuration); + return Ok(()); } else { - info!( - self.log, - "No RSS configuration found at {}", - rss_config_path.to_string_lossy() - ); + info!(self.log, "RSS configuration has not been fully applied yet",); } + // Wait for either: + // - All the peers to re-load an old plan (if one exists) + // - Enough peers to create a new plan (if one does not exist) + let maybe_plan = self.load_plan().await?; + let expectation = if let Some(plan) = &maybe_plan { + PeerExpectation::LoadOldPlan(plan.keys().map(|a| *a.ip()).collect()) + } else { + PeerExpectation::CreateNewPlan(config.requests.len()) + }; + let addrs = self.wait_for_peers(expectation).await?; + info!(self.log, "Enough peers exist to enact RSS plan"); + + // If we created a plan, reuse it. Otherwise, create a new plan. + // + // NOTE: This is a "point-of-no-return" -- before sending any requests + // to neighboring sleds, the plan must be recorded to durable storage. + // This way, if the RSS power-cycles, it can idempotently execute the + // same allocation plan. + let plan = if let Some(plan) = maybe_plan { + info!(self.log, "Re-using existing allocation plan"); + plan + } else { + info!(self.log, "Creating new allocation plan"); + self.create_plan(config, addrs).await? + }; + // Issue the dataset initialization requests to all sleds. - futures::future::join_all( - config.requests.iter().map(|request| async move { - info!(self.log, "observing request: {:#?}", request); - let dur = std::time::Duration::from_secs(60); - let client = reqwest::ClientBuilder::new() - .connect_timeout(dur) - .timeout(dur) - .build()?; - let client = sled_agent_client::Client::new_with_client( - &format!("http://{}", request.sled_address), - client, - self.log.new(o!("SledAgentClient" => request.sled_address)), + futures::future::join_all(plan.iter().map( + |(bootstrap_addr, allocation)| async move { + info!( + self.log, + "Sending request: {:#?}", allocation.initialization_request ); - info!(self.log, "sending dataset requests..."); - for dataset in &request.datasets { - let filesystem_put = || async { - info!(self.log, "creating new filesystem: {:?}", dataset); - client.filesystem_put(&dataset.clone().into()) - .await - .map_err(BackoffError::transient)?; - Ok::< - (), - BackoffError< - sled_agent_client::Error, - >, - >(()) - }; - let log_failure = |error, _| { - warn!(self.log, "failed to create filesystem"; "error" => ?error); - }; - retry_notify( - internal_service_policy(), - filesystem_put, - log_failure, - ).await?; - } + // First, connect to the Bootstrap Agent and tell it to + // initialize the Sled Agent with the specified subnet. + self.initialize_sled_agent( + *bootstrap_addr, + &allocation.initialization_request, + ) + .await?; + info!( + self.log, + "Initialized sled agent on sled with bootstrap address: {}", + bootstrap_addr + ); + + // Next, initialize any datasets on sleds that need it. + let sled_address = SocketAddr::V6(get_sled_address( + *allocation.initialization_request.subnet.as_ref(), + )); + self.initialize_datasets( + sled_address, + &allocation.services_request.datasets, + ) + .await?; Ok(()) - }) - ).await.into_iter().collect::, SetupServiceError>>()?; + }, + )) + .await + .into_iter() + .collect::>()?; + + info!(self.log, "Finished setting up agents and datasets"); // Issue service initialization requests. // // Note that this must happen *after* the dataset initialization, // to ensure that CockroachDB has been initialized before Nexus // starts. - futures::future::join_all( - config.requests.iter().map(|request| async move { - info!(self.log, "observing request: {:#?}", request); - let dur = std::time::Duration::from_secs(60); - let client = reqwest::ClientBuilder::new() - .connect_timeout(dur) - .timeout(dur) - .build()?; - let client = sled_agent_client::Client::new_with_client( - &format!("http://{}", request.sled_address), - client, - self.log.new(o!("SledAgentClient" => request.sled_address)), - ); + futures::future::join_all(plan.iter().map( + |(_, allocation)| async move { + let sled_address = SocketAddr::V6(get_sled_address( + *allocation.initialization_request.subnet.as_ref(), + )); + self.initialize_services( + sled_address, + &allocation.services_request.services, + ) + .await?; + Ok(()) + }, + )) + .await + .into_iter() + .collect::, SetupServiceError>>()?; - info!(self.log, "sending service requests..."); - let services_put = || async { - info!(self.log, "initializing sled services: {:?}", request.services); - client.services_put( - &sled_agent_client::types::ServiceEnsureBody { - services: request.services.iter().map(|s| s.clone().into()).collect() - }) - .await - .map_err(BackoffError::transient)?; - Ok::< - (), - BackoffError< - sled_agent_client::Error, - >, - >(()) - }; - let log_failure = |error, _| { - warn!(self.log, "failed to initialize services"; "error" => ?error); - }; - retry_notify( - internal_service_policy(), - services_put, - log_failure, - ).await?; - Ok::<(), SetupServiceError>(()) - }) - ).await.into_iter().collect::, SetupServiceError>>()?; + info!(self.log, "Finished setting up services"); // Finally, make sure the configuration is saved so we don't inject // the requests on the next iteration. - tokio::fs::write(rss_config_path, config_str).await?; + tokio::fs::rename(rss_plan_path(), rss_completed_plan_path).await?; + + // TODO Questions to consider: + // - What if a sled comes online *right after* this setup? How does + // it get a /64? + Ok(()) } } diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index 0c8c68331a5..00c141a0358 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -8,23 +8,36 @@ use super::config::Config; use super::http_entrypoints::api as http_api; use super::sled_agent::SledAgent; use crate::nexus::NexusClient; -use slog::Drain; - use omicron_common::backoff::{ internal_service_policy, retry_notify, BackoffError, }; +use slog::Drain; +use std::net::{SocketAddr, SocketAddrV6}; use std::sync::Arc; +use uuid::Uuid; /// Packages up a [`SledAgent`], running the sled agent API under a Dropshot /// server wired up to the sled agent pub struct Server { /// Dropshot server for the API. http_server: dropshot::HttpServer, + _nexus_notifier_handle: tokio::task::JoinHandle<()>, } impl Server { + pub fn address(&self) -> SocketAddr { + self.http_server.local_addr() + } + + pub fn id(&self) -> Uuid { + self.http_server.app_private().id() + } + /// Starts a SledAgent server - pub async fn start(config: &Config) -> Result { + pub async fn start( + config: &Config, + addr: SocketAddrV6, + ) -> Result { let (drain, registration) = slog_dtrace::with_drain( config.log.to_logger("sled-agent").map_err(|message| { format!("initializing logger: {}", message) @@ -50,13 +63,17 @@ impl Server { "component" => "SledAgent", "server" => config.id.clone().to_string() )); - let sled_agent = SledAgent::new(&config, sa_log, nexus_client.clone()) - .await - .map_err(|e| e.to_string())?; + let sled_agent = + SledAgent::new(&config, sa_log, nexus_client.clone(), addr) + .await + .map_err(|e| e.to_string())?; + let mut dropshot_config = dropshot::ConfigDropshot::default(); + dropshot_config.request_body_max_bytes = 1024 * 1024; + dropshot_config.bind_address = SocketAddr::V6(addr); let dropshot_log = log.new(o!("component" => "dropshot")); let http_server = dropshot::HttpServerStarter::new( - &config.dropshot, + &dropshot_config, http_api(), sled_agent, &dropshot_log, @@ -64,42 +81,48 @@ impl Server { .map_err(|error| format!("initializing server: {}", error))? .start(); - // Notify the control plane that we're up, and continue trying this - // until it succeeds. We retry with an randomized, capped exponential - // backoff. - // - // TODO-robustness if this returns a 400 error, we probably want to - // return a permanent error from the `notify_nexus` closure. - let sa_address = http_server.local_addr(); - let notify_nexus = || async { - info!( - log, - "contacting server nexus, registering sled: {}", config.id - ); - nexus_client - .cpapi_sled_agents_post( - &config.id, - &nexus_client::types::SledAgentStartupInfo { - sa_address: sa_address.to_string(), - }, - ) - .await - .map_err(BackoffError::transient) - }; - let log_notification_failure = |_, delay| { - warn!( - log, - "failed to contact nexus, will retry in {:?}", delay; - ); - }; - retry_notify( - internal_service_policy(), - notify_nexus, - log_notification_failure, - ) - .await - .expect("Expected an infinite retry loop contacting Nexus"); - Ok(Server { http_server }) + let sled_address = http_server.local_addr(); + let sled_id = config.id; + let nexus_notifier_handle = tokio::task::spawn(async move { + // Notify the control plane that we're up, and continue trying this + // until it succeeds. We retry with an randomized, capped exponential + // backoff. + // + // TODO-robustness if this returns a 400 error, we probably want to + // return a permanent error from the `notify_nexus` closure. + let notify_nexus = || async { + info!( + log, + "contacting server nexus, registering sled: {}", sled_id + ); + nexus_client + .cpapi_sled_agents_post( + &sled_id, + &nexus_client::types::SledAgentStartupInfo { + sa_address: sled_address.to_string(), + }, + ) + .await + .map_err(BackoffError::transient) + }; + let log_notification_failure = |_, delay| { + warn!( + log, + "failed to contact nexus, will retry in {:?}", delay; + ); + }; + retry_notify( + internal_service_policy(), + notify_nexus, + log_notification_failure, + ) + .await + .expect("Expected an infinite retry loop contacting Nexus"); + }); + Ok(Server { + http_server, + _nexus_notifier_handle: nexus_notifier_handle, + }) } /// Wait for the given server to shut down diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index cdf65d9cf8c..df3e9e816a4 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -21,7 +21,7 @@ use omicron_common::api::{ internal::nexus::UpdateArtifact, }; use slog::Logger; -use std::net::SocketAddr; +use std::net::{SocketAddr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -68,6 +68,9 @@ impl From for omicron_common::api::external::Error { /// /// Contains both a connection to the Nexus, as well as managed instances. pub struct SledAgent { + // ID of the Sled + id: Uuid, + // Component of Sled Agent responsible for storage and dataset management. storage: StorageManager, @@ -86,6 +89,7 @@ impl SledAgent { config: &Config, log: Logger, nexus_client: Arc, + sled_address: SocketAddrV6, ) -> Result { let id = &config.id; let vlan = config.vlan; @@ -110,7 +114,8 @@ impl SledAgent { // configuration file. Zones::ensure_has_global_zone_v6_address( config.data_link.clone(), - config.dropshot.bind_address.ip(), + *sled_address.ip(), + "sled6", )?; // Identify all existing zones which should be managed by the Sled @@ -168,7 +173,17 @@ impl SledAgent { ServiceManager::new(log.clone(), config.data_link.clone(), None) .await?; - Ok(SledAgent { storage, instances, nexus_client, services }) + Ok(SledAgent { + id: config.id, + storage, + instances, + nexus_client, + services, + }) + } + + pub fn id(&self) -> Uuid { + self.id } /// Ensures that particular services should be initialized. diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index ab98a688b54..ff09437993e 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -208,6 +208,7 @@ impl DatasetInfo { ) -> Result<(), Error> { match self.kind { DatasetKind::CockroachDb { .. } => { + info!(log, "start_zone: Loading CRDB manifest"); // Load the CRDB manifest. zone.run_cmd(&[ crate::illumos::zone::SVCCFG, @@ -216,6 +217,11 @@ impl DatasetInfo { ])?; // Set parameters which are passed to the CRDB binary. + info!( + log, + "start_zone: setting CRDB's config/listen_addr: {}", + address + ); zone.run_cmd(&[ crate::illumos::zone::SVCCFG, "-s", @@ -223,6 +229,8 @@ impl DatasetInfo { "setprop", &format!("config/listen_addr={}", address), ])?; + + info!(log, "start_zone: setting CRDB's config/store"); zone.run_cmd(&[ crate::illumos::zone::SVCCFG, "-s", @@ -234,6 +242,7 @@ impl DatasetInfo { // // Set these addresses, use "start" instead of // "start-single-node". + info!(log, "start_zone: setting CRDB's config/join_addrs"); zone.run_cmd(&[ crate::illumos::zone::SVCCFG, "-s", @@ -244,6 +253,7 @@ impl DatasetInfo { // Refresh the manifest with the new properties we set, // so they become "effective" properties when the service is enabled. + info!(log, "start_zone: refreshing manifest"); zone.run_cmd(&[ crate::illumos::zone::SVCCFG, "-s", @@ -251,6 +261,7 @@ impl DatasetInfo { "refresh", ])?; + info!(log, "start_zone: enabling CRDB service"); zone.run_cmd(&[ crate::illumos::zone::SVCADM, "enable", @@ -259,6 +270,7 @@ impl DatasetInfo { ])?; // Await liveness of the cluster. + info!(log, "start_zone: awaiting liveness of CRDB"); let check_health = || async { let http_addr = SocketAddr::new(address.ip(), 8080); reqwest::get(format!("http://{}/health?ready=1", http_addr)) @@ -658,6 +670,7 @@ impl StorageWorker { nexus_notifications: &mut FuturesOrdered>>, request: &NewFilesystemRequest, ) -> Result<(), Error> { + info!(self.log, "add_dataset: {:?}", request); let mut pools = self.pools.lock().await; let name = ZpoolName::new(request.zpool_id); let pool = pools.get_mut(&name).ok_or_else(|| { diff --git a/sled-agent/tests/test_commands.rs b/sled-agent/tests/integration_tests/commands.rs similarity index 100% rename from sled-agent/tests/test_commands.rs rename to sled-agent/tests/integration_tests/commands.rs diff --git a/sled-agent/tests/integration_tests/mod.rs b/sled-agent/tests/integration_tests/mod.rs new file mode 100644 index 00000000000..6c6686f6543 --- /dev/null +++ b/sled-agent/tests/integration_tests/mod.rs @@ -0,0 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +mod commands; +#[cfg(target_os = "illumos")] +mod multicast; diff --git a/sled-agent/tests/integration_tests/multicast.rs b/sled-agent/tests/integration_tests/multicast.rs new file mode 100644 index 00000000000..1aa9f8b103f --- /dev/null +++ b/sled-agent/tests/integration_tests/multicast.rs @@ -0,0 +1,87 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use omicron_sled_agent::bootstrap; +use omicron_sled_agent::illumos::addrobj::AddrObject; +use omicron_sled_agent::illumos::{dladm, zone}; +use std::io; +use std::net::IpAddr; + +struct AddressCleanup { + addrobj: AddrObject, +} + +impl Drop for AddressCleanup { + fn drop(&mut self) { + let _ = zone::Zones::delete_address(None, &self.addrobj); + } +} + +#[tokio::test] +async fn test_multicast_bootstrap_address() { + // Setup the bootstrap address. + // + // This modifies global state of the target machine, creating + // an address named "testbootstrap6", akin to what the bootstrap + // agent should do. + let link = dladm::Dladm::find_physical().unwrap(); + let address = bootstrap::agent::bootstrap_address(link.clone()).unwrap(); + let address_name = "testbootstrap6"; + let addrobj = AddrObject::new(&link.0, address_name).unwrap(); + zone::Zones::ensure_has_global_zone_v6_address( + Some(link), + *address.ip(), + address_name, + ) + .unwrap(); + + // Cleanup-on-drop removal of the bootstrap address. + let _cleanup = AddressCleanup { addrobj }; + + // Create the multicast pair. + let loopback = true; + let interface = 0; + let (sender, listener) = bootstrap::multicast::new_ipv6_udp_pair( + address.ip(), + loopback, + interface, + ) + .unwrap(); + + // Create a receiver task which reads for messages that have + // been broadcast, verifies the message, and returns the + // calling address. + let message = b"Hello World!"; + let receiver_task_handle = tokio::task::spawn(async move { + let mut buf = vec![0u8; 32]; + let (len, addr) = listener.recv_from(&mut buf).await?; + assert_eq!(message.len(), len); + assert_eq!(message, &buf[..message.len()]); + assert_eq!(addr.ip(), IpAddr::V6(*address.ip())); + Ok::<_, io::Error>(addr) + }); + + // Send a message repeatedly, and exit successfully if we + // manage to receive the response. + tokio::pin!(receiver_task_handle); + let mut send_count = 0; + loop { + tokio::select! { + result = sender.send_to(message, bootstrap::multicast::multicast_address()) => { + assert_eq!(message.len(), result.unwrap()); + send_count += 1; + if send_count > 10 { + panic!("10 multicast UDP messages sent with no response"); + } + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + } + result = &mut receiver_task_handle => { + let addr = result.unwrap().unwrap(); + eprintln!("Receiver received message: {:#?}", addr); + assert_eq!(addr.ip(), IpAddr::V6(*address.ip())); + break; + } + } + } +} diff --git a/sled-agent/tests/mod.rs b/sled-agent/tests/mod.rs new file mode 100644 index 00000000000..325a59bfe25 --- /dev/null +++ b/sled-agent/tests/mod.rs @@ -0,0 +1,17 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Integration tests for the Sled Agent. +//! +//! Why use this weird layer of indirection, you might ask? Cargo chooses to +//! compile *each file* within the "tests/" subdirectory as a separate crate. +//! This means that doing "file-granularity" conditional compilation is +//! difficult, since a file like "test_for_illumos_only.rs" would get compiled +//! and tested regardless of the contents of "mod.rs". +//! +//! However, by lumping all tests into a submodule, all integration tests are +//! joined into a single crate, which itself can filter individual files +//! by (for example) choice of target OS. + +mod integration_tests; diff --git a/smf/nexus/config.toml b/smf/nexus/config.toml index 02f663f7c94..d4030efc9d4 100644 --- a/smf/nexus/config.toml +++ b/smf/nexus/config.toml @@ -18,15 +18,15 @@ schemes_external = ["spoof", "session_cookie"] [database] # URL for connecting to the database -url = "postgresql://root@[fd00:1de::5]:32221/omicron?sslmode=disable" +url = "postgresql://root@[fd00:1122:3344:1::2]:32221/omicron?sslmode=disable" [dropshot_external] # IP address and TCP port on which to listen for the external API -bind_address = "[fd00:1de::7]:12220" +bind_address = "[fd00:1122:3344:1::3]:12220" [dropshot_internal] # IP address and TCP port on which to listen for the internal API -bind_address = "[fd00:1de::7]:12221" +bind_address = "[fd00:1122:3344:1::3]:12221" [log] # Show log messages of this level and more severe @@ -42,4 +42,4 @@ mode = "stderr-terminal" # Configuration for interacting with the timeseries database [timeseries_db] -address = "[fd00:1de::8]:8123" +address = "[fd00:1122:3344:1::5]:8123" diff --git a/smf/oximeter/config.toml b/smf/oximeter/config.toml index 78eb1027842..a4812d01fd1 100644 --- a/smf/oximeter/config.toml +++ b/smf/oximeter/config.toml @@ -2,10 +2,10 @@ id = "1da65e5b-210c-4859-a7d7-200c1e659972" # Internal address of nexus -nexus_address = "[fd00:1de::7]:12221" +nexus_address = "[fd00:1122:3344:1::3]:12221" [db] -address = "[fd00:1de::8]:8123" +address = "[fd00:1122:3344:1::5]:8123" batch_size = 1000 batch_interval = 5 # In seconds @@ -14,4 +14,4 @@ level = "debug" mode = "stderr-terminal" [dropshot] -bind_address = "[fd00:1de::6]:12223" +bind_address = "[fd00:1122:3344:1::4]:12223" diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index 03eb99d01bb..ad8993c1ae3 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -1,46 +1,56 @@ # RSS (Rack Setup Service) "stand-in" configuration. +# The /56 subnet for the rack. +# Also implies the /48 AZ subnet. +# |............| <- This /48 is the AZ Subnet +# |...............| <- This /56 is the Rack Subnet +rack_subnet = "fd00:1122:3344:1::" + [[request]] -sled_address = "[fd00:1de::1]:12345" # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate crucible datasets. [[request.dataset]] zpool_uuid = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1de::9]:32345" +address = "[fd00:1122:3344:1::6]:32345" dataset_kind.type = "crucible" [[request.dataset]] zpool_uuid = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03" -address = "[fd00:1de::10]:32345" +address = "[fd00:1122:3344:1::7]:32345" dataset_kind.type = "crucible" [[request.dataset]] zpool_uuid = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03" -address = "[fd00:1de::11]:32345" +address = "[fd00:1122:3344:1::8]:32345" dataset_kind.type = "crucible" [[request.dataset]] zpool_uuid = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1de::5]:32221" +address = "[fd00:1122:3344:1::2]:32221" dataset_kind.type = "cockroach_db" dataset_kind.all_addresses = [ - "[fd00:1de::5]:32221", + "[fd00:1122:3344:1::2]:32221", ] # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate clickhouse datasets. [[request.dataset]] zpool_uuid = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1de::8]:8123" +address = "[fd00:1122:3344:1::5]:8123" dataset_kind.type = "clickhouse" [[request.service]] name = "nexus" -addresses = [ "[fd00:1de::7]:12220", "[fd00:1de::7]:12221" ] +addresses = [ + "[fd00:1122:3344:1::3]:12220", + "[fd00:1122:3344:1::3]:12221", +] # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate Oximeter services. [[request.service]] name = "oximeter" -addresses = [ "[fd00:1de::6]:12223" ] +addresses = [ + "[fd00:1122:3344:1::4]:12223", +] diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index 96dc361254d..6dfe87fe9bf 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -2,9 +2,10 @@ id = "fb0f7546-4d46-40ca-9d56-cbb810684ca7" -bootstrap_address = "[::]:12346" +# TODO: Remove this address + # Internal address of Nexus -nexus_address = "[fd00:1de::7]:12221" +nexus_address = "[fd00:1122:3344:01::3]:12221" # A file-backed zpool can be manually created with the following: # $ truncate -s 10GB testpool.vdev @@ -20,16 +21,6 @@ zpools = [ # $ dladm show-phys -p -o LINK # data_link = "igb0" -# Address of the Sled Agent itself -# -# With the usage of non-global zones, we no longer can use localhost addresses, -# as Nexus (within a Zone) is effectively "on a different machine" from the -# sled agent. -[dropshot] -bind_address = "[fd00:1de::1]:12345" - [log] level = "info" mode = "stderr-terminal" - -