diff --git a/common/src/address.rs b/common/src/address.rs index 708fbff12bd..6013932d47a 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -54,7 +54,9 @@ const GZ_ADDRESS_INDEX: usize = 2; pub const RSS_RESERVED_ADDRESSES: u16 = 10; /// Wraps an [`Ipv6Network`] with a compile-time prefix length. -#[derive(Debug, Clone, Copy, JsonSchema, Serialize, Deserialize, PartialEq)] +#[derive( + Debug, Clone, Copy, JsonSchema, Serialize, Deserialize, Hash, PartialEq, Eq, +)] pub struct Ipv6Subnet { net: Ipv6Net, } diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index db76fa2134b..c0e92841ed7 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -999,7 +999,7 @@ impl JsonSchema for Ipv4Net { } /// An `Ipv6Net` represents a IPv6 subnetwork, including the address and network mask. -#[derive(Clone, Copy, Debug, Deserialize, Hash, PartialEq, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)] pub struct Ipv6Net(pub ipnetwork::Ipv6Network); impl Ipv6Net { diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index 0aa9ba42f6d..a948b9414a9 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -17,8 +17,7 @@ use crate::illumos::dladm::{self, Dladm, PhysicalLink}; use crate::illumos::zone::Zones; use crate::server::Server as SledServer; use crate::sp::SpHandle; -use ddm_admin_client::types::Ipv6Prefix; -use omicron_common::address::get_sled_address; +use omicron_common::address::{get_sled_address, Ipv6Subnet}; use omicron_common::api::external::{Error as ExternalError, MacAddr}; use omicron_common::backoff::{ internal_service_policy, retry_notify, BackoffError, @@ -88,6 +87,7 @@ pub(crate) struct Agent { sled_agent: Mutex>, sled_config: SledConfig, sp: Option, + ddmd_client: DdmAdminClient, } fn get_sled_agent_request_path() -> PathBuf { @@ -174,10 +174,8 @@ impl Agent { // Start trying to notify ddmd of our bootstrap address so it can // advertise it to other sleds. - tokio::spawn(advertise_bootstrap_address_via_ddmd( - ba_log.clone(), - address, - )); + let ddmd_client = DdmAdminClient::new(log.clone())?; + ddmd_client.advertise_prefix(Ipv6Subnet::new(address)); let agent = Agent { log: ba_log, @@ -188,6 +186,7 @@ impl Agent { sled_agent: Mutex::new(None), sled_config, sp, + ddmd_client, }; let request_path = get_sled_agent_request_path(); @@ -290,6 +289,19 @@ impl Agent { err, })?; + // Start trying to notify ddmd of our sled prefix so it can + // advertise it to other sleds. + // + // TODO-security This ddmd_client is used to advertise both this + // (underlay) address and our bootstrap address. Bootstrap addresses are + // unauthenticated (connections made on them are auth'd via sprockets), + // but underlay addresses should be exchanged via authenticated channels + // between ddmd instances. It's TBD how that will work, but presumably + // we'll need to do something different here for underlay vs bootstrap + // addrs (either talk to a differently-configured ddmd, or include info + // indicating which kind of address we're advertising). + self.ddmd_client.advertise_prefix(request.subnet); + Ok(SledAgentResponse { id: self.sled_config.id }) } @@ -469,21 +481,6 @@ impl Agent { } } -async fn advertise_bootstrap_address_via_ddmd(log: Logger, address: Ipv6Addr) { - let prefix = Ipv6Prefix { addr: address, mask: 64 }; - retry_notify(internal_service_policy(), || async { - let client = DdmAdminClient::new(log.clone())?; - client.advertise_prefix(prefix).await?; - Ok(()) - }, |err, duration| { - info!( - log, - "Failed to notify ddmd of our address (will retry after {duration:?}"; - "err" => %err, - ); - }).await.unwrap(); -} - #[cfg(test)] mod tests { use super::*; diff --git a/sled-agent/src/bootstrap/ddm_admin_client.rs b/sled-agent/src/bootstrap/ddm_admin_client.rs index 16729543aa2..cbf150c22b5 100644 --- a/sled-agent/src/bootstrap/ddm_admin_client.rs +++ b/sled-agent/src/bootstrap/ddm_admin_client.rs @@ -6,6 +6,10 @@ use ddm_admin_client::types::Ipv6Prefix; use ddm_admin_client::Client; +use omicron_common::address::Ipv6Subnet; +use omicron_common::address::SLED_PREFIX; +use omicron_common::backoff::internal_service_policy; +use omicron_common::backoff::retry_notify; use slog::Logger; use std::net::Ipv6Addr; use std::net::SocketAddr; @@ -54,20 +58,32 @@ impl DdmAdminClient { Ok(DdmAdminClient { client, log }) } - /// Instruct ddmd to advertise the given prefix to peer sleds. - pub async fn advertise_prefix( - &self, - prefix: Ipv6Prefix, - ) -> Result<(), DdmError> { - // TODO-cleanup Why does the generated openapi client require a `&Vec` - // instead of a `&[]`? - info!( - self.log, "Sending prefix to ddmd for advertisement"; - "prefix" => ?prefix, - ); - let prefixes = vec![prefix]; - self.client.advertise_prefixes(&prefixes).await?; - Ok(()) + /// Spawns a background task to instruct ddmd to advertise the given prefix + /// to peer sleds. + pub fn advertise_prefix(&self, address: Ipv6Subnet) { + let me = self.clone(); + tokio::spawn(async move { + let prefix = + Ipv6Prefix { addr: address.net().network(), mask: SLED_PREFIX }; + retry_notify(internal_service_policy(), || async { + info!( + me.log, "Sending prefix to ddmd for advertisement"; + "prefix" => ?prefix, + ); + + // TODO-cleanup Why does the generated openapi client require a + // `&Vec` instead of a `&[]`? + let prefixes = vec![prefix]; + me.client.advertise_prefixes(&prefixes).await?; + Ok(()) + }, |err, duration| { + info!( + me.log, + "Failed to notify ddmd of our address (will retry after {duration:?}"; + "err" => %err, + ); + }).await.unwrap(); + }); } /// Returns the addresses of connected sleds. diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index be11bfb2a6d..adf2731eb9a 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -4,6 +4,7 @@ //! Support for miscellaneous services managed by the sled. +use crate::bootstrap::ddm_admin_client::{DdmAdminClient, DdmError}; use crate::illumos::dladm::{Etherstub, EtherstubVnic}; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; @@ -12,7 +13,10 @@ use crate::illumos::zone::AddressRequest; use crate::params::{ServiceEnsureBody, ServiceRequest, ServiceType}; use crate::zone::Zones; use dropshot::ConfigDropshot; -use omicron_common::address::{Ipv6Subnet, OXIMETER_PORT, RACK_PREFIX}; +use omicron_common::address::Ipv6Subnet; +use omicron_common::address::OXIMETER_PORT; +use omicron_common::address::RACK_PREFIX; +use omicron_common::address::SLED_PREFIX; use omicron_common::nexus_config::{ self, DeploymentConfig as NexusDeploymentConfig, }; @@ -63,6 +67,9 @@ pub enum Error { #[error(transparent)] ZoneInstall(#[from] crate::illumos::running_zone::InstallZoneError), + #[error("Error contacting ddmd: {0}")] + DdmError(#[from] DdmError), + #[error("Failed to add GZ addresses: {message}: {err}")] GzAddress { message: String, @@ -126,6 +133,8 @@ pub struct ServiceManager { underlay_vnic: EtherstubVnic, underlay_address: Ipv6Addr, rack_id: Uuid, + ddmd_client: DdmAdminClient, + advertised_prefixes: Mutex>>, } impl ServiceManager { @@ -148,14 +157,17 @@ impl ServiceManager { rack_id: Uuid, ) -> Result { debug!(log, "Creating new ServiceManager"); + let log = log.new(o!("component" => "ServiceManager")); let mgr = Self { - log: log.new(o!("component" => "ServiceManager")), + log: log.clone(), config, zones: Mutex::new(vec![]), vnic_allocator: VnicAllocator::new("Service", etherstub), underlay_vnic, underlay_address, rack_id, + ddmd_client: DdmAdminClient::new(log)?, + advertised_prefixes: Mutex::new(HashSet::new()), }; let config_path = mgr.services_config_path(); @@ -194,6 +206,18 @@ impl ServiceManager { self.config.all_svcs_config_path.clone() } + // Advertise the /64 prefix of `address`, unless we already have. + // + // This method only blocks long enough to check our HashSet of + // already-advertised prefixes; the actual request to ddmd to advertise the + // prefix is spawned onto a background task. + async fn advertise_prefix_of_address(&self, address: Ipv6Addr) { + let subnet = Ipv6Subnet::new(address); + if self.advertised_prefixes.lock().await.insert(subnet) { + self.ddmd_client.advertise_prefix(subnet); + } + } + // Populates `existing_zones` according to the requests in `services`. // // At the point this function is invoked, IP addresses have already been @@ -250,7 +274,7 @@ impl ServiceManager { } info!(self.log, "GZ addresses: {:#?}", service.gz_addresses); - for addr in &service.gz_addresses { + for &addr in &service.gz_addresses { info!( self.log, "Ensuring GZ address {} exists", @@ -260,7 +284,7 @@ impl ServiceManager { let addr_name = service.name.replace(&['-', '_'][..], ""); Zones::ensure_has_global_zone_v6_address( self.underlay_vnic.clone(), - *addr, + addr, &addr_name, ) .map_err(|err| Error::GzAddress { @@ -270,6 +294,10 @@ impl ServiceManager { ), err, })?; + + // If this address is in a new ipv6 prefix, notify maghemite so + // it can advertise it to other sleds. + self.advertise_prefix_of_address(addr).await; } let gateway = if !service.gz_addresses.is_empty() {