Advertise underlay and gz service prefixes via maghemite

common/src/address.rs

@@ -54,7 +54,9 @@ const GZ_ADDRESS_INDEX: usize = 2;
 pub const RSS_RESERVED_ADDRESSES: u16 = 10;
 
 /// Wraps an [`Ipv6Network`] with a compile-time prefix length.
-#[derive(Debug, Clone, Copy, JsonSchema, Serialize, Deserialize, PartialEq)]
+#[derive(
+    Debug, Clone, Copy, JsonSchema, Serialize, Deserialize, Hash, PartialEq, Eq,
+)]
 pub struct Ipv6Subnet<const N: u8> {
     net: Ipv6Net,
 }

common/src/api/external/mod.rs

@@ -999,7 +999,7 @@ impl JsonSchema for Ipv4Net {
 }
 
 /// An `Ipv6Net` represents a IPv6 subnetwork, including the address and network mask.
-#[derive(Clone, Copy, Debug, Deserialize, Hash, PartialEq, Serialize)]
+#[derive(Clone, Copy, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
 pub struct Ipv6Net(pub ipnetwork::Ipv6Network);
 
 impl Ipv6Net {

sled-agent/src/bootstrap/agent.rs

@@ -17,8 +17,7 @@ use crate::illumos::dladm::{self, Dladm, PhysicalLink};
 use crate::illumos::zone::Zones;
 use crate::server::Server as SledServer;
 use crate::sp::SpHandle;
-use ddm_admin_client::types::Ipv6Prefix;
-use omicron_common::address::get_sled_address;
+use omicron_common::address::{get_sled_address, Ipv6Subnet};
 use omicron_common::api::external::{Error as ExternalError, MacAddr};
 use omicron_common::backoff::{
     internal_service_policy, retry_notify, BackoffError,
@@ -88,6 +87,7 @@ pub(crate) struct Agent {
     sled_agent: Mutex<Option<SledServer>>,
     sled_config: SledConfig,
     sp: Option<SpHandle>,
+    ddmd_client: DdmAdminClient,
 }
 
 fn get_sled_agent_request_path() -> PathBuf {
@@ -174,10 +174,8 @@ impl Agent {
 
         // Start trying to notify ddmd of our bootstrap address so it can
         // advertise it to other sleds.
-        tokio::spawn(advertise_bootstrap_address_via_ddmd(
-            ba_log.clone(),
-            address,
-        ));
+        let ddmd_client = DdmAdminClient::new(log.clone())?;
+        ddmd_client.advertise_prefix(Ipv6Subnet::new(address));
 
         let agent = Agent {
             log: ba_log,
@@ -188,6 +186,7 @@ impl Agent {
             sled_agent: Mutex::new(None),
             sled_config,
             sp,
+            ddmd_client,
         };
 
         let request_path = get_sled_agent_request_path();
@@ -290,6 +289,19 @@ impl Agent {
             err,
         })?;
 
+        // Start trying to notify ddmd of our sled prefix so it can
+        // advertise it to other sleds.
+        //
+        // TODO-security This ddmd_client is used to advertise both this
+        // (underlay) address and our bootstrap address. Bootstrap addresses are
+        // unauthenticated (connections made on them are auth'd via sprockets),
+        // but underlay addresses should be exchanged via authenticated channels
+        // between ddmd instances. It's TBD how that will work, but presumably
+        // we'll need to do something different here for underlay vs bootstrap
+        // addrs (either talk to a differently-configured ddmd, or include info
+        // indicating which kind of address we're advertising).
+        self.ddmd_client.advertise_prefix(request.subnet);
+
         Ok(SledAgentResponse { id: self.sled_config.id })
     }
 
@@ -469,21 +481,6 @@ impl Agent {
     }
 }
 
-async fn advertise_bootstrap_address_via_ddmd(log: Logger, address: Ipv6Addr) {
-    let prefix = Ipv6Prefix { addr: address, mask: 64 };
-    retry_notify(internal_service_policy(), || async {
-        let client = DdmAdminClient::new(log.clone())?;
-        client.advertise_prefix(prefix).await?;
-        Ok(())
-    }, |err, duration| {
-        info!(
-            log,
-            "Failed to notify ddmd of our address (will retry after {duration:?}";
-            "err" => %err,
-        );
-    }).await.unwrap();
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

sled-agent/src/bootstrap/ddm_admin_client.rs

@@ -6,6 +6,10 @@
 
 use ddm_admin_client::types::Ipv6Prefix;
 use ddm_admin_client::Client;
+use omicron_common::address::Ipv6Subnet;
+use omicron_common::address::SLED_PREFIX;
+use omicron_common::backoff::internal_service_policy;
+use omicron_common::backoff::retry_notify;
 use slog::Logger;
 use std::net::Ipv6Addr;
 use std::net::SocketAddr;
@@ -54,20 +58,32 @@ impl DdmAdminClient {
         Ok(DdmAdminClient { client, log })
     }
 
-    /// Instruct ddmd to advertise the given prefix to peer sleds.
-    pub async fn advertise_prefix(
-        &self,
-        prefix: Ipv6Prefix,
-    ) -> Result<(), DdmError> {
-        // TODO-cleanup Why does the generated openapi client require a `&Vec`
-        // instead of a `&[]`?
-        info!(
-            self.log, "Sending prefix to ddmd for advertisement";
-            "prefix" => ?prefix,
-        );
-        let prefixes = vec![prefix];
-        self.client.advertise_prefixes(&prefixes).await?;
-        Ok(())
+    /// Spawns a background task to instruct ddmd to advertise the given prefix
+    /// to peer sleds.
+    pub fn advertise_prefix(&self, address: Ipv6Subnet<SLED_PREFIX>) {
+        let me = self.clone();
+        tokio::spawn(async move {
+            let prefix =
+                Ipv6Prefix { addr: address.net().network(), mask: SLED_PREFIX };
+            retry_notify(internal_service_policy(), || async {
+                info!(
+                    me.log, "Sending prefix to ddmd for advertisement";
+                    "prefix" => ?prefix,
+                );
+
+                // TODO-cleanup Why does the generated openapi client require a
+                // `&Vec` instead of a `&[]`?
+                let prefixes = vec![prefix];
+                me.client.advertise_prefixes(&prefixes).await?;
+                Ok(())
+            }, |err, duration| {
+                info!(
+                    me.log,
+                    "Failed to notify ddmd of our address (will retry after {duration:?}";
+                    "err" => %err,
+                );
+            }).await.unwrap();
+        });
     }
 
     /// Returns the addresses of connected sleds.

sled-agent/src/services.rs

@@ -4,6 +4,7 @@
 
 //! Support for miscellaneous services managed by the sled.
 
+use crate::bootstrap::ddm_admin_client::{DdmAdminClient, DdmError};
 use crate::illumos::dladm::{Etherstub, EtherstubVnic};
 use crate::illumos::running_zone::{InstalledZone, RunningZone};
 use crate::illumos::vnic::VnicAllocator;
@@ -12,7 +13,10 @@ use crate::illumos::zone::AddressRequest;
 use crate::params::{ServiceEnsureBody, ServiceRequest, ServiceType};
 use crate::zone::Zones;
 use dropshot::ConfigDropshot;
-use omicron_common::address::{Ipv6Subnet, OXIMETER_PORT, RACK_PREFIX};
+use omicron_common::address::Ipv6Subnet;
+use omicron_common::address::OXIMETER_PORT;
+use omicron_common::address::RACK_PREFIX;
+use omicron_common::address::SLED_PREFIX;
 use omicron_common::nexus_config::{
     self, DeploymentConfig as NexusDeploymentConfig,
 };
@@ -63,6 +67,9 @@ pub enum Error {
     #[error(transparent)]
     ZoneInstall(#[from] crate::illumos::running_zone::InstallZoneError),
 
+    #[error("Error contacting ddmd: {0}")]
+    DdmError(#[from] DdmError),
+
     #[error("Failed to add GZ addresses: {message}: {err}")]
     GzAddress {
         message: String,
@@ -126,6 +133,8 @@ pub struct ServiceManager {
     underlay_vnic: EtherstubVnic,
     underlay_address: Ipv6Addr,
     rack_id: Uuid,
+    ddmd_client: DdmAdminClient,
+    advertised_prefixes: Mutex<HashSet<Ipv6Subnet<SLED_PREFIX>>>,
 }
 
 impl ServiceManager {
@@ -148,14 +157,17 @@ impl ServiceManager {
         rack_id: Uuid,
     ) -> Result<Self, Error> {
         debug!(log, "Creating new ServiceManager");
+        let log = log.new(o!("component" => "ServiceManager"));
         let mgr = Self {
-            log: log.new(o!("component" => "ServiceManager")),
+            log: log.clone(),
             config,
             zones: Mutex::new(vec![]),
             vnic_allocator: VnicAllocator::new("Service", etherstub),
             underlay_vnic,
             underlay_address,
             rack_id,
+            ddmd_client: DdmAdminClient::new(log)?,
+            advertised_prefixes: Mutex::new(HashSet::new()),
         };
 
         let config_path = mgr.services_config_path();
@@ -194,6 +206,18 @@ impl ServiceManager {
         self.config.all_svcs_config_path.clone()
     }
 
+    // Advertise the /64 prefix of `address`, unless we already have.
+    //
+    // This method only blocks long enough to check our HashSet of
+    // already-advertised prefixes; the actual request to ddmd to advertise the
+    // prefix is spawned onto a background task.
+    async fn advertise_prefix_of_address(&self, address: Ipv6Addr) {
+        let subnet = Ipv6Subnet::new(address);
+        if self.advertised_prefixes.lock().await.insert(subnet) {
+            self.ddmd_client.advertise_prefix(subnet);
+        }
+    }
+
     // Populates `existing_zones` according to the requests in `services`.
     //
     // At the point this function is invoked, IP addresses have already been
@@ -250,7 +274,7 @@ impl ServiceManager {
             }
 
             info!(self.log, "GZ addresses: {:#?}", service.gz_addresses);
-            for addr in &service.gz_addresses {
+            for &addr in &service.gz_addresses {
                 info!(
                     self.log,
                     "Ensuring GZ address {} exists",
@@ -260,7 +284,7 @@ impl ServiceManager {
                 let addr_name = service.name.replace(&['-', '_'][..], "");
                 Zones::ensure_has_global_zone_v6_address(
                     self.underlay_vnic.clone(),
-                    *addr,
+                    addr,
                     &addr_name,
                 )
                 .map_err(|err| Error::GzAddress {
@@ -270,6 +294,10 @@ impl ServiceManager {
                     ),
                     err,
                 })?;
+
+                // If this address is in a new ipv6 prefix, notify maghemite so
+                // it can advertise it to other sleds.
+                self.advertise_prefix_of_address(addr).await;
             }
 
             let gateway = if !service.gz_addresses.is_empty() {