[nexus][sled-agent] Generate rack ID in RSS, plumb it to Nexus

common/src/nexus_config.rs

@@ -102,6 +102,8 @@ pub enum Database {
 pub struct DeploymentConfig {
     /// Uuid of the Nexus instance
     pub id: Uuid,
+    /// Uuid of the Rack where Nexus is executing.
+    pub rack_id: Uuid,
     /// Dropshot configuration for external API server
     pub dropshot_external: ConfigDropshot,
     /// Dropshot configuration for internal API server

common/src/sql/dbinit.sql

@@ -75,6 +75,9 @@ CREATE TABLE omicron.public.sled (
     time_deleted TIMESTAMPTZ,
     rcgen INT NOT NULL,
 
+    /* FK into the Rack table */
+    rack_id UUID NOT NULL,
+
     /* The IP address and bound port of the sled agent server. */
     ip INET NOT NULL,
     port INT4 CHECK (port BETWEEN 0 AND 65535) NOT NULL,
@@ -83,6 +86,11 @@ CREATE TABLE omicron.public.sled (
     last_used_address INET NOT NULL
 );
 
+/* Add an index which lets us look up sleds on a rack */
+CREATE INDEX ON omicron.public.sled (
+    rack_id
+) WHERE time_deleted IS NULL;
+
 /*
  * Services
  */

nexus/examples/config.toml

@@ -36,6 +36,7 @@ address = "[::1]:8123"
 [deployment]
 # Identifier for this instance of Nexus
 id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
+rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 
 [deployment.dropshot_external]
 # IP address and TCP port on which to listen for the external API

nexus/src/app/sled.rs

@@ -31,7 +31,7 @@ impl super::Nexus {
         address: SocketAddrV6,
     ) -> Result<(), Error> {
         info!(self.log, "registered sled agent"; "sled_uuid" => id.to_string());
-        let sled = db::model::Sled::new(id, address);
+        let sled = db::model::Sled::new(id, address, self.rack_id);
         self.db_datastore.sled_upsert(sled).await?;
         Ok(())
     }

nexus/src/config.rs

@@ -329,6 +329,7 @@ mod test {
             max_vpc_ipv4_subnet_prefix = 27
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
+            rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
@@ -348,6 +349,9 @@ mod test {
             Config {
                 deployment: DeploymentConfig {
                     id: "28b90dc4-c22a-65ba-f49a-f051fe01208f".parse().unwrap(),
+                    rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f"
+                        .parse()
+                        .unwrap(),
                     dropshot_external: ConfigDropshot {
                         bind_address: "10.1.2.3:4567"
                             .parse::<SocketAddr>()
@@ -407,6 +411,7 @@ mod test {
             address = "[::1]:8123"
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
+            rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
@@ -448,6 +453,7 @@ mod test {
             address = "[::1]:8123"
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
+            rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024
@@ -503,6 +509,7 @@ mod test {
             max_vpc_ipv4_subnet_prefix = 100
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
+            rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
             request_body_max_bytes = 1024

nexus/src/db/datastore.rs

@@ -4034,8 +4034,9 @@ mod test {
             0,
             0,
         );
+        let rack_id = Uuid::new_v4();
         let sled_id = Uuid::new_v4();
-        let sled = Sled::new(sled_id, bogus_addr.clone());
+        let sled = Sled::new(sled_id, bogus_addr.clone(), rack_id);
         datastore.sled_upsert(sled).await.unwrap();
         sled_id
     }
@@ -4391,14 +4392,15 @@ mod test {
         let opctx =
             OpContext::for_tests(logctx.log.new(o!()), datastore.clone());
 
+        let rack_id = Uuid::new_v4();
         let addr1 = "[fd00:1de::1]:12345".parse().unwrap();
         let sled1_id = "0de4b299-e0b4-46f0-d528-85de81a7095f".parse().unwrap();
-        let sled1 = db::model::Sled::new(sled1_id, addr1);
+        let sled1 = db::model::Sled::new(sled1_id, addr1, rack_id);
         datastore.sled_upsert(sled1).await.unwrap();
 
         let addr2 = "[fd00:1df::1]:12345".parse().unwrap();
         let sled2_id = "66285c18-0c79-43e0-e54f-95271f271314".parse().unwrap();
-        let sled2 = db::model::Sled::new(sled2_id, addr2);
+        let sled2 = db::model::Sled::new(sled2_id, addr2, rack_id);
         datastore.sled_upsert(sled2).await.unwrap();
 
         let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap();

nexus/src/db/model/sled.rs

@@ -21,6 +21,8 @@ pub struct Sled {
     time_deleted: Option<DateTime<Utc>>,
     rcgen: Generation,
 
+    pub rack_id: Uuid,
+
     // ServiceAddress (Sled Agent).
     pub ip: ipv6::Ipv6Addr,
     pub port: SqlU16,
@@ -30,7 +32,7 @@ pub struct Sled {
 }
 
 impl Sled {
-    pub fn new(id: Uuid, addr: SocketAddrV6) -> Self {
+    pub fn new(id: Uuid, addr: SocketAddrV6, rack_id: Uuid) -> Self {
         let last_used_address = {
             let mut segments = addr.ip().segments();
             segments[7] += omicron_common::address::RSS_RESERVED_ADDRESSES;
@@ -40,6 +42,7 @@ impl Sled {
             identity: SledIdentity::new(id),
             time_deleted: None,
             rcgen: Generation::new(),
+            rack_id,
             ip: ipv6::Ipv6Addr::from(addr.ip()),
             port: addr.port().into(),
             last_used_address,

nexus/src/db/schema.rs

@@ -297,6 +297,7 @@ table! {
         time_deleted -> Nullable<Timestamptz>,
         rcgen -> Int8,
 
+        rack_id -> Uuid,
         ip -> Inet,
         port -> Int4,
         last_used_address -> Inet,

nexus/src/lib.rs

@@ -36,7 +36,6 @@ use external_api::http_entrypoints::external_api;
 use internal_api::http_entrypoints::internal_api;
 use slog::Logger;
 use std::sync::Arc;
-use uuid::Uuid;
 
 #[macro_use]
 extern crate slog;
@@ -82,15 +81,15 @@ impl Server {
     /// Start a nexus server.
     pub async fn start(
         config: &Config,
-        rack_id: Uuid,
         log: &Logger,
     ) -> Result<Server, String> {
         let log = log.new(o!("name" => config.deployment.id.to_string()));
         info!(log, "setting up nexus server");
 
         let ctxlog = log.new(o!("component" => "ServerContext"));
 
-        let apictx = ServerContext::new(rack_id, ctxlog, &config)?;
+        let apictx =
+            ServerContext::new(config.deployment.rack_id, ctxlog, &config)?;
 
         let http_server_starter_external = dropshot::HttpServerStarter::new(
             &config.deployment.dropshot_external,
@@ -167,8 +166,7 @@ pub async fn run_server(config: &Config) -> Result<(), String> {
     } else {
         debug!(log, "registered DTrace probes");
     }
-    let rack_id = Uuid::new_v4();
-    let server = Server::start(config, rack_id, &log).await?;
+    let server = Server::start(config, &log).await?;
     server.register_as_producer().await;
     server.wait_for_finish().await
 }

nexus/test-utils/src/lib.rs

@@ -90,7 +90,6 @@ pub async fn test_setup_with_config(
     config: &mut omicron_nexus::Config,
 ) -> ControlPlaneTestContext {
     let logctx = LogContext::new(test_name, &config.pkg.log);
-    let rack_id = Uuid::parse_str(RACK_UUID).unwrap();
     let log = &logctx.log;
 
     // Start up CockroachDB.
@@ -104,9 +103,8 @@ pub async fn test_setup_with_config(
         nexus_config::Database::FromUrl { url: database.pg_config().clone() };
     config.pkg.timeseries_db.address.set_port(clickhouse.port());
 
-    let server = omicron_nexus::Server::start(&config, rack_id, &logctx.log)
-        .await
-        .unwrap();
+    let server =
+        omicron_nexus::Server::start(&config, &logctx.log).await.unwrap();
     server
         .apictx
         .nexus

nexus/tests/config.test.toml

@@ -39,6 +39,7 @@ max_vpc_ipv4_subnet_prefix = 29
 # Identifier for this instance of Nexus.
 # NOTE: The test suite always overrides this.
 id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
+rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 
 #
 # NOTE: for the test suite, the port MUST be 0 (in order to bind to any

sled-agent/src/bootstrap/agent.rs

@@ -245,6 +245,7 @@ impl Agent {
             &self.sled_config,
             self.parent_log.clone(),
             sled_address,
+            request.rack_id,
         )
         .await
         .map_err(|e| {

sled-agent/src/bootstrap/params.rs

@@ -8,13 +8,20 @@ use super::trust_quorum::ShareDistribution;
 use omicron_common::address::{Ipv6Subnet, SLED_PREFIX};
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
+use uuid::Uuid;
 
 /// Configuration information for launching a Sled Agent.
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub struct SledAgentRequest {
+    /// Uuid of the Sled Agent to be created.
+    pub id: Uuid,
+
     /// Portion of the IP space to be managed by the Sled Agent.
     pub subnet: Ipv6Subnet<SLED_PREFIX>,
 
+    /// Uuid of the rack to which this sled agent belongs.
+    pub rack_id: Uuid,
+
     /// Share of the rack secret for this Sled Agent.
     // TODO-cleanup This is currently optional because we don't do trust quorum
     // shares for single-node deployments (i.e., most dev/test environments),

sled-agent/src/rack_setup/service.rs

@@ -357,6 +357,7 @@ impl ServiceInner {
                 (request, (idx, bootstrap_addr))
             });
 
+        let rack_id = Uuid::new_v4();
         let allocations = requests_and_sleds.map(|(request, sled)| {
             let (idx, bootstrap_addr) = sled;
             info!(
@@ -373,7 +374,9 @@ impl ServiceInner {
                 bootstrap_addr,
                 SledAllocation {
                     initialization_request: SledAgentRequest {
+                        id: Uuid::new_v4(),
                         subnet,
+                        rack_id,
                         trust_quorum_share: maybe_rack_secret_shares
                             .as_mut()
                             .map(|shares_iter| {

sled-agent/src/server.rs

@@ -38,6 +38,7 @@ impl Server {
         config: &Config,
         log: Logger,
         addr: SocketAddrV6,
+        rack_id: Uuid,
     ) -> Result<Server, String> {
         info!(log, "setting up sled agent server");
 
@@ -47,10 +48,15 @@ impl Server {
             client_log,
         ));
 
-        let sled_agent =
-            SledAgent::new(&config, log.clone(), nexus_client.clone(), addr)
-                .await
-                .map_err(|e| e.to_string())?;
+        let sled_agent = SledAgent::new(
+            &config,
+            log.clone(),
+            nexus_client.clone(),
+            addr,
+            rack_id,
+        )
+        .await
+        .map_err(|e| e.to_string())?;
 
         let mut dropshot_config = dropshot::ConfigDropshot::default();
         dropshot_config.request_body_max_bytes = 1024 * 1024;

sled-agent/src/services.rs

@@ -25,6 +25,7 @@ use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use tokio::io::AsyncWriteExt;
 use tokio::sync::Mutex;
+use uuid::Uuid;
 
 // The filename of ServiceManager's internal storage.
 const SERVICE_CONFIG_FILENAME: &str = "service.toml";
@@ -124,6 +125,7 @@ pub struct ServiceManager {
     vnic_allocator: VnicAllocator,
     underlay_vnic: EtherstubVnic,
     underlay_address: Ipv6Addr,
+    rack_id: Uuid,
 }
 
 impl ServiceManager {
@@ -143,6 +145,7 @@ impl ServiceManager {
         underlay_vnic: EtherstubVnic,
         underlay_address: Ipv6Addr,
         config: Config,
+        rack_id: Uuid,
     ) -> Result<Self, Error> {
         debug!(log, "Creating new ServiceManager");
         let mgr = Self {
@@ -152,6 +155,7 @@ impl ServiceManager {
             vnic_allocator: VnicAllocator::new("Service", etherstub),
             underlay_vnic,
             underlay_address,
+            rack_id,
         };
 
         let config_path = mgr.services_config_path();
@@ -316,6 +320,7 @@ impl ServiceManager {
                     // cannot be known at packaging time.
                     let deployment_config = NexusDeploymentConfig {
                         id: service.id,
+                        rack_id: self.rack_id,
                         dropshot_external: ConfigDropshot {
                             bind_address: SocketAddr::V6(external_address),
                             request_body_max_bytes: 1048576,
@@ -702,6 +707,7 @@ mod test {
             EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()),
             Ipv6Addr::LOCALHOST,
             test_config.make_config(),
+            Uuid::new_v4(),
         )
         .await
         .unwrap();
@@ -728,6 +734,7 @@ mod test {
             EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()),
             Ipv6Addr::LOCALHOST,
             test_config.make_config(),
+            Uuid::new_v4(),
         )
         .await
         .unwrap();
@@ -756,6 +763,7 @@ mod test {
             EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()),
             Ipv6Addr::LOCALHOST,
             test_config.make_config(),
+            Uuid::new_v4(),
         )
         .await
         .unwrap();
@@ -773,6 +781,7 @@ mod test {
             EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()),
             Ipv6Addr::LOCALHOST,
             test_config.make_config(),
+            Uuid::new_v4(),
         )
         .await
         .unwrap();
@@ -797,6 +806,7 @@ mod test {
             EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()),
             Ipv6Addr::LOCALHOST,
             test_config.make_config(),
+            Uuid::new_v4(),
         )
         .await
         .unwrap();
@@ -816,6 +826,7 @@ mod test {
             EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()),
             Ipv6Addr::LOCALHOST,
             config,
+            Uuid::new_v4(),
         )
         .await
         .unwrap();

sled-agent/src/sled_agent.rs

@@ -116,6 +116,7 @@ impl SledAgent {
         log: Logger,
         nexus_client: Arc<NexusClient>,
         sled_address: SocketAddrV6,
+        rack_id: Uuid,
     ) -> Result<SledAgent, Error> {
         let id = &config.id;
 
@@ -266,6 +267,7 @@ impl SledAgent {
             etherstub_vnic.clone(),
             *sled_address.ip(),
             services::Config::default(),
+            rack_id,
         )
         .await?;