Blueprints: store sled state (#5663)

This adds the `sled_state` map described in RFD 457 to `Blueprint`. The
motivating driver here is decommissioning expunged sleds, which is on
the path to addressing #5625, but this PR only adds the field - the
planner carries forward states from its inputs, but otherwise makes no
changes, and the executor doesn't read the new field at all.

nexus/db-model/src/deployment.rs

@@ -9,11 +9,13 @@ use crate::inventory::ZoneType;
 use crate::omicron_zone_config::{OmicronZone, OmicronZoneNic};
 use crate::schema::{
     blueprint, bp_omicron_physical_disk, bp_omicron_zone, bp_omicron_zone_nic,
-    bp_sled_omicron_physical_disks, bp_sled_omicron_zones, bp_target,
+    bp_sled_omicron_physical_disks, bp_sled_omicron_zones, bp_sled_state,
+    bp_target,
 };
 use crate::typed_uuid::DbTypedUuid;
 use crate::{
-    impl_enum_type, ipv6, Generation, MacAddr, Name, SqlU16, SqlU32, SqlU8,
+    impl_enum_type, ipv6, Generation, MacAddr, Name, SledState, SqlU16, SqlU32,
+    SqlU8,
 };
 use chrono::{DateTime, Utc};
 use ipnetwork::IpNetwork;
@@ -103,6 +105,15 @@ impl From<BpTarget> for nexus_types::deployment::BlueprintTarget {
     }
 }
 
+/// See [`nexus_types::deployment::Blueprint::sled_state`].
+#[derive(Queryable, Clone, Debug, Selectable, Insertable)]
+#[diesel(table_name = bp_sled_state)]
+pub struct BpSledState {
+    pub blueprint_id: Uuid,
+    pub sled_id: DbTypedUuid<SledKind>,
+    pub sled_state: SledState,
+}
+
 /// See [`nexus_types::deployment::BlueprintPhysicalDisksConfig`].
 #[derive(Queryable, Clone, Debug, Selectable, Insertable)]
 #[diesel(table_name = bp_sled_omicron_physical_disks)]

nexus/db-model/src/schema.rs

@@ -1504,6 +1504,15 @@ table! {
     }
 }
 
+table! {
+    bp_sled_state (blueprint_id, sled_id) {
+        blueprint_id -> Uuid,
+        sled_id -> Uuid,
+
+        sled_state -> crate::SledStateEnum,
+    }
+}
+
 table! {
     bp_sled_omicron_physical_disks (blueprint_id, sled_id) {
         blueprint_id -> Uuid,

nexus/db-model/src/schema_versions.rs

@@ -17,7 +17,7 @@ use std::collections::BTreeMap;
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(60, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(61, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy<Vec<KnownVersion>> = Lazy::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(61, "blueprint-add-sled-state"),
         KnownVersion::new(60, "add-lookup-vmm-by-sled-id-index"),
         KnownVersion::new(59, "enforce-first-as-default"),
         KnownVersion::new(58, "insert-default-allowlist"),

nexus/db-queries/src/db/datastore/deployment.rs

@@ -40,12 +40,14 @@ use nexus_db_model::BpOmicronZone;
 use nexus_db_model::BpOmicronZoneNic;
 use nexus_db_model::BpSledOmicronPhysicalDisks;
 use nexus_db_model::BpSledOmicronZones;
+use nexus_db_model::BpSledState;
 use nexus_db_model::BpTarget;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintMetadata;
 use nexus_types::deployment::BlueprintPhysicalDisksConfig;
 use nexus_types::deployment::BlueprintTarget;
 use nexus_types::deployment::BlueprintZonesConfig;
+use nexus_types::external_api::views::SledState;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::ListResultVec;
@@ -109,6 +111,16 @@ impl DataStore {
         let row_blueprint = DbBlueprint::from(blueprint);
         let blueprint_id = row_blueprint.id;
 
+        let sled_states = blueprint
+            .sled_state
+            .iter()
+            .map(|(&sled_id, &state)| BpSledState {
+                blueprint_id,
+                sled_id: sled_id.into(),
+                sled_state: state.into(),
+            })
+            .collect::<Vec<_>>();
+
         let sled_omicron_physical_disks = blueprint
             .blueprint_disks
             .iter()
@@ -187,6 +199,16 @@ impl DataStore {
                     .await?;
             }
 
+            // Insert all the sled states for this blueprint.
+            {
+                use db::schema::bp_sled_state::dsl as sled_state;
+
+                let _ = diesel::insert_into(sled_state::bp_sled_state)
+                    .values(sled_states)
+                    .execute_async(&conn)
+                    .await?;
+            }
+
             // Insert all physical disks for this blueprint.
 
             {
@@ -290,6 +312,41 @@ impl DataStore {
             )
         };
 
+        // Load the sled states for this blueprint.
+        let sled_state: BTreeMap<SledUuid, SledState> = {
+            use db::schema::bp_sled_state::dsl;
+
+            let mut sled_state = BTreeMap::new();
+            let mut paginator = Paginator::new(SQL_BATCH_SIZE);
+            while let Some(p) = paginator.next() {
+                let batch = paginated(
+                    dsl::bp_sled_state,
+                    dsl::sled_id,
+                    &p.current_pagparams(),
+                )
+                .filter(dsl::blueprint_id.eq(blueprint_id))
+                .select(BpSledState::as_select())
+                .load_async(&*conn)
+                .await
+                .map_err(|e| {
+                    public_error_from_diesel(e, ErrorHandler::Server)
+                })?;
+
+                paginator = p.found_batch(&batch, &|s| s.sled_id);
+
+                for s in batch {
+                    let old = sled_state
+                        .insert(s.sled_id.into(), s.sled_state.into());
+                    bail_unless!(
+                        old.is_none(),
+                        "found duplicate sled ID in bp_sled_state: {}",
+                        s.sled_id
+                    );
+                }
+            }
+            sled_state
+        };
+
         // Read this blueprint's `bp_sled_omicron_zones` rows, which describes
         // the `OmicronZonesConfig` generation number for each sled that is a
         // part of this blueprint. Construct the BTreeMap we ultimately need,
@@ -550,6 +607,7 @@ impl DataStore {
             id: blueprint_id,
             blueprint_zones,
             blueprint_disks,
+            sled_state,
             parent_blueprint_id,
             internal_dns_version,
             external_dns_version,
@@ -578,6 +636,7 @@ impl DataStore {
 
         let (
             nblueprints,
+            nsled_states,
             nsled_physical_disks,
             nphysical_disks,
             nsled_agent_zones,
@@ -617,6 +676,17 @@ impl DataStore {
                     ));
                 }
 
+                // Remove rows associated with sled states.
+                let nsled_states = {
+                    use db::schema::bp_sled_state::dsl;
+                    diesel::delete(
+                        dsl::bp_sled_state
+                            .filter(dsl::blueprint_id.eq(blueprint_id)),
+                    )
+                    .execute_async(&conn)
+                    .await?
+                };
+
                 // Remove rows associated with Omicron physical disks
                 let nsled_physical_disks = {
                     use db::schema::bp_sled_omicron_physical_disks::dsl;
@@ -670,6 +740,7 @@ impl DataStore {
 
                 Ok((
                     nblueprints,
+                    nsled_states,
                     nsled_physical_disks,
                     nphysical_disks,
                     nsled_agent_zones,
@@ -688,6 +759,7 @@ impl DataStore {
         info!(&opctx.log, "removed blueprint";
             "blueprint_id" => blueprint_id.to_string(),
             "nblueprints" => nblueprints,
+            "nsled_states" => nsled_states,
             "nsled_physical_disks" => nsled_physical_disks,
             "nphysical_disks" => nphysical_disks,
             "nsled_agent_zones" => nsled_agent_zones,
@@ -1267,7 +1339,6 @@ mod tests {
     use nexus_types::external_api::views::PhysicalDiskPolicy;
     use nexus_types::external_api::views::PhysicalDiskState;
     use nexus_types::external_api::views::SledPolicy;
-    use nexus_types::external_api::views::SledState;
     use nexus_types::inventory::Collection;
     use omicron_common::address::Ipv6Subnet;
     use omicron_common::disk::DiskIdentity;

nexus/db-queries/src/db/datastore/rack.rs

@@ -984,6 +984,7 @@ mod test {
         BlueprintZoneDisposition, OmicronZoneExternalSnatIp,
     };
     use nexus_types::external_api::shared::SiloIdentityMode;
+    use nexus_types::external_api::views::SledState;
     use nexus_types::identity::Asset;
     use nexus_types::internal_api::params::DnsRecord;
     use nexus_types::inventory::NetworkInterface;
@@ -1016,6 +1017,7 @@ mod test {
                     id: Uuid::new_v4(),
                     blueprint_zones: BTreeMap::new(),
                     blueprint_disks: BTreeMap::new(),
+                    sled_state: BTreeMap::new(),
                     parent_blueprint_id: None,
                     internal_dns_version: *Generation::new(),
                     external_dns_version: *Generation::new(),
@@ -1255,6 +1257,12 @@ mod test {
         }
     }
 
+    fn sled_states_active(
+        sled_ids: impl Iterator<Item = SledUuid>,
+    ) -> BTreeMap<SledUuid, SledState> {
+        sled_ids.map(|sled_id| (sled_id, SledState::Active)).collect()
+    }
+
     #[tokio::test]
     async fn rack_set_initialized_with_services() {
         let test_name = "rack_set_initialized_with_services";
@@ -1487,6 +1495,7 @@ mod test {
         }
         let blueprint = Blueprint {
             id: Uuid::new_v4(),
+            sled_state: sled_states_active(blueprint_zones.keys().copied()),
             blueprint_zones,
             blueprint_disks: BTreeMap::new(),
             parent_blueprint_id: None,
@@ -1742,6 +1751,7 @@ mod test {
         }
         let blueprint = Blueprint {
             id: Uuid::new_v4(),
+            sled_state: sled_states_active(blueprint_zones.keys().copied()),
             blueprint_zones,
             blueprint_disks: BTreeMap::new(),
             parent_blueprint_id: None,
@@ -1953,6 +1963,7 @@ mod test {
         }
         let blueprint = Blueprint {
             id: Uuid::new_v4(),
+            sled_state: sled_states_active(blueprint_zones.keys().copied()),
             blueprint_zones,
             blueprint_disks: BTreeMap::new(),
             parent_blueprint_id: None,
@@ -2091,6 +2102,7 @@ mod test {
         }
         let blueprint = Blueprint {
             id: Uuid::new_v4(),
+            sled_state: sled_states_active(blueprint_zones.keys().copied()),
             blueprint_zones,
             blueprint_disks: BTreeMap::new(),
             parent_blueprint_id: None,