From b68b90aeddabcab74ac037c130df713eff6b8d7c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 16 Nov 2022 15:44:40 -0500 Subject: [PATCH 01/20] [sled-agent][nexus][RSS] Implement RFD 278: RSS to Nexus Handoff --- Cargo.lock | 1 + common/src/address.rs | 1 + nexus/db-model/Cargo.toml | 1 + nexus/db-model/src/dataset.rs | 31 +- nexus/db-model/src/ipv6.rs | 12 +- nexus/db-model/src/service_kind.rs | 8 +- nexus/src/app/rack.rs | 53 +- nexus/src/app/sled.rs | 4 +- nexus/src/db/datastore/mod.rs | 20 +- nexus/src/db/datastore/rack.rs | 72 ++- nexus/src/internal_api/http_entrypoints.rs | 11 +- nexus/tests/integration_tests/datasets.rs | 8 +- nexus/types/src/internal_api/params.rs | 53 +- openapi/nexus-internal.json | 139 ++++- openapi/sled-agent.json | 39 ++ sled-agent/src/bin/sled-agent.rs | 3 +- sled-agent/src/bootstrap/agent.rs | 12 +- sled-agent/src/bootstrap/server.rs | 1 - sled-agent/src/config.rs | 3 - sled-agent/src/http_entrypoints.rs | 14 +- sled-agent/src/params.rs | 5 + sled-agent/src/rack_setup/config.rs | 27 +- sled-agent/src/rack_setup/mod.rs | 1 + sled-agent/src/rack_setup/plan/mod.rs | 8 + sled-agent/src/rack_setup/plan/service.rs | 354 +++++++++++ sled-agent/src/rack_setup/plan/sled.rs | 245 ++++++++ sled-agent/src/rack_setup/service.rs | 679 ++++++++++----------- sled-agent/src/sled_agent.rs | 16 +- sled-agent/src/sp/mod.rs | 6 +- sled-agent/src/sp/simulated.rs | 4 - sled-agent/src/storage_manager.rs | 8 + smf/sled-agent/config-rss.toml | 57 +- smf/sled-agent/config.toml | 2 - 33 files changed, 1338 insertions(+), 560 deletions(-) create mode 100644 sled-agent/src/rack_setup/plan/mod.rs create mode 100644 sled-agent/src/rack_setup/plan/service.rs create mode 100644 sled-agent/src/rack_setup/plan/sled.rs diff --git a/Cargo.lock b/Cargo.lock index 6f2cd138475..e0de2f5c6ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2993,6 +2993,7 @@ dependencies = [ "db-macros", "diesel", "hex", + "internal-dns-client", "ipnetwork", "macaddr", "newtype_derive", diff --git a/common/src/address.rs b/common/src/address.rs index b597ff93caa..46b1e63bd08 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -34,6 +34,7 @@ pub const SLED_AGENT_PORT: u16 = 12345; /// The port propolis-server listens on inside the propolis zone. pub const PROPOLIS_PORT: u16 = 12400; pub const COCKROACH_PORT: u16 = 32221; +pub const CRUCIBLE_PORT: u16 = 32345; pub const CLICKHOUSE_PORT: u16 = 8123; pub const OXIMETER_PORT: u16 = 12223; pub const DENDRITE_PORT: u16 = 12224; diff --git a/nexus/db-model/Cargo.toml b/nexus/db-model/Cargo.toml index 22953a4ad98..c23d5a7c920 100644 --- a/nexus/db-model/Cargo.toml +++ b/nexus/db-model/Cargo.toml @@ -12,6 +12,7 @@ anyhow = "1.0" chrono = { version = "0.4", features = ["serde"] } diesel = { version = "2.0.2", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } hex = "0.4.3" +internal-dns-client = { path = "../../internal-dns-client" } ipnetwork = "0.20" macaddr = { version = "1.0.1", features = [ "serde_std" ]} newtype_derive = "0.1.6" diff --git a/nexus/db-model/src/dataset.rs b/nexus/db-model/src/dataset.rs index d68097af274..d58cde4fb4b 100644 --- a/nexus/db-model/src/dataset.rs +++ b/nexus/db-model/src/dataset.rs @@ -4,11 +4,14 @@ use super::{DatasetKind, Generation, Region, SqlU16}; use crate::collection::DatastoreCollectionConfig; +use crate::ipv6; use crate::schema::{dataset, region}; use chrono::{DateTime, Utc}; use db_macros::Asset; +use internal_dns_client::names::{BackendName, ServiceName, AAAA, SRV}; +use nexus_types::identity::Asset; use serde::{Deserialize, Serialize}; -use std::net::SocketAddr; +use std::net::{Ipv6Addr, SocketAddrV6}; use uuid::Uuid; /// Database representation of a Dataset. @@ -35,10 +38,10 @@ pub struct Dataset { pub pool_id: Uuid, - ip: ipnetwork::IpNetwork, + ip: ipv6::Ipv6Addr, port: SqlU16, - kind: DatasetKind, + pub kind: DatasetKind, pub size_used: Option, } @@ -46,7 +49,7 @@ impl Dataset { pub fn new( id: Uuid, pool_id: Uuid, - addr: SocketAddr, + addr: SocketAddrV6, kind: DatasetKind, ) -> Self { let size_used = match kind { @@ -65,12 +68,26 @@ impl Dataset { } } - pub fn address(&self) -> SocketAddr { + pub fn address(&self) -> SocketAddrV6 { self.address_with_port(self.port.into()) } - pub fn address_with_port(&self, port: u16) -> SocketAddr { - SocketAddr::new(self.ip.ip(), port) + pub fn address_with_port(&self, port: u16) -> SocketAddrV6 { + SocketAddrV6::new(Ipv6Addr::from(self.ip), port, 0, 0) + } + + pub fn aaaa(&self) -> AAAA { + AAAA::Zone(self.id()) + } + + pub fn srv(&self) -> SRV { + match self.kind { + DatasetKind::Crucible => { + SRV::Backend(BackendName::Crucible, self.id()) + } + DatasetKind::Clickhouse => SRV::Service(ServiceName::Clickhouse), + DatasetKind::Cockroach => SRV::Service(ServiceName::Cockroach), + } } } diff --git a/nexus/db-model/src/ipv6.rs b/nexus/db-model/src/ipv6.rs index 2b494100825..60f7c0558c6 100644 --- a/nexus/db-model/src/ipv6.rs +++ b/nexus/db-model/src/ipv6.rs @@ -16,9 +16,19 @@ use diesel::sql_types::Inet; use ipnetwork::IpNetwork; use ipnetwork::Ipv6Network; use omicron_common::api::external::Error; +use serde::{Deserialize, Serialize}; #[derive( - Clone, Copy, AsExpression, FromSqlRow, PartialEq, Ord, PartialOrd, Eq, + Clone, + Copy, + AsExpression, + FromSqlRow, + PartialEq, + Ord, + PartialOrd, + Eq, + Deserialize, + Serialize, )] #[diesel(sql_type = Inet)] pub struct Ipv6Addr(std::net::Ipv6Addr); diff --git a/nexus/db-model/src/service_kind.rs b/nexus/db-model/src/service_kind.rs index 9b6e08bee1c..363fd11717c 100644 --- a/nexus/db-model/src/service_kind.rs +++ b/nexus/db-model/src/service_kind.rs @@ -7,11 +7,11 @@ use nexus_types::internal_api; use serde::{Deserialize, Serialize}; impl_enum_type!( - #[derive(SqlType, Debug, QueryId)] + #[derive(Clone, SqlType, Debug, QueryId)] #[diesel(postgres_type(name = "service_kind"))] pub struct ServiceKindEnum; - #[derive(Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)] + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)] #[diesel(sql_type = ServiceKindEnum)] pub enum ServiceKind; @@ -29,7 +29,9 @@ impl From for ServiceKind { internal_api::params::ServiceKind::InternalDNS => { ServiceKind::InternalDNS } - internal_api::params::ServiceKind::Nexus => ServiceKind::Nexus, + internal_api::params::ServiceKind::Nexus { .. } => { + ServiceKind::Nexus + } internal_api::params::ServiceKind::Oximeter => { ServiceKind::Oximeter } diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index dcc7ce92dbc..8b00bb7195b 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -8,7 +8,7 @@ use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::lookup::LookupPath; -use crate::internal_api::params::ServicePutRequest; +use crate::internal_api::params::RackInitializationRequest; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; @@ -57,12 +57,13 @@ impl super::Nexus { &self, opctx: &OpContext, rack_id: Uuid, - services: Vec, + request: RackInitializationRequest, ) -> Result<(), Error> { opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; // Convert from parameter -> DB type. - let services: Vec<_> = services + let services: Vec<_> = request + .services .into_iter() .map(|svc| { db::model::Service::new( @@ -74,10 +75,54 @@ impl super::Nexus { }) .collect(); + // TODO: If nexus, add a pool? + + let datasets: Vec<_> = request + .datasets + .into_iter() + .map(|dataset| { + db::model::Dataset::new( + dataset.dataset_id, + dataset.zpool_id, + dataset.request.address, + dataset.request.kind.into(), + ) + }) + .collect(); + self.db_datastore - .rack_set_initialized(opctx, rack_id, services) + .rack_set_initialized(opctx, rack_id, services, datasets) .await?; Ok(()) } + + /// Awaits the initialization of the rack. + /// + /// This will occur by either: + /// 1. RSS invoking the internal API, handing off responsibility, or + /// 2. Re-reading a value from the DB, if the rack has already been + /// initialized. + /// + /// See RFD 278 for additional context. + pub async fn await_rack_initialization(&self, opctx: &OpContext) { + loop { + let result = self.rack_lookup(&opctx, &self.rack_id).await; + match result { + Ok(rack) => { + if rack.initialized { + return; + } + info!( + self.log, + "Still waiting for rack initialization: {:?}", rack + ); + } + Err(e) => { + warn!(self.log, "Cannot look up rack: {}", e); + } + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + } } diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index bdb52465586..e51f7c7af8a 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -18,7 +18,7 @@ use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use sled_agent_client::Client as SledAgentClient; -use std::net::{Ipv6Addr, SocketAddr}; +use std::net::{Ipv6Addr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -126,7 +126,7 @@ impl super::Nexus { &self, id: Uuid, zpool_id: Uuid, - address: SocketAddr, + address: SocketAddrV6, kind: DatasetKind, ) -> Result<(), Error> { info!(self.log, "upserting dataset"; "zpool_id" => zpool_id.to_string(), "dataset_id" => id.to_string(), "address" => address.to_string()); diff --git a/nexus/src/db/datastore/mod.rs b/nexus/src/db/datastore/mod.rs index f636e510ddb..1e779c796f4 100644 --- a/nexus/src/db/datastore/mod.rs +++ b/nexus/src/db/datastore/mod.rs @@ -264,9 +264,7 @@ mod test { use omicron_test_utils::dev; use ref_cast::RefCast; use std::collections::HashSet; - use std::net::Ipv6Addr; - use std::net::SocketAddrV6; - use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -514,8 +512,7 @@ mod test { // ... and datasets within that zpool. let dataset_count = REGION_REDUNDANCY_THRESHOLD * 2; - let bogus_addr = - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0); let dataset_ids: Vec = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); for id in &dataset_ids { @@ -614,8 +611,7 @@ mod test { // ... and datasets within that zpool. let dataset_count = REGION_REDUNDANCY_THRESHOLD; - let bogus_addr = - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0); let dataset_ids: Vec = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); for id in &dataset_ids { @@ -691,8 +687,7 @@ mod test { // ... and datasets within that zpool. let dataset_count = REGION_REDUNDANCY_THRESHOLD - 1; - let bogus_addr = - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0); let dataset_ids: Vec = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); for id in &dataset_ids { @@ -748,8 +743,7 @@ mod test { // ... and datasets within that zpool. let dataset_count = REGION_REDUNDANCY_THRESHOLD; - let bogus_addr = - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0); let dataset_ids: Vec = (0..dataset_count).map(|_| Uuid::new_v4()).collect(); for id in &dataset_ids { @@ -1025,14 +1019,14 @@ mod test { // Initialize the Rack. let result = datastore - .rack_set_initialized(&opctx, rack.id(), vec![]) + .rack_set_initialized(&opctx, rack.id(), vec![], vec![]) .await .unwrap(); assert!(result.initialized); // Re-initialize the rack (check for idempotency) let result = datastore - .rack_set_initialized(&opctx, rack.id(), vec![]) + .rack_set_initialized(&opctx, rack.id(), vec![], vec![]) .await .unwrap(); assert!(result.initialized); diff --git a/nexus/src/db/datastore/rack.rs b/nexus/src/db/datastore/rack.rs index 415e7e83623..f95f4da6db8 100644 --- a/nexus/src/db/datastore/rack.rs +++ b/nexus/src/db/datastore/rack.rs @@ -14,9 +14,11 @@ use crate::db::error::public_error_from_diesel_pool; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::identity::Asset; +use crate::db::model::Dataset; use crate::db::model::Rack; use crate::db::model::Service; use crate::db::model::Sled; +use crate::db::model::Zpool; use crate::db::pagination::paginated; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; @@ -83,19 +85,21 @@ impl DataStore { opctx: &OpContext, rack_id: Uuid, services: Vec, + datasets: Vec, ) -> UpdateResult { use db::schema::rack::dsl as rack_dsl; - use db::schema::service::dsl as service_dsl; #[derive(Debug)] enum RackInitError { ServiceInsert { err: AsyncInsertError, sled_id: Uuid, svc_id: Uuid }, + DatasetInsert { err: AsyncInsertError, zpool_id: Uuid }, RackUpdate(PoolError), } type TxnError = TransactionError; // NOTE: This operation could likely be optimized with a CTE, but given // the low-frequency of calls, this optimization has been deferred. + let log = opctx.log.clone(); self.pool_authorized(opctx) .await? .transaction_async(|conn| async move { @@ -111,25 +115,25 @@ impl DataStore { )) })?; if rack.initialized { + info!(log, "Early exit: Rack already initialized"); return Ok(rack); } - // Otherwise, insert services and set rack.initialized = true. + // Otherwise, insert services and datasets for svc in services { + use db::schema::service::dsl; let sled_id = svc.sled_id; >::insert_resource( sled_id, - diesel::insert_into(service_dsl::service) + diesel::insert_into(dsl::service) .values(svc.clone()) - .on_conflict(service_dsl::id) + .on_conflict(dsl::id) .do_update() .set(( - service_dsl::time_modified.eq(Utc::now()), - service_dsl::sled_id - .eq(excluded(service_dsl::sled_id)), - service_dsl::ip.eq(excluded(service_dsl::ip)), - service_dsl::kind - .eq(excluded(service_dsl::kind)), + dsl::time_modified.eq(Utc::now()), + dsl::sled_id.eq(excluded(dsl::sled_id)), + dsl::ip.eq(excluded(dsl::ip)), + dsl::kind.eq(excluded(dsl::kind)), )), ) .insert_and_get_result_async(&conn) @@ -142,7 +146,36 @@ impl DataStore { }) })?; } - diesel::update(rack_dsl::rack) + info!(log, "Inserted services"); + for dataset in datasets { + use db::schema::dataset::dsl; + let zpool_id = dataset.pool_id; + >::insert_resource( + zpool_id, + diesel::insert_into(dsl::dataset) + .values(dataset.clone()) + .on_conflict(dsl::id) + .do_update() + .set(( + dsl::time_modified.eq(Utc::now()), + dsl::pool_id.eq(excluded(dsl::pool_id)), + dsl::ip.eq(excluded(dsl::ip)), + dsl::port.eq(excluded(dsl::port)), + dsl::kind.eq(excluded(dsl::kind)), + )), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|err| { + TxnError::CustomError(RackInitError::DatasetInsert { + err, + zpool_id, + }) + })?; + } + info!(log, "Inserted datasets"); + + let rack = diesel::update(rack_dsl::rack) .filter(rack_dsl::id.eq(rack_id)) .set(( rack_dsl::initialized.eq(true), @@ -155,10 +188,25 @@ impl DataStore { TxnError::CustomError(RackInitError::RackUpdate( PoolError::from(e), )) - }) + })?; + Ok(rack) }) .await .map_err(|e| match e { + TxnError::CustomError(RackInitError::DatasetInsert { + err, + zpool_id, + }) => match err { + AsyncInsertError::CollectionNotFound => { + Error::ObjectNotFound { + type_name: ResourceType::Zpool, + lookup_type: LookupType::ById(zpool_id), + } + } + AsyncInsertError::DatabaseError(e) => { + public_error_from_diesel_pool(e, ErrorHandler::Server) + } + }, TxnError::CustomError(RackInitError::ServiceInsert { err, sled_id, diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs index 692dd82b9b6..307a3f02941 100644 --- a/nexus/src/internal_api/http_entrypoints.rs +++ b/nexus/src/internal_api/http_entrypoints.rs @@ -7,8 +7,9 @@ use crate::context::OpContext; use crate::ServerContext; use super::params::{ - DatasetPutRequest, DatasetPutResponse, OximeterInfo, ServicePutRequest, - SledAgentStartupInfo, ZpoolPutRequest, ZpoolPutResponse, + DatasetPutRequest, DatasetPutResponse, OximeterInfo, + RackInitializationRequest, SledAgentStartupInfo, ZpoolPutRequest, + ZpoolPutResponse, }; use dropshot::endpoint; use dropshot::ApiDescription; @@ -101,15 +102,15 @@ struct RackPathParam { async fn rack_initialization_complete( rqctx: Arc>>, path_params: Path, - info: TypedBody>, + info: TypedBody, ) -> Result { let apictx = rqctx.context(); let nexus = &apictx.nexus; let path = path_params.into_inner(); - let svcs = info.into_inner(); + let request = info.into_inner(); let opctx = OpContext::for_internal_api(&rqctx).await; - nexus.rack_initialize(&opctx, path.rack_id, svcs).await?; + nexus.rack_initialize(&opctx, path.rack_id, request).await?; Ok(HttpResponseUpdatedNoContent()) } diff --git a/nexus/tests/integration_tests/datasets.rs b/nexus/tests/integration_tests/datasets.rs index a5dcc13e06c..e3f454d436c 100644 --- a/nexus/tests/integration_tests/datasets.rs +++ b/nexus/tests/integration_tests/datasets.rs @@ -8,7 +8,7 @@ use omicron_common::api::external::ByteCount; use omicron_nexus::internal_api::params::{ DatasetKind, DatasetPutRequest, ZpoolPutRequest, }; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{Ipv6Addr, SocketAddrV6}; use uuid::Uuid; use nexus_test_utils::SLED_AGENT_UUID; @@ -39,8 +39,7 @@ async fn test_dataset_put_success(cptestctx: &ControlPlaneTestContext) { .await .unwrap(); - let address = - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0); let kind = DatasetKind::Crucible; let request = DatasetPutRequest { address, kind }; let dataset_id = Uuid::new_v4(); @@ -72,8 +71,7 @@ async fn test_dataset_put_bad_zpool_returns_not_found( let dataset_put_url = format!("/zpools/{}/dataset/{}", zpool_id, dataset_id); - let address = - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080); + let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0); let kind = DatasetKind::Crucible; let request = DatasetPutRequest { address, kind }; diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index 098059fbbba..9f738efd244 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -7,6 +7,7 @@ use omicron_common::api::external::ByteCount; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fmt; +use std::net::IpAddr; use std::net::Ipv6Addr; use std::net::SocketAddr; use std::net::SocketAddrV6; @@ -93,7 +94,7 @@ impl FromStr for DatasetKind { pub struct DatasetPutRequest { /// Address on which a service is responding to requests for the /// dataset. - pub address: SocketAddr, + pub address: SocketAddrV6, /// Type of dataset being inserted. pub kind: DatasetKind, @@ -122,10 +123,16 @@ pub struct DatasetPutResponse { #[derive( Debug, Serialize, Deserialize, JsonSchema, Clone, Copy, PartialEq, Eq, )] -#[serde(rename_all = "snake_case")] +#[serde(rename_all = "snake_case", tag = "type", content = "content")] pub enum ServiceKind { InternalDNS, - Nexus, + Nexus { + // TODO(https://github.com/oxidecomputer/omicron/issues/1530): + // While it's true that Nexus will only run with a single address, + // we want to convey information about the available pool of addresses + // when handing off from RSS -> Nexus. + external_address: IpAddr, + }, Oximeter, Dendrite, Tfport, @@ -136,7 +143,7 @@ impl fmt::Display for ServiceKind { use ServiceKind::*; let s = match self { InternalDNS => "internal_dns", - Nexus => "nexus", + Nexus { .. } => "nexus", Oximeter => "oximeter", Dendrite => "dendrite", Tfport => "tfport", @@ -145,24 +152,6 @@ impl fmt::Display for ServiceKind { } } -impl FromStr for ServiceKind { - type Err = omicron_common::api::external::Error; - - fn from_str(s: &str) -> Result { - use ServiceKind::*; - match s { - "nexus" => Ok(Nexus), - "oximeter" => Ok(Oximeter), - "internal_dns" => Ok(InternalDNS), - "dendrite" => Ok(Dendrite), - "tfport" => Ok(Tfport), - _ => Err(Self::Err::InternalError { - internal_message: format!("Unknown service kind: {}", s), - }), - } - } -} - /// Describes a service on a sled #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct ServicePutRequest { @@ -176,6 +165,26 @@ pub struct ServicePutRequest { pub kind: ServiceKind, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct DatasetCreateRequest { + pub zpool_id: Uuid, + pub dataset_id: Uuid, + pub request: DatasetPutRequest, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct RackInitializationRequest { + pub services: Vec, + pub datasets: Vec, + // TODO(https://github.com/oxidecomputer/omicron/issues/1530): + // While it's true that Nexus will only run with a single address, + // we want to convey information about the available pool of addresses + // when handing off from RSS -> Nexus. + + // TODO(https://github.com/oxidecomputer/omicron/issues/1528): + // Support passing x509 cert info. +} + /// Message used to notify Nexus that this oximeter instance is up and running. #[derive(Debug, Clone, Copy, JsonSchema, Serialize, Deserialize)] pub struct OximeterInfo { diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index e5864ca4e83..b92bfab9fce 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -255,11 +255,7 @@ "content": { "application/json": { "schema": { - "title": "Array_of_ServicePutRequest", - "type": "array", - "items": { - "$ref": "#/components/schemas/ServicePutRequest" - } + "$ref": "#/components/schemas/RackInitializationRequest" } } }, @@ -704,6 +700,27 @@ "value" ] }, + "DatasetCreateRequest": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string", + "format": "uuid" + }, + "request": { + "$ref": "#/components/schemas/DatasetPutRequest" + }, + "zpool_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "dataset_id", + "request", + "zpool_id" + ] + }, "DatasetKind": { "description": "Describes the purpose of the dataset.", "type": "string", @@ -1800,6 +1817,27 @@ } ] }, + "RackInitializationRequest": { + "type": "object", + "properties": { + "datasets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DatasetCreateRequest" + } + }, + "services": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ServicePutRequest" + } + } + }, + "required": [ + "datasets", + "services" + ] + }, "Sample": { "description": "A concrete type representing a single, timestamped measurement from a timeseries.", "type": "object", @@ -1832,13 +1870,90 @@ }, "ServiceKind": { "description": "Describes the purpose of the service.", - "type": "string", - "enum": [ - "internal_d_n_s", - "nexus", - "oximeter", - "dendrite", - "tfport" + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internal_d_n_s" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "content": { + "type": "object", + "properties": { + "external_address": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "external_address" + ] + }, + "type": { + "type": "string", + "enum": [ + "nexus" + ] + } + }, + "required": [ + "content", + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "oximeter" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "dendrite" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "tfport" + ] + } + }, + "required": [ + "type" + ] + } ] }, "ServicePutRequest": { diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 9ceb3436bfc..7a440bc28b0 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -390,6 +390,33 @@ } } } + }, + "/zpools": { + "get": { + "operationId": "zpools_get", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Zpool", + "type": "array", + "items": { + "$ref": "#/components/schemas/Zpool" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } } }, "components": { @@ -1858,6 +1885,18 @@ "oximeter", "switch" ] + }, + "Zpool": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "id" + ] } } } diff --git a/sled-agent/src/bin/sled-agent.rs b/sled-agent/src/bin/sled-agent.rs index ffa88dbcda3..4cc97fd0247 100644 --- a/sled-agent/src/bin/sled-agent.rs +++ b/sled-agent/src/bin/sled-agent.rs @@ -15,6 +15,7 @@ use omicron_sled_agent::rack_setup::config::SetupServiceConfig as RssConfig; use omicron_sled_agent::sp::SimSpConfig; use omicron_sled_agent::{config::Config as SledConfig, server as sled_server}; use std::path::PathBuf; +use uuid::Uuid; #[derive(Debug, Parser)] #[clap( @@ -98,7 +99,7 @@ async fn do_run() -> Result<(), CmdError> { // Configure and run the Bootstrap server. let bootstrap_config = BootstrapConfig { - id: config.id, + id: Uuid::new_v4(), bind_address: bootstrap_address, log: config.log.clone(), rss_config, diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index cb0be2a82a8..43444853bcf 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -136,7 +136,6 @@ impl Agent { ) -> Result<(Self, TrustQuorumMembership), BootstrapError> { let ba_log = log.new(o!( "component" => "BootstrapAgent", - "server" => sled_config.id.to_string(), )); // We expect this directory to exist - ensure that it does, before any @@ -241,7 +240,14 @@ impl Agent { // Server already exists, return it. info!(&self.log, "Sled Agent already loaded"); - if &server.address().ip() != sled_address.ip() { + if server.id() != request.id { + let err_str = format!( + "Sled Agent already running with UUID {}, but {} was requested", + server.id(), + request.id, + ); + return Err(BootstrapError::SledError(err_str)); + } else if &server.address().ip() != sled_address.ip() { let err_str = format!( "Sled Agent already running on address {}, but {} was requested", server.address().ip(), @@ -320,7 +326,7 @@ impl Agent { // indicating which kind of address we're advertising). self.ddmd_client.advertise_prefix(request.subnet); - Ok(SledAgentResponse { id: self.sled_config.id }) + Ok(SledAgentResponse { id: request.id }) } /// Communicates with peers, sharing secrets, until the rack has been diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index d79d82e47b3..e0747bbc2b0 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -84,7 +84,6 @@ impl Server { info!(log, "detecting (real or simulated) SP"); let sp = SpHandle::detect( config.sp_config.as_ref().map(|c| &c.local_sp), - &sled_config, &log, ) .await diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 7986b3b6477..e02448c646c 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -10,13 +10,10 @@ use crate::illumos::zpool::ZpoolName; use dropshot::ConfigLogging; use serde::Deserialize; use std::path::{Path, PathBuf}; -use uuid::Uuid; /// Configuration for a sled agent #[derive(Clone, Debug, Deserialize)] pub struct Config { - /// Unique id for the sled - pub id: Uuid, /// Configuration for the sled agent debug log pub log: ConfigLogging, /// Optionally force the sled to self-identify as a scrimlet (or gimlet, diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 89a3db00d9a..ddaf9be7b3d 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -7,7 +7,7 @@ use crate::params::{ DatasetEnsureBody, DiskEnsureBody, InstanceEnsureBody, InstanceSerialConsoleData, InstanceSerialConsoleRequest, ServiceEnsureBody, - VpcFirewallRulesEnsureBody, + VpcFirewallRulesEnsureBody, Zpool, }; use crate::serial::ByteOffset; use dropshot::{ @@ -33,6 +33,7 @@ type SledApiDescription = ApiDescription; pub fn api() -> SledApiDescription { fn register_endpoints(api: &mut SledApiDescription) -> Result<(), String> { api.register(services_put)?; + api.register(zpools_get)?; api.register(filesystem_put)?; api.register(instance_put)?; api.register(disk_put)?; @@ -66,6 +67,17 @@ async fn services_put( Ok(HttpResponseUpdatedNoContent()) } +#[endpoint { + method = GET, + path = "/zpools", +}] +async fn zpools_get( + rqctx: Arc>, +) -> Result>, HttpError> { + let sa = rqctx.context(); + Ok(HttpResponseOk(sa.zpools_get().await.map_err(|e| Error::from(e))?)) +} + #[endpoint { method = PUT, path = "/filesystem", diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index a63a5947db7..8f3996a13b7 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -230,6 +230,11 @@ pub struct InstanceSerialConsoleData { pub last_byte_offset: u64, } +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct Zpool { + pub id: Uuid, +} + // The type of networking 'ASIC' the Dendrite service is expected to manage #[derive( Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Copy, Hash, diff --git a/sled-agent/src/rack_setup/config.rs b/sled-agent/src/rack_setup/config.rs index 576344d6f9d..3938608465b 100644 --- a/sled-agent/src/rack_setup/config.rs +++ b/sled-agent/src/rack_setup/config.rs @@ -6,13 +6,12 @@ use crate::bootstrap::params::Gateway; use crate::config::ConfigError; -use crate::params::{DatasetEnsureBody, ServiceZoneRequest}; use omicron_common::address::{ get_64_subnet, Ipv6Subnet, AZ_PREFIX, RACK_PREFIX, SLED_PREFIX, }; use serde::Deserialize; use serde::Serialize; -use std::net::Ipv6Addr; +use std::net::{IpAddr, Ipv6Addr}; use std::path::Path; /// Configuration for the "rack setup service", which is controlled during @@ -29,9 +28,6 @@ use std::path::Path; pub struct SetupServiceConfig { pub rack_subnet: Ipv6Addr, - #[serde(default, rename = "request")] - pub requests: Vec, - /// The minimum number of sleds required to unlock the rack secret. /// /// If this value is less than 2, no rack secret will be created on startup; @@ -40,22 +36,11 @@ pub struct SetupServiceConfig { /// Internet gateway information. pub gateway: Gateway, -} - -/// A request to initialize a sled. -#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] -pub struct HardcodedSledRequest { - /// Datasets to be created. - #[serde(default, rename = "dataset")] - pub datasets: Vec, - - /// Services to be instantiated. - #[serde(default, rename = "service_zone")] - pub service_zones: Vec, - /// DNS Services to be instantiated. - #[serde(default, rename = "dns_service")] - pub dns_services: Vec, + /// The address on which Nexus should serve an external interface. + // TODO(https://github.com/oxidecomputer/omicron/issues/1530): Eventually, + // this should be pulled from a pool of addresses. + pub nexus_external_address: IpAddr, } impl SetupServiceConfig { @@ -90,9 +75,9 @@ mod test { fn test_subnets() { let cfg = SetupServiceConfig { rack_subnet: "fd00:1122:3344:0100::".parse().unwrap(), - requests: vec![], rack_secret_threshold: 0, gateway: Gateway { address: None, mac: macaddr::MacAddr6::nil() }, + nexus_external_address: "192.168.1.20".parse().unwrap(), }; assert_eq!( diff --git a/sled-agent/src/rack_setup/mod.rs b/sled-agent/src/rack_setup/mod.rs index e947ff99ef0..4df85a7727f 100644 --- a/sled-agent/src/rack_setup/mod.rs +++ b/sled-agent/src/rack_setup/mod.rs @@ -5,4 +5,5 @@ //! Rack Setup Service pub mod config; +mod plan; pub mod service; diff --git a/sled-agent/src/rack_setup/plan/mod.rs b/sled-agent/src/rack_setup/plan/mod.rs new file mode 100644 index 00000000000..2343a3be2e6 --- /dev/null +++ b/sled-agent/src/rack_setup/plan/mod.rs @@ -0,0 +1,8 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Rack Setup Service plan generation + +pub mod service; +pub mod sled; diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs new file mode 100644 index 00000000000..d38995751ce --- /dev/null +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -0,0 +1,354 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Plan generation for "where should services be initialized". + +use crate::params::{ + DatasetEnsureBody, ServiceType, ServiceZoneRequest, ZoneType, +}; +use crate::rack_setup::config::SetupServiceConfig as Config; +use omicron_common::address::{ + get_switch_zone_address, Ipv6Subnet, ReservedRackSubnet, DNS_PORT, + DNS_SERVER_PORT, RSS_RESERVED_ADDRESSES, SLED_PREFIX, +}; +use omicron_common::backoff::{ + internal_service_policy, retry_notify, BackoffError, +}; +use serde::{Deserialize, Serialize}; +use sled_agent_client::{ + types as SledAgentTypes, Client as SledAgentClient, Error as SledAgentError, +}; +use slog::Logger; +use std::collections::HashMap; +use std::net::{Ipv6Addr, SocketAddrV6}; +use std::path::{Path, PathBuf}; +use thiserror::Error; +use uuid::Uuid; + +// The number of Nexus instances to create from RSS. +const NEXUS_COUNT: usize = 1; + +// The number of CRDB instances to create from RSS. +const CRDB_COUNT: usize = 1; + +// TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove +// when Nexus provisions Oximeter. +const OXIMETER_COUNT: usize = 1; +// TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove +// when Nexus provisions Clickhouse. +const CLICKHOUSE_COUNT: usize = 1; +// TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove. +// when Nexus provisions Crucible. +const MINIMUM_ZPOOL_COUNT: usize = 3; + +fn rss_service_plan_path() -> PathBuf { + Path::new(omicron_common::OMICRON_CONFIG_PATH).join("rss-service-plan.toml") +} + +/// Describes errors which may occur while generating a plan for services. +#[derive(Error, Debug)] +pub enum PlanError { + #[error("I/O error while {message}: {err}")] + Io { + message: String, + #[source] + err: std::io::Error, + }, + + #[error("Cannot deserialize TOML file at {path}: {err}")] + Toml { path: PathBuf, err: toml::de::Error }, + + #[error("Error making HTTP request to Sled Agent: {0}")] + SledApi(#[from] SledAgentError), + + #[error("Error initializing sled via sled-agent: {0}")] + SledInitialization(String), + + #[error("Failed to construct an HTTP client: {0}")] + HttpClient(reqwest::Error), +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct SledRequest { + /// Datasets to be created. + #[serde(default, rename = "dataset")] + pub datasets: Vec, + + /// Services to be instantiated. + #[serde(default, rename = "service")] + pub services: Vec, + + /// DNS Services to be instantiated. + #[serde(default, rename = "dns_service")] + pub dns_services: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Plan { + pub services: HashMap, +} + +impl Plan { + pub async fn load(log: &Logger) -> Result, PlanError> { + // If we already created a plan for this RSS to allocate + // services to sleds, re-use that existing plan. + let rss_service_plan_path = rss_service_plan_path(); + if rss_service_plan_path.exists() { + info!(log, "RSS plan already created, loading from file"); + + let plan: Self = toml::from_str( + &tokio::fs::read_to_string(&rss_service_plan_path) + .await + .map_err(|err| PlanError::Io { + message: format!( + "Loading RSS plan {rss_service_plan_path:?}" + ), + err, + })?, + ) + .map_err(|err| PlanError::Toml { + path: rss_service_plan_path, + err, + })?; + Ok(Some(plan)) + } else { + Ok(None) + } + } + // Gets zpool UUIDs from the sled. + async fn get_zpools_from_sled( + log: &Logger, + address: SocketAddrV6, + ) -> Result, PlanError> { + let dur = std::time::Duration::from_secs(60); + let client = reqwest::ClientBuilder::new() + .connect_timeout(dur) + .timeout(dur) + .build() + .map_err(PlanError::HttpClient)?; + let client = SledAgentClient::new_with_client( + &format!("http://{}", address), + client, + log.new(o!("SledAgentClient" => address.to_string())), + ); + + let get_zpools = || async { + let zpools: Vec = client + .zpools_get() + .await + .map(|response| { + response + .into_inner() + .into_iter() + .map(|zpool| zpool.id) + .collect() + }) + .map_err(|err| { + BackoffError::transient(PlanError::SledApi(err)) + })?; + + // TODO(https://github.com/oxidecomputer/omicron/issues/732): + // We're currently waiting for ALL zpools to appear, so RSS can be + // responsible for provisioning Crucible datasets. + // + // Once this responsibility shifts to Nexus, we actually only + // need enough zpools to provision CRDB. + if zpools.len() < MINIMUM_ZPOOL_COUNT { + return Err(BackoffError::transient( + PlanError::SledInitialization( + "Awaiting zpools".to_string(), + ), + )); + } + + Ok(zpools) + }; + let log_failure = |error, _| { + warn!(log, "failed to get zpools"; "error" => ?error); + }; + let zpools = + retry_notify(internal_service_policy(), get_zpools, log_failure) + .await?; + + Ok(zpools) + } + + pub async fn create( + log: &Logger, + config: &Config, + sled_addrs: &Vec, + ) -> Result { + let reserved_rack_subnet = ReservedRackSubnet::new(config.az_subnet()); + let dns_subnets = reserved_rack_subnet.get_dns_subnets(); + + let mut allocations = vec![]; + + for idx in 0..sled_addrs.len() { + let sled_address = sled_addrs[idx]; + let sled_subnet_index = + u8::try_from(idx + 1).expect("Too many peers!"); + let subnet = config.sled_subnet(sled_subnet_index); + let zpools = Self::get_zpools_from_sled(log, sled_address).await?; + let mut addr_alloc = AddressBumpAllocator::new(subnet); + + let mut request = SledRequest::default(); + + // The first enumerated sleds get assigned the responsibility + // of hosting Nexus. + if idx < NEXUS_COUNT { + let address = addr_alloc.next().expect("Not enough addrs"); + request.services.push(ServiceZoneRequest { + id: Uuid::new_v4(), + zone_type: ZoneType::Nexus, + addresses: vec![address], + gz_addresses: vec![], + services: vec![ServiceType::Nexus { + internal_ip: address, + external_ip: config.nexus_external_address, + }], + }) + } + + // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove + if idx < OXIMETER_COUNT { + let address = addr_alloc.next().expect("Not enough addrs"); + request.services.push(ServiceZoneRequest { + id: Uuid::new_v4(), + zone_type: ZoneType::Oximeter, + addresses: vec![address], + gz_addresses: vec![], + services: vec![ServiceType::Oximeter], + }) + } + + // The first enumerated sleds host the CRDB datasets, using + // zpools described from the underlying config file. + if idx < CRDB_COUNT { + let address = SocketAddrV6::new( + addr_alloc.next().expect("Not enough addrs"), + omicron_common::address::COCKROACH_PORT, + 0, + 0, + ); + request.datasets.push(DatasetEnsureBody { + id: Uuid::new_v4(), + zpool_id: zpools[0], + dataset_kind: crate::params::DatasetKind::CockroachDb { + all_addresses: vec![address], + }, + address, + }); + } + + // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove + if idx < CLICKHOUSE_COUNT { + let address = SocketAddrV6::new( + addr_alloc.next().expect("Not enough addrs"), + omicron_common::address::CLICKHOUSE_PORT, + 0, + 0, + ); + request.datasets.push(DatasetEnsureBody { + id: Uuid::new_v4(), + zpool_id: zpools[0], + dataset_kind: crate::params::DatasetKind::Clickhouse, + address, + }); + } + + // Each zpool gets a crucible zone. + // + // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove + for zpool_id in zpools { + let address = SocketAddrV6::new( + addr_alloc.next().expect("Not enough addrs"), + omicron_common::address::CRUCIBLE_PORT, + 0, + 0, + ); + request.datasets.push(DatasetEnsureBody { + id: Uuid::new_v4(), + zpool_id, + dataset_kind: crate::params::DatasetKind::Crucible, + address, + }); + } + + // The first enumerated sleds get assigned the additional + // responsibility of being internal DNS servers. + if idx < dns_subnets.len() { + let dns_subnet = &dns_subnets[idx]; + let dns_addr = dns_subnet.dns_address().ip(); + request.dns_services.push(ServiceZoneRequest { + id: Uuid::new_v4(), + zone_type: ZoneType::InternalDNS, + addresses: vec![dns_addr], + gz_addresses: vec![dns_subnet.gz_address().ip()], + services: vec![ServiceType::InternalDns { + server_address: SocketAddrV6::new( + dns_addr, + DNS_SERVER_PORT, + 0, + 0, + ), + dns_address: SocketAddrV6::new( + dns_addr, DNS_PORT, 0, 0, + ), + }], + }); + } + + allocations.push((sled_address, request)); + } + + let mut services = std::collections::HashMap::new(); + for (addr, allocation) in allocations { + services.insert(addr, allocation); + } + + let plan = Self { services }; + + // Once we've constructed a plan, write it down to durable storage. + let serialized_plan = + toml::Value::try_from(&plan).unwrap_or_else(|e| { + panic!("Cannot serialize configuration: {:#?}: {}", plan, e) + }); + let plan_str = toml::to_string(&serialized_plan) + .expect("Cannot turn config to string"); + + info!(log, "Plan serialized as: {}", plan_str); + let path = rss_service_plan_path(); + tokio::fs::write(&path, plan_str).await.map_err(|err| { + PlanError::Io { + message: format!("Storing RSS service plan to {path:?}"), + err, + } + })?; + info!(log, "Service plan written to storage"); + + Ok(plan) + } +} + +struct AddressBumpAllocator { + last_addr: Ipv6Addr, +} + +// TODO: Testable? +// TODO: Could exist in another file? +impl AddressBumpAllocator { + fn new(subnet: Ipv6Subnet) -> Self { + Self { last_addr: get_switch_zone_address(subnet) } + } + + fn next(&mut self) -> Option { + let mut segments: [u16; 8] = self.last_addr.segments(); + segments[7] = segments[7].checked_add(1)?; + if segments[7] > RSS_RESERVED_ADDRESSES { + return None; + } + self.last_addr = Ipv6Addr::from(segments); + Some(self.last_addr) + } +} diff --git a/sled-agent/src/rack_setup/plan/sled.rs b/sled-agent/src/rack_setup/plan/sled.rs new file mode 100644 index 00000000000..cde70deb690 --- /dev/null +++ b/sled-agent/src/rack_setup/plan/sled.rs @@ -0,0 +1,245 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Plan generation for "how should sleds be initialized". + +use crate::bootstrap::{ + config::BOOTSTRAP_AGENT_PORT, + params::SledAgentRequest, + trust_quorum::{RackSecret, ShareDistribution}, +}; +use crate::rack_setup::config::SetupServiceConfig as Config; +use serde::{Deserialize, Serialize}; +use slog::Logger; +use sprockets_host::Ed25519Certificate; +use std::collections::HashMap; +use std::net::{Ipv6Addr, SocketAddrV6}; +use std::path::{Path, PathBuf}; +use thiserror::Error; +use uuid::Uuid; + +fn rss_sled_plan_path() -> PathBuf { + Path::new(omicron_common::OMICRON_CONFIG_PATH).join("rss-sled-plan.toml") +} + +pub fn generate_rack_secret<'a>( + rack_secret_threshold: usize, + member_device_id_certs: &'a [Ed25519Certificate], + log: &Logger, +) -> Result< + Option + 'a>, + PlanError, +> { + // We do not generate a rack secret if we only have a single sled or if our + // config specifies that the threshold for unlock is only a single sled. + let total_shares = member_device_id_certs.len(); + if total_shares <= 1 { + info!(log, "Skipping rack secret creation (only one sled present)"); + return Ok(None); + } + + if rack_secret_threshold <= 1 { + warn!( + log, + concat!( + "Skipping rack secret creation due to config", + " (despite discovery of {} bootstrap agents)" + ), + total_shares, + ); + return Ok(None); + } + + let secret = RackSecret::new(); + let (shares, verifier) = secret + .split(rack_secret_threshold, total_shares) + .map_err(PlanError::SplitRackSecret)?; + + Ok(Some(shares.into_iter().map(move |share| ShareDistribution { + threshold: rack_secret_threshold, + verifier: verifier.clone(), + share, + member_device_id_certs: member_device_id_certs.to_vec(), + }))) +} + +/// Describes errors which may occur while generating a plan for sleds. +#[derive(Error, Debug)] +pub enum PlanError { + #[error("I/O error while {message}: {err}")] + Io { + message: String, + #[source] + err: std::io::Error, + }, + + #[error("Cannot deserialize TOML file at {path}: {err}")] + Toml { path: PathBuf, err: toml::de::Error }, + + #[error("Failed to split rack secret: {0:?}")] + SplitRackSecret(vsss_rs::Error), +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Plan { + pub rack_id: Uuid, + pub sleds: HashMap, + // TODO: Consider putting the rack subnet here? This may be operator-driven + // in the future, so it should exist in the "plan". + // + // TL;DR: The more we decouple rom "rss-config.toml", the easier it'll be to + // switch to an operator-driven interface. +} + +impl Plan { + pub async fn load(log: &Logger) -> Result, PlanError> { + // If we already created a plan for this RSS to allocate + // subnets/requests to sleds, re-use that existing plan. + let rss_sled_plan_path = rss_sled_plan_path(); + if rss_sled_plan_path.exists() { + info!(log, "RSS plan already created, loading from file"); + + let plan: Self = toml::from_str( + &tokio::fs::read_to_string(&rss_sled_plan_path).await.map_err( + |err| PlanError::Io { + message: format!( + "Loading RSS plan {rss_sled_plan_path:?}" + ), + err, + }, + )?, + ) + .map_err(|err| PlanError::Toml { path: rss_sled_plan_path, err })?; + Ok(Some(plan)) + } else { + Ok(None) + } + } + + pub async fn create( + log: &Logger, + config: &Config, + bootstrap_addrs: Vec, + ) -> Result { + let rack_id = Uuid::new_v4(); + + let bootstrap_addrs = bootstrap_addrs.into_iter().enumerate(); + let allocations = bootstrap_addrs.map(|(idx, bootstrap_addr)| { + info!(log, "Creating plan for the sled at {:?}", bootstrap_addr); + let bootstrap_addr = + SocketAddrV6::new(bootstrap_addr, BOOTSTRAP_AGENT_PORT, 0, 0); + let sled_subnet_index = + u8::try_from(idx + 1).expect("Too many peers!"); + let subnet = config.sled_subnet(sled_subnet_index); + + ( + bootstrap_addr, + SledAgentRequest { + id: Uuid::new_v4(), + subnet, + gateway: config.gateway.clone(), + rack_id, + }, + ) + }); + + info!(log, "Serializing plan"); + + let mut sleds = std::collections::HashMap::new(); + for (addr, allocation) in allocations { + sleds.insert(addr, allocation); + } + + let plan = Self { rack_id, sleds }; + + // Once we've constructed a plan, write it down to durable storage. + let serialized_plan = + toml::Value::try_from(&plan).unwrap_or_else(|e| { + panic!("Cannot serialize configuration: {:#?}: {}", plan, e) + }); + let plan_str = toml::to_string(&serialized_plan) + .expect("Cannot turn config to string"); + + info!(log, "Plan serialized as: {}", plan_str); + let path = rss_sled_plan_path(); + tokio::fs::write(&path, plan_str).await.map_err(|err| { + PlanError::Io { + message: format!("Storing RSS sled plan to {path:?}"), + err, + } + })?; + info!(log, "Sled plan written to storage"); + + Ok(plan) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use omicron_test_utils::dev::test_setup_log; + use sprockets_common::certificates::Ed25519Signature; + use sprockets_common::certificates::KeyType; + use std::collections::HashSet; + + fn dummy_certs(n: usize) -> Vec { + vec![ + Ed25519Certificate { + subject_key_type: KeyType::DeviceId, + subject_public_key: sprockets_host::Ed25519PublicKey([0; 32]), + signer_key_type: KeyType::Manufacturing, + signature: Ed25519Signature([0; 64]), + }; + n + ] + } + + #[test] + fn test_generate_rack_secret() { + let logctx = test_setup_log("test_generate_rack_secret"); + + // No secret generated if we have <= 1 sled + assert!(generate_rack_secret(10, &dummy_certs(1), &logctx.log) + .unwrap() + .is_none()); + + // No secret generated if threshold <= 1 + assert!(generate_rack_secret(1, &dummy_certs(10), &logctx.log) + .unwrap() + .is_none()); + + // Secret generation fails if threshold > total sleds + assert!(matches!( + generate_rack_secret(10, &dummy_certs(5), &logctx.log), + Err(PlanError::SplitRackSecret(_)) + )); + + // Secret generation succeeds if threshold <= total shares and both are + // > 1, and the returned iterator satifies: + // + // * total length == total shares + // * each share is distinct + for total_shares in 2..=32 { + for threshold in 2..=total_shares { + let certs = dummy_certs(total_shares); + let shares = + generate_rack_secret(threshold, &certs, &logctx.log) + .unwrap() + .unwrap(); + + assert_eq!(shares.len(), total_shares); + + // `Share` doesn't implement `Hash`, but it's a newtype around + // `Vec` (which does). Unwrap the newtype to check that all + // shares are distinct. + let shares_set = shares + .map(|share_dist| share_dist.share.0) + .collect::>(); + assert_eq!(shares_set.len(), total_shares); + } + } + + logctx.cleanup_successful(); + } +} diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index ee0ab306787..ae4e9ec986a 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -4,21 +4,33 @@ //! Rack Setup Service implementation -use super::config::{HardcodedSledRequest, SetupServiceConfig as Config}; -use crate::bootstrap::config::BOOTSTRAP_AGENT_PORT; +use super::config::SetupServiceConfig as Config; use crate::bootstrap::ddm_admin_client::{DdmAdminClient, DdmError}; use crate::bootstrap::params::SledAgentRequest; use crate::bootstrap::rss_handle::BootstrapAgentHandle; -use crate::bootstrap::trust_quorum::{RackSecret, ShareDistribution}; -use crate::params::{ServiceType, ServiceZoneRequest, ZoneType}; -use internal_dns_client::multiclient::{DnsError, Updater as DnsUpdater}; -use omicron_common::address::{ - get_sled_address, ReservedRackSubnet, DNS_PORT, DNS_SERVER_PORT, +use crate::params::{DatasetEnsureBody, ServiceType, ServiceZoneRequest}; +use crate::rack_setup::plan::service::{ + Plan as ServicePlan, PlanError as ServicePlanError, }; +use crate::rack_setup::plan::sled::{ + generate_rack_secret, Plan as SledPlan, PlanError as SledPlanError, +}; +use internal_dns_client::multiclient::{ + DnsError, Resolver as DnsResolver, Updater as DnsUpdater, +}; +use internal_dns_client::names::{ServiceName, SRV}; +use nexus_client::{ + types as NexusTypes, Client as NexusClient, Error as NexusError, +}; +use omicron_common::address::{get_sled_address, NEXUS_INTERNAL_PORT}; use omicron_common::backoff::{ - internal_service_policy, retry_notify, BackoffError, + internal_service_policy, internal_service_policy_with_max, retry_notify, + BackoffError, }; use serde::{Deserialize, Serialize}; +use sled_agent_client::{ + types as SledAgentTypes, Client as SledAgentClient, Error as SledAgentError, +}; use slog::Logger; use sprockets_host::Ed25519Certificate; use std::collections::{HashMap, HashSet}; @@ -27,7 +39,9 @@ use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use std::path::PathBuf; use thiserror::Error; use tokio::sync::OnceCell; -use uuid::Uuid; + +// The minimum number of sleds to initialize the rack. +const MINIMUM_SLED_COUNT: usize = 1; /// Describes errors which may occur while operating the setup service. #[derive(Error, Debug)] @@ -39,27 +53,33 @@ pub enum SetupServiceError { err: std::io::Error, }, + #[error("Cannot create plan for sled services: {0}")] + ServicePlan(#[from] ServicePlanError), + + #[error("Cannot create plan for sled setup: {0}")] + SledPlan(#[from] SledPlanError), + + #[error("Bad configuration for setting up rack: {0}")] + BadConfig(String), + #[error("Error initializing sled via sled-agent: {0}")] SledInitialization(String), #[error("Error making HTTP request to Sled Agent: {0}")] - SledApi(#[from] sled_agent_client::Error), + SledApi(#[from] SledAgentError), + + #[error("Error making HTTP request to Nexus: {0}")] + NexusApi(#[from] NexusError), #[error("Error contacting ddmd: {0}")] DdmError(#[from] DdmError), - #[error("Cannot deserialize TOML file at {path}: {err}")] - Toml { path: PathBuf, err: toml::de::Error }, - #[error("Failed to monitor for peers: {0}")] PeerMonitor(#[from] tokio::sync::broadcast::error::RecvError), #[error("Failed to construct an HTTP client: {0}")] HttpClient(reqwest::Error), - #[error("Failed to split rack secret: {0:?}")] - SplitRackSecret(vsss_rs::Error), - #[error("Failed to access DNS servers: {0}")] Dns(#[from] DnsError), } @@ -68,7 +88,6 @@ pub enum SetupServiceError { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] struct SledAllocation { initialization_request: SledAgentRequest, - services_request: HardcodedSledRequest, } /// The interface to the Rack Setup Service. @@ -123,12 +142,12 @@ impl RackSetupService { } } -fn rss_plan_path() -> std::path::PathBuf { +fn rss_plan_path() -> PathBuf { std::path::Path::new(omicron_common::OMICRON_CONFIG_PATH) .join("rss-plan.toml") } -fn rss_completed_plan_path() -> std::path::PathBuf { +fn rss_completed_plan_path() -> PathBuf { std::path::Path::new(omicron_common::OMICRON_CONFIG_PATH) .join("rss-plan-completed.toml") } @@ -164,8 +183,8 @@ impl ServiceInner { async fn initialize_datasets( &self, - sled_address: SocketAddr, - datasets: &Vec, + sled_address: SocketAddrV6, + datasets: &Vec, ) -> Result<(), SetupServiceError> { let dur = std::time::Duration::from_secs(60); @@ -174,10 +193,10 @@ impl ServiceInner { .timeout(dur) .build() .map_err(SetupServiceError::HttpClient)?; - let client = sled_agent_client::Client::new_with_client( + let client = SledAgentClient::new_with_client( &format!("http://{}", sled_address), client, - self.log.new(o!("SledAgentClient" => sled_address)), + self.log.new(o!("SledAgentClient" => sled_address.to_string())), ); info!(self.log, "sending dataset requests..."); @@ -188,14 +207,7 @@ impl ServiceInner { .filesystem_put(&dataset.clone().into()) .await .map_err(BackoffError::transient)?; - Ok::< - (), - BackoffError< - sled_agent_client::Error< - sled_agent_client::types::Error, - >, - >, - >(()) + Ok::<(), BackoffError>>(()) }; let log_failure = |error, _| { warn!(self.log, "failed to create filesystem"; "error" => ?error); @@ -207,12 +219,35 @@ impl ServiceInner { ) .await?; } + + let mut records = HashMap::new(); + for dataset in datasets { + records + .entry(dataset.srv()) + .or_insert_with(Vec::new) + .push((dataset.aaaa(), dataset.address())); + } + let records_put = || async { + self.dns_servers + .get() + .expect("DNS servers must be initialized first") + .insert_dns_records(&records) + .await + .map_err(BackoffError::transient)?; + Ok::<(), BackoffError>(()) + }; + let log_failure = |error, _| { + warn!(self.log, "failed to set DNS records"; "error" => ?error); + }; + retry_notify(internal_service_policy(), records_put, log_failure) + .await?; + Ok(()) } async fn initialize_services( &self, - sled_address: SocketAddr, + sled_address: SocketAddrV6, services: &Vec, ) -> Result<(), SetupServiceError> { let dur = std::time::Duration::from_secs(60); @@ -221,17 +256,17 @@ impl ServiceInner { .timeout(dur) .build() .map_err(SetupServiceError::HttpClient)?; - let client = sled_agent_client::Client::new_with_client( + let client = SledAgentClient::new_with_client( &format!("http://{}", sled_address), client, - self.log.new(o!("SledAgentClient" => sled_address)), + self.log.new(o!("SledAgentClient" => sled_address.to_string())), ); info!(self.log, "sending service requests..."); let services_put = || async { info!(self.log, "initializing sled services: {:?}", services); client - .services_put(&sled_agent_client::types::ServiceEnsureBody { + .services_put(&SledAgentTypes::ServiceEnsureBody { services: services .iter() .map(|s| s.clone().into()) @@ -239,156 +274,42 @@ impl ServiceInner { }) .await .map_err(BackoffError::transient)?; - Ok::< - (), - BackoffError< - sled_agent_client::Error, - >, - >(()) + Ok::<(), BackoffError>>(()) }; let log_failure = |error, _| { warn!(self.log, "failed to initialize services"; "error" => ?error); }; retry_notify(internal_service_policy(), services_put, log_failure) .await?; - Ok(()) - } - - async fn load_plan( - &self, - ) -> Result>, SetupServiceError> - { - // If we already created a plan for this RSS to allocate - // subnets/requests to sleds, re-use that existing plan. - let rss_plan_path = rss_plan_path(); - if rss_plan_path.exists() { - info!(self.log, "RSS plan already created, loading from file"); - - let plan: std::collections::HashMap = - toml::from_str( - &tokio::fs::read_to_string(&rss_plan_path).await.map_err( - |err| SetupServiceError::Io { - message: format!( - "Loading RSS plan {rss_plan_path:?}" - ), - err, - }, - )?, - ) - .map_err(|err| SetupServiceError::Toml { - path: rss_plan_path, - err, - })?; - Ok(Some(plan)) - } else { - Ok(None) - } - } - async fn create_plan( - &self, - config: &Config, - bootstrap_addrs: Vec, - ) -> Result, SetupServiceError> { - let bootstrap_addrs = bootstrap_addrs.into_iter().enumerate(); - let reserved_rack_subnet = ReservedRackSubnet::new(config.az_subnet()); - let dns_subnets = reserved_rack_subnet.get_dns_subnets(); - - info!(self.log, "dns_subnets: {:#?}", dns_subnets); - - let requests_and_sleds = - bootstrap_addrs.map(|(idx, bootstrap_addr)| { - // If a sled was explicitly requested from the RSS configuration, - // use that. Otherwise, just give it a "default" (empty) set of - // services. - let mut request = { - if idx < config.requests.len() { - config.requests[idx].clone() - } else { - HardcodedSledRequest::default() + // Insert DNS records, if the DNS servers have been initialized + if let Some(dns_servers) = self.dns_servers.get() { + let mut records = HashMap::new(); + for zone in services { + for service in &zone.services { + if let Some(addr) = zone.address(&service) { + records + .entry(zone.srv(&service)) + .or_insert_with(Vec::new) + .push((zone.aaaa(), addr)); } - }; - - // The first enumerated sleds get assigned the additional - // responsibility of being internal DNS servers. - if idx < dns_subnets.len() { - let dns_subnet = &dns_subnets[idx]; - let dns_addr = dns_subnet.dns_address().ip(); - request.dns_services.push(ServiceZoneRequest { - id: Uuid::new_v4(), - zone_type: ZoneType::InternalDNS, - addresses: vec![dns_addr], - gz_addresses: vec![dns_subnet.gz_address().ip()], - services: vec![ServiceType::InternalDns { - server_address: SocketAddrV6::new( - dns_addr, - DNS_SERVER_PORT, - 0, - 0, - ), - dns_address: SocketAddrV6::new( - dns_addr, DNS_PORT, 0, 0, - ), - }], - }); } - - (request, (idx, bootstrap_addr)) - }); - - let rack_id = Uuid::new_v4(); - let allocations = requests_and_sleds.map(|(request, sled)| { - let (idx, bootstrap_addr) = sled; - info!( - self.log, - "Creating plan for the sled at {:?}", bootstrap_addr - ); - let bootstrap_addr = - SocketAddrV6::new(bootstrap_addr, BOOTSTRAP_AGENT_PORT, 0, 0); - let sled_subnet_index = - u8::try_from(idx + 1).expect("Too many peers!"); - let subnet = config.sled_subnet(sled_subnet_index); - - ( - bootstrap_addr, - SledAllocation { - initialization_request: SledAgentRequest { - id: Uuid::new_v4(), - subnet, - rack_id, - gateway: config.gateway.clone(), - }, - services_request: request, - }, - ) - }); - - info!(self.log, "Serializing plan"); - - let mut plan = std::collections::HashMap::new(); - for (addr, allocation) in allocations { - plan.insert(addr, allocation); - } - - // Once we've constructed a plan, write it down to durable storage. - let serialized_plan = - toml::Value::try_from(&plan).unwrap_or_else(|e| { - panic!("Cannot serialize configuration: {:#?}: {}", plan, e) - }); - let plan_str = toml::to_string(&serialized_plan) - .expect("Cannot turn config to string"); - - info!(self.log, "Plan serialized as: {}", plan_str); - let path = rss_plan_path(); - tokio::fs::write(&path, plan_str).await.map_err(|err| { - SetupServiceError::Io { - message: format!("Storing RSS plan to {path:?}"), - err, } - })?; - info!(self.log, "Plan written to storage"); + let records_put = || async { + dns_servers + .insert_dns_records(&records) + .await + .map_err(BackoffError::transient)?; + Ok::<(), BackoffError>(()) + }; + let log_failure = |error, _| { + warn!(self.log, "failed to set DNS records"; "error" => ?error); + }; + retry_notify(internal_service_policy(), records_put, log_failure) + .await?; + } - Ok(plan) + Ok(()) } // Waits for sufficient neighbors to exist so the initial set of requests @@ -459,6 +380,120 @@ impl ServiceInner { Ok(addrs) } + async fn handoff_to_nexus( + &self, + config: &Config, + sled_plan: &SledPlan, + service_plan: &ServicePlan, + ) -> Result<(), SetupServiceError> { + info!(self.log, "Handing off control to Nexus"); + + let resolver = DnsResolver::new(&config.az_subnet()) + .expect("Failed to create DNS resolver"); + let ip = resolver + .lookup_ip(SRV::Service(ServiceName::Nexus)) + .await + .expect("Failed to lookup IP"); + let nexus_address = SocketAddr::new(ip, NEXUS_INTERNAL_PORT); + + info!(self.log, "Nexus address: {}", nexus_address.to_string()); + + let nexus_client = NexusClient::new( + &format!("http://{}", nexus_address), + self.log.new(o!("component" => "NexusClient")), + ); + + // Ensure we can quickly look up "Sled Agent Address" -> "UUID of sled". + // + // We need the ID when passing info to Nexus. + let mut id_map = HashMap::new(); + for (_, sled_request) in sled_plan.sleds.iter() { + id_map + .insert(get_sled_address(sled_request.subnet), sled_request.id); + } + + // Convert all the information we have about services and datasets into + // a format which can be processed by Nexus. + let mut services: Vec = vec![]; + let mut datasets: Vec = vec![]; + for (addr, service_request) in service_plan.services.iter() { + let sled_id = *id_map + .get(addr) + .expect("Sled address in service plan, but not sled plan"); + + for zone in service_request + .services + .iter() + .chain(service_request.dns_services.iter()) + { + for svc in &zone.services { + let kind = match svc { + ServiceType::Nexus { external_ip, internal_ip: _ } => { + NexusTypes::ServiceKind::Nexus { + external_address: *external_ip, + } + } + ServiceType::InternalDns { .. } => { + NexusTypes::ServiceKind::InternalDNS + } + ServiceType::Oximeter => { + NexusTypes::ServiceKind::Oximeter + } + // TODO TODO TODO + ServiceType::ManagementGatewayService => todo!(), + ServiceType::Dendrite { .. } => { + NexusTypes::ServiceKind::Dendrite + } + // TODO TODO TODO + ServiceType::Tfport { .. } => todo!(), + }; + + services.push(NexusTypes::ServicePutRequest { + service_id: zone.id, + sled_id, + // TODO: Should this be a vec, or a single value? + address: zone.addresses[0], + kind, + }) + } + } + + for dataset in service_request.datasets.iter() { + datasets.push(NexusTypes::DatasetCreateRequest { + zpool_id: dataset.zpool_id, + dataset_id: dataset.id, + request: NexusTypes::DatasetPutRequest { + address: dataset.address.to_string(), + kind: dataset.dataset_kind.clone().into(), + }, + }) + } + } + + let request = + NexusTypes::RackInitializationRequest { services, datasets }; + + let notify_nexus = || async { + nexus_client + .rack_initialization_complete(&sled_plan.rack_id, &request) + .await + .map_err(BackoffError::transient) + }; + let log_failure = |err, _| { + info!(self.log, "Failed to handoff to nexus: {err}"); + }; + + retry_notify( + internal_service_policy_with_max(std::time::Duration::from_secs(1)), + notify_nexus, + log_failure, + ) + .await?; + + info!(self.log, "Handoff to Nexus is complete"); + Ok(()) + } + // In lieu of having an operator send requests to all sleds via an // initialization service, the sled-agent configuration may allow for the // automated injection of setup requests from a sled. @@ -466,19 +501,23 @@ impl ServiceInner { // This method has a few distinct phases, identified by files in durable // storage: // - // 1. ALLOCATION PLAN CREATION. When the RSS starts up for the first time, - // it creates an allocation plan to provision subnets and services - // to an initial set of sleds. - // - // This plan is stored at "rss_plan_path()". + // 1. SLED ALLOCATION PLAN CREATION. When the RSS starts up for the first + // time, it creates an allocation plan to provision subnets to an initial + // set of sleds. // - // 2. ALLOCATION PLAN EXECUTION. The RSS then carries out this plan, making + // 2. SLED ALLOCATION PLAN EXECUTION. The RSS then carries out this plan, making // requests to the sleds enumerated within the "allocation plan". // - // 3. MARKING SETUP COMPLETE. Once the RSS has successfully initialized the - // rack, the "rss_plan_path()" file is renamed to - // "rss_completed_plan_path()". This indicates that the plan executed - // successfully, and no work remains. + // 3. SERVICE ALLOCATION PLAN CREATION. Now that Sled Agents are executing + // on their respsective subnets, they can be queried to create an + // allocation plan for services. + // + // 4. SERVICE ALLOCATION PLAN EXECUTION. RSS requests that the services + // outlined in the aforementioned step are created. + // + // 5. MARKING SETUP COMPLETE. Once the RSS has successfully initialized the + // rack, a marker file is created at "rss_completed_plan_path()". This + // indicates that the plan executed successfully, and no work remains. async fn inject_rack_setup_requests( &self, config: &Config, @@ -499,6 +538,14 @@ impl ServiceInner { self.log, "RSS configuration looks like it has already been applied", ); + + let sled_plan = SledPlan::load(&self.log) + .await? + .expect("Sled plan should exist if completed marker exists"); + let service_plan = ServicePlan::load(&self.log) + .await? + .expect("Service plan should exist if completed marker exists"); + self.handoff_to_nexus(&config, &sled_plan, &service_plan).await?; return Ok(()); } else { info!(self.log, "RSS configuration has not been fully applied yet",); @@ -507,11 +554,13 @@ impl ServiceInner { // Wait for either: // - All the peers to re-load an old plan (if one exists) // - Enough peers to create a new plan (if one does not exist) - let maybe_plan = self.load_plan().await?; - let expectation = if let Some(plan) = &maybe_plan { - PeerExpectation::LoadOldPlan(plan.keys().map(|a| *a.ip()).collect()) + let maybe_sled_plan = SledPlan::load(&self.log).await?; + let expectation = if let Some(plan) = &maybe_sled_plan { + PeerExpectation::LoadOldPlan( + plan.sleds.keys().map(|a| *a.ip()).collect(), + ) } else { - PeerExpectation::CreateNewPlan(config.requests.len()) + PeerExpectation::CreateNewPlan(MINIMUM_SLED_COUNT) }; let addrs = self .wait_for_peers(expectation, local_bootstrap_agent.our_address()) @@ -522,14 +571,14 @@ impl ServiceInner { // // NOTE: This is a "point-of-no-return" -- before sending any requests // to neighboring sleds, the plan must be recorded to durable storage. - // This way, if the RSS power-cycles, it can idempotently execute the - // same allocation plan. - let plan = if let Some(plan) = maybe_plan { + // This way, if the RSS power-cycles, it can idempotently provide the + // same subnets to the same sleds. + let plan = if let Some(plan) = maybe_sled_plan { info!(self.log, "Re-using existing allocation plan"); plan } else { info!(self.log, "Creating new allocation plan"); - self.create_plan(config, addrs).await? + SledPlan::create(&self.log, &config, addrs).await? }; // Generate our rack secret, unless we're in the single-sled case. @@ -549,7 +598,7 @@ impl ServiceInner { // addrs, which would remove the need for this assertion. assert_eq!( rack_secret_shares.len(), - plan.len(), + plan.sleds.len(), concat!( "Number of trust quorum members does not match ", "number of sleds in the plan" @@ -560,11 +609,12 @@ impl ServiceInner { // Forward the sled initialization requests to our sled-agent. local_bootstrap_agent .initialize_sleds( - plan.iter() - .map(move |(bootstrap_addr, allocation)| { + plan.sleds + .iter() + .map(move |(bootstrap_addr, initialization_request)| { ( *bootstrap_addr, - allocation.initialization_request.clone(), + initialization_request.clone(), maybe_rack_secret_shares .as_mut() .map(|shares| shares.next().unwrap()), @@ -575,22 +625,36 @@ impl ServiceInner { .await .map_err(SetupServiceError::SledInitialization)?; + // Now that sled agents have been initialized, we can create + // a service allocation plan. + let sled_addresses: Vec<_> = plan + .sleds + .values() + .map(|initialization_request| { + get_sled_address(initialization_request.subnet) + }) + .collect(); + let service_plan = + if let Some(plan) = ServicePlan::load(&self.log).await? { + plan + } else { + ServicePlan::create(&self.log, &config, &sled_addresses).await? + }; + // Set up internal DNS services. futures::future::join_all( - plan.iter() - .filter(|(_, allocation)| { + service_plan + .services + .iter() + .filter(|(_, services_request)| { // Only send requests to sleds that are supposed to be running // DNS services. - !allocation.services_request.dns_services.is_empty() + !services_request.dns_services.is_empty() }) - .map(|(_, allocation)| async move { - let sled_address = SocketAddr::V6(get_sled_address( - allocation.initialization_request.subnet, - )); - + .map(|(sled_address, services_request)| async move { self.initialize_services( - sled_address, - &allocation.services_request.dns_services, + *sled_address, + &services_request.dns_services, ) .await?; Ok(()) @@ -610,30 +674,16 @@ impl ServiceInner { .expect("DNS servers should only be set once"); // Issue the dataset initialization requests to all sleds. - futures::future::join_all(plan.values().map(|allocation| async move { - let sled_address = SocketAddr::V6(get_sled_address( - allocation.initialization_request.subnet, - )); - self.initialize_datasets( - sled_address, - &allocation.services_request.datasets, - ) - .await?; - - let mut records = HashMap::new(); - for dataset in &allocation.services_request.datasets { - records - .entry(dataset.srv()) - .or_insert_with(Vec::new) - .push((dataset.aaaa(), dataset.address())); - } - self.dns_servers - .get() - .expect("DNS servers must be initialized first") - .insert_dns_records(&records) + futures::future::join_all(service_plan.services.iter().map( + |(sled_address, services_request)| async move { + self.initialize_datasets( + *sled_address, + &services_request.datasets, + ) .await?; - Ok(()) - })) + Ok(()) + }, + )) .await .into_iter() .collect::>()?; @@ -642,42 +692,32 @@ impl ServiceInner { // Issue service initialization requests. // - // Note that this must happen *after* the dataset initialization, + // NOTE: This must happen *after* the dataset initialization, // to ensure that CockroachDB has been initialized before Nexus // starts. - futures::future::join_all(plan.values().map(|allocation| async move { - let sled_address = SocketAddr::V6(get_sled_address( - allocation.initialization_request.subnet, - )); - - let all_zones = allocation - .services_request - .service_zones - .iter() - .chain(allocation.services_request.dns_services.iter()) - .map(|s| s.clone()) - .collect::>(); - - self.initialize_services(sled_address, &all_zones).await?; - - let mut records = HashMap::new(); - for zone in &all_zones { - for service in &zone.services { - if let Some(addr) = zone.address(service) { - records - .entry(zone.srv(service)) - .or_insert_with(Vec::new) - .push((zone.aaaa(), addr)) - } - } - } - self.dns_servers - .get() - .expect("DNS servers must be initialized first") - .insert_dns_records(&records) - .await?; - Ok(()) - })) + // + // If Nexus was more resilient to concurrent initialization + // of CRDB, this requirement could be relaxed. + futures::future::join_all(service_plan.services.iter().map( + |(sled_address, services_request)| async move { + // With the current implementation of "initialize_services", + // we must provide the set of *all* services that should be + // executing on a sled. + // + // This means re-requesting the DNS service, even if it is + // already running - this is fine, however, as the receiving + // sled agent doesn't modify the already-running service. + let all_services = services_request + .services + .iter() + .chain(services_request.dns_services.iter()) + .map(|s| s.clone()) + .collect::>(); + + self.initialize_services(*sled_address, &all_services).await?; + Ok(()) + }, + )) .await .into_iter() .collect::, SetupServiceError>>()?; @@ -696,6 +736,10 @@ impl ServiceInner { }, )?; + // At this point, even if we reboot, we must not try to manage sleds, + // services, or DNS records. + self.handoff_to_nexus(&config, &plan, &service_plan).await?; + // TODO Questions to consider: // - What if a sled comes online *right after* this setup? How does // it get a /64? @@ -703,112 +747,3 @@ impl ServiceInner { Ok(()) } } - -fn generate_rack_secret<'a>( - rack_secret_threshold: usize, - member_device_id_certs: &'a [Ed25519Certificate], - log: &Logger, -) -> Result< - Option + 'a>, - SetupServiceError, -> { - // We do not generate a rack secret if we only have a single sled or if our - // config specifies that the threshold for unlock is only a single sled. - let total_shares = member_device_id_certs.len(); - if total_shares <= 1 { - info!(log, "Skipping rack secret creation (only one sled present)"); - return Ok(None); - } - - if rack_secret_threshold <= 1 { - warn!( - log, - concat!( - "Skipping rack secret creation due to config", - " (despite discovery of {} bootstrap agents)" - ), - total_shares, - ); - return Ok(None); - } - - let secret = RackSecret::new(); - let (shares, verifier) = secret - .split(rack_secret_threshold, total_shares) - .map_err(SetupServiceError::SplitRackSecret)?; - - Ok(Some(shares.into_iter().map(move |share| ShareDistribution { - threshold: rack_secret_threshold, - verifier: verifier.clone(), - share, - member_device_id_certs: member_device_id_certs.to_vec(), - }))) -} - -#[cfg(test)] -mod tests { - use super::*; - use omicron_test_utils::dev::test_setup_log; - use sprockets_common::certificates::Ed25519Signature; - use sprockets_common::certificates::KeyType; - - fn dummy_certs(n: usize) -> Vec { - vec![ - Ed25519Certificate { - subject_key_type: KeyType::DeviceId, - subject_public_key: sprockets_host::Ed25519PublicKey([0; 32]), - signer_key_type: KeyType::Manufacturing, - signature: Ed25519Signature([0; 64]), - }; - n - ] - } - - #[test] - fn test_generate_rack_secret() { - let logctx = test_setup_log("test_generate_rack_secret"); - - // No secret generated if we have <= 1 sled - assert!(generate_rack_secret(10, &dummy_certs(1), &logctx.log) - .unwrap() - .is_none()); - - // No secret generated if threshold <= 1 - assert!(generate_rack_secret(1, &dummy_certs(10), &logctx.log) - .unwrap() - .is_none()); - - // Secret generation fails if threshold > total sleds - assert!(matches!( - generate_rack_secret(10, &dummy_certs(5), &logctx.log), - Err(SetupServiceError::SplitRackSecret(_)) - )); - - // Secret generation succeeds if threshold <= total shares and both are - // > 1, and the returned iterator satifies: - // - // * total length == total shares - // * each share is distinct - for total_shares in 2..=32 { - for threshold in 2..=total_shares { - let certs = dummy_certs(total_shares); - let shares = - generate_rack_secret(threshold, &certs, &logctx.log) - .unwrap() - .unwrap(); - - assert_eq!(shares.len(), total_shares); - - // `Share` doesn't implement `Hash`, but it's a newtype around - // `Vec` (which does). Unwrap the newtype to check that all - // shares are distinct. - let shares_set = shares - .map(|share_dist| share_dist.share.0) - .collect::>(); - assert_eq!(shares_set.len(), total_shares); - } - } - - logctx.cleanup_successful(); - } -} diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 60ef5c68826..a3b8d1d65b0 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -19,7 +19,7 @@ use crate::params::{ DatasetKind, DendriteAsic, DiskStateRequested, InstanceHardware, InstanceMigrateParams, InstanceRuntimeStateRequested, InstanceSerialConsoleData, ServiceEnsureBody, ServiceType, - ServiceZoneRequest, VpcFirewallRule, ZoneType, + ServiceZoneRequest, VpcFirewallRule, ZoneType, Zpool, }; use crate::services::{self, ServiceManager}; use crate::storage_manager::StorageManager; @@ -206,8 +206,6 @@ impl SledAgent { lazy_nexus_client: LazyNexusClient, request: SledAgentRequest, ) -> Result { - let id = config.id; - // Pass the "parent_log" to all subcomponents that want to set their own // "component" value. let parent_log = log.clone(); @@ -215,7 +213,7 @@ impl SledAgent { // Use "log" for ourself. let log = log.new(o!( "component" => "SledAgent", - "sled_id" => id.to_string(), + "sled_id" => request.id.to_string(), )); info!(&log, "created sled agent"); @@ -310,7 +308,7 @@ impl SledAgent { let storage = StorageManager::new( &parent_log, - id, + request.id, lazy_nexus_client.clone(), etherstub.clone(), *sled_address.ip(), @@ -356,7 +354,7 @@ impl SledAgent { let sled_agent = SledAgent { inner: Arc::new(SledAgentInner { - id, + id: request.id, config: config.clone(), subnet: request.subnet, storage, @@ -554,6 +552,12 @@ impl SledAgent { Ok(()) } + /// Gets the sled's current list of all zpools. + pub async fn zpools_get(&self) -> Result, Error> { + let zpools = self.inner.storage.get_zpools().await?; + Ok(zpools) + } + /// Ensures that a filesystem type exists within the zpool. pub async fn filesystem_ensure( &self, diff --git a/sled-agent/src/sp/mod.rs b/sled-agent/src/sp/mod.rs index d5e89673dd3..b664e1ac125 100644 --- a/sled-agent/src/sp/mod.rs +++ b/sled-agent/src/sp/mod.rs @@ -4,7 +4,6 @@ //! Interface to a (simulated or real) SP / RoT. -use crate::config::Config as SledConfig; use crate::config::ConfigError; use crate::illumos; use crate::illumos::dladm::CreateVnicError; @@ -86,11 +85,10 @@ impl SpHandle { /// A return value of `Ok(None)` means no SP is available. pub async fn detect( sp_config: Option<&GimletConfig>, - sled_config: &SledConfig, log: &Logger, ) -> Result, SpError> { - let inner = if let Some(config) = sp_config { - let sim_sp = SimulatedSp::start(config, sled_config, log).await?; + let inner = if let Some(config) = sp_config.as_ref() { + let sim_sp = SimulatedSp::start(config, log).await?; Some(Inner::SimulatedSp(sim_sp)) } else { None diff --git a/sled-agent/src/sp/simulated.rs b/sled-agent/src/sp/simulated.rs index a37deabc816..3b74713a0d8 100644 --- a/sled-agent/src/sp/simulated.rs +++ b/sled-agent/src/sp/simulated.rs @@ -5,7 +5,6 @@ //! Implementation of a simulated SP / RoT. use super::SpError; -use crate::config::Config as SledConfig; use crate::illumos::dladm::Dladm; use crate::zone::Zones; use slog::Logger; @@ -36,7 +35,6 @@ pub(super) struct SimulatedSp { impl SimulatedSp { pub(super) async fn start( sp_config: &GimletConfig, - sled_config: &SledConfig, log: &Logger, ) -> Result { // Is our simulated SP going to bind to addresses (acting like @@ -80,7 +78,6 @@ impl SimulatedSp { info!(log, "starting simulated gimlet SP"); let sp_log = log.new(o!( "component" => "sp-sim", - "server" => sled_config.id.clone().to_string(), )); let sp = Arc::new( sp_sim::Gimlet::spawn(&sp_config, sp_log) @@ -92,7 +89,6 @@ impl SimulatedSp { info!(log, "starting simulated gimlet RoT"); let rot_log = log.new(o!( "component" => "rot-sim", - "server" => sled_config.id.clone().to_string(), )); let transport = SimRotTransport { sp: Arc::clone(&sp), responses: VecDeque::new() }; diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index 37401eaf73a..2ad90e251d0 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -956,6 +956,14 @@ impl StorageManager { Ok(()) } + pub async fn get_zpools(&self) -> Result, Error> { + let pools = self.pools.lock().await; + Ok(pools + .keys() + .map(|zpool| crate::params::Zpool { id: zpool.id() }) + .collect()) + } + pub async fn upsert_filesystem( &self, zpool_id: Uuid, diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index c641cdf4c73..d2207758135 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -27,60 +27,5 @@ rack_secret_threshold = 1 # how-to-run.adoc for details on how to determine the value for your network. mac = "00:0d:b9:54:fe:e4" -[[request]] - -# TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus -# should allocate crucible datasets. -[[request.dataset]] -id = "09a9a25f-2602-4e2f-9630-31af9c492c3e" -zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1122:3344:0101::7]:32345" -dataset_kind.type = "crucible" - -[[request.dataset]] -id = "2713b37a-3043-4ed5-aaff-f38200e45cfb" -zpool_id = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03" -address = "[fd00:1122:3344:0101::8]:32345" -dataset_kind.type = "crucible" - -[[request.dataset]] -id = "ffd16cad-e5d5-495e-9c59-4312a3857d91" -zpool_id = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03" -address = "[fd00:1122:3344:0101::9]:32345" -dataset_kind.type = "crucible" - -[[request.dataset]] -id = "4d08fc19-3d5f-4f6b-9c48-925f8eac7255" -zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1122:3344:0101::3]:32221" -dataset_kind.type = "cockroach_db" -dataset_kind.all_addresses = [ "[fd00:1122:3344:0101::3]:32221" ] - -# TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus -# should allocate clickhouse datasets. -[[request.dataset]] -id = "a3505b41-a592-420b-84f2-3d76bf0e0a81" -zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1122:3344:0101::6]:8123" -dataset_kind.type = "clickhouse" - -[[request.service_zone]] -id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" -zone_type = "nexus" -addresses = [ "fd00:1122:3344:0101::4" ] -gz_addresses = [] -[[request.service_zone.services]] -type = "nexus" -internal_ip = "fd00:1122:3344:0101::4" # NOTE: In the lab, use "172.20.15.226" -external_ip = "192.168.1.20" - -# TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus -# should allocate Oximeter services. -[[request.service_zone]] -id = "1da65e5b-210c-4859-a7d7-200c1e659972" -zone_type = "oximeter" -addresses = [ "fd00:1122:3344:0101::5" ] -gz_addresses = [] -[[request.service_zone.services]] -type = "oximeter" +nexus_external_address = "192.168.1.20" diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index cf082e7dfcc..c63ed43c37a 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -1,7 +1,5 @@ # Sled Agent Configuration -id = "fb0f7546-4d46-40ca-9d56-cbb810684ca7" - # Identifies if the sled should be unconditionally treated as a scrimlet. # # If this is set to "true", the sled agent treats itself as a scrimlet. From bf9fe662d455edb60de0e6afdbba9d5c7b417392 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 16 Nov 2022 16:12:36 -0500 Subject: [PATCH 02/20] Remove dataset notify --- nexus/src/internal_api/http_entrypoints.rs | 37 +--------- nexus/tests/integration_tests/datasets.rs | 86 ---------------------- nexus/tests/integration_tests/mod.rs | 1 - nexus/types/src/internal_api/params.rs | 19 ----- openapi/nexus-internal.json | 80 -------------------- sled-agent/src/mocks/mod.rs | 11 +-- sled-agent/src/sim/storage.rs | 24 +----- sled-agent/src/storage_manager.rs | 65 +--------------- 8 files changed, 11 insertions(+), 312 deletions(-) delete mode 100644 nexus/tests/integration_tests/datasets.rs diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs index 307a3f02941..01d160b7e6c 100644 --- a/nexus/src/internal_api/http_entrypoints.rs +++ b/nexus/src/internal_api/http_entrypoints.rs @@ -7,9 +7,8 @@ use crate::context::OpContext; use crate::ServerContext; use super::params::{ - DatasetPutRequest, DatasetPutResponse, OximeterInfo, - RackInitializationRequest, SledAgentStartupInfo, ZpoolPutRequest, - ZpoolPutResponse, + OximeterInfo, RackInitializationRequest, SledAgentStartupInfo, + ZpoolPutRequest, ZpoolPutResponse, }; use dropshot::endpoint; use dropshot::ApiDescription; @@ -40,7 +39,6 @@ pub fn internal_api() -> NexusApiDescription { api.register(sled_agent_put)?; api.register(rack_initialization_complete)?; api.register(zpool_put)?; - api.register(dataset_put)?; api.register(cpapi_instances_put)?; api.register(cpapi_disks_put)?; api.register(cpapi_volume_remove_read_only_parent)?; @@ -140,37 +138,6 @@ async fn zpool_put( Ok(HttpResponseOk(ZpoolPutResponse {})) } -#[derive(Deserialize, JsonSchema)] -struct DatasetPathParam { - zpool_id: Uuid, - dataset_id: Uuid, -} - -/// Report that a dataset within a pool has come online. -#[endpoint { - method = PUT, - path = "/zpools/{zpool_id}/dataset/{dataset_id}", - }] -async fn dataset_put( - rqctx: Arc>>, - path_params: Path, - info: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.nexus; - let path = path_params.into_inner(); - let info = info.into_inner(); - nexus - .upsert_dataset( - path.dataset_id, - path.zpool_id, - info.address, - info.kind.into(), - ) - .await?; - Ok(HttpResponseOk(DatasetPutResponse { reservation: None, quota: None })) -} - /// Path parameters for Instance requests (internal API) #[derive(Deserialize, JsonSchema)] struct InstancePathParam { diff --git a/nexus/tests/integration_tests/datasets.rs b/nexus/tests/integration_tests/datasets.rs deleted file mode 100644 index e3f454d436c..00000000000 --- a/nexus/tests/integration_tests/datasets.rs +++ /dev/null @@ -1,86 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use http::method::Method; -use http::StatusCode; -use omicron_common::api::external::ByteCount; -use omicron_nexus::internal_api::params::{ - DatasetKind, DatasetPutRequest, ZpoolPutRequest, -}; -use std::net::{Ipv6Addr, SocketAddrV6}; -use uuid::Uuid; - -use nexus_test_utils::SLED_AGENT_UUID; -use nexus_test_utils_macros::nexus_test; - -type ControlPlaneTestContext = - nexus_test_utils::ControlPlaneTestContext; - -// Tests the "normal" case of dataset_put: inserting a dataset within a known -// zpool. -// -// This will typically be invoked by the Sled Agent, after performing inventory. -#[nexus_test] -async fn test_dataset_put_success(cptestctx: &ControlPlaneTestContext) { - let client = &cptestctx.internal_client; - - let zpool_id = Uuid::new_v4(); - let zpool_put_url = - format!("/sled-agents/{}/zpools/{}", SLED_AGENT_UUID, zpool_id); - let request = ZpoolPutRequest { size: ByteCount::from_gibibytes_u32(1) }; - client - .make_request( - Method::PUT, - &zpool_put_url, - Some(request), - StatusCode::OK, - ) - .await - .unwrap(); - - let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0); - let kind = DatasetKind::Crucible; - let request = DatasetPutRequest { address, kind }; - let dataset_id = Uuid::new_v4(); - let dataset_put_url = - format!("/zpools/{}/dataset/{}", zpool_id, dataset_id); - - client - .make_request( - Method::PUT, - &dataset_put_url, - Some(request), - StatusCode::OK, - ) - .await - .unwrap(); -} - -// Tests a failure case of dataset_put: Inserting a dataset into a zpool that -// does not exist. -#[nexus_test] -async fn test_dataset_put_bad_zpool_returns_not_found( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.internal_client; - - // A zpool with the "nil" UUID should not exist. - let zpool_id = Uuid::nil(); - let dataset_id = Uuid::new_v4(); - let dataset_put_url = - format!("/zpools/{}/dataset/{}", zpool_id, dataset_id); - - let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0); - let kind = DatasetKind::Crucible; - let request = DatasetPutRequest { address, kind }; - - client - .make_request_error_body( - Method::PUT, - &dataset_put_url, - request, - StatusCode::NOT_FOUND, - ) - .await; -} diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs index 293511b91e1..454114d53e0 100644 --- a/nexus/tests/integration_tests/mod.rs +++ b/nexus/tests/integration_tests/mod.rs @@ -8,7 +8,6 @@ mod authz; mod basic; mod commands; mod console_api; -mod datasets; mod device_auth; mod disks; mod images; diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index 9f738efd244..bfd45fd3da2 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -100,25 +100,6 @@ pub struct DatasetPutRequest { pub kind: DatasetKind, } -/// Describes which ZFS properties should be set for a particular allocated -/// dataset. -// TODO: This could be useful for indicating quotas, or -// for Nexus instructing the Sled Agent "what to format, and where". -// -// For now, the Sled Agent is a bit more proactive about allocation -// decisions - see the "storage manager" section of the Sled Agent for -// more details. Nexus, in response, merely advises minimums/maximums -// for dataset sizes. -#[derive(Serialize, Deserialize, JsonSchema)] -pub struct DatasetPutResponse { - /// A minimum reservation size for a filesystem. - /// Refer to ZFS native properties for more detail. - pub reservation: Option, - /// A maximum quota on filesystem usage. - /// Refer to ZFS native properties for more detail. - pub quota: Option, -} - /// Describes the purpose of the service. #[derive( Debug, Serialize, Deserialize, JsonSchema, Clone, Copy, PartialEq, Eq, diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index b92bfab9fce..a1376266640 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -398,62 +398,6 @@ } } } - }, - "/zpools/{zpool_id}/dataset/{dataset_id}": { - "put": { - "summary": "Report that a dataset within a pool has come online.", - "operationId": "dataset_put", - "parameters": [ - { - "in": "path", - "name": "dataset_id", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - }, - "style": "simple" - }, - { - "in": "path", - "name": "zpool_id", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - }, - "style": "simple" - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DatasetPutRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DatasetPutResponse" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } } }, "components": { @@ -752,30 +696,6 @@ "kind" ] }, - "DatasetPutResponse": { - "description": "Describes which ZFS properties should be set for a particular allocated dataset.", - "type": "object", - "properties": { - "quota": { - "nullable": true, - "description": "A maximum quota on filesystem usage. Refer to ZFS native properties for more detail.", - "allOf": [ - { - "$ref": "#/components/schemas/ByteCount" - } - ] - }, - "reservation": { - "nullable": true, - "description": "A minimum reservation size for a filesystem. Refer to ZFS native properties for more detail.", - "allOf": [ - { - "$ref": "#/components/schemas/ByteCount" - } - ] - } - } - }, "Datum": { "description": "A `Datum` is a single sampled data point from a metric.", "oneOf": [ diff --git a/sled-agent/src/mocks/mod.rs b/sled-agent/src/mocks/mod.rs index 46796db5c58..1c45848805b 100644 --- a/sled-agent/src/mocks/mod.rs +++ b/sled-agent/src/mocks/mod.rs @@ -6,9 +6,8 @@ use mockall::mock; use nexus_client::types::{ - DatasetPutRequest, DatasetPutResponse, DiskRuntimeState, - InstanceRuntimeState, SledAgentStartupInfo, UpdateArtifactKind, - ZpoolPutRequest, ZpoolPutResponse, + DiskRuntimeState, InstanceRuntimeState, SledAgentStartupInfo, + UpdateArtifactKind, ZpoolPutRequest, ZpoolPutResponse, }; use slog::Logger; use uuid::Uuid; @@ -50,11 +49,5 @@ mock! { zpool_id: &Uuid, info: &ZpoolPutRequest, ) -> Result; - pub async fn dataset_put( - &self, - zpool_id: &Uuid, - dataset_id: &Uuid, - info: &DatasetPutRequest, - ) -> Result; } } diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 58c880eb20d..e3acac19df1 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -15,9 +15,7 @@ use crucible_agent_client::types::{ CreateRegion, Region, RegionId, RunningSnapshot, Snapshot, State, }; use futures::lock::Mutex; -use nexus_client::types::{ - ByteCount, DatasetKind, DatasetPutRequest, ZpoolPutRequest, -}; +use nexus_client::types::{ByteCount, ZpoolPutRequest}; use slog::Logger; use std::collections::HashMap; use std::net::{IpAddr, SocketAddr}; @@ -328,7 +326,7 @@ impl CrucibleData { /// /// Contains both the data and the HTTP server. pub struct CrucibleServer { - server: dropshot::HttpServer>, + _server: dropshot::HttpServer>, data: Arc, } @@ -354,11 +352,7 @@ impl CrucibleServer { .start(); info!(&log, "Created Simulated Crucible Server"; "address" => server.local_addr()); - CrucibleServer { server, data } - } - - fn address(&self) -> SocketAddr { - self.server.local_addr() + CrucibleServer { _server: server, data } } pub fn data(&self) -> Arc { @@ -464,7 +458,7 @@ impl Storage { /// Adds a Dataset to the sled's simulated storage and notifies Nexus. pub async fn insert_dataset(&mut self, zpool_id: Uuid, dataset_id: Uuid) { // Update our local data - let dataset = self + let _ = self .zpools .get_mut(&zpool_id) .expect("Zpool does not exist") @@ -476,16 +470,6 @@ impl Storage { ); self.next_crucible_port += 100; - - // Notify Nexus - let request = DatasetPutRequest { - address: dataset.address().to_string(), - kind: DatasetKind::Crucible, - }; - self.nexus_client - .dataset_put(&zpool_id, &dataset_id, &request) - .await - .expect("Failed to notify Nexus about new Dataset"); } pub async fn get_dataset( diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index 2ad90e251d0..03526fc7a59 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -15,7 +15,7 @@ use crate::params::DatasetKind; use futures::stream::FuturesOrdered; use futures::FutureExt; use futures::StreamExt; -use nexus_client::types::{DatasetPutRequest, ZpoolPutRequest}; +use nexus_client::types::ZpoolPutRequest; use omicron_common::api::external::{ByteCount, ByteCountRangeError}; use omicron_common::backoff; use schemars::JsonSchema; @@ -667,59 +667,12 @@ impl StorageWorker { ); } - // Adds a "notification to nexus" to `nexus_notifications`, - // informing it about the addition of `datasets` to `pool_id`. - fn add_datasets_notify( - &self, - nexus_notifications: &mut FuturesOrdered>>, - datasets: Vec<(Uuid, SocketAddrV6, DatasetKind)>, - pool_id: Uuid, - ) { - let lazy_nexus_client = self.lazy_nexus_client.clone(); - let notify_nexus = move || { - let lazy_nexus_client = lazy_nexus_client.clone(); - let datasets = datasets.clone(); - async move { - let nexus = lazy_nexus_client.get().await.map_err(|e| { - backoff::BackoffError::transient(e.to_string()) - })?; - - for (id, address, kind) in datasets { - let request = DatasetPutRequest { - address: address.to_string(), - kind: kind.into(), - }; - nexus.dataset_put(&pool_id, &id, &request).await.map_err( - |e| backoff::BackoffError::transient(e.to_string()), - )?; - } - Ok(()) - } - }; - let log = self.log.clone(); - let log_post_failure = move |_, delay| { - warn!( - log, - "failed to notify nexus about datasets, will retry in {:?}", delay; - ); - }; - nexus_notifications.push_back( - backoff::retry_notify( - backoff::internal_service_policy(), - notify_nexus, - log_post_failure, - ) - .boxed(), - ); - } - // TODO: a lot of these functions act on the `FuturesOrdered` - should // that just be a part of the "worker" struct? // Attempts to add a dataset within a zpool, according to `request`. async fn add_dataset( &self, - nexus_notifications: &mut FuturesOrdered>>, request: &NewFilesystemRequest, ) -> Result<(), Error> { info!(self.log, "add_dataset: {:?}", request); @@ -766,12 +719,6 @@ impl StorageWorker { err, })?; - self.add_datasets_notify( - nexus_notifications, - vec![(id, dataset_info.address, dataset_info.kind)], - pool.id(), - ); - Ok(()) } @@ -865,21 +812,15 @@ impl StorageWorker { } } - // Notify Nexus of the zpool and all datasets within. + // Notify Nexus of the zpool. self.add_zpool_notify( &mut nexus_notifications, pool.id(), size, ); - - self.add_datasets_notify( - &mut nexus_notifications, - datasets, - pool.id(), - ); }, Some(request) = self.new_filesystems_rx.recv() => { - let result = self.add_dataset(&mut nexus_notifications, &request).await; + let result = self.add_dataset(&request).await; let _ = request.responder.send(result); } } From c12f2a886c0a5f89b0b776766752beb7dd0ed1d1 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 16 Nov 2022 16:18:55 -0500 Subject: [PATCH 03/20] Update deploy script to leave nexus address alone --- .github/buildomat/jobs/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 07b48ba9a4b..0a199e26372 100644 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -147,7 +147,7 @@ pfexec ipadm create-addr -T static -a 192.168.1.199/24 igb0/sidehatch # address for upstream connectivity. # tar xf out/omicron-sled-agent.tar pkg/config-rss.toml -sed -e '/\[gateway\]/,/\[request\]/ s/^.*address =.*$/address = "192.168.1.199"/' \ +sed -e 's/^# address =.*$/address = "192.168.1.199"/' \ -e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \ -i pkg/config-rss.toml tar rf out/omicron-sled-agent.tar pkg/config-rss.toml From 37a4457d623a24e86151925a0b932b3963db209e Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 16 Nov 2022 16:36:45 -0500 Subject: [PATCH 04/20] Fix discovery of nexus IP in e2e tests --- end-to-end-tests/src/helpers/ctx.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/end-to-end-tests/src/helpers/ctx.rs b/end-to-end-tests/src/helpers/ctx.rs index 2708f7d5e2b..b3293aeb4db 100644 --- a/end-to-end-tests/src/helpers/ctx.rs +++ b/end-to-end-tests/src/helpers/ctx.rs @@ -100,16 +100,7 @@ pub fn nexus_addr() -> SocketAddr { .join("../smf/sled-agent/config-rss.toml"); if rss_config_path.exists() { if let Ok(config) = SetupServiceConfig::from_file(rss_config_path) { - for request in config.requests { - for zone in request.service_zones { - for service in zone.services { - if let ServiceType::Nexus { external_ip, .. } = service - { - return (external_ip, 80).into(); - } - } - } - } + return (config.nexus_external_address, 80).into(); } } From 809ac6463a47bc1d1823f4586fd6775d31ac9ee7 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 16 Nov 2022 16:53:49 -0500 Subject: [PATCH 05/20] Nexus actually awaiting RSS handoff --- nexus/src/lib.rs | 76 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index ef29550f7da..616a3d48a51 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -66,23 +66,24 @@ pub fn run_openapi_internal() -> Result<(), String> { .map_err(|e| e.to_string()) } -/// Packages up a [`Nexus`], running both external and internal HTTP API servers -/// wired up to Nexus -pub struct Server { +/// A partially-initialized Nexus server, which exposes an internal interface, +/// but is not ready to receive external requests. +pub struct InternalServer<'a> { /// shared state used by API request handlers pub apictx: Arc, - /// dropshot servers for external API - pub http_servers_external: Vec>>, /// dropshot server for internal API pub http_server_internal: dropshot::HttpServer>, + + config: &'a Config, + log: Logger, } -impl Server { +impl<'a> InternalServer<'a> { /// Start a nexus server. pub async fn start( - config: &Config, + config: &'a Config, log: &Logger, - ) -> Result { + ) -> Result, String> { let log = log.new(o!("name" => config.deployment.id.to_string())); info!(log, "setting up nexus server"); @@ -102,6 +103,32 @@ impl Server { .map_err(|error| format!("initializing internal server: {}", error))?; let http_server_internal = server_starter_internal.start(); + Ok(Self { apictx, http_server_internal, config, log }) + } +} + +/// Packages up a [`Nexus`], running both external and internal HTTP API servers +/// wired up to Nexus +pub struct Server { + /// shared state used by API request handlers + pub apictx: Arc, + /// dropshot servers for external API + pub http_servers_external: Vec>>, + /// dropshot server for internal API + pub http_server_internal: dropshot::HttpServer>, +} + +impl Server { + pub async fn start(internal: InternalServer<'_>) -> Result { + let apictx = internal.apictx; + let http_server_internal = internal.http_server_internal; + let log = internal.log; + let config = internal.config; + + // Wait until RSS handoff completes. + let opctx = apictx.nexus.opctx_for_service_balancer(); + apictx.nexus.await_rack_initialization(&opctx).await; + // Launch the external server(s). let http_servers_external = config .deployment @@ -165,9 +192,33 @@ impl Server { #[async_trait::async_trait] impl nexus_test_interface::NexusServer for Server { async fn start_and_populate(config: &Config, log: &Logger) -> Self { - let server = Server::start(config, log).await.unwrap(); - server.apictx.nexus.wait_for_populate().await.unwrap(); - server + let internal_server = + InternalServer::start(config, &log).await.unwrap(); + internal_server.apictx.nexus.wait_for_populate().await.unwrap(); + + // Perform the "handoff from RSS". + // + // However, RSS isn't running, so we'll do the handoff ourselves. + let opctx = internal_server.apictx.nexus.opctx_for_service_balancer(); + internal_server + .apictx + .nexus + .rack_initialize( + &opctx, + config.deployment.rack_id, + // NOTE: In the context of this test utility, we arguably do have an + // instance of CRDB and Nexus running. However, as this info isn't + // necessary for most tests, we pass no information here. + internal_api::params::RackInitializationRequest { + services: vec![], + datasets: vec![], + }, + ) + .await + .expect("Could not initialize rack"); + + // Start the Nexus external API. + Server::start(internal_server).await.unwrap() } fn get_http_servers_external(&self) -> Vec { @@ -206,7 +257,8 @@ pub async fn run_server(config: &Config) -> Result<(), String> { } else { debug!(log, "registered DTrace probes"); } - let server = Server::start(config, &log).await?; + let internal_server = InternalServer::start(config, &log).await?; + let server = Server::start(internal_server).await?; server.register_as_producer().await; server.wait_for_finish().await } From 4ffa1bdd4fd58b967accf20f4874e783398d0342 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 16 Nov 2022 23:44:03 -0500 Subject: [PATCH 06/20] test fixes: service balancer role, dataset upserting from tests --- Cargo.lock | 1 + nexus/src/db/fixed_data/role_assignment.rs | 14 ++++++++++ nexus/src/lib.rs | 21 ++++++++++++++- nexus/test-interface/Cargo.toml | 1 + nexus/test-interface/src/lib.rs | 31 +++++++++++++++++++++- nexus/test-utils/src/resource_helpers.rs | 31 +++++++++++++++++++--- nexus/tests/integration_tests/disks.rs | 12 ++++----- sled-agent/src/sim/sled_agent.rs | 4 +-- sled-agent/src/sim/storage.rs | 20 ++++++++++---- 9 files changed, 116 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0de2f5c6ea..81f16533716 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3044,6 +3044,7 @@ dependencies = [ "dropshot", "omicron-common", "slog", + "uuid", ] [[package]] diff --git a/nexus/src/db/fixed_data/role_assignment.rs b/nexus/src/db/fixed_data/role_assignment.rs index 43635518c9a..7d7ddffab64 100644 --- a/nexus/src/db/fixed_data/role_assignment.rs +++ b/nexus/src/db/fixed_data/role_assignment.rs @@ -25,6 +25,20 @@ lazy_static! { role_builtin::FLEET_ADMIN.role_name, ), + // The "USER_SERVICE_BALANCER" user gets the "admin" role on the + // Fleet. + // + // This is necessary as services exist as resources implied by + // "FLEET" - if they ever become more fine-grained, this scope + // could also become smaller. + RoleAssignment::new( + IdentityType::UserBuiltin, + user_builtin::USER_SERVICE_BALANCER.id, + role_builtin::FLEET_ADMIN.resource_type, + *FLEET_ID, + role_builtin::FLEET_ADMIN.role_name, + ), + // The "internal-read" user gets the "viewer" role on the sole // Fleet. This will grant them the ability to read various control // plane data (like the list of sleds), which is in turn used to diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index 616a3d48a51..9d04b0ad5ea 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -33,8 +33,9 @@ pub use crucible_agent_client; use external_api::http_entrypoints::external_api; use internal_api::http_entrypoints::internal_api; use slog::Logger; -use std::net::SocketAddr; +use std::net::{SocketAddr, SocketAddrV6}; use std::sync::Arc; +use uuid::Uuid; #[macro_use] extern crate slog; @@ -232,6 +233,24 @@ impl nexus_test_interface::NexusServer for Server { self.http_server_internal.local_addr() } + async fn upsert_crucible_dataset( + &self, + id: Uuid, + zpool_id: Uuid, + address: SocketAddrV6, + ) { + self.apictx + .nexus + .upsert_dataset( + id, + zpool_id, + address, + crate::db::model::DatasetKind::Crucible, + ) + .await + .unwrap(); + } + async fn close(mut self) { for server in self.http_servers_external { server.close().await.unwrap(); diff --git a/nexus/test-interface/Cargo.toml b/nexus/test-interface/Cargo.toml index 1df5091c5ec..da66c43ef20 100644 --- a/nexus/test-interface/Cargo.toml +++ b/nexus/test-interface/Cargo.toml @@ -9,3 +9,4 @@ async-trait = "0.1.56" dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main" , features = [ "usdt-probes" ] } omicron-common = { path = "../../common" } slog = { version = "2.7" } +uuid = { version = "1.2.1", features = [ "serde", "v4" ] } diff --git a/nexus/test-interface/src/lib.rs b/nexus/test-interface/src/lib.rs index c130dde0802..b4251fae364 100644 --- a/nexus/test-interface/src/lib.rs +++ b/nexus/test-interface/src/lib.rs @@ -34,7 +34,8 @@ use async_trait::async_trait; use omicron_common::nexus_config::Config; use slog::Logger; -use std::net::SocketAddr; +use std::net::{SocketAddr, SocketAddrV6}; +use uuid::Uuid; #[async_trait] pub trait NexusServer { @@ -42,5 +43,33 @@ pub trait NexusServer { fn get_http_servers_external(&self) -> Vec; fn get_http_server_internal(&self) -> SocketAddr; + + // Previously, as a dataset was created (within the sled agent), + // we'd use an internal API from Nexus to record that the dataset + // now exists. In other words, Sled Agent was in control, by telling + // Nexus when it should record persistent information about datasets. + // + // However, as of https://github.com/oxidecomputer/omicron/pull/1954, + // control over dataset provisioning is shifting to Nexus. There is + // a short window where RSS controls dataset provisioning, but afterwards, + // Nexus should be calling the shots on "when to provision datasets". + // + // For test purposes, we have many situations where we want to carve up + // zpools and datasets precisely for disk-based tests. As a result, we + // *want* tests (namely, an entity outside of Nexus) to have this control. + // + // This test-based API provides one such mechanism of control. + // + // TODO: In the future, we *could* re-structure our tests to more rigorously + // use the "RackInitializationRequest" handoff, but this would require + // creating all our Zpools and Datasets before performing handoff to Nexus. + // However, doing so would let us remove this test-only API. + async fn upsert_crucible_dataset( + &self, + id: Uuid, + zpool_id: Uuid, + address: SocketAddrV6, + ); + async fn close(self); } diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index ecccbe03fc0..8e656c33062 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -11,6 +11,7 @@ use dropshot::test_util::ClientTestContext; use dropshot::HttpErrorResponseBody; use dropshot::Method; use http::StatusCode; +use nexus_test_interface::NexusServer; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Disk; use omicron_common::api::external::IdentityMetadataCreateParams; @@ -434,7 +435,9 @@ impl DiskTest { pub const DEFAULT_ZPOOL_SIZE_GIB: u32 = 10; // Creates fake physical storage, an organization, and a project. - pub async fn new(cptestctx: &ControlPlaneTestContext) -> Self { + pub async fn new( + cptestctx: &ControlPlaneTestContext, + ) -> Self { let sled_agent = cptestctx.sled_agent.sled_agent.clone(); let mut disk_test = Self { sled_agent, zpools: vec![] }; @@ -442,14 +445,18 @@ impl DiskTest { // Create three Zpools, each 10 GiB, each with one Crucible dataset. for _ in 0..3 { disk_test - .add_zpool_with_dataset(Self::DEFAULT_ZPOOL_SIZE_GIB) + .add_zpool_with_dataset(cptestctx, Self::DEFAULT_ZPOOL_SIZE_GIB) .await; } disk_test } - pub async fn add_zpool_with_dataset(&mut self, gibibytes: u32) { + pub async fn add_zpool_with_dataset( + &mut self, + cptestctx: &ControlPlaneTestContext, + gibibytes: u32, + ) { let zpool = TestZpool { id: Uuid::new_v4(), size: ByteCount::from_gibibytes_u32(gibibytes), @@ -459,7 +466,10 @@ impl DiskTest { self.sled_agent.create_zpool(zpool.id, zpool.size.to_bytes()).await; for dataset in &zpool.datasets { - self.sled_agent.create_crucible_dataset(zpool.id, dataset.id).await; + let address = self + .sled_agent + .create_crucible_dataset(zpool.id, dataset.id) + .await; // By default, regions are created immediately. let crucible = self @@ -469,6 +479,19 @@ impl DiskTest { crucible .set_create_callback(Box::new(|_| RegionState::Created)) .await; + + let address = match address { + std::net::SocketAddr::V6(addr) => addr, + _ => panic!("Unsupported address type: {address} "), + }; + + cptestctx + .server + .upsert_crucible_dataset(dataset.id, zpool.id, address) + .await; + + // TODO: Upsert dataset?? + // cptestctx.name } self.zpools.push(zpool); diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index 22a96cde435..14c8b3d21c8 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -832,9 +832,9 @@ async fn test_disk_backed_by_multiple_region_sets( assert_eq!(10, DiskTest::DEFAULT_ZPOOL_SIZE_GIB); // Create another three zpools, all 10 gibibytes, each with one dataset - test.add_zpool_with_dataset(10).await; - test.add_zpool_with_dataset(10).await; - test.add_zpool_with_dataset(10).await; + test.add_zpool_with_dataset(cptestctx, 10).await; + test.add_zpool_with_dataset(cptestctx, 10).await; + test.add_zpool_with_dataset(cptestctx, 10).await; create_org_and_project(client).await; @@ -1077,9 +1077,9 @@ async fn test_multiple_disks_multiple_zpools( // Assert default is still 10 GiB assert_eq!(10, DiskTest::DEFAULT_ZPOOL_SIZE_GIB); - test.add_zpool_with_dataset(10).await; - test.add_zpool_with_dataset(10).await; - test.add_zpool_with_dataset(10).await; + test.add_zpool_with_dataset(cptestctx, 10).await; + test.add_zpool_with_dataset(cptestctx, 10).await; + test.add_zpool_with_dataset(cptestctx, 10).await; create_org_and_project(client).await; diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index bc59b4a0d2d..f70353d7975 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -276,8 +276,8 @@ impl SledAgent { &self, zpool_id: Uuid, dataset_id: Uuid, - ) { - self.storage.lock().await.insert_dataset(zpool_id, dataset_id).await; + ) -> SocketAddr { + self.storage.lock().await.insert_dataset(zpool_id, dataset_id).await } /// Returns a crucible dataset within a particular zpool. diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index e3acac19df1..5e3727ebb67 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -326,7 +326,7 @@ impl CrucibleData { /// /// Contains both the data and the HTTP server. pub struct CrucibleServer { - _server: dropshot::HttpServer>, + server: dropshot::HttpServer>, data: Arc, } @@ -352,7 +352,11 @@ impl CrucibleServer { .start(); info!(&log, "Created Simulated Crucible Server"; "address" => server.local_addr()); - CrucibleServer { _server: server, data } + CrucibleServer { server, data } + } + + fn address(&self) -> SocketAddr { + self.server.local_addr() } pub fn data(&self) -> Arc { @@ -455,10 +459,14 @@ impl Storage { .expect("Failed to notify Nexus about new Zpool"); } - /// Adds a Dataset to the sled's simulated storage and notifies Nexus. - pub async fn insert_dataset(&mut self, zpool_id: Uuid, dataset_id: Uuid) { + /// Adds a Dataset to the sled's simulated storage. + pub async fn insert_dataset( + &mut self, + zpool_id: Uuid, + dataset_id: Uuid, + ) -> SocketAddr { // Update our local data - let _ = self + let dataset = self .zpools .get_mut(&zpool_id) .expect("Zpool does not exist") @@ -470,6 +478,8 @@ impl Storage { ); self.next_crucible_port += 100; + + dataset.address() } pub async fn get_dataset( From cfa957301c388f5db740a6e4fc5b9bb8ba3aba9e Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 17 Nov 2022 00:40:23 -0500 Subject: [PATCH 07/20] Bump allocator tests --- sled-agent/src/rack_setup/plan/service.rs | 60 ++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index d38995751ce..e9aa398b1b9 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -335,8 +335,6 @@ struct AddressBumpAllocator { last_addr: Ipv6Addr, } -// TODO: Testable? -// TODO: Could exist in another file? impl AddressBumpAllocator { fn new(subnet: Ipv6Subnet) -> Self { Self { last_addr: get_switch_zone_address(subnet) } @@ -352,3 +350,61 @@ impl AddressBumpAllocator { Some(self.last_addr) } } + +#[cfg(test)] +mod tests { + use super::*; + + const EXPECTED_RESERVED_ADDRESSES: u16 = 2; + const EXPECTED_USABLE_ADDRESSES: u16 = + RSS_RESERVED_ADDRESSES - EXPECTED_RESERVED_ADDRESSES; + + #[test] + fn bump_allocator_basics() { + let address = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0); + let subnet = Ipv6Subnet::::new(address); + + let mut allocator = AddressBumpAllocator::new(subnet); + assert_eq!( + allocator.next().unwrap(), + Ipv6Addr::new( + 0xfd00, + 0, + 0, + 0, + 0, + 0, + 0, + EXPECTED_RESERVED_ADDRESSES + 1 + ), + ); + assert_eq!( + allocator.next().unwrap(), + Ipv6Addr::new( + 0xfd00, + 0, + 0, + 0, + 0, + 0, + 0, + EXPECTED_RESERVED_ADDRESSES + 2 + ), + ); + } + + #[test] + fn bump_allocator_exhaustion() { + let address = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0); + let subnet = Ipv6Subnet::::new(address); + + let mut allocator = AddressBumpAllocator::new(subnet); + for i in 0..EXPECTED_USABLE_ADDRESSES { + assert!( + allocator.next().is_some(), + "Could not allocate {i}-th address" + ); + } + assert!(allocator.next().is_none(), "Expected allocation to fail"); + } +} From 82dd525c7613634062a88bce563b2fe075159801 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 17 Nov 2022 00:43:29 -0500 Subject: [PATCH 08/20] re-order config fields --- smf/sled-agent/config-rss.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index d2207758135..3efd7d0cc84 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -11,6 +11,9 @@ rack_subnet = "fd00:1122:3344:0100::" # For values less than 2, no rack secret will be generated. rack_secret_threshold = 1 +# NOTE: In the lab, use "172.20.15.226" +nexus_external_address = "192.168.1.20" + [gateway] # IP address of Internet gateway @@ -26,6 +29,3 @@ rack_secret_threshold = 1 # in their local network, using the current workaround methods in OPTE. See # how-to-run.adoc for details on how to determine the value for your network. mac = "00:0d:b9:54:fe:e4" - -# NOTE: In the lab, use "172.20.15.226" -nexus_external_address = "192.168.1.20" From b10db8f706b62729c7c69b10647b8395d05e59a1 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 17 Nov 2022 11:29:23 -0500 Subject: [PATCH 09/20] Nexus needs to find CRDB via DNS --- sled-agent/src/services.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 405b29b92e0..cc587adf8ef 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -46,13 +46,11 @@ use omicron_common::address::SLED_PREFIX; use omicron_common::nexus_config::{ self, DeploymentConfig as NexusDeploymentConfig, }; -use omicron_common::postgres_config::PostgresConfigWithUrl; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::path::{Path, PathBuf}; -use std::str::FromStr; use std::sync::Arc; use tokio::io::AsyncWriteExt; use tokio::sync::oneshot; @@ -623,9 +621,15 @@ impl ServiceManager { // one for HTTPS (port 443). dropshot_external: vec![ dropshot::ConfigDropshot { - bind_address: SocketAddr::new(*external_ip, 443), + bind_address: SocketAddr::new( + *external_ip, + 443, + ), request_body_max_bytes: 1048576, - tls: Some(dropshot::ConfigTls { cert_file, key_file }), + tls: Some(dropshot::ConfigTls { + cert_file, + key_file, + }), }, dropshot::ConfigDropshot { bind_address: SocketAddr::new(*external_ip, 80), @@ -634,19 +638,17 @@ impl ServiceManager { }, ], dropshot_internal: dropshot::ConfigDropshot { - bind_address: SocketAddr::new(IpAddr::V6(*internal_ip), NEXUS_INTERNAL_PORT), + bind_address: SocketAddr::new( + IpAddr::V6(*internal_ip), + NEXUS_INTERNAL_PORT, + ), request_body_max_bytes: 1048576, ..Default::default() }, subnet: Ipv6Subnet::::new( self.inner.underlay_address, ), - // TODO: Switch to inferring this URL by DNS. - database: nexus_config::Database::FromUrl { - url: PostgresConfigWithUrl::from_str( - "postgresql://root@[fd00:1122:3344:0101::3]:32221/omicron?sslmode=disable" - ).unwrap(), - } + database: nexus_config::Database::FromDns, }; // Copy the partial config file to the expected location. From ac5e9f6c3cd2f01fca4dc604b2964bc32fad78d1 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 17 Nov 2022 12:08:53 -0500 Subject: [PATCH 10/20] Fix RSS config files, better docs --- sled-agent/src/rack_setup/service.rs | 70 ++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index ae4e9ec986a..20e602cc226 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -2,7 +2,58 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Rack Setup Service implementation +//! Rack Setup Service (RSS) implementation +//! +//! RSS triggers the initialization of: +//! - Sled Agents (giving them underlay addresses) +//! - Trust Quorum (coordinating between Sled Agents) +//! - Services (such as internal DNS, CRDB, Nexus) +//! - DNS records for those services +//! - Handoff to Nexus, for control of Control Plane management +//! +//! # Phases and Configuration Files +//! +//! Rack setup occurs in distinct phases which are denoted by the prescence of +//! configuration files. +//! +//! - /var/oxide/rss-sled-plan.toml (Sled Plan) +//! - /var/oxide/rss-service-plan.toml (Service Plan) +//! - /var/oxide/rss-plan-completed.marker (Plan Execution Complete) +//! +//! ## Sled Plan +//! +//! When RSS starts, it presumably is executing on a single sled, and must +//! communicate with other sleds on the bootstrap network to discover neighbors. +//! RSS uses the bootstrap network to identify peers, assign them subnets and +//! UUIDs, and initialize a trust quorum. Once RSS decides these values +//! (see: [crate::rack_setup::plan::sled] for more details) it commits them +//! to a local file as the "Sled Plan", before sending requests. +//! +//! As a result, restarting RSS should result in retransmission of the same +//! values, as long as the same configuration file is used. +//! +//! ## Service Plan +//! +//! After the trust quorum is established and Sled Agents are executing across +//! the rack, RSS can make the call on "what services should run where", +//! ensuring the minimal set of services necessary to execute Nexus are +//! operational (see: [crate::rack_setup::plan::service]). Critically, +//! these include: +//! - Internal DNS: Necessary so internal services can discover each other +//! - CockroachDB: Necessary for Nexus to operate +//! - Nexus itself +//! +//! Once the distribution of these services is decided (which sled should run +//! what service? On what zpools should CockroachDB be provisioned?) it is +//! committed to the Service Plan, and executed. +//! +//! ## Execution Complete +//! +//! Once the both the Sled and Service plans have finished execution, handoff of +//! control to Nexus can occur. +//! covers this in more detail, but in short, RSS creates a "marker" file after +//! completing execution, and unconditionally calls the "handoff to Nexus" API +//! thereafter. use super::config::SetupServiceConfig as Config; use crate::bootstrap::ddm_admin_client::{DdmAdminClient, DdmError}; @@ -142,14 +193,9 @@ impl RackSetupService { } } -fn rss_plan_path() -> PathBuf { - std::path::Path::new(omicron_common::OMICRON_CONFIG_PATH) - .join("rss-plan.toml") -} - fn rss_completed_plan_path() -> PathBuf { std::path::Path::new(omicron_common::OMICRON_CONFIG_PATH) - .join("rss-plan-completed.toml") + .join("rss-plan-completed.marker") } // Describes the options when awaiting for peers. @@ -724,14 +770,10 @@ impl ServiceInner { info!(self.log, "Finished setting up services"); - // Finally, make sure the configuration is saved so we don't inject - // the requests on the next iteration. - let plan_path = rss_plan_path(); - tokio::fs::rename(&plan_path, &rss_completed_plan_path).await.map_err( + // Finally, mark that we've completed executing the plans. + tokio::fs::File::create(&rss_completed_plan_path).await.map_err( |err| SetupServiceError::Io { - message: format!( - "renaming {plan_path:?} to {rss_completed_plan_path:?}" - ), + message: format!("creating {rss_completed_plan_path:?}"), err, }, )?; From b9ddc9efdf54c3a3a1c134b1a1e4a9f130b4f406 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 17 Nov 2022 12:29:54 -0500 Subject: [PATCH 11/20] No doc links to private modules, I guess --- sled-agent/src/rack_setup/service.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 20e602cc226..e66826f69df 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -25,9 +25,8 @@ //! When RSS starts, it presumably is executing on a single sled, and must //! communicate with other sleds on the bootstrap network to discover neighbors. //! RSS uses the bootstrap network to identify peers, assign them subnets and -//! UUIDs, and initialize a trust quorum. Once RSS decides these values -//! (see: [crate::rack_setup::plan::sled] for more details) it commits them -//! to a local file as the "Sled Plan", before sending requests. +//! UUIDs, and initialize a trust quorum. Once RSS decides these values it +//! commits them to a local file as the "Sled Plan", before sending requests. //! //! As a result, restarting RSS should result in retransmission of the same //! values, as long as the same configuration file is used. @@ -37,8 +36,7 @@ //! After the trust quorum is established and Sled Agents are executing across //! the rack, RSS can make the call on "what services should run where", //! ensuring the minimal set of services necessary to execute Nexus are -//! operational (see: [crate::rack_setup::plan::service]). Critically, -//! these include: +//! operational. Critically, these include: //! - Internal DNS: Necessary so internal services can discover each other //! - CockroachDB: Necessary for Nexus to operate //! - Nexus itself From cf6a07bb859ecb531691d75b79a025a980fbaa91 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 17 Nov 2022 14:13:30 -0500 Subject: [PATCH 12/20] store config in sled plan --- sled-agent/src/rack_setup/plan/sled.rs | 11 +++++------ sled-agent/src/rack_setup/service.rs | 8 +++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/sled-agent/src/rack_setup/plan/sled.rs b/sled-agent/src/rack_setup/plan/sled.rs index cde70deb690..04167f98a7f 100644 --- a/sled-agent/src/rack_setup/plan/sled.rs +++ b/sled-agent/src/rack_setup/plan/sled.rs @@ -85,11 +85,10 @@ pub enum PlanError { pub struct Plan { pub rack_id: Uuid, pub sleds: HashMap, - // TODO: Consider putting the rack subnet here? This may be operator-driven - // in the future, so it should exist in the "plan". - // - // TL;DR: The more we decouple rom "rss-config.toml", the easier it'll be to - // switch to an operator-driven interface. + + // Store the provided RSS configuration as part of the sled plan; if it + // changes after reboot, we need to know. + pub config: Config, } impl Plan { @@ -151,7 +150,7 @@ impl Plan { sleds.insert(addr, allocation); } - let plan = Self { rack_id, sleds }; + let plan = Self { rack_id, sleds, config: config.clone() }; // Once we've constructed a plan, write it down to durable storage. let serialized_plan = diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index e66826f69df..7752ef82f63 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -586,6 +586,11 @@ impl ServiceInner { let sled_plan = SledPlan::load(&self.log) .await? .expect("Sled plan should exist if completed marker exists"); + if &sled_plan.config != config { + return Err(SetupServiceError::BadConfig( + "Configuration changed".to_string(), + )); + } let service_plan = ServicePlan::load(&self.log) .await? .expect("Service plan should exist if completed marker exists"); @@ -622,8 +627,9 @@ impl ServiceInner { plan } else { info!(self.log, "Creating new allocation plan"); - SledPlan::create(&self.log, &config, addrs).await? + SledPlan::create(&self.log, config, addrs).await? }; + let config = &plan.config; // Generate our rack secret, unless we're in the single-sled case. let mut maybe_rack_secret_shares = generate_rack_secret( From f3c6a34948b828950886f51fa135a7419762aa1a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 17 Nov 2022 14:19:41 -0500 Subject: [PATCH 13/20] Service translation --- sled-agent/src/rack_setup/service.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 7752ef82f63..3e22111a111 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -483,13 +483,12 @@ impl ServiceInner { ServiceType::Oximeter => { NexusTypes::ServiceKind::Oximeter } - // TODO TODO TODO - ServiceType::ManagementGatewayService => todo!(), - ServiceType::Dendrite { .. } => { - NexusTypes::ServiceKind::Dendrite + _ => { + return Err(SetupServiceError::BadConfig(format!( + "RSS should not request service of type: {}", + svc + ))); } - // TODO TODO TODO - ServiceType::Tfport { .. } => todo!(), }; services.push(NexusTypes::ServicePutRequest { From 07823273c0812d9d547daf500498476fe1dd79df Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 17 Nov 2022 16:48:07 -0500 Subject: [PATCH 14/20] comment cleanup --- nexus/src/app/rack.rs | 2 +- nexus/test-utils/src/resource_helpers.rs | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 8b00bb7195b..b2ec94d817c 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -75,7 +75,7 @@ impl super::Nexus { }) .collect(); - // TODO: If nexus, add a pool? + // TODO(https://github.com/oxidecomputer/omicron/issues/1958): If nexus, add a pool? let datasets: Vec<_> = request .datasets diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 8e656c33062..da344d40657 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -489,9 +489,6 @@ impl DiskTest { .server .upsert_crucible_dataset(dataset.id, zpool.id, address) .await; - - // TODO: Upsert dataset?? - // cptestctx.name } self.zpools.push(zpool); From 8d899c513a13bb809327fa888b70f1cede76e50f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Nov 2022 15:26:04 -0500 Subject: [PATCH 15/20] Collapse services and dns_services --- sled-agent/src/rack_setup/plan/service.rs | 6 +-- sled-agent/src/rack_setup/service.rs | 61 +++++++++++------------ 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index e9aa398b1b9..cf270954745 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -78,10 +78,6 @@ pub struct SledRequest { /// Services to be instantiated. #[serde(default, rename = "service")] pub services: Vec, - - /// DNS Services to be instantiated. - #[serde(default, rename = "dns_service")] - pub dns_services: Vec, } #[derive(Debug, Serialize, Deserialize)] @@ -280,7 +276,7 @@ impl Plan { if idx < dns_subnets.len() { let dns_subnet = &dns_subnets[idx]; let dns_addr = dns_subnet.dns_address().ip(); - request.dns_services.push(ServiceZoneRequest { + request.services.push(ServiceZoneRequest { id: Uuid::new_v4(), zone_type: ZoneType::InternalDNS, addresses: vec![dns_addr], diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 3e22111a111..d305dd40a9e 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -57,7 +57,9 @@ use super::config::SetupServiceConfig as Config; use crate::bootstrap::ddm_admin_client::{DdmAdminClient, DdmError}; use crate::bootstrap::params::SledAgentRequest; use crate::bootstrap::rss_handle::BootstrapAgentHandle; -use crate::params::{DatasetEnsureBody, ServiceType, ServiceZoneRequest}; +use crate::params::{ + DatasetEnsureBody, ServiceType, ServiceZoneRequest, ZoneType, +}; use crate::rack_setup::plan::service::{ Plan as ServicePlan, PlanError as ServicePlanError, }; @@ -465,11 +467,7 @@ impl ServiceInner { .get(addr) .expect("Sled address in service plan, but not sled plan"); - for zone in service_request - .services - .iter() - .chain(service_request.dns_services.iter()) - { + for zone in &service_request.services { for svc in &zone.services { let kind = match svc { ServiceType::Nexus { external_ip, internal_ip: _ } => { @@ -691,24 +689,26 @@ impl ServiceInner { }; // Set up internal DNS services. - futures::future::join_all( - service_plan - .services - .iter() - .filter(|(_, services_request)| { - // Only send requests to sleds that are supposed to be running - // DNS services. - !services_request.dns_services.is_empty() - }) - .map(|(sled_address, services_request)| async move { - self.initialize_services( - *sled_address, - &services_request.dns_services, - ) - .await?; - Ok(()) - }), - ) + futures::future::join_all(service_plan.services.iter().map( + |(sled_address, services_request)| async move { + let dns_services: Vec<_> = services_request + .services + .iter() + .filter_map(|svc| { + if matches!(svc.zone_type, ZoneType::InternalDNS) { + Some(svc.clone()) + } else { + None + } + }) + .collect(); + if !dns_services.is_empty() { + self.initialize_services(*sled_address, &dns_services) + .await?; + } + Ok(()) + }, + )) .await .into_iter() .collect::>()?; @@ -756,14 +756,11 @@ impl ServiceInner { // This means re-requesting the DNS service, even if it is // already running - this is fine, however, as the receiving // sled agent doesn't modify the already-running service. - let all_services = services_request - .services - .iter() - .chain(services_request.dns_services.iter()) - .map(|s| s.clone()) - .collect::>(); - - self.initialize_services(*sled_address, &all_services).await?; + self.initialize_services( + *sled_address, + &services_request.services, + ) + .await?; Ok(()) }, )) From d223fb9b1798e03ae7debdebacc9cdfaa8344666 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Nov 2022 15:30:17 -0500 Subject: [PATCH 16/20] rss_completed_marker_path naming --- sled-agent/src/rack_setup/service.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index d305dd40a9e..5a37c2d6285 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -193,7 +193,7 @@ impl RackSetupService { } } -fn rss_completed_plan_path() -> PathBuf { +fn rss_completed_marker_path() -> PathBuf { std::path::Path::new(omicron_common::OMICRON_CONFIG_PATH) .join("rss-plan-completed.marker") } @@ -557,7 +557,7 @@ impl ServiceInner { // outlined in the aforementioned step are created. // // 5. MARKING SETUP COMPLETE. Once the RSS has successfully initialized the - // rack, a marker file is created at "rss_completed_plan_path()". This + // rack, a marker file is created at "rss_completed_marker_path()". This // indicates that the plan executed successfully, and no work remains. async fn inject_rack_setup_requests( &self, @@ -570,8 +570,8 @@ impl ServiceInner { // Check if a previous RSS plan has completed successfully. // // If it has, the system should be up-and-running. - let rss_completed_plan_path = rss_completed_plan_path(); - if rss_completed_plan_path.exists() { + let rss_completed_marker_path = rss_completed_marker_path(); + if rss_completed_marker_path.exists() { // TODO(https://github.com/oxidecomputer/omicron/issues/724): If the // running configuration doesn't match Config, we could try to // update things. @@ -771,9 +771,9 @@ impl ServiceInner { info!(self.log, "Finished setting up services"); // Finally, mark that we've completed executing the plans. - tokio::fs::File::create(&rss_completed_plan_path).await.map_err( + tokio::fs::File::create(&rss_completed_marker_path).await.map_err( |err| SetupServiceError::Io { - message: format!("creating {rss_completed_plan_path:?}"), + message: format!("creating {rss_completed_marker_path:?}"), err, }, )?; From c53a0bf3c48569611d98839117c62a5f9ed3bf62 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Nov 2022 15:32:34 -0500 Subject: [PATCH 17/20] or_default --- sled-agent/src/rack_setup/service.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 5a37c2d6285..780b300b553 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -266,11 +266,11 @@ impl ServiceInner { .await?; } - let mut records = HashMap::new(); + let mut records: HashMap<_, Vec<_>> = HashMap::new(); for dataset in datasets { records .entry(dataset.srv()) - .or_insert_with(Vec::new) + .or_default() .push((dataset.aaaa(), dataset.address())); } let records_put = || async { From 362f7e13ca360c240d49ee1019ca9d4435c9e6f6 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Nov 2022 15:36:52 -0500 Subject: [PATCH 18/20] Updated doc comment --- sled-agent/src/rack_setup/service.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 780b300b553..c4438f8896a 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -22,11 +22,12 @@ //! //! ## Sled Plan //! -//! When RSS starts, it presumably is executing on a single sled, and must -//! communicate with other sleds on the bootstrap network to discover neighbors. -//! RSS uses the bootstrap network to identify peers, assign them subnets and -//! UUIDs, and initialize a trust quorum. Once RSS decides these values it -//! commits them to a local file as the "Sled Plan", before sending requests. +//! RSS should start as a service executing on a Sidecar-attached Gimlet +//! (Scrimlet). It must communicate with other sleds on the bootstrap network to +//! discover neighbors. RSS uses the bootstrap network to identify peers, assign +//! them subnets and UUIDs, and initialize a trust quorum. Once RSS decides +//! these values it commits them to a local file as the "Sled Plan", before +//! sending requests. //! //! As a result, restarting RSS should result in retransmission of the same //! values, as long as the same configuration file is used. From 4ec1e2843b9506b2dc57596dd1cae6502d541914 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 1 Dec 2022 13:47:49 -0500 Subject: [PATCH 19/20] Add log statement about rack init --- nexus/src/app/rack.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index b2ec94d817c..9aaac5c7dea 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -111,6 +111,7 @@ impl super::Nexus { match result { Ok(rack) => { if rack.initialized { + info!(self.log, "Rack initialized"); return; } info!( From a6593901500f442723a0d2f82d60d667c4cab425 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 1 Dec 2022 14:31:06 -0500 Subject: [PATCH 20/20] Add doc comments --- sled-agent/src/rack_setup/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sled-agent/src/rack_setup/mod.rs b/sled-agent/src/rack_setup/mod.rs index 4df85a7727f..cfabca62090 100644 --- a/sled-agent/src/rack_setup/mod.rs +++ b/sled-agent/src/rack_setup/mod.rs @@ -4,6 +4,8 @@ //! Rack Setup Service +/// Configuration files which automate input to RSS. pub mod config; mod plan; +/// The main implementation of the RSS service. pub mod service;