diff --git a/Cargo.lock b/Cargo.lock index 268e855769e..284c7f25a0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2243,12 +2243,16 @@ dependencies = [ name = "internal-dns-client" version = "0.1.0" dependencies = [ + "omicron-common", "progenitor", "reqwest", "serde", "serde_json", "slog", "structopt", + "trust-dns-proto", + "trust-dns-resolver", + "uuid", ] [[package]] @@ -2798,6 +2802,7 @@ dependencies = [ "thiserror", "tokio", "tokio-postgres", + "toml", "uuid", ] @@ -2877,6 +2882,7 @@ dependencies = [ "http", "httptest", "hyper", + "internal-dns-client", "ipnetwork", "lazy_static", "libc", @@ -2972,6 +2978,7 @@ dependencies = [ "expectorate", "futures", "http", + "internal-dns-client", "ipnetwork", "libc", "macaddr", diff --git a/common/Cargo.toml b/common/Cargo.toml index aa3b8943800..cd47bef1169 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -30,6 +30,7 @@ structopt = "0.3" thiserror = "1.0" tokio = { version = "1.18", features = [ "full" ] } tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] } +toml = "0.5.9" uuid = { version = "1.1.0", features = [ "serde", "v4" ] } parse-display = "0.5.4" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } diff --git a/common/src/address.rs b/common/src/address.rs index d1f86fbb82c..7d6b37a34ee 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -27,12 +27,23 @@ pub const DNS_REDUNDANCY: usize = 1; /// reserved for DNS servers. pub const MAX_DNS_REDUNDANCY: usize = 5; +/// The port for the UDP-based internal DNS name server. pub const DNS_PORT: u16 = 53; +/// The port for the HTTP-based internal DNS dropshot server. pub const DNS_SERVER_PORT: u16 = 5353; pub const SLED_AGENT_PORT: u16 = 12345; /// The port propolis-server listens on inside the propolis zone. pub const PROPOLIS_PORT: u16 = 12400; +pub const COCKROACH_PORT: u16 = 32221; +pub const CRUCIBLE_PORT: u16 = 32345; + +pub const NEXUS_EXTERNAL_PORT: u16 = 12220; +pub const NEXUS_INTERNAL_PORT: u16 = 12221; + +pub const COCKROACH_DNS_NAME: &str = + "_cockroachdb._tcp.control-plane.oxide.internal"; + // Anycast is a mechanism in which a single IP address is shared by multiple // devices, and the destination is located based on routing distance. @@ -124,14 +135,16 @@ impl ReservedRackSubnet { } } -const SLED_AGENT_ADDRESS_INDEX: usize = 1; +pub const SLED_AGENT_ADDRESS_INDEX: u16 = 1; +/// The maximum number of addresses per sled subnet reserved for RSS. +pub const RSS_RESERVED_ADDRESSES: u16 = 10; /// Return the sled agent address for a subnet. /// /// This address will come from the first address of the [`SLED_PREFIX`] subnet. pub fn get_sled_address(sled_subnet: Ipv6Subnet) -> SocketAddrV6 { let sled_agent_ip = - sled_subnet.net().iter().nth(SLED_AGENT_ADDRESS_INDEX).unwrap(); + sled_subnet.net().iter().nth(SLED_AGENT_ADDRESS_INDEX.into()).unwrap(); SocketAddrV6::new(sled_agent_ip, SLED_AGENT_PORT, 0, 0) } diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 2f74c8e5597..7c4d77065ad 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -500,6 +500,7 @@ pub enum ResourceType { Instance, NetworkInterface, Rack, + Service, Sled, SagaDbg, Snapshot, diff --git a/common/src/lib.rs b/common/src/lib.rs index 2a933283425..d90ecdb7333 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -24,7 +24,8 @@ pub mod address; pub mod api; pub mod backoff; pub mod cmd; -pub mod config; +pub mod nexus_config; +pub mod postgres_config; #[macro_export] macro_rules! generate_logging_api { diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs new file mode 100644 index 00000000000..f1325ae336d --- /dev/null +++ b/common/src/nexus_config.rs @@ -0,0 +1,128 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Configuration parameters to Nexus that are usually only known +//! at runtime. + +use super::address::{Ipv6Subnet, RACK_PREFIX}; +use super::postgres_config::PostgresConfigWithUrl; +use dropshot::ConfigDropshot; +use serde::{Deserialize, Serialize}; +use serde_with::serde_as; +use serde_with::DisplayFromStr; +use std::fmt; +use std::path::{Path, PathBuf}; +use uuid::Uuid; + +#[derive(Debug)] +pub struct LoadError { + pub path: PathBuf, + pub kind: LoadErrorKind, +} + +#[derive(Debug)] +pub struct InvalidTunable { + pub tunable: String, + pub message: String, +} + +impl std::fmt::Display for InvalidTunable { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "invalid \"{}\": \"{}\"", self.tunable, self.message) + } +} +impl std::error::Error for InvalidTunable {} + +#[derive(Debug)] +pub enum LoadErrorKind { + Io(std::io::Error), + Parse(toml::de::Error), + InvalidTunable(InvalidTunable), +} + +impl From<(PathBuf, std::io::Error)> for LoadError { + fn from((path, err): (PathBuf, std::io::Error)) -> Self { + LoadError { path, kind: LoadErrorKind::Io(err) } + } +} + +impl From<(PathBuf, toml::de::Error)> for LoadError { + fn from((path, err): (PathBuf, toml::de::Error)) -> Self { + LoadError { path, kind: LoadErrorKind::Parse(err) } + } +} + +impl std::error::Error for LoadError {} + +impl fmt::Display for LoadError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self.kind { + LoadErrorKind::Io(e) => { + write!(f, "read \"{}\": {}", self.path.display(), e) + } + LoadErrorKind::Parse(e) => { + write!(f, "parse \"{}\": {}", self.path.display(), e) + } + LoadErrorKind::InvalidTunable(inner) => { + write!( + f, + "invalid tunable \"{}\": {}", + self.path.display(), + inner, + ) + } + } + } +} + +impl std::cmp::PartialEq for LoadError { + fn eq(&self, other: &std::io::Error) -> bool { + if let LoadErrorKind::Io(e) = &self.kind { + e.kind() == other.kind() + } else { + false + } + } +} + +#[serde_as] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +#[allow(clippy::large_enum_variant)] +pub enum Database { + FromDns, + FromUrl { + #[serde_as(as = "DisplayFromStr")] + url: PostgresConfigWithUrl, + }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct RuntimeConfig { + /// Uuid of the Nexus instance + pub id: Uuid, + /// Dropshot configuration for external API server + pub dropshot_external: ConfigDropshot, + /// Dropshot configuration for internal API server + pub dropshot_internal: ConfigDropshot, + /// Portion of the IP space to be managed by the Rack. + pub subnet: Ipv6Subnet, + /// DB configuration. + pub database: Database, +} + +impl RuntimeConfig { + /// Load a `RuntimeConfig` from the given TOML file + /// + /// This config object can then be used to create a new `Nexus`. + /// The format is described in the README. + pub fn from_file>(path: P) -> Result { + let path = path.as_ref(); + let file_contents = std::fs::read_to_string(path) + .map_err(|e| (path.to_path_buf(), e))?; + let config_parsed: Self = toml::from_str(&file_contents) + .map_err(|e| (path.to_path_buf(), e))?; + Ok(config_parsed) + } +} diff --git a/common/src/config.rs b/common/src/postgres_config.rs similarity index 100% rename from common/src/config.rs rename to common/src/postgres_config.rs diff --git a/common/src/sql/dbinit.sql b/common/src/sql/dbinit.sql index 5d592073635..4ee16c314f4 100644 --- a/common/src/sql/dbinit.sql +++ b/common/src/sql/dbinit.sql @@ -72,6 +72,34 @@ CREATE TABLE omicron.public.sled ( last_used_address INET NOT NULL ); +/* + * Services + */ + +CREATE TABLE omicron.public.service ( + /* Identity metadata (asset) */ + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + + /* FK into the Sled table */ + sled_id UUID NOT NULL, + + /* The IP address and bound port of the service. */ + ip INET NOT NULL, + port INT4 CHECK (port BETWEEN 0 AND 65535) NOT NULL +); + +/* Add an index which lets us look up the services on a sled */ +CREATE INDEX ON omicron.public.service ( + sled_id +); + +CREATE TYPE omicron.public.service_kind AS ENUM ( + 'nexus', + 'oximeter' +); + /* * ZPools of Storage, attached to Sleds. * Typically these are backed by a single physical disk. diff --git a/internal-dns-client/Cargo.toml b/internal-dns-client/Cargo.toml index 22e28c91bc9..0ac6ecba610 100644 --- a/internal-dns-client/Cargo.toml +++ b/internal-dns-client/Cargo.toml @@ -5,9 +5,13 @@ edition = "2021" license = "MPL-2.0" [dependencies] +omicron-common = { path = "../common" } progenitor = { git = "https://github.com/oxidecomputer/progenitor" } +reqwest = { version = "0.11", features = ["json", "rustls-tls", "stream"] } serde = { version = "1.0", features = [ "derive" ] } serde_json = "1.0" slog = { version = "2.5.0", features = [ "max_level_trace", "release_max_level_debug" ] } structopt = "0.3" -reqwest = { version = "0.11", features = ["json", "rustls-tls", "stream"] } +trust-dns-proto = "0.21" +trust-dns-resolver = "0.21" +uuid = { version = "1.1.0", features = [ "v4", "serde" ] } diff --git a/internal-dns-client/src/lib.rs b/internal-dns-client/src/lib.rs index 49daa3d58ae..f7ce56f8521 100644 --- a/internal-dns-client/src/lib.rs +++ b/internal-dns-client/src/lib.rs @@ -16,3 +16,6 @@ progenitor::generate_api!( slog::debug!(log, "client response"; "result" => ?result); }), ); + +pub mod multiclient; +pub mod names; diff --git a/internal-dns-client/src/multiclient.rs b/internal-dns-client/src/multiclient.rs new file mode 100644 index 00000000000..e01fb5a2139 --- /dev/null +++ b/internal-dns-client/src/multiclient.rs @@ -0,0 +1,145 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::types::{DnsKv, DnsRecord, DnsRecordKey, Srv}; +use omicron_common::address::{ + Ipv6Subnet, ReservedRackSubnet, AZ_PREFIX, DNS_PORT, DNS_SERVER_PORT, +}; +use omicron_common::backoff::{ + internal_service_policy, retry_notify, BackoffError, +}; +use slog::{info, warn, Logger}; +use std::net::{SocketAddr, SocketAddrV6}; +use trust_dns_resolver::config::{ + NameServerConfig, Protocol, ResolverConfig, ResolverOpts, +}; +use trust_dns_resolver::TokioAsyncResolver; + +type DnsError = crate::Error; + +/// A connection used to update multiple DNS servers. +pub struct Updater { + clients: Vec, +} + +impl Updater { + pub fn new(subnet: Ipv6Subnet, log: Logger) -> Self { + let clients = ReservedRackSubnet::new(subnet) + .get_dns_subnets() + .into_iter() + .map(|dns_subnet| { + let addr = dns_subnet.dns_address().ip(); + info!(log, "Adding DNS server: {}", addr); + crate::Client::new( + &format!("http://[{}]:{}", addr, DNS_SERVER_PORT), + log.clone(), + ) + }) + .collect::>(); + + Self { clients } + } + + /// Utility function to insert: + /// - A set of uniquely-named AAAA records, each corresponding to an address + /// - An SRV record, pointing to each of the AAAA records. + pub async fn insert_dns_records( + &self, + log: &Logger, + aaaa: Vec<(crate::names::AAAA, SocketAddrV6)>, + srv_key: crate::names::SRV, + ) -> Result<(), DnsError> { + let mut records = Vec::with_capacity(aaaa.len() + 1); + + // Add one DnsKv per AAAA, each with a single record. + records.extend(aaaa.iter().map(|(name, addr)| DnsKv { + key: DnsRecordKey { name: name.to_string() }, + records: vec![DnsRecord::Aaaa(*addr.ip())], + })); + + // Add the DnsKv for the SRV, with a record for each AAAA. + records.push(DnsKv { + key: DnsRecordKey { name: srv_key.to_string() }, + records: aaaa + .iter() + .map(|(name, addr)| { + DnsRecord::Srv(Srv { + prio: 0, + weight: 0, + port: addr.port(), + target: name.to_string(), + }) + }) + .collect::>(), + }); + + let set_record = || async { + self.dns_records_set(&records) + .await + .map_err(BackoffError::transient)?; + Ok::<(), BackoffError>(()) + }; + let log_failure = |error, _| { + warn!(log, "Failed to set DNS records"; "error" => ?error); + }; + + retry_notify(internal_service_policy(), set_record, log_failure) + .await?; + Ok(()) + } + + /// Sets a records on all DNS servers. + /// + /// Returns an error if setting the record fails on any server. + pub async fn dns_records_set<'a>( + &'a self, + body: &'a Vec, + ) -> Result<(), DnsError> { + // TODO: Could be sent concurrently. + for client in &self.clients { + client.dns_records_set(body).await?; + } + + Ok(()) + } + + /// Deletes records in all DNS servers. + /// + /// Returns an error if deleting the record fails on any server. + pub async fn dns_records_delete<'a>( + &'a self, + body: &'a Vec, + ) -> Result<(), DnsError> { + // TODO: Could be sent concurrently + for client in &self.clients { + client.dns_records_delete(body).await?; + } + Ok(()) + } +} + +/// Creates a resolver using all internal DNS name servers. +pub fn create_resolver( + subnet: Ipv6Subnet, +) -> Result { + let mut rc = ResolverConfig::new(); + let dns_ips = ReservedRackSubnet::new(subnet) + .get_dns_subnets() + .into_iter() + .map(|subnet| subnet.dns_address().ip()) + .collect::>(); + + for dns_ip in dns_ips { + rc.add_name_server(NameServerConfig { + socket_addr: SocketAddr::V6(SocketAddrV6::new( + dns_ip, DNS_PORT, 0, 0, + )), + protocol: Protocol::Udp, + tls_dns_name: None, + trust_nx_responses: false, + bind_addr: None, + }); + } + TokioAsyncResolver::tokio(rc, ResolverOpts::default()) +} diff --git a/internal-dns-client/src/names.rs b/internal-dns-client/src/names.rs new file mode 100644 index 00000000000..6384ec9e503 --- /dev/null +++ b/internal-dns-client/src/names.rs @@ -0,0 +1,55 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::fmt; +use uuid::Uuid; + +const DNS_ZONE: &str = "control-plane.oxide.internal"; + +pub enum SRV { + /// A service identified and accessed by name, such as "nexus", "CRDB", etc. + /// + /// This is used in cases where services are interchangeable. + Service(String), + + /// A service identified by name and a unique identifier. + /// + /// This is used in cases where services are not interchangeable, such as + /// for the Sled agent. + Backend(String, Uuid), +} + +impl fmt::Display for SRV { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self { + SRV::Service(name) => { + write!(f, "_{}._tcp.{}", name, DNS_ZONE) + } + SRV::Backend(name, id) => { + write!(f, "_{}._tcp.{}.{}", name, id, DNS_ZONE) + } + } + } +} + +pub enum AAAA { + /// Identifies an AAAA record for a sled. + Sled(Uuid), + + /// Identifies an AAAA record for a zone within a sled. + Zone(Uuid), +} + +impl fmt::Display for AAAA { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self { + AAAA::Sled(id) => { + write!(f, "{}.sled.{}", id, DNS_ZONE) + } + AAAA::Zone(id) => { + write!(f, "{}.host.{}", id, DNS_ZONE) + } + } + } +} diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index d4147586332..df1af6cf331 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -16,6 +16,7 @@ base64 = "0.13.0" bb8 = "0.8.0" cookie = "0.16" crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "cd74a23ea42ce5e673923a00faf31b0a920191cc" } +db-macros = { path = "src/db/db-macros" } diesel = { version = "2.0.0-rc.0", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } diesel-dtrace = { git = "https://github.com/oxidecomputer/diesel-dtrace" } fatfs = "0.3.5" @@ -24,7 +25,7 @@ headers = "0.3.7" hex = "0.4.3" http = "0.2.7" hyper = "0.14" -db-macros = { path = "src/db/db-macros" } +internal-dns-client = { path = "../internal-dns-client" } ipnetwork = "0.18" lazy_static = "1.4.0" libc = "0.2.126" diff --git a/nexus/benches/setup_benchmark.rs b/nexus/benches/setup_benchmark.rs index c4c27bd2a97..24584670ce5 100644 --- a/nexus/benches/setup_benchmark.rs +++ b/nexus/benches/setup_benchmark.rs @@ -19,7 +19,7 @@ async fn do_full_setup() { // Wraps exclusively the CockroachDB portion of setup/teardown. async fn do_crdb_setup() { let cfg = nexus_test_utils::load_test_config(); - let logctx = LogContext::new("crdb_setup", &cfg.log); + let logctx = LogContext::new("crdb_setup", &cfg.pkg.log); let mut db = test_setup_database(&logctx.log).await; db.cleanup().await.unwrap(); } diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index 7900813cae0..22889ab1be9 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -2,9 +2,6 @@ # Oxide API: example configuration file # -# Identifier for this instance of Nexus -id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" - [console] # Directory for static assets. Absolute path or relative to CWD. static_dir = "nexus/static" # TODO: figure out value @@ -20,21 +17,6 @@ session_absolute_timeout_minutes = 480 # TODO(https://github.com/oxidecomputer/omicron/issues/372): Remove "spoof". schemes_external = ["spoof", "session_cookie"] -[database] -# URL for connecting to the database -url = "postgresql://root@127.0.0.1:32221/omicron?sslmode=disable" - -[dropshot_external] -# IP address and TCP port on which to listen for the external API -bind_address = "127.0.0.1:12220" -# Allow larger request bodies (1MiB) to accomodate firewall endpoints (one -# rule is ~500 bytes) -request_body_max_bytes = 1048576 - -[dropshot_internal] -# IP address and TCP port on which to listen for the internal API -bind_address = "127.0.0.1:12221" - [log] # Show log messages of this level and more severe level = "info" @@ -51,6 +33,29 @@ mode = "stderr-terminal" [timeseries_db] address = "[::1]:8123" +[runtime] +# Identifier for this instance of Nexus +id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" + +[runtime.dropshot_external] +# IP address and TCP port on which to listen for the external API +bind_address = "127.0.0.1:12220" +# Allow larger request bodies (1MiB) to accomodate firewall endpoints (one +# rule is ~500 bytes) +request_body_max_bytes = 1048576 + +[runtime.dropshot_internal] +# IP address and TCP port on which to listen for the internal API +bind_address = "127.0.0.1:12221" + +[runtime.subnet] +net = "fd00:1122:3344:0100::/56" + +[runtime.database] +# URL for connecting to the database +type = "from_url" +url = "postgresql://root@127.0.0.1:32221/omicron?sslmode=disable" + # Tunable configuration parameters, for testing or experimentation [tunables] diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index ce20065fa1f..1c3620de7e7 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -112,7 +112,7 @@ impl Nexus { authz: Arc, ) -> Arc { let pool = Arc::new(pool); - let my_sec_id = db::SecId::from(config.id); + let my_sec_id = db::SecId::from(config.runtime.id); let db_datastore = Arc::new(db::DataStore::new(Arc::clone(&pool))); let sec_store = Arc::new(db::CockroachDbSecStore::new( my_sec_id, @@ -127,7 +127,7 @@ impl Nexus { sec_store, )); let timeseries_client = - oximeter_db::Client::new(config.timeseries_db.address, &log); + oximeter_db::Client::new(config.pkg.timeseries_db.address, &log); // TODO-cleanup We may want a first-class subsystem for managing startup // background tasks. It could use a Future for each one, a status enum @@ -143,7 +143,7 @@ impl Nexus { populate_start(populate_ctx, Arc::clone(&db_datastore)); let nexus = Nexus { - id: config.id, + id: config.runtime.id, rack_id, log: log.new(o!()), api_rack_identity: db::model::RackIdentity::new(rack_id), @@ -153,8 +153,8 @@ impl Nexus { recovery_task: std::sync::Mutex::new(None), populate_status, timeseries_client, - updates_config: config.updates.clone(), - tunables: config.tunables.clone(), + updates_config: config.pkg.updates.clone(), + tunables: config.pkg.tunables.clone(), opctx_alloc: OpContext::for_background( log.new(o!("component" => "InstanceAllocator")), Arc::clone(&authz), diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index e7ae088640a..98f01920a60 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -9,6 +9,7 @@ use crate::db; use crate::db::identity::Asset; use crate::db::lookup::LookupPath; use crate::db::model::DatasetKind; +use crate::db::model::ServiceKind; use crate::internal_api::params::ZpoolPutRequest; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; @@ -142,4 +143,20 @@ impl super::Nexus { self.db_datastore.dataset_upsert(dataset).await?; Ok(()) } + + // Services + + /// Upserts a Service into the database, updating it if it already exists. + pub async fn upsert_service( + &self, + id: Uuid, + sled_id: Uuid, + address: SocketAddrV6, + kind: ServiceKind, + ) -> Result<(), Error> { + info!(self.log, "upserting service"; "sled_id" => sled_id.to_string(), "service_id" => id.to_string()); + let service = db::model::Service::new(id, sled_id, address, kind); + self.db_datastore.service_upsert(service).await?; + Ok(()) + } } diff --git a/nexus/src/config.rs b/nexus/src/config.rs index 11b2c8d861e..d5bf6a2a2f9 100644 --- a/nexus/src/config.rs +++ b/nexus/src/config.rs @@ -5,15 +5,13 @@ //! Interfaces for parsing configuration files and working with a nexus server //! configuration -use crate::db; use anyhow::anyhow; -use dropshot::ConfigDropshot; use dropshot::ConfigLogging; +use omicron_common::nexus_config::{InvalidTunable, LoadError, RuntimeConfig}; use serde::Deserialize; use serde::Serialize; use serde_with::DeserializeFromStr; use serde_with::SerializeDisplay; -use std::fmt; use std::net::SocketAddr; use std::path::{Path, PathBuf}; @@ -124,22 +122,15 @@ impl Default for Tunables { /// Configuration for a nexus server #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Config { - /// Dropshot configuration for external API server - pub dropshot_external: ConfigDropshot, - /// Dropshot configuration for internal API server - pub dropshot_internal: ConfigDropshot, - /// Identifier for this instance of Nexus - pub id: uuid::Uuid, +pub struct PackageConfig { /// Console-related tunables pub console: ConsoleConfig, /// Server-wide logging configuration. pub log: ConfigLogging, - /// Database parameters - pub database: db::Config, /// Authentication-related configuration pub authn: AuthnConfig, /// Timeseries database configuration. + // TODO: Should this be removed? Nexus needs to initialize it. pub timeseries_db: TimeseriesDbConfig, /// Updates-related configuration. Updates APIs return 400 Bad Request when this is /// unconfigured. @@ -150,74 +141,28 @@ pub struct Config { pub tunables: Tunables, } -#[derive(Debug)] -pub struct InvalidTunable { - tunable: String, - message: String, -} - -impl std::fmt::Display for InvalidTunable { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "invalid \"{}\": \"{}\"", self.tunable, self.message) - } -} - -impl std::error::Error for InvalidTunable {} - -#[derive(Debug)] -pub struct LoadError { - path: PathBuf, - kind: LoadErrorKind, -} -#[derive(Debug)] -pub enum LoadErrorKind { - Io(std::io::Error), - Parse(toml::de::Error), - InvalidTunable(InvalidTunable), -} - -impl From<(PathBuf, std::io::Error)> for LoadError { - fn from((path, err): (PathBuf, std::io::Error)) -> Self { - LoadError { path, kind: LoadErrorKind::Io(err) } - } -} - -impl From<(PathBuf, toml::de::Error)> for LoadError { - fn from((path, err): (PathBuf, toml::de::Error)) -> Self { - LoadError { path, kind: LoadErrorKind::Parse(err) } - } -} - -impl std::error::Error for LoadError {} +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub struct Config { + /// Configuration parameters known at compile-time. + #[serde(flatten)] + pub pkg: PackageConfig, -impl fmt::Display for LoadError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match &self.kind { - LoadErrorKind::Io(e) => { - write!(f, "read \"{}\": {}", self.path.display(), e) - } - LoadErrorKind::Parse(e) => { - write!(f, "parse \"{}\": {}", self.path.display(), e) - } - LoadErrorKind::InvalidTunable(inner) => { - write!( - f, - "invalid tunable \"{}\": {}", - self.path.display(), - inner, - ) - } - } - } + /// A variety of configuration parameters only known at runtime. + pub runtime: RuntimeConfig, } -impl std::cmp::PartialEq for LoadError { - fn eq(&self, other: &std::io::Error) -> bool { - if let LoadErrorKind::Io(e) = &self.kind { - e.kind() == other.kind() - } else { - false - } +impl Config { + /// Load a `PackageConfig` from the given TOML file + /// + /// This config object can then be used to create a new `Nexus`. + /// The format is described in the README. + pub fn from_file>(path: P) -> Result { + let path = path.as_ref(); + let file_contents = std::fs::read_to_string(path) + .map_err(|e| (path.to_path_buf(), e))?; + let config_parsed: Self = toml::from_str(&file_contents) + .map_err(|e| (path.to_path_buf(), e))?; + Ok(config_parsed) } } @@ -255,36 +200,24 @@ impl std::fmt::Display for SchemeName { } } -impl Config { - /// Load a `Config` from the given TOML file - /// - /// This config object can then be used to create a new `Nexus`. - /// The format is described in the README. - pub fn from_file>(path: P) -> Result { - let path = path.as_ref(); - let file_contents = std::fs::read_to_string(path) - .map_err(|e| (path.to_path_buf(), e))?; - let config_parsed: Config = toml::from_str(&file_contents) - .map_err(|e| (path.to_path_buf(), e))?; - Ok(config_parsed) - } -} - #[cfg(test)] mod test { use super::Tunables; use super::{ - AuthnConfig, Config, ConsoleConfig, LoadError, LoadErrorKind, + AuthnConfig, Config, ConsoleConfig, LoadError, PackageConfig, SchemeName, TimeseriesDbConfig, UpdatesConfig, }; - use crate::db; use dropshot::ConfigDropshot; use dropshot::ConfigLogging; use dropshot::ConfigLoggingIfExists; use dropshot::ConfigLoggingLevel; use libc; + use omicron_common::address::{Ipv6Subnet, RACK_PREFIX}; + use omicron_common::nexus_config::{ + Database, LoadErrorKind, RuntimeConfig, + }; use std::fs; - use std::net::SocketAddr; + use std::net::{Ipv6Addr, SocketAddr}; use std::path::Path; use std::path::PathBuf; @@ -355,7 +288,7 @@ mod test { let error = read_config("empty", "").expect_err("expected failure"); if let LoadErrorKind::Parse(error) = &error.kind { assert_eq!(error.line_col(), None); - assert_eq!(error.to_string(), "missing field `dropshot_external`"); + assert_eq!(error.to_string(), "missing field `runtime`"); } else { panic!( "Got an unexpected error, expected Parse but got {:?}", @@ -373,7 +306,6 @@ mod test { let config = read_config( "valid", r##" - id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" [console] static_dir = "tests/static" cache_control_max_age_minutes = 10 @@ -381,14 +313,6 @@ mod test { session_absolute_timeout_minutes = 480 [authn] schemes_external = [] - [dropshot_external] - bind_address = "10.1.2.3:4567" - request_body_max_bytes = 1024 - [dropshot_internal] - bind_address = "10.1.2.3:4568" - request_body_max_bytes = 1024 - [database] - url = "postgresql://127.0.0.1?sslmode=disable" [log] mode = "file" level = "debug" @@ -401,6 +325,18 @@ mod test { default_base_url = "http://example.invalid/" [tunables] max_vpc_ipv4_subnet_prefix = 27 + [runtime] + id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" + [runtime.dropshot_external] + bind_address = "10.1.2.3:4567" + request_body_max_bytes = 1024 + [runtime.dropshot_internal] + bind_address = "10.1.2.3:4568" + request_body_max_bytes = 1024 + [runtime.subnet] + net = "::/56" + [runtime.database] + type = "from_dns" "##, ) .unwrap(); @@ -408,51 +344,51 @@ mod test { assert_eq!( config, Config { - id: "28b90dc4-c22a-65ba-f49a-f051fe01208f".parse().unwrap(), - console: ConsoleConfig { - static_dir: "tests/static".parse().unwrap(), - cache_control_max_age_minutes: 10, - session_idle_timeout_minutes: 60, - session_absolute_timeout_minutes: 480 - }, - authn: AuthnConfig { schemes_external: Vec::new() }, - dropshot_external: ConfigDropshot { - bind_address: "10.1.2.3:4567" - .parse::() - .unwrap(), - ..Default::default() - }, - dropshot_internal: ConfigDropshot { - bind_address: "10.1.2.3:4568" - .parse::() - .unwrap(), - ..Default::default() - }, - log: ConfigLogging::File { - level: ConfigLoggingLevel::Debug, - if_exists: ConfigLoggingIfExists::Fail, - path: "/nonexistent/path".to_string() + runtime: RuntimeConfig { + id: "28b90dc4-c22a-65ba-f49a-f051fe01208f".parse().unwrap(), + dropshot_external: ConfigDropshot { + bind_address: "10.1.2.3:4567" + .parse::() + .unwrap(), + ..Default::default() + }, + dropshot_internal: ConfigDropshot { + bind_address: "10.1.2.3:4568" + .parse::() + .unwrap(), + ..Default::default() + }, + subnet: Ipv6Subnet::::new(Ipv6Addr::LOCALHOST), + database: Database::FromDns, }, - database: db::Config { - url: "postgresql://127.0.0.1?sslmode=disable" - .parse() - .unwrap() + pkg: PackageConfig { + console: ConsoleConfig { + static_dir: "tests/static".parse().unwrap(), + cache_control_max_age_minutes: 10, + session_idle_timeout_minutes: 60, + session_absolute_timeout_minutes: 480 + }, + authn: AuthnConfig { schemes_external: Vec::new() }, + log: ConfigLogging::File { + level: ConfigLoggingLevel::Debug, + if_exists: ConfigLoggingIfExists::Fail, + path: "/nonexistent/path".to_string() + }, + timeseries_db: TimeseriesDbConfig { + address: "[::1]:8123".parse().unwrap() + }, + updates: Some(UpdatesConfig { + trusted_root: PathBuf::from("/path/to/root.json"), + default_base_url: "http://example.invalid/".into(), + }), + tunables: Tunables { max_vpc_ipv4_subnet_prefix: 27 }, }, - timeseries_db: TimeseriesDbConfig { - address: "[::1]:8123".parse().unwrap() - }, - updates: Some(UpdatesConfig { - trusted_root: PathBuf::from("/path/to/root.json"), - default_base_url: "http://example.invalid/".into(), - }), - tunables: Tunables { max_vpc_ipv4_subnet_prefix: 27 }, } ); let config = read_config( "valid", r##" - id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" [console] static_dir = "tests/static" cache_control_max_age_minutes = 10 @@ -460,14 +396,6 @@ mod test { session_absolute_timeout_minutes = 480 [authn] schemes_external = [ "spoof", "session_cookie" ] - [dropshot_external] - bind_address = "10.1.2.3:4567" - request_body_max_bytes = 1024 - [dropshot_internal] - bind_address = "10.1.2.3:4568" - request_body_max_bytes = 1024 - [database] - url = "postgresql://127.0.0.1?sslmode=disable" [log] mode = "file" level = "debug" @@ -475,12 +403,24 @@ mod test { if_exists = "fail" [timeseries_db] address = "[::1]:8123" + [runtime] + id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" + [runtime.dropshot_external] + bind_address = "10.1.2.3:4567" + request_body_max_bytes = 1024 + [runtime.dropshot_internal] + bind_address = "10.1.2.3:4568" + request_body_max_bytes = 1024 + [runtime.subnet] + net = "::/56" + [runtime.database] + type = "from_dns" "##, ) .unwrap(); assert_eq!( - config.authn.schemes_external, + config.pkg.authn.schemes_external, vec![SchemeName::Spoof, SchemeName::SessionCookie], ); } @@ -490,7 +430,6 @@ mod test { let error = read_config( "bad authn.schemes_external", r##" - id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" [console] static_dir = "tests/static" cache_control_max_age_minutes = 10 @@ -498,14 +437,6 @@ mod test { session_absolute_timeout_minutes = 480 [authn] schemes_external = ["trust-me"] - [dropshot_external] - bind_address = "10.1.2.3:4567" - request_body_max_bytes = 1024 - [dropshot_internal] - bind_address = "10.1.2.3:4568" - request_body_max_bytes = 1024 - [database] - url = "postgresql://127.0.0.1?sslmode=disable" [log] mode = "file" level = "debug" @@ -513,14 +444,29 @@ mod test { if_exists = "fail" [timeseries_db] address = "[::1]:8123" + [runtime] + id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" + [runtime.dropshot_external] + bind_address = "10.1.2.3:4567" + request_body_max_bytes = 1024 + [runtime.dropshot_internal] + bind_address = "10.1.2.3:4568" + request_body_max_bytes = 1024 + [runtime.subnet] + net = "::/56" + [runtime.database] + type = "from_dns" "##, ) .expect_err("expected failure"); if let LoadErrorKind::Parse(error) = &error.kind { - assert!(error.to_string().starts_with( - "unsupported authn scheme: \"trust-me\" \ - for key `authn.schemes_external`" - )); + assert!( + error + .to_string() + .starts_with("unsupported authn scheme: \"trust-me\""), + "error = {}", + error.to_string() + ); } else { panic!( "Got an unexpected error, expected Parse but got {:?}", @@ -534,7 +480,6 @@ mod test { let error = read_config( "invalid_ipv4_prefix_tunable", r##" - id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" [console] static_dir = "tests/static" cache_control_max_age_minutes = 10 @@ -542,14 +487,6 @@ mod test { session_absolute_timeout_minutes = 480 [authn] schemes_external = [] - [dropshot_external] - bind_address = "10.1.2.3:4567" - request_body_max_bytes = 1024 - [dropshot_internal] - bind_address = "10.1.2.3:4568" - request_body_max_bytes = 1024 - [database] - url = "postgresql://127.0.0.1?sslmode=disable" [log] mode = "file" level = "debug" @@ -562,6 +499,18 @@ mod test { default_base_url = "http://example.invalid/" [tunables] max_vpc_ipv4_subnet_prefix = 100 + [runtime] + id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" + [runtime.dropshot_external] + bind_address = "10.1.2.3:4567" + request_body_max_bytes = 1024 + [runtime.dropshot_internal] + bind_address = "10.1.2.3:4568" + request_body_max_bytes = 1024 + [runtime.subnet] + net = "::/56" + [runtime.database] + type = "from_dns" "##, ) .expect_err("Expected failure"); diff --git a/nexus/src/context.rs b/nexus/src/context.rs index f0d9e6b13a0..61763eb7d6b 100644 --- a/nexus/src/context.rs +++ b/nexus/src/context.rs @@ -18,7 +18,12 @@ use authn::external::session_cookie::HttpAuthnSessionCookie; use authn::external::spoof::HttpAuthnSpoof; use authn::external::HttpAuthnScheme; use chrono::{DateTime, Duration, Utc}; +use omicron_common::address::{ + Ipv6Subnet, AZ_PREFIX, COCKROACH_DNS_NAME, COCKROACH_PORT, +}; use omicron_common::api::external::Error; +use omicron_common::nexus_config; +use omicron_common::postgres_config::PostgresConfigWithUrl; use oximeter::types::ProducerRegistry; use oximeter_instruments::http::{HttpService, LatencyTracker}; use slog::Logger; @@ -26,6 +31,7 @@ use std::collections::BTreeMap; use std::env; use std::fmt::Debug; use std::path::PathBuf; +use std::str::FromStr; use std::sync::Arc; use std::time::Instant; use std::time::SystemTime; @@ -67,13 +73,13 @@ pub struct ConsoleConfig { impl ServerContext { /// Create a new context with the given rack id and log. This creates the /// underlying nexus as well. - pub fn new( + pub async fn new( rack_id: Uuid, log: Logger, - pool: db::Pool, config: &config::Config, ) -> Result, String> { let nexus_schemes = config + .pkg .authn .schemes_external .iter() @@ -90,7 +96,8 @@ impl ServerContext { let internal_authn = Arc::new(authn::Context::internal_api()); let authz = Arc::new(authz::Authz::new(&log)); let create_tracker = |name: &str| { - let target = HttpService { name: name.to_string(), id: config.id }; + let target = + HttpService { name: name.to_string(), id: config.runtime.id }; const START_LATENCY_DECADE: i8 = -6; const END_LATENCY_DECADE: i8 = 3; LatencyTracker::with_latency_decades( @@ -102,7 +109,7 @@ impl ServerContext { }; let internal_latencies = create_tracker("nexus-internal"); let external_latencies = create_tracker("nexus-external"); - let producer_registry = ProducerRegistry::with_id(config.id); + let producer_registry = ProducerRegistry::with_id(config.runtime.id); producer_registry .register_producer(internal_latencies.clone()) .unwrap(); @@ -113,11 +120,11 @@ impl ServerContext { // Support both absolute and relative paths. If configured dir is // absolute, use it directly. If not, assume it's relative to the // current working directory. - let static_dir = if config.console.static_dir.is_absolute() { - Some(config.console.static_dir.to_owned()) + let static_dir = if config.pkg.console.static_dir.is_absolute() { + Some(config.pkg.console.static_dir.to_owned()) } else { env::current_dir() - .map(|root| root.join(&config.console.static_dir)) + .map(|root| root.join(&config.pkg.console.static_dir)) .ok() }; @@ -132,6 +139,36 @@ impl ServerContext { // like console index.html. leaving that out for now so we don't break // nexus in dev for everyone + // Set up DNS Client + let az_subnet = + Ipv6Subnet::::new(config.runtime.subnet.net().ip()); + info!(log, "Setting up resolver on subnet: {:?}", az_subnet); + let resolver = + internal_dns_client::multiclient::create_resolver(az_subnet) + .map_err(|e| format!("Failed to create DNS resolver: {}", e))?; + + // Set up DB pool + let url = match &config.runtime.database { + nexus_config::Database::FromUrl { url } => url.clone(), + nexus_config::Database::FromDns => { + info!(log, "Accessing DB url from DNS"); + let response = resolver + .lookup_ip(COCKROACH_DNS_NAME) + .await + .map_err(|e| format!("Failed to lookup IP: {}", e))?; + let address = response.iter().next().ok_or_else(|| { + "no addresses returned from DNS resolver".to_string() + })?; + info!(log, "DB addreess: {}", address); + PostgresConfigWithUrl::from_str(&format!( + "postgresql://root@[{}]:{}/omicron?sslmode=disable", + address, COCKROACH_PORT + )) + .map_err(|e| format!("Cannot parse Postgres URL: {}", e))? + } + }; + let pool = db::Pool::new(&db::Config { url }); + Ok(Arc::new(ServerContext { nexus: Nexus::new_with_id( rack_id, @@ -149,14 +186,14 @@ impl ServerContext { producer_registry, console_config: ConsoleConfig { session_idle_timeout: Duration::minutes( - config.console.session_idle_timeout_minutes.into(), + config.pkg.console.session_idle_timeout_minutes.into(), ), session_absolute_timeout: Duration::minutes( - config.console.session_absolute_timeout_minutes.into(), + config.pkg.console.session_absolute_timeout_minutes.into(), ), static_dir, cache_control_max_age: Duration::minutes( - config.console.cache_control_max_age_minutes.into(), + config.pkg.console.cache_control_max_age_minutes.into(), ), }, })) diff --git a/nexus/src/db/config.rs b/nexus/src/db/config.rs index b4066ce3cbe..afe51bca66d 100644 --- a/nexus/src/db/config.rs +++ b/nexus/src/db/config.rs @@ -4,7 +4,7 @@ //! Nexus database configuration -use omicron_common::config::PostgresConfigWithUrl; +use omicron_common::postgres_config::PostgresConfigWithUrl; use serde::Deserialize; use serde::Serialize; use serde_with::serde_as; diff --git a/nexus/src/db/datastore.rs b/nexus/src/db/datastore.rs index dc30987277a..2031d645392 100644 --- a/nexus/src/db/datastore.rs +++ b/nexus/src/db/datastore.rs @@ -39,7 +39,6 @@ use crate::db::fixed_data::silo::DEFAULT_SILO; use crate::db::lookup::LookupPath; use crate::db::model::DatabaseString; use crate::db::model::IncompleteVpc; -use crate::db::model::Vpc; use crate::db::queries::network_interface; use crate::db::queries::vpc::InsertVpcQuery; use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery; @@ -53,8 +52,8 @@ use crate::db::{ InstanceRuntimeState, Name, NetworkInterface, Organization, OrganizationUpdate, OximeterInfo, ProducerEndpoint, Project, ProjectUpdate, Region, RoleAssignment, RoleBuiltin, RouterRoute, - RouterRouteUpdate, Silo, SiloUser, Sled, SshKey, - UpdateAvailableArtifact, UserBuiltin, Volume, VpcFirewallRule, + RouterRouteUpdate, Service, Silo, SiloUser, Sled, SshKey, + UpdateAvailableArtifact, UserBuiltin, Volume, Vpc, VpcFirewallRule, VpcRouter, VpcRouterUpdate, VpcSubnet, VpcSubnetUpdate, VpcUpdate, Zpool, }, @@ -260,6 +259,46 @@ impl DataStore { }) } + /// Stores a new service in the database. + pub async fn service_upsert( + &self, + service: Service, + ) -> CreateResult { + use db::schema::service::dsl; + + let sled_id = service.sled_id; + Sled::insert_resource( + sled_id, + diesel::insert_into(dsl::service) + .values(service.clone()) + .on_conflict(dsl::id) + .do_update() + .set(( + dsl::time_modified.eq(Utc::now()), + dsl::sled_id.eq(excluded(dsl::sled_id)), + dsl::ip.eq(excluded(dsl::ip)), + dsl::port.eq(excluded(dsl::port)), + )), + ) + .insert_and_get_result_async(self.pool()) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { + type_name: ResourceType::Sled, + lookup_type: LookupType::ById(sled_id), + }, + AsyncInsertError::DatabaseError(e) => { + public_error_from_diesel_pool( + e, + ErrorHandler::Conflict( + ResourceType::Service, + &service.id().to_string(), + ), + ) + } + }) + } + fn get_allocated_regions_query( volume_id: Uuid, ) -> impl RunnableQuery<(Dataset, Region)> { @@ -3968,7 +4007,7 @@ mod test { // Test sled-specific IPv6 address allocation #[tokio::test] async fn test_sled_ipv6_address_allocation() { - use crate::db::model::STATIC_IPV6_ADDRESS_OFFSET; + use omicron_common::address::RSS_RESERVED_ADDRESSES as STATIC_IPV6_ADDRESS_OFFSET; use std::net::Ipv6Addr; let logctx = dev::test_setup_log("test_sled_ipv6_address_allocation"); diff --git a/nexus/src/db/model/mod.rs b/nexus/src/db/model/mod.rs index 552e39b549c..c6ab9155aa2 100644 --- a/nexus/src/db/model/mod.rs +++ b/nexus/src/db/model/mod.rs @@ -32,6 +32,8 @@ mod rack; mod region; mod role_assignment; mod role_builtin; +mod service; +mod service_kind; mod silo; mod silo_user; mod sled; @@ -78,6 +80,8 @@ pub use rack::*; pub use region::*; pub use role_assignment::*; pub use role_builtin::*; +pub use service::*; +pub use service_kind::*; pub use silo::*; pub use silo_user::*; pub use sled::*; diff --git a/nexus/src/db/model/service.rs b/nexus/src/db/model/service.rs new file mode 100644 index 00000000000..426f1e6aad5 --- /dev/null +++ b/nexus/src/db/model/service.rs @@ -0,0 +1,43 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::{ServiceKind, SqlU16}; +use crate::db::ipv6; +use crate::db::schema::service; +use db_macros::Asset; +use std::net::SocketAddrV6; +use uuid::Uuid; + +#[derive(Queryable, Insertable, Debug, Clone, Selectable, Asset)] +#[diesel(table_name = service)] +pub struct Service { + #[diesel(embed)] + identity: ServiceIdentity, + + // Sled to which this Zpool belongs. + pub sled_id: Uuid, + + // ServiceAddress (Sled Agent). + pub ip: ipv6::Ipv6Addr, + pub port: SqlU16, + + kind: ServiceKind, +} + +impl Service { + pub fn new( + id: Uuid, + sled_id: Uuid, + addr: SocketAddrV6, + kind: ServiceKind, + ) -> Self { + Self { + identity: ServiceIdentity::new(id), + sled_id, + ip: addr.ip().into(), + port: addr.port().into(), + kind, + } + } +} diff --git a/nexus/src/db/model/service_kind.rs b/nexus/src/db/model/service_kind.rs new file mode 100644 index 00000000000..6cd4e72d0db --- /dev/null +++ b/nexus/src/db/model/service_kind.rs @@ -0,0 +1,33 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::impl_enum_type; +use crate::internal_api; +use serde::{Deserialize, Serialize}; +use std::io::Write; + +impl_enum_type!( + #[derive(SqlType, Debug, QueryId)] + #[diesel(postgres_type(name = "service_kind"))] + pub struct ServiceKindEnum; + + #[derive(Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)] + #[diesel(sql_type = ServiceKindEnum)] + pub enum ServiceKind; + + // Enum values + Nexus => b"nexus" + Oximeter => b"oximeter" +); + +impl From for ServiceKind { + fn from(k: internal_api::params::ServiceKind) -> Self { + match k { + internal_api::params::ServiceKind::Nexus => ServiceKind::Nexus, + internal_api::params::ServiceKind::Oximeter => { + ServiceKind::Oximeter + } + } + } +} diff --git a/nexus/src/db/model/sled.rs b/nexus/src/db/model/sled.rs index 17e6bce01a9..ad756c3473f 100644 --- a/nexus/src/db/model/sled.rs +++ b/nexus/src/db/model/sled.rs @@ -5,7 +5,7 @@ use super::{Generation, SqlU16}; use crate::db::collection_insert::DatastoreCollection; use crate::db::ipv6; -use crate::db::schema::{sled, zpool}; +use crate::db::schema::{service, sled, zpool}; use chrono::{DateTime, Utc}; use db_macros::Asset; use std::net::Ipv6Addr; @@ -29,19 +29,11 @@ pub struct Sled { pub last_used_address: ipv6::Ipv6Addr, } -// TODO-correctness: We need a small offset here, while services and -// their addresses are still hardcoded in the mock RSS config file at -// `./smf/sled-agent/config-rss.toml`. This avoids conflicts with those -// addresses, but should be removed when they are entirely under the -// control of Nexus or RSS. -// -// See https://github.com/oxidecomputer/omicron/issues/732 for tracking issue. -pub(crate) const STATIC_IPV6_ADDRESS_OFFSET: u16 = 20; impl Sled { pub fn new(id: Uuid, addr: SocketAddrV6) -> Self { let last_used_address = { let mut segments = addr.ip().segments(); - segments[7] += STATIC_IPV6_ADDRESS_OFFSET; + segments[7] += omicron_common::address::RSS_RESERVED_ADDRESSES; ipv6::Ipv6Addr::from(Ipv6Addr::from(segments)) }; Self { @@ -73,3 +65,10 @@ impl DatastoreCollection for Sled { type CollectionTimeDeletedColumn = sled::dsl::time_deleted; type CollectionIdColumn = zpool::dsl::sled_id; } + +impl DatastoreCollection for Sled { + type CollectionId = Uuid; + type GenerationNumberColumn = sled::dsl::rcgen; + type CollectionTimeDeletedColumn = sled::dsl::time_deleted; + type CollectionIdColumn = service::dsl::sled_id; +} diff --git a/nexus/src/db/schema.rs b/nexus/src/db/schema.rs index 37a9164228c..c8f9b0c34a7 100644 --- a/nexus/src/db/schema.rs +++ b/nexus/src/db/schema.rs @@ -264,6 +264,21 @@ table! { } } +table! { + service (id) { + id -> Uuid, + time_created -> Timestamptz, + time_modified -> Timestamptz, + + sled_id -> Uuid, + + ip -> Inet, + port -> Int4, + + kind -> crate::db::model::ServiceKindEnum, + } +} + table! { zpool (id) { id -> Uuid, @@ -469,6 +484,7 @@ allow_tables_to_appear_in_same_query!( saga, saga_node_event, console_session, + service, sled, router_route, vpc, diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs index 4df80916d77..7dd9146df3f 100644 --- a/nexus/src/internal_api/http_entrypoints.rs +++ b/nexus/src/internal_api/http_entrypoints.rs @@ -7,8 +7,8 @@ use crate::context::OpContext; use crate::ServerContext; use super::params::{ - DatasetPutRequest, DatasetPutResponse, OximeterInfo, SledAgentStartupInfo, - ZpoolPutRequest, ZpoolPutResponse, + DatasetPutRequest, DatasetPutResponse, OximeterInfo, ServicePutRequest, + SledAgentStartupInfo, ZpoolPutRequest, ZpoolPutResponse, }; use dropshot::endpoint; use dropshot::ApiDescription; @@ -37,6 +37,7 @@ type NexusApiDescription = ApiDescription>; pub fn internal_api() -> NexusApiDescription { fn register_endpoints(api: &mut NexusApiDescription) -> Result<(), String> { api.register(cpapi_sled_agents_post)?; + api.register(service_put)?; api.register(zpool_put)?; api.register(dataset_put)?; api.register(cpapi_instances_put)?; @@ -87,6 +88,37 @@ async fn cpapi_sled_agents_post( apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await } +#[derive(Deserialize, JsonSchema)] +struct ServicePathParam { + sled_id: Uuid, + service_id: Uuid, +} + +/// Report that a service should be running on a sled. +#[endpoint { + method = PUT, + path = "/sled_agents/{sled_id}/services/{service_id}", + }] +async fn service_put( + rqctx: Arc>>, + path_params: Path, + info: TypedBody, +) -> Result { + let apictx = rqctx.context(); + let nexus = &apictx.nexus; + let path = path_params.into_inner(); + let info = info.into_inner(); + nexus + .upsert_service( + path.service_id, + path.sled_id, + info.address, + info.kind.into(), + ) + .await?; + Ok(HttpResponseUpdatedNoContent()) +} + /// Path parameters for Sled Agent requests (internal API) #[derive(Deserialize, JsonSchema)] struct ZpoolPathParam { diff --git a/nexus/src/internal_api/params.rs b/nexus/src/internal_api/params.rs index 365b43a81cc..d10946bdc34 100644 --- a/nexus/src/internal_api/params.rs +++ b/nexus/src/internal_api/params.rs @@ -98,6 +98,50 @@ pub struct DatasetPutResponse { pub quota: Option, } +/// Describes the purpose of the service. +#[derive(Debug, Serialize, Deserialize, JsonSchema, Clone, Copy, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ServiceKind { + Nexus, + Oximeter, +} + +impl fmt::Display for ServiceKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use ServiceKind::*; + let s = match self { + Nexus => "nexus", + Oximeter => "oximeter", + }; + write!(f, "{}", s) + } +} + +impl FromStr for ServiceKind { + type Err = omicron_common::api::external::Error; + + fn from_str(s: &str) -> Result { + use ServiceKind::*; + match s { + "nexus" => Ok(Nexus), + "oximeter" => Ok(Oximeter), + _ => Err(Self::Err::InternalError { + internal_message: format!("Unknown service kind: {}", s), + }), + } + } +} + +/// Describes a service on a sled +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct ServicePutRequest { + /// Address on which a service is responding to requests. + pub address: SocketAddrV6, + + /// Type of service being inserted. + pub kind: ServiceKind, +} + /// Message used to notify Nexus that this oximeter instance is up and running. #[derive(Debug, Clone, Copy, JsonSchema, Serialize, Deserialize)] pub struct OximeterInfo { diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index e56503c3c09..61abe04b1ba 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -29,7 +29,7 @@ pub mod updates; // public for testing pub use app::test_interfaces::TestInterfaces; pub use app::Nexus; -pub use config::Config; +pub use config::{Config, PackageConfig}; pub use context::ServerContext; pub use crucible_agent_client; use external_api::http_entrypoints::external_api; @@ -85,15 +85,15 @@ impl Server { rack_id: Uuid, log: &Logger, ) -> Result { - let log = log.new(o!("name" => config.id.to_string())); + let log = log.new(o!("name" => config.runtime.id.to_string())); info!(log, "setting up nexus server"); let ctxlog = log.new(o!("component" => "ServerContext")); - let pool = db::Pool::new(&config.database); - let apictx = ServerContext::new(rack_id, ctxlog, pool, &config)?; + + let apictx = ServerContext::new(rack_id, ctxlog, &config).await?; let http_server_starter_external = dropshot::HttpServerStarter::new( - &config.dropshot_external, + &config.runtime.dropshot_external, external_api(), Arc::clone(&apictx), &log.new(o!("component" => "dropshot_external")), @@ -101,7 +101,7 @@ impl Server { .map_err(|error| format!("initializing external server: {}", error))?; let http_server_starter_internal = dropshot::HttpServerStarter::new( - &config.dropshot_internal, + &config.runtime.dropshot_internal, internal_api(), Arc::clone(&apictx), &log.new(o!("component" => "dropshot_internal")), @@ -153,12 +153,12 @@ impl Server { /// Run an instance of the [Server]. pub async fn run_server(config: &Config) -> Result<(), String> { use slog::Drain; - let (drain, registration) = slog_dtrace::with_drain( - config - .log - .to_logger("nexus") - .map_err(|message| format!("initializing logger: {}", message))?, - ); + let (drain, registration) = + slog_dtrace::with_drain( + config.pkg.log.to_logger("nexus").map_err(|message| { + format!("initializing logger: {}", message) + })?, + ); let log = slog::Logger::root(drain.fuse(), slog::o!()); if let slog_dtrace::ProbeRegistration::Failed(e) = registration { let msg = format!("failed to register DTrace probes: {}", e); diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index a53ad85d585..e4eb744e2fa 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -11,6 +11,7 @@ use dropshot::ConfigLogging; use dropshot::ConfigLoggingLevel; use omicron_common::api::external::IdentityMetadata; use omicron_common::api::internal::nexus::ProducerEndpoint; +use omicron_common::nexus_config; use omicron_sled_agent::sim; use omicron_test_utils::dev; use oximeter_collector::Oximeter; @@ -75,7 +76,7 @@ pub fn load_test_config() -> omicron_nexus::Config { let config_file_path = Path::new("tests/config.test.toml"); let mut config = omicron_nexus::Config::from_file(config_file_path) .expect("failed to load config.test.toml"); - config.id = Uuid::new_v4(); + config.runtime.id = Uuid::new_v4(); config } @@ -88,7 +89,7 @@ pub async fn test_setup_with_config( test_name: &str, config: &mut omicron_nexus::Config, ) -> ControlPlaneTestContext { - let logctx = LogContext::new(test_name, &config.log); + let logctx = LogContext::new(test_name, &config.pkg.log); let rack_id = Uuid::parse_str(RACK_UUID).unwrap(); let log = &logctx.log; @@ -99,8 +100,9 @@ pub async fn test_setup_with_config( let clickhouse = dev::clickhouse::ClickHouseInstance::new(0).await.unwrap(); // Store actual address/port information for the databases after they start. - config.database.url = database.pg_config().clone(); - config.timeseries_db.address.set_port(clickhouse.port()); + config.runtime.database = + nexus_config::Database::FromUrl { url: database.pg_config().clone() }; + config.pkg.timeseries_db.address.set_port(clickhouse.port()); let server = omicron_nexus::Server::start(&config, rack_id, &logctx.log) .await diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 9b8f1f42731..2fc4ddba192 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -2,10 +2,6 @@ # Oxide API: configuration file for test suite # -# Identifier for this instance of Nexus. -# NOTE: The test suite always overrides this. -id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" - [console] # Directory for static assets. Absolute path or relative to CWD. static_dir = "tests/static" @@ -17,27 +13,6 @@ session_absolute_timeout_minutes = 480 [authn] schemes_external = [ "spoof", "session_cookie" ] -# -# NOTE: for the test suite, the database URL will be replaced with one -# appropriate for the database that's started by the test runner. -# -[database] -url = "postgresql://root@127.0.0.1:0/omicron?sslmode=disable" - -# -# NOTE: for the test suite, the port MUST be 0 (in order to bind to any -# available port) because the test suite will be running many servers -# concurrently. -# -[dropshot_external] -bind_address = "127.0.0.1:0" -request_body_max_bytes = 1048576 - -# port must be 0. see above -[dropshot_internal] -bind_address = "127.0.0.1:0" -request_body_max_bytes = 1048576 - # # NOTE: for the test suite, if mode = "file", the file path MUST be the sentinel # string "UNUSED". The actual path will be generated by the test suite for each @@ -59,3 +34,33 @@ address = "[::1]:0" [tunables] # Allow small subnets, so we can test IP address exhaustion easily / quickly max_vpc_ipv4_subnet_prefix = 29 + +[runtime] +# Identifier for this instance of Nexus. +# NOTE: The test suite always overrides this. +id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" + +# +# NOTE: for the test suite, the port MUST be 0 (in order to bind to any +# available port) because the test suite will be running many servers +# concurrently. +# +[runtime.dropshot_external] +bind_address = "127.0.0.1:0" +request_body_max_bytes = 1048576 + +# port must be 0. see above +[runtime.dropshot_internal] +bind_address = "127.0.0.1:0" +request_body_max_bytes = 1048576 + +[runtime.subnet] +net = "fd00:1122:3344:0100::/56" + +# +# NOTE: for the test suite, the database URL will be replaced with one +# appropriate for the database that's started by the test runner. +# +[runtime.database] +type = "from_url" +url = "postgresql://root@127.0.0.1:0/omicron?sslmode=disable" diff --git a/nexus/tests/integration_tests/authn_http.rs b/nexus/tests/integration_tests/authn_http.rs index 7125a52ea90..e0234da1b97 100644 --- a/nexus/tests/integration_tests/authn_http.rs +++ b/nexus/tests/integration_tests/authn_http.rs @@ -277,7 +277,7 @@ async fn start_whoami_server( sessions: HashMap, ) -> TestContext { let config = nexus_test_utils::load_test_config(); - let logctx = LogContext::new(test_name, &config.log); + let logctx = LogContext::new(test_name, &config.pkg.log); let whoami_api = { let mut whoami_api = ApiDescription::new(); @@ -299,7 +299,7 @@ async fn start_whoami_server( TestContext::new( whoami_api, server_state, - &config.dropshot_external, + &config.runtime.dropshot_external, Some(logctx), log, ) diff --git a/nexus/tests/integration_tests/commands.rs b/nexus/tests/integration_tests/commands.rs index 7d3855d5a6c..ac770c137e3 100644 --- a/nexus/tests/integration_tests/commands.rs +++ b/nexus/tests/integration_tests/commands.rs @@ -76,8 +76,7 @@ fn test_nexus_invalid_config() { assert_eq!( stderr_text, format!( - "nexus: parse \"{}\": missing field \ - `dropshot_external`\n", + "nexus: parse \"{}\": missing field `runtime`\n", config_path.display() ), ); diff --git a/nexus/tests/integration_tests/console_api.rs b/nexus/tests/integration_tests/console_api.rs index e84c65c0fe1..779e94470eb 100644 --- a/nexus/tests/integration_tests/console_api.rs +++ b/nexus/tests/integration_tests/console_api.rs @@ -196,7 +196,7 @@ async fn test_assets(cptestctx: &ControlPlaneTestContext) { #[tokio::test] async fn test_absolute_static_dir() { let mut config = load_test_config(); - config.console.static_dir = current_dir().unwrap().join("tests/static"); + config.pkg.console.static_dir = current_dir().unwrap().join("tests/static"); let cptestctx = test_setup_with_config("test_absolute_static_dir", &mut config).await; let testctx = &cptestctx.external_client; diff --git a/nexus/tests/integration_tests/updates.rs b/nexus/tests/integration_tests/updates.rs index 1bfa25d0a2c..c09ca0b7fea 100644 --- a/nexus/tests/integration_tests/updates.rs +++ b/nexus/tests/integration_tests/updates.rs @@ -62,7 +62,7 @@ async fn test_update_end_to_end() { let mut api = ApiDescription::new(); api.register(static_content).unwrap(); let context = FileServerContext { base: tuf_repo.path().to_owned() }; - let logctx = LogContext::new("test_update_end_to_end", &config.log); + let logctx = LogContext::new("test_update_end_to_end", &config.pkg.log); let server = HttpServerStarter::new(&dropshot_config, api, context, &logctx.log) .unwrap() @@ -70,7 +70,7 @@ async fn test_update_end_to_end() { let local_addr = server.local_addr(); // stand up the test environment - config.updates = Some(UpdatesConfig { + config.pkg.updates = Some(UpdatesConfig { trusted_root: tuf_repo.path().join("metadata").join("1.root.json"), default_base_url: format!("http://{}/", local_addr), }); diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 4fc85bf1bb6..411a82a40db 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -267,6 +267,55 @@ } } }, + "/sled_agents/{sled_id}/services/{service_id}": { + "put": { + "summary": "Report that a service should be running on a sled.", + "operationId": "service_put", + "parameters": [ + { + "in": "path", + "name": "service_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + }, + "style": "simple" + }, + { + "in": "path", + "name": "sled_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + }, + "style": "simple" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServicePutRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/sled_agents/{sled_id}/zpools/{zpool_id}": { "put": { "summary": "Report that a pool for a specified sled has come online.", @@ -1691,6 +1740,36 @@ "timeseries_name" ] }, + "ServiceKind": { + "description": "Describes the purpose of the service.", + "type": "string", + "enum": [ + "nexus", + "oximeter" + ] + }, + "ServicePutRequest": { + "description": "Describes a service on a sled", + "type": "object", + "properties": { + "address": { + "description": "Address on which a service is responding to requests.", + "type": "string" + }, + "kind": { + "description": "Type of service being inserted.", + "allOf": [ + { + "$ref": "#/components/schemas/ServiceKind" + } + ] + } + }, + "required": [ + "address", + "kind" + ] + }, "SledAgentStartupInfo": { "description": "Sent by a sled agent on startup to Nexus to request further instruction", "type": "object", diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 273082a7500..51e2d20f9ab 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -247,6 +247,10 @@ "dataset_kind": { "$ref": "#/components/schemas/DatasetKind" }, + "id": { + "type": "string", + "format": "uuid" + }, "zpool_id": { "type": "string", "format": "uuid" @@ -255,6 +259,7 @@ "required": [ "address", "dataset_kind", + "id", "zpool_id" ] }, @@ -959,6 +964,7 @@ ] }, "ServiceRequest": { + "description": "Describes a request to create a service. This information should be sufficient for a Sled Agent to start a zone containing the requested service.", "type": "object", "properties": { "addresses": { @@ -976,13 +982,71 @@ "format": "ipv6" } }, + "id": { + "type": "string", + "format": "uuid" + }, "name": { "type": "string" + }, + "service_type": { + "$ref": "#/components/schemas/ServiceType" } }, "required": [ "addresses", - "name" + "id", + "name", + "service_type" + ] + }, + "ServiceType": { + "description": "Describes service-specific parameters.", + "oneOf": [ + { + "type": "object", + "properties": { + "external_address": { + "type": "string" + }, + "internal_address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "nexus" + ] + } + }, + "required": [ + "external_address", + "internal_address", + "type" + ] + }, + { + "type": "object", + "properties": { + "dns_address": { + "type": "string" + }, + "server_address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "internal_dns" + ] + } + }, + "required": [ + "dns_address", + "server_address", + "type" + ] + } ] }, "Slot": { @@ -1144,4 +1208,4 @@ } } } -} \ No newline at end of file +} diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index a2b322bc1c9..540981987d8 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -16,6 +16,7 @@ chrono = { version = "0.4", features = [ "serde" ] } crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "cd74a23ea42ce5e673923a00faf31b0a920191cc" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } futures = "0.3.21" +internal-dns-client = { path = "../internal-dns-client" } ipnetwork = "0.18" libc = "0.2.126" macaddr = { version = "1.0.1", features = [ "serde_std" ] } diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 1c713a69067..a371bcec035 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -9,9 +9,7 @@ use omicron_common::api::internal::nexus::{ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display, Formatter, Result as FormatResult}; -use std::net::IpAddr; -use std::net::Ipv6Addr; -use std::net::{SocketAddr, SocketAddrV6}; +use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; use uuid::Uuid; /// Information required to construct a virtual network interface for a guest @@ -165,7 +163,7 @@ pub struct InstanceRuntimeStateRequested { pub enum DatasetKind { CockroachDb { /// The addresses of all nodes within the cluster. - all_addresses: Vec, + all_addresses: Vec, }, Crucible, Clickhouse, @@ -213,6 +211,8 @@ impl std::fmt::Display for DatasetKind { /// instantiated when the dataset is detected. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] pub struct DatasetEnsureBody { + // The UUID of the dataset, as well as the service using it directly. + pub id: Uuid, // The name (and UUID) of the Zpool which we are inserting into. pub zpool_id: Uuid, // The type of the filesystem. @@ -235,14 +235,51 @@ impl From for sled_agent_client::types::DatasetEnsureBody { zpool_id: p.zpool_id, dataset_kind: p.dataset_kind.into(), address: p.address.to_string(), + id: p.id, } } } +/// Describes service-specific parameters. +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +// Struct variant enums require some assistance for serialization to TOML. +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ServiceType { + Nexus { internal_address: SocketAddrV6, external_address: SocketAddrV6 }, + InternalDns { server_address: SocketAddrV6, dns_address: SocketAddrV6 }, +} + +impl From for sled_agent_client::types::ServiceType { + fn from(s: ServiceType) -> Self { + use sled_agent_client::types::ServiceType as AutoSt; + use ServiceType as St; + + match s { + St::Nexus { internal_address, external_address } => AutoSt::Nexus { + internal_address: internal_address.to_string(), + external_address: external_address.to_string(), + }, + St::InternalDns { server_address, dns_address } => { + AutoSt::InternalDns { + server_address: server_address.to_string(), + dns_address: dns_address.to_string(), + } + } + } + } +} + +/// Describes a request to create a service. This information +/// should be sufficient for a Sled Agent to start a zone +/// containing the requested service. #[derive( Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] pub struct ServiceRequest { + // The UUID of the service to be initialized. + pub id: Uuid, // The name of the service to be created. pub name: String, // The addresses on which the service should listen for requests. @@ -256,14 +293,18 @@ pub struct ServiceRequest { // is necessary to allow inter-zone traffic routing. #[serde(default)] pub gz_addresses: Vec, + // Any other service-specific parameters. + pub service_type: ServiceType, } impl From for sled_agent_client::types::ServiceRequest { fn from(s: ServiceRequest) -> Self { Self { + id: s.id, name: s.name, addresses: s.addresses, gz_addresses: s.gz_addresses, + service_type: s.service_type.into(), } } } diff --git a/sled-agent/src/rack_setup/config.rs b/sled-agent/src/rack_setup/config.rs index 26f3ce8a321..dfb24de1de0 100644 --- a/sled-agent/src/rack_setup/config.rs +++ b/sled-agent/src/rack_setup/config.rs @@ -5,7 +5,6 @@ //! Interfaces for working with RSS config. use crate::config::ConfigError; -use crate::params::{DatasetEnsureBody, ServiceRequest}; use omicron_common::address::{ get_64_subnet, Ipv6Subnet, AZ_PREFIX, RACK_PREFIX, SLED_PREFIX, }; @@ -13,6 +12,7 @@ use serde::Deserialize; use serde::Serialize; use std::net::Ipv6Addr; use std::path::Path; +use uuid::Uuid; /// Configuration for the "rack setup service", which is controlled during /// bootstrap. @@ -29,23 +29,23 @@ pub struct SetupServiceConfig { pub rack_subnet: Ipv6Addr, #[serde(default, rename = "request")] - pub requests: Vec, + pub requests: Vec, +} + +/// Hard-coded configurations for where to place CRDB datasets. +/// +/// Converts into a [`crate::params::DatasetEnsureBody`]. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct CockroachDataset { + pub zpool_uuid: Uuid, } /// A request to initialize a sled. #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] -pub struct SledRequest { +pub struct HardcodedSledRequest { /// Datasets to be created. #[serde(default, rename = "dataset")] - pub datasets: Vec, - - /// Services to be instantiated. - #[serde(default, rename = "service")] - pub services: Vec, - - /// DNS Services to be instantiated. - #[serde(default, rename = "dns_service")] - pub dns_services: Vec, + pub datasets: Vec, } impl SetupServiceConfig { diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 0fef7054d26..2796be2d79c 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -4,13 +4,16 @@ //! Rack Setup Service implementation -use super::config::{SetupServiceConfig as Config, SledRequest}; -use crate::bootstrap::config::BOOTSTRAP_AGENT_PORT; -use crate::bootstrap::discovery::PeerMonitorObserver; -use crate::bootstrap::params::SledAgentRequest; -use crate::bootstrap::rss_handle::BootstrapAgentHandle; -use crate::params::ServiceRequest; -use omicron_common::address::{get_sled_address, ReservedRackSubnet}; +use super::config::SetupServiceConfig as Config; +use crate::bootstrap::{ + config::BOOTSTRAP_AGENT_PORT, discovery::PeerMonitorObserver, + params::SledAgentRequest, rss_handle::BootstrapAgentHandle, +}; +use crate::params::{DatasetEnsureBody, ServiceRequest, ServiceType}; +use omicron_common::address::{ + get_sled_address, ReservedRackSubnet, DNS_PORT, DNS_SERVER_PORT, + NEXUS_EXTERNAL_PORT, NEXUS_INTERNAL_PORT, RSS_RESERVED_ADDRESSES, +}; use omicron_common::backoff::{ internal_service_policy, retry_notify, BackoffError, }; @@ -20,7 +23,11 @@ use std::collections::{HashMap, HashSet}; use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use std::path::PathBuf; use thiserror::Error; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, OnceCell}; +use uuid::Uuid; + +// The number of Nexus instances to create from RSS. +const NEXUS_COUNT: usize = 1; /// Describes errors which may occur while operating the setup service. #[derive(Error, Debug)] @@ -49,6 +56,25 @@ pub enum SetupServiceError { #[error("Failed to construct a sprockets proxy: {0}")] SprocketsProxy(#[from] sprockets_proxy::Error), + + // XXX CLEAN UP + #[error(transparent)] + Dns(#[from] internal_dns_client::Error), +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct SledRequest { + /// Datasets to be created. + #[serde(default, rename = "dataset")] + pub datasets: Vec, + + /// Services to be instantiated. + #[serde(default, rename = "service")] + pub services: Vec, + + /// DNS Services to be instantiated. + #[serde(default, rename = "dns_service")] + pub dns_services: Vec, } // The workload / information allocated to a single sled. @@ -130,15 +156,42 @@ enum PeerExpectation { CreateNewPlan(usize), } +struct AddressBumpAllocator { + last_addr: Ipv6Addr, +} + +// TODO: Testable? +// TODO: Could exist in another file? +impl AddressBumpAllocator { + fn new(sled_addr: Ipv6Addr) -> Self { + Self { last_addr: sled_addr } + } + + fn next(&mut self) -> Option { + let mut segments: [u16; 8] = self.last_addr.segments(); + segments[7] = segments[7].checked_add(1)?; + if segments[7] > RSS_RESERVED_ADDRESSES { + return None; + } + self.last_addr = Ipv6Addr::from(segments); + Some(self.last_addr) + } +} + /// The implementation of the Rack Setup Service. struct ServiceInner { log: Logger, peer_monitor: Mutex, + dns_servers: OnceCell, } impl ServiceInner { fn new(log: Logger, peer_monitor: PeerMonitorObserver) -> Self { - ServiceInner { log, peer_monitor: Mutex::new(peer_monitor) } + ServiceInner { + log, + peer_monitor: Mutex::new(peer_monitor), + dns_servers: OnceCell::new(), + } } async fn initialize_datasets( @@ -186,13 +239,44 @@ impl ServiceInner { ) .await?; } + + // CRDB datasets are treated as services. + // + // XXX: Hardcoding CRDB? + let crdb_datasets = datasets.iter().filter(|dataset| { + matches!( + dataset.dataset_kind, + crate::params::DatasetKind::CockroachDb { .. } + ) + }); + + let aaaa = crdb_datasets + .map(|dataset| { + ( + internal_dns_client::names::AAAA::Zone(dataset.id), + dataset.address, + ) + }) + .collect::>(); + let srv_key = + internal_dns_client::names::SRV::Service("cockroachdb".into()); + + self.dns_servers + .get() + .expect("DNS servers must be initialized first") + .insert_dns_records(&self.log, aaaa, srv_key) + .await?; + + // TODO: add dns records for non-crdb datasets too + // TODO: alternatively, REMOVE THEM! Make RSS set up crdb exclusively. + Ok(()) } async fn initialize_services( &self, sled_address: SocketAddr, - services: &Vec, + services: &Vec, ) -> Result<(), SetupServiceError> { let dur = std::time::Duration::from_secs(60); let client = reqwest::ClientBuilder::new() @@ -277,25 +361,83 @@ impl ServiceInner { let requests_and_sleds = bootstrap_addrs.map(|(idx, bootstrap_addr)| { - // If a sled was explicitly requested from the RSS configuration, - // use that. Otherwise, just give it a "default" (empty) set of - // services. - let mut request = { - if idx < config.requests.len() { - config.requests[idx].clone() - } else { - SledRequest::default() + let sled_subnet_index = + u8::try_from(idx + 1).expect("Too many peers!"); + let subnet = config.sled_subnet(sled_subnet_index); + let mut addr_alloc = + AddressBumpAllocator::new(*get_sled_address(subnet).ip()); + + let mut request = SledRequest::default(); + + // The first enumerated sleds get assigned the responsibility + // of hosting Nexus. + if idx < NEXUS_COUNT { + let address = addr_alloc.next().expect("Not enough addrs"); + request.services.push(ServiceRequest { + id: Uuid::new_v4(), + name: "nexus".to_string(), + addresses: vec![address], + gz_addresses: vec![], + service_type: ServiceType::Nexus { + internal_address: SocketAddrV6::new( + address, + NEXUS_INTERNAL_PORT, + 0, + 0, + ), + external_address: SocketAddrV6::new( + address, + NEXUS_EXTERNAL_PORT, + 0, + 0, + ), + }, + }) + } + + // The first enumerated sleds host the CRDB datasets, using + // zpools described from the underlying config file. + if idx < config.requests.len() { + for dataset in &config.requests[idx].datasets { + let address = SocketAddrV6::new( + addr_alloc.next().expect("Not enough addrs"), + omicron_common::address::COCKROACH_PORT, + 0, + 0, + ); + request.datasets.push(DatasetEnsureBody { + id: Uuid::new_v4(), + zpool_id: dataset.zpool_uuid, + dataset_kind: + crate::params::DatasetKind::CockroachDb { + all_addresses: vec![address], + }, + address, + }); } - }; + } - // The first enumerated addresses get assigned the additional + // The first enumerated sleds get assigned the additional // responsibility of being internal DNS servers. if idx < dns_subnets.len() { let dns_subnet = &dns_subnets[idx]; + let dns_addr = dns_subnet.dns_address().ip(); request.dns_services.push(ServiceRequest { + id: Uuid::new_v4(), name: "internal-dns".to_string(), - addresses: vec![dns_subnet.dns_address().ip()], + addresses: vec![dns_addr], gz_addresses: vec![dns_subnet.gz_address().ip()], + service_type: ServiceType::InternalDns { + server_address: SocketAddrV6::new( + dns_addr, + DNS_SERVER_PORT, + 0, + 0, + ), + dns_address: SocketAddrV6::new( + dns_addr, DNS_PORT, 0, 0, + ), + }, }); } @@ -331,8 +473,10 @@ impl ServiceInner { } // Once we've constructed a plan, write it down to durable storage. - let serialized_plan = toml::Value::try_from(&plan) - .expect("Cannot serialize configuration"); + let serialized_plan = + toml::Value::try_from(&plan).unwrap_or_else(|e| { + panic!("Cannot serialize configuration: {:#?}: {}", plan, e) + }); let plan_str = toml::to_string(&serialized_plan) .expect("Cannot turn config to string"); @@ -502,6 +646,33 @@ impl ServiceInner { .into_iter() .collect::>()?; + let dns_servers = internal_dns_client::multiclient::Updater::new( + config.az_subnet(), + self.log.new(o!("client" => "DNS")), + ); + self.dns_servers + .set(dns_servers) + .map_err(|_| ()) + .expect("Already set DNS servers"); + + // XXX Test record insertion + /* + insert_dns_record( + &self.log, + &dns_servers, + "hello.world", + Ipv6Addr::new(0xfd, 0, 0, 0, 0, 0, 0, 0x1), + ).await?; + + // XXX test record retreival + + let resolver = internal_dns_client::multiclient::create_resolver(config.az_subnet()) + .expect("Failed to create DNS resolver"); + let response = resolver.lookup_ip(name.to_owned() + ".").await.expect("Failed to lookup IP"); + let address = response.iter().next().expect("no addresses returned from DNS resolver"); + assert_eq!(address, addr); + */ + // Issue the dataset initialization requests to all sleds. futures::future::join_all(plan.iter().map( |(_, allocation)| async move { diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 72444a79b17..ac10da4732d 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -7,17 +7,30 @@ use crate::illumos::dladm::{Etherstub, EtherstubVnic}; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; +use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::zone::AddressRequest; -use crate::params::{ServiceEnsureBody, ServiceRequest}; +use crate::params::{ServiceEnsureBody, ServiceRequest, ServiceType}; use crate::zone::Zones; -use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT}; +use dropshot::ConfigDropshot; +use omicron_common::address::{Ipv6Subnet, RACK_PREFIX}; +use omicron_common::nexus_config::{self, RuntimeConfig as NexusRuntimeConfig}; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; -use std::net::{IpAddr, Ipv6Addr}; +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; use std::path::{Path, PathBuf}; +use tokio::io::AsyncWriteExt; use tokio::sync::Mutex; +// The filename of ServiceManager's internal storage. +const SERVICE_CONFIG_FILENAME: &str = "service.toml"; +// The filename of a half-completed config, in need of parameters supplied at +// runtime. +const PARTIAL_CONFIG_FILENAME: &str = "config-partial.toml"; +// The filename of a completed config, merging the partial config with +// additional appended parameters known at runtime. +const COMPLETE_CONFIG_FILENAME: &str = "config.toml"; + #[derive(thiserror::Error, Debug)] pub enum Error { #[error("Cannot serialize TOML to file {path}: {err}")] @@ -69,13 +82,40 @@ impl From for omicron_common::api::external::Error { /// The default path to service configuration, if one is not /// explicitly provided. pub fn default_services_config_path() -> PathBuf { - Path::new(omicron_common::OMICRON_CONFIG_PATH).join("services.toml") + Path::new(omicron_common::OMICRON_CONFIG_PATH).join(SERVICE_CONFIG_FILENAME) +} + +/// Configuration parameters which modify the [`ServiceManager`]'s behavior. +/// +/// These are typically used to make testing easier; production usage +/// should generally prefer to use the defaults. +pub struct Config { + /// The path for the ServiceManager to store information about + /// all running services. + pub all_svcs_config_path: PathBuf, + /// A function which returns the path the directory holding the + /// service's configuration file. + pub get_svc_config_dir: Box PathBuf + Send + Sync>, +} + +impl Default for Config { + fn default() -> Self { + Self { + all_svcs_config_path: default_services_config_path(), + get_svc_config_dir: Box::new(|zone_name: &str, svc_name: &str| { + PathBuf::from(ZONE_ZFS_DATASET_MOUNTPOINT) + .join(PathBuf::from(zone_name)) + .join("root") + .join(format!("var/svc/manifest/site/{}", svc_name)) + }), + } + } } /// Manages miscellaneous Sled-local services. pub struct ServiceManager { log: Logger, - config_path: Option, + config: Config, zones: Mutex>, vnic_allocator: VnicAllocator, underlay_vnic: EtherstubVnic, @@ -98,12 +138,12 @@ impl ServiceManager { etherstub: Etherstub, underlay_vnic: EtherstubVnic, underlay_address: Ipv6Addr, - config_path: Option, + config: Config, ) -> Result { debug!(log, "Creating new ServiceManager"); let mgr = Self { log: log.new(o!("component" => "ServiceManager")), - config_path, + config, zones: Mutex::new(vec![]), vnic_allocator: VnicAllocator::new("Service", etherstub), underlay_vnic, @@ -143,11 +183,7 @@ impl ServiceManager { // Returns either the path to the explicitly provided config path, or // chooses the default one. fn services_config_path(&self) -> PathBuf { - if let Some(path) = &self.config_path { - path.clone() - } else { - default_services_config_path() - } + self.config.all_svcs_config_path.clone() } // Populates `existing_zones` according to the requests in `services`. @@ -268,16 +304,65 @@ impl ServiceManager { let smf_name = format!("svc:/system/illumos/{}", service.name); let default_smf_name = format!("{}:default", smf_name); - match service.name.as_str() { - "internal-dns" => { - info!(self.log, "Setting up internal-dns service"); - let address = - service.addresses.get(0).ok_or_else(|| { - Error::BadServiceRequest { - service: service.name.clone(), - message: "Not enough addresses".to_string(), - } + match service.service_type { + ServiceType::Nexus { internal_address, external_address } => { + info!(self.log, "Setting up Nexus service"); + + // Nexus takes a separate config file for parameters which + // cannot be known at packaging time. + let runtime_config = NexusRuntimeConfig { + id: service.id, + dropshot_external: ConfigDropshot { + bind_address: SocketAddr::V6(external_address), + ..Default::default() + }, + dropshot_internal: ConfigDropshot { + bind_address: SocketAddr::V6(internal_address), + ..Default::default() + }, + subnet: Ipv6Subnet::::new( + self.underlay_address, + ), + database: nexus_config::Database::FromDns, + }; + + // Copy the partial config file to the expected location. + let config_dir = (self.config.get_svc_config_dir)( + running_zone.name(), + &service.name, + ); + let partial_config_path = + config_dir.join(PARTIAL_CONFIG_FILENAME); + let config_path = config_dir.join(COMPLETE_CONFIG_FILENAME); + tokio::fs::copy(partial_config_path, &config_path) + .await + .map_err(|err| Error::Io { + path: config_path.clone(), + err, })?; + + // Serialize the configuration and append it into the file. + let serialized_cfg = toml::Value::try_from(&runtime_config) + .expect("Cannot serialize config"); + let mut map = toml::map::Map::new(); + map.insert("runtime".to_string(), serialized_cfg); + let config_str = toml::to_string(&map).map_err(|err| { + Error::TomlSerialize { path: config_path.clone(), err } + })?; + let mut file = tokio::fs::OpenOptions::new() + .append(true) + .open(&config_path) + .await + .map_err(|err| Error::Io { + path: config_path.clone(), + err, + })?; + file.write_all(config_str.as_bytes()).await.map_err( + |err| Error::Io { path: config_path.clone(), err }, + )?; + } + ServiceType::InternalDns { server_address, dns_address } => { + info!(self.log, "Setting up internal-dns service"); running_zone .run_cmd(&[ crate::illumos::zone::SVCCFG, @@ -286,14 +371,12 @@ impl ServiceManager { "setprop", &format!( "config/server_address=[{}]:{}", - address, DNS_SERVER_PORT + server_address.ip(), + server_address.port(), ), ]) .map_err(|err| Error::ZoneCommand { - intent: format!( - "Setting DNS server address [{}]:{}", - address, DNS_SERVER_PORT - ), + intent: "set server address".to_string(), err, })?; @@ -305,14 +388,12 @@ impl ServiceManager { "setprop", &format!( "config/dns_address=[{}]:{}", - address, DNS_PORT + dns_address.ip(), + dns_address.port(), ), ]) .map_err(|err| Error::ZoneCommand { - intent: format!( - "Setting DNS address [{}]:{}", - address, DNS_SERVER_PORT - ), + intent: "Set DNS address".to_string(), err, })?; @@ -327,18 +408,12 @@ impl ServiceManager { ]) .map_err(|err| Error::ZoneCommand { intent: format!( - "Refreshing DNS service config for {}", + "Refresh SMF manifest {}", default_smf_name ), err, })?; } - _ => { - info!( - self.log, - "Service name {} did not match", service.name - ); - } } debug!(self.log, "enabling service"); @@ -438,7 +513,9 @@ mod test { svc, zone::MockZones, }; + use std::net::{Ipv6Addr, SocketAddrV6}; use std::os::unix::process::ExitStatusExt; + use uuid::Uuid; const SVC_NAME: &str = "my_svc"; const EXPECTED_ZONE_NAME: &str = "oxz_my_svc"; @@ -488,14 +565,29 @@ mod test { } // Prepare to call "ensure" for a new service, then actually call "ensure". - async fn ensure_new_service(mgr: &ServiceManager) { + async fn ensure_new_service(mgr: &ServiceManager, id: Uuid) { let _expectations = expect_new_service(); mgr.ensure(ServiceEnsureBody { services: vec![ServiceRequest { + id, name: SVC_NAME.to_string(), addresses: vec![], gz_addresses: vec![], + service_type: ServiceType::Nexus { + internal_address: SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ), + external_address: SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ), + }, }], }) .await @@ -504,12 +596,27 @@ mod test { // Prepare to call "ensure" for a service which already exists. We should // return the service without actually installing a new zone. - async fn ensure_existing_service(mgr: &ServiceManager) { + async fn ensure_existing_service(mgr: &ServiceManager, id: Uuid) { mgr.ensure(ServiceEnsureBody { services: vec![ServiceRequest { + id, name: SVC_NAME.to_string(), addresses: vec![], gz_addresses: vec![], + service_type: ServiceType::Nexus { + internal_address: SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ), + external_address: SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ), + }, }], }) .await @@ -533,26 +640,56 @@ mod test { drop(mgr); } + struct TestConfig { + config_dir: tempfile::TempDir, + } + + impl TestConfig { + async fn new() -> Self { + let config_dir = tempfile::TempDir::new().unwrap(); + tokio::fs::File::create( + config_dir.path().join(PARTIAL_CONFIG_FILENAME), + ) + .await + .unwrap(); + Self { config_dir } + } + + fn make_config(&self) -> Config { + let all_svcs_config_path = + self.config_dir.path().join(SERVICE_CONFIG_FILENAME); + let svc_config_dir = self.config_dir.path().to_path_buf(); + Config { + all_svcs_config_path, + get_svc_config_dir: Box::new( + move |_zone_name: &str, _svc_name: &str| { + svc_config_dir.clone() + }, + ), + } + } + } + #[tokio::test] #[serial_test::serial] async fn test_ensure_service() { let logctx = omicron_test_utils::dev::test_setup_log("test_ensure_service"); let log = logctx.log.clone(); + let test_config = TestConfig::new().await; - let config_dir = tempfile::TempDir::new().unwrap(); - let config = config_dir.path().join("services.toml"); let mgr = ServiceManager::new( log, Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, - Some(config), + test_config.make_config(), ) .await .unwrap(); - ensure_new_service(&mgr).await; + let id = Uuid::new_v4(); + ensure_new_service(&mgr, id).await; drop_service_manager(mgr); logctx.cleanup_successful(); @@ -565,21 +702,21 @@ mod test { "test_ensure_service_which_already_exists", ); let log = logctx.log.clone(); + let test_config = TestConfig::new().await; - let config_dir = tempfile::TempDir::new().unwrap(); - let config = config_dir.path().join("services.toml"); let mgr = ServiceManager::new( log, Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, - Some(config), + test_config.make_config(), ) .await .unwrap(); - ensure_new_service(&mgr).await; - ensure_existing_service(&mgr).await; + let id = Uuid::new_v4(); + ensure_new_service(&mgr, id).await; + ensure_existing_service(&mgr, id).await; drop_service_manager(mgr); logctx.cleanup_successful(); @@ -591,9 +728,7 @@ mod test { let logctx = omicron_test_utils::dev::test_setup_log( "test_services_are_recreated_on_reboot", ); - - let config_dir = tempfile::TempDir::new().unwrap(); - let config = config_dir.path().join("services.toml"); + let test_config = TestConfig::new().await; // First, spin up a ServiceManager, create a new service, and tear it // down. @@ -602,11 +737,13 @@ mod test { Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, - Some(config.clone()), + test_config.make_config(), ) .await .unwrap(); - ensure_new_service(&mgr).await; + + let id = Uuid::new_v4(); + ensure_new_service(&mgr, id).await; drop_service_manager(mgr); // Before we re-create the service manager - notably, using the same @@ -617,7 +754,7 @@ mod test { Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, - Some(config.clone()), + test_config.make_config(), ) .await .unwrap(); @@ -632,9 +769,7 @@ mod test { let logctx = omicron_test_utils::dev::test_setup_log( "test_services_do_not_persist_without_config", ); - - let config_dir = tempfile::TempDir::new().unwrap(); - let config = config_dir.path().join("services.toml"); + let test_config = TestConfig::new().await; // First, spin up a ServiceManager, create a new service, and tear it // down. @@ -643,16 +778,18 @@ mod test { Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, - Some(config.clone()), + test_config.make_config(), ) .await .unwrap(); - ensure_new_service(&mgr).await; + let id = Uuid::new_v4(); + ensure_new_service(&mgr, id).await; drop_service_manager(mgr); // Next, delete the config. This means the service we just created will // not be remembered on the next initialization. - std::fs::remove_file(&config).unwrap(); + let config = test_config.make_config(); + std::fs::remove_file(&config.all_svcs_config_path).unwrap(); // Observe that the old service is not re-initialized. let mgr = ServiceManager::new( @@ -660,7 +797,7 @@ mod test { Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, - Some(config.clone()), + config, ) .await .unwrap(); diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index c0c2ff649c8..5f8f1e500ab 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -16,7 +16,7 @@ use crate::params::{ DatasetKind, DiskStateRequested, InstanceHardware, InstanceMigrateParams, InstanceRuntimeStateRequested, ServiceEnsureBody, }; -use crate::services::ServiceManager; +use crate::services::{self, ServiceManager}; use crate::storage_manager::StorageManager; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, @@ -245,7 +245,7 @@ impl SledAgent { etherstub.clone(), etherstub_vnic.clone(), *sled_address.ip(), - None, + services::Config::default(), ) .await?; diff --git a/smf/nexus/config.toml b/smf/nexus/config-partial.toml similarity index 53% rename from smf/nexus/config.toml rename to smf/nexus/config-partial.toml index d73d7a90cfc..b77ffc3137f 100644 --- a/smf/nexus/config.toml +++ b/smf/nexus/config-partial.toml @@ -1,10 +1,7 @@ # -# Oxide API: example configuration file +# Oxide API: partial configuration file # -# Identifier for this instance of Nexus -id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" - [console] # Directory for static assets. Absolute path or relative to CWD. static_dir = "/var/nexus/static" @@ -16,18 +13,6 @@ session_absolute_timeout_minutes = 480 # TODO(https://github.com/oxidecomputer/omicron/issues/372): Remove "spoof". schemes_external = ["spoof", "session_cookie"] -[database] -# URL for connecting to the database -url = "postgresql://root@[fd00:1122:3344:0101::2]:32221/omicron?sslmode=disable" - -[dropshot_external] -# IP address and TCP port on which to listen for the external API -bind_address = "[fd00:1122:3344:0101::3]:12220" - -[dropshot_internal] -# IP address and TCP port on which to listen for the internal API -bind_address = "[fd00:1122:3344:0101::3]:12221" - [log] # Show log messages of this level and more severe level = "info" diff --git a/smf/nexus/manifest.xml b/smf/nexus/manifest.xml index 0b8da2ff62f..3ff92b2fbac 100644 --- a/smf/nexus/manifest.xml +++ b/smf/nexus/manifest.xml @@ -11,6 +11,14 @@ type='service'> + + + + + + diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index d8113cf4d1b..83e0ee5fd40 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -10,20 +10,20 @@ rack_subnet = "fd00:1122:3344:0100::" # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate crucible datasets. -[[request.dataset]] -zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1122:3344:0101::6]:32345" -dataset_kind.type = "crucible" - -[[request.dataset]] -zpool_id = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03" -address = "[fd00:1122:3344:0101::7]:32345" -dataset_kind.type = "crucible" - -[[request.dataset]] -zpool_id = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03" -address = "[fd00:1122:3344:0101::8]:32345" -dataset_kind.type = "crucible" +# [[request.dataset]] +# zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" +# address = "[fd00:1122:3344:0101::6]:32345" +# dataset_kind.type = "crucible" +# +# [[request.dataset]] +# zpool_id = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03" +# address = "[fd00:1122:3344:0101::7]:32345" +# dataset_kind.type = "crucible" +# +# [[request.dataset]] +# zpool_id = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03" +# address = "[fd00:1122:3344:0101::8]:32345" +# dataset_kind.type = "crucible" [[request.dataset]] zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" @@ -33,19 +33,19 @@ dataset_kind.all_addresses = [ "[fd00:1122:3344:0101::2]:32221" ] # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate clickhouse datasets. -[[request.dataset]] -zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1122:3344:0101::5]:8123" -dataset_kind.type = "clickhouse" +# [[request.dataset]] +# zpool_id = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" +# address = "[fd00:1122:3344:0101::5]:8123" +# dataset_kind.type = "clickhouse" -[[request.service]] -name = "nexus" -addresses = [ "fd00:1122:3344:0101::3" ] -gz_addresses = [] +# [[request.service]] +# name = "nexus" +# addresses = [ "fd00:1122:3344:0101::3" ] +# gz_addresses = [] # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate Oximeter services. -[[request.service]] -name = "oximeter" -addresses = [ "fd00:1122:3344:0101::4" ] -gz_addresses = [] +# [[request.service]] +# name = "oximeter" +# addresses = [ "fd00:1122:3344:0101::4" ] +# gz_addresses = [] diff --git a/smf/sled-agent/manifest.xml b/smf/sled-agent/manifest.xml index 378b77776c8..96f029d96e0 100644 --- a/smf/sled-agent/manifest.xml +++ b/smf/sled-agent/manifest.xml @@ -28,6 +28,10 @@ type='service'> + + + diff --git a/test-utils/src/dev/db.rs b/test-utils/src/dev/db.rs index 5449bfc4139..b7112ae1a37 100644 --- a/test-utils/src/dev/db.rs +++ b/test-utils/src/dev/db.rs @@ -8,7 +8,7 @@ use crate::dev::poll; use anyhow::anyhow; use anyhow::bail; use anyhow::Context; -use omicron_common::config::PostgresConfigWithUrl; +use omicron_common::postgres_config::PostgresConfigWithUrl; use std::ffi::{OsStr, OsString}; use std::fmt; use std::ops::Deref;