[WIP] Use dns services instead of hard-coded IP addresses

common/Cargo.toml

@@ -30,6 +30,7 @@ structopt = "0.3"
 thiserror = "1.0"
 tokio = { version = "1.18", features = [ "full" ] }
 tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] }
+toml = "0.5.9"
 uuid = { version = "1.1.0", features = [ "serde", "v4" ] }
 parse-display = "0.5.4"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }

common/src/address.rs

@@ -27,12 +27,23 @@ pub const DNS_REDUNDANCY: usize = 1;
 /// reserved for DNS servers.
 pub const MAX_DNS_REDUNDANCY: usize = 5;
 
+/// The port for the UDP-based internal DNS name server.
 pub const DNS_PORT: u16 = 53;
+/// The port for the HTTP-based internal DNS dropshot server.
 pub const DNS_SERVER_PORT: u16 = 5353;
 pub const SLED_AGENT_PORT: u16 = 12345;
 
 /// The port propolis-server listens on inside the propolis zone.
 pub const PROPOLIS_PORT: u16 = 12400;
+pub const COCKROACH_PORT: u16 = 32221;
+pub const CRUCIBLE_PORT: u16 = 32345;
+
+pub const NEXUS_EXTERNAL_PORT: u16 = 12220;
+pub const NEXUS_INTERNAL_PORT: u16 = 12221;
+
+pub const COCKROACH_DNS_NAME: &str =
+    "_cockroachdb._tcp.control-plane.oxide.internal";
+
 
 // Anycast is a mechanism in which a single IP address is shared by multiple
 // devices, and the destination is located based on routing distance.
@@ -124,14 +135,16 @@ impl ReservedRackSubnet {
     }
 }
 
-const SLED_AGENT_ADDRESS_INDEX: usize = 1;
+pub const SLED_AGENT_ADDRESS_INDEX: u16 = 1;
+/// The maximum number of addresses per sled subnet reserved for RSS.
+pub const RSS_RESERVED_ADDRESSES: u16 = 10;
 
 /// Return the sled agent address for a subnet.
 ///
 /// This address will come from the first address of the [`SLED_PREFIX`] subnet.
 pub fn get_sled_address(sled_subnet: Ipv6Subnet<SLED_PREFIX>) -> SocketAddrV6 {
     let sled_agent_ip =
-        sled_subnet.net().iter().nth(SLED_AGENT_ADDRESS_INDEX).unwrap();
+        sled_subnet.net().iter().nth(SLED_AGENT_ADDRESS_INDEX.into()).unwrap();
     SocketAddrV6::new(sled_agent_ip, SLED_AGENT_PORT, 0, 0)
 }
 

common/src/api/external/mod.rs

@@ -500,6 +500,7 @@ pub enum ResourceType {
     Instance,
     NetworkInterface,
     Rack,
+    Service,
     Sled,
     SagaDbg,
     Snapshot,

common/src/lib.rs

@@ -24,7 +24,8 @@ pub mod address;
 pub mod api;
 pub mod backoff;
 pub mod cmd;
-pub mod config;
+pub mod nexus_config;
+pub mod postgres_config;
 
 #[macro_export]
 macro_rules! generate_logging_api {

common/src/nexus_config.rs

@@ -0,0 +1,128 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Configuration parameters to Nexus that are usually only known
+//! at runtime.
+
+use super::address::{Ipv6Subnet, RACK_PREFIX};
+use super::postgres_config::PostgresConfigWithUrl;
+use dropshot::ConfigDropshot;
+use serde::{Deserialize, Serialize};
+use serde_with::serde_as;
+use serde_with::DisplayFromStr;
+use std::fmt;
+use std::path::{Path, PathBuf};
+use uuid::Uuid;
+
+#[derive(Debug)]
+pub struct LoadError {
+    pub path: PathBuf,
+    pub kind: LoadErrorKind,
+}
+
+#[derive(Debug)]
+pub struct InvalidTunable {
+    pub tunable: String,
+    pub message: String,
+}
+
+impl std::fmt::Display for InvalidTunable {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "invalid \"{}\": \"{}\"", self.tunable, self.message)
+    }
+}
+impl std::error::Error for InvalidTunable {}
+
+#[derive(Debug)]
+pub enum LoadErrorKind {
+    Io(std::io::Error),
+    Parse(toml::de::Error),
+    InvalidTunable(InvalidTunable),
+}
+
+impl From<(PathBuf, std::io::Error)> for LoadError {
+    fn from((path, err): (PathBuf, std::io::Error)) -> Self {
+        LoadError { path, kind: LoadErrorKind::Io(err) }
+    }
+}
+
+impl From<(PathBuf, toml::de::Error)> for LoadError {
+    fn from((path, err): (PathBuf, toml::de::Error)) -> Self {
+        LoadError { path, kind: LoadErrorKind::Parse(err) }
+    }
+}
+
+impl std::error::Error for LoadError {}
+
+impl fmt::Display for LoadError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.kind {
+            LoadErrorKind::Io(e) => {
+                write!(f, "read \"{}\": {}", self.path.display(), e)
+            }
+            LoadErrorKind::Parse(e) => {
+                write!(f, "parse \"{}\": {}", self.path.display(), e)
+            }
+            LoadErrorKind::InvalidTunable(inner) => {
+                write!(
+                    f,
+                    "invalid tunable \"{}\": {}",
+                    self.path.display(),
+                    inner,
+                )
+            }
+        }
+    }
+}
+
+impl std::cmp::PartialEq<std::io::Error> for LoadError {
+    fn eq(&self, other: &std::io::Error) -> bool {
+        if let LoadErrorKind::Io(e) = &self.kind {
+            e.kind() == other.kind()
+        } else {
+            false
+        }
+    }
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+#[allow(clippy::large_enum_variant)]
+pub enum Database {
+    FromDns,
+    FromUrl {
+        #[serde_as(as = "DisplayFromStr")]
+        url: PostgresConfigWithUrl,
+    },
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+pub struct RuntimeConfig {
+    /// Uuid of the Nexus instance
+    pub id: Uuid,
+    /// Dropshot configuration for external API server
+    pub dropshot_external: ConfigDropshot,
+    /// Dropshot configuration for internal API server
+    pub dropshot_internal: ConfigDropshot,
+    /// Portion of the IP space to be managed by the Rack.
+    pub subnet: Ipv6Subnet<RACK_PREFIX>,
+    /// DB configuration.
+    pub database: Database,
+}
+
+impl RuntimeConfig {
+    /// Load a `RuntimeConfig` from the given TOML file
+    ///
+    /// This config object can then be used to create a new `Nexus`.
+    /// The format is described in the README.
+    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self, LoadError> {
+        let path = path.as_ref();
+        let file_contents = std::fs::read_to_string(path)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        let config_parsed: Self = toml::from_str(&file_contents)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        Ok(config_parsed)
+    }
+}

common/src/sql/dbinit.sql

@@ -72,6 +72,34 @@ CREATE TABLE omicron.public.sled (
     last_used_address INET NOT NULL
 );
 
+/*
+ * Services
+ */
+
+CREATE TABLE omicron.public.service (
+    /* Identity metadata (asset) */
+    id UUID PRIMARY KEY,
+    time_created TIMESTAMPTZ NOT NULL,
+    time_modified TIMESTAMPTZ NOT NULL,
+
+    /* FK into the Sled table */
+    sled_id UUID NOT NULL,
+
+    /* The IP address and bound port of the service. */
+    ip INET NOT NULL,
+    port INT4 CHECK (port BETWEEN 0 AND 65535) NOT NULL
+);
+
+/* Add an index which lets us look up the services on a sled */
+CREATE INDEX ON omicron.public.service (
+    sled_id
+);
+
+CREATE TYPE omicron.public.service_kind AS ENUM (
+  'nexus',
+  'oximeter'
+);
+
 /*
  * ZPools of Storage, attached to Sleds.
  * Typically these are backed by a single physical disk.

internal-dns-client/Cargo.toml

@@ -5,9 +5,13 @@ edition = "2021"
 license = "MPL-2.0"
 
 [dependencies]
+omicron-common = { path = "../common" }
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
+reqwest = { version = "0.11", features = ["json", "rustls-tls", "stream"] }
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1.0"
 slog = { version = "2.5.0", features = [ "max_level_trace", "release_max_level_debug" ] }
 structopt = "0.3"
-reqwest = { version = "0.11", features = ["json", "rustls-tls", "stream"] }
+trust-dns-proto = "0.21"
+trust-dns-resolver = "0.21"
+uuid = { version = "1.1.0", features = [ "v4", "serde" ] }

internal-dns-client/src/lib.rs

@@ -16,3 +16,6 @@ progenitor::generate_api!(
         slog::debug!(log, "client response"; "result" => ?result);
     }),
 );
+
+pub mod multiclient;
+pub mod names;