Service management and rss handoff

common/src/address.rs

@@ -20,12 +20,12 @@ pub const SLED_PREFIX: u8 = 64;
 /// The amount of redundancy for DNS servers.
 ///
 /// Must be less than MAX_DNS_REDUNDANCY.
-pub const DNS_REDUNDANCY: usize = 1;
+pub const DNS_REDUNDANCY: u32 = 1;
 /// The maximum amount of redundancy for DNS servers.
 ///
 /// This determines the number of addresses which are
 /// reserved for DNS servers.
-pub const MAX_DNS_REDUNDANCY: usize = 5;
+pub const MAX_DNS_REDUNDANCY: u32 = 5;
 
 pub const DNS_PORT: u16 = 53;
 pub const DNS_SERVER_PORT: u16 = 5353;
@@ -34,10 +34,12 @@ pub const SLED_AGENT_PORT: u16 = 12345;
 /// The port propolis-server listens on inside the propolis zone.
 pub const PROPOLIS_PORT: u16 = 12400;
 pub const COCKROACH_PORT: u16 = 32221;
+pub const CRUCIBLE_PORT: u16 = 32345;
 pub const CLICKHOUSE_PORT: u16 = 8123;
 pub const OXIMETER_PORT: u16 = 12223;
 pub const DENDRITE_PORT: u16 = 12224;
 
+pub const NEXUS_EXTERNAL_PORT: u16 = 12220;
 pub const NEXUS_INTERNAL_PORT: u16 = 12221;
 
 // Anycast is a mechanism in which a single IP address is shared by multiple
@@ -85,6 +87,11 @@ pub struct DnsSubnet {
 }
 
 impl DnsSubnet {
+    /// Creates a subnet, given an address for the DNS server itself.
+    pub fn from_dns_address(address: Ipv6Addr) -> Self {
+        Self { subnet: Ipv6Subnet::new(address) }
+    }
+
     /// Returns the DNS server address within the subnet.
     ///
     /// This is the first address within the subnet.
@@ -180,7 +187,7 @@ mod test {
 
         // Observe the first DNS subnet within this reserved rack subnet.
         let dns_subnets = rack_subnet.get_dns_subnets();
-        assert_eq!(DNS_REDUNDANCY, dns_subnets.len());
+        assert_eq!(DNS_REDUNDANCY, dns_subnets.len() as u32);
 
         // The DNS address and GZ address should be only differing by one.
         assert_eq!(

common/src/sql/dbinit.sql

@@ -89,10 +89,11 @@ CREATE TABLE omicron.public.sled (
     last_used_address INET NOT NULL
 );
 
-/* Add an index which lets us look up sleds on a rack */
+/* Add an index which lets us look up the sleds on a rack */
 CREATE INDEX ON omicron.public.sled (
     rack_id
-) WHERE time_deleted IS NULL;
+) WHERE
+    time_deleted IS NULL;
 
 CREATE INDEX ON omicron.public.sled (
     id
@@ -125,7 +126,40 @@ CREATE TABLE omicron.public.service (
 
 /* Add an index which lets us look up the services on a sled */
 CREATE INDEX ON omicron.public.service (
-    sled_id
+    sled_id,
+    kind
+);
+
+/* Add an index which lets us look up services of a particular kind on a sled */
+CREATE INDEX ON omicron.public.service (
+    kind
+);
+
+/*
+ * Additional context for services of "kind = nexus"
+ * This table should be treated as an optional extension
+ * of the service table itself.
+ */
+CREATE TABLE omicron.public.nexus_service (
+    id UUID PRIMARY KEY,
+
+    /* FK to the service table */
+    service_id UUID NOT NULL,
+    /* FK to the instance_external_ip table */
+    external_ip_id UUID NOT NULL,
+    /* FK to the nexus_certificate table */
+    certificate_id UUID NOT NULL
+);
+
+/*
+ * Information about x509 certificates used to serve Nexus' external interface.
+ * These certificates may be used by multiple instantiations of the Nexus
+ * service simultaneously.
+ */
+CREATE TABLE omicron.public.nexus_certificate (
+    id UUID PRIMARY KEY,
+    public_cert BYTES NOT NULL,
+    private_key BYTES NOT NULL
 );
 
 /*
@@ -148,6 +182,11 @@ CREATE TABLE omicron.public.Zpool (
     total_size INT NOT NULL
 );
 
+/* Create an index which allows looking up all zpools on a sled */
+CREATE INDEX on omicron.public.Zpool (
+    sled_id
+) WHERE time_deleted IS NULL;
+
 CREATE TYPE omicron.public.dataset_kind AS ENUM (
   'crucible',
   'cockroach',
@@ -184,6 +223,11 @@ CREATE TABLE omicron.public.Dataset (
     )
 );
 
+/* Create an index which allows looking up all datasets in a pool */
+CREATE INDEX on omicron.public.Dataset (
+    pool_id
+) WHERE time_deleted IS NULL;
+
 /* Create an index on the size usage for Crucible's allocation */
 CREATE INDEX on omicron.public.Dataset (
     size_used

docs/how-to-run.adoc

@@ -154,11 +154,11 @@ When we deploy, we're effectively creating a number of different zones
 for all the components that make up Omicron (Nexus, Clickhouse, Crucible, etc).
 Since all these services run in different zones they cannot communicate with
 each other (and Sled Agent in the global zone) via `localhost`. In practice,
-we'll assign addresses as per RFD 63 as well as incorporating DNS based
+we assign addresses as per RFD 63 as well as incorporating DNS based
 service discovery.
 
-For the purposes of local development today, we specify some hardcoded IPv6
-unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1::/64`.
+For the purposes of local development today, we specify some hardcoded IP
+addresses.
 
 If you'd like to modify these values to suit your local network, you can modify
 them within the https://github.com/oxidecomputer/omicron/tree/main/smf[`smf/` subdirectory].
@@ -171,15 +171,6 @@ be set as a default route for the Nexus zone.
 |===================================================================================================
 | Service                    | Endpoint
 | Sled Agent: Bootstrap      | Derived from MAC address of physical data link.
-| Sled Agent: Dropshot API   | `[fd00:1122:3344:0101::1]:12345`
-| Cockroach DB               | `[fd00:1122:3344:0101::2]:32221`
-| Nexus: Internal API        | `[fd00:1122:3344:0101::3]:12221`
-| Oximeter                   | `[fd00:1122:3344:0101::4]:12223`
-| Clickhouse                 | `[fd00:1122:3344:0101::5]:8123`
-| Crucible Downstairs 1      | `[fd00:1122:3344:0101::6]:32345`
-| Crucible Downstairs 2      | `[fd00:1122:3344:0101::7]:32345`
-| Crucible Downstairs 3      | `[fd00:1122:3344:0101::8]:32345`
-| Internal DNS Service       | `[fd00:1122:3344:0001::1]:5353`
 | Nexus: External API        | `192.168.1.20:80`
 | Internet Gateway           | None, but can be set in `smf/sled-agent/config-rss.toml`
 |===================================================================================================

nexus/Cargo.toml

@@ -125,6 +125,7 @@ nexus-test-utils-macros = { path = "test-utils-macros" }
 nexus-test-utils = { path = "test-utils" }
 omicron-test-utils = { path = "../test-utils" }
 openapiv3 = "1.0"
+pretty_assertions = "1.3"
 regex = "1.6.0"
 subprocess = "0.2.9"
 term = "0.7"

nexus/db-model/Cargo.toml

@@ -12,6 +12,7 @@ anyhow = "1.0"
 chrono = { version = "0.4", features = ["serde"] }
 diesel = { version = "2.0.2", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] }
 hex = "0.4.3"
+internal-dns-client = { path = "../../internal-dns-client" }
 ipnetwork = "0.20"
 macaddr = { version = "1.0.1", features = [ "serde_std" ]}
 newtype_derive = "0.1.6"

nexus/db-model/src/dataset.rs

@@ -4,11 +4,14 @@
 
 use super::{DatasetKind, Generation, Region, SqlU16};
 use crate::collection::DatastoreCollectionConfig;
+use crate::ipv6;
 use crate::schema::{dataset, region};
 use chrono::{DateTime, Utc};
 use db_macros::Asset;
+use internal_dns_client::names::{BackendName, ServiceName, AAAA, SRV};
+use nexus_types::identity::Asset;
 use serde::{Deserialize, Serialize};
-use std::net::SocketAddr;
+use std::net::{Ipv6Addr, SocketAddrV6};
 use uuid::Uuid;
 
 /// Database representation of a Dataset.
@@ -35,18 +38,18 @@ pub struct Dataset {
 
     pub pool_id: Uuid,
 
-    ip: ipnetwork::IpNetwork,
+    ip: ipv6::Ipv6Addr,
     port: SqlU16,
 
-    kind: DatasetKind,
+    pub kind: DatasetKind,
     pub size_used: Option<i64>,
 }
 
 impl Dataset {
     pub fn new(
         id: Uuid,
         pool_id: Uuid,
-        addr: SocketAddr,
+        addr: SocketAddrV6,
         kind: DatasetKind,
     ) -> Self {
         let size_used = match kind {
@@ -65,12 +68,26 @@ impl Dataset {
         }
     }
 
-    pub fn address(&self) -> SocketAddr {
+    pub fn address(&self) -> SocketAddrV6 {
         self.address_with_port(self.port.into())
     }
 
-    pub fn address_with_port(&self, port: u16) -> SocketAddr {
-        SocketAddr::new(self.ip.ip(), port)
+    pub fn address_with_port(&self, port: u16) -> SocketAddrV6 {
+        SocketAddrV6::new(Ipv6Addr::from(self.ip), port, 0, 0)
+    }
+
+    pub fn aaaa(&self) -> AAAA {
+        AAAA::Zone(self.id())
+    }
+
+    pub fn srv(&self) -> SRV {
+        match self.kind {
+            DatasetKind::Crucible => {
+                SRV::Backend(BackendName::Crucible, self.id())
+            }
+            DatasetKind::Clickhouse => SRV::Service(ServiceName::Clickhouse),
+            DatasetKind::Cockroach => SRV::Service(ServiceName::Cockroach),
+        }
     }
 }
 

nexus/db-model/src/dataset_kind.rs

@@ -4,14 +4,15 @@
 
 use super::impl_enum_type;
 use nexus_types::internal_api;
+use omicron_common::address::{CLICKHOUSE_PORT, COCKROACH_PORT, CRUCIBLE_PORT};
 use serde::{Deserialize, Serialize};
 
 impl_enum_type!(
-    #[derive(SqlType, Debug, QueryId)]
+    #[derive(Clone, SqlType, Debug, QueryId)]
     #[diesel(postgres_type(name = "dataset_kind"))]
     pub struct DatasetKindEnum;
 
-    #[derive(Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
+    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
     #[diesel(sql_type = DatasetKindEnum)]
     pub enum DatasetKind;
 
@@ -21,6 +22,16 @@ impl_enum_type!(
     Clickhouse => b"clickhouse"
 );
 
+impl DatasetKind {
+    pub fn port(&self) -> u16 {
+        match self {
+            DatasetKind::Crucible => CRUCIBLE_PORT,
+            DatasetKind::Cockroach => COCKROACH_PORT,
+            DatasetKind::Clickhouse => CLICKHOUSE_PORT,
+        }
+    }
+}
+
 impl From<internal_api::params::DatasetKind> for DatasetKind {
     fn from(k: internal_api::params::DatasetKind) -> Self {
         match k {

nexus/db-model/src/ipv6.rs

@@ -16,9 +16,19 @@ use diesel::sql_types::Inet;
 use ipnetwork::IpNetwork;
 use ipnetwork::Ipv6Network;
 use omicron_common::api::external::Error;
+use serde::{Deserialize, Serialize};
 
 #[derive(
-    Clone, Copy, AsExpression, FromSqlRow, PartialEq, Ord, PartialOrd, Eq,
+    Clone,
+    Copy,
+    AsExpression,
+    FromSqlRow,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Eq,
+    Deserialize,
+    Serialize,
 )]
 #[diesel(sql_type = Inet)]
 pub struct Ipv6Addr(std::net::Ipv6Addr);

nexus/db-model/src/queries/mod.rs

@@ -5,3 +5,4 @@
 //! Subqueries used in CTEs.
 
 pub mod region_allocation;
+pub mod service_provision;

nexus/db-model/src/queries/region_allocation.rs

@@ -143,11 +143,12 @@ diesel::allow_tables_to_appear_in_same_query!(
     dataset,
 );
 
+diesel::allow_tables_to_appear_in_same_query!(candidate_regions, zpool,);
+
 diesel::allow_tables_to_appear_in_same_query!(
     do_insert,
     candidate_regions,
     dataset,
-    zpool,
 );
 
 diesel::allow_tables_to_appear_in_same_query!(candidate_zpools, dataset,);

nexus/db-model/src/queries/service_provision.rs

@@ -0,0 +1,77 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Service prosioning subqueries used by CTEs.
+
+table! {
+    sled_allocation_pool {
+        id -> Uuid,
+    }
+}
+
+table! {
+    previously_allocated_services {
+        id -> Uuid,
+        time_created -> Timestamptz,
+        time_modified -> Timestamptz,
+
+        sled_id -> Uuid,
+        ip -> Inet,
+        kind -> crate::ServiceKindEnum,
+    }
+}
+
+table! {
+    old_service_count (count) {
+        count -> Int8,
+    }
+}
+
+table! {
+    new_service_count (count) {
+        count -> Int8,
+    }
+}
+
+table! {
+    candidate_sleds {
+        id -> Uuid,
+    }
+}
+
+table! {
+    new_internal_ips {
+        id -> Uuid,
+        last_used_address -> Inet,
+    }
+}
+
+diesel::allow_tables_to_appear_in_same_query!(
+    candidate_sleds,
+    new_internal_ips,
+);
+
+table! {
+    candidate_services {
+        id -> Uuid,
+        time_created -> Timestamptz,
+        time_modified -> Timestamptz,
+
+        sled_id -> Uuid,
+        ip -> Inet,
+        kind -> crate::ServiceKindEnum,
+    }
+}
+
+table! {
+    inserted_services {
+        id -> Uuid,
+        time_created -> Timestamptz,
+        time_modified -> Timestamptz,
+
+        sled_id -> Uuid,
+        ip -> Inet,
+        kind -> crate::ServiceKindEnum,
+    }
+}