[nexus] Populate IP pool, nexus service information, during rack setup

common/src/sql/dbinit.sql

@@ -136,6 +136,15 @@ CREATE INDEX ON omicron.public.service (
     sled_id
 );
 
+-- Extended information for services where "service.kind = nexus"
+-- The "id" columng of this table should match "id" column of the
+-- "omicron.public.service" table exactly.
+CREATE TABLE omicron.public.nexus_service (
+    id UUID PRIMARY KEY,
+    -- The external IP address used for Nexus' external interface.
+    external_ip_id UUID NOT NULL
+);
+
 CREATE TYPE omicron.public.physical_disk_kind AS ENUM (
   'm2',
   'u2'

nexus/db-model/src/external_ip.rs

@@ -18,6 +18,7 @@ use nexus_types::external_api::shared;
 use nexus_types::external_api::views;
 use omicron_common::api::external::Error;
 use std::convert::TryFrom;
+use std::net::IpAddr;
 use uuid::Uuid;
 
 impl_enum_type!(
@@ -90,6 +91,8 @@ pub struct IncompleteExternalIp {
     kind: IpKind,
     instance_id: Option<Uuid>,
     pool_id: Uuid,
+    // Optional address requesting that a specific IP address be allocated.
+    explicit_ip: Option<IpNetwork>,
 }
 
 impl IncompleteExternalIp {
@@ -106,6 +109,7 @@ impl IncompleteExternalIp {
             kind: IpKind::SNat,
             instance_id: Some(instance_id),
             pool_id,
+            explicit_ip: None,
         }
     }
 
@@ -118,6 +122,7 @@ impl IncompleteExternalIp {
             kind: IpKind::Ephemeral,
             instance_id: Some(instance_id),
             pool_id,
+            explicit_ip: None,
         }
     }
 
@@ -135,6 +140,24 @@ impl IncompleteExternalIp {
             kind: IpKind::Floating,
             instance_id: None,
             pool_id,
+            explicit_ip: None,
+        }
+    }
+
+    pub fn for_service_explicit(
+        id: Uuid,
+        pool_id: Uuid,
+        address: IpAddr,
+    ) -> Self {
+        Self {
+            id,
+            name: None,
+            description: None,
+            time_created: Utc::now(),
+            kind: IpKind::Service,
+            instance_id: None,
+            pool_id,
+            explicit_ip: Some(IpNetwork::from(address)),
         }
     }
 
@@ -147,6 +170,7 @@ impl IncompleteExternalIp {
             kind: IpKind::Service,
             instance_id: None,
             pool_id,
+            explicit_ip: None,
         }
     }
 
@@ -177,6 +201,10 @@ impl IncompleteExternalIp {
     pub fn pool_id(&self) -> &Uuid {
         &self.pool_id
     }
+
+    pub fn explicit_ip(&self) -> &Option<IpNetwork> {
+        &self.explicit_ip
+    }
 }
 
 impl TryFrom<IpKind> for shared::IpKind {

nexus/db-model/src/lib.rs

@@ -36,6 +36,7 @@ mod l4_port_range;
 mod macaddr;
 mod name;
 mod network_interface;
+mod nexus_service;
 mod organization;
 mod oximeter_info;
 mod physical_disk;
@@ -114,6 +115,7 @@ pub use ipv6net::*;
 pub use l4_port_range::*;
 pub use name::*;
 pub use network_interface::*;
+pub use nexus_service::*;
 pub use organization::*;
 pub use oximeter_info::*;
 pub use physical_disk::*;

nexus/db-model/src/nexus_service.rs

@@ -0,0 +1,20 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use crate::schema::nexus_service;
+use uuid::Uuid;
+
+/// Nexus-specific extended service information.
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = nexus_service)]
+pub struct NexusService {
+    pub id: Uuid,
+    pub external_ip_id: Uuid,
+}
+
+impl NexusService {
+    pub fn new(id: Uuid, external_ip_id: Uuid) -> Self {
+        Self { id, external_ip_id }
+    }
+}

nexus/db-model/src/schema.rs

@@ -408,6 +408,13 @@ table! {
     }
 }
 
+table! {
+    nexus_service (id) {
+        id -> Uuid,
+        external_ip_id -> Uuid,
+    }
+}
+
 table! {
     physical_disk (id) {
         id -> Uuid,

nexus/src/app/rack.rs

@@ -65,22 +65,6 @@ impl super::Nexus {
     ) -> Result<(), Error> {
         opctx.authorize(authz::Action::Modify, &authz::FLEET).await?;
 
-        // Convert from parameter -> DB type.
-        let services: Vec<_> = request
-            .services
-            .into_iter()
-            .map(|svc| {
-                db::model::Service::new(
-                    svc.service_id,
-                    svc.sled_id,
-                    svc.address,
-                    svc.kind.into(),
-                )
-            })
-            .collect();
-
-        // TODO(https://github.com/oxidecomputer/omicron/issues/1958): If nexus, add a pool?
-
         let datasets: Vec<_> = request
             .datasets
             .into_iter()
@@ -94,6 +78,7 @@ impl super::Nexus {
             })
             .collect();
 
+        let service_ip_pool_ranges = request.internal_services_ip_pool_ranges;
         let certificates: Vec<_> = request
             .certs
             .into_iter()
@@ -126,8 +111,9 @@ impl super::Nexus {
             .rack_set_initialized(
                 opctx,
                 rack_id,
-                services,
+                request.services,
                 datasets,
+                service_ip_pool_ranges,
                 certificates,
             )
             .await?;

nexus/src/db/datastore/external_ip.rs

@@ -14,17 +14,19 @@ use crate::db::model::ExternalIp;
 use crate::db::model::IncompleteExternalIp;
 use crate::db::model::IpKind;
 use crate::db::model::Name;
+use crate::db::pool::DbConnection;
 use crate::db::queries::external_ip::NextExternalIp;
 use crate::db::update_and_check::UpdateAndCheck;
 use crate::db::update_and_check::UpdateStatus;
-use async_bb8_diesel::AsyncRunQueryDsl;
+use async_bb8_diesel::{AsyncRunQueryDsl, PoolError};
 use chrono::Utc;
 use diesel::prelude::*;
 use nexus_types::identity::Resource;
 use omicron_common::api::external::CreateResult;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::Name as ExternalName;
+use std::net::IpAddr;
 use std::str::FromStr;
 use uuid::Uuid;
 
@@ -83,28 +85,65 @@ impl DataStore {
         opctx: &OpContext,
         data: IncompleteExternalIp,
     ) -> CreateResult<ExternalIp> {
-        NextExternalIp::new(data)
-            .get_result_async(self.pool_authorized(opctx).await?)
-            .await
-            .map_err(|e| {
-                use async_bb8_diesel::ConnectionError::Query;
-                use async_bb8_diesel::PoolError::Connection;
-                use diesel::result::Error::NotFound;
-                match e {
-                    Connection(Query(NotFound)) => Error::invalid_request(
-                        "No external IP addresses available",
-                    ),
-                    _ => public_error_from_diesel_pool(e, ErrorHandler::Server),
+        let conn = self.pool_authorized(opctx).await?;
+        Self::allocate_external_ip_on_connection(conn, data).await
+    }
+
+    /// Variant of [Self::allocate_external_ip] which may be called from a
+    /// transaction context.
+    pub(crate) async fn allocate_external_ip_on_connection<ConnErr>(
+        conn: &(impl async_bb8_diesel::AsyncConnection<DbConnection, ConnErr>
+              + Sync),
+        data: IncompleteExternalIp,
+    ) -> CreateResult<ExternalIp>
+    where
+        ConnErr: From<diesel::result::Error> + Send + 'static,
+        PoolError: From<ConnErr>,
+    {
+        let explicit_ip = data.explicit_ip().is_some();
+        NextExternalIp::new(data).get_result_async(conn).await.map_err(|e| {
+            use async_bb8_diesel::ConnectionError::Query;
+            use async_bb8_diesel::PoolError::Connection;
+            use diesel::result::Error::NotFound;
+            let e = PoolError::from(e);
+            match e {
+                Connection(Query(NotFound)) => {
+                    if explicit_ip {
+                        Error::invalid_request(
+                            "Requested external IP address not available",
+                        )
+                    } else {
+                        Error::invalid_request(
+                            "No external IP addresses available",
+                        )
+                    }
                 }
-            })
+                _ => crate::db::queries::external_ip::from_pool(e),
+            }
+        })
+    }
+
+    /// Allocates an explicit IP address for an internal service.
+    ///
+    /// Unlike the other IP allocation requests, this does not search for an
+    /// available IP address, it asks for one explicitly.
+    pub async fn allocate_explicit_service_ip(
+        &self,
+        opctx: &OpContext,
+        ip_id: Uuid,
+        ip: IpAddr,
+    ) -> CreateResult<ExternalIp> {
+        let (.., pool) = self.ip_pools_service_lookup(opctx).await?;
+        let data =
+            IncompleteExternalIp::for_service_explicit(ip_id, pool.id(), ip);
+        self.allocate_external_ip(opctx, data).await
     }
 
     /// Deallocate the external IP address with the provided ID.
     ///
     /// To support idempotency, such as in saga operations, this method returns
     /// an extra boolean, rather than the usual `DeleteResult`. The meaning of
     /// return values are:
-    ///
     /// - `Ok(true)`: The record was deleted during this call
     /// - `Ok(false)`: The record was already deleted, such as by a previous
     /// call

nexus/src/db/datastore/ip_pool.rs

@@ -21,10 +21,11 @@ use crate::db::model::IpPoolRange;
 use crate::db::model::IpPoolUpdate;
 use crate::db::model::Name;
 use crate::db::pagination::paginated;
+use crate::db::pool::DbConnection;
 use crate::db::queries::ip_pool::FilterOverlappingIpRanges;
 use crate::external_api::params;
 use crate::external_api::shared::IpRange;
-use async_bb8_diesel::AsyncRunQueryDsl;
+use async_bb8_diesel::{AsyncRunQueryDsl, PoolError};
 use chrono::Utc;
 use diesel::prelude::*;
 use ipnetwork::IpNetwork;
@@ -287,6 +288,24 @@ impl DataStore {
         authz_pool: &authz::IpPool,
         range: &IpRange,
     ) -> CreateResult<IpPoolRange> {
+        let conn = self.pool_authorized(opctx).await?;
+        Self::ip_pool_add_range_on_connection(conn, opctx, authz_pool, range)
+            .await
+    }
+
+    /// Variant of [Self::ip_pool_add_range] which may be called from a
+    /// transaction context.
+    pub(crate) async fn ip_pool_add_range_on_connection<ConnErr>(
+        conn: &(impl async_bb8_diesel::AsyncConnection<DbConnection, ConnErr>
+              + Sync),
+        opctx: &OpContext,
+        authz_pool: &authz::IpPool,
+        range: &IpRange,
+    ) -> CreateResult<IpPoolRange>
+    where
+        ConnErr: From<diesel::result::Error> + Send + 'static,
+        PoolError: From<ConnErr>,
+    {
         use db::schema::ip_pool_range::dsl;
         opctx.authorize(authz::Action::CreateChild, authz_pool).await?;
         let pool_id = authz_pool.id();
@@ -295,7 +314,7 @@ impl DataStore {
         let insert_query =
             diesel::insert_into(dsl::ip_pool_range).values(filter_subquery);
         IpPool::insert_resource(pool_id, insert_query)
-            .insert_and_get_result_async(self.pool_authorized(opctx).await?)
+            .insert_and_get_result_async(conn)
             .await
             .map_err(|e| {
                 use async_bb8_diesel::ConnectionError::Query;

nexus/src/db/datastore/mod.rs

@@ -81,6 +81,9 @@ pub use volume::CrucibleResources;
 // TODO: This should likely turn into a configuration option.
 pub(crate) const REGION_REDUNDANCY_THRESHOLD: usize = 3;
 
+/// The name of the built-in IP pool for Oxide services.
+pub const SERVICE_IP_POOL_NAME: &str = "oxide-service-pool";
+
 // Represents a query that is ready to be executed.
 //
 // This helper trait lets the statement either be executed or explained.
@@ -236,12 +239,20 @@ pub async fn datastore_test(
         authn::Context::internal_db_init(),
         Arc::clone(&datastore),
     );
+
+    // TODO: Can we just call "Populate" instead of doing this?
+    let rack_id = Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap();
     datastore.load_builtin_users(&opctx).await.unwrap();
     datastore.load_builtin_roles(&opctx).await.unwrap();
     datastore.load_builtin_role_asgns(&opctx).await.unwrap();
     datastore.load_builtin_silos(&opctx).await.unwrap();
     datastore.load_silo_users(&opctx).await.unwrap();
     datastore.load_silo_user_role_assignments(&opctx).await.unwrap();
+    datastore
+        .load_builtin_fleet_virtual_provisioning_collection(&opctx)
+        .await
+        .unwrap();
+    datastore.load_builtin_rack_data(&opctx, rack_id).await.unwrap();
 
     // Create an OpContext with the credentials of "test-privileged" for general
     // testing.
@@ -1067,14 +1078,28 @@ mod test {
 
         // Initialize the Rack.
         let result = datastore
-            .rack_set_initialized(&opctx, rack.id(), vec![], vec![], vec![])
+            .rack_set_initialized(
+                &opctx,
+                rack.id(),
+                vec![],
+                vec![],
+                vec![],
+                vec![],
+            )
             .await
             .unwrap();
         assert!(result.initialized);
 
         // Re-initialize the rack (check for idempotency)
         let result = datastore
-            .rack_set_initialized(&opctx, rack.id(), vec![], vec![], vec![])
+            .rack_set_initialized(
+                &opctx,
+                rack.id(),
+                vec![],
+                vec![],
+                vec![],
+                vec![],
+            )
             .await
             .unwrap();
         assert!(result.initialized);