diff --git a/common/src/address.rs b/common/src/address.rs index 48ec7c46edc..8e128103433 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -421,6 +421,14 @@ impl IpRange { IpRange::V6(ip6) => IpRangeIter::V6(ip6.iter()), } } + + // Has to be u128 to accommodate IPv6 + pub fn len(&self) -> u128 { + match self { + IpRange::V4(ip4) => u128::from(ip4.len()), + IpRange::V6(ip6) => ip6.len(), + } + } } impl From for IpRange { @@ -508,6 +516,12 @@ impl Ipv4Range { pub fn iter(&self) -> Ipv4RangeIter { Ipv4RangeIter { next: Some(self.first.into()), last: self.last.into() } } + + pub fn len(&self) -> u32 { + let start_num = u32::from(self.first); + let end_num = u32::from(self.last); + end_num - start_num + 1 + } } impl From for Ipv4Range { @@ -564,6 +578,12 @@ impl Ipv6Range { pub fn iter(&self) -> Ipv6RangeIter { Ipv6RangeIter { next: Some(self.first.into()), last: self.last.into() } } + + pub fn len(&self) -> u128 { + let start_num = u128::from(self.first); + let end_num = u128::from(self.last); + end_num - start_num + 1 + } } impl From for Ipv6Range { @@ -781,6 +801,19 @@ mod test { ); } + #[test] + fn test_ip_range_length() { + let lo = Ipv4Addr::new(10, 0, 0, 1); + let hi = Ipv4Addr::new(10, 0, 0, 3); + let range = IpRange::try_from((lo, hi)).unwrap(); + assert_eq!(range.len(), 3); + + let lo = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1); + let hi = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 1, 3); + let range = IpRange::try_from((lo, hi)).unwrap(); + assert_eq!(range.len(), 2u128.pow(16) + 3); + } + #[test] fn test_ipv6_subnet_deserialize() { let value = json!({ diff --git a/nexus/db-model/src/utilization.rs b/nexus/db-model/src/utilization.rs index b0e6324bc9b..e82600d64dc 100644 --- a/nexus/db-model/src/utilization.rs +++ b/nexus/db-model/src/utilization.rs @@ -55,3 +55,40 @@ impl From for views::Utilization { } } } + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Ipv4Utilization { + pub allocated: u32, + pub capacity: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Ipv6Utilization { + pub allocated: u128, + pub capacity: u128, +} + +// Not really a DB model, just the result of a datastore function +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IpPoolUtilization { + pub ipv4: Ipv4Utilization, + pub ipv6: Ipv6Utilization, +} + +impl From for views::Ipv4Utilization { + fn from(util: Ipv4Utilization) -> Self { + Self { allocated: util.allocated, capacity: util.capacity } + } +} + +impl From for views::Ipv6Utilization { + fn from(util: Ipv6Utilization) -> Self { + Self { allocated: util.allocated, capacity: util.capacity } + } +} + +impl From for views::IpPoolUtilization { + fn from(util: IpPoolUtilization) -> Self { + Self { ipv4: util.ipv4.into(), ipv6: util.ipv6.into() } + } +} diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs index a9062e6ee3e..c0f4e0eea87 100644 --- a/nexus/db-queries/src/db/datastore/ip_pool.rs +++ b/nexus/db-queries/src/db/datastore/ip_pool.rs @@ -51,6 +51,16 @@ use omicron_common::api::external::UpdateResult; use ref_cast::RefCast; use uuid::Uuid; +pub struct IpsAllocated { + pub ipv4: i64, + pub ipv6: i64, +} + +pub struct IpsCapacity { + pub ipv4: u32, + pub ipv6: u128, +} + impl DataStore { /// List IP Pools pub async fn ip_pools_list( @@ -324,6 +334,80 @@ impl DataStore { }) } + pub async fn ip_pool_allocated_count( + &self, + opctx: &OpContext, + authz_pool: &authz::IpPool, + ) -> Result { + opctx.authorize(authz::Action::Read, authz_pool).await?; + + use db::schema::external_ip; + use diesel::dsl::sql; + use diesel::sql_types::BigInt; + + let (ipv4, ipv6) = external_ip::table + .filter(external_ip::ip_pool_id.eq(authz_pool.id())) + .filter(external_ip::time_deleted.is_null()) + .select(( + sql::("count(*) FILTER (WHERE family(ip) = 4)"), + sql::("count(*) FILTER (WHERE family(ip) = 6)"), + )) + .first_async::<(i64, i64)>( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(IpsAllocated { ipv4, ipv6 }) + } + + pub async fn ip_pool_total_capacity( + &self, + opctx: &OpContext, + authz_pool: &authz::IpPool, + ) -> Result { + opctx.authorize(authz::Action::Read, authz_pool).await?; + opctx.authorize(authz::Action::ListChildren, authz_pool).await?; + + use db::schema::ip_pool_range; + + let ranges = ip_pool_range::table + .filter(ip_pool_range::ip_pool_id.eq(authz_pool.id())) + .filter(ip_pool_range::time_deleted.is_null()) + .select(IpPoolRange::as_select()) + // This is a rare unpaginated DB query, which means we are + // vulnerable to a resource exhaustion attack in which someone + // creates a very large number of ranges in order to make this + // query slow. In order to mitigate that, we limit the query to the + // (current) max allowed page size, effectively making this query + // exactly as vulnerable as if it were paginated. If there are more + // than 10,000 ranges in a pool, we will undercount, but I have a + // hard time seeing that as a practical problem. + .limit(10000) + .get_results_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_pool), + ) + })?; + + let mut ipv4: u32 = 0; + let mut ipv6: u128 = 0; + + for range in &ranges { + let r = IpRange::from(range); + match r { + IpRange::V4(r) => ipv4 += r.len(), + IpRange::V6(r) => ipv6 += r.len(), + } + } + Ok(IpsCapacity { ipv4, ipv6 }) + } + pub async fn ip_pool_silo_list( &self, opctx: &OpContext, @@ -846,10 +930,14 @@ mod test { use crate::authz; use crate::db::datastore::test_utils::datastore_test; - use crate::db::model::{IpPool, IpPoolResource, IpPoolResourceType}; + use crate::db::model::{ + IpPool, IpPoolResource, IpPoolResourceType, Project, + }; use assert_matches::assert_matches; use nexus_test_utils::db::test_setup_database; + use nexus_types::external_api::params; use nexus_types::identity::Resource; + use omicron_common::address::{IpRange, Ipv4Range, Ipv6Range}; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::{ DataPageParams, Error, IdentityMetadataCreateParams, LookupType, @@ -1075,4 +1163,155 @@ mod test { db.cleanup().await.unwrap(); logctx.cleanup_successful(); } + + #[tokio::test] + async fn test_ip_pool_utilization() { + let logctx = dev::test_setup_log("test_ip_utilization"); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + let authz_silo = opctx.authn.silo_required().unwrap(); + let project = Project::new( + authz_silo.id(), + params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "my-project".parse().unwrap(), + description: "".to_string(), + }, + }, + ); + let (.., project) = + datastore.project_create(&opctx, project).await.unwrap(); + + // create an IP pool for the silo, add a range to it, and link it to the silo + let identity = IdentityMetadataCreateParams { + name: "my-pool".parse().unwrap(), + description: "".to_string(), + }; + let pool = datastore + .ip_pool_create(&opctx, IpPool::new(&identity)) + .await + .expect("Failed to create IP pool"); + let authz_pool = authz::IpPool::new( + authz::FLEET, + pool.id(), + LookupType::ById(pool.id()), + ); + + // capacity of zero because there are no ranges + let max_ips = datastore + .ip_pool_total_capacity(&opctx, &authz_pool) + .await + .unwrap(); + assert_eq!(max_ips.ipv4, 0); + assert_eq!(max_ips.ipv6, 0); + + let range = IpRange::V4( + Ipv4Range::new( + std::net::Ipv4Addr::new(10, 0, 0, 1), + std::net::Ipv4Addr::new(10, 0, 0, 5), + ) + .unwrap(), + ); + datastore + .ip_pool_add_range(&opctx, &authz_pool, &range) + .await + .expect("Could not add range"); + + // now it has a capacity of 5 because we added the range + let max_ips = datastore + .ip_pool_total_capacity(&opctx, &authz_pool) + .await + .unwrap(); + assert_eq!(max_ips.ipv4, 5); + assert_eq!(max_ips.ipv6, 0); + + let link = IpPoolResource { + ip_pool_id: pool.id(), + resource_type: IpPoolResourceType::Silo, + resource_id: authz_silo.id(), + is_default: true, + }; + datastore + .ip_pool_link_silo(&opctx, link) + .await + .expect("Could not link pool to silo"); + + let ip_count = datastore + .ip_pool_allocated_count(&opctx, &authz_pool) + .await + .unwrap(); + assert_eq!(ip_count.ipv4, 0); + assert_eq!(ip_count.ipv6, 0); + + let identity = IdentityMetadataCreateParams { + name: "my-ip".parse().unwrap(), + description: "".to_string(), + }; + let ip = datastore + .allocate_floating_ip(&opctx, project.id(), identity, None, None) + .await + .expect("Could not allocate floating IP"); + assert_eq!(ip.ip.to_string(), "10.0.0.1/32"); + + let ip_count = datastore + .ip_pool_allocated_count(&opctx, &authz_pool) + .await + .unwrap(); + assert_eq!(ip_count.ipv4, 1); + assert_eq!(ip_count.ipv6, 0); + + // allocating one has nothing to do with total capacity + let max_ips = datastore + .ip_pool_total_capacity(&opctx, &authz_pool) + .await + .unwrap(); + assert_eq!(max_ips.ipv4, 5); + assert_eq!(max_ips.ipv6, 0); + + let ipv6_range = IpRange::V6( + Ipv6Range::new( + std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 10), + std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 1, 20), + ) + .unwrap(), + ); + datastore + .ip_pool_add_range(&opctx, &authz_pool, &ipv6_range) + .await + .expect("Could not add range"); + + // now test with additional v6 range + let max_ips = datastore + .ip_pool_total_capacity(&opctx, &authz_pool) + .await + .unwrap(); + assert_eq!(max_ips.ipv4, 5); + assert_eq!(max_ips.ipv6, 11 + 65536); + + // add a giant range for fun + let ipv6_range = IpRange::V6( + Ipv6Range::new( + std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 1, 21), + std::net::Ipv6Addr::new( + 0xfd00, 0, 0, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ), + ) + .unwrap(), + ); + datastore + .ip_pool_add_range(&opctx, &authz_pool, &ipv6_range) + .await + .expect("Could not add range"); + + let max_ips = datastore + .ip_pool_total_capacity(&opctx, &authz_pool) + .await + .unwrap(); + assert_eq!(max_ips.ipv4, 5); + assert_eq!(max_ips.ipv6, 1208925819614629174706166); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } } diff --git a/nexus/src/app/utilization.rs b/nexus/src/app/utilization.rs index 526ebc9470e..bd5f5d002ab 100644 --- a/nexus/src/app/utilization.rs +++ b/nexus/src/app/utilization.rs @@ -4,6 +4,9 @@ //! Insights into capacity and utilization +use nexus_db_model::IpPoolUtilization; +use nexus_db_model::Ipv4Utilization; +use nexus_db_model::Ipv6Utilization; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -30,4 +33,55 @@ impl super::Nexus { ) -> ListResultVec { self.db_datastore.silo_utilization_list(opctx, pagparams).await } + + pub async fn ip_pool_utilization_view( + &self, + opctx: &OpContext, + pool_lookup: &lookup::IpPool<'_>, + ) -> Result { + let (.., authz_pool) = + pool_lookup.lookup_for(authz::Action::Read).await?; + + let allocated = self + .db_datastore + .ip_pool_allocated_count(opctx, &authz_pool) + .await?; + let capacity = self + .db_datastore + .ip_pool_total_capacity(opctx, &authz_pool) + .await?; + + // we have one query for v4 and v6 allocated and one for both v4 and + // v6 capacity so we can do two queries instead 4, but in the response + // we want them paired by IP version + Ok(IpPoolUtilization { + ipv4: Ipv4Utilization { + // This one less trivial to justify than the u128 conversion + // below because an i64 could obviously be too big for u32. + // In addition to the fact that it is unlikely for anyone to + // allocate 4 billion IPs, we rely on the fact that there can + // only be 2^32 IPv4 addresses, period. + allocated: u32::try_from(allocated.ipv4).map_err(|_e| { + Error::internal_error(&format!( + "Failed to convert i64 {} IPv4 address count to u32", + allocated.ipv4 + )) + })?, + capacity: capacity.ipv4, + }, + ipv6: Ipv6Utilization { + // SQL represents counts as signed integers for historical + // or compatibility reasons even though they can't really be + // negative, and Diesel follows that. We assume here that it + // will be a positive i64. + allocated: u128::try_from(allocated.ipv6).map_err(|_e| { + Error::internal_error(&format!( + "Failed to convert i64 {} IPv6 address count to u128", + allocated.ipv6 + )) + })?, + capacity: capacity.ipv6, + }, + }) + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index aa037e072f6..9c8d9cef50a 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -130,6 +130,7 @@ pub(crate) fn external_api() -> NexusApiDescription { api.register(ip_pool_update)?; // Variants for internal services api.register(ip_pool_service_view)?; + api.register(ip_pool_utilization_view)?; // Operator-Accessible IP Pool Range API api.register(ip_pool_range_list)?; @@ -1557,6 +1558,31 @@ async fn ip_pool_update( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } +/// Fetch IP pool utilization +#[endpoint { + method = GET, + path = "/v1/system/ip-pools/{pool}/utilization", + tags = ["system/networking"], +}] +async fn ip_pool_utilization_view( + rqctx: RequestContext>, + path_params: Path, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.nexus; + let pool_selector = path_params.into_inner().pool; + // We do not prevent the service pool from being fetched by name or ID + // like we do for update, delete, associate. + let pool_lookup = nexus.ip_pool_lookup(&opctx, &pool_selector)?; + let utilization = + nexus.ip_pool_utilization_view(&opctx, &pool_lookup).await?; + Ok(HttpResponseOk(utilization.into())) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + /// List IP pool's linked silos #[endpoint { method = GET, diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index f4b5d9fa463..b67028a9960 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -631,6 +631,38 @@ pub async fn create_router( .unwrap() } +pub async fn assert_ip_pool_utilization( + client: &ClientTestContext, + pool_name: &str, + ipv4_allocated: u32, + ipv4_capacity: u32, + ipv6_allocated: u128, + ipv6_capacity: u128, +) { + let url = format!("/v1/system/ip-pools/{}/utilization", pool_name); + let utilization: views::IpPoolUtilization = object_get(client, &url).await; + assert_eq!( + utilization.ipv4.allocated, ipv4_allocated, + "IP pool '{}': expected {} IPv4 allocated, got {:?}", + pool_name, ipv4_allocated, utilization.ipv4.allocated + ); + assert_eq!( + utilization.ipv4.capacity, ipv4_capacity, + "IP pool '{}': expected {} IPv4 capacity, got {:?}", + pool_name, ipv4_capacity, utilization.ipv4.capacity + ); + assert_eq!( + utilization.ipv6.allocated, ipv6_allocated, + "IP pool '{}': expected {} IPv6 allocated, got {:?}", + pool_name, ipv6_allocated, utilization.ipv6.allocated + ); + assert_eq!( + utilization.ipv6.capacity, ipv6_capacity, + "IP pool '{}': expected {} IPv6 capacity, got {:?}", + pool_name, ipv6_capacity, utilization.ipv6.capacity + ); +} + /// Grant a role on a resource to a user /// /// * `grant_resource_url`: URL of the resource we're granting the role on diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index f18b2d961d2..7d2ff2596ab 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -656,6 +656,8 @@ pub static DEMO_IP_POOL_PROJ_URL: Lazy = Lazy::new(|| { }); pub static DEMO_IP_POOL_URL: Lazy = Lazy::new(|| format!("/v1/system/ip-pools/{}", *DEMO_IP_POOL_NAME)); +pub static DEMO_IP_POOL_UTILIZATION_URL: Lazy = + Lazy::new(|| format!("{}/utilization", *DEMO_IP_POOL_URL)); pub static DEMO_IP_POOL_UPDATE: Lazy = Lazy::new(|| params::IpPoolUpdate { identity: IdentityMetadataUpdateParams { @@ -1104,6 +1106,16 @@ pub static VERIFY_ENDPOINTS: Lazy> = Lazy::new(|| { ], }, + // IP pool utilization + VerifyEndpoint { + url: &DEMO_IP_POOL_UTILIZATION_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Get, + ], + }, + // IP Pool endpoint (Oxide services) VerifyEndpoint { url: &DEMO_IP_POOL_SERVICE_URL, diff --git a/nexus/tests/integration_tests/external_ips.rs b/nexus/tests/integration_tests/external_ips.rs index 25c74bfd7f5..fa884f8731c 100644 --- a/nexus/tests/integration_tests/external_ips.rs +++ b/nexus/tests/integration_tests/external_ips.rs @@ -17,6 +17,7 @@ use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::assert_ip_pool_utilization; use nexus_test_utils::resource_helpers::create_default_ip_pool; use nexus_test_utils::resource_helpers::create_floating_ip; use nexus_test_utils::resource_helpers::create_instance_with; @@ -151,6 +152,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) { // automatically linked to current silo create_default_ip_pool(&client).await; + assert_ip_pool_utilization(client, "default", 0, 65536, 0, 0).await; + let other_pool_range = IpRange::V4( Ipv4Range::new(Ipv4Addr::new(10, 1, 0, 1), Ipv4Addr::new(10, 1, 0, 5)) .unwrap(), @@ -158,6 +161,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) { // not automatically linked to currently silo. see below create_ip_pool(&client, "other-pool", Some(other_pool_range)).await; + assert_ip_pool_utilization(client, "other-pool", 0, 5, 0, 0).await; + let project = create_project(client, PROJECT_NAME).await; // Create with no chosen IP and fallback to default pool. @@ -175,6 +180,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) { assert_eq!(fip.instance_id, None); assert_eq!(fip.ip, IpAddr::from(Ipv4Addr::new(10, 0, 0, 0))); + assert_ip_pool_utilization(client, "default", 1, 65536, 0, 0).await; + // Create with chosen IP and fallback to default pool. let fip_name = FIP_NAMES[1]; let ip_addr = "10.0.12.34".parse().unwrap(); @@ -191,6 +198,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) { assert_eq!(fip.instance_id, None); assert_eq!(fip.ip, ip_addr); + assert_ip_pool_utilization(client, "default", 2, 65536, 0, 0).await; + // Creating with other-pool fails with 404 until it is linked to the current silo let fip_name = FIP_NAMES[2]; let params = params::FloatingIpCreate { @@ -206,6 +215,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) { object_create_error(client, &url, ¶ms, StatusCode::NOT_FOUND).await; assert_eq!(error.message, "not found: ip-pool with name \"other-pool\""); + assert_ip_pool_utilization(client, "other-pool", 0, 5, 0, 0).await; + // now link the pool and everything should work with the exact same params let silo_id = DEFAULT_SILO.id(); link_ip_pool(&client, "other-pool", &silo_id, false).await; @@ -217,6 +228,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) { assert_eq!(fip.instance_id, None); assert_eq!(fip.ip, IpAddr::from(Ipv4Addr::new(10, 1, 0, 1))); + assert_ip_pool_utilization(client, "other-pool", 1, 5, 0, 0).await; + // Create with chosen IP from fleet-scoped named pool. let fip_name = FIP_NAMES[3]; let ip_addr = "10.1.0.5".parse().unwrap(); @@ -232,6 +245,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) { assert_eq!(fip.project_id, project.identity.id); assert_eq!(fip.instance_id, None); assert_eq!(fip.ip, ip_addr); + + assert_ip_pool_utilization(client, "other-pool", 2, 5, 0, 0).await; } #[nexus_test] @@ -586,6 +601,8 @@ async fn test_external_ip_live_attach_detach( create_default_ip_pool(&client).await; let project = create_project(client, PROJECT_NAME).await; + assert_ip_pool_utilization(client, "default", 0, 65536, 0, 0).await; + // Create 2 instances, and a floating IP for each instance. // One instance will be started, and one will be stopped. let mut fips = vec![]; @@ -602,6 +619,9 @@ async fn test_external_ip_live_attach_detach( ); } + // 2 floating IPs have been allocated + assert_ip_pool_utilization(client, "default", 2, 65536, 0, 0).await; + let mut instances = vec![]; for (i, start) in [false, true].iter().enumerate() { let instance = instance_for_external_ips( @@ -633,6 +653,10 @@ async fn test_external_ip_live_attach_detach( instances.push(instance); } + // the two instances above were deliberately not given ephemeral IPs, but + // they still always get SNAT IPs, so we went from 2 to 4 + assert_ip_pool_utilization(client, "default", 4, 65536, 0, 0).await; + // Attach a floating IP and ephemeral IP to each instance. let mut recorded_ephs = vec![]; for (instance, fip) in instances.iter().zip(&fips) { @@ -675,6 +699,10 @@ async fn test_external_ip_live_attach_detach( recorded_ephs.push(eph_resp); } + // now 6 because an ephemeral IP was added for each instance. floating IPs + // were attached, but they were already allocated + assert_ip_pool_utilization(client, "default", 6, 65536, 0, 0).await; + // Detach a floating IP and ephemeral IP from each instance. for (instance, fip) in instances.iter().zip(&fips) { let instance_name = instance.identity.name.as_str(); @@ -705,6 +733,9 @@ async fn test_external_ip_live_attach_detach( ); } + // 2 ephemeral go away on detachment but still 2 floating and 2 SNAT + assert_ip_pool_utilization(client, "default", 4, 65536, 0, 0).await; + // Finally, two kind of funny tests. There is special logic in the handler // for the case where the floating IP is specified by name but the instance // by ID and vice versa, so we want to test both combinations. @@ -763,6 +794,9 @@ async fn test_external_ip_live_attach_detach( fetch_instance_external_ips(client, instance_name, PROJECT_NAME).await; assert_eq!(eip_list.len(), 1); assert_eq!(eip_list[0].ip(), fips[1].ip); + + // none of that changed the number of allocated IPs + assert_ip_pool_utilization(client, "default", 4, 65536, 0, 0).await; } #[nexus_test] diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 309036256f9..5d8bcc2136a 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -20,6 +20,7 @@ use nexus_test_interface::NexusServer; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::assert_ip_pool_utilization; use nexus_test_utils::resource_helpers::create_default_ip_pool; use nexus_test_utils::resource_helpers::create_disk; use nexus_test_utils::resource_helpers::create_floating_ip; @@ -3905,10 +3906,14 @@ async fn test_instance_ephemeral_ip_from_correct_pool( create_ip_pool(&client, "pool1", Some(range1)).await; link_ip_pool(&client, "pool1", &DEFAULT_SILO.id(), /*default*/ true).await; + assert_ip_pool_utilization(client, "pool1", 0, 5, 0, 0).await; + // second pool is associated with the silo but not default create_ip_pool(&client, "pool2", Some(range2)).await; link_ip_pool(&client, "pool2", &DEFAULT_SILO.id(), /*default*/ false).await; + assert_ip_pool_utilization(client, "pool2", 0, 5, 0, 0).await; + // Create an instance with pool name blank, expect IP from default pool create_instance_with_pool(client, "pool1-inst", None).await; @@ -3917,6 +3922,10 @@ async fn test_instance_ephemeral_ip_from_correct_pool( ip.ip() >= range1.first_address() && ip.ip() <= range1.last_address(), "Expected ephemeral IP to come from pool1" ); + // 1 ephemeral + 1 snat + assert_ip_pool_utilization(client, "pool1", 2, 5, 0, 0).await; + // pool2 unaffected + assert_ip_pool_utilization(client, "pool2", 0, 5, 0, 0).await; // Create an instance explicitly using the non-default "other-pool". create_instance_with_pool(client, "pool2-inst", Some("pool2")).await; @@ -3926,6 +3935,13 @@ async fn test_instance_ephemeral_ip_from_correct_pool( "Expected ephemeral IP to come from pool2" ); + // added 1 snat because snat comes from default pool. + // https://github.com/oxidecomputer/omicron/issues/5043 is about changing that + assert_ip_pool_utilization(client, "pool1", 3, 5, 0, 0).await; + + // ephemeral IP comes from specified pool + assert_ip_pool_utilization(client, "pool2", 1, 5, 0, 0).await; + // make pool2 default and create instance with default pool. check that it now it comes from pool2 let _: views::IpPoolSiloLink = object_put( client, @@ -3941,6 +3957,11 @@ async fn test_instance_ephemeral_ip_from_correct_pool( "Expected ephemeral IP to come from pool2" ); + // pool1 unchanged + assert_ip_pool_utilization(client, "pool1", 3, 5, 0, 0).await; + // +1 snat (now that pool2 is default) and +1 ephemeral + assert_ip_pool_utilization(client, "pool2", 3, 5, 0, 0).await; + // try to delete association with pool1, but it fails because there is an // instance with an IP from the pool in this silo let pool1_silo_url = @@ -3956,8 +3977,14 @@ async fn test_instance_ephemeral_ip_from_correct_pool( // stop and delete instances with IPs from pool1. perhaps surprisingly, that // includes pool2-inst also because the SNAT IP comes from the default pool // even when different pool is specified for the ephemeral IP - stop_instance(&cptestctx, "pool1-inst").await; - stop_instance(&cptestctx, "pool2-inst").await; + stop_and_delete_instance(&cptestctx, "pool1-inst").await; + stop_and_delete_instance(&cptestctx, "pool2-inst").await; + + // pool1 is down to 0 because it had 1 snat + 1 ephemeral from pool1-inst + // and 1 snat from pool2-inst + assert_ip_pool_utilization(client, "pool1", 0, 5, 0, 0).await; + // pool2 drops one because it had 1 ephemeral from pool2-inst + assert_ip_pool_utilization(client, "pool2", 2, 5, 0, 0).await; // now unlink works object_delete(client, &pool1_silo_url).await; @@ -3992,7 +4019,7 @@ async fn test_instance_ephemeral_ip_from_correct_pool( assert_eq!(error.message, "not found: ip-pool with name \"pool1\""); } -async fn stop_instance( +async fn stop_and_delete_instance( cptestctx: &ControlPlaneTestContext, instance_name: &str, ) { @@ -4146,9 +4173,7 @@ async fn test_instance_attach_several_external_ips( create_ip_pool(&client, "default", Some(default_pool_range)).await; link_ip_pool(&client, "default", &DEFAULT_SILO.id(), true).await; - // this doesn't work as a replacement for the above. figure out why and - // probably delete it - // create_default_ip_pool(&client).await; + assert_ip_pool_utilization(client, "default", 0, 10, 0, 0).await; // Create several floating IPs for the instance, totalling 8 IPs. let mut external_ip_create = @@ -4177,6 +4202,9 @@ async fn test_instance_attach_several_external_ips( ) .await; + // 1 ephemeral + 7 floating + 1 SNAT + assert_ip_pool_utilization(client, "default", 9, 10, 0, 0).await; + // Verify that all external IPs are visible on the instance and have // been allocated in order. let external_ips = @@ -4331,6 +4359,8 @@ async fn test_instance_create_in_silo(cptestctx: &ControlPlaneTestContext) { create_ip_pool(&client, "default", None).await; link_ip_pool(&client, "default", &silo.identity.id, true).await; + assert_ip_pool_utilization(client, "default", 0, 65536, 0, 0).await; + // Create test projects NexusRequest::objects_post( client, diff --git a/nexus/tests/integration_tests/ip_pools.rs b/nexus/tests/integration_tests/ip_pools.rs index f1d8825d0e8..c8390e8ce0d 100644 --- a/nexus/tests/integration_tests/ip_pools.rs +++ b/nexus/tests/integration_tests/ip_pools.rs @@ -15,6 +15,7 @@ use nexus_db_queries::db::fixed_data::silo::INTERNAL_SILO_ID; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::assert_ip_pool_utilization; use nexus_test_utils::resource_helpers::create_instance; use nexus_test_utils::resource_helpers::create_ip_pool; use nexus_test_utils::resource_helpers::create_project; @@ -757,6 +758,47 @@ async fn create_pool(client: &ClientTestContext, name: &str) -> IpPool { .unwrap() } +// This is mostly about testing the total field with huge numbers. +// testing allocated is done in a bunch of other places. look for +// assert_ip_pool_utilization calls +#[nexus_test] +async fn test_ip_pool_utilization_total(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + + create_pool(client, "p0").await; + + assert_ip_pool_utilization(client, "p0", 0, 0, 0, 0).await; + + let add_url = "/v1/system/ip-pools/p0/ranges/add"; + + // add just 5 addresses to get the party started + let range = IpRange::V4( + Ipv4Range::new( + std::net::Ipv4Addr::new(10, 0, 0, 1), + std::net::Ipv4Addr::new(10, 0, 0, 5), + ) + .unwrap(), + ); + object_create::(client, &add_url, &range).await; + + assert_ip_pool_utilization(client, "p0", 0, 5, 0, 0).await; + + // now let's add a gigantic range just for fun + let big_range = IpRange::V6( + Ipv6Range::new( + std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1), + std::net::Ipv6Addr::new( + 0xfd00, 0, 0, 0, 0xffff, 0xfff, 0xffff, 0xffff, + ), + ) + .unwrap(), + ); + object_create::(client, &add_url, &big_range).await; + + assert_ip_pool_utilization(client, "p0", 0, 5, 0, 18446480190918885375) + .await; +} + // Data for testing overlapping IP ranges struct TestRange { // A starting IP range that should be inserted correctly diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 3b954ec6ec6..ba79d75f224 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -164,6 +164,7 @@ ip_pool_silo_list GET /v1/system/ip-pools/{pool}/sil ip_pool_silo_unlink DELETE /v1/system/ip-pools/{pool}/silos/{silo} ip_pool_silo_update PUT /v1/system/ip-pools/{pool}/silos/{silo} ip_pool_update PUT /v1/system/ip-pools/{pool} +ip_pool_utilization_view GET /v1/system/ip-pools/{pool}/utilization ip_pool_view GET /v1/system/ip-pools/{pool} networking_address_lot_block_list GET /v1/system/networking/address-lot/{address_lot}/blocks networking_address_lot_create POST /v1/system/networking/address-lot diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index a3e87b162e9..fcea302f725 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -305,6 +305,82 @@ pub struct IpPool { pub identity: IdentityMetadata, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct Ipv4Utilization { + /// The number of IPv4 addresses allocated from this pool + pub allocated: u32, + /// The total number of IPv4 addresses in the pool, i.e., the sum of the + /// lengths of the IPv4 ranges. Unlike IPv6 capacity, can be a 32-bit + /// integer because there are only 2^32 IPv4 addresses. + pub capacity: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct Ipv6Utilization { + /// The number of IPv6 addresses allocated from this pool. A 128-bit integer + /// string to match the capacity field. + #[serde(with = "U128String")] + pub allocated: u128, + + /// The total number of IPv6 addresses in the pool, i.e., the sum of the + /// lengths of the IPv6 ranges. An IPv6 range can contain up to 2^128 + /// addresses, so we represent this value in JSON as a numeric string with a + /// custom "uint128" format. + #[serde(with = "U128String")] + pub capacity: u128, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct IpPoolUtilization { + /// Number of allocated and total available IPv4 addresses in pool + pub ipv4: Ipv4Utilization, + /// Number of allocated and total available IPv6 addresses in pool + pub ipv6: Ipv6Utilization, +} + +// Custom struct for serializing/deserializing u128 as a string. The serde +// docs will suggest using a module (or serialize_with and deserialize_with +// functions), but as discussed in the comments on the UserData de/serializer, +// schemars wants this to be a type, so it has to be a struct. +struct U128String; +impl U128String { + pub fn serialize(value: &u128, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&value.to_string()) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + s.parse().map_err(serde::de::Error::custom) + } +} + +impl JsonSchema for U128String { + fn schema_name() -> String { + "String".to_string() + } + + fn json_schema( + _: &mut schemars::gen::SchemaGenerator, + ) -> schemars::schema::Schema { + schemars::schema::SchemaObject { + instance_type: Some(schemars::schema::InstanceType::String.into()), + format: Some("uint128".to_string()), + ..Default::default() + } + .into() + } + + fn is_referenceable() -> bool { + false + } +} + /// An IP pool in the context of a silo #[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct SiloIpPool { diff --git a/openapi/nexus.json b/openapi/nexus.json index cacd875accc..7516a896a74 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -5723,6 +5723,44 @@ } } }, + "/v1/system/ip-pools/{pool}/utilization": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Fetch IP pool utilization", + "operationId": "ip_pool_utilization_view", + "parameters": [ + { + "in": "path", + "name": "pool", + "description": "Name or ID of the IP pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IpPoolUtilization" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/system/ip-pools-service": { "get": { "tags": [ @@ -13652,6 +13690,31 @@ } } }, + "IpPoolUtilization": { + "type": "object", + "properties": { + "ipv4": { + "description": "Number of allocated and total available IPv4 addresses in pool", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Utilization" + } + ] + }, + "ipv6": { + "description": "Number of allocated and total available IPv6 addresses in pool", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Utilization" + } + ] + } + }, + "required": [ + "ipv4", + "ipv6" + ] + }, "IpRange": { "oneOf": [ { @@ -13697,6 +13760,27 @@ "last" ] }, + "Ipv4Utilization": { + "type": "object", + "properties": { + "allocated": { + "description": "The number of IPv4 addresses allocated from this pool", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "capacity": { + "description": "The total number of IPv4 addresses in the pool, i.e., the sum of the lengths of the IPv4 ranges. Unlike IPv6 capacity, can be a 32-bit integer because there are only 2^32 IPv4 addresses.", + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "allocated", + "capacity" + ] + }, "Ipv6Net": { "example": "fd12:3456::/64", "title": "An IPv6 subnet", @@ -13722,6 +13806,25 @@ "last" ] }, + "Ipv6Utilization": { + "type": "object", + "properties": { + "allocated": { + "description": "The number of IPv6 addresses allocated from this pool. A 128-bit integer string to match the capacity field.", + "type": "string", + "format": "uint128" + }, + "capacity": { + "description": "The total number of IPv6 addresses in the pool, i.e., the sum of the lengths of the IPv6 ranges. An IPv6 range can contain up to 2^128 addresses, so we represent this value in JSON as a numeric string with a custom \"uint128\" format.", + "type": "string", + "format": "uint128" + } + }, + "required": [ + "allocated", + "capacity" + ] + }, "L4PortRange": { "example": "22", "title": "A range of IP ports",