From fe87906420c670982529daf4caa7d060b2a22fd5 Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Tue, 11 Nov 2025 18:29:23 +0000 Subject: [PATCH 1/5] Add dual-stack IP config for internal NICs - Add dual-stack VPC private address configuration type and include it in the shared NetworkInterface type. - Add database model support for reading / writing the dual-stack NIC type to the database, handling serialization into optional fields. - Update all the callsites to handle the new dual-stack-aware type. - Add a bunch of conversions for the APIs which rely on that new type, of which there are many. This also adds the conversions and older types into the `sled-agent-types` crate, so they can be used in a few places that don't directly depend on the `sled-agent-api` crate itself, notably reconfigurator and `nexus-inventory`. - Update the sled-agent reconciler to deserialize previous versions of its sled-configuration ledgers, convert them, and write them out again as the new versions. - Updates the OPTE `PortManager` type with the new dual-stack support. This is only for private IP addresses, though. We still need some work to support OPTE ports with dual stack external addresses. This is about half of #9247, the VPC-private part. - Closes #9246 --- .../api/internal/{shared.rs => shared/mod.rs} | 64 +- .../internal/shared/network_interface/mod.rs | 39 + .../internal/shared/network_interface/v1.rs | 46 + .../internal/shared/network_interface/v2.rs | 445 + dev-tools/reconfigurator-cli/src/lib.rs | 24 +- illumos-utils/src/opte/mod.rs | 117 +- illumos-utils/src/opte/port.rs | 48 +- illumos-utils/src/opte/port_manager.rs | 593 +- illumos-utils/src/running_zone.rs | 4 +- nexus/db-model/src/network_interface.rs | 146 +- nexus/db-model/src/omicron_zone_config.rs | 51 +- .../db-queries/src/db/datastore/deployment.rs | 14 +- .../deployment/external_networking.rs | 151 +- .../src/db/datastore/network_interface.rs | 101 +- nexus/db-queries/src/db/datastore/probe.rs | 49 +- nexus/db-queries/src/db/datastore/rack.rs | 141 +- nexus/db-queries/src/db/datastore/vpc.rs | 9 +- nexus/inventory/src/examples.rs | 18 +- nexus/networking/src/firewall_rules.rs | 39 +- nexus/reconfigurator/blippy/src/checks.rs | 72 +- .../tests/integration/blueprint_edit.rs | 35 +- .../reconfigurator/execution/src/database.rs | 12 +- .../planning/src/blueprint_builder/builder.rs | 36 +- .../allocators/external_networking.rs | 193 +- nexus/reconfigurator/planning/src/example.rs | 20 +- nexus/reconfigurator/planning/src/planner.rs | 15 +- .../output/planner_nonprovisionable_2_2a.txt | 18 +- nexus/test-utils/src/lib.rs | 48 +- nexus/tests/integration_tests/vpc_routers.rs | 17 +- .../types/src/deployment/network_resources.rs | 2 + nexus/types/src/external_api/shared.rs | 9 +- openapi/nexus-lockstep.json | 142 +- .../sled-agent/sled-agent-7.0.0-90da02.json | 8699 +++++++++++++++++ openapi/sled-agent/sled-agent-latest.json | 2 +- sled-agent/api/src/lib.rs | 101 +- sled-agent/config-reconciler/src/ledger.rs | 19 +- .../src/ledger/legacy_configs.rs | 186 +- sled-agent/src/rack_setup/plan/service.rs | 75 +- sled-agent/src/services.rs | 85 +- sled-agent/src/sim/server.rs | 33 +- sled-agent/src/sim/sled_agent.rs | 22 +- sled-agent/types/src/inventory/mod.rs | 11 + .../{api/src => types/src/inventory}/v3.rs | 280 +- sled-agent/types/src/inventory/v6.rs | 770 ++ sled-agent/types/src/lib.rs | 1 + .../types/src/{probes.rs => probes/mod.rs} | 3 + sled-agent/types/src/probes/v1.rs | 72 + 47 files changed, 12062 insertions(+), 1015 deletions(-) rename common/src/api/internal/{shared.rs => shared/mod.rs} (96%) create mode 100644 common/src/api/internal/shared/network_interface/mod.rs create mode 100644 common/src/api/internal/shared/network_interface/v1.rs create mode 100644 common/src/api/internal/shared/network_interface/v2.rs create mode 100644 openapi/sled-agent/sled-agent-7.0.0-90da02.json create mode 100644 sled-agent/types/src/inventory/mod.rs rename sled-agent/{api/src => types/src/inventory}/v3.rs (67%) create mode 100644 sled-agent/types/src/inventory/v6.rs rename sled-agent/types/src/{probes.rs => probes/mod.rs} (97%) create mode 100644 sled-agent/types/src/probes/v1.rs diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared/mod.rs similarity index 96% rename from common/src/api/internal/shared.rs rename to common/src/api/internal/shared/mod.rs index 65c9a47e554..00720ce598d 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared/mod.rs @@ -4,6 +4,7 @@ //! Types shared between Nexus and Sled Agent. +use super::nexus::HostIdentifier; use crate::{ address::NUM_SOURCE_NAT_PORTS, api::external::{self, BfdMode, ImportExportPolicy, Name, Vni}, @@ -21,60 +22,11 @@ use std::{ use strum::EnumCount; use uuid::Uuid; -use super::nexus::HostIdentifier; - -/// The type of network interface -#[derive( - Clone, - Copy, - Debug, - Eq, - PartialEq, - Ord, - PartialOrd, - Deserialize, - Serialize, - JsonSchema, - Hash, - Diffable, -)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum NetworkInterfaceKind { - /// A vNIC attached to a guest instance - Instance { id: Uuid }, - /// A vNIC associated with an internal service - Service { id: Uuid }, - /// A vNIC associated with a probe - Probe { id: Uuid }, -} +pub mod network_interface; -/// Information required to construct a virtual network interface -#[derive( - Clone, - Debug, - Deserialize, - Serialize, - JsonSchema, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - Diffable, -)] -pub struct NetworkInterface { - pub id: Uuid, - pub kind: NetworkInterfaceKind, - pub name: Name, - pub ip: IpAddr, - pub mac: external::MacAddr, - pub subnet: IpNet, - pub vni: Vni, - pub primary: bool, - pub slot: u8, - #[serde(default)] - pub transit_ips: Vec, -} +// Re-export latest version of all NIC-related types. +pub use network_interface::NetworkInterfaceKind; +pub use network_interface::v2::*; /// An IP address and port range used for source NAT, i.e., making /// outbound network connections from guests or services. @@ -776,7 +728,7 @@ impl TryFrom<&[ipnetwork::IpNetwork]> for IpAllowList { /// A VPC route resolved into a concrete target. #[derive( - Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, + Clone, Copy, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] pub struct ResolvedVpcRoute { pub dest: IpNet, @@ -973,12 +925,12 @@ impl JsonSchema for DatasetKind { } fn json_schema( - gen: &mut schemars::gen::SchemaGenerator, + generator: &mut schemars::gen::SchemaGenerator, ) -> schemars::schema::Schema { // The schema is a bit more complicated than this -- it's either one of // the fixed values or a string starting with "zone/" -- but this is // good enough for now. - let mut schema = ::json_schema(gen).into_object(); + let mut schema = ::json_schema(generator).into_object(); schema.metadata().description = Some( "The kind of dataset. See the `DatasetKind` enum \ in omicron-common for possible values." diff --git a/common/src/api/internal/shared/network_interface/mod.rs b/common/src/api/internal/shared/network_interface/mod.rs new file mode 100644 index 00000000000..a5c3615b054 --- /dev/null +++ b/common/src/api/internal/shared/network_interface/mod.rs @@ -0,0 +1,39 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Shared network-interface types. + +use daft::Diffable; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use uuid::Uuid; + +pub mod v1; +pub mod v2; + +/// The type of network interface +#[derive( + Clone, + Copy, + Debug, + Eq, + PartialEq, + Ord, + PartialOrd, + Deserialize, + Serialize, + JsonSchema, + Hash, + Diffable, +)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum NetworkInterfaceKind { + /// A vNIC attached to a guest instance + Instance { id: Uuid }, + /// A vNIC associated with an internal service + Service { id: Uuid }, + /// A vNIC associated with a probe + Probe { id: Uuid }, +} diff --git a/common/src/api/internal/shared/network_interface/v1.rs b/common/src/api/internal/shared/network_interface/v1.rs new file mode 100644 index 00000000000..47cb16a2724 --- /dev/null +++ b/common/src/api/internal/shared/network_interface/v1.rs @@ -0,0 +1,46 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Network interface types version 1 + +use std::net::IpAddr; + +use crate::api::external; +use crate::api::external::Name; +use crate::api::external::Vni; +use crate::api::internal::shared::NetworkInterfaceKind; +use daft::Diffable; +use oxnet::IpNet; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use uuid::Uuid; + +/// Information required to construct a virtual network interface +#[derive( + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Diffable, +)] +pub struct NetworkInterface { + pub id: Uuid, + pub kind: NetworkInterfaceKind, + pub name: Name, + pub ip: IpAddr, + pub mac: external::MacAddr, + pub subnet: IpNet, + pub vni: Vni, + pub primary: bool, + pub slot: u8, + #[serde(default)] + pub transit_ips: Vec, +} diff --git a/common/src/api/internal/shared/network_interface/v2.rs b/common/src/api/internal/shared/network_interface/v2.rs new file mode 100644 index 00000000000..dff6be71c66 --- /dev/null +++ b/common/src/api/internal/shared/network_interface/v2.rs @@ -0,0 +1,445 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Network interface types version 2 + +use crate::api::external; +use crate::api::external::Name; +use crate::api::external::Vni; +use crate::api::internal::shared::network_interface::NetworkInterfaceKind; +use daft::Diffable; +use oxnet::IpNet; +use oxnet::Ipv4Net; +use oxnet::Ipv6Net; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; +use uuid::Uuid; + +/// Information required to construct a virtual network interface +#[derive( + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Diffable, +)] +pub struct NetworkInterface { + pub id: Uuid, + pub kind: NetworkInterfaceKind, + pub name: Name, + pub ip_config: PrivateIpConfig, + pub mac: external::MacAddr, + pub vni: Vni, + pub primary: bool, + pub slot: u8, +} + +impl TryFrom for NetworkInterface { + type Error = external::Error; + + fn try_from( + value: super::v1::NetworkInterface, + ) -> Result { + let super::v1::NetworkInterface { + id, + kind, + name, + ip, + mac, + subnet, + vni, + primary, + slot, + transit_ips, + } = value; + let ip_config = match (ip, subnet) { + (IpAddr::V4(ip), IpNet::V4(subnet)) => { + let transit_ips = transit_ips + .into_iter() + .map(|net| { + let IpNet::V4(subnet) = net else { + return Err(external::Error::invalid_request( + "Expected an IPv4 transit IP subnet, but found IPv6", + )); + }; + Ok(subnet) + }) + .collect::>()?; + PrivateIpConfig::V4(PrivateIpv4Config::new_with_transit_ips( + ip, + subnet, + transit_ips, + )?) + } + (IpAddr::V6(ip), IpNet::V6(subnet)) => { + let transit_ips = transit_ips + .into_iter() + .map(|net| { + let IpNet::V6(subnet) = net else { + return Err(external::Error::invalid_request( + "Expected an IPv6 transit IP subnet, but found IPv4", + )); + }; + Ok(subnet) + }) + .collect::>()?; + PrivateIpConfig::V6(PrivateIpv6Config::new_with_transit_ips( + ip, + subnet, + transit_ips, + )?) + } + (IpAddr::V4(_), IpNet::V6(_)) | (IpAddr::V6(_), IpNet::V4(_)) => { + return Err(external::Error::invalid_request( + "IP address and subnet must have the same IP version", + )); + } + }; + Ok(Self { id, kind, name, ip_config, mac, vni, primary, slot }) + } +} + +impl TryFrom for super::v1::NetworkInterface { + type Error = external::Error; + + fn try_from(value: NetworkInterface) -> Result { + let NetworkInterface { + id, + kind, + name, + ip_config: ip, + mac, + vni, + primary, + slot, + } = value; + let (ip, subnet, transit_ips) = match ip { + PrivateIpConfig::V4(v4) => ( + IpAddr::V4(v4.ip), + IpNet::V4(v4.subnet), + v4.transit_ips.into_iter().map(IpNet::V4).collect(), + ), + PrivateIpConfig::V6(v6) => ( + IpAddr::V6(v6.ip), + IpNet::V6(v6.subnet), + v6.transit_ips.into_iter().map(IpNet::V6).collect(), + ), + PrivateIpConfig::DualStack { .. } => { + return Err(external::Error::invalid_request( + "Cannot convert dual-stack v2 NetworkInterface to v1", + )); + } + }; + Ok(Self { + id, + kind, + name, + ip, + mac, + subnet, + vni, + primary, + slot, + transit_ips, + }) + } +} + +#[derive(Clone, Debug, thiserror::Error)] +pub enum PrivateIpConfigError { + #[error("IP subnet {subnet} does not contain the requested addres {ip}")] + IpNotInSubnet { subnet: IpNet, ip: IpAddr }, +} + +impl From for external::Error { + fn from(e: PrivateIpConfigError) -> external::Error { + match e { + PrivateIpConfigError::IpNotInSubnet { .. } => { + external::Error::invalid_request(e.to_string()) + } + } + } +} + +/// VPC-private IP address configuration for a network interface. +#[derive( + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Diffable, +)] +#[serde(rename_all = "snake_case")] +pub enum PrivateIpConfig { + /// The interface has only an IPv4 configuration. + V4(PrivateIpv4Config), + /// The interface has only an IPv6 configuration. + V6(PrivateIpv6Config), + /// The interface is dual-stack. + DualStack { + /// The interface's IPv4 configuration. + v4: PrivateIpv4Config, + /// The interface's IPv6 configuration. + v6: PrivateIpv6Config, + }, +} + +impl PrivateIpConfig { + /// Construct an IPv4 IP configuration, with no transit IPs. + /// + /// An error is returned if the IP address is not within the subnet. + pub fn new_ipv4( + addr: Ipv4Addr, + subnet: Ipv4Net, + ) -> Result { + PrivateIpv4Config::new(addr, subnet).map(PrivateIpConfig::V4) + } + + /// Construct an IPv6 IP configuration, with no transit IPs. + /// + /// An error is returned if the IP address is not within the subnet. + pub fn new_ipv6( + addr: Ipv6Addr, + subnet: Ipv6Net, + ) -> Result { + PrivateIpv6Config::new(addr, subnet).map(PrivateIpConfig::V6) + } + + /// Return the IPv4 configuration, if one exists. + pub fn ipv4_config(&self) -> Option<&PrivateIpv4Config> { + match &self { + PrivateIpConfig::V4(v4) | PrivateIpConfig::DualStack { v4, .. } => { + Some(v4) + } + PrivateIpConfig::V6(_) => None, + } + } + + /// Return the IPv6 configuration, if one exists. + pub fn ipv6_config(&self) -> Option<&PrivateIpv6Config> { + match &self { + PrivateIpConfig::V6(v6) | PrivateIpConfig::DualStack { v6, .. } => { + Some(v6) + } + PrivateIpConfig::V4(_) => None, + } + } + + /// Return the IPv4 address for this configuration, if it exists. + pub fn ipv4_addr(&self) -> Option<&Ipv4Addr> { + self.ipv4_config().map(PrivateIpv4Config::ip) + } + + /// Return the IPv6 address for this configuration, if it exists. + pub fn ipv6_addr(&self) -> Option<&Ipv6Addr> { + self.ipv6_config().map(PrivateIpv6Config::ip) + } + + /// Return the IPv4 subnet for this configuration, if it exists. + pub fn ipv4_subnet(&self) -> Option<&Ipv4Net> { + self.ipv4_config().map(PrivateIpv4Config::subnet) + } + + /// Return the IPv6 subnet for this configuration, if it exists. + pub fn ipv6_subnet(&self) -> Option<&Ipv6Net> { + self.ipv6_config().map(PrivateIpv6Config::subnet) + } + + /// Return the IPv4 transit IPs, if they exist. + pub fn ipv4_transit_ips(&self) -> Option<&[Ipv4Net]> { + self.ipv4_config().map(|c| c.transit_ips.as_slice()) + } + + /// Return the IPv6 transit IPs, if they exist. + pub fn ipv6_transit_ips(&self) -> Option<&[Ipv6Net]> { + self.ipv6_config().map(|c| c.transit_ips.as_slice()) + } + + /// Return all transit IPs, of any IP version. + pub fn all_transit_ips(&self) -> impl Iterator + '_ { + let v4 = self + .ipv4_transit_ips() + .into_iter() + .flatten() + .copied() + .map(Into::into); + let v6 = self + .ipv6_transit_ips() + .into_iter() + .flatten() + .copied() + .map(Into::into); + v4.chain(v6) + } + + /// Return true if this is an IPv4-only configuration. + pub fn is_ipv4_only(&self) -> bool { + matches!(self, PrivateIpConfig::V4(_)) + } + + /// Return true if this is an IPv6-only configuration. + pub fn is_ipv6_only(&self) -> bool { + matches!(self, PrivateIpConfig::V6(_)) + } + + /// Return true if this is a dual-stack configuration. + pub fn is_dual_stack(&self) -> bool { + matches!(self, PrivateIpConfig::DualStack { .. }) + } + + /// Return true if this configuration has the provided IP address. + pub fn has_addr(&self, ip: &IpAddr) -> bool { + match ip { + IpAddr::V4(ipv4) => { + self.ipv4_addr().map(|ip| ip == ipv4).unwrap_or(false) + } + IpAddr::V6(ipv6) => { + self.ipv6_addr().map(|ip| ip == ipv6).unwrap_or(false) + } + } + } +} + +/// VPC-private IPv4 configuration for a network interface. +#[derive( + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Diffable, +)] +pub struct PrivateIpv4Config { + /// VPC-private IP address. + ip: Ipv4Addr, + /// The IP subnet. + subnet: Ipv4Net, + /// Additional networks on which the interface can send / receive traffic. + #[serde(default)] + pub transit_ips: Vec, +} + +impl PrivateIpv4Config { + /// Construct a new IPv4 configuration. + /// + /// This fails if the provided address is not within the subnet. + pub fn new( + ip: Ipv4Addr, + subnet: Ipv4Net, + ) -> Result { + Self::new_with_transit_ips(ip, subnet, vec![]) + } + + /// Construct a new IPv4 configuration, with transit IPs. + /// + /// This fails if the provided address is not within the subnet. + pub fn new_with_transit_ips( + ip: Ipv4Addr, + subnet: Ipv4Net, + transit_ips: Vec, + ) -> Result { + if subnet.contains(ip) { + Ok(Self { ip, subnet, transit_ips }) + } else { + Err(PrivateIpConfigError::IpNotInSubnet { + subnet: subnet.into(), + ip: ip.into(), + }) + } + } + + /// Return the IPv4 address. + pub fn ip(&self) -> &Ipv4Addr { + &self.ip + } + + /// Return the IPv4 subnet. + pub fn subnet(&self) -> &Ipv4Net { + &self.subnet + } +} + +/// VPC-private IPv6 configuration for a network interface. +#[derive( + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Diffable, +)] +pub struct PrivateIpv6Config { + /// VPC-private IP address. + ip: Ipv6Addr, + /// The IP subnet. + subnet: Ipv6Net, + /// Additional networks on which the interface can send / receive traffic. + pub transit_ips: Vec, +} + +impl PrivateIpv6Config { + /// Construct a new IPv6 configuration with no transit IPs. + /// + /// This fails if the provided address is not within the subnet. + pub fn new( + ip: Ipv6Addr, + subnet: Ipv6Net, + ) -> Result { + Self::new_with_transit_ips(ip, subnet, vec![]) + } + + /// Construct a new IPv4 configuration, with transit IPs. + /// + /// This fails if the provided address is not within the subnet. + pub fn new_with_transit_ips( + ip: Ipv6Addr, + subnet: Ipv6Net, + transit_ips: Vec, + ) -> Result { + if subnet.contains(ip) { + Ok(Self { ip, subnet, transit_ips }) + } else { + Err(PrivateIpConfigError::IpNotInSubnet { + subnet: subnet.into(), + ip: ip.into(), + }) + } + } + + /// Return the IPv6 address. + pub fn ip(&self) -> &Ipv6Addr { + &self.ip + } + + /// Return the IPv6 subnet. + pub fn subnet(&self) -> &Ipv6Net { + &self.subnet + } +} diff --git a/dev-tools/reconfigurator-cli/src/lib.rs b/dev-tools/reconfigurator-cli/src/lib.rs index 13a794867a3..8e4bb8aab05 100644 --- a/dev-tools/reconfigurator-cli/src/lib.rs +++ b/dev-tools/reconfigurator-cli/src/lib.rs @@ -71,6 +71,7 @@ use std::collections::BTreeSet; use std::convert::Infallible; use std::fmt::{self, Write}; use std::io::IsTerminal; +use std::net::IpAddr; use std::num::ParseIntError; use std::str::FromStr; use swrite::{SWrite, swrite, swriteln}; @@ -162,11 +163,32 @@ impl ReconfiguratorSim { builder .add_omicron_zone_external_ip(zone.id, external_ip) .context("adding omicron zone external IP")?; + + // TODO-completeness: This needs to support dual-stack zones. + // See https://github.com/oxidecomputer/omicron/issues/9288 and + // related issues. + let maybe_ip = if matches!(external_ip.ip(), IpAddr::V4(_)) { + nic.ip_config.ipv4_addr().copied().map(IpAddr::V4) + } else { + nic.ip_config.ipv6_addr().copied().map(IpAddr::V6) + }; + let ip = maybe_ip.with_context(|| { + format!( + "Omicron zone has an external and private IP \ + configurations of different IP versions. \ + zone_id={} zone_kind={} \ + external_ip={} private_ip_config={:?}", + zone.id, + zone.zone_type.kind().report_str(), + external_ip.ip(), + nic.ip_config, + ) + })?; let nic = OmicronZoneNic { // TODO-cleanup use `TypedUuid` everywhere id: VnicUuid::from_untyped_uuid(nic.id), mac: nic.mac, - ip: nic.ip, + ip, slot: nic.slot, primary: nic.primary, }; diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 9f5c25462c5..806b3360d3c 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -17,9 +17,9 @@ mod port; mod port_manager; pub use firewall_rules::opte_firewall_rules; -use ipnetwork::IpNetwork; use macaddr::MacAddr6; use omicron_common::api::internal::shared; +use omicron_common::api::internal::shared::PrivateIpConfig; pub use oxide_vpc::api::BoundaryServices; pub use oxide_vpc::api::DhcpCfg; use oxide_vpc::api::IpCidr; @@ -34,14 +34,26 @@ pub use port::Port; pub use port_manager::PortCreateParams; pub use port_manager::PortManager; pub use port_manager::PortTicket; -use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; /// Information about the gateway for an OPTE port #[derive(Debug, Clone, Copy)] #[allow(dead_code)] pub struct Gateway { mac: MacAddr6, - ip: IpAddr, + ips: GatewayIps, +} + +// IP addresses for an OPTE gateway. +#[derive(Clone, Copy, Debug)] +enum GatewayIps { + // IPv4-only gateway. + V4(Ipv4Addr), + // IPv6-only gateway. + V6(Ipv6Addr), + // Dual-stack gateway. + DualStack { v4: Ipv4Addr, v6: Ipv6Addr }, } // The MAC address that OPTE exposes to guest NICs, i.e., the MAC of the virtual @@ -54,20 +66,47 @@ const OPTE_VIRTUAL_GATEWAY_MAC: MacAddr6 = MacAddr6::new(0xa8, 0x40, 0x25, 0xff, 0x77, 0x77); impl Gateway { - pub fn from_subnet(subnet: &IpNetwork) -> Self { - Self { - mac: OPTE_VIRTUAL_GATEWAY_MAC, + /// Construct information about the gateway from an IP configuration. + pub fn from_ip_config(ip: &PrivateIpConfig) -> Self { + let ips = + match ip { + PrivateIpConfig::V4(v4) => { + let ip = v4.subnet().first_host(); + GatewayIps::V4(ip) + } + PrivateIpConfig::V6(v6) => { + let ip = + v6.subnet().iter().nth(1).expect( + "IPv6 subnet must have at least 2 addresses", + ); + GatewayIps::V6(ip) + } + PrivateIpConfig::DualStack { v4, v6 } => { + let v4 = v4.subnet().first_host(); + let v6 = + v6.subnet().iter().nth(1).expect( + "IPv6 subnet must have at least 2 addresses", + ); + GatewayIps::DualStack { v4, v6 } + } + }; + Self { mac: OPTE_VIRTUAL_GATEWAY_MAC, ips } + } - // See RFD 21, section 2.2 table 1 - ip: subnet - .iter() - .nth(1) - .expect("IP subnet must have at least 2 addresses"), + /// Return the gateway's IPv4 address, if it exists. + pub fn ipv4_addr(&self) -> Option<&Ipv4Addr> { + match &self.ips { + GatewayIps::V4(v4) | GatewayIps::DualStack { v4, .. } => Some(&v4), + GatewayIps::V6(_) => None, } } - pub fn ip(&self) -> &IpAddr { - &self.ip + /// Return the gateway's IPv6 address, if it exists. + pub fn ipv6_addr(&self) -> Option<&Ipv6Addr> { + match &self.ips { + GatewayIps::V6(v6) | GatewayIps::DualStack { v6, .. } => Some(&v6), + GatewayIps::V4(_) => None, + } } } @@ -106,3 +145,55 @@ fn router_target_opte(target: &shared::RouterTarget) -> RouterTarget { VpcSubnet(net) => RouterTarget::VpcSubnet(net_to_cidr(*net)), } } + +#[cfg(test)] +mod tests { + use super::Gateway; + use super::GatewayIps; + use omicron_common::api::internal::shared::PrivateIpConfig; + use omicron_common::api::internal::shared::PrivateIpv4Config; + use omicron_common::api::internal::shared::PrivateIpv6Config; + use oxnet::Ipv4Net; + use oxnet::Ipv6Net; + use std::net::Ipv4Addr; + use std::net::Ipv6Addr; + + #[test] + fn convert_private_ip_config_to_gateway_ips() { + let v4 = PrivateIpv4Config::new( + Ipv4Addr::new(10, 0, 0, 5), + Ipv4Net::new(Ipv4Addr::new(10, 0, 0, 0), 24).unwrap(), + ) + .unwrap(); + let v6 = PrivateIpv6Config::new( + Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 5), + Ipv6Net::new(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0), 64) + .unwrap(), + ) + .unwrap(); + let cfg = PrivateIpConfig::DualStack { v4: v4.clone(), v6: v6.clone() }; + + let expected_v4_gateway = Ipv4Addr::new(10, 0, 0, 1); + let expected_v6_gateway = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1); + let GatewayIps::V4(gateway) = + Gateway::from_ip_config(&PrivateIpConfig::V4(v4)).ips + else { + panic!("Expected IPv4 OPTE gateway IP addresses"); + }; + assert_eq!(gateway, expected_v4_gateway); + let GatewayIps::V6(gateway) = + Gateway::from_ip_config(&PrivateIpConfig::V6(v6)).ips + else { + panic!("Expected IPv6 OPTE gateway IP addresses"); + }; + assert_eq!(gateway, expected_v6_gateway); + + let GatewayIps::DualStack { v4: ipv4, v6: ipv6 } = + Gateway::from_ip_config(&cfg).ips + else { + panic!("Expected dual-stack OPTE gateway IP addresses"); + }; + assert_eq!(ipv4, expected_v4_gateway); + assert_eq!(ipv6, expected_v6_gateway); + } +} diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 966103d01ac..07d050bcaec 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -9,26 +9,27 @@ use crate::opte::Handle; use crate::opte::Vni; use macaddr::MacAddr6; use omicron_common::api::external; +use omicron_common::api::internal::shared::PrivateIpConfig; use omicron_common::api::internal::shared::RouterId; use omicron_common::api::internal::shared::RouterKind; -use oxnet::IpNet; -use std::net::IpAddr; +use oxnet::Ipv4Net; +use oxnet::Ipv6Net; +use std::net::Ipv4Addr; +use std::net::Ipv6Addr; use std::sync::Arc; #[derive(Debug)] pub struct PortData { /// Name of the port as identified by OPTE pub(crate) name: String, - /// IP address within the VPC Subnet - pub(crate) ip: IpAddr, + /// The VPC-private IP configuration for the port. + pub(crate) ip: PrivateIpConfig, /// VPC-private MAC address pub(crate) mac: MacAddr6, /// Emulated PCI slot for the guest NIC, passed to Propolis pub(crate) slot: u8, /// Geneve VNI for the VPC pub(crate) vni: Vni, - /// Subnet the port belong to within the VPC. - pub(crate) subnet: IpNet, /// Information about the virtual gateway, aka OPTE pub(crate) gateway: Gateway, } @@ -80,8 +81,14 @@ impl Port { Self { inner: Arc::new(PortInner(data)) } } - pub fn ip(&self) -> &IpAddr { - &self.inner.ip + /// Return the VPC-private IPv4 address, if it exists. + pub fn ipv4_addr(&self) -> Option<&Ipv4Addr> { + self.inner.ip.ipv4_addr() + } + + /// Return the VPC-private IPv6 address, if it exists. + pub fn ipv6_addr(&self) -> Option<&Ipv6Addr> { + self.inner.ip.ipv6_addr() } pub fn name(&self) -> &str { @@ -101,8 +108,14 @@ impl Port { &self.inner.vni } - pub fn subnet(&self) -> &IpNet { - &self.inner.subnet + /// Return the VPC-private IPv4 subnet, if it exists. + pub fn ipv4_subnet(&self) -> Option<&Ipv4Net> { + self.inner.ip.ipv4_subnet() + } + + /// Return the VPC-private IPv6 subnet, if it exists. + pub fn ipv6_subnet(&self) -> Option<&Ipv6Net> { + self.inner.ip.ipv6_subnet() } pub fn slot(&self) -> u8 { @@ -115,10 +128,17 @@ impl Port { RouterId { vni, kind: RouterKind::System } } - pub fn custom_router_key(&self) -> RouterId { - RouterId { - kind: RouterKind::Custom(*self.subnet()), + pub fn custom_ipv4_router_key(&self) -> Option { + self.ipv4_subnet().copied().map(|subnet| RouterId { + kind: RouterKind::Custom(subnet.into()), + ..self.system_router_key() + }) + } + + pub fn custom_ipv6_router_key(&self) -> Option { + self.ipv6_subnet().copied().map(|subnet| RouterId { + kind: RouterKind::Custom(subnet.into()), ..self.system_router_key() - } + }) } } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 97eba85e621..4471a16704b 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -12,7 +12,8 @@ use crate::opte::Port; use crate::opte::Vni; use crate::opte::opte_firewall_rules; use crate::opte::port::PortData; -use ipnetwork::IpNetwork; +use ipnetwork::Ipv4Network; +use ipnetwork::Ipv6Network; use macaddr::MacAddr6; use omicron_common::api::external; use omicron_common::api::internal::shared::ExternalIpGatewayMap; @@ -24,6 +25,7 @@ use omicron_common::api::internal::shared::ResolvedVpcRoute; use omicron_common::api::internal::shared::ResolvedVpcRouteSet; use omicron_common::api::internal::shared::ResolvedVpcRouteState; use omicron_common::api::internal::shared::RouterId; +use omicron_common::api::internal::shared::RouterKind; use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget; use omicron_common::api::internal::shared::RouterVersion; use omicron_common::api::internal::shared::SourceNatConfig; @@ -34,9 +36,10 @@ use oxide_vpc::api::DhcpCfg; use oxide_vpc::api::Direction; use oxide_vpc::api::ExternalIpCfg; use oxide_vpc::api::IpCfg; -use oxide_vpc::api::IpCidr; use oxide_vpc::api::Ipv4Cfg; +use oxide_vpc::api::Ipv4Cidr; use oxide_vpc::api::Ipv6Cfg; +use oxide_vpc::api::Ipv6Cidr; use oxide_vpc::api::MacAddr; use oxide_vpc::api::RouterClass; use oxide_vpc::api::SNat4Cfg; @@ -108,6 +111,11 @@ impl PortManagerInner { /// Parameters needed to create and configure an OPTE port. pub struct PortCreateParams<'a> { pub nic: &'a NetworkInterface, + // TODO-completeness: These should ideally be grouped together into a type + // that ensures they're all of the same IP version, and that the IP stack + // for the external and VPC-private addresses match. + // + // See https://github.com/oxidecomputer/omicron/issues/9318. pub source_nat: Option, pub ephemeral_ip: Option, pub floating_ips: &'a [IpAddr], @@ -115,6 +123,164 @@ pub struct PortCreateParams<'a> { pub dhcp_config: DhcpCfg, } +impl<'a> TryFrom<&PortCreateParams<'a>> for IpCfg { + type Error = Error; + + fn try_from(params: &PortCreateParams) -> Result { + let ip_config = ¶ms.nic.ip_config; + let has_ipv4_stack = ip_config.ipv4_addr().is_some(); + let has_ipv6_stack = ip_config.ipv6_addr().is_some(); + + // First, attempt to convert the IPv4 configuration. + // + // The important part here is ensuring that we have an external and + // VPC-private configuration for each IP stack. That is, if we have an + // IPv4 external Ephemeral IP, we also have an IPv4 VPC-private address. + let v4 = match ip_config.ipv4_config() { + None => None, + Some(ipv4_config) => { + let gateway_ip = ipv4_config.subnet().first_addr().into(); + let vpc_subnet = + Ipv4Cidr::from(Ipv4Network::from(*ipv4_config.subnet())); + let private_ip = (*ipv4_config.ip()).into(); + let external_ips = build_external_ipv4_config( + params.source_nat.as_ref(), + params.ephemeral_ip.as_ref(), + ¶ms.floating_ips, + has_ipv4_stack, + )?; + Some(Ipv4Cfg { + vpc_subnet, + private_ip, + gateway_ip, + external_ips, + }) + } + }; + + // Now the same conversion, but for IPv6. Again, we need to ensure that, + // if we have an external IPv6 address, we also have a VPC-private IPv6 + // address to translate that into. + let v6 = match ip_config.ipv6_config() { + None => None, + Some(ipv6_config) => { + let gateway_ip = ipv6_config.subnet().first_addr().into(); + let vpc_subnet = + Ipv6Cidr::from(Ipv6Network::from(*ipv6_config.subnet())); + let private_ip = (*ipv6_config.ip()).into(); + let external_ips = build_external_ipv6_config( + params.source_nat.as_ref(), + params.ephemeral_ip.as_ref(), + ¶ms.floating_ips, + has_ipv6_stack, + )?; + Some(Ipv6Cfg { + vpc_subnet, + private_ip, + gateway_ip, + external_ips, + }) + } + }; + + // Now build the full configuration, either single- or dual-stack. + let cfg = match (v4, v6) { + (Some(ipv4), Some(ipv6)) => IpCfg::DualStack { ipv4, ipv6 }, + (Some(v4), None) => IpCfg::Ipv4(v4), + (None, Some(v6)) => IpCfg::Ipv6(v6), + (None, None) => unreachable!(), + }; + Ok(cfg) + } +} + +// Build an ExternalIpCfg from parameters. +fn build_external_ipv4_config( + source_nat: Option<&SourceNatConfig>, + ephemeral_ip: Option<&IpAddr>, + floating_ips: &[IpAddr], + has_ipv4_stack: bool, +) -> Result, Error> { + // Convert the SNAT configuration. + // + // It's not an error to have an SNAT address of a different IP + // version right now, _as long as_ we have a VPC-private address + // that _does_ have the right version. + let snat = match source_nat { + None => None, + Some(snat) => match snat.ip { + IpAddr::V4(ipv4) if has_ipv4_stack => Some(SNat4Cfg { + external_ip: ipv4.into(), + ports: snat.port_range(), + }), + IpAddr::V4(_) => { + return Err(Error::InvalidPortIpConfig); + } + IpAddr::V6(_) => None, + }, + }; + + // Convert the Ephemeral address, again ensuring that we have a + // VPC-private IP stack to support the external address. + let ephemeral_ip = match ephemeral_ip { + Some(IpAddr::V4(ipv4)) if has_ipv4_stack => Some((*ipv4).into()), + Some(IpAddr::V4(_)) => return Err(Error::InvalidPortIpConfig), + None | Some(IpAddr::V6(_)) => None, + }; + + // And all the Floating IPs, still ensuring we can support them. + let floating_ips = floating_ips + .iter() + .filter_map(|ip| match ip { + IpAddr::V4(ipv4) if has_ipv4_stack => Some(Ok((*ipv4).into())), + IpAddr::V4(_) => Some(Err(Error::InvalidPortIpConfig)), + IpAddr::V6(_) => None, + }) + .collect::>()?; + Ok(ExternalIpCfg { snat, ephemeral_ip, floating_ips }) +} + +// Build an OPTE External IPv6 configuration from parameters. +fn build_external_ipv6_config( + source_nat: Option<&SourceNatConfig>, + ephemeral_ip: Option<&IpAddr>, + floating_ips: &[IpAddr], + has_ipv6_stack: bool, +) -> Result, Error> { + // Convert the SNAT configuration. + let snat = match source_nat { + None => None, + Some(snat) => match snat.ip { + IpAddr::V6(ipv6) if has_ipv6_stack => Some(SNat6Cfg { + external_ip: ipv6.into(), + ports: snat.port_range(), + }), + IpAddr::V6(_) => { + return Err(Error::InvalidPortIpConfig); + } + IpAddr::V4(_) => None, + }, + }; + + // Convert the Ephemeral address. + let ephemeral_ip = match ephemeral_ip { + Some(IpAddr::V6(ipv6)) if has_ipv6_stack => Some((*ipv6).into()), + Some(IpAddr::V6(_)) => return Err(Error::InvalidPortIpConfig), + None | Some(IpAddr::V4(_)) => None, + }; + + // And all the Floating IPs, still ensuring we can support them. + let floating_ips = floating_ips + .iter() + .filter_map(|ip| match ip { + IpAddr::V6(ipv6) if has_ipv6_stack => Some(Ok((*ipv6).into())), + IpAddr::V6(_) => Some(Err(Error::InvalidPortIpConfig)), + IpAddr::V4(_) => None, + }) + .collect::>()?; + Ok(ExternalIpCfg { snat, ephemeral_ip, floating_ips }) +} + /// The port manager controls all OPTE ports on a single host. #[derive(Debug, Clone)] pub struct PortManager { @@ -145,6 +311,7 @@ impl PortManager { &self, params: PortCreateParams, ) -> Result<(Port, PortTicket), Error> { + let ip_cfg = IpCfg::try_from(¶ms)?; let PortCreateParams { nic, source_nat, @@ -153,121 +320,15 @@ impl PortManager { firewall_rules, dhcp_config, } = params; - let is_service = matches!(nic.kind, NetworkInterfaceKind::Service { .. }); let is_instance = matches!(nic.kind, NetworkInterfaceKind::Instance { .. }); - let mac = *nic.mac; let vni = Vni::new(nic.vni).unwrap(); - let subnet = IpNetwork::from(nic.subnet); - let vpc_subnet = IpCidr::from(subnet); - let gateway = Gateway::from_subnet(&subnet); - - // Describe the external IP addresses for this port. - macro_rules! ip_cfg { - ($ip:expr, $log_prefix:literal, $ip_t:path, $cidr_t:path, - $ipcfg_e:path, $ipcfg_t:ident, $snat_t:ident) => {{ - let $cidr_t(vpc_subnet) = vpc_subnet else { - error!( - self.inner.log, - concat!($log_prefix, " subnet"); - "subnet" => ?vpc_subnet, - ); - return Err(Error::InvalidPortIpConfig); - }; - let $ip_t(gateway_ip) = gateway.ip else { - error!( - self.inner.log, - concat!($log_prefix, " gateway"); - "gateway_ip" => ?gateway.ip, - ); - return Err(Error::InvalidPortIpConfig); - }; - let snat = match source_nat { - Some(snat) => { - let $ip_t(snat_ip) = snat.ip else { - error!( - self.inner.log, - concat!($log_prefix, " SNAT config"); - "snat_ip" => ?snat.ip, - ); - return Err(Error::InvalidPortIpConfig); - }; - let ports = snat.port_range(); - Some($snat_t { external_ip: snat_ip.into(), ports }) - } - None => None, - }; - let ephemeral_ip = match ephemeral_ip { - Some($ip_t(ip)) => Some(ip.into()), - Some(_) => { - error!( - self.inner.log, - concat!($log_prefix, " ephemeral IP"); - "ephemeral_ip" => ?ephemeral_ip, - ); - return Err(Error::InvalidPortIpConfig); - } - None => None, - }; - let floating_ips: Vec<_> = floating_ips - .iter() - .copied() - .map(|ip| match ip { - $ip_t(ip) => Ok(ip.into()), - _ => { - error!( - self.inner.log, - concat!($log_prefix, " ephemeral IP"); - "ephemeral_ip" => ?ephemeral_ip, - ); - Err(Error::InvalidPortIpConfig) - } - }) - .collect::, _>>()?; - - $ipcfg_e($ipcfg_t { - vpc_subnet, - private_ip: $ip.into(), - gateway_ip: gateway_ip.into(), - external_ips: ExternalIpCfg { - ephemeral_ip, - snat, - floating_ips, - }, - }) - }} - } - - // Build the port's IP configuration as either IPv4 or IPv6 - // depending on the IP that was assigned to the NetworkInterface. - // We use a macro here to be DRY - // TODO-completeness: Support both dual stack - let ip_cfg = match nic.ip { - IpAddr::V4(ip) => ip_cfg!( - ip, - "Expected IPv4", - IpAddr::V4, - IpCidr::Ip4, - IpCfg::Ipv4, - Ipv4Cfg, - SNat4Cfg - ), - IpAddr::V6(ip) => ip_cfg!( - ip, - "Expected IPv6", - IpAddr::V6, - IpCidr::Ip6, - IpCfg::Ipv6, - Ipv6Cfg, - SNat6Cfg - ), - }; - + let gateway = Gateway::from_ip_config(&nic.ip_config); let vpc_cfg = VpcCfg { - ip_cfg: ip_cfg.clone(), + ip_cfg, guest_mac: MacAddr::from(nic.mac.into_array()), gateway_mac: MacAddr::from(gateway.mac.into_array()), vni, @@ -310,11 +371,10 @@ impl PortManager { let ticket = PortTicket::new(nic.id, nic.kind, self.inner.clone()); let port = Port::new(PortData { name: port_name.clone(), - ip: nic.ip, + ip: nic.ip_config.clone(), mac, slot: nic.slot, vni, - subnet: nic.subnet, gateway, }); let old = ports.insert((nic.id, nic.kind), port.clone()); @@ -360,79 +420,93 @@ impl PortManager { rules, })?; - // Check locally to see whether we have any routes from the - // control plane for this port already installed. If not, - // create a record to show that we're interested in receiving - // those routes. - let mut route_map = self.inner.routes.lock().unwrap(); - let system_routes = - route_map.entry(port.system_router_key()).or_insert_with(|| { - let mut routes = HashSet::new(); - if is_service { - // Always insert a rule targeting the _system VPC Internet Gateway_. - // This may be sent later from Nexus, but we need it during - // bootstrapping NTP or other very early services, before the - // control plane database has been started. - let target = ApiRouterTarget::InternetGateway( - InternetGatewayRouterTarget::System, - ); - routes.insert(ResolvedVpcRoute { - dest: IpNet::V4( - Ipv4Net::new(Ipv4Addr::UNSPECIFIED, 0).unwrap(), - ), - target, - }); - routes.insert(ResolvedVpcRoute { - dest: IpNet::V6( - Ipv6Net::new(Ipv6Addr::UNSPECIFIED, 0).unwrap(), - ), - target, - }); - } - RouteSet { version: None, routes, active_ports: 0 } - }); - system_routes.active_ports += 1; - - // Clone is needed to get borrowck on our side, sadly. - let system_routes = system_routes.clone(); - - let custom_routes = route_map - .entry(port.custom_router_key()) - .or_insert_with(|| RouteSet { + // Create the default set of routes for a new port. + // + // This creates an empty route set for instance ports, but adds a rule + // targeting the System VPC Internet Gateway for service ports. + // + // We need this during bootstrapping NTP or other very early services, + // before the control plane database has started. + let default_route_set = |is_service: bool| { + let mut route_set = RouteSet { version: None, - routes: HashSet::default(), + routes: HashSet::new(), active_ports: 0, + }; + if !is_service { + return route_set; + } + let target = ApiRouterTarget::InternetGateway( + InternetGatewayRouterTarget::System, + ); + route_set.routes.insert(ResolvedVpcRoute { + dest: IpNet::V4( + Ipv4Net::new(Ipv4Addr::UNSPECIFIED, 0).unwrap(), + ), + target, }); - custom_routes.active_ports += 1; - - for (class, routes) in [ - (RouterClass::System, &system_routes), - (RouterClass::Custom, custom_routes), - ] { - for route in &routes.routes { - let route = AddRouterEntryReq { + route_set.routes.insert(ResolvedVpcRoute { + dest: IpNet::V6( + Ipv6Net::new(Ipv6Addr::UNSPECIFIED, 0).unwrap(), + ), + target, + }); + route_set + }; + + // Add routes to a new port and the shared set of all routes in this + // port manager. + // + // As new ports are created, they need both System router entries, but + // also Custom entries targeting things like the VPC Subnet or another + // instance. These are added to the port, but after updating the shared + // set of all routes maintained by this manager. + let add_routes = |route_map: &mut HashMap, + key: RouterId| + -> Result<(), Error> { + let route_set = route_map + .entry(key) + .or_insert_with(|| default_route_set(is_service)); + route_set.active_ports += 1; + let class = match key.kind { + RouterKind::System => RouterClass::System, + RouterKind::Custom(_) => RouterClass::Custom, + }; + for route in &route_set.routes { + let request = AddRouterEntryReq { class, port_name: port_name.clone(), dest: super::net_to_cidr(route.dest), target: super::router_target_opte(&route.target), }; - - hdl.add_router_entry(&route)?; - + hdl.add_router_entry(&request)?; debug!( self.inner.log, "Added router entry"; "port_name" => &port_name, - "route" => ?route, + "route" => ?request, ); } + Ok(()) + }; + + // Actually add all system and custom router entries relevant for this + // new port. + let mut route_map = self.inner.routes.lock().unwrap(); + add_routes(&mut route_map, port.system_router_key())?; + if let Some(key) = port.custom_ipv4_router_key() { + add_routes(&mut route_map, key)?; } + if let Some(key) = port.custom_ipv6_router_key() { + add_routes(&mut route_map, key)?; + } + drop(route_map); // If there are any transit IPs set, allow them through. // TODO: Currently set only in initial state. // This, external IPs, and cfg'able state // (DHCP?) are probably worth being managed by an RPW. - for block in &nic.transit_ips { + for block in nic.ip_config.all_transit_ips() { // In principle if this were an operation on an existing // port, we would explicitly undo the In addition if the // Out addition fails. @@ -441,12 +515,12 @@ impl PortManager { // number of rules are specified. hdl.allow_cidr( &port_name, - super::net_to_cidr(*block), + super::net_to_cidr(block), Direction::In, )?; hdl.allow_cidr( &port_name, - super::net_to_cidr(*block), + super::net_to_cidr(block), Direction::Out, )?; @@ -507,8 +581,8 @@ impl PortManager { continue; } _ => ( - new.routes.difference(&old.routes).cloned().collect(), - old.routes.difference(&new.routes).cloned().collect(), + new.routes.difference(&old.routes).copied().collect(), + old.routes.difference(&new.routes).copied().collect(), ), }; deltas.insert(new.id, (to_add, to_delete)); @@ -532,15 +606,18 @@ impl PortManager { // Propagate deltas out to all ports. for port in ports.values() { - let system_id = port.system_router_key(); - let system_delta = deltas.get(&system_id); - - let custom_id = port.custom_router_key(); - let custom_delta = deltas.get(&custom_id); + // Fetch deltas for all router keys: system, IPv4 subnet, and IPv6 + // subnet. + let system_delta = deltas.get(&port.system_router_key()); + let custom_ipv4_delta = + port.custom_ipv4_router_key().and_then(|k| deltas.get(&k)); + let custom_ipv6_delta = + port.custom_ipv6_router_key().and_then(|k| deltas.get(&k)); for (class, delta) in [ (RouterClass::System, system_delta), - (RouterClass::Custom, custom_delta), + (RouterClass::Custom, custom_ipv4_delta), + (RouterClass::Custom, custom_ipv6_delta), ] { let Some((to_add, to_delete)) = delta else { debug!(self.inner.log, "vpc route ensure: no delta"); @@ -644,90 +721,47 @@ impl PortManager { let inet_gw_map = egw_lock.get(&nic_id).cloned(); drop(egw_lock); - // XXX: duplicates parts of macro logic in `create_port`. - macro_rules! ext_ip_cfg { - ($ip:expr, $log_prefix:literal, $ip_t:path, $cidr_t:path, - $ipcfg_e:path, $ipcfg_t:ident, $snat_t:ident) => {{ - let snat = match source_nat { - Some(snat) => { - let $ip_t(snat_ip) = snat.ip else { - error!( - self.inner.log, - concat!($log_prefix, " SNAT config"); - "snat_ip" => ?snat.ip, - ); - return Err(Error::InvalidPortIpConfig); - }; - let ports = snat.port_range(); - Some($snat_t { external_ip: snat_ip.into(), ports }) - } - None => None, - }; - let ephemeral_ip = match ephemeral_ip { - Some($ip_t(ip)) => Some(ip.into()), - Some(_) => { - error!( - self.inner.log, - concat!($log_prefix, " ephemeral IP"); - "ephemeral_ip" => ?ephemeral_ip, - ); - return Err(Error::InvalidPortIpConfig); - } - None => None, - }; - let floating_ips: Vec<_> = floating_ips - .iter() - .copied() - .map(|ip| match ip { - $ip_t(ip) => Ok(ip.into()), - _ => { - error!( - self.inner.log, - concat!($log_prefix, " ephemeral IP"); - "ephemeral_ip" => ?ephemeral_ip, - ); - Err(Error::InvalidPortIpConfig) - } - }) - .collect::, _>>()?; - - ExternalIpCfg { - ephemeral_ip, - snat, - floating_ips, - } - }} - } - - // TODO-completeness: support dual-stack. We'll need to explicitly store - // a v4 and a v6 ephemeral IP + SNat + gateway + ... in `InstanceInner` - // to have enough info to build both. - let mut v4_cfg = None; - let mut v6_cfg = None; - match port.gateway().ip { - IpAddr::V4(_) => { - v4_cfg = Some(ext_ip_cfg!( - ip, - "Expected IPv4", - IpAddr::V4, - IpCidr::Ip4, - IpCfg::Ipv4, - Ipv4Cfg, - SNat4Cfg - )) - } - IpAddr::V6(_) => { - v6_cfg = Some(ext_ip_cfg!( - ip, - "Expected IPv6", - IpAddr::V6, - IpCidr::Ip6, - IpCfg::Ipv6, - Ipv6Cfg, - SNat6Cfg - )) + let has_ipv4_stack = port.ipv4_addr().is_some(); + let has_ipv6_stack = port.ipv6_addr().is_some(); + let external_ips_v4 = build_external_ipv4_config( + source_nat.as_ref(), + ephemeral_ip.as_ref(), + floating_ips, + has_ipv4_stack, + )?; + let external_ips_v6 = build_external_ipv6_config( + source_nat.as_ref(), + ephemeral_ip.as_ref(), + floating_ips, + has_ipv6_stack, + )?; + + // The above functions building the external address configuration only + // fail if we're provided external addresses from an IP version we don't + // have a VPC-private IP stack for. E.g., IPv6 external IPs for an + // IPv4-only interface. If we're provided IPv6 external IPs and we have + // both an IPv4 and IPv6 interface, then those methods succeed, but + // return an `ExternalIpCfg` where all the fields are empty. + // + // However, the `SetExternalIpsReq` method accepts an _option_ around + // those values. Those should be None if there are zero addresses of the + // corresponding version in the parameters. In that case, all the fields + // of the `ExternalIpCfg`s are None or empty. This function does that + // conversion for us. + fn convert_empty_ip_cfg( + cfg: ExternalIpCfg, + ) -> Option> { + if cfg.snat.is_none() + && cfg.ephemeral_ip.is_none() + && cfg.floating_ips.is_empty() + { + None + } else { + Some(cfg) } } + let external_ips_v4 = convert_empty_ip_cfg(external_ips_v4); + let external_ips_v6 = convert_empty_ip_cfg(external_ips_v6); let inet_gw_map = if let Some(map) = inet_gw_map { Some( @@ -741,8 +775,8 @@ impl PortManager { let req = SetExternalIpsReq { port_name: port.name().into(), - external_ips_v4: v4_cfg, - external_ips_v6: v6_cfg, + external_ips_v4, + external_ips_v6, inet_gw_map, }; let hdl = Handle::new()?; @@ -913,15 +947,15 @@ impl PortTicket { drop(ports); // Cleanup the set of subnets we want to receive routes for. - let mut routes = self.manager.routes.lock().unwrap(); - for key in [port.system_router_key(), port.custom_router_key()] { + let remove_key = |routes: &mut HashMap, + key: RouterId| { let should_remove = routes .get_mut(&key) .map(|v| { v.active_ports = v.active_ports.saturating_sub(1); v.active_ports == 0 }) - .unwrap_or_default(); + .unwrap_or(false); if should_remove { routes.remove(&key); @@ -931,8 +965,15 @@ impl PortTicket { "id" => ?&key, ); } + }; + let mut routes = self.manager.routes.lock().unwrap(); + remove_key(&mut routes, port.system_router_key()); + if let Some(key) = port.custom_ipv4_router_key() { + remove_key(&mut routes, key); + } + if let Some(key) = port.custom_ipv6_router_key() { + remove_key(&mut routes, key); } - debug!( self.manager.log, "Removed OPTE port from manager"; @@ -974,8 +1015,8 @@ mod tests { external::{MacAddr, Vni}, internal::shared::{ InternetGatewayRouterTarget, NetworkInterface, - NetworkInterfaceKind, ResolvedVpcRoute, ResolvedVpcRouteSet, - RouterTarget, RouterVersion, SourceNatConfig, + NetworkInterfaceKind, PrivateIpConfig, ResolvedVpcRoute, + ResolvedVpcRouteSet, RouterTarget, RouterVersion, SourceNatConfig, }, }; use omicron_test_utils::dev::test_setup_log; @@ -1015,12 +1056,18 @@ mod tests { // At this point, we'll insert a single default route, because this is a // service point, from `0.0.0.0/0 -> InternetGateway(None)`, and then // add this route to OPTE. - let private_ipv4_addr0 = IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4)); - let private_ipv4_addr1 = IpAddr::V4(Ipv4Addr::new(172, 20, 0, 5)); + let private_subnet = + Ipv4Net::new(Ipv4Addr::new(172, 20, 0, 0), 24).unwrap(); + let private_ipv4_addr0 = Ipv4Addr::new(172, 20, 0, 4); + let ip_config0 = + PrivateIpConfig::new_ipv4(private_ipv4_addr0, private_subnet) + .unwrap(); + let private_ipv4_addr1 = Ipv4Addr::new(172, 20, 0, 5); + let ip_config1 = + PrivateIpConfig::new_ipv4(private_ipv4_addr1, private_subnet) + .unwrap(); let public_ipv4_addr0 = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 4)); let public_ipv4_addr1 = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5)); - let private_subnet = - IpNet::V4(Ipv4Net::new(Ipv4Addr::new(172, 20, 0, 0), 24).unwrap()); const MAX_PORT: u16 = (1 << 14) - 1; let (port0, _ticket0) = manager .create_port(PortCreateParams { @@ -1028,15 +1075,13 @@ mod tests { id: Uuid::new_v4(), kind: NetworkInterfaceKind::Service { id: Uuid::new_v4() }, name: "opte0".parse().unwrap(), - ip: private_ipv4_addr0, + ip_config: ip_config0, mac: MacAddr(MacAddr6::new( 0xa8, 0x40, 0x25, 0x00, 0x00, 0x01, )), - subnet: private_subnet, vni: SERVICES_VPC_VNI, primary: true, slot: 0, - transit_ips: Vec::new(), }, source_nat: Some( SourceNatConfig::new(public_ipv4_addr0, 0, MAX_PORT) @@ -1203,15 +1248,13 @@ mod tests { id: Uuid::new_v4(), kind: NetworkInterfaceKind::Service { id: Uuid::new_v4() }, name: "opte1".parse().unwrap(), - ip: private_ipv4_addr1, + ip_config: ip_config1, mac: MacAddr(MacAddr6::new( 0xa8, 0x40, 0x25, 0x00, 0x00, 0x02, )), - subnet: private_subnet, vni: SERVICES_VPC_VNI, primary: true, slot: 0, - transit_ips: Vec::new(), }, source_nat: Some( SourceNatConfig::new(public_ipv4_addr1, 0, MAX_PORT) diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index 4cb80c24495..483ea5c26a6 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -23,7 +23,7 @@ use omicron_common::zone_images::ZoneImageFileSource; use omicron_uuid_kinds::OmicronZoneUuid; pub use oxlog::is_oxide_smf_log_file; use slog::{Logger, error, info, o, warn}; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::{Ipv4Addr, Ipv6Addr}; use std::sync::Arc; #[cfg(target_os = "illumos")] use std::sync::OnceLock; @@ -572,7 +572,7 @@ impl RunningZone { } })?; let zone = Some(self.inner.name.as_ref()); - if let IpAddr::V4(gateway) = port.gateway().ip() { + if let Some(gateway) = port.gateway().ipv4_addr() { let addr = Zones::ensure_address(zone, &addrobj, AddressRequest::Dhcp) .await?; diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs index 9ea9fe59172..0cf3168f905 100644 --- a/nexus/db-model/src/network_interface.rs +++ b/nexus/db-model/src/network_interface.rs @@ -19,6 +19,10 @@ use nexus_db_schema::schema::service_network_interface; use nexus_sled_agent_shared::inventory::ZoneKind; use nexus_types::external_api::params; use nexus_types::identity::Resource; +use omicron_common::api::external::Error; +use omicron_common::api::internal::shared::PrivateIpConfig; +use omicron_common::api::internal::shared::PrivateIpv4Config; +use omicron_common::api::internal::shared::PrivateIpv6Config; use omicron_common::api::{external, internal}; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; @@ -102,15 +106,86 @@ pub struct NetworkInterface { impl NetworkInterface { pub fn into_internal( self, - subnet: oxnet::IpNet, - ) -> internal::shared::NetworkInterface { - // TODO-completeness: Handle IP Subnets of either version. - // https://github.com/oxidecomputer/omicron/issues/9246. - assert!( - matches!(subnet, oxnet::IpNet::V4(_)), - "Only IPv4 VPC Subnets are currently supported" - ); - internal::shared::NetworkInterface { + ipv4_subnet: oxnet::Ipv4Net, + ipv6_subnet: oxnet::Ipv6Net, + ) -> Result { + let ip_config = match (self.ipv4, self.ipv6) { + (None, None) => unreachable!(), + (None, Some(ip)) => { + // Check that all transit IPs are IPv6. + let transit_ips = self + .transit_ips + .iter() + .map(|net| { + let IpNetwork::V6(net) = net else { + return Err(Error::internal_error(&format!( + "NIC with ID '{}' is IPv6-only, but has \ + IPv4 transit IPs", + self.id(), + ))); + }; + Ok(Ipv6Net::from(*net)) + }) + .collect::>()?; + PrivateIpConfig::V6(PrivateIpv6Config::new_with_transit_ips( + *ip, + ipv6_subnet, + transit_ips, + )?) + } + (Some(ip), None) => { + // Check that all transit IPs are IPv4. + let transit_ips = self + .transit_ips + .iter() + .map(|net| { + let IpNetwork::V4(net) = net else { + return Err(Error::internal_error(&format!( + "NIC with ID '{}' is IPv4-only, but has \ + IPv6 transit IPs", + self.id(), + ))); + }; + Ok(Ipv4Net::from(*net)) + }) + .collect::>()?; + PrivateIpConfig::V4(PrivateIpv4Config::new_with_transit_ips( + *ip, + ipv4_subnet, + transit_ips, + )?) + } + (Some(ipv4), Some(ipv6)) => { + let ipv4_transit_ips = self + .transit_ips + .iter() + .filter_map(|net| match net { + IpNetwork::V4(net) => Some(Ipv4Net::from(*net)), + IpNetwork::V6(_) => None, + }) + .collect(); + let ipv6_transit_ips = self + .transit_ips + .iter() + .filter_map(|net| match net { + IpNetwork::V6(net) => Some(Ipv6Net::from(*net)), + IpNetwork::V4(_) => None, + }) + .collect(); + let v4 = PrivateIpv4Config::new_with_transit_ips( + *ipv4, + ipv4_subnet, + ipv4_transit_ips, + )?; + let v6 = PrivateIpv6Config::new_with_transit_ips( + *ipv6, + ipv6_subnet, + ipv6_transit_ips, + )?; + PrivateIpConfig::DualStack { v4, v6 } + } + }; + Ok(internal::shared::NetworkInterface { id: self.id(), kind: match self.kind { NetworkInterfaceKind::Instance => { @@ -130,16 +205,12 @@ impl NetworkInterface { } }, name: self.name().clone(), - // TODO-completeness: Handle one or both IP addresses when - // addressing https://github.com/oxidecomputer/omicron/issues/9246. - ip: self.ipv4.expect("only IPv4 interfaces are supported").into(), + ip_config, mac: self.mac.into(), - subnet, vni: external::Vni::try_from(0).unwrap(), primary: self.primary, slot: *self.slot, - transit_ips: self.transit_ips.into_iter().map(Into::into).collect(), - } + }) } } @@ -208,36 +279,34 @@ impl ServiceNetworkInterface { } // TODO-remove: Remove this when we support dual-stack service NICs. See -// https://github.com/oxidecomputer/omicron/issues/9246. +// https://github.com/oxidecomputer/omicron/issues/9314. #[derive(Debug, thiserror::Error)] #[error( - "Service NIC {nic_id} has an IPv6 address ({ip}); \ - only a single IPv4 address is supported" + "Service NIC {nic_id} is dual-stack, \ + only a single IPv4 or IPv6 address is supported" )] -pub struct ServiceNicNotIpv4OnlyError { +pub struct DualStackServiceNicError { pub nic_id: Uuid, - pub ip: std::net::Ipv6Addr, } impl TryFrom<&'_ ServiceNetworkInterface> for nexus_types::deployment::OmicronZoneNic { - type Error = ServiceNicNotIpv4OnlyError; + type Error = DualStackServiceNicError; fn try_from(nic: &ServiceNetworkInterface) -> Result { - if let Some(ipv6) = nic.ipv6 { - return Err(ServiceNicNotIpv4OnlyError { - nic_id: nic.id(), - ip: *ipv6, - }); - } - let Some(ip) = nic.ipv4 else { - unreachable!("must be single-stack IPv4"); + let ip = match (nic.ipv4, nic.ipv6) { + (None, None) => unreachable!("database constraint ensures this"), + (None, Some(ip)) => ip.into(), + (Some(ip), None) => ip.into(), + (Some(_), Some(_)) => { + return Err(DualStackServiceNicError { nic_id: nic.id() }); + } }; Ok(Self { id: VnicUuid::from_untyped_uuid(nic.id()), mac: *nic.mac, - ip: ip.into(), + ip, slot: *nic.slot, primary: nic.primary, }) @@ -496,6 +565,23 @@ impl IpConfig { } } + /// Construct a dual-stack IP configuration with explicit IP addresses. + pub fn new_dual_stack( + ipv4: std::net::Ipv4Addr, + ipv6: std::net::Ipv6Addr, + ) -> Self { + IpConfig::DualStack { + v4: Ipv4Config { + ip: Ipv4Assignment::Explicit(ipv4), + transit_ips: Vec::new(), + }, + v6: Ipv6Config { + ip: Ipv6Assignment::Explicit(ipv6), + transit_ips: Vec::new(), + }, + } + } + /// Construct an IP configuration with both IPv4 / IPv6 addresses and no /// transit IPs. pub fn auto_dual_stack() -> Self { diff --git a/nexus/db-model/src/omicron_zone_config.rs b/nexus/db-model/src/omicron_zone_config.rs index 74e2b9637b9..9cbed0fbd4a 100644 --- a/nexus/db-model/src/omicron_zone_config.rs +++ b/nexus/db-model/src/omicron_zone_config.rs @@ -17,8 +17,11 @@ use anyhow::{Context, anyhow, bail, ensure}; use ipnetwork::IpNetwork; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_types::inventory::NetworkInterface; -use omicron_common::api::internal::shared::NetworkInterfaceKind; +use omicron_common::api::internal::shared::{ + NetworkInterfaceKind, PrivateIpConfig, +}; use omicron_uuid_kinds::{GenericUuid, OmicronZoneUuid}; +use oxnet::{Ipv4Net, Ipv6Net}; use std::net::{IpAddr, SocketAddr, SocketAddrV6}; use uuid::Uuid; @@ -149,12 +152,33 @@ impl OmicronZoneNic { id to match the zone's id ({zone_id})", ); + // TODO-completeness: Support dual-stack NICs for Omicron zones. + // See https://github.com/oxidecomputer/omicron/issues/9314. + let (ip, subnet) = match &nic.ip_config { + PrivateIpConfig::V4(ipv4) => ( + IpNetwork::V4((*ipv4.ip()).into()), + IpNetwork::V4((*ipv4.subnet()).into()), + ), + PrivateIpConfig::V6(ipv6) => ( + IpNetwork::V6((*ipv6.ip()).into()), + IpNetwork::V6((*ipv6.subnet()).into()), + ), + PrivateIpConfig::DualStack { .. } => { + bail!( + "Found a dual-stack NIC, which isn't yet supported. \ + nic_id=\"{}\" zone_id=\"{}\"", + nic.id, + zone_id, + ); + } + }; + Ok(Self { id: nic.id, name: Name::from(nic.name.clone()), - ip: IpNetwork::from(nic.ip), + ip, mac: MacAddr::from(nic.mac), - subnet: IpNetwork::from(nic.subnet), + subnet, vni: SqlU32::from(u32::from(nic.vni)), is_primary: nic.primary, slot: SqlU8::from(nic.slot), @@ -165,9 +189,26 @@ impl OmicronZoneNic { self, zone_id: OmicronZoneUuid, ) -> anyhow::Result { + let ip = match (self.ip.ip(), self.subnet) { + (IpAddr::V4(addr), IpNetwork::V4(net)) => { + PrivateIpConfig::new_ipv4(addr, Ipv4Net::from(net))? + } + (IpAddr::V6(addr), IpNetwork::V6(net)) => { + PrivateIpConfig::new_ipv6(addr, Ipv6Net::from(net))? + } + (IpAddr::V4(_), IpNetwork::V6(_)) + | (IpAddr::V6(_), IpNetwork::V4(_)) => bail!( + "OmicronZoneNic has a mix of IPv4 and IPv6 \ + addresses and subnets! nic_id=\"{}\" ip=\"{}\" \ + subnet=\"{}\"", + self.id, + self.ip.ip(), + self.subnet, + ), + }; Ok(NetworkInterface { id: self.id, - ip: self.ip.ip(), + ip_config: ip, kind: NetworkInterfaceKind::Service { id: zone_id.into_untyped_uuid(), }, @@ -177,8 +218,6 @@ impl OmicronZoneNic { slot: *self.slot, vni: omicron_common::api::external::Vni::try_from(*self.vni) .context("parsing VNI")?, - subnet: self.subnet.into(), - transit_ips: vec![], }) } } diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 6bfb26db881..dcc2f6c10b0 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -3176,6 +3176,7 @@ mod tests { use omicron_common::api::external::Vni; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; + use omicron_common::api::internal::shared::PrivateIpConfig; use omicron_common::disk::DiskIdentity; use omicron_common::disk::M2Slot; use omicron_common::update::ArtifactId; @@ -3188,12 +3189,10 @@ mod tests { use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; - use oxnet::IpNet; use pretty_assertions::assert_eq; use rand::Rng; use std::collections::BTreeSet; use std::mem; - use std::net::IpAddr; use std::net::Ipv4Addr; use std::net::Ipv6Addr; use std::net::SocketAddrV6; @@ -4329,6 +4328,11 @@ mod tests { .await .expect("add range to service ip pool"); let zone_id = OmicronZoneUuid::new_v4(); + let ip_config = PrivateIpConfig::new_ipv4( + Ipv4Addr::new(172, 30, 2, 6), + oxnet::Ipv4Net::new(Ipv4Addr::new(172, 30, 2, 0), 24).unwrap(), + ) + .unwrap(); blueprint .sleds .get_mut(&sled_id) @@ -4357,15 +4361,11 @@ mod tests { id: *zone_id.as_untyped_uuid(), }, name: Name::from_str("mynic").unwrap(), - ip: "172.30.2.6".parse().unwrap(), + ip_config, mac: MacAddr::random_system(), - subnet: IpNet::host_net(IpAddr::V6( - Ipv6Addr::LOCALHOST, - )), vni: Vni::random(), primary: true, slot: 1, - transit_ips: vec![], }, external_tls: false, external_dns_servers: vec![], diff --git a/nexus/db-queries/src/db/datastore/deployment/external_networking.rs b/nexus/db-queries/src/db/datastore/deployment/external_networking.rs index 0741c38dd5f..d900ca83b9a 100644 --- a/nexus/db-queries/src/db/datastore/deployment/external_networking.rs +++ b/nexus/db-queries/src/db/datastore/deployment/external_networking.rs @@ -24,6 +24,7 @@ use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::IpVersion; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; +use omicron_common::api::internal::shared::PrivateIpConfig; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; use slog::Logger; @@ -281,7 +282,21 @@ impl DataStore { log: &Logger, ) -> Result { // See the comment in is_external_ip_already_allocated(). - if cfg!(any(test, feature = "testing")) && nic.ip.is_loopback() { + // + // TODO-completeness: Ensure this works for dual-stack Omicron service + // zone NICs. See https://github.com/oxidecomputer/omicron/issues/9313. + if cfg!(any(test, feature = "testing")) + && nic + .ip_config + .ipv4_addr() + .map(|ip| ip.is_loopback()) + .unwrap_or(false) + && nic + .ip_config + .ipv6_addr() + .map(|ip| ip.is_loopback()) + .unwrap_or(false) + { return Ok(true); } @@ -301,20 +316,10 @@ impl DataStore { // because that would require an extra DB lookup. We'll assume if // these main properties are correct, the subnet is too. for allocated_nic in &allocated_nics { - // TODO-completeness: Need support for dual-stack internal - // network interfaces. See - // https://github.com/oxidecomputer/omicron/issues/9246. - // - // This should not be possible to hit until we actually allow - // creating a NIC with a VPC-private IP address. - let Some(ipv4) = allocated_nic.ipv4 else { - return Err(Error::internal_error(&format!( - "Allocated NICs should be single-stack IPv4, but \ - NIC with id '{}' is missing an IPv4 address", - allocated_nic.identity.id, - ))); - }; - if std::net::IpAddr::from(ipv4) == nic.ip + if allocated_nic.ipv4.map(Into::into).as_ref() + == nic.ip_config.ipv4_addr() + && allocated_nic.ipv6.map(Into::into).as_ref() + == nic.ip_config.ipv6_addr() && *allocated_nic.mac == nic.mac && *allocated_nic.slot == nic.slot && allocated_nic.primary == nic.primary @@ -437,14 +442,12 @@ impl DataStore { return Ok(()); } - // TODO-completeness: Handle dual-stack `shared::NetworkInterface`s. - // See https://github.com/oxidecomputer/omicron/issues/9246. - let std::net::IpAddr::V4(ip) = nic.ip else { - return Err(Error::internal_error(&format!( - "Unexpectedly found a service NIC without an IPv4 \ - address, nic_id=\"{}\"", - nic.id, - ))); + let ip_config = match &nic.ip_config { + PrivateIpConfig::V4(ipv4) => IpConfig::from_ipv4(*ipv4.ip()), + PrivateIpConfig::V6(ipv6) => IpConfig::from_ipv6(*ipv6.ip()), + PrivateIpConfig::DualStack { v4, v6 } => { + IpConfig::new_dual_stack(*v4.ip(), *v6.ip()) + } }; let nic_arg = IncompleteNetworkInterface::new_service( nic.id, @@ -454,7 +457,7 @@ impl DataStore { name: nic.name.clone(), description: format!("{} service vNIC", zone_kind.report_str()), }, - IpConfig::from_ipv4(ip), + ip_config, nic.mac, nic.slot, )?; @@ -540,7 +543,7 @@ mod tests { use omicron_uuid_kinds::ExternalZpoolUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; - use oxnet::IpNet; + use std::collections::BTreeSet; use std::net::IpAddr; use std::net::SocketAddr; use uuid::Uuid; @@ -599,22 +602,24 @@ mod tests { id: ExternalIpUuid::new_v4(), ip: external_ips.next().expect("exhausted external_ips"), }; + let nexus_private_ip_config = PrivateIpConfig::new_ipv4( + NEXUS_OPTE_IPV4_SUBNET + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES) + .unwrap(), + *NEXUS_OPTE_IPV4_SUBNET, + ) + .unwrap(); let nexus_nic = NetworkInterface { id: Uuid::new_v4(), kind: NetworkInterfaceKind::Service { id: nexus_id.into_untyped_uuid(), }, name: "test-nexus".parse().expect("bad name"), - ip: NEXUS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES) - .unwrap() - .into(), + ip_config: nexus_private_ip_config, mac: random_system_mac_iter.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }; let dns_id = OmicronZoneUuid::new_v4(); @@ -625,22 +630,24 @@ mod tests { 0, ), }; + let dns_private_ip_config = PrivateIpConfig::new_ipv4( + DNS_OPTE_IPV4_SUBNET + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES) + .unwrap(), + *DNS_OPTE_IPV4_SUBNET, + ) + .unwrap(); let dns_nic = NetworkInterface { id: Uuid::new_v4(), kind: NetworkInterfaceKind::Service { id: dns_id.into_untyped_uuid(), }, name: "test-external-dns".parse().expect("bad name"), - ip: DNS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES) - .unwrap() - .into(), + ip_config: dns_private_ip_config, mac: random_system_mac_iter.next().unwrap(), - subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }; // Boundary NTP: @@ -654,22 +661,24 @@ mod tests { ) .unwrap(), }; + let ntp_private_ip_config = PrivateIpConfig::new_ipv4( + NTP_OPTE_IPV4_SUBNET + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES) + .unwrap(), + *NTP_OPTE_IPV4_SUBNET, + ) + .unwrap(); let ntp_nic = NetworkInterface { id: Uuid::new_v4(), kind: NetworkInterfaceKind::Service { id: ntp_id.into_untyped_uuid(), }, name: "test-external-ntp".parse().expect("bad name"), - ip: NTP_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES) - .unwrap() - .into(), + ip_config: ntp_private_ip_config, mac: random_system_mac_iter.next().unwrap(), - subnet: IpNet::from(*NTP_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }; Self { @@ -861,13 +870,14 @@ mod tests { assert_eq!(db_nexus_nics[0].vpc_id, NEXUS_VPC_SUBNET.vpc_id); assert_eq!(db_nexus_nics[0].subnet_id, NEXUS_VPC_SUBNET.id()); assert_eq!(*db_nexus_nics[0].mac, self.nexus_nic.mac); - // TODO-completeness: Handle the `nexus_nic` being dual-stack as - // well. See https://github.com/oxidecomputer/omicron/issues/9246. assert_eq!( - db_nexus_nics[0].ipv4.map(IpAddr::from), - Some(self.nexus_nic.ip) + db_nexus_nics[0].ipv4, + self.nexus_nic.ip_config.ipv4_addr().copied().map(Into::into), + ); + assert_eq!( + db_nexus_nics[0].ipv6, + self.nexus_nic.ip_config.ipv6_addr().copied().map(Into::into), ); - assert!(db_nexus_nics[0].ipv6.is_none()); assert_eq!(*db_nexus_nics[0].slot, self.nexus_nic.slot); assert_eq!(db_nexus_nics[0].primary, self.nexus_nic.primary); @@ -887,11 +897,13 @@ mod tests { assert_eq!(db_dns_nics[0].vpc_id, DNS_VPC_SUBNET.vpc_id); assert_eq!(db_dns_nics[0].subnet_id, DNS_VPC_SUBNET.id()); assert_eq!(*db_dns_nics[0].mac, self.dns_nic.mac); - // TODO-completeness: Handle the `nexus_nic` being dual-stack as - // well. See https://github.com/oxidecomputer/omicron/issues/9246. assert_eq!( - db_nexus_nics[0].ipv4.map(IpAddr::from), - Some(self.nexus_nic.ip) + db_nexus_nics[0].ipv4, + self.nexus_nic.ip_config.ipv4_addr().copied().map(Into::into), + ); + assert_eq!( + db_nexus_nics[0].ipv6, + self.nexus_nic.ip_config.ipv6_addr().copied().map(Into::into), ); assert!(db_nexus_nics[0].ipv6.is_none()); assert_eq!(*db_dns_nics[0].slot, self.dns_nic.slot); @@ -913,11 +925,13 @@ mod tests { assert_eq!(db_ntp_nics[0].vpc_id, NTP_VPC_SUBNET.vpc_id); assert_eq!(db_ntp_nics[0].subnet_id, NTP_VPC_SUBNET.id()); assert_eq!(*db_ntp_nics[0].mac, self.ntp_nic.mac); - // TODO-completeness: Handle the `nexus_nic` being dual-stack as - // well. See https://github.com/oxidecomputer/omicron/issues/9246. assert_eq!( - db_nexus_nics[0].ipv4.map(IpAddr::from), - Some(self.nexus_nic.ip) + db_nexus_nics[0].ipv4, + self.nexus_nic.ip_config.ipv4_addr().copied().map(Into::into), + ); + assert_eq!( + db_nexus_nics[0].ipv6, + self.nexus_nic.ip_config.ipv6_addr().copied().map(Into::into), ); assert!(db_nexus_nics[0].ipv6.is_none()); assert_eq!(*db_ntp_nics[0].slot, self.ntp_nic.slot); @@ -1192,7 +1206,22 @@ mod tests { as &dyn Fn(OmicronZoneUuid, &mut NetworkInterface) -> String, // non-matching IP &|zone_id, nic| { - nic.ip = bogus_ip; + // Take the last IP still in the subnet. + if let Some(subnet) = nic.ip_config.ipv4_subnet() { + let new = + PrivateIpConfig::new_ipv4(subnet.last_addr(), *subnet) + .unwrap(); + nic.ip_config = new; + } else if let Some(subnet) = nic.ip_config.ipv6_subnet() { + let new = + PrivateIpConfig::new_ipv6(subnet.last_addr(), *subnet) + .unwrap(); + nic.ip_config = new; + } else { + todo!( + "See https://github.com/oxidecomputer/omicron/issues/9313" + ); + } format!("zone {zone_id} already has 1 non-matching NIC") }, ] { @@ -1381,13 +1410,15 @@ mod tests { id: zone_id.into_untyped_uuid(), }, name: "test-external-dns".parse().unwrap(), - ip: opte_ip_iter.next().unwrap().into(), + ip_config: PrivateIpConfig::new_ipv4( + opte_ip_iter.next().unwrap(), + *DNS_OPTE_IPV4_SUBNET, + ) + .unwrap(), mac: mac_iter.next().unwrap(), - subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, }, ), diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 43a6efa479a..462bc734563 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -19,6 +19,7 @@ use crate::db::model::Name; use crate::db::model::NetworkInterface; use crate::db::model::NetworkInterfaceKind; use crate::db::model::NetworkInterfaceUpdate; +use crate::db::model::SqlU8; use crate::db::model::VpcSubnet; use crate::db::pagination::Paginator; use crate::db::pagination::paginated; @@ -45,6 +46,11 @@ use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::internal::shared::PrivateIpConfig; +use omicron_common::api::internal::shared::PrivateIpv4Config; +use omicron_common::api::internal::shared::PrivateIpv6Config; +use oxnet::Ipv4Net; +use oxnet::Ipv6Net; use ref_cast::RefCast; use uuid::Uuid; @@ -63,19 +69,19 @@ struct NicInfo { #[diesel(select_expression = nexus_db_schema::schema::network_interface::ip)] ipv4: Option, #[diesel(select_expression = nexus_db_schema::schema::network_interface::ipv6)] - _ipv6: Option, + ipv6: Option, #[diesel(select_expression = nexus_db_schema::schema::network_interface::mac)] mac: db::model::MacAddr, #[diesel(select_expression = nexus_db_schema::schema::vpc_subnet::ipv4_block)] ipv4_block: db::model::Ipv4Net, #[diesel(select_expression = nexus_db_schema::schema::vpc_subnet::ipv6_block)] - _ipv6_block: db::model::Ipv6Net, + ipv6_block: db::model::Ipv6Net, #[diesel(select_expression = nexus_db_schema::schema::vpc::vni)] vni: db::model::Vni, #[diesel(select_expression = nexus_db_schema::schema::network_interface::is_primary)] primary: bool, #[diesel(select_expression = nexus_db_schema::schema::network_interface::slot)] - slot: i16, + slot: SqlU8, #[diesel(select_expression = nexus_db_schema::schema::network_interface::transit_ips)] transit_ips: Vec, } @@ -91,21 +97,78 @@ impl TryFrom omicron_common::api::internal::shared::NetworkInterface, Self::Error, > { - // TODO-completeness: Support IPv6 and dual-stack NICs in the Nexus <-> - // sled-agent API. That includes the IPs themselves and the VPC Subnets. - // - // See https://github.com/oxidecomputer/omicron/issues/9246. - // - // This whole method can become the infallible `From` again when that's - // resolved. - let Some(ipv4) = nic.ipv4 else { - return Err(Error::internal_error(&format!( - "Found internal NIC without an IPv4 address: \ - nic_id=\"{}\", parent_id=\"{}\"", - nic.id, nic.parent_id, - ))); + let ip = match (nic.ipv4, nic.ipv6) { + (None, None) => { + return Err(Error::internal_error( + "Found NIC with no VPC-private IP addresses at all", + )); + } + (Some(ipv4), None) => { + let transit_ips = nic + .transit_ips + .iter() + .map(|net| { + let ipnetwork::IpNetwork::V4(net) = net else { + return Err(Error::internal_error( + "Found NIC with IPv4 address only, but \ + which has IPv6 transit IPs", + )); + }; + Ok(Ipv4Net::from(*net)) + }) + .collect::>()?; + PrivateIpConfig::V4(PrivateIpv4Config::new_with_transit_ips( + *ipv4, + *nic.ipv4_block, + transit_ips, + )?) + } + (None, Some(ipv6)) => { + let transit_ips = nic + .transit_ips + .iter() + .map(|net| { + let ipnetwork::IpNetwork::V6(net) = net else { + return Err(Error::internal_error( + "Found NIC with IPv6 address only, but \ + which has IPv4 transit IPs", + )); + }; + Ok(Ipv6Net::from(*net)) + }) + .collect::>()?; + PrivateIpConfig::V6(PrivateIpv6Config::new_with_transit_ips( + *ipv6, + *nic.ipv6_block, + transit_ips, + )?) + } + (Some(ipv4), Some(ipv6)) => { + let mut ipv4_transit_ips = Vec::new(); + let mut ipv6_transit_ips = Vec::new(); + for net in nic.transit_ips.iter() { + match net { + ipnetwork::IpNetwork::V4(net) => { + ipv4_transit_ips.push(Ipv4Net::from(*net)) + } + ipnetwork::IpNetwork::V6(net) => { + ipv6_transit_ips.push(Ipv6Net::from(*net)) + } + } + } + let v4 = PrivateIpv4Config::new_with_transit_ips( + *ipv4, + *nic.ipv4_block, + ipv4_transit_ips, + )?; + let v6 = PrivateIpv6Config::new_with_transit_ips( + *ipv6, + *nic.ipv6_block, + ipv6_transit_ips, + )?; + PrivateIpConfig::DualStack { v4, v6 } + } }; - let ip_subnet = oxnet::IpNet::V4(nic.ipv4_block.0); let kind = match nic.kind { NetworkInterfaceKind::Instance => { omicron_common::api::internal::shared::NetworkInterfaceKind::Instance{ id: nic.parent_id } @@ -121,13 +184,11 @@ impl TryFrom id: nic.id, kind, name: nic.name.into(), - ip: ipv4.into(), + ip_config: ip, mac: nic.mac.0, - subnet: ip_subnet, vni: nic.vni.0, primary: nic.primary, slot: u8::try_from(nic.slot).unwrap(), - transit_ips: nic.transit_ips.iter().map(|v| (*v).into()).collect(), }) } } diff --git a/nexus/db-queries/src/db/datastore/probe.rs b/nexus/db-queries/src/db/datastore/probe.rs index 7794604467a..725c34a108b 100644 --- a/nexus/db-queries/src/db/datastore/probe.rs +++ b/nexus/db-queries/src/db/datastore/probe.rs @@ -118,10 +118,19 @@ impl super::DataStore { public_error_from_diesel(e, ErrorHandler::Server) })?; - let mut interface: NetworkInterface = - interface.into_internal(db_subnet.ipv4_block.0.into()); - - interface.vni = vni.0; + let interface = NetworkInterface { + vni: vni.0, + ..interface.into_internal( + db_subnet.ipv4_block.0, + db_subnet.ipv6_block.0, + )? + }; + + // TODO ProbeInfo still uses version 1 of the network interface. We + // need to support version 2, which allows dual-stack IP + // configurations. + // See https://github.com/oxidecomputer/omicron/issues/9248. + let interface = interface.try_into()?; result.push(ProbeInfo { id: probe.id(), @@ -162,9 +171,17 @@ impl super::DataStore { let vni = self.resolve_vpc_to_vni(opctx, interface.vpc_id).await?; - let mut interface: NetworkInterface = - interface.into_internal(db_subnet.ipv4_block.0.into()); - interface.vni = vni.0; + let interface = NetworkInterface { + vni: vni.0, + ..interface + .into_internal(db_subnet.ipv4_block.0, db_subnet.ipv6_block.0)? + }; + + // TODO ProbeInfo still uses version 1 of the network interface. We + // need to support version 2, which allows dual-stack IP + // configurations. + // See https://github.com/oxidecomputer/omicron/issues/9248. + let interface = interface.try_into()?; Ok(ProbeInfo { id: probe.id(), @@ -382,6 +399,7 @@ impl super::DataStore { external_ip::dsl::kind, nexus_db_model::NetworkInterface::as_select(), vpc_subnet::dsl::ipv4_block, + vpc_subnet::dsl::ipv6_block, vpc::dsl::vni, )) .get_results_async::<( @@ -391,14 +409,13 @@ impl super::DataStore { nexus_db_model::SqlU16, nexus_db_model::IpKind, nexus_db_model::NetworkInterface, - // TODO: Need to extract IPv6 block when we support dual-stack - // external IP addresses. See - // https://github.com/oxidecomputer/omicron/issues/9318. nexus_db_model::Ipv4Net, + nexus_db_model::Ipv6Net, nexus_db_model::Vni, )>(&*self.pool_connection_authorized(opctx).await?) .await - .map(|rows| { + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + .and_then(|rows| { rows.into_iter() .map( |( @@ -409,6 +426,7 @@ impl super::DataStore { kind, nic, ipv4_block, + ipv6_block, vni, )| { let kind = db_ip_kind_to_sled(kind); @@ -421,18 +439,17 @@ impl super::DataStore { }]; let interface = NetworkInterface { vni: vni.0, - ..nic.into_internal((*ipv4_block).into()) + ..nic.into_internal(*ipv4_block, *ipv6_block)? }; - ProbeCreate { + Ok(ProbeCreate { id: ProbeUuid::from_untyped_uuid(probe_id), external_ips, interface, - } + }) }, ) - .collect() + .collect::>() }) - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Remove a probe from the data store. diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 92854041b6b..0f91da3bfc4 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -69,14 +69,13 @@ use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::UserId; +use omicron_common::api::internal::shared::PrivateIpConfig; use omicron_common::bail_unless; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::SiloUserUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use slog_error_chain::InlineErrorChain; -use std::net::IpAddr; -use std::net::Ipv4Addr; use std::sync::{Arc, OnceLock}; use uuid::Uuid; @@ -551,18 +550,24 @@ impl DataStore { let zone_type = &zone_config.zone_type; let zone_report_str = zone_type.kind().report_str(); - // Extract an IPv4 address from the `shared::NetworkInterface` object. - // - // TODO-completeness: Handle IPv6 interface addresses. See - // https://github.com/oxidecomputer/omicron/issues/9246. - let extract_ipv4 = |nic: &NetworkInterface| -> Result { - let IpAddr::V4(ipv4) = nic.ip else { - return Err(Error::invalid_request( - "IPv6 addresses are not yet supported", - )); + // TODO-completeness: Support dual-stack NICs for services. See + // https://github.com/oxidecomputer/omicron/issues/9313. + let extract_ip_config = + |nic: &NetworkInterface| -> Result { + match &nic.ip_config { + PrivateIpConfig::V4(ipv4) => { + Ok(IpConfig::from_ipv4(*ipv4.ip())) + } + PrivateIpConfig::V6(ipv6) => { + Ok(IpConfig::from_ipv6(*ipv6.ip())) + } + PrivateIpConfig::DualStack { .. } => { + Err(Error::invalid_request( + "Dual-stack service NICs are not yet supported", + )) + } + } }; - Ok(ipv4) - }; let service_ip_nic = match zone_type { BlueprintZoneType::ExternalDns( @@ -570,7 +575,8 @@ impl DataStore { ) => { let external_ip = OmicronZoneExternalIp::Floating(dns_address.into_ip()); - let ip = extract_ipv4(nic).map_err(RackInitError::AddingNic)?; + let ip_config = + extract_ip_config(nic).map_err(RackInitError::AddingNic)?; let db_nic = IncompleteNetworkInterface::new_service( nic.id, zone_config.id.into_untyped_uuid(), @@ -582,7 +588,7 @@ impl DataStore { zone_report_str ), }, - IpConfig::from_ipv4(ip), + ip_config, nic.mac, nic.slot, ) @@ -595,7 +601,8 @@ impl DataStore { .. }) => { let external_ip = OmicronZoneExternalIp::Floating(*external_ip); - let ip = extract_ipv4(nic).map_err(RackInitError::AddingNic)?; + let ip_config = + extract_ip_config(nic).map_err(RackInitError::AddingNic)?; let db_nic = IncompleteNetworkInterface::new_service( nic.id, zone_config.id.into_untyped_uuid(), @@ -607,7 +614,7 @@ impl DataStore { zone_report_str ), }, - IpConfig::from_ipv4(ip), + ip_config, nic.mac, nic.slot, ) @@ -618,7 +625,8 @@ impl DataStore { blueprint_zone_type::BoundaryNtp { external_ip, nic, .. }, ) => { let external_ip = OmicronZoneExternalIp::Snat(*external_ip); - let ip = extract_ipv4(nic).map_err(RackInitError::AddingNic)?; + let ip_config = + extract_ip_config(nic).map_err(RackInitError::AddingNic)?; let db_nic = IncompleteNetworkInterface::new_service( nic.id, zone_config.id.into_untyped_uuid(), @@ -630,7 +638,7 @@ impl DataStore { zone_report_str ), }, - IpConfig::from_ipv4(ip), + ip_config, nic.mac, nic.slot, ) @@ -1108,7 +1116,6 @@ mod test { use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; - use oxnet::IpNet; use std::collections::{BTreeMap, HashMap}; use std::net::Ipv6Addr; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -1434,21 +1441,31 @@ mod test { let external_dns_pip = DNS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) .unwrap(); + let external_dns_pip_config = + PrivateIpConfig::new_ipv4(external_dns_pip, *DNS_OPTE_IPV4_SUBNET) + .unwrap(); let external_dns_id = OmicronZoneUuid::new_v4(); let nexus_ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 6)); let nexus_pip = NEXUS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) .unwrap(); + let nexus_pip_config = + PrivateIpConfig::new_ipv4(nexus_pip, *NEXUS_OPTE_IPV4_SUBNET) + .unwrap(); let nexus_id = OmicronZoneUuid::new_v4(); let ntp1_ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 5)); let ntp1_pip = NTP_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) .unwrap(); + let ntp1_pip_config = + PrivateIpConfig::new_ipv4(ntp1_pip, *NTP_OPTE_IPV4_SUBNET).unwrap(); let ntp1_id = OmicronZoneUuid::new_v4(); let ntp2_ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 5)); let ntp2_pip = NTP_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 2) .unwrap(); + let ntp2_pip_config = + PrivateIpConfig::new_ipv4(ntp2_pip, *NTP_OPTE_IPV4_SUBNET).unwrap(); let ntp2_id = OmicronZoneUuid::new_v4(); let ntp3_id = OmicronZoneUuid::new_v4(); let mut macs = MacAddr::iter_system(); @@ -1476,13 +1493,11 @@ mod test { id: external_dns_id.into_untyped_uuid(), }, name: "external-dns".parse().unwrap(), - ip: external_dns_pip.into(), + ip_config: external_dns_pip_config, mac: macs.next().unwrap(), - subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, }, ), @@ -1504,13 +1519,11 @@ mod test { id: ntp1_id.into_untyped_uuid(), }, name: "ntp1".parse().unwrap(), - ip: ntp1_pip.into(), + ip_config: ntp1_pip_config, mac: macs.next().unwrap(), - subnet: IpNet::from(*NTP_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), @@ -1551,13 +1564,11 @@ mod test { id: nexus_id.into_untyped_uuid(), }, name: "nexus".parse().unwrap(), - ip: nexus_pip.into(), + ip_config: nexus_pip_config, mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, nexus_generation: *Generation::new(), }, @@ -1580,13 +1591,11 @@ mod test { id: ntp2_id.into_untyped_uuid(), }, name: "ntp2".parse().unwrap(), - ip: ntp2_pip.into(), + ip_config: ntp2_pip_config, mac: macs.next().unwrap(), - subnet: IpNet::from(*NTP_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), @@ -1781,9 +1790,15 @@ mod test { let nexus_pip1 = NEXUS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) .unwrap(); + let nexus_pip1_config = + PrivateIpConfig::new_ipv4(nexus_pip1, *NEXUS_OPTE_IPV4_SUBNET) + .unwrap(); let nexus_pip2 = NEXUS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 2) .unwrap(); + let nexus_pip2_config = + PrivateIpConfig::new_ipv4(nexus_pip2, *NEXUS_OPTE_IPV4_SUBNET) + .unwrap(); let mut macs = MacAddr::iter_system(); let mut blueprint_zones = BTreeMap::new(); @@ -1811,13 +1826,11 @@ mod test { id: nexus_id1.into_untyped_uuid(), }, name: "nexus1".parse().unwrap(), - ip: nexus_pip1.into(), + ip_config: nexus_pip1_config, mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, nexus_generation: *Generation::new(), }, @@ -1845,15 +1858,11 @@ mod test { id: nexus_id2.into_untyped_uuid(), }, name: "nexus2".parse().unwrap(), - ip: nexus_pip2.into(), + ip_config: nexus_pip2_config, mac: macs.next().unwrap(), - subnet: oxnet::IpNet::from( - *NEXUS_OPTE_IPV4_SUBNET, - ), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, nexus_generation: *Generation::new(), }, @@ -2073,6 +2082,9 @@ mod test { let nexus_pip = NEXUS_OPTE_IPV6_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES as u128 + 1) .unwrap(); + let nexus_pip_config = + PrivateIpConfig::new_ipv6(nexus_pip, *NEXUS_OPTE_IPV6_SUBNET) + .unwrap(); let mut macs = MacAddr::iter_system(); let mut blueprint_zones = BTreeMap::new(); @@ -2099,13 +2111,11 @@ mod test { id: nexus_id.into_untyped_uuid(), }, name: "nexus1".parse().unwrap(), - ip: nexus_pip.into(), + ip_config: nexus_pip_config, mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV6_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, nexus_generation: *Generation::new(), }, @@ -2174,28 +2184,8 @@ mod test { ..Default::default() }, ) - .await; - - // IPv6 addresses aren't fully supported right now. See - // https://github.com/oxidecomputer/omicron/issues/1716. When that is - // fully-addressed, this will start to fail and we can remove this - // block to restore the previous test coverage. - let Err(Error::InvalidRequest { message }) = &rack else { - panic!( - "Expected an error initializing a rack with an IPv6 address, \ - until they are fully-supported. Found {rack:#?}" - ); - }; - assert_eq!( - message.external_message(), - "IPv6 addresses are not yet supported" - ); - let Ok(rack) = rack else { - db.terminate().await; - logctx.cleanup_successful(); - return; - }; - + .await + .expect("an initialized rack"); assert_eq!(rack.id(), rack_id()); assert!(rack.initialized); @@ -2318,6 +2308,9 @@ mod test { let nexus_pip = NEXUS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) .unwrap(); + let nexus_pip_config = + PrivateIpConfig::new_ipv4(nexus_pip, *NEXUS_OPTE_IPV4_SUBNET) + .unwrap(); let nexus_id = OmicronZoneUuid::new_v4(); let mut macs = MacAddr::iter_system(); let mut blueprint_zones = BTreeMap::new(); @@ -2344,13 +2337,11 @@ mod test { id: nexus_id.into_untyped_uuid(), }, name: "nexus".parse().unwrap(), - ip: nexus_pip.into(), + ip_config: nexus_pip_config, mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, nexus_generation: *Generation::new(), }, @@ -2426,10 +2417,16 @@ mod test { let external_dns_pip = DNS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) .unwrap(); + let external_dns_pip_config = + PrivateIpConfig::new_ipv4(external_dns_pip, *DNS_OPTE_IPV4_SUBNET) + .unwrap(); let nexus_id = OmicronZoneUuid::new_v4(); let nexus_pip = NEXUS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) .unwrap(); + let nexus_pip_config = + PrivateIpConfig::new_ipv4(nexus_pip, *NEXUS_OPTE_IPV4_SUBNET) + .unwrap(); let mut macs = MacAddr::iter_system(); let mut blueprint_zones = BTreeMap::new(); @@ -2455,13 +2452,11 @@ mod test { id: external_dns_id.into_untyped_uuid(), }, name: "external-dns".parse().unwrap(), - ip: external_dns_pip.into(), + ip_config: external_dns_pip_config, mac: macs.next().unwrap(), - subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, }, ), @@ -2488,13 +2483,11 @@ mod test { id: nexus_id.into_untyped_uuid(), }, name: "nexus".parse().unwrap(), - ip: nexus_pip.into(), + ip_config: nexus_pip_config, mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, nexus_generation: *Generation::new(), }, diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index b679bd63f6e..099828297a8 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -3293,9 +3293,10 @@ mod tests { .zone_type .external_networking() .expect("external networking for zone type"); - let IpAddr::V4(ip) = nic.ip else { - panic!("Expected an IPv4 address for this NIC"); - }; + let ip = nic + .ip_config + .ipv4_addr() + .expect("an IPv4 address for this NIC"); IncompleteNetworkInterface::new_service( nic.id, zone_config.id.into_untyped_uuid(), @@ -3304,7 +3305,7 @@ mod tests { name: nic.name.clone(), description: nic.name.to_string(), }, - IpConfig::from_ipv4(ip), + IpConfig::from_ipv4(*ip), nic.mac, nic.slot, ) diff --git a/nexus/inventory/src/examples.rs b/nexus/inventory/src/examples.rs index 2b48b5ed670..df581aee3fb 100644 --- a/nexus/inventory/src/examples.rs +++ b/nexus/inventory/src/examples.rs @@ -52,6 +52,7 @@ use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; +use sled_agent_types::inventory::v6::OmicronZonesConfig as OmicronZonesConfigV6; use sled_agent_types::zone_images::MupdateOverrideNonBootInfo; use sled_agent_types::zone_images::MupdateOverrideNonBootMismatch; use sled_agent_types::zone_images::MupdateOverrideNonBootResult; @@ -373,12 +374,23 @@ pub fn representative() -> Representative { // (2) pretty-printing each one with `json --in-place --file FILENAME` // (3) adjusting the format slightly with // `jq '{ generation: .omicron_generation, zones: .zones }'` + // + // Note that these files are in an older format of the zone configuration + // types. Rather than rewrite these, we're relying on existing conversion + // code, which we already have in order to support the sled-agent's + // versioned API in any case. let sled14_data = include_str!("../example-data/madrid-sled14.json"); let sled16_data = include_str!("../example-data/madrid-sled16.json"); let sled17_data = include_str!("../example-data/madrid-sled17.json"); - let sled14: OmicronZonesConfig = serde_json::from_str(sled14_data).unwrap(); - let sled16: OmicronZonesConfig = serde_json::from_str(sled16_data).unwrap(); - let sled17: OmicronZonesConfig = serde_json::from_str(sled17_data).unwrap(); + let sled14_v6: OmicronZonesConfigV6 = + serde_json::from_str(sled14_data).unwrap(); + let sled16_v6: OmicronZonesConfigV6 = + serde_json::from_str(sled16_data).unwrap(); + let sled17_v6: OmicronZonesConfigV6 = + serde_json::from_str(sled17_data).unwrap(); + let sled14 = OmicronZonesConfig::try_from(sled14_v6).unwrap(); + let sled16 = OmicronZonesConfig::try_from(sled16_v6).unwrap(); + let sled17 = OmicronZonesConfig::try_from(sled17_v6).unwrap(); // Convert these to `OmicronSledConfig`s. We'll start with empty disks and // datasets for now, and add to them below for sled14. diff --git a/nexus/networking/src/firewall_rules.rs b/nexus/networking/src/firewall_rules.rs index 49768969b40..b2defa9fd52 100644 --- a/nexus/networking/src/firewall_rules.rs +++ b/nexus/networking/src/firewall_rules.rs @@ -284,7 +284,7 @@ pub async fn resolve_firewall_rules_for_sled_agent( .get(vpc.name()) .unwrap_or(&no_interfaces) .iter() - .filter(|nic| nic.ip == *addr) + .filter(|nic| nic.ip_config.has_addr(addr)) .for_each(&mut push_target_nic); } external::VpcFirewallRuleTarget::IpNet(net) => { @@ -292,14 +292,17 @@ pub async fn resolve_firewall_rules_for_sled_agent( .get(vpc.name()) .unwrap_or(&no_interfaces) .iter() - .filter(|nic| match (net, nic.ip) { - (IpNet::V4(net), IpAddr::V4(ip)) => { - net.contains(ip) - } - (IpNet::V6(net), IpAddr::V6(ip)) => { - net.contains(ip) - } - (_, _) => false, + .filter(|nic| match net { + IpNet::V4(net) => nic + .ip_config + .ipv4_addr() + .map(|ip| net.contains(*ip)) + .unwrap_or(false), + IpNet::V6(net) => nic + .ip_config + .ipv6_addr() + .map(|ip| net.contains(*ip)) + .unwrap_or(false), }) .for_each(&mut push_target_nic); } @@ -366,9 +369,21 @@ pub async fn resolve_firewall_rules_for_sled_agent( .get(&name) .unwrap_or(&no_interfaces) { - host_addrs.insert(HostIdentifier::Ip( - IpNet::host_net(interface.ip), - )); + // Insert both IPv4 and / or IPv6 addresses. + if let Some(ipv4) = + interface.ip_config.ipv4_addr() + { + host_addrs.insert(HostIdentifier::Ip( + IpNet::host_net(IpAddr::V4(*ipv4)), + )); + } + if let Some(ipv6) = + interface.ip_config.ipv6_addr() + { + host_addrs.insert(HostIdentifier::Ip( + IpNet::host_net(IpAddr::V6(*ipv6)), + )); + } } } external::VpcFirewallRuleHostFilter::Subnet(name) => { diff --git a/nexus/reconfigurator/blippy/src/checks.rs b/nexus/reconfigurator/blippy/src/checks.rs index db66d337180..6a7134ccd9b 100644 --- a/nexus/reconfigurator/blippy/src/checks.rs +++ b/nexus/reconfigurator/blippy/src/checks.rs @@ -193,17 +193,34 @@ fn check_external_networking(blippy: &mut Blippy<'_>) { } } - // There should be no duplicate NIC IPs or MACs. - if let Some(prev_zone) = used_nic_ips.insert(nic.ip, zone) { - blippy.push_sled_note( - sled_id, - Severity::Fatal, - SledKind::DuplicateNicIp { - zone1: prev_zone.clone(), - zone2: zone.clone(), - ip: nic.ip, - }, - ); + // There should be no duplicate NIC IPs (of either version) or MACs. + if let Some(ipv4) = nic.ip_config.ipv4_addr() { + let ip = std::net::IpAddr::V4(*ipv4); + if let Some(prev_zone) = used_nic_ips.insert(ip, zone) { + blippy.push_sled_note( + sled_id, + Severity::Fatal, + SledKind::DuplicateNicIp { + zone1: prev_zone.clone(), + zone2: zone.clone(), + ip, + }, + ); + } + } + if let Some(ipv6) = nic.ip_config.ipv6_addr() { + let ip = std::net::IpAddr::V6(*ipv6); + if let Some(prev_zone) = used_nic_ips.insert(ip, zone) { + blippy.push_sled_note( + sled_id, + Severity::Fatal, + SledKind::DuplicateNicIp { + zone1: prev_zone.clone(), + zone2: zone.clone(), + ip, + }, + ); + } } if let Some(prev_zone) = used_nic_macs.insert(nic.mac, zone) { blippy.push_sled_note( @@ -1047,18 +1064,18 @@ mod tests { nexus_iter.next().expect("at least two Nexus zones"); assert_ne!(nexus0_sled_id, nexus1_sled_id); - let dup_ip = nexus0 + let dup_ip = &nexus0 .zone_type .external_networking() .expect("Nexus has external networking") .1 - .ip; + .ip_config; match &mut nexus1.zone_type { BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { nic, .. }) => { - nic.ip = dup_ip; + nic.ip_config = dup_ip.clone(); } _ => unreachable!("this is a Nexus zone"), }; @@ -1070,7 +1087,11 @@ mod tests { kind: Box::new(SledKind::DuplicateNicIp { zone1: nexus0.clone(), zone2: nexus1.clone(), - ip: dup_ip, + ip: dup_ip + .ipv4_addr() + .copied() + .expect("an IPv4 address") + .into(), }), }, }]; @@ -2127,13 +2148,28 @@ fn check_planning_input_network_records_appear_in_blueprint( let zone_type = &z.zone_type; match zone_type { BlueprintZoneType::BoundaryNtp(ntp) => { - all_boundary_ntp_nic_ips.insert(ntp.nic.ip); + if let Some(ip) = ntp.nic.ip_config.ipv4_addr().copied() { + all_boundary_ntp_nic_ips.insert(ip.into()); + } + if let Some(ip) = ntp.nic.ip_config.ipv6_addr().copied() { + all_boundary_ntp_nic_ips.insert(ip.into()); + } } BlueprintZoneType::Nexus(nexus) => { - all_nexus_nic_ips.insert(nexus.nic.ip); + if let Some(ip) = nexus.nic.ip_config.ipv4_addr().copied() { + all_nexus_nic_ips.insert(ip.into()); + } + if let Some(ip) = nexus.nic.ip_config.ipv6_addr().copied() { + all_nexus_nic_ips.insert(ip.into()); + } } BlueprintZoneType::ExternalDns(dns) => { - all_external_dns_nic_ips.insert(dns.nic.ip); + if let Some(ip) = dns.nic.ip_config.ipv4_addr().copied() { + all_external_dns_nic_ips.insert(ip.into()); + } + if let Some(ip) = dns.nic.ip_config.ipv6_addr().copied() { + all_external_dns_nic_ips.insert(ip.into()); + } } _ => (), } diff --git a/nexus/reconfigurator/cli-integration-tests/tests/integration/blueprint_edit.rs b/nexus/reconfigurator/cli-integration-tests/tests/integration/blueprint_edit.rs index 0ffeee4a304..abb87170109 100644 --- a/nexus/reconfigurator/cli-integration-tests/tests/integration/blueprint_edit.rs +++ b/nexus/reconfigurator/cli-integration-tests/tests/integration/blueprint_edit.rs @@ -66,6 +66,7 @@ async fn test_blueprint_edit(cptestctx: &ControlPlaneTestContext) { .blueprint_target_get_current_full(&opctx) .await .expect("failed to read current target blueprint"); + let mut disk_test = DiskTest::new(&cptestctx).await; disk_test.add_blueprint_disks(&initial_blueprint).await; @@ -170,9 +171,25 @@ async fn test_blueprint_edit(cptestctx: &ControlPlaneTestContext) { // Run this reconfigurator-cli invocation. write_json(&saved_state1_path, &state1).unwrap(); let exec = Exec::cmd(path_to_cli()).arg(&script1_path); - let (exit_status, _, stderr_text) = run_command(exec); + let (exit_status, stdout_text, stderr_text) = run_command(exec); assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text); + // Save the CLI stdout / stderr. + // + // The CLI intentionally doesn't bail if any of the steps fail. Record the + // output in a file in the tempdir, so that we can see any errors we happen + // to catch. + let stdout_path1 = tmpdir_path.join("reconfigurator-cli-script1.stdout"); + let stderr_path1 = tmpdir_path.join("reconfigurator-cli-script1.stderr"); + for (output, path) in + [(&stdout_text, &stdout_path1), (&stdout_text, &stderr_path1)] + { + println!("writing reconfigurator-cli script1 output to {path}"); + std::fs::write(&path, &output) + .with_context(|| format!("write CLI output to {}", path)) + .unwrap(); + } + // Load the new file and find the new blueprint name. let state2: UnstableReconfiguratorState = read_json(&saved_state2_path).unwrap(); @@ -197,6 +214,18 @@ async fn test_blueprint_edit(cptestctx: &ControlPlaneTestContext) { let (exit_status, _, stderr_text) = run_command(exec); assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text); + // Save the output again. + let stdout_path2 = tmpdir_path.join("reconfigurator-cli-script2.stdout"); + let stderr_path2 = tmpdir_path.join("reconfigurator-cli-script2.stderr"); + for (output, path) in + [(&stdout_text, &stdout_path2), (&stdout_text, &stderr_path2)] + { + println!("writing reconfigurator-cli script2 output to {path}"); + std::fs::write(&path, &output) + .with_context(|| format!("write CLI output to {}", path)) + .unwrap(); + } + // Load the blueprint we just wrote. let new_blueprint2: Blueprint = read_json(&new_blueprint_path).unwrap(); assert_eq!(new_blueprint, new_blueprint2); @@ -245,6 +274,10 @@ async fn test_blueprint_edit(cptestctx: &ControlPlaneTestContext) { script1_path, script2_path, new_blueprint_path, + stdout_path1, + stderr_path1, + stdout_path2, + stderr_path2, ] { std::fs::remove_file(&path) .with_context(|| format!("remove {}", path)) diff --git a/nexus/reconfigurator/execution/src/database.rs b/nexus/reconfigurator/execution/src/database.rs index 075ecb7af33..ab67029a665 100644 --- a/nexus/reconfigurator/execution/src/database.rs +++ b/nexus/reconfigurator/execution/src/database.rs @@ -89,6 +89,7 @@ mod test { use omicron_common::api::external::Generation; use omicron_common::api::external::MacAddr; use omicron_common::api::external::Vni; + use omicron_common::api::internal::shared::PrivateIpConfig; use omicron_common::zpool_name::ZpoolName; use omicron_test_utils::dev; use omicron_uuid_kinds::BlueprintUuid; @@ -110,6 +111,11 @@ mod test { let blueprint_id = BlueprintUuid::new_v4(); let sled_id = SledUuid::new_v4(); + let ip_config = PrivateIpConfig::new_ipv4( + "192.168.1.1".parse().unwrap(), + "192.168.1.0/24".parse().unwrap(), + ) + .unwrap(); let zones: IdOrdMap = nexus_zones .into_iter() .map(|(zone_id, disposition, nexus_generation)| BlueprintZoneConfig { @@ -131,15 +137,11 @@ mod test { id: zone_id.into_untyped_uuid(), }, name: "test-nic".parse().unwrap(), - ip: "192.168.1.1".parse().unwrap(), + ip_config: ip_config.clone(), mac: MacAddr::random_system(), - subnet: ipnetwork::IpNetwork::V4( - "192.168.1.0/24".parse().unwrap() - ).into(), vni: Vni::try_from(100).unwrap(), primary: true, slot: 0, - transit_ips: Vec::new(), }, nexus_generation, }), diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 50b653abfd0..fc24a9ec0c0 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -1396,23 +1396,17 @@ impl<'a> BlueprintBuilder<'a> { external_ip: ExternalNetworkingChoice, ) -> Result<(), Error> { let id = self.rng.sled_rng(sled_id).next_zone(); - let ExternalNetworkingChoice { - external_ip, - nic_ip, - nic_subnet, - nic_mac, - } = external_ip; + let ExternalNetworkingChoice { external_ip, nic_ip_config, nic_mac } = + external_ip; let nic = NetworkInterface { id: self.rng.sled_rng(sled_id).next_network_interface(), kind: NetworkInterfaceKind::Service { id: id.into_untyped_uuid() }, name: format!("external-dns-{id}").parse().unwrap(), - ip: nic_ip, + ip_config: nic_ip_config, mac: nic_mac, - subnet: nic_subnet, vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }; let underlay_address = self.sled_alloc_ip(sled_id)?; @@ -1599,12 +1593,8 @@ impl<'a> BlueprintBuilder<'a> { nexus_generation: Generation, ) -> Result<(), Error> { let nexus_id = self.rng.sled_rng(sled_id).next_zone(); - let ExternalNetworkingChoice { - external_ip, - nic_ip, - nic_subnet, - nic_mac, - } = external_ip; + let ExternalNetworkingChoice { external_ip, nic_ip_config, nic_mac } = + external_ip; let external_ip = OmicronZoneExternalFloatingIp { id: self.rng.sled_rng(sled_id).next_external_ip(), ip: external_ip, @@ -1616,13 +1606,11 @@ impl<'a> BlueprintBuilder<'a> { id: nexus_id.into_untyped_uuid(), }, name: format!("nexus-{nexus_id}").parse().unwrap(), - ip: nic_ip, + ip_config: nic_ip_config, mac: nic_mac, - subnet: nic_subnet, vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }; let ip = self.sled_alloc_ip(sled_id)?; @@ -1904,12 +1892,8 @@ impl<'a> BlueprintBuilder<'a> { // Add the new boundary NTP zone. let new_zone_id = self.rng.sled_rng(sled_id).next_zone(); - let ExternalSnatNetworkingChoice { - snat_cfg, - nic_ip, - nic_subnet, - nic_mac, - } = external_ip; + let ExternalSnatNetworkingChoice { snat_cfg, nic_ip_config, nic_mac } = + external_ip; let external_ip = OmicronZoneExternalSnatIp { id: self.rng.sled_rng(sled_id).next_external_ip(), snat_cfg, @@ -1920,13 +1904,11 @@ impl<'a> BlueprintBuilder<'a> { id: new_zone_id.into_untyped_uuid(), }, name: format!("ntp-{new_zone_id}").parse().unwrap(), - ip: nic_ip, + ip_config: nic_ip_config, mac: nic_mac, - subnet: nic_subnet, vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }; let underlay_ip = self.sled_alloc_ip(sled_id)?; diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs b/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs index 53703f6a7ab..4ffad16b5c9 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs @@ -20,8 +20,9 @@ use omicron_common::address::NTP_OPTE_IPV4_SUBNET; use omicron_common::address::NTP_OPTE_IPV6_SUBNET; use omicron_common::address::NUM_SOURCE_NAT_PORTS; use omicron_common::api::external::MacAddr; +use omicron_common::api::internal::shared::PrivateIpConfig; +use omicron_common::api::internal::shared::PrivateIpConfigError; use omicron_common::api::internal::shared::SourceNatConfigError; -use oxnet::IpNet; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashSet; @@ -46,6 +47,8 @@ pub enum ExternalNetworkingError { ExhaustedOpteIps { kind: ZoneKind }, #[error("attempted to add duplicate external DNS IP: {ip}")] AddDuplicateExternalDnsIp { ip: IpAddr }, + #[error("invalid private IP configuration")] + InvalidPrivateIpConfig(#[from] PrivateIpConfigError), } #[derive(Debug)] @@ -152,30 +155,30 @@ impl ExternalNetworkingAllocator { for z in running_omicron_zones { let zone_type = &z.zone_type; match zone_type { - BlueprintZoneType::BoundaryNtp(ntp) => match ntp.nic.ip { - IpAddr::V4(ip) => { - if !existing_boundary_ntp_v4_ips.insert(ip) { + BlueprintZoneType::BoundaryNtp(ntp) => { + if let Some(ip) = ntp.nic.ip_config.ipv4_addr() { + if !existing_boundary_ntp_v4_ips.insert(*ip) { bail!("duplicate Boundary NTP NIC IP: {ip}"); } } - IpAddr::V6(ip) => { - if !existing_boundary_ntp_v6_ips.insert(ip) { + if let Some(ip) = ntp.nic.ip_config.ipv6_addr() { + if !existing_boundary_ntp_v6_ips.insert(*ip) { bail!("duplicate Boundary NTP NIC IP: {ip}"); } } - }, - BlueprintZoneType::Nexus(nexus) => match nexus.nic.ip { - IpAddr::V4(ip) => { - if !existing_nexus_v4_ips.insert(ip) { + } + BlueprintZoneType::Nexus(nexus) => { + if let Some(ip) = nexus.nic.ip_config.ipv4_addr() { + if !existing_nexus_v4_ips.insert(*ip) { bail!("duplicate Nexus NIC IP: {ip}"); } } - IpAddr::V6(ip) => { - if !existing_nexus_v6_ips.insert(ip) { + if let Some(ip) = nexus.nic.ip_config.ipv6_addr() { + if !existing_nexus_v6_ips.insert(*ip) { bail!("duplicate Nexus NIC IP: {ip}"); } } - }, + } BlueprintZoneType::ExternalDns(dns) => { if !used_external_dns_ips.insert(dns.dns_address.addr.ip()) { @@ -184,16 +187,14 @@ impl ExternalNetworkingAllocator { dns.dns_address.addr ); } - match dns.nic.ip { - IpAddr::V4(ip) => { - if !existing_external_dns_v4_ips.insert(ip) { - bail!("duplicate external DNS IP: {ip}"); - } + if let Some(ip) = dns.nic.ip_config.ipv4_addr() { + if !existing_external_dns_v4_ips.insert(*ip) { + bail!("duplicate external DNS IP: {ip}"); } - IpAddr::V6(ip) => { - if !existing_external_dns_v6_ips.insert(ip) { - bail!("duplicate external DNS IP: {ip}"); - } + } + if let Some(ip) = dns.nic.ip_config.ipv6_addr() { + if !existing_external_dns_v6_ips.insert(*ip) { + bail!("duplicate external DNS IP: {ip}"); } } } @@ -301,132 +302,116 @@ impl ExternalNetworkingAllocator { pub fn for_new_nexus( &mut self, ) -> Result { + // TODO-completeness: Support dual-stack external networking for + // services. See https://github.com/oxidecomputer/omicron/issues/8949 + // and https://github.com/oxidecomputer/omicron/issues/9288. let external_ip = self.external_ip_alloc.claim_next_exclusive_ip()?; - let (nic_ip, nic_subnet) = match external_ip { - IpAddr::V4(_) => ( - self.nexus_v4_ips - .next() - .ok_or(ExternalNetworkingError::ExhaustedOpteIps { + let nic_ip_config = match external_ip { + IpAddr::V4(_) => { + let ip = self.nexus_v4_ips.next().ok_or( + ExternalNetworkingError::ExhaustedOpteIps { kind: ZoneKind::Nexus, - })? - .into(), - IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), - ), - IpAddr::V6(_) => ( - self.nexus_v6_ips - .next() - .ok_or(ExternalNetworkingError::ExhaustedOpteIps { + }, + )?; + PrivateIpConfig::new_ipv4(ip, *NEXUS_OPTE_IPV4_SUBNET)? + } + IpAddr::V6(_) => { + let ip = self.nexus_v6_ips.next().ok_or( + ExternalNetworkingError::ExhaustedOpteIps { kind: ZoneKind::Nexus, - })? - .into(), - IpNet::from(*NEXUS_OPTE_IPV6_SUBNET), - ), + }, + )?; + PrivateIpConfig::new_ipv6(ip, *NEXUS_OPTE_IPV6_SUBNET)? + } }; let nic_mac = self .available_system_macs .next() .ok_or(ExternalNetworkingError::NoSystemMacAddressAvailable)?; - Ok(ExternalNetworkingChoice { - external_ip, - nic_ip, - nic_subnet, - nic_mac, - }) + Ok(ExternalNetworkingChoice { external_ip, nic_ip_config, nic_mac }) } pub fn for_new_boundary_ntp( &mut self, ) -> Result { + // TODO-completeness: Support dual-stack external networking for + // services. See https://github.com/oxidecomputer/omicron/issues/8949. let snat_cfg = self.external_ip_alloc.claim_next_snat_ip()?; - let (nic_ip, nic_subnet) = match snat_cfg.ip { - IpAddr::V4(_) => ( - self.boundary_ntp_v4_ips - .next() - .ok_or(ExternalNetworkingError::ExhaustedOpteIps { + let nic_ip_config = match snat_cfg.ip { + IpAddr::V4(_) => { + let ip = self.boundary_ntp_v4_ips.next().ok_or( + ExternalNetworkingError::ExhaustedOpteIps { kind: ZoneKind::BoundaryNtp, - })? - .into(), - IpNet::from(*NTP_OPTE_IPV4_SUBNET), - ), - IpAddr::V6(_) => ( - self.boundary_ntp_v6_ips - .next() - .ok_or(ExternalNetworkingError::ExhaustedOpteIps { + }, + )?; + PrivateIpConfig::new_ipv4(ip, *NTP_OPTE_IPV4_SUBNET)? + } + IpAddr::V6(_) => { + let ip = self.boundary_ntp_v6_ips.next().ok_or( + ExternalNetworkingError::ExhaustedOpteIps { kind: ZoneKind::BoundaryNtp, - })? - .into(), - IpNet::from(*NTP_OPTE_IPV6_SUBNET), - ), + }, + )?; + PrivateIpConfig::new_ipv6(ip, *NTP_OPTE_IPV6_SUBNET)? + } }; let nic_mac = self .available_system_macs .next() .ok_or(ExternalNetworkingError::NoSystemMacAddressAvailable)?; - Ok(ExternalSnatNetworkingChoice { - snat_cfg, - nic_ip, - nic_subnet, - nic_mac, - }) + Ok(ExternalSnatNetworkingChoice { snat_cfg, nic_ip_config, nic_mac }) } pub fn for_new_external_dns( &mut self, ) -> Result { + // TODO-completeness: Support dual-stack external networking for + // services. See https://github.com/oxidecomputer/omicron/issues/8949. let external_ip = self .available_external_dns_ips .pop_first() .ok_or(ExternalNetworkingError::NoExternalDnsIpAvailable)?; - let (nic_ip, nic_subnet) = match external_ip { - IpAddr::V4(_) => ( - self.external_dns_v4_ips - .next() - .ok_or(ExternalNetworkingError::ExhaustedOpteIps { + let nic_ip_config = match external_ip { + IpAddr::V4(_) => { + let ip = self.external_dns_v4_ips.next().ok_or( + ExternalNetworkingError::ExhaustedOpteIps { kind: ZoneKind::ExternalDns, - })? - .into(), - IpNet::from(*DNS_OPTE_IPV4_SUBNET), - ), - IpAddr::V6(_) => ( - self.external_dns_v6_ips - .next() - .ok_or(ExternalNetworkingError::ExhaustedOpteIps { + }, + )?; + PrivateIpConfig::new_ipv4(ip, *DNS_OPTE_IPV4_SUBNET)? + } + IpAddr::V6(_) => { + let ip = self.external_dns_v6_ips.next().ok_or( + ExternalNetworkingError::ExhaustedOpteIps { kind: ZoneKind::ExternalDns, - })? - .into(), - IpNet::from(*DNS_OPTE_IPV6_SUBNET), - ), + }, + )?; + PrivateIpConfig::new_ipv6(ip, *DNS_OPTE_IPV6_SUBNET)? + } }; let nic_mac = self .available_system_macs .next() .ok_or(ExternalNetworkingError::NoSystemMacAddressAvailable)?; - Ok(ExternalNetworkingChoice { - external_ip, - nic_ip, - nic_subnet, - nic_mac, - }) + Ok(ExternalNetworkingChoice { external_ip, nic_ip_config, nic_mac }) } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct ExternalNetworkingChoice { pub external_ip: IpAddr, - pub nic_ip: IpAddr, - pub nic_subnet: IpNet, + pub nic_ip_config: PrivateIpConfig, pub nic_mac: MacAddr, } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct ExternalSnatNetworkingChoice { pub snat_cfg: SourceNatConfig, - pub nic_ip: IpAddr, - pub nic_subnet: IpNet, + pub nic_ip_config: PrivateIpConfig, pub nic_mac: MacAddr, } @@ -913,6 +898,12 @@ pub mod test { let make_external_dns = |index, disposition| { let id = OmicronZoneUuid::new_v4(); let pool_name = ZpoolName::new_external(ZpoolUuid::new_v4()); + let ip_addr = DNS_OPTE_IPV4_SUBNET + .addr_iter() + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + index) + .unwrap(); + let ip = PrivateIpConfig::new_ipv4(ip_addr, *DNS_OPTE_IPV4_SUBNET) + .unwrap(); BlueprintZoneConfig { disposition, id, @@ -938,17 +929,11 @@ pub mod test { id: id.into_untyped_uuid(), }, name: format!("test-{index}").parse().unwrap(), - ip: DNS_OPTE_IPV4_SUBNET - .addr_iter() - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + index) - .unwrap() - .into(), + ip_config: ip, mac: MacAddr::iter_system().nth(index).unwrap(), - subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: Vec::new(), }, }, ), diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index c2291538cb8..562965926f6 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -749,6 +749,24 @@ impl ExampleSystemBuilder { input_builder .add_omicron_zone_external_ip(service_id, external_ip) .expect("failed to add Omicron zone external IP"); + // TODO-completess: Support dual-stack Omicron zone NICs. + // See https://github.com/oxidecomputer/omicron/issues/9314 + assert!( + !nic.ip_config.is_dual_stack(), + "Dual-stack OmicronZoneNics are not yet supported" + ); + let ip = nic + .ip_config + .ipv4_addr() + .copied() + .map(IpAddr::V4) + .unwrap_or_else(|| { + nic.ip_config + .ipv6_addr() + .copied() + .map(IpAddr::V6) + .expect("must have at least one IP address") + }); input_builder .add_omicron_zone_nic( service_id, @@ -756,7 +774,7 @@ impl ExampleSystemBuilder { // TODO-cleanup use `TypedUuid` everywhere id: VnicUuid::from_untyped_uuid(nic.id), mac: nic.mac, - ip: nic.ip, + ip, slot: nic.slot, primary: nic.primary, }, diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index f8528f3d45f..19519f08747 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -2607,6 +2607,7 @@ pub(crate) mod test { use omicron_common::api::external::Vni; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; + use omicron_common::api::internal::shared::PrivateIpConfig; use omicron_common::api::internal::shared::SourceNatConfig; use omicron_common::disk::DatasetKind; use omicron_common::disk::DiskIdentity; @@ -2620,6 +2621,7 @@ pub(crate) mod test { use omicron_uuid_kinds::ExternalIpUuid; use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::ZpoolUuid; + use oxnet::Ipv6Net; use semver::Version; use slog_error_chain::InlineErrorChain; use std::collections::BTreeMap; @@ -7101,6 +7103,11 @@ pub(crate) mod test { ) => address, _ => panic!("should be internal NTP?"), }; + let ip_config = PrivateIpConfig::new_ipv6( + Ipv6Addr::LOCALHOST, + Ipv6Net::new(Ipv6Addr::LOCALHOST, 64).unwrap(), + ) + .unwrap(); // The contents here are all lies, but it's just stored // as plain-old-data for the purposes of this test, so @@ -7117,17 +7124,11 @@ pub(crate) mod test { id: Uuid::new_v4(), }, name: "ntp-0".parse().unwrap(), - ip: IpAddr::V6(Ipv6Addr::LOCALHOST), + ip_config, mac: MacAddr::random_system(), - subnet: oxnet::IpNet::new( - IpAddr::V6(Ipv6Addr::LOCALHOST), - 8, - ) - .unwrap(), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt index 6d57765149f..51308402ef6 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt @@ -352,7 +352,16 @@ mismatched zone type: after: Nexus( name: Name( "nexus-5a8e9719-62bd-40be-81b1-20b85970740b", ), - ip: 172.30.2.5, + ip: V4( + PrivateIpv4Config { + ip: 172.30.2.5, + subnet: Ipv4Net { + addr: 172.30.2.0, + width: 24, + }, + transit_ips: [], + }, + ), mac: MacAddr( MacAddr6( [ @@ -365,18 +374,11 @@ mismatched zone type: after: Nexus( ], ), ), - subnet: V4( - Ipv4Net { - addr: 172.30.2.0, - width: 24, - }, - ), vni: Vni( 100, ), primary: true, slot: 0, - transit_ips: [], }, external_tls: false, external_dns_servers: [], diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 6fb1b59cb1d..d023d4a327c 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -66,6 +66,7 @@ use nexus_types::deployment::blueprint_zone_type; use nexus_types::external_api::views::SledState; use nexus_types::internal_api::params::DnsConfigParams; use omicron_common::address::DNS_OPTE_IPV4_SUBNET; +use omicron_common::address::DNS_OPTE_IPV6_SUBNET; use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; use omicron_common::address::NTP_OPTE_IPV4_SUBNET; use omicron_common::address::NTP_PORT; @@ -80,6 +81,7 @@ use omicron_common::api::internal::nexus::ProducerKind; use omicron_common::api::internal::shared::DatasetKind; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; +use omicron_common::api::internal::shared::PrivateIpConfig; use omicron_common::api::internal::shared::SourceNatConfig; use omicron_common::api::internal::shared::SwitchLocation; use omicron_common::disk::CompressionAlgorithm; @@ -965,6 +967,13 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { .mac_addrs .next() .expect("ran out of MAC addresses"); + let ip_config = PrivateIpConfig::new_ipv4( + NEXUS_OPTE_IPV4_SUBNET + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1 + which) + .unwrap(), + *NEXUS_OPTE_IPV4_SUBNET, + ) + .unwrap(); self.blueprint_zones.push(BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, id, @@ -988,20 +997,15 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { lockstep_port, nic: NetworkInterface { id: Uuid::new_v4(), - ip: NEXUS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1 + which) - .unwrap() - .into(), kind: NetworkInterfaceKind::Service { id: id.into_untyped_uuid(), }, mac, name: format!("nexus-{}", id).parse().unwrap(), + ip_config, primary: true, slot: 0, - subnet: (*NEXUS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, - transit_ips: vec![], }, nexus_generation: Generation::new(), }), @@ -1340,6 +1344,9 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { let internal_ip = NTP_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) .unwrap(); + let private_ip_config = + PrivateIpConfig::new_ipv4(internal_ip, *NTP_OPTE_IPV4_SUBNET) + .unwrap(); let external_ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)); let address = format!("[::1]:{NTP_PORT}").parse().unwrap(); // localhost let zone_id = OmicronZoneUuid::new_v4(); @@ -1365,16 +1372,14 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { kind: NetworkInterfaceKind::Service { id: zone_id.into_untyped_uuid(), }, - ip: internal_ip.into(), + ip_config: private_ip_config, mac, name: format!("boundary-ntp-{zone_id}") .parse() .unwrap(), primary: true, slot: 0, - subnet: (*NTP_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, - transit_ips: vec![], }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), @@ -1444,6 +1449,24 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { .to_string() .parse() .unwrap(); + + let ip_config = if dns.dns_server.local_address().is_ipv4() { + PrivateIpConfig::new_ipv4( + DNS_OPTE_IPV4_SUBNET + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) + .unwrap(), + *DNS_OPTE_IPV4_SUBNET, + ) + .unwrap() + } else { + PrivateIpConfig::new_ipv6( + DNS_OPTE_IPV6_SUBNET + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES as u128 + 1) + .unwrap(), + *DNS_OPTE_IPV6_SUBNET, + ) + .unwrap() + }; self.blueprint_zones.push(BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, id: zone_id, @@ -1458,10 +1481,6 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { http_address: dropshot_address, nic: NetworkInterface { id: Uuid::new_v4(), - ip: DNS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) - .unwrap() - .into(), kind: NetworkInterfaceKind::Service { id: zone_id.into_untyped_uuid(), }, @@ -1469,11 +1488,10 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { name: format!("external-dns-{}", zone_id) .parse() .unwrap(), + ip_config, primary: true, slot: 0, - subnet: (*DNS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, - transit_ips: vec![], }, }, ), diff --git a/nexus/tests/integration_tests/vpc_routers.rs b/nexus/tests/integration_tests/vpc_routers.rs index ce72e605c56..eb16bfcf6d8 100644 --- a/nexus/tests/integration_tests/vpc_routers.rs +++ b/nexus/tests/integration_tests/vpc_routers.rs @@ -640,10 +640,19 @@ async fn test_vpc_routers_custom_delivered_to_instance( .await; assert_eq!(last_routes[0].0, new_system); - assert!(new_custom.contains(&ResolvedVpcRoute { - dest: "2.0.7.0/24".parse().unwrap(), - target: RouterTarget::Ip(instance_nics[INSTANCE_NAMES[1]][0].ip), - })); + assert!( + new_custom.contains(&ResolvedVpcRoute { + dest: "2.0.7.0/24".parse().unwrap(), + target: RouterTarget::Ip( + instance_nics[INSTANCE_NAMES[1]][0] + .ip_config + .ipv4_addr() + .copied() + .unwrap() + .into() + ), + }) + ); // Swapping router should change the installed routes at that sled. set_custom_router( diff --git a/nexus/types/src/deployment/network_resources.rs b/nexus/types/src/deployment/network_resources.rs index d1c37089943..7dbfe208a91 100644 --- a/nexus/types/src/deployment/network_resources.rs +++ b/nexus/types/src/deployment/network_resources.rs @@ -298,6 +298,8 @@ pub struct OmicronZoneExternalSnatIp { pub struct OmicronZoneNic { pub id: VnicUuid, pub mac: MacAddr, + // TODO-completeness: Support dual-stack NICs for Omicron zones. See + // https://github.com/oxidecomputer/omicron/issues/9314. pub ip: IpAddr, pub slot: u8, pub primary: bool, diff --git a/nexus/types/src/external_api/shared.rs b/nexus/types/src/external_api/shared.rs index 42a64b72c46..eb0752c4062 100644 --- a/nexus/types/src/external_api/shared.rs +++ b/nexus/types/src/external_api/shared.rs @@ -11,7 +11,7 @@ use anyhow::Context; use chrono::DateTime; use chrono::Utc; use omicron_common::api::external::Name; -use omicron_common::api::internal::shared::NetworkInterface; +use omicron_common::api::internal::shared::network_interface::v1::NetworkInterface as NetworkInterfaceV1; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::SiloGroupUuid; use omicron_uuid_kinds::SiloUserUuid; @@ -698,7 +698,12 @@ pub struct ProbeInfo { #[schemars(with = "Uuid")] pub sled: SledUuid, pub external_ips: Vec, - pub interface: NetworkInterface, + // NOTE: This type currently appears in both the external and internal APIs. + // It's not used in the internal API anymore, and we've not yet expanded the + // external API to support dual-stack NICs. When we do, this whole type + // needs a new version the external API, and the internal API needs to + // continue to refer to this original version. + pub interface: NetworkInterfaceV1, } #[derive(Debug, Clone, JsonSchema, Serialize, Deserialize)] diff --git a/openapi/nexus-lockstep.json b/openapi/nexus-lockstep.json index 70879988e1b..b15f091c36a 100644 --- a/openapi/nexus-lockstep.json +++ b/openapi/nexus-lockstep.json @@ -5608,8 +5608,7 @@ "format": "uuid" }, "ip": { - "type": "string", - "format": "ip" + "$ref": "#/components/schemas/PrivateIpConfig" }, "kind": { "$ref": "#/components/schemas/NetworkInterfaceKind" @@ -5628,16 +5627,6 @@ "format": "uint8", "minimum": 0 }, - "subnet": { - "$ref": "#/components/schemas/IpNet" - }, - "transit_ips": { - "default": [], - "type": "array", - "items": { - "$ref": "#/components/schemas/IpNet" - } - }, "vni": { "$ref": "#/components/schemas/Vni" } @@ -5650,7 +5639,6 @@ "name", "primary", "slot", - "subnet", "vni" ] }, @@ -7387,6 +7375,134 @@ "speed400_g" ] }, + "PrivateIpConfig": { + "description": "VPC-private IP address configuration for a network interface.", + "oneOf": [ + { + "description": "The interface has only an IPv4 configuration.", + "type": "object", + "properties": { + "v4": { + "$ref": "#/components/schemas/PrivateIpv4Config" + } + }, + "required": [ + "v4" + ], + "additionalProperties": false + }, + { + "description": "The interface has only an IPv6 configuration.", + "type": "object", + "properties": { + "v6": { + "$ref": "#/components/schemas/PrivateIpv6Config" + } + }, + "required": [ + "v6" + ], + "additionalProperties": false + }, + { + "description": "The interface is dual-stack.", + "type": "object", + "properties": { + "dual_stack": { + "type": "object", + "properties": { + "v4": { + "description": "The interface's IPv4 configuration.", + "allOf": [ + { + "$ref": "#/components/schemas/PrivateIpv4Config" + } + ] + }, + "v6": { + "description": "The interface's IPv6 configuration.", + "allOf": [ + { + "$ref": "#/components/schemas/PrivateIpv6Config" + } + ] + } + }, + "required": [ + "v4", + "v6" + ] + } + }, + "required": [ + "dual_stack" + ], + "additionalProperties": false + } + ] + }, + "PrivateIpv4Config": { + "description": "VPC-private IPv4 configuration for a network interface.", + "type": "object", + "properties": { + "ip": { + "description": "VPC-private IP address.", + "type": "string", + "format": "ipv4" + }, + "subnet": { + "description": "The IP subnet.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Net" + } + ] + }, + "transit_ips": { + "description": "Additional networks on which the interface can send / receive traffic.", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Net" + } + } + }, + "required": [ + "ip", + "subnet" + ] + }, + "PrivateIpv6Config": { + "description": "VPC-private IPv6 configuration for a network interface.", + "type": "object", + "properties": { + "ip": { + "description": "VPC-private IP address.", + "type": "string", + "format": "ipv6" + }, + "subnet": { + "description": "The IP subnet.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Net" + } + ] + }, + "transit_ips": { + "description": "Additional networks on which the interface can send / receive traffic.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Net" + } + } + }, + "required": [ + "ip", + "subnet", + "transit_ips" + ] + }, "QuiesceState": { "description": "See [`QuiesceStatus`] for more on Nexus quiescing.\n\nAt any given time, Nexus is always in one of these states:\n\n```text Undetermined (have not loaded persistent state; don't know yet) | | load persistent state and find we're not quiescing v Running (normal operation) | | quiesce starts v DrainingSagas (no new sagas are allowed, but some are still running) | | no more sagas running v DrainingDb (no sagas running; no new db connections may be | acquired by Nexus at-large, but some are still held) | | no more database connections held v RecordingQuiesce (everything is quiesced aside from one connection being | used to record our final quiesced state) | | finish recording quiesce state in database v Quiesced (no sagas running, no database connections in use) ```\n\nQuiescing is (currently) a one-way trip: once a Nexus process starts quiescing, it will never go back to normal operation. It will never go back to an earlier stage, either.", "oneOf": [ diff --git a/openapi/sled-agent/sled-agent-7.0.0-90da02.json b/openapi/sled-agent/sled-agent-7.0.0-90da02.json new file mode 100644 index 00000000000..94496170835 --- /dev/null +++ b/openapi/sled-agent/sled-agent-7.0.0-90da02.json @@ -0,0 +1,8699 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Oxide Sled Agent API", + "description": "API for interacting with individual sleds", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "7.0.0" + }, + "paths": { + "/artifacts": { + "get": { + "operationId": "artifact_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactListResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/artifacts/{sha256}": { + "put": { + "operationId": "artifact_put", + "parameters": [ + { + "in": "path", + "name": "sha256", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + }, + { + "in": "query", + "name": "generation", + "required": true, + "schema": { + "$ref": "#/components/schemas/Generation" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactPutResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/artifacts/{sha256}/copy-from-depot": { + "post": { + "operationId": "artifact_copy_from_depot", + "parameters": [ + { + "in": "path", + "name": "sha256", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + }, + { + "in": "query", + "name": "generation", + "required": true, + "schema": { + "$ref": "#/components/schemas/Generation" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactCopyFromDepotBody" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "successfully enqueued operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactCopyFromDepotResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/artifacts-config": { + "get": { + "operationId": "artifact_config_get", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "artifact_config_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/bootstore/status": { + "get": { + "summary": "Get the internal state of the local bootstore node", + "operationId": "bootstore_status", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BootstoreStatus" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/debug/switch-zone-policy": { + "get": { + "summary": "A debugging endpoint only used by `omdb` that allows us to test", + "description": "restarting the switch zone without restarting sled-agent. See for context.", + "operationId": "debug_operator_switch_zone_policy_get", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorSwitchZonePolicy" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "A debugging endpoint only used by `omdb` that allows us to test", + "description": "restarting the switch zone without restarting sled-agent. See for context.\n\nSetting the switch zone policy is asynchronous and inherently racy with the standard process of starting the switch zone. If the switch zone is in the process of being started or stopped when this policy is changed, the new policy may not take effect until that transition completes.", + "operationId": "debug_operator_switch_zone_policy_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorSwitchZonePolicy" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/disks/{disk_id}": { + "put": { + "operationId": "disk_put", + "parameters": [ + { + "in": "path", + "name": "disk_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiskEnsureBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiskRuntimeState" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/eip-gateways": { + "put": { + "summary": "Update per-NIC IP address <-> internet gateway mappings.", + "operationId": "set_eip_gateways", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExternalIpGatewayMap" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/inventory": { + "get": { + "summary": "Fetch basic information about this sled", + "operationId": "inventory", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Inventory" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/network-bootstore-config": { + "get": { + "summary": "This API endpoint is only reading the local sled agent's view of the", + "description": "bootstore. The boostore is a distributed data store that is eventually consistent. Reads from individual nodes may not represent the latest state.", + "operationId": "read_network_bootstore_config_cache", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "write_network_bootstore_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/omicron-config": { + "put": { + "operationId": "omicron_config_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OmicronSledConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/probes": { + "put": { + "summary": "Update the entire set of probe zones on this sled.", + "description": "Probe zones are used to debug networking configuration. They look similar to instances, in that they have an OPTE port on a VPC subnet and external addresses, but no actual VM.", + "operationId": "probes_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProbeSet" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/sled-identifiers": { + "get": { + "summary": "Fetch sled identifiers", + "operationId": "sled_identifiers", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledIdentifiers" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/sled-role": { + "get": { + "operationId": "sled_role_get", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledRole" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/sleds": { + "put": { + "summary": "Add a sled to a rack that was already initialized via RSS", + "operationId": "sled_add", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddSledRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/dladm-info": { + "get": { + "operationId": "support_dladm_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/health-check": { + "get": { + "operationId": "support_health_check", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/ipadm-info": { + "get": { + "operationId": "support_ipadm_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/logs/download/{zone}": { + "get": { + "summary": "This endpoint returns a zip file of a zone's logs organized by service.", + "operationId": "support_logs_download", + "parameters": [ + { + "in": "path", + "name": "zone", + "description": "The zone for which one would like to collect logs for", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "max_rotated", + "description": "The max number of rotated logs to include in the final support bundle", + "required": true, + "schema": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + } + }, + "/support/logs/zones": { + "get": { + "summary": "This endpoint returns a list of known zones on a sled that have service", + "description": "logs that can be collected into a support bundle.", + "operationId": "support_logs", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_String", + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/nvmeadm-info": { + "get": { + "operationId": "support_nvmeadm_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/pargs-info": { + "get": { + "operationId": "support_pargs_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/pfiles-info": { + "get": { + "operationId": "support_pfiles_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/pstack-info": { + "get": { + "operationId": "support_pstack_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SledDiagnosticsQueryOutput", + "type": "array", + "items": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/zfs-info": { + "get": { + "operationId": "support_zfs_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/zoneadm-info": { + "get": { + "operationId": "support_zoneadm_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support/zpool-info": { + "get": { + "operationId": "support_zpool_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledDiagnosticsQueryOutput" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}": { + "get": { + "summary": "List all support bundles within a particular dataset", + "operationId": "support_bundle_list", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SupportBundleMetadata", + "type": "array", + "items": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}": { + "post": { + "summary": "Starts creation of a support bundle within a particular dataset", + "description": "Callers should transfer chunks of the bundle with \"support_bundle_transfer\", and then call \"support_bundle_finalize\" once the bundle has finished transferring.\n\nIf a support bundle was previously created without being finalized successfully, this endpoint will reset the state.\n\nIf a support bundle was previously created and finalized successfully, this endpoint will return metadata indicating that it already exists.", + "operationId": "support_bundle_start_creation", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a support bundle from a particular dataset", + "operationId": "support_bundle_delete", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/download": { + "get": { + "summary": "Fetch a support bundle from a particular dataset", + "operationId": "support_bundle_download", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + }, + "head": { + "summary": "Fetch metadata about a support bundle from a particular dataset", + "operationId": "support_bundle_head", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/download/{file}": { + "get": { + "summary": "Fetch a file within a support bundle from a particular dataset", + "operationId": "support_bundle_download_file", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "file", + "description": "The path of the file within the support bundle to query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + }, + "head": { + "summary": "Fetch metadata about a file within a support bundle from a particular dataset", + "operationId": "support_bundle_head_file", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "file", + "description": "The path of the file within the support bundle to query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/finalize": { + "post": { + "summary": "Finalizes the creation of a support bundle", + "description": "If the requested hash matched the bundle, the bundle is created. Otherwise, an error is returned.", + "operationId": "support_bundle_finalize", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + }, + { + "in": "query", + "name": "hash", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/index": { + "get": { + "summary": "Fetch the index (list of files within a support bundle)", + "operationId": "support_bundle_index", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + }, + "head": { + "summary": "Fetch metadata about the list of files within a support bundle", + "operationId": "support_bundle_head_index", + "parameters": [ + { + "in": "header", + "name": "range", + "description": "A request to access a portion of the resource, such as `bytes=0-499`\n\nSee: ", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/transfer": { + "put": { + "summary": "Transfers a chunk of a support bundle within a particular dataset", + "operationId": "support_bundle_transfer", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetUuid" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/ZpoolUuid" + } + }, + { + "in": "query", + "name": "offset", + "required": true, + "schema": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/switch-ports": { + "post": { + "operationId": "uplink_ensure", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SwitchPorts" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v2p": { + "get": { + "summary": "List v2p mappings present on sled", + "operationId": "list_v2p", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_VirtualNetworkInterfaceHost", + "type": "array", + "items": { + "$ref": "#/components/schemas/VirtualNetworkInterfaceHost" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Create a mapping from a virtual NIC to a physical host", + "operationId": "set_v2p", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VirtualNetworkInterfaceHost" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a mapping from a virtual NIC to a physical host", + "operationId": "del_v2p", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VirtualNetworkInterfaceHost" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}": { + "put": { + "operationId": "vmm_register", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceEnsureBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledVmmState" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "vmm_unregister", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmUnregisterResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/disks/{disk_id}/snapshot": { + "post": { + "summary": "Take a snapshot of a disk that is attached to an instance", + "operationId": "vmm_issue_disk_snapshot_request", + "parameters": [ + { + "in": "path", + "name": "disk_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmIssueDiskSnapshotRequestBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmIssueDiskSnapshotRequestResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/external-ip": { + "put": { + "operationId": "vmm_put_external_ip", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceExternalIpBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "vmm_delete_external_ip", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceExternalIpBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/state": { + "get": { + "operationId": "vmm_get_state", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledVmmState" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "vmm_put_state", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/PropolisUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmPutStateBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmPutStateResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vpc/{vpc_id}/firewall/rules": { + "put": { + "operationId": "vpc_firewall_rules_put", + "parameters": [ + { + "in": "path", + "name": "vpc_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcFirewallRulesEnsureBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vpc-routes": { + "get": { + "summary": "Get the current versions of VPC routing rules.", + "operationId": "list_vpc_routes", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ResolvedVpcRouteState", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResolvedVpcRouteState" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Update VPC routing rules.", + "operationId": "set_vpc_routes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Array_of_ResolvedVpcRouteSet", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResolvedVpcRouteSet" + } + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones": { + "get": { + "summary": "List the zones that are currently managed by the sled agent.", + "operationId": "zones_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_String", + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundle-cleanup": { + "post": { + "summary": "Trigger a zone bundle cleanup.", + "operationId": "zone_bundle_cleanup", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_CleanupCount", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/CleanupCount" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundle-cleanup/context": { + "get": { + "summary": "Return context used by the zone-bundle cleanup task.", + "operationId": "zone_bundle_cleanup_context", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CleanupContext" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Update context used by the zone-bundle cleanup task.", + "operationId": "zone_bundle_cleanup_context_update", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CleanupContextUpdate" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundle-cleanup/utilization": { + "get": { + "summary": "Return utilization information about all zone bundles.", + "operationId": "zone_bundle_utilization", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_BundleUtilization", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/BundleUtilization" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundles": { + "get": { + "summary": "List all zone bundles that exist, even for now-deleted zones.", + "operationId": "zone_bundle_list_all", + "parameters": [ + { + "in": "query", + "name": "filter", + "description": "An optional substring used to filter zone bundles.", + "schema": { + "nullable": true, + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ZoneBundleMetadata", + "type": "array", + "items": { + "$ref": "#/components/schemas/ZoneBundleMetadata" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundles/{zone_name}": { + "get": { + "summary": "List the zone bundles that are available for a running zone.", + "operationId": "zone_bundle_list", + "parameters": [ + { + "in": "path", + "name": "zone_name", + "description": "The name of the zone.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ZoneBundleMetadata", + "type": "array", + "items": { + "$ref": "#/components/schemas/ZoneBundleMetadata" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/zones/bundles/{zone_name}/{bundle_id}": { + "get": { + "summary": "Fetch the binary content of a single zone bundle.", + "operationId": "zone_bundle_get", + "parameters": [ + { + "in": "path", + "name": "bundle_id", + "description": "The ID for this bundle itself.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "zone_name", + "description": "The name of the zone this bundle is derived from.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a zone bundle.", + "operationId": "zone_bundle_delete", + "parameters": [ + { + "in": "path", + "name": "bundle_id", + "description": "The ID for this bundle itself.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "zone_name", + "description": "The name of the zone this bundle is derived from.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "AddSledRequest": { + "description": "A request to Add a given sled after rack initialization has occurred", + "type": "object", + "properties": { + "sled_id": { + "$ref": "#/components/schemas/BaseboardId" + }, + "start_request": { + "$ref": "#/components/schemas/StartSledAgentRequest" + } + }, + "required": [ + "sled_id", + "start_request" + ] + }, + "ArtifactConfig": { + "type": "object", + "properties": { + "artifacts": { + "type": "array", + "items": { + "type": "string", + "format": "hex string (32 bytes)" + }, + "uniqueItems": true + }, + "generation": { + "$ref": "#/components/schemas/Generation" + } + }, + "required": [ + "artifacts", + "generation" + ] + }, + "ArtifactCopyFromDepotBody": { + "type": "object", + "properties": { + "depot_base_url": { + "type": "string" + } + }, + "required": [ + "depot_base_url" + ] + }, + "ArtifactCopyFromDepotResponse": { + "type": "object" + }, + "ArtifactListResponse": { + "type": "object", + "properties": { + "generation": { + "$ref": "#/components/schemas/Generation" + }, + "list": { + "type": "object", + "additionalProperties": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + } + }, + "required": [ + "generation", + "list" + ] + }, + "ArtifactPutResponse": { + "type": "object", + "properties": { + "datasets": { + "description": "The number of valid M.2 artifact datasets we found on the sled. There is typically one of these datasets for each functional M.2.", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "successful_writes": { + "description": "The number of valid writes to the M.2 artifact datasets. This should be less than or equal to the number of artifact datasets.", + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "datasets", + "successful_writes" + ] + }, + "Baseboard": { + "description": "Describes properties that should uniquely identify a Gimlet.", + "oneOf": [ + { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "model": { + "type": "string" + }, + "revision": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "gimlet" + ] + } + }, + "required": [ + "identifier", + "model", + "revision", + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "unknown" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "model": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "pc" + ] + } + }, + "required": [ + "identifier", + "model", + "type" + ] + } + ] + }, + "BaseboardId": { + "description": "A representation of a Baseboard ID as used in the inventory subsystem This type is essentially the same as a `Baseboard` except it doesn't have a revision or HW type (Gimlet, PC, Unknown).", + "type": "object", + "properties": { + "part_number": { + "description": "Oxide Part Number", + "type": "string" + }, + "serial_number": { + "description": "Serial number (unique for a given part number)", + "type": "string" + } + }, + "required": [ + "part_number", + "serial_number" + ] + }, + "BfdMode": { + "description": "BFD connection mode.", + "type": "string", + "enum": [ + "single_hop", + "multi_hop" + ] + }, + "BfdPeerConfig": { + "type": "object", + "properties": { + "detection_threshold": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "local": { + "nullable": true, + "type": "string", + "format": "ip" + }, + "mode": { + "$ref": "#/components/schemas/BfdMode" + }, + "remote": { + "type": "string", + "format": "ip" + }, + "required_rx": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "switch": { + "$ref": "#/components/schemas/SwitchLocation" + } + }, + "required": [ + "detection_threshold", + "mode", + "remote", + "required_rx", + "switch" + ] + }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "checker": { + "nullable": true, + "description": "Checker to apply to incoming messages.", + "default": null, + "type": "string" + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Net" + } + }, + "shaper": { + "nullable": true, + "description": "Shaper to apply to outgoing messages.", + "default": null, + "type": "string" + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "allowed_export": { + "description": "Define export policy for a peer.", + "default": { + "type": "no_filtering" + }, + "allOf": [ + { + "$ref": "#/components/schemas/ImportExportPolicy" + } + ] + }, + "allowed_import": { + "description": "Define import policy for a peer.", + "default": { + "type": "no_filtering" + }, + "allOf": [ + { + "$ref": "#/components/schemas/ImportExportPolicy" + } + ] + }, + "asn": { + "description": "The autonomous system number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "communities": { + "description": "Include the provided communities in updates sent to the peer.", + "default": [], + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "connect_retry": { + "nullable": true, + "description": "The interval in seconds between peer connection retry attempts.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "delay_open": { + "nullable": true, + "description": "How long to delay sending open messages to a peer. In seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "enforce_first_as": { + "description": "Enforce that the first AS in paths received from this peer is the peer's AS.", + "default": false, + "type": "boolean" + }, + "hold_time": { + "nullable": true, + "description": "How long to keep a session alive without a keepalive in seconds. Defaults to 6.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "idle_hold_time": { + "nullable": true, + "description": "How long to keep a peer in idle after a state machine reset in seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "keepalive": { + "nullable": true, + "description": "The interval to send keepalive messages at.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "local_pref": { + "nullable": true, + "description": "Apply a local preference to routes received from this peer.", + "default": null, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "md5_auth_key": { + "nullable": true, + "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, + "type": "string" + }, + "min_ttl": { + "nullable": true, + "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "multi_exit_discriminator": { + "nullable": true, + "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + }, + "remote_asn": { + "nullable": true, + "description": "Require that a peer has a specified ASN.", + "default": null, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "vlan_id": { + "nullable": true, + "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, + "BlobStorageBackend": { + "description": "A storage backend for a disk whose initial contents are given explicitly by the specification.", + "type": "object", + "properties": { + "base64": { + "description": "The disk's initial contents, encoded as a base64 string.", + "type": "string" + }, + "readonly": { + "description": "Indicates whether the storage is read-only.", + "type": "boolean" + } + }, + "required": [ + "base64", + "readonly" + ], + "additionalProperties": false + }, + "Board": { + "description": "A VM's mainboard.", + "type": "object", + "properties": { + "chipset": { + "description": "The chipset to expose to guest software.", + "allOf": [ + { + "$ref": "#/components/schemas/Chipset" + } + ] + }, + "cpuid": { + "nullable": true, + "description": "The CPUID values to expose to the guest. If `None`, bhyve will derive default values from the host's CPUID values.", + "allOf": [ + { + "$ref": "#/components/schemas/Cpuid" + } + ] + }, + "cpus": { + "description": "The number of virtual logical processors attached to this VM.", + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "guest_hv_interface": { + "description": "The hypervisor platform to expose to the guest. The default is a bhyve-compatible interface with no additional features.\n\nFor compatibility with older versions of Propolis, this field is only serialized if it specifies a non-default interface.", + "allOf": [ + { + "$ref": "#/components/schemas/GuestHypervisorInterface" + } + ] + }, + "memory_mb": { + "description": "The amount of guest RAM attached to this VM.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "chipset", + "cpus", + "memory_mb" + ], + "additionalProperties": false + }, + "BootImageHeader": { + "type": "object", + "properties": { + "data_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "flags": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "image_name": { + "type": "string" + }, + "image_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "sha256": { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "minItems": 32, + "maxItems": 32 + }, + "target_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "data_size", + "flags", + "image_name", + "image_size", + "sha256", + "target_size" + ] + }, + "BootOrderEntry": { + "description": "An entry in the boot order stored in a [`BootSettings`] component.", + "type": "object", + "properties": { + "id": { + "description": "The ID of another component in the spec that Propolis should try to boot from.\n\nCurrently, only disk device components are supported.", + "allOf": [ + { + "$ref": "#/components/schemas/SpecKey" + } + ] + } + }, + "required": [ + "id" + ] + }, + "BootPartitionContents": { + "type": "object", + "properties": { + "boot_disk": { + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/M2Slot" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/M2Slot" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + }, + "slot_a": { + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/BootPartitionDetails" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/BootPartitionDetails" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + }, + "slot_b": { + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/BootPartitionDetails" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/BootPartitionDetails" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + } + }, + "required": [ + "boot_disk", + "slot_a", + "slot_b" + ] + }, + "BootPartitionDetails": { + "type": "object", + "properties": { + "artifact_hash": { + "type": "string", + "format": "hex string (32 bytes)" + }, + "artifact_size": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "header": { + "$ref": "#/components/schemas/BootImageHeader" + } + }, + "required": [ + "artifact_hash", + "artifact_size", + "header" + ] + }, + "BootSettings": { + "description": "Settings supplied to the guest's firmware image that specify the order in which it should consider its options when selecting a device to try to boot from.", + "type": "object", + "properties": { + "order": { + "description": "An ordered list of components to attempt to boot from.", + "type": "array", + "items": { + "$ref": "#/components/schemas/BootOrderEntry" + } + } + }, + "required": [ + "order" + ], + "additionalProperties": false + }, + "BootstoreStatus": { + "type": "object", + "properties": { + "accepted_connections": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "established_connections": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EstablishedConnection" + } + }, + "fsm_ledger_generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "fsm_state": { + "type": "string" + }, + "negotiating_connections": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "network_config_ledger_generation": { + "nullable": true, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "peers": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": [ + "accepted_connections", + "established_connections", + "fsm_ledger_generation", + "fsm_state", + "negotiating_connections", + "peers" + ] + }, + "BundleUtilization": { + "description": "The portion of a debug dataset used for zone bundles.", + "type": "object", + "properties": { + "bytes_available": { + "description": "The total number of bytes available for zone bundles.\n\nThis is `dataset_quota` multiplied by the context's storage limit.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "bytes_used": { + "description": "Total bundle usage, in bytes.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "dataset_quota": { + "description": "The total dataset quota, in bytes.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "bytes_available", + "bytes_used", + "dataset_quota" + ] + }, + "ByteCount": { + "description": "Byte count to express memory or storage capacity.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "Chipset": { + "description": "A kind of virtual chipset.", + "oneOf": [ + { + "description": "An Intel 440FX-compatible chipset.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "i440_fx" + ] + }, + "value": { + "$ref": "#/components/schemas/I440Fx" + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + }, + "CleanupContext": { + "description": "Context provided for the zone bundle cleanup task.", + "type": "object", + "properties": { + "period": { + "description": "The period on which automatic checks and cleanup is performed.", + "allOf": [ + { + "$ref": "#/components/schemas/CleanupPeriod" + } + ] + }, + "priority": { + "description": "The priority ordering for keeping old bundles.", + "allOf": [ + { + "$ref": "#/components/schemas/PriorityOrder" + } + ] + }, + "storage_limit": { + "description": "The limit on the dataset quota available for zone bundles.", + "allOf": [ + { + "$ref": "#/components/schemas/StorageLimit" + } + ] + } + }, + "required": [ + "period", + "priority", + "storage_limit" + ] + }, + "CleanupContextUpdate": { + "description": "Parameters used to update the zone bundle cleanup context.", + "type": "object", + "properties": { + "period": { + "nullable": true, + "description": "The new period on which automatic cleanups are run.", + "allOf": [ + { + "$ref": "#/components/schemas/Duration" + } + ] + }, + "priority": { + "nullable": true, + "description": "The priority ordering for preserving old zone bundles.", + "allOf": [ + { + "$ref": "#/components/schemas/PriorityOrder" + } + ] + }, + "storage_limit": { + "nullable": true, + "description": "The new limit on the underlying dataset quota allowed for bundles.", + "type": "integer", + "format": "uint8", + "minimum": 0 + } + } + }, + "CleanupCount": { + "description": "The count of bundles / bytes removed during a cleanup operation.", + "type": "object", + "properties": { + "bundles": { + "description": "The number of bundles removed.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "bytes": { + "description": "The number of bytes removed.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "bundles", + "bytes" + ] + }, + "CleanupPeriod": { + "description": "A period on which bundles are automatically cleaned up.", + "allOf": [ + { + "$ref": "#/components/schemas/Duration" + } + ] + }, + "ComponentV0": { + "oneOf": [ + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/VirtioDisk" + }, + "type": { + "type": "string", + "enum": [ + "virtio_disk" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/NvmeDisk" + }, + "type": { + "type": "string", + "enum": [ + "nvme_disk" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/VirtioNic" + }, + "type": { + "type": "string", + "enum": [ + "virtio_nic" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/SerialPort" + }, + "type": { + "type": "string", + "enum": [ + "serial_port" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/PciPciBridge" + }, + "type": { + "type": "string", + "enum": [ + "pci_pci_bridge" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/QemuPvpanic" + }, + "type": { + "type": "string", + "enum": [ + "qemu_pvpanic" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/BootSettings" + }, + "type": { + "type": "string", + "enum": [ + "boot_settings" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/SoftNpuPciPort" + }, + "type": { + "type": "string", + "enum": [ + "soft_npu_pci_port" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/SoftNpuPort" + }, + "type": { + "type": "string", + "enum": [ + "soft_npu_port" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/SoftNpuP9" + }, + "type": { + "type": "string", + "enum": [ + "soft_npu_p9" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/P9fs" + }, + "type": { + "type": "string", + "enum": [ + "p9fs" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/MigrationFailureInjector" + }, + "type": { + "type": "string", + "enum": [ + "migration_failure_injector" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/CrucibleStorageBackend" + }, + "type": { + "type": "string", + "enum": [ + "crucible_storage_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/FileStorageBackend" + }, + "type": { + "type": "string", + "enum": [ + "file_storage_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/BlobStorageBackend" + }, + "type": { + "type": "string", + "enum": [ + "blob_storage_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/VirtioNetworkBackend" + }, + "type": { + "type": "string", + "enum": [ + "virtio_network_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "component": { + "$ref": "#/components/schemas/DlpiNetworkBackend" + }, + "type": { + "type": "string", + "enum": [ + "dlpi_network_backend" + ] + } + }, + "required": [ + "component", + "type" + ], + "additionalProperties": false + } + ] + }, + "CompressionAlgorithm": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "on" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "off" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "gzip" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "level": { + "$ref": "#/components/schemas/GzipLevel" + }, + "type": { + "type": "string", + "enum": [ + "gzip_n" + ] + } + }, + "required": [ + "level", + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lz4" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lzjb" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "zle" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, + "ConfigReconcilerInventory": { + "description": "Describes the last attempt made by the sled-agent-config-reconciler to reconcile the current sled config against the actual state of the sled.", + "type": "object", + "properties": { + "boot_partitions": { + "$ref": "#/components/schemas/BootPartitionContents" + }, + "datasets": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ConfigReconcilerInventoryResult" + } + }, + "external_disks": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ConfigReconcilerInventoryResult" + } + }, + "last_reconciled_config": { + "$ref": "#/components/schemas/OmicronSledConfig" + }, + "orphaned_datasets": { + "title": "IdOrdMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/OrphanedDataset" + } + ], + "path": "iddqd::IdOrdMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/OrphanedDataset" + }, + "uniqueItems": true + }, + "remove_mupdate_override": { + "nullable": true, + "description": "The result of removing the mupdate override file on disk.\n\n`None` if `remove_mupdate_override` was not provided in the sled config.", + "allOf": [ + { + "$ref": "#/components/schemas/RemoveMupdateOverrideInventory" + } + ] + }, + "zones": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ConfigReconcilerInventoryResult" + } + } + }, + "required": [ + "boot_partitions", + "datasets", + "external_disks", + "last_reconciled_config", + "orphaned_datasets", + "zones" + ] + }, + "ConfigReconcilerInventoryResult": { + "oneOf": [ + { + "type": "object", + "properties": { + "result": { + "type": "string", + "enum": [ + "ok" + ] + } + }, + "required": [ + "result" + ] + }, + { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "result": { + "type": "string", + "enum": [ + "err" + ] + } + }, + "required": [ + "message", + "result" + ] + } + ] + }, + "ConfigReconcilerInventoryStatus": { + "description": "Status of the sled-agent-config-reconciler task.", + "oneOf": [ + { + "description": "The reconciler task has not yet run for the first time since sled-agent started.", + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "not_yet_run" + ] + } + }, + "required": [ + "status" + ] + }, + { + "description": "The reconciler task is actively running.", + "type": "object", + "properties": { + "config": { + "$ref": "#/components/schemas/OmicronSledConfig" + }, + "running_for": { + "$ref": "#/components/schemas/Duration" + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "status": { + "type": "string", + "enum": [ + "running" + ] + } + }, + "required": [ + "config", + "running_for", + "started_at", + "status" + ] + }, + { + "description": "The reconciler task is currently idle, but previously did complete a reconciliation attempt.\n\nThis variant does not include the `OmicronSledConfig` used in the last attempt, because that's always available via [`ConfigReconcilerInventory::last_reconciled_config`].", + "type": "object", + "properties": { + "completed_at": { + "type": "string", + "format": "date-time" + }, + "ran_for": { + "$ref": "#/components/schemas/Duration" + }, + "status": { + "type": "string", + "enum": [ + "idle" + ] + } + }, + "required": [ + "completed_at", + "ran_for", + "status" + ] + } + ] + }, + "Cpuid": { + "description": "A set of CPUID values to expose to a guest.", + "type": "object", + "properties": { + "entries": { + "description": "A list of CPUID leaves/subleaves and their associated values.\n\nPropolis servers require that each entry's `leaf` be unique and that it falls in either the \"standard\" (0 to 0xFFFF) or \"extended\" (0x8000_0000 to 0x8000_FFFF) function ranges, since these are the only valid input ranges currently defined by Intel and AMD. See the Intel 64 and IA-32 Architectures Software Developer's Manual (June 2024) Table 3-17 and the AMD64 Architecture Programmer's Manual (March 2024) Volume 3's documentation of the CPUID instruction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/CpuidEntry" + } + }, + "vendor": { + "description": "The CPU vendor to emulate.\n\nCPUID leaves in the extended range (0x8000_0000 to 0x8000_FFFF) have vendor-defined semantics. Propolis uses this value to determine these semantics when deciding whether it needs to specialize the supplied template values for these leaves.", + "allOf": [ + { + "$ref": "#/components/schemas/CpuidVendor" + } + ] + } + }, + "required": [ + "entries", + "vendor" + ], + "additionalProperties": false + }, + "CpuidEntry": { + "description": "A full description of a CPUID leaf/subleaf and the values it produces.", + "type": "object", + "properties": { + "eax": { + "description": "The value to return in eax.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "ebx": { + "description": "The value to return in ebx.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "ecx": { + "description": "The value to return in ecx.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "edx": { + "description": "The value to return in edx.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "leaf": { + "description": "The leaf (function) number for this entry.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "subleaf": { + "nullable": true, + "description": "The subleaf (index) number for this entry, if it uses subleaves.", + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "eax", + "ebx", + "ecx", + "edx", + "leaf" + ], + "additionalProperties": false + }, + "CpuidVendor": { + "description": "A CPU vendor to use when interpreting the meanings of CPUID leaves in the extended ID range (0x80000000 to 0x8000FFFF).", + "type": "string", + "enum": [ + "amd", + "intel" + ] + }, + "CrucibleStorageBackend": { + "description": "A Crucible storage backend.", + "type": "object", + "properties": { + "readonly": { + "description": "Indicates whether the storage is read-only.", + "type": "boolean" + }, + "request_json": { + "description": "A serialized `[crucible_client_types::VolumeConstructionRequest]`. This is stored in serialized form so that breaking changes to the definition of a `VolumeConstructionRequest` do not inadvertently break instance spec deserialization.\n\nWhen using a spec to initialize a new instance, the spec author must ensure this request is well-formed and can be deserialized by the version of `crucible_client_types` used by the target Propolis.", + "type": "string" + } + }, + "required": [ + "readonly", + "request_json" + ], + "additionalProperties": false + }, + "DatasetConfig": { + "description": "Configuration information necessary to request a single dataset.\n\nThese datasets are tracked directly by Nexus.", + "type": "object", + "properties": { + "compression": { + "description": "The compression mode to be used by the dataset", + "allOf": [ + { + "$ref": "#/components/schemas/CompressionAlgorithm" + } + ] + }, + "id": { + "description": "The UUID of the dataset being requested", + "allOf": [ + { + "$ref": "#/components/schemas/DatasetUuid" + } + ] + }, + "name": { + "description": "The dataset's name", + "allOf": [ + { + "$ref": "#/components/schemas/DatasetName" + } + ] + }, + "quota": { + "nullable": true, + "description": "The upper bound on the amount of storage used by this dataset", + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + }, + "reservation": { + "nullable": true, + "description": "The lower bound on the amount of storage usable by this dataset", + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + } + }, + "required": [ + "compression", + "id", + "name" + ] + }, + "DatasetKind": { + "description": "The kind of dataset. See the `DatasetKind` enum in omicron-common for possible values.", + "type": "string" + }, + "DatasetName": { + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/DatasetKind" + }, + "pool_name": { + "$ref": "#/components/schemas/ZpoolName" + } + }, + "required": [ + "kind", + "pool_name" + ] + }, + "DatasetUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::DatasetUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "DhcpConfig": { + "description": "DHCP configuration for a port\n\nNot present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we use `InstanceRuntimeState::hostname` for this value.", + "type": "object", + "properties": { + "dns_servers": { + "description": "DNS servers to send to the instance\n\n(DHCPv4 option 6; DHCPv6 option 23)", + "type": "array", + "items": { + "type": "string", + "format": "ip" + } + }, + "host_domain": { + "nullable": true, + "description": "DNS zone this instance's hostname belongs to (e.g. the `project.example` part of `instance1.project.example`)\n\n(DHCPv4 option 15; used in DHCPv6 option 39)", + "type": "string" + }, + "search_domains": { + "description": "DNS search domains\n\n(DHCPv4 option 119; DHCPv6 option 24)", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "dns_servers", + "search_domains" + ] + }, + "DiskEnsureBody": { + "description": "Sent from to a sled agent to establish the runtime state of a Disk", + "type": "object", + "properties": { + "initial_runtime": { + "description": "Last runtime state of the Disk known to Nexus (used if the agent has never seen this Disk before).", + "allOf": [ + { + "$ref": "#/components/schemas/DiskRuntimeState" + } + ] + }, + "target": { + "description": "requested runtime state of the Disk", + "allOf": [ + { + "$ref": "#/components/schemas/DiskStateRequested" + } + ] + } + }, + "required": [ + "initial_runtime", + "target" + ] + }, + "DiskIdentity": { + "description": "Uniquely identifies a disk.", + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "serial": { + "type": "string" + }, + "vendor": { + "type": "string" + } + }, + "required": [ + "model", + "serial", + "vendor" + ] + }, + "DiskRuntimeState": { + "description": "Runtime state of the Disk, which includes its attach state and some minimal metadata", + "type": "object", + "properties": { + "disk_state": { + "description": "runtime state of the Disk", + "allOf": [ + { + "$ref": "#/components/schemas/DiskState" + } + ] + }, + "gen": { + "description": "generation number for this state", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "time_updated": { + "description": "timestamp for this information", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "disk_state", + "gen", + "time_updated" + ] + }, + "DiskState": { + "description": "State of a Disk", + "oneOf": [ + { + "description": "Disk is being initialized", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "creating" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is ready but detached from any Instance", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "detached" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is ready to receive blocks from an external source", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "import_ready" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is importing blocks from a URL", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "importing_from_url" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is importing blocks from bulk writes", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "importing_from_bulk_writes" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is being finalized to state Detached", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "finalizing" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is undergoing maintenance", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "maintenance" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is being attached to the given Instance", + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "attaching" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "description": "Disk is attached to the given Instance", + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "attached" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "description": "Disk is being detached from the given Instance", + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "detaching" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "description": "Disk has been destroyed", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "destroyed" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is unavailable", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "faulted" + ] + } + }, + "required": [ + "state" + ] + } + ] + }, + "DiskStateRequested": { + "description": "Used to request a Disk state change", + "oneOf": [ + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "detached" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "attached" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "destroyed" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "faulted" + ] + } + }, + "required": [ + "state" + ] + } + ] + }, + "DiskVariant": { + "type": "string", + "enum": [ + "U2", + "M2" + ] + }, + "DlpiNetworkBackend": { + "description": "A network backend associated with a DLPI VNIC on the host.", + "type": "object", + "properties": { + "vnic_name": { + "description": "The name of the VNIC to use as a backend.", + "type": "string" + } + }, + "required": [ + "vnic_name" + ], + "additionalProperties": false + }, + "Duration": { + "type": "object", + "properties": { + "nanos": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "secs": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "nanos", + "secs" + ] + }, + "EarlyNetworkConfig": { + "description": "Network configuration required to bring up the control plane\n\nThe fields in this structure are those from [`crate::rack_init::RackInitializeRequest`] necessary for use beyond RSS. This is just for the initial rack configuration and cold boot purposes. Updates come from Nexus.", + "type": "object", + "properties": { + "body": { + "$ref": "#/components/schemas/EarlyNetworkConfigBody" + }, + "generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "body", + "generation", + "schema_version" + ] + }, + "EarlyNetworkConfigBody": { + "description": "This is the actual configuration of EarlyNetworking.\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "type": "object", + "properties": { + "ntp_servers": { + "description": "The external NTP server addresses.", + "type": "array", + "items": { + "type": "string" + } + }, + "rack_network_config": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RackNetworkConfigV2" + } + ] + } + }, + "required": [ + "ntp_servers" + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "EstablishedConnection": { + "type": "object", + "properties": { + "addr": { + "type": "string" + }, + "baseboard": { + "$ref": "#/components/schemas/Baseboard" + } + }, + "required": [ + "addr", + "baseboard" + ] + }, + "ExternalIp": { + "description": "An external IP address used by a probe.", + "type": "object", + "properties": { + "first_port": { + "description": "The first port used by the address.", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "ip": { + "description": "The external IP address.", + "type": "string", + "format": "ip" + }, + "kind": { + "description": "The kind of address this is.", + "allOf": [ + { + "$ref": "#/components/schemas/IpKind" + } + ] + }, + "last_port": { + "description": "The last port used by the address.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "first_port", + "ip", + "kind", + "last_port" + ] + }, + "ExternalIpGatewayMap": { + "description": "Per-NIC mappings from external IP addresses to the Internet Gateways which can choose them as a source.", + "type": "object", + "properties": { + "mappings": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + }, + "uniqueItems": true + } + } + } + }, + "required": [ + "mappings" + ] + }, + "FileStorageBackend": { + "description": "A storage backend backed by a file in the host system's file system.", + "type": "object", + "properties": { + "block_size": { + "description": "Block size of the backend", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "path": { + "description": "A path to a file that backs a disk.", + "type": "string" + }, + "readonly": { + "description": "Indicates whether the storage is read-only.", + "type": "boolean" + }, + "workers": { + "nullable": true, + "description": "Optional worker threads for the file backend, exposed for testing only.", + "type": "integer", + "format": "uint", + "minimum": 1 + } + }, + "required": [ + "block_size", + "path", + "readonly" + ], + "additionalProperties": false + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "GuestHypervisorInterface": { + "description": "A hypervisor interface to expose to the guest.", + "oneOf": [ + { + "description": "Expose a bhyve-like interface (\"bhyve bhyve \" as the hypervisor ID in leaf 0x4000_0000 and no additional leaves or features).", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "bhyve" + ] + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "description": "Expose a Hyper-V-compatible hypervisor interface with the supplied features enabled.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "hyper_v" + ] + }, + "value": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HyperVFeatureFlag" + }, + "uniqueItems": true + } + }, + "required": [ + "features" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "value" + ], + "additionalProperties": false + } + ] + }, + "GzipLevel": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "HostIdentifier": { + "description": "A `HostIdentifier` represents either an IP host or network (v4 or v6), or an entire VPC (identified by its VNI). It is used in firewall rule host filters.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "$ref": "#/components/schemas/IpNet" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc" + ] + }, + "value": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "HostPhase2DesiredContents": { + "description": "Describes the desired contents of a host phase 2 slot (i.e., the boot partition on one of the internal M.2 drives).", + "oneOf": [ + { + "description": "Do not change the current contents.\n\nWe use this value when we've detected a sled has been mupdated (and we don't want to overwrite phase 2 images until we understand how to recover from that mupdate) and as the default value when reading an [`OmicronSledConfig`] that was ledgered before this concept existed.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "current_contents" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Set the phase 2 slot to the given artifact.\n\nThe artifact will come from an unpacked and distributed TUF repo.", + "type": "object", + "properties": { + "hash": { + "type": "string", + "format": "hex string (32 bytes)" + }, + "type": { + "type": "string", + "enum": [ + "artifact" + ] + } + }, + "required": [ + "hash", + "type" + ] + } + ] + }, + "HostPhase2DesiredSlots": { + "description": "Describes the desired contents for both host phase 2 slots.", + "type": "object", + "properties": { + "slot_a": { + "$ref": "#/components/schemas/HostPhase2DesiredContents" + }, + "slot_b": { + "$ref": "#/components/schemas/HostPhase2DesiredContents" + } + }, + "required": [ + "slot_a", + "slot_b" + ] + }, + "HostPortConfig": { + "type": "object", + "properties": { + "addrs": { + "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool). May also include an optional VLAN ID.", + "type": "array", + "items": { + "$ref": "#/components/schemas/UplinkAddressConfig" + } + }, + "lldp": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/LldpPortConfig" + } + ] + }, + "port": { + "description": "Switchport to use for external connectivity", + "type": "string" + }, + "tx_eq": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] + } + }, + "required": [ + "addrs", + "port" + ] + }, + "Hostname": { + "title": "An RFC-1035-compliant hostname", + "description": "A hostname identifies a host on a network, and is usually a dot-delimited sequence of labels, where each label contains only letters, digits, or the hyphen. See RFCs 1035 and 952 for more details.", + "type": "string", + "pattern": "^([a-zA-Z0-9]+[a-zA-Z0-9\\-]*(? for background.", + "oneOf": [ + { + "description": "Start the switch zone if a switch is present.\n\nThis is the default policy.", + "type": "object", + "properties": { + "policy": { + "type": "string", + "enum": [ + "start_if_switch_present" + ] + } + }, + "required": [ + "policy" + ] + }, + { + "description": "Even if a switch zone is present, stop the switch zone.", + "type": "object", + "properties": { + "policy": { + "type": "string", + "enum": [ + "stop_despite_switch_presence" + ] + } + }, + "required": [ + "policy" + ] + } + ] + }, + "OrphanedDataset": { + "type": "object", + "properties": { + "available": { + "$ref": "#/components/schemas/ByteCount" + }, + "id": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/DatasetUuid" + } + ] + }, + "mounted": { + "type": "boolean" + }, + "name": { + "$ref": "#/components/schemas/DatasetName" + }, + "reason": { + "type": "string" + }, + "used": { + "$ref": "#/components/schemas/ByteCount" + } + }, + "required": [ + "available", + "mounted", + "name", + "reason", + "used" + ] + }, + "P9fs": { + "description": "Describes a filesystem to expose through a P9 device.\n\nThis is only supported by Propolis servers compiled with the `falcon` feature.", + "type": "object", + "properties": { + "chunk_size": { + "description": "The chunk size to use in the 9P protocol. Vanilla Helios images should use 8192. Falcon Helios base images and Linux can use up to 65536.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "pci_path": { + "description": "The PCI path at which to attach the guest to this P9 filesystem.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + }, + "source": { + "description": "The host source path to mount into the guest.", + "type": "string" + }, + "target": { + "description": "The 9P target filesystem tag.", + "type": "string" + } + }, + "required": [ + "chunk_size", + "pci_path", + "source", + "target" + ], + "additionalProperties": false + }, + "PciPath": { + "description": "A PCI bus/device/function tuple.", + "type": "object", + "properties": { + "bus": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "device": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "function": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "bus", + "device", + "function" + ] + }, + "PciPciBridge": { + "description": "A PCI-PCI bridge.", + "type": "object", + "properties": { + "downstream_bus": { + "description": "The logical bus number of this bridge's downstream bus. Other devices may use this bus number in their PCI paths to indicate they should be attached to this bridge's bus.", + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "pci_path": { + "description": "The PCI path at which to attach this bridge.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "downstream_bus", + "pci_path" + ], + "additionalProperties": false + }, + "PhysicalDiskUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::PhysicalDiskUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "PortConfigV2": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses and optional vlan IDs", + "type": "array", + "items": { + "$ref": "#/components/schemas/UplinkAddressConfig" + } + }, + "autoneg": { + "description": "Whether or not to set autonegotiation", + "default": false, + "type": "boolean" + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "lldp": { + "nullable": true, + "description": "LLDP configuration for this port", + "allOf": [ + { + "$ref": "#/components/schemas/LldpPortConfig" + } + ] + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "tx_eq": { + "nullable": true, + "description": "TX-EQ configuration for this port", + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] + }, + "uplink_port_fec": { + "nullable": true, + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_speed" + ] + }, + "PortFec": { + "description": "Switchport FEC options", + "type": "string", + "enum": [ + "firecode", + "none", + "rs" + ] + }, + "PortSpeed": { + "description": "Switchport Speed options", + "type": "string", + "enum": [ + "speed0_g", + "speed1_g", + "speed10_g", + "speed25_g", + "speed40_g", + "speed50_g", + "speed100_g", + "speed200_g", + "speed400_g" + ] + }, + "PriorityDimension": { + "description": "A dimension along with bundles can be sorted, to determine priority.", + "oneOf": [ + { + "description": "Sorting by time, with older bundles with lower priority.", + "type": "string", + "enum": [ + "time" + ] + }, + { + "description": "Sorting by the cause for creating the bundle.", + "type": "string", + "enum": [ + "cause" + ] + } + ] + }, + "PriorityOrder": { + "description": "The priority order for bundles during cleanup.\n\nBundles are sorted along the dimensions in [`PriorityDimension`], with each dimension appearing exactly once. During cleanup, lesser-priority bundles are pruned first, to maintain the dataset quota. Note that bundles are sorted by each dimension in the order in which they appear, with each dimension having higher priority than the next.", + "type": "array", + "items": { + "$ref": "#/components/schemas/PriorityDimension" + }, + "minItems": 2, + "maxItems": 2 + }, + "PrivateIpConfig": { + "description": "VPC-private IP address configuration for a network interface.", + "oneOf": [ + { + "description": "The interface has only an IPv4 configuration.", + "type": "object", + "properties": { + "v4": { + "$ref": "#/components/schemas/PrivateIpv4Config" + } + }, + "required": [ + "v4" + ], + "additionalProperties": false + }, + { + "description": "The interface has only an IPv6 configuration.", + "type": "object", + "properties": { + "v6": { + "$ref": "#/components/schemas/PrivateIpv6Config" + } + }, + "required": [ + "v6" + ], + "additionalProperties": false + }, + { + "description": "The interface is dual-stack.", + "type": "object", + "properties": { + "dual_stack": { + "type": "object", + "properties": { + "v4": { + "description": "The interface's IPv4 configuration.", + "allOf": [ + { + "$ref": "#/components/schemas/PrivateIpv4Config" + } + ] + }, + "v6": { + "description": "The interface's IPv6 configuration.", + "allOf": [ + { + "$ref": "#/components/schemas/PrivateIpv6Config" + } + ] + } + }, + "required": [ + "v4", + "v6" + ] + } + }, + "required": [ + "dual_stack" + ], + "additionalProperties": false + } + ] + }, + "PrivateIpv4Config": { + "description": "VPC-private IPv4 configuration for a network interface.", + "type": "object", + "properties": { + "ip": { + "description": "VPC-private IP address.", + "type": "string", + "format": "ipv4" + }, + "subnet": { + "description": "The IP subnet.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Net" + } + ] + }, + "transit_ips": { + "description": "Additional networks on which the interface can send / receive traffic.", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Net" + } + } + }, + "required": [ + "ip", + "subnet" + ] + }, + "PrivateIpv6Config": { + "description": "VPC-private IPv6 configuration for a network interface.", + "type": "object", + "properties": { + "ip": { + "description": "VPC-private IP address.", + "type": "string", + "format": "ipv6" + }, + "subnet": { + "description": "The IP subnet.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Net" + } + ] + }, + "transit_ips": { + "description": "Additional networks on which the interface can send / receive traffic.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv6Net" + } + } + }, + "required": [ + "ip", + "subnet", + "transit_ips" + ] + }, + "ProbeCreate": { + "description": "Parameters used to create a probe.", + "type": "object", + "properties": { + "external_ips": { + "description": "The external IP addresses assigned to the probe.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ExternalIp" + } + }, + "id": { + "description": "The ID for the probe.", + "allOf": [ + { + "$ref": "#/components/schemas/ProbeUuid" + } + ] + }, + "interface": { + "description": "The probe's networking interface.", + "allOf": [ + { + "$ref": "#/components/schemas/NetworkInterface" + } + ] + } + }, + "required": [ + "external_ips", + "id", + "interface" + ] + }, + "ProbeSet": { + "description": "A set of probes that the target sled should run.", + "type": "object", + "properties": { + "probes": { + "title": "IdHashMap", + "description": "The exact set of probes to run.", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/ProbeCreate" + } + ], + "path": "iddqd::IdHashMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/ProbeCreate" + }, + "uniqueItems": true + } + }, + "required": [ + "probes" + ] + }, + "ProbeUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::ProbeUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "QemuPvpanic": { + "type": "object", + "properties": { + "enable_isa": { + "description": "Enable the QEMU PVPANIC ISA bus device (I/O port 0x505).", + "type": "boolean" + } + }, + "required": [ + "enable_isa" + ], + "additionalProperties": false + }, + "RackNetworkConfigV2": { + "description": "Initial network configuration", + "type": "object", + "properties": { + "bfd": { + "description": "BFD configuration for connecting the rack to external networks", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/BfdPeerConfig" + } + }, + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, + "infra_ip_first": { + "description": "First ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "infra_ip_last": { + "description": "Last ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "ports": { + "description": "Uplinks for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/PortConfigV2" + } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Net" + } + }, + "required": [ + "bgp", + "infra_ip_first", + "infra_ip_last", + "ports", + "rack_subnet" + ] + }, + "RemoveMupdateOverrideBootSuccessInventory": { + "description": "Status of removing the mupdate override on the boot disk.", + "oneOf": [ + { + "description": "The mupdate override was successfully removed.", + "type": "string", + "enum": [ + "removed" + ] + }, + { + "description": "No mupdate override was found.\n\nThis is considered a success for idempotency reasons.", + "type": "string", + "enum": [ + "no_override" + ] + } + ] + }, + "RemoveMupdateOverrideInventory": { + "description": "Status of removing the mupdate override in the inventory.", + "type": "object", + "properties": { + "boot_disk_result": { + "description": "The result of removing the mupdate override on the boot disk.", + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/RemoveMupdateOverrideBootSuccessInventory" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/RemoveMupdateOverrideBootSuccessInventory" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + }, + "non_boot_message": { + "description": "What happened on non-boot disks.\n\nWe aren't modeling this out in more detail, because we plan to not try and keep ledgered data in sync across both disks in the future.", + "type": "string" + } + }, + "required": [ + "boot_disk_result", + "non_boot_message" + ] + }, + "ResolvedVpcFirewallRule": { + "description": "VPC firewall rule after object name resolution has been performed by Nexus", + "type": "object", + "properties": { + "action": { + "$ref": "#/components/schemas/VpcFirewallRuleAction" + }, + "direction": { + "$ref": "#/components/schemas/VpcFirewallRuleDirection" + }, + "filter_hosts": { + "nullable": true, + "type": "array", + "items": { + "$ref": "#/components/schemas/HostIdentifier" + }, + "uniqueItems": true + }, + "filter_ports": { + "nullable": true, + "type": "array", + "items": { + "$ref": "#/components/schemas/L4PortRange" + } + }, + "filter_protocols": { + "nullable": true, + "type": "array", + "items": { + "$ref": "#/components/schemas/VpcFirewallRuleProtocol" + } + }, + "priority": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "status": { + "$ref": "#/components/schemas/VpcFirewallRuleStatus" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/NetworkInterface" + } + } + }, + "required": [ + "action", + "direction", + "priority", + "status", + "targets" + ] + }, + "ResolvedVpcRoute": { + "description": "A VPC route resolved into a concrete target.", + "type": "object", + "properties": { + "dest": { + "$ref": "#/components/schemas/IpNet" + }, + "target": { + "$ref": "#/components/schemas/RouterTarget" + } + }, + "required": [ + "dest", + "target" + ] + }, + "ResolvedVpcRouteSet": { + "description": "An updated set of routes for a given VPC and/or subnet.", + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/RouterId" + }, + "routes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ResolvedVpcRoute" + }, + "uniqueItems": true + }, + "version": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RouterVersion" + } + ] + } + }, + "required": [ + "id", + "routes" + ] + }, + "ResolvedVpcRouteState": { + "description": "Version information for routes on a given VPC subnet.", + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/RouterId" + }, + "version": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RouterVersion" + } + ] + } + }, + "required": [ + "id" + ] + }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + }, + "rib_priority": { + "nullable": true, + "description": "The RIB priority (i.e. Admin Distance) associated with this route.", + "default": null, + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "vlan_id": { + "nullable": true, + "description": "The VLAN id associated with this route.", + "default": null, + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "destination", + "nexthop" + ] + }, + "RouterId": { + "description": "Identifier for a VPC and/or subnet.", + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/RouterKind" + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "kind", + "vni" + ] + }, + "RouterKind": { + "description": "The scope of a set of VPC router rules.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "system" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "subnet": { + "$ref": "#/components/schemas/IpNet" + }, + "type": { + "type": "string", + "enum": [ + "custom" + ] + } + }, + "required": [ + "subnet", + "type" + ] + } + ] + }, + "RouterTarget": { + "description": "The target for a given router entry.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "drop" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internet_gateway" + ] + }, + "value": { + "$ref": "#/components/schemas/InternetGatewayRouterTarget" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc_subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/IpNet" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "RouterVersion": { + "description": "Information on the current parent router (and version) of a route set according to the control plane.", + "type": "object", + "properties": { + "router_id": { + "type": "string", + "format": "uuid" + }, + "version": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "router_id", + "version" + ] + }, + "SerialPort": { + "description": "A serial port device.", + "type": "object", + "properties": { + "num": { + "description": "The serial port number for this port.", + "allOf": [ + { + "$ref": "#/components/schemas/SerialPortNumber" + } + ] + } + }, + "required": [ + "num" + ], + "additionalProperties": false + }, + "SerialPortNumber": { + "description": "A serial port identifier, which determines what I/O ports a guest can use to access a port.", + "type": "string", + "enum": [ + "com1", + "com2", + "com3", + "com4" + ] + }, + "SledCpuFamily": { + "description": "Identifies the kind of CPU present on a sled, determined by reading CPUID.\n\nThis is intended to broadly support the control plane answering the question \"can I run this instance on that sled?\" given an instance with either no or some CPU platform requirement. It is not enough information for more precise placement questions - for example, is a CPU a high-frequency part or many-core part? We don't include Genoa here, but in that CPU family there are high frequency parts, many-core parts, and large-cache parts. To support those questions (or satisfactorily answer #8730) we would need to collect additional information and send it along.", + "oneOf": [ + { + "description": "The CPU vendor or its family number don't correspond to any of the known family variants.", + "type": "string", + "enum": [ + "unknown" + ] + }, + { + "description": "AMD Milan processors (or very close). Could be an actual Milan in a Gimlet, a close-to-Milan client Zen 3 part, or Zen 4 (for which Milan is the greatest common denominator).", + "type": "string", + "enum": [ + "amd_milan" + ] + }, + { + "description": "AMD Turin processors (or very close). Could be an actual Turin in a Cosmo, or a close-to-Turin client Zen 5 part.", + "type": "string", + "enum": [ + "amd_turin" + ] + }, + { + "description": "AMD Turin Dense processors. There are no \"Turin Dense-like\" CPUs unlike other cases, so this means a bona fide Zen 5c Turin Dense part.", + "type": "string", + "enum": [ + "amd_turin_dense" + ] + } + ] + }, + "SledDiagnosticsQueryOutput": { + "oneOf": [ + { + "type": "object", + "properties": { + "success": { + "type": "object", + "properties": { + "command": { + "description": "The command and its arguments.", + "type": "string" + }, + "exit_code": { + "nullable": true, + "description": "The exit code if one was present when the command exited.", + "type": "integer", + "format": "int32" + }, + "exit_status": { + "description": "The exit status of the command. This will be the exit code (if any) and exit reason such as from a signal.", + "type": "string" + }, + "stdio": { + "description": "Any stdout/stderr produced by the command.", + "type": "string" + } + }, + "required": [ + "command", + "exit_status", + "stdio" + ] + } + }, + "required": [ + "success" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "failure": { + "type": "object", + "properties": { + "error": { + "description": "The reason the command failed to execute.", + "type": "string" + } + }, + "required": [ + "error" + ] + } + }, + "required": [ + "failure" + ], + "additionalProperties": false + } + ] + }, + "SledIdentifiers": { + "description": "Identifiers for a single sled.\n\nThis is intended primarily to be used in timeseries, to identify sled from which metric data originates.", + "type": "object", + "properties": { + "model": { + "description": "Model name of the sled", + "type": "string" + }, + "rack_id": { + "description": "Control plane ID of the rack this sled is a member of", + "type": "string", + "format": "uuid" + }, + "revision": { + "description": "Revision number of the sled", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "serial": { + "description": "Serial number of the sled", + "type": "string" + }, + "sled_id": { + "description": "Control plane ID for the sled itself", + "type": "string", + "format": "uuid" + } + }, + "required": [ + "model", + "rack_id", + "revision", + "serial", + "sled_id" + ] + }, + "SledRole": { + "description": "Describes the role of the sled within the rack.\n\nNote that this may change if the sled is physically moved within the rack.", + "oneOf": [ + { + "description": "The sled is a general compute sled.", + "type": "string", + "enum": [ + "gimlet" + ] + }, + { + "description": "The sled is attached to the network switch, and has additional responsibilities.", + "type": "string", + "enum": [ + "scrimlet" + ] + } + ] + }, + "SledUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::SledUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "SledVmmState": { + "description": "A wrapper type containing a sled's total knowledge of the state of a VMM.", + "type": "object", + "properties": { + "migration_in": { + "nullable": true, + "description": "The current state of any inbound migration to this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/MigrationRuntimeState" + } + ] + }, + "migration_out": { + "nullable": true, + "description": "The state of any outbound migration from this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/MigrationRuntimeState" + } + ] + }, + "vmm_state": { + "description": "The most recent state of the sled's VMM process.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmRuntimeState" + } + ] + } + }, + "required": [ + "vmm_state" + ] + }, + "SoftNpuP9": { + "description": "Describes a PCI device that shares host files with the guest using the P9 protocol.\n\nThis is only supported by Propolis servers compiled with the `falcon` feature.", + "type": "object", + "properties": { + "pci_path": { + "description": "The PCI path at which to attach the guest to this port.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "pci_path" + ], + "additionalProperties": false + }, + "SoftNpuPciPort": { + "description": "Describes a SoftNPU PCI device.\n\nThis is only supported by Propolis servers compiled with the `falcon` feature.", + "type": "object", + "properties": { + "pci_path": { + "description": "The PCI path at which to attach the guest to this port.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "pci_path" + ], + "additionalProperties": false + }, + "SoftNpuPort": { + "description": "Describes a port in a SoftNPU emulated ASIC.\n\nThis is only supported by Propolis servers compiled with the `falcon` feature.", + "type": "object", + "properties": { + "backend_id": { + "description": "The name of the port's associated DLPI backend.", + "allOf": [ + { + "$ref": "#/components/schemas/SpecKey" + } + ] + }, + "link_name": { + "description": "The data link name for this port.", + "type": "string" + } + }, + "required": [ + "backend_id", + "link_name" + ], + "additionalProperties": false + }, + "SourceNatConfig": { + "description": "An IP address and port range used for source NAT, i.e., making outbound network connections from guests or services.", + "type": "object", + "properties": { + "first_port": { + "description": "The first port used for source NAT, inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "ip": { + "description": "The external address provided to the instance or service.", + "type": "string", + "format": "ip" + }, + "last_port": { + "description": "The last port used for source NAT, also inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "first_port", + "ip", + "last_port" + ] + }, + "SpecKey": { + "description": "A key identifying a component in an instance spec.", + "oneOf": [ + { + "title": "uuid", + "allOf": [ + { + "type": "string", + "format": "uuid" + } + ] + }, + { + "title": "name", + "allOf": [ + { + "type": "string" + } + ] + } + ] + }, + "StartSledAgentRequest": { + "description": "Configuration information for launching a Sled Agent.", + "type": "object", + "properties": { + "body": { + "$ref": "#/components/schemas/StartSledAgentRequestBody" + }, + "generation": { + "description": "The current generation number of data as stored in CRDB.\n\nThe initial generation is set during RSS time and then only mutated by Nexus. For now, we don't actually anticipate mutating this data, but we leave open the possiblity.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "body", + "generation", + "schema_version" + ] + }, + "StartSledAgentRequestBody": { + "description": "This is the actual app level data of `StartSledAgentRequest`\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "type": "object", + "properties": { + "id": { + "description": "Uuid of the Sled Agent to be created.", + "allOf": [ + { + "$ref": "#/components/schemas/SledUuid" + } + ] + }, + "is_lrtq_learner": { + "description": "Is this node an LRTQ learner node?\n\nWe only put the node into learner mode if `use_trust_quorum` is also true.", + "type": "boolean" + }, + "rack_id": { + "description": "Uuid of the rack to which this sled agent belongs.", + "type": "string", + "format": "uuid" + }, + "subnet": { + "description": "Portion of the IP space to be managed by the Sled Agent.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Subnet" + } + ] + }, + "use_trust_quorum": { + "description": "Use trust quorum for key generation", + "type": "boolean" + } + }, + "required": [ + "id", + "is_lrtq_learner", + "rack_id", + "subnet", + "use_trust_quorum" + ] + }, + "StorageLimit": { + "description": "The limit on space allowed for zone bundles, as a percentage of the overall dataset's quota.", + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "SupportBundleMetadata": { + "description": "Metadata about a support bundle", + "type": "object", + "properties": { + "state": { + "$ref": "#/components/schemas/SupportBundleState" + }, + "support_bundle_id": { + "$ref": "#/components/schemas/SupportBundleUuid" + } + }, + "required": [ + "state", + "support_bundle_id" + ] + }, + "SupportBundleState": { + "type": "string", + "enum": [ + "complete", + "incomplete" + ] + }, + "SupportBundleUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::SupportBundleUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "SwitchLocation": { + "description": "Identifies switch physical location", + "oneOf": [ + { + "description": "Switch in upper slot", + "type": "string", + "enum": [ + "switch0" + ] + }, + { + "description": "Switch in lower slot", + "type": "string", + "enum": [ + "switch1" + ] + } + ] + }, + "SwitchPorts": { + "description": "A set of switch uplinks.", + "type": "object", + "properties": { + "uplinks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HostPortConfig" + } + } + }, + "required": [ + "uplinks" + ] + }, + "TxEqConfig": { + "description": "Per-port tx-eq overrides. This can be used to fine-tune the transceiver equalization settings to improve signal integrity.", + "type": "object", + "properties": { + "main": { + "nullable": true, + "description": "Main tap", + "type": "integer", + "format": "int32" + }, + "post1": { + "nullable": true, + "description": "Post-cursor tap1", + "type": "integer", + "format": "int32" + }, + "post2": { + "nullable": true, + "description": "Post-cursor tap2", + "type": "integer", + "format": "int32" + }, + "pre1": { + "nullable": true, + "description": "Pre-cursor tap1", + "type": "integer", + "format": "int32" + }, + "pre2": { + "nullable": true, + "description": "Pre-cursor tap2", + "type": "integer", + "format": "int32" + } + } + }, + "UplinkAddressConfig": { + "type": "object", + "properties": { + "address": { + "$ref": "#/components/schemas/IpNet" + }, + "vlan_id": { + "nullable": true, + "description": "The VLAN id (if any) associated with this address.", + "default": null, + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "address" + ] + }, + "VirtioDisk": { + "description": "A disk that presents a virtio-block interface to the guest.", + "type": "object", + "properties": { + "backend_id": { + "description": "The name of the disk's backend component.", + "allOf": [ + { + "$ref": "#/components/schemas/SpecKey" + } + ] + }, + "pci_path": { + "description": "The PCI bus/device/function at which this disk should be attached.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "backend_id", + "pci_path" + ], + "additionalProperties": false + }, + "VirtioNetworkBackend": { + "description": "A network backend associated with a virtio-net (viona) VNIC on the host.", + "type": "object", + "properties": { + "vnic_name": { + "description": "The name of the viona VNIC to use as a backend.", + "type": "string" + } + }, + "required": [ + "vnic_name" + ], + "additionalProperties": false + }, + "VirtioNic": { + "description": "A network card that presents a virtio-net interface to the guest.", + "type": "object", + "properties": { + "backend_id": { + "description": "The name of the device's backend.", + "allOf": [ + { + "$ref": "#/components/schemas/SpecKey" + } + ] + }, + "interface_id": { + "description": "A caller-defined correlation identifier for this interface. If Propolis is configured to collect network interface kstats in its Oximeter metrics, the metric series for this interface will be associated with this identifier.", + "type": "string", + "format": "uuid" + }, + "pci_path": { + "description": "The PCI path at which to attach this device.", + "allOf": [ + { + "$ref": "#/components/schemas/PciPath" + } + ] + } + }, + "required": [ + "backend_id", + "interface_id", + "pci_path" + ], + "additionalProperties": false + }, + "VirtualNetworkInterfaceHost": { + "description": "A mapping from a virtual NIC to a physical host", + "type": "object", + "properties": { + "physical_host_ip": { + "type": "string", + "format": "ipv6" + }, + "virtual_ip": { + "type": "string", + "format": "ip" + }, + "virtual_mac": { + "$ref": "#/components/schemas/MacAddr" + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "physical_host_ip", + "virtual_ip", + "virtual_mac", + "vni" + ] + }, + "VmmIssueDiskSnapshotRequestBody": { + "type": "object", + "properties": { + "snapshot_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "snapshot_id" + ] + }, + "VmmIssueDiskSnapshotRequestResponse": { + "type": "object", + "properties": { + "snapshot_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "snapshot_id" + ] + }, + "VmmPutStateBody": { + "description": "The body of a request to move a previously-ensured instance into a specific runtime state.", + "type": "object", + "properties": { + "state": { + "description": "The state into which the instance should be driven.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmStateRequested" + } + ] + } + }, + "required": [ + "state" + ] + }, + "VmmPutStateResponse": { + "description": "The response sent from a request to move an instance into a specific runtime state.", + "type": "object", + "properties": { + "updated_runtime": { + "nullable": true, + "description": "The current runtime state of the instance after handling the request to change its state. If the instance's state did not change, this field is `None`.", + "allOf": [ + { + "$ref": "#/components/schemas/SledVmmState" + } + ] + } + } + }, + "VmmRuntimeState": { + "description": "The dynamic runtime properties of an individual VMM process.", + "type": "object", + "properties": { + "gen": { + "description": "The generation number for this VMM's state.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "state": { + "description": "The last state reported by this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmState" + } + ] + }, + "time_updated": { + "description": "Timestamp for the VMM's state.", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "gen", + "state", + "time_updated" + ] + }, + "VmmSpec": { + "description": "Specifies the virtual hardware configuration of a new Propolis VMM in the form of a Propolis instance specification.\n\nSled-agent expects that when an instance spec is provided alongside an `InstanceSledLocalConfig` to initialize a new instance, the NIC IDs in that config's network interface list will match the IDs of the virtio network backends in the instance spec.", + "allOf": [ + { + "$ref": "#/components/schemas/InstanceSpecV0" + } + ] + }, + "VmmState": { + "description": "One of the states that a VMM can be in.", + "oneOf": [ + { + "description": "The VMM is initializing and has not started running guest CPUs yet.", + "type": "string", + "enum": [ + "starting" + ] + }, + { + "description": "The VMM has finished initializing and may be running guest CPUs.", + "type": "string", + "enum": [ + "running" + ] + }, + { + "description": "The VMM is shutting down.", + "type": "string", + "enum": [ + "stopping" + ] + }, + { + "description": "The VMM's guest has stopped, and the guest will not run again, but the VMM process may not have released all of its resources yet.", + "type": "string", + "enum": [ + "stopped" + ] + }, + { + "description": "The VMM is being restarted or its guest OS is rebooting.", + "type": "string", + "enum": [ + "rebooting" + ] + }, + { + "description": "The VMM is part of a live migration.", + "type": "string", + "enum": [ + "migrating" + ] + }, + { + "description": "The VMM process reported an internal failure.", + "type": "string", + "enum": [ + "failed" + ] + }, + { + "description": "The VMM process has been destroyed and its resources have been released.", + "type": "string", + "enum": [ + "destroyed" + ] + } + ] + }, + "VmmStateRequested": { + "description": "Requestable running state of an Instance.\n\nA subset of [`omicron_common::api::external::InstanceState`].", + "oneOf": [ + { + "description": "Run this instance by migrating in from a previous running incarnation of the instance.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "migration_target" + ] + }, + "value": { + "$ref": "#/components/schemas/InstanceMigrationTargetParams" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Start the instance if it is not already running.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "running" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Stop the instance.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "stopped" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Immediately reset the instance, as though it had stopped and immediately began to run again.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "reboot" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, + "VmmUnregisterResponse": { + "description": "The response sent from a request to unregister an instance.", + "type": "object", + "properties": { + "updated_runtime": { + "nullable": true, + "description": "The current state of the instance after handling the request to unregister it. If the instance's state did not change, this field is `None`.", + "allOf": [ + { + "$ref": "#/components/schemas/SledVmmState" + } + ] + } + } + }, + "Vni": { + "description": "A Geneve Virtual Network Identifier", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "VpcFirewallIcmpFilter": { + "type": "object", + "properties": { + "code": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/IcmpParamRange" + } + ] + }, + "icmp_type": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "icmp_type" + ] + }, + "VpcFirewallRuleAction": { + "type": "string", + "enum": [ + "allow", + "deny" + ] + }, + "VpcFirewallRuleDirection": { + "type": "string", + "enum": [ + "inbound", + "outbound" + ] + }, + "VpcFirewallRuleProtocol": { + "description": "The protocols that may be specified in a firewall rule's filter", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "tcp" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "udp" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "icmp" + ] + }, + "value": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/VpcFirewallIcmpFilter" + } + ] + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "VpcFirewallRuleStatus": { + "type": "string", + "enum": [ + "disabled", + "enabled" + ] + }, + "VpcFirewallRulesEnsureBody": { + "description": "Update firewall rules for a VPC", + "type": "object", + "properties": { + "rules": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ResolvedVpcFirewallRule" + } + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "rules", + "vni" + ] + }, + "ZoneArtifactInventory": { + "description": "Inventory representation of a single zone artifact on a boot disk.\n\nPart of [`ZoneManifestBootInventory`].", + "type": "object", + "properties": { + "expected_hash": { + "description": "The expected digest of the file's contents.", + "type": "string", + "format": "hex string (32 bytes)" + }, + "expected_size": { + "description": "The expected size of the file, in bytes.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "file_name": { + "description": "The name of the zone file on disk, for example `nexus.tar.gz`. Zone files are always \".tar.gz\".", + "type": "string" + }, + "path": { + "description": "The full path to the zone file.", + "type": "string", + "format": "Utf8PathBuf" + }, + "status": { + "description": "The status of the artifact.\n\nThis is `Ok(())` if the artifact is present and matches the expected size and digest, or an error message if it is missing or does not match.", + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "type": "null" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "type": "string", + "enum": [ + null + ] + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + } + }, + "required": [ + "expected_hash", + "expected_size", + "file_name", + "path", + "status" + ] + }, + "ZoneBundleCause": { + "description": "The reason or cause for a zone bundle, i.e., why it was created.", + "oneOf": [ + { + "description": "Some other, unspecified reason.", + "type": "string", + "enum": [ + "other" + ] + }, + { + "description": "A zone bundle taken when a sled agent finds a zone that it does not expect to be running.", + "type": "string", + "enum": [ + "unexpected_zone" + ] + }, + { + "description": "An instance zone was terminated.", + "type": "string", + "enum": [ + "terminated_instance" + ] + } + ] + }, + "ZoneBundleId": { + "description": "An identifier for a zone bundle.", + "type": "object", + "properties": { + "bundle_id": { + "description": "The ID for this bundle itself.", + "type": "string", + "format": "uuid" + }, + "zone_name": { + "description": "The name of the zone this bundle is derived from.", + "type": "string" + } + }, + "required": [ + "bundle_id", + "zone_name" + ] + }, + "ZoneBundleMetadata": { + "description": "Metadata about a zone bundle.", + "type": "object", + "properties": { + "cause": { + "description": "The reason or cause a bundle was created.", + "allOf": [ + { + "$ref": "#/components/schemas/ZoneBundleCause" + } + ] + }, + "id": { + "description": "Identifier for this zone bundle", + "allOf": [ + { + "$ref": "#/components/schemas/ZoneBundleId" + } + ] + }, + "time_created": { + "description": "The time at which this zone bundle was created.", + "type": "string", + "format": "date-time" + }, + "version": { + "description": "A version number for this zone bundle.", + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "cause", + "id", + "time_created", + "version" + ] + }, + "ZoneImageResolverInventory": { + "description": "Inventory representation of zone image resolver status and health.", + "type": "object", + "properties": { + "mupdate_override": { + "description": "The mupdate override status.", + "allOf": [ + { + "$ref": "#/components/schemas/MupdateOverrideInventory" + } + ] + }, + "zone_manifest": { + "description": "The zone manifest status.", + "allOf": [ + { + "$ref": "#/components/schemas/ZoneManifestInventory" + } + ] + } + }, + "required": [ + "mupdate_override", + "zone_manifest" + ] + }, + "ZoneManifestBootInventory": { + "description": "Inventory representation of zone artifacts on the boot disk.\n\nPart of [`ZoneManifestInventory`].", + "type": "object", + "properties": { + "artifacts": { + "title": "IdOrdMap", + "description": "The artifacts on disk.", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/ZoneArtifactInventory" + } + ], + "path": "iddqd::IdOrdMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/ZoneArtifactInventory" + }, + "uniqueItems": true + }, + "source": { + "description": "The manifest source.\n\nIn production this is [`OmicronZoneManifestSource::Installinator`], but in some development and testing flows Sled Agent synthesizes zone manifests. In those cases, the source is [`OmicronZoneManifestSource::SledAgent`].", + "allOf": [ + { + "$ref": "#/components/schemas/OmicronZoneManifestSource" + } + ] + } + }, + "required": [ + "artifacts", + "source" + ] + }, + "ZoneManifestInventory": { + "description": "Inventory representation of a zone manifest.\n\nPart of [`ZoneImageResolverInventory`].\n\nA zone manifest is a listing of all the zones present in a system's install dataset. This struct contains information about the install dataset gathered from a system.", + "type": "object", + "properties": { + "boot_disk_path": { + "description": "The full path to the zone manifest file on the boot disk.", + "type": "string", + "format": "Utf8PathBuf" + }, + "boot_inventory": { + "description": "The manifest read from the boot disk, and whether the manifest is valid.", + "x-rust-type": { + "crate": "std", + "parameters": [ + { + "$ref": "#/components/schemas/ZoneManifestBootInventory" + }, + { + "type": "string" + } + ], + "path": "::std::result::Result", + "version": "*" + }, + "oneOf": [ + { + "type": "object", + "properties": { + "ok": { + "$ref": "#/components/schemas/ZoneManifestBootInventory" + } + }, + "required": [ + "ok" + ] + }, + { + "type": "object", + "properties": { + "err": { + "type": "string" + } + }, + "required": [ + "err" + ] + } + ] + }, + "non_boot_status": { + "title": "IdOrdMap", + "description": "Information about the install dataset on non-boot disks.", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/ZoneManifestNonBootInventory" + } + ], + "path": "iddqd::IdOrdMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/ZoneManifestNonBootInventory" + }, + "uniqueItems": true + } + }, + "required": [ + "boot_disk_path", + "boot_inventory", + "non_boot_status" + ] + }, + "ZoneManifestNonBootInventory": { + "description": "Inventory representation of a zone manifest on a non-boot disk.\n\nUnlike [`ZoneManifestBootInventory`] which is structured since Reconfigurator makes decisions based on it, information about non-boot disks is purely advisory. For simplicity, we store information in an unstructured format.", + "type": "object", + "properties": { + "is_valid": { + "description": "Whether the status is valid.", + "type": "boolean" + }, + "message": { + "description": "A message describing the status.\n\nIf `is_valid` is true, then the message describes the list of artifacts found and their hashes.\n\nIf `is_valid` is false, then this message describes the reason for the invalid status. This could include errors reading the zone manifest, or zone file mismatches.", + "type": "string" + }, + "path": { + "description": "The full path to the zone manifest JSON on the non-boot disk.", + "type": "string", + "format": "Utf8PathBuf" + }, + "zpool_id": { + "description": "The ID of the non-boot zpool.", + "allOf": [ + { + "$ref": "#/components/schemas/InternalZpoolUuid" + } + ] + } + }, + "required": [ + "is_valid", + "message", + "path", + "zpool_id" + ] + }, + "ZpoolName": { + "title": "The name of a Zpool", + "description": "Zpool names are of the format ox{i,p}_. They are either Internal or External, and should be unique", + "type": "string", + "pattern": "^ox[ip]_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "ZpoolUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::ZpoolUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, + "PropolisUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::PropolisUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/sled-agent/sled-agent-latest.json b/openapi/sled-agent/sled-agent-latest.json index a3a44e12369..da63ba1eb13 120000 --- a/openapi/sled-agent/sled-agent-latest.json +++ b/openapi/sled-agent/sled-agent-latest.json @@ -1 +1 @@ -sled-agent-6.0.0-d37dd7.json \ No newline at end of file +sled-agent-7.0.0-90da02.json \ No newline at end of file diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 4ee3de420cf..d6d37f981dd 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -33,7 +33,9 @@ use omicron_uuid_kinds::{ }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use sled_agent_types::probes::ProbeSet; +use sled_agent_types::inventory::v3; +use sled_agent_types::inventory::v6; +use sled_agent_types::probes; use sled_agent_types::{ bootstore::BootstoreStatus, disk::DiskEnsureBody, @@ -53,9 +55,6 @@ use sled_diagnostics::SledDiagnosticsQueryOutput; use tufaceous_artifact::ArtifactHash; use uuid::Uuid; -/// Copies of data types that changed between v3 and v4. -mod v3; - api_versions!([ // WHEN CHANGING THE API (part 1 of 2): // @@ -68,6 +67,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (7, ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES), (6, ADD_PROBE_PUT_ENDPOINT), (5, NEWTYPE_UUID_BUMP), (4, ADD_NEXUS_LOCKSTEP_PORT_TO_INVENTORY), @@ -328,13 +328,27 @@ pub trait SledAgentApi { #[endpoint { method = PUT, path = "/omicron-config", - versions = VERSION_ADD_NEXUS_LOCKSTEP_PORT_TO_INVENTORY.., + versions = VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES.. }] async fn omicron_config_put( rqctx: RequestContext, body: TypedBody, ) -> Result; + #[endpoint { + method = PUT, + path = "/omicron-config", + versions = + VERSION_ADD_NEXUS_LOCKSTEP_PORT_TO_INVENTORY..VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES, + }] + async fn v6_omicron_config_put( + rqctx: RequestContext, + body: TypedBody, + ) -> Result { + let body = body.try_map(OmicronSledConfig::try_from)?; + Self::omicron_config_put(rqctx, body).await + } + #[endpoint { operation_id = "omicron_config_put", method = PUT, @@ -345,7 +359,8 @@ pub trait SledAgentApi { rqctx: RequestContext, body: TypedBody, ) -> Result { - Self::omicron_config_put(rqctx, body.map(Into::into)).await + let body = body.try_map(OmicronSledConfig::try_from)?; + Self::omicron_config_put(rqctx, body).await } #[endpoint { @@ -359,6 +374,7 @@ pub trait SledAgentApi { #[endpoint { method = PUT, path = "/vmms/{propolis_id}", + versions = VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES.., }] async fn vmm_register( rqctx: RequestContext, @@ -366,6 +382,21 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result, HttpError>; + #[endpoint { + operation_id = "vmm_register", + method = PUT, + path = "/vmms/{propolis_id}", + versions = ..VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES, + }] + async fn v6_vmm_register( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError> { + let body = body.try_map(InstanceEnsureBody::try_from)?; + Self::vmm_register(rqctx, path_params, body).await + } + #[endpoint { method = DELETE, path = "/vmms/{propolis_id}", @@ -486,6 +517,7 @@ pub trait SledAgentApi { #[endpoint { method = PUT, path = "/vpc/{vpc_id}/firewall/rules", + versions = VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES.., }] async fn vpc_firewall_rules_put( rqctx: RequestContext, @@ -493,6 +525,21 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result; + #[endpoint { + operation_id = "vpc_firewall_rules_put", + method = PUT, + path = "/vpc/{vpc_id}/firewall/rules", + versions = ..VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES, + }] + async fn v6_vpc_firewall_rules_put( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result { + let body = body.try_map(VpcFirewallRulesEnsureBody::try_from)?; + Self::vpc_firewall_rules_put(rqctx, path_params, body).await + } + /// Create a mapping from a virtual NIC to a physical host // Keep interface_id to maintain parity with the simulated sled agent, which // requires interface_id on the path. @@ -570,12 +617,27 @@ pub trait SledAgentApi { #[endpoint { method = GET, path = "/inventory", - versions = VERSION_ADD_NEXUS_LOCKSTEP_PORT_TO_INVENTORY.., + versions = VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES.., }] async fn inventory( rqctx: RequestContext, ) -> Result, HttpError>; + /// Fetch basic information about this sled + #[endpoint { + operation_id = "inventory", + method = GET, + path = "/inventory", + versions = + VERSION_ADD_NEXUS_LOCKSTEP_PORT_TO_INVENTORY..VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES, + }] + async fn v6_inventory( + rqctx: RequestContext, + ) -> Result, HttpError> { + let HttpResponseOk(inventory) = Self::inventory(rqctx).await?; + inventory.try_into().map_err(HttpError::from).map(HttpResponseOk) + } + /// Fetch basic information about this sled #[endpoint { operation_id = "inventory", @@ -587,7 +649,7 @@ pub trait SledAgentApi { rqctx: RequestContext, ) -> Result, HttpError> { let HttpResponseOk(inventory) = Self::inventory(rqctx).await?; - Ok(HttpResponseOk(inventory.into())) + inventory.try_into().map_err(HttpError::from).map(HttpResponseOk) } /// Fetch sled identifiers @@ -803,12 +865,31 @@ pub trait SledAgentApi { #[endpoint { method = PUT, path = "/probes", - versions = VERSION_ADD_PROBE_PUT_ENDPOINT.., + versions = VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES.., }] async fn probes_put( request_context: RequestContext, - body: TypedBody, + body: TypedBody, ) -> Result; + + /// Update the entire set of probe zones on this sled. + /// + /// Probe zones are used to debug networking configuration. They look + /// similar to instances, in that they have an OPTE port on a VPC subnet and + /// external addresses, but no actual VM. + #[endpoint { + method = PUT, + path = "/probes", + versions = + VERSION_ADD_PROBE_PUT_ENDPOINT..VERSION_ADD_DUAL_STACK_SHARED_NETWORK_INTERFACES, + }] + async fn v6_probes_put( + request_context: RequestContext, + body: TypedBody, + ) -> Result { + let body = body.try_map(TryInto::try_into)?; + Self::probes_put(request_context, body).await + } } #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] diff --git a/sled-agent/config-reconciler/src/ledger.rs b/sled-agent/config-reconciler/src/ledger.rs index 9d5041a0f7b..f1f92c950ef 100644 --- a/sled-agent/config-reconciler/src/ledger.rs +++ b/sled-agent/config-reconciler/src/ledger.rs @@ -30,6 +30,7 @@ use tufaceous_artifact::ArtifactHash; use crate::InternalDisksReceiver; use crate::SledAgentArtifactStore; +use crate::ledger::legacy_configs::try_convert_v6_sled_config; mod legacy_configs; @@ -633,18 +634,30 @@ async fn load_sled_config( let paths = config_datasets .iter() .map(|p| p.join(CONFIG_LEDGER_FILENAME)) - .collect(); + .collect::>(); info!( log, "Attempting to load sled config from ledger"; "paths" => ?paths, ); - if let Some(config) = Ledger::new(log, paths).await { + if let Some(config) = Ledger::new(log, paths.clone()).await { info!(log, "Ledger of sled config exists"); return CurrentSledConfig::Ledgered(Box::new(config.into_inner())); } // If we have no ledgered config, see if we can convert from the previous - // triple of legacy ledgers. + // version of the format. + if let Some(config) = try_convert_v6_sled_config(log, paths).await { + info!( + log, + "Ledger of sled config exists, but it was formatted as \ + version 6, with single-stack NICs. It has been rewritten \ + to the current version", + ); + return CurrentSledConfig::Ledgered(Box::new(config)); + } + + // If we have no ledgered config, see if we can convert from the even + // more-previous triple of legacy ledgers. if let Some(config) = convert_legacy_ledgers(&config_datasets, log).await { info!(log, "Converted legacy triple of ledgers into new sled config"); return CurrentSledConfig::Ledgered(Box::new(config)); diff --git a/sled-agent/config-reconciler/src/ledger/legacy_configs.rs b/sled-agent/config-reconciler/src/ledger/legacy_configs.rs index 04a535bec8c..76afb40d12b 100644 --- a/sled-agent/config-reconciler/src/ledger/legacy_configs.rs +++ b/sled-agent/config-reconciler/src/ledger/legacy_configs.rs @@ -2,13 +2,11 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Module for converting the legacy triple of sled config files (disks, -//! datasets, and zones) into the current unified [`OmicronSledConfig`]. +//! Module for converting older formats of the sled configuration files. use camino::Utf8PathBuf; use nexus_sled_agent_shared::inventory::HostPhase2DesiredSlots; use nexus_sled_agent_shared::inventory::OmicronSledConfig; -use nexus_sled_agent_shared::inventory::OmicronZoneConfig; use omicron_common::api::external::Generation; use omicron_common::disk::DatasetsConfig; use omicron_common::disk::OmicronPhysicalDisksConfig; @@ -16,6 +14,8 @@ use omicron_common::ledger::Ledger; use omicron_common::ledger::Ledgerable; use serde::Deserialize; use serde::Serialize; +use sled_agent_types::inventory::v6::OmicronSledConfig as OmicronSledConfigV6; +use sled_agent_types::inventory::v6::OmicronZoneConfig as OmicronZoneConfigV6; use slog::Logger; use slog::error; use slog::warn; @@ -27,6 +27,38 @@ const LEGACY_DISKS_LEDGER_FILENAME: &str = "omicron-physical-disks.json"; const LEGACY_DATASETS_LEDGER_FILENAME: &str = "omicron-datasets.json"; const LEGACY_ZONES_LEDGER_FILENAME: &str = "omicron-zones.json"; +/// Convert from version 6 of the sled-configuration to the current, if +/// possible. The later version includes dual-stack private IP configuration in +/// our internal network interface types. +/// +/// # Panics +/// +/// This panics if the conversion fails. That might happen if we somehow +/// serialized a NIC with an IPv4 address, but on an IPv6 subnet. That should +/// not be possible, but we have no way of recovering into the current format if +/// we do encounter that. +pub(super) async fn try_convert_v6_sled_config( + log: &Logger, + datasets: Vec, +) -> Option { + let old = + Ledger::::new(log, datasets.clone()).await?; + let new_config = old.into_inner().0.try_into().unwrap_or_else(|e| { + panic!( + "Failed to convert OmicronSledConfigV6 to the current version: {e}" + ) + }); + write_converted_ledger( + log, + datasets, + new_config, + LegacyKind::SingleStackNic, + ) + .await +} + +/// Convert the legacy triple of sled config files (disks, datasets, and zones) +/// into the current unified [`OmicronSledConfig`]. pub(super) async fn convert_legacy_ledgers( config_datasets: &[Utf8PathBuf], log: &Logger, @@ -95,13 +127,69 @@ pub(super) async fn convert_legacy_ledgers( // Perform the actual merge; this is infallible. let sled_config = merge_old_configs(disks, datasets, zones); + // At this point, we've converted from the old, 3-config world into the + // 1-config world, but we've also converted into an older version of the + // configuration type itself. The 3-file format predates support for + // dual-stack network interfaces. + // + // Convert the merged configuration into this new format, and write that out + // instead. This conversion _is_ fallible. Unfortunately, if it fails, + // there's nothing we can do. That conversion is determinstic, so doing it + // again won't change the result. + let sled_config = OmicronSledConfig::try_from(sled_config) + .unwrap_or_else(|e| panic!( + "Failed to convert OmicronSledConfigV6 to the current version: {e}" + )); + // Write the newly-merged config to disk. let new_config_paths = config_datasets .iter() .map(|p| p.join(CONFIG_LEDGER_FILENAME)) .collect::>(); - let mut config_ledger = - Ledger::new_with(log, new_config_paths.clone(), sled_config); + let new_ledger = write_converted_ledger( + log, + new_config_paths, + sled_config, + LegacyKind::ThreeLedgerFormat, + ) + .await?; + + // We've successfully written and reread our new combined config; remove the + // old legacy ledgers. + for old_ledger_path in + disk_paths.iter().chain(dataset_paths.iter()).chain(zone_paths.iter()) + { + if let Err(err) = tokio::fs::remove_file(old_ledger_path).await { + // There isn't really anything we can do other than warn here; + // future attempts to read the ledger will find the combined config + // we wrote above, so we'll just leak the legacy configs here and + // rely on support procedures to confirm we don't hit this during + // the transition period. + warn!( + log, + "Failed to remove legacy ledger"; + "path" => %old_ledger_path, + InlineErrorChain::new(&err), + ); + } + } + + Some(new_ledger) +} + +#[derive(Debug)] +enum LegacyKind { + ThreeLedgerFormat, + SingleStackNic, +} + +async fn write_converted_ledger( + log: &Logger, + paths: Vec, + sled_config: OmicronSledConfig, + kind: LegacyKind, +) -> Option { + let mut config_ledger = Ledger::new_with(log, paths.clone(), sled_config); match config_ledger.commit().await { Ok(()) => (), @@ -112,7 +200,8 @@ pub(super) async fn convert_legacy_ledgers( warn!( log, "Failed to write new sled config ledger built \ - from legacy ledgers"; + from legacy ledgers"; + "legacy_kind" => ?kind, InlineErrorChain::new(&err), ); return Some(config_ledger.into_inner()); @@ -121,21 +210,22 @@ pub(super) async fn convert_legacy_ledgers( // Be paranoid before removing the legacy ledgers: confirm we can read back // the new combined config. - match Ledger::new(log, new_config_paths.clone()).await { + match Ledger::new(log, paths.clone()).await { Some(reread_config) => { // Check that the contents we wrote match the contents we read. No // one should be modifying this file concurrently, so a failure here - // means we've ledgered incorrect data, which could be disasterous. + // means we've ledgered incorrect data, which could be disastrous. // Log an error and at least try remove the ledgers we just wrote, // then use the config we cobbled together from the legacy ledgers. if config_ledger.data() != reread_config.data() { error!( log, "Reading just-ledgered config returns unexpected contents!"; + "legacy_kind" => ?kind, "written" => ?config_ledger.data(), "read" => ?reread_config.data(), ); - for p in &new_config_paths { + for p in &paths { if let Err(err) = tokio::fs::remove_file(p).await { // We're in really big trouble now: we've written a // bogus ledger and can't remove it. We cannot safely @@ -143,6 +233,7 @@ pub(super) async fn convert_legacy_ledgers( error!( log, "Wrote bogus ledger and then failed to remove it!"; + "legacy_kind" => ?kind, "path" => %p, InlineErrorChain::new(&err), ); @@ -170,32 +261,13 @@ pub(super) async fn convert_legacy_ledgers( // try to remove the legacy ledgers. warn!( log, "Failed to read ledgered config we just wrote!"; - "paths" => ?new_config_paths, + "legacy_kind" => ?kind, + "paths" => ?paths, ); return Some(config_ledger.into_inner()); } } - // We've successfully written and reread our new combined config; remove the - // old legacy ledgers. - for old_ledger_path in - disk_paths.iter().chain(dataset_paths.iter()).chain(zone_paths.iter()) - { - if let Err(err) = tokio::fs::remove_file(old_ledger_path).await { - // There isn't really anything we can do other than warn here; - // future attempts to read the ledger will find the combined config - // we wrote above, so we'll just leak the legacy configs here and - // rely on support procedures to confirm we don't hit this during - // the transition period. - warn!( - log, - "Failed to remove legacy ledger"; - "path" => %old_ledger_path, - InlineErrorChain::new(&err), - ); - } - } - Some(config_ledger.into_inner()) } @@ -203,8 +275,8 @@ fn merge_old_configs( disks: OmicronPhysicalDisksConfig, datasets: DatasetsConfig, zones: OmicronZonesConfigLocal, -) -> OmicronSledConfig { - OmicronSledConfig { +) -> OmicronSledConfigV6 { + OmicronSledConfigV6 { // Take the zone generation as the overall config generation; this is // consistent with Reconfigurator's transition from three configs to // one. @@ -243,12 +315,26 @@ impl Ledgerable for OmicronZonesConfigLocal { #[derive(Debug, Clone, Deserialize, Serialize)] #[cfg_attr(test, derive(schemars::JsonSchema))] struct OmicronZoneConfigLocal { - zone: OmicronZoneConfig, + zone: OmicronZoneConfigV6, #[serde(rename = "root")] #[cfg_attr(test, schemars(with = "String"))] _root: Utf8PathBuf, } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(transparent)] +struct OmicronSledConfigLocal(OmicronSledConfigV6); + +impl Ledgerable for OmicronSledConfigLocal { + fn is_newer_than(&self, other: &Self) -> bool { + self.0.generation > other.0.generation + } + + fn generation_bump(&mut self) { + self.0.generation = self.0.generation.next() + } +} + #[cfg(test)] pub(super) mod tests { use crate::ledger::CONFIG_LEDGER_FILENAME; @@ -279,6 +365,40 @@ pub(super) mod tests { ); } + #[tokio::test] + async fn can_convert_v6_config_version() { + let logctx = dev::test_setup_log("can_convert_v6_config_version"); + let tempdir = Utf8TempDir::new().expect("created tempdir"); + + // Copy version 6 into a tempdir. + println!("logging to {}", tempdir.path()); + let dst_file_name = Utf8PathBuf::from( + Utf8PathBuf::from(MERGED_CONFIG_PATH).file_name().unwrap(), + ); + let dst_file = tempdir.path().join(&dst_file_name); + tokio::fs::copy(MERGED_CONFIG_PATH, &dst_file) + .await + .expect("Copy old config into tempdir"); + println!("copied {} => {}", MERGED_CONFIG_PATH, dst_file); + + // Convert, which will rewrite the config as well. + let converted = + try_convert_v6_sled_config(&logctx.log, vec![dst_file.clone()]) + .await + .expect("Should have found and converted v6 config"); + + // And make sure it matches the new, directly loaded and converted from + // disk. + let new_as_v6: OmicronSledConfigV6 = serde_json::from_str( + tokio::fs::read_to_string(dst_file).await.unwrap().as_str(), + ) + .expect("successfully converted config"); + let new = OmicronSledConfig::try_from(new_as_v6) + .expect("successfully converted v6 config"); + assert_eq!(new, converted); + logctx.cleanup_successful(); + } + #[test] fn test_merge_old_configs() { let disks: OmicronPhysicalDisksConfig = { diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 124eae7eb92..09a44a3cc87 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -28,8 +28,8 @@ use omicron_common::address::{ }; use omicron_common::api::external::{Generation, MacAddr, Vni}; use omicron_common::api::internal::shared::{ - NetworkInterface, NetworkInterfaceKind, SourceNatConfig, - SourceNatConfigError, + NetworkInterface, NetworkInterfaceKind, PrivateIpConfig, + PrivateIpConfigError, SourceNatConfig, SourceNatConfigError, }; use omicron_common::backoff::{ BackoffError, retry_notify_ext, retry_policy_internal_service_aggressive, @@ -92,6 +92,9 @@ pub enum PlanError { #[error("Unexpected dataset kind: {0}")] UnexpectedDataset(String), + + #[error("invalid private IP configuration")] + InvalidPrivateIpConfig(#[from] PrivateIpConfigError), } #[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] @@ -1035,15 +1038,17 @@ impl ServicePortBuilder { }; let external_ip = self.external_dns_ips.next()?; - let (ip, subnet) = match external_ip { - IpAddr::V4(_) => ( - self.dns_v4_ips.next().unwrap().into(), - (*DNS_OPTE_IPV4_SUBNET).into(), - ), - IpAddr::V6(_) => ( - self.dns_v6_ips.next().unwrap().into(), - (*DNS_OPTE_IPV6_SUBNET).into(), - ), + let ip_config = match external_ip { + IpAddr::V4(_) => PrivateIpConfig::new_ipv4( + self.dns_v4_ips.next().unwrap(), + *DNS_OPTE_IPV4_SUBNET, + ) + .ok()?, + IpAddr::V6(_) => PrivateIpConfig::new_ipv6( + self.dns_v6_ips.next().unwrap(), + *DNS_OPTE_IPV6_SUBNET, + ) + .ok()?, }; let nic = NetworkInterface { @@ -1053,13 +1058,11 @@ impl ServicePortBuilder { id: svc_id.into_untyped_uuid(), }, name: format!("external-dns-{svc_id}").parse().unwrap(), - ip, + ip_config, mac: self.random_mac(), - subnet, vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }; Some((nic, external_ip)) @@ -1076,15 +1079,15 @@ impl ServicePortBuilder { .next_internal_service_ip() .ok_or_else(|| PlanError::ServiceIp("Nexus"))?; - let (ip, subnet) = match external_ip { - IpAddr::V4(_) => ( - self.nexus_v4_ips.next().unwrap().into(), - (*NEXUS_OPTE_IPV4_SUBNET).into(), - ), - IpAddr::V6(_) => ( - self.nexus_v6_ips.next().unwrap().into(), - (*NEXUS_OPTE_IPV6_SUBNET).into(), - ), + let ip_config = match external_ip { + IpAddr::V4(_) => PrivateIpConfig::new_ipv4( + self.nexus_v4_ips.next().unwrap(), + *NEXUS_OPTE_IPV4_SUBNET, + )?, + IpAddr::V6(_) => PrivateIpConfig::new_ipv6( + self.nexus_v6_ips.next().unwrap(), + *NEXUS_OPTE_IPV6_SUBNET, + )?, }; let nic = NetworkInterface { @@ -1094,13 +1097,11 @@ impl ServicePortBuilder { id: svc_id.into_untyped_uuid(), }, name: format!("nexus-{svc_id}").parse().unwrap(), - ip, + ip_config, mac: self.random_mac(), - subnet, vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }; Ok((nic, external_ip)) @@ -1136,15 +1137,15 @@ impl ServicePortBuilder { } }; - let (ip, subnet) = match snat_ip { - IpAddr::V4(_) => ( - self.ntp_v4_ips.next().unwrap().into(), - (*NTP_OPTE_IPV4_SUBNET).into(), - ), - IpAddr::V6(_) => ( - self.ntp_v6_ips.next().unwrap().into(), - (*NTP_OPTE_IPV6_SUBNET).into(), - ), + let ip_config = match snat_ip { + IpAddr::V4(_) => PrivateIpConfig::new_ipv4( + self.ntp_v4_ips.next().unwrap(), + *NTP_OPTE_IPV4_SUBNET, + )?, + IpAddr::V6(_) => PrivateIpConfig::new_ipv6( + self.ntp_v6_ips.next().unwrap(), + *NTP_OPTE_IPV6_SUBNET, + )?, }; let nic = NetworkInterface { @@ -1154,13 +1155,11 @@ impl ServicePortBuilder { id: svc_id.into_untyped_uuid(), }, name: format!("ntp-{svc_id}").parse().unwrap(), - ip, + ip_config, mac: self.random_mac(), - subnet, vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }; Ok((nic, snat_cfg)) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 80833805d1c..2aac653caba 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -81,7 +81,7 @@ use omicron_common::address::{ use omicron_common::address::{Ipv6Subnet, NEXUS_TECHPORT_EXTERNAL_PORT}; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::{ - HostPortConfig, RackNetworkConfig, SledIdentifiers, + HostPortConfig, PrivateIpConfig, RackNetworkConfig, SledIdentifiers, }; use omicron_common::backoff::{ BackoffError, retry_notify, retry_policy_internal_service_aggressive, @@ -1306,8 +1306,22 @@ impl ServiceManager { })?; let opte_interface = port.name(); - let opte_gateway = port.gateway().ip().to_string(); - let opte_ip = port.ip().to_string(); + + // TODO-completeness: This needs to support dual-stack OPTE ports. + // See https://github.com/oxidecomputer/omicron/issues/9309. + let opte_gateway = + port.gateway().ipv4_addr().map(|ip| ip.to_string()).unwrap_or_else( + || { + port.gateway() + .ipv6_addr() + .expect("at least one IP address") + .to_string() + }, + ); + let opte_ip = + port.ipv4_addr().map(|ip| ip.to_string()).unwrap_or_else(|| { + port.ipv6_addr().expect("at least one IP address").to_string() + }); let mut config_builder = PropertyGroupBuilder::new("config"); config_builder = config_builder @@ -1973,8 +1987,16 @@ impl ServiceManager { // We need to tell external_dns to listen on its OPTE port IP // address, which comes from `nic`. Attach the port from its // true external DNS address (`dns_address`). - let dns_address = - SocketAddr::new(nic.ip, dns_address.port()).to_string(); + // + // Make sure we take the VPC-private IP address with the same + // version as the external address. + let private_ip = Self::private_ip_for_external_address( + dns_address.ip(), + &nic.ip_config, + config.zone_type.kind(), + )?; + let private_dns_address = + SocketAddr::new(private_ip, dns_address.port()).to_string(); let external_dns_config = PropertyGroupBuilder::new("config") .add_property( @@ -1982,7 +2004,11 @@ impl ServiceManager { "astring", http_address.to_string(), ) - .add_property("dns_address", "astring", dns_address); + .add_property( + "dns_address", + "astring", + private_dns_address, + ); let external_dns_service = ServiceBuilder::new("oxide/external_dns").add_instance( ServiceInstanceBuilder::new("default") @@ -2282,6 +2308,8 @@ impl ServiceManager { lockstep_port, external_tls, external_dns_servers, + external_ip, + nic, .. }, id, @@ -2303,7 +2331,6 @@ impl ServiceManager { // external IP automatically. let opte_interface_setup = Self::opte_interface_set_up_install(&installed_zone)?; - let port_idx = 0; let port = installed_zone .opte_ports() @@ -2316,9 +2343,16 @@ impl ServiceManager { }, ) })?; - let opte_ip = port.ip(); let opte_iface_name = port.name(); + // Fetch the private IP of the same IP version as the external + // IP address. + let private_ip = Self::private_ip_for_external_address( + *external_ip, + &nic.ip_config, + config.zone_type.kind(), + )?; + // Nexus takes a separate config file for parameters // which cannot be known at packaging time. let nexus_port = if *external_tls { 443 } else { 80 }; @@ -2330,7 +2364,9 @@ impl ServiceManager { dropshot_external: ConfigDropshotWithTls { tls: *external_tls, dropshot: dropshot::ConfigDropshot { - bind_address: SocketAddr::new(*opte_ip, nexus_port), + bind_address: SocketAddr::new( + private_ip, nexus_port, + ), default_request_body_max_bytes: 1048576, default_handler_task_mode: HandlerTaskMode::Detached, @@ -4084,6 +4120,37 @@ impl ServiceManager { ) .await; } + + fn private_ip_for_external_address( + external_ip: IpAddr, + ip_config: &PrivateIpConfig, + kind: ZoneKind, + ) -> Result { + let maybe_private_ip = if external_ip.is_ipv6() { + ip_config.ipv6_addr().copied().map(IpAddr::V6) + } else { + ip_config.ipv4_addr().copied().map(IpAddr::V4) + }; + maybe_private_ip.ok_or_else(|| { + let external_ip_version = + if external_ip.is_ipv6() { "6" } else { "4" }; + let private_ip_stack = if ip_config.is_ipv4_only() { + "IPv4" + } else if ip_config.is_ipv6_only() { + "IPv6" + } else { + "dual-stack" + }; + Error::BadServiceRequest { + service: kind.report_str().to_string(), + message: format!( + "External IP address is IPv{}, but VPC-private \ + IP configuration is {}", + external_ip_version, private_ip_stack, + ), + } + }) + } } fn internal_dns_addrobj_name(gz_address_index: u32) -> String { diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index e90b916d0a1..0158f8778f8 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -46,6 +46,7 @@ use omicron_common::api::external::Generation; use omicron_common::api::external::MacAddr; use omicron_common::api::external::Vni; use omicron_common::api::internal::nexus::Certificate; +use omicron_common::api::internal::shared::PrivateIpConfig; use omicron_common::backoff::{ BackoffError, retry_notify, retry_policy_internal_service_aggressive, }; @@ -432,8 +433,14 @@ pub async fn run_standalone_server( let mut internal_services_ip_pool_ranges = vec![]; let mut macs = MacAddr::iter_system(); if let Some(nexus_external_addr) = rss_args.nexus_external_addr { - let ip = nexus_external_addr.ip(); + let external_ip = nexus_external_addr.ip(); let id = OmicronZoneUuid::new_v4(); + let private_ip = NEXUS_OPTE_IPV4_SUBNET + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) + .unwrap(); + let ip_config = + PrivateIpConfig::new_ipv4(private_ip, *NEXUS_OPTE_IPV4_SUBNET) + .context("creating private IP configuration")?; zones.insert(BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, @@ -444,23 +451,18 @@ pub async fn run_standalone_server( SocketAddr::V6(a) => a, }, lockstep_port: nexus_lockstep_port, - external_ip: from_ipaddr_to_external_floating_ip(ip), + external_ip: from_ipaddr_to_external_floating_ip(external_ip), nic: nexus_types::inventory::NetworkInterface { id: Uuid::new_v4(), kind: NetworkInterfaceKind::Service { id: id.into_untyped_uuid(), }, name: "nexus".parse().unwrap(), - ip: NEXUS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) - .unwrap() - .into(), + ip_config, mac: macs.next().unwrap(), - subnet: (*NEXUS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, external_tls: false, external_dns_servers: vec![], @@ -470,7 +472,7 @@ pub async fn run_standalone_server( image_source: BlueprintZoneImageSource::InstallDataset, }); - internal_services_ip_pool_ranges.push(match ip { + internal_services_ip_pool_ranges.push(match external_ip { IpAddr::V4(addr) => { IpRange::V4(Ipv4Range { first: addr, last: addr }) } @@ -485,6 +487,12 @@ pub async fn run_standalone_server( { let ip = *external_dns_internal_addr.ip(); let id = OmicronZoneUuid::new_v4(); + let private_ip = DNS_OPTE_IPV4_SUBNET + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) + .unwrap(); + let ip_config = + PrivateIpConfig::new_ipv4(private_ip, *DNS_OPTE_IPV4_SUBNET) + .context("creating private IP configuration")?; let pool_name = ZpoolName::new_external(ZpoolUuid::new_v4()); zones.insert(BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, @@ -502,16 +510,11 @@ pub async fn run_standalone_server( id: id.into_untyped_uuid(), }, name: "external-dns".parse().unwrap(), - ip: DNS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) - .unwrap() - .into(), + ip_config, mac: macs.next().unwrap(), - subnet: (*DNS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], }, }, ), diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 4648490b2a0..f9a835f60c7 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -338,13 +338,21 @@ impl SledAgent { let mut routes = self.vpc_routes.lock().unwrap(); for nic in &local_config.nics { - let my_routers = [ - RouterId { vni: nic.vni, kind: RouterKind::System }, - RouterId { vni: nic.vni, kind: RouterKind::Custom(nic.subnet) }, - ]; - - for router in my_routers { - routes.entry(router).or_default(); + let kinds = std::iter::once(RouterKind::System) + .chain( + nic.ip_config + .ipv4_subnet() + .copied() + .map(|subnet| RouterKind::Custom(subnet.into())), + ) + .chain( + nic.ip_config + .ipv6_subnet() + .copied() + .map(|subnet| RouterKind::Custom(subnet.into())), + ); + for kind in kinds { + routes.entry(RouterId { vni: nic.vni, kind }).or_default(); } } diff --git a/sled-agent/types/src/inventory/mod.rs b/sled-agent/types/src/inventory/mod.rs new file mode 100644 index 00000000000..16146e1c93d --- /dev/null +++ b/sled-agent/types/src/inventory/mod.rs @@ -0,0 +1,11 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Sled-agent types for inventory. +//! +//! This is intended to contain old versions of the inventory types. The current +//! version should be in `nexus-sled-agent-shared`. + +pub mod v3; +pub mod v6; diff --git a/sled-agent/api/src/v3.rs b/sled-agent/types/src/inventory/v3.rs similarity index 67% rename from sled-agent/api/src/v3.rs rename to sled-agent/types/src/inventory/v3.rs index a6476fe79b7..8581f2bbf69 100644 --- a/sled-agent/api/src/v3.rs +++ b/sled-agent/types/src/inventory/v3.rs @@ -2,32 +2,49 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::collections::BTreeMap; -use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; -use std::time::Duration; - -use chrono::{DateTime, Utc}; -use iddqd::{IdOrdItem, IdOrdMap, id_upcast}; -use nexus_sled_agent_shared::inventory::{ - self, BootPartitionContents, ConfigReconcilerInventoryResult, - HostPhase2DesiredSlots, InventoryDataset, InventoryDisk, InventoryZpool, - OmicronZoneDataset, OmicronZoneImageSource, OrphanedDataset, - RemoveMupdateOverrideInventory, SledRole, ZoneImageResolverInventory, -}; +use chrono::DateTime; +use chrono::Utc; +use iddqd::IdOrdItem; +use iddqd::IdOrdMap; +use iddqd::id_upcast; +use nexus_sled_agent_shared::inventory; +use nexus_sled_agent_shared::inventory::BootPartitionContents; +use nexus_sled_agent_shared::inventory::ConfigReconcilerInventoryResult; +use nexus_sled_agent_shared::inventory::HostPhase2DesiredSlots; +use nexus_sled_agent_shared::inventory::InventoryDataset; +use nexus_sled_agent_shared::inventory::InventoryDisk; +use nexus_sled_agent_shared::inventory::InventoryZpool; +use nexus_sled_agent_shared::inventory::OmicronZoneDataset; +use nexus_sled_agent_shared::inventory::OmicronZoneImageSource; +use nexus_sled_agent_shared::inventory::OrphanedDataset; +use nexus_sled_agent_shared::inventory::RemoveMupdateOverrideInventory; +use nexus_sled_agent_shared::inventory::SledRole; +use nexus_sled_agent_shared::inventory::ZoneImageResolverInventory; use omicron_common::address::NEXUS_LOCKSTEP_PORT; -use omicron_common::{ - api::external::{ByteCount, Generation}, - api::internal::shared::{NetworkInterface, SourceNatConfig}, - disk::{DatasetConfig, OmicronPhysicalDiskConfig}, - zpool_name::ZpoolName, -}; -use omicron_uuid_kinds::{ - DatasetUuid, MupdateOverrideUuid, OmicronZoneUuid, PhysicalDiskUuid, - SledUuid, -}; +use omicron_common::api::external; +use omicron_common::api::external::ByteCount; +use omicron_common::api::external::Generation; +use omicron_common::api::internal::shared::SourceNatConfig; +use omicron_common::api::internal::shared::network_interface::v1::NetworkInterface; +use omicron_common::disk::DatasetConfig; +use omicron_common::disk::OmicronPhysicalDiskConfig; +use omicron_common::zpool_name::ZpoolName; +use omicron_uuid_kinds::DatasetUuid; +use omicron_uuid_kinds::MupdateOverrideUuid; +use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::PhysicalDiskUuid; +use omicron_uuid_kinds::SledUuid; use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use sled_hardware_types::{Baseboard, SledCpuFamily}; +use serde::Deserialize; +use serde::Serialize; +use sled_hardware_types::Baseboard; +use sled_hardware_types::SledCpuFamily; +use std::collections::BTreeMap; +use std::net::IpAddr; +use std::net::Ipv6Addr; +use std::net::SocketAddr; +use std::net::SocketAddrV6; +use std::time::Duration; /// Identity and basic status information about this sled agent #[derive(Deserialize, Serialize, JsonSchema)] @@ -49,9 +66,20 @@ pub struct Inventory { pub zone_image_resolver: ZoneImageResolverInventory, } -impl From for Inventory { - fn from(value: inventory::Inventory) -> Self { - Self { +impl TryFrom for Inventory { + type Error = external::Error; + + fn try_from(value: inventory::Inventory) -> Result { + let ledgered_sled_config = value + .ledgered_sled_config + .map(OmicronSledConfig::try_from) + .transpose()?; + let reconciler_status = value.reconciler_status.try_into()?; + let last_reconciliation = value + .last_reconciliation + .map(ConfigReconcilerInventory::try_from) + .transpose()?; + Ok(Self { sled_id: value.sled_id, sled_agent_address: value.sled_agent_address, sled_role: value.sled_role, @@ -63,11 +91,11 @@ impl From for Inventory { disks: value.disks, zpools: value.zpools, datasets: value.datasets, - ledgered_sled_config: value.ledgered_sled_config.map(Into::into), - reconciler_status: value.reconciler_status.into(), - last_reconciliation: value.last_reconciliation.map(Into::into), + ledgered_sled_config, + reconciler_status, + last_reconciliation, zone_image_resolver: value.zone_image_resolver, - } + }) } } @@ -88,29 +116,45 @@ pub struct OmicronSledConfig { pub host_phase_2: HostPhase2DesiredSlots, } -impl From for inventory::OmicronSledConfig { - fn from(value: OmicronSledConfig) -> Self { - Self { +impl TryFrom for inventory::OmicronSledConfig { + type Error = external::Error; + + fn try_from(value: OmicronSledConfig) -> Result { + let zones = value + .zones + .into_iter() + .map(TryInto::try_into) + .collect::>()?; + Ok(Self { generation: value.generation, disks: value.disks, datasets: value.datasets, - zones: value.zones.into_iter().map(Into::into).collect(), + zones, remove_mupdate_override: value.remove_mupdate_override, host_phase_2: value.host_phase_2, - } + }) } } -impl From for OmicronSledConfig { - fn from(value: inventory::OmicronSledConfig) -> Self { - Self { +impl TryFrom for OmicronSledConfig { + type Error = external::Error; + + fn try_from( + value: inventory::OmicronSledConfig, + ) -> Result { + let zones = value + .zones + .into_iter() + .map(TryInto::try_into) + .collect::>()?; + Ok(Self { generation: value.generation, disks: value.disks, datasets: value.datasets, - zones: value.zones.into_iter().map(Into::into).collect(), + zones, remove_mupdate_override: value.remove_mupdate_override, host_phase_2: value.host_phase_2, - } + }) } } @@ -141,25 +185,31 @@ impl IdOrdItem for OmicronZoneConfig { id_upcast!(); } -impl From for inventory::OmicronZoneConfig { - fn from(value: OmicronZoneConfig) -> Self { - Self { +impl TryFrom for inventory::OmicronZoneConfig { + type Error = external::Error; + + fn try_from(value: OmicronZoneConfig) -> Result { + Ok(Self { id: value.id, filesystem_pool: value.filesystem_pool, - zone_type: value.zone_type.into(), + zone_type: value.zone_type.try_into()?, image_source: value.image_source, - } + }) } } -impl From for OmicronZoneConfig { - fn from(value: inventory::OmicronZoneConfig) -> Self { - Self { +impl TryFrom for OmicronZoneConfig { + type Error = external::Error; + + fn try_from( + value: inventory::OmicronZoneConfig, + ) -> Result { + Ok(Self { id: value.id, filesystem_pool: value.filesystem_pool, - zone_type: value.zone_type.into(), + zone_type: value.zone_type.try_into()?, image_source: value.image_source, - } + }) } } @@ -256,8 +306,10 @@ pub enum OmicronZoneType { }, } -impl From for inventory::OmicronZoneType { - fn from(value: OmicronZoneType) -> Self { +impl TryFrom for inventory::OmicronZoneType { + type Error = external::Error; + + fn try_from(value: OmicronZoneType) -> Result { match value { OmicronZoneType::BoundaryNtp { address, @@ -266,53 +318,58 @@ impl From for inventory::OmicronZoneType { domain, nic, snat_cfg, - } => Self::BoundaryNtp { + } => Ok(Self::BoundaryNtp { address, ntp_servers, dns_servers, domain, - nic, + nic: nic.try_into()?, snat_cfg, - }, + }), OmicronZoneType::Clickhouse { address, dataset } => { - Self::Clickhouse { address, dataset } + Ok(Self::Clickhouse { address, dataset }) } OmicronZoneType::ClickhouseKeeper { address, dataset } => { - Self::ClickhouseKeeper { address, dataset } + Ok(Self::ClickhouseKeeper { address, dataset }) } OmicronZoneType::ClickhouseServer { address, dataset } => { - Self::ClickhouseServer { address, dataset } + Ok(Self::ClickhouseServer { address, dataset }) } OmicronZoneType::CockroachDb { address, dataset } => { - Self::CockroachDb { address, dataset } + Ok(Self::CockroachDb { address, dataset }) } OmicronZoneType::Crucible { address, dataset } => { - Self::Crucible { address, dataset } + Ok(Self::Crucible { address, dataset }) } OmicronZoneType::CruciblePantry { address } => { - Self::CruciblePantry { address } + Ok(Self::CruciblePantry { address }) } OmicronZoneType::ExternalDns { dataset, http_address, dns_address, nic, - } => Self::ExternalDns { dataset, http_address, dns_address, nic }, + } => Ok(Self::ExternalDns { + dataset, + http_address, + dns_address, + nic: nic.try_into()?, + }), OmicronZoneType::InternalDns { dataset, http_address, dns_address, gz_address, gz_address_index, - } => Self::InternalDns { + } => Ok(Self::InternalDns { dataset, http_address, dns_address, gz_address, gz_address_index, - }, + }), OmicronZoneType::InternalNtp { address } => { - Self::InternalNtp { address } + Ok(Self::InternalNtp { address }) } OmicronZoneType::Nexus { internal_address, @@ -320,21 +377,27 @@ impl From for inventory::OmicronZoneType { nic, external_tls, external_dns_servers, - } => Self::Nexus { + } => Ok(Self::Nexus { internal_address, lockstep_port: NEXUS_LOCKSTEP_PORT, external_ip, - nic, + nic: nic.try_into()?, external_tls, external_dns_servers, - }, - OmicronZoneType::Oximeter { address } => Self::Oximeter { address }, + }), + OmicronZoneType::Oximeter { address } => { + Ok(Self::Oximeter { address }) + } } } } -impl From for OmicronZoneType { - fn from(value: inventory::OmicronZoneType) -> Self { +impl TryFrom for OmicronZoneType { + type Error = external::Error; + + fn try_from( + value: inventory::OmicronZoneType, + ) -> Result { match value { inventory::OmicronZoneType::BoundaryNtp { address, @@ -343,55 +406,60 @@ impl From for OmicronZoneType { domain, nic, snat_cfg, - } => Self::BoundaryNtp { + } => Ok(Self::BoundaryNtp { address, ntp_servers, dns_servers, domain, - nic, + nic: nic.try_into()?, snat_cfg, - }, + }), inventory::OmicronZoneType::Clickhouse { address, dataset } => { - Self::Clickhouse { address, dataset } + Ok(Self::Clickhouse { address, dataset }) } inventory::OmicronZoneType::ClickhouseKeeper { address, dataset, - } => Self::ClickhouseKeeper { address, dataset }, + } => Ok(Self::ClickhouseKeeper { address, dataset }), inventory::OmicronZoneType::ClickhouseServer { address, dataset, - } => Self::ClickhouseServer { address, dataset }, + } => Ok(Self::ClickhouseServer { address, dataset }), inventory::OmicronZoneType::CockroachDb { address, dataset } => { - Self::CockroachDb { address, dataset } + Ok(Self::CockroachDb { address, dataset }) } inventory::OmicronZoneType::Crucible { address, dataset } => { - Self::Crucible { address, dataset } + Ok(Self::Crucible { address, dataset }) } inventory::OmicronZoneType::CruciblePantry { address } => { - Self::CruciblePantry { address } + Ok(Self::CruciblePantry { address }) } inventory::OmicronZoneType::ExternalDns { dataset, http_address, dns_address, nic, - } => Self::ExternalDns { dataset, http_address, dns_address, nic }, + } => Ok(Self::ExternalDns { + dataset, + http_address, + dns_address, + nic: nic.try_into()?, + }), inventory::OmicronZoneType::InternalDns { dataset, http_address, dns_address, gz_address, gz_address_index, - } => Self::InternalDns { + } => Ok(Self::InternalDns { dataset, http_address, dns_address, gz_address, gz_address_index, - }, + }), inventory::OmicronZoneType::InternalNtp { address } => { - Self::InternalNtp { address } + Ok(Self::InternalNtp { address }) } inventory::OmicronZoneType::Nexus { internal_address, @@ -400,15 +468,15 @@ impl From for OmicronZoneType { nic, external_tls, external_dns_servers, - } => Self::Nexus { + } => Ok(Self::Nexus { internal_address, external_ip, - nic, + nic: nic.try_into()?, external_tls, external_dns_servers, - }, + }), inventory::OmicronZoneType::Oximeter { address } => { - Self::Oximeter { address } + Ok(Self::Oximeter { address }) } } } @@ -432,17 +500,23 @@ pub struct ConfigReconcilerInventory { pub remove_mupdate_override: Option, } -impl From for ConfigReconcilerInventory { - fn from(value: inventory::ConfigReconcilerInventory) -> Self { - Self { - last_reconciled_config: value.last_reconciled_config.into(), +impl TryFrom + for ConfigReconcilerInventory +{ + type Error = external::Error; + + fn try_from( + value: inventory::ConfigReconcilerInventory, + ) -> Result { + Ok(Self { + last_reconciled_config: value.last_reconciled_config.try_into()?, external_disks: value.external_disks, datasets: value.datasets, orphaned_datasets: value.orphaned_datasets, zones: value.zones, boot_partitions: value.boot_partitions, remove_mupdate_override: value.remove_mupdate_override, - } + }) } } @@ -468,27 +542,31 @@ pub enum ConfigReconcilerInventoryStatus { Idle { completed_at: DateTime, ran_for: Duration }, } -impl From +impl TryFrom for ConfigReconcilerInventoryStatus { - fn from(value: inventory::ConfigReconcilerInventoryStatus) -> Self { + type Error = external::Error; + + fn try_from( + value: inventory::ConfigReconcilerInventoryStatus, + ) -> Result { match value { inventory::ConfigReconcilerInventoryStatus::NotYetRun => { - Self::NotYetRun + Ok(Self::NotYetRun) } inventory::ConfigReconcilerInventoryStatus::Running { config, started_at, running_for, - } => Self::Running { - config: Box::new((*config).into()), + } => Ok(Self::Running { + config: Box::new((*config).try_into()?), started_at, running_for, - }, + }), inventory::ConfigReconcilerInventoryStatus::Idle { completed_at, ran_for, - } => Self::Idle { completed_at, ran_for }, + } => Ok(Self::Idle { completed_at, ran_for }), } } } diff --git a/sled-agent/types/src/inventory/v6.rs b/sled-agent/types/src/inventory/v6.rs new file mode 100644 index 00000000000..8e504cdedc3 --- /dev/null +++ b/sled-agent/types/src/inventory/v6.rs @@ -0,0 +1,770 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Sled-agent API types that changed from v6 to v7. + +use crate::instance::InstanceMetadata; +use crate::instance::VmmSpec; +use chrono::DateTime; +use chrono::Utc; +use iddqd::IdOrdItem; +use iddqd::IdOrdMap; +use iddqd::id_upcast; +use nexus_sled_agent_shared::inventory; +use nexus_sled_agent_shared::inventory::BootPartitionContents; +use nexus_sled_agent_shared::inventory::ConfigReconcilerInventoryResult; +use nexus_sled_agent_shared::inventory::HostPhase2DesiredSlots; +use nexus_sled_agent_shared::inventory::InventoryDataset; +use nexus_sled_agent_shared::inventory::InventoryDisk; +use nexus_sled_agent_shared::inventory::InventoryZpool; +use nexus_sled_agent_shared::inventory::OmicronZoneDataset; +use nexus_sled_agent_shared::inventory::OmicronZoneImageSource; +use nexus_sled_agent_shared::inventory::OrphanedDataset; +use nexus_sled_agent_shared::inventory::RemoveMupdateOverrideInventory; +use nexus_sled_agent_shared::inventory::SledRole; +use nexus_sled_agent_shared::inventory::ZoneImageResolverInventory; +use omicron_common::api::external; +use omicron_common::api::external::ByteCount; +use omicron_common::api::external::Generation; +use omicron_common::api::external::Hostname; +use omicron_common::api::internal::nexus::{HostIdentifier, VmmRuntimeState}; +use omicron_common::api::internal::shared::DhcpConfig; +use omicron_common::api::internal::shared::SourceNatConfig; +use omicron_common::api::internal::shared::network_interface::v1::NetworkInterface; +use omicron_common::disk::DatasetConfig; +use omicron_common::disk::OmicronPhysicalDiskConfig; +use omicron_common::zpool_name::ZpoolName; +use omicron_uuid_kinds::DatasetUuid; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::MupdateOverrideUuid; +use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::PhysicalDiskUuid; +use omicron_uuid_kinds::SledUuid; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use sled_hardware_types::Baseboard; +use sled_hardware_types::SledCpuFamily; +use std::collections::BTreeMap; +use std::collections::HashSet; +use std::net::IpAddr; +use std::net::Ipv6Addr; +use std::net::SocketAddr; +use std::net::SocketAddrV6; +use std::time::Duration; +use uuid::Uuid; + +/// Identity and basic status information about this sled agent +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct Inventory { + pub sled_id: SledUuid, + pub sled_agent_address: SocketAddrV6, + pub sled_role: SledRole, + pub baseboard: Baseboard, + pub usable_hardware_threads: u32, + pub usable_physical_ram: ByteCount, + pub cpu_family: SledCpuFamily, + pub reservoir_size: ByteCount, + pub disks: Vec, + pub zpools: Vec, + pub datasets: Vec, + pub ledgered_sled_config: Option, + pub reconciler_status: ConfigReconcilerInventoryStatus, + pub last_reconciliation: Option, + pub zone_image_resolver: ZoneImageResolverInventory, +} + +impl TryFrom for Inventory { + type Error = external::Error; + + fn try_from(value: inventory::Inventory) -> Result { + let ledgered_sled_config = + value.ledgered_sled_config.map(TryInto::try_into).transpose()?; + let reconciler_status = value.reconciler_status.try_into()?; + let last_reconciliation = + value.last_reconciliation.map(TryInto::try_into).transpose()?; + Ok(Self { + sled_id: value.sled_id, + sled_agent_address: value.sled_agent_address, + sled_role: value.sled_role, + baseboard: value.baseboard, + usable_hardware_threads: value.usable_hardware_threads, + usable_physical_ram: value.usable_physical_ram, + cpu_family: value.cpu_family, + reservoir_size: value.reservoir_size, + disks: value.disks, + zpools: value.zpools, + datasets: value.datasets, + ledgered_sled_config, + reconciler_status, + last_reconciliation, + zone_image_resolver: value.zone_image_resolver, + }) + } +} + +/// Describes the set of Reconfigurator-managed configuration elements of a sled +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct OmicronSledConfig { + pub generation: Generation, + #[serde( + with = "iddqd::id_ord_map::IdOrdMapAsMap::" + )] + pub disks: IdOrdMap, + #[serde(with = "iddqd::id_ord_map::IdOrdMapAsMap::")] + pub datasets: IdOrdMap, + #[serde(with = "iddqd::id_ord_map::IdOrdMapAsMap::")] + pub zones: IdOrdMap, + pub remove_mupdate_override: Option, + #[serde(default = "HostPhase2DesiredSlots::current_contents")] + pub host_phase_2: HostPhase2DesiredSlots, +} + +impl TryFrom for inventory::OmicronSledConfig { + type Error = external::Error; + + fn try_from(value: OmicronSledConfig) -> Result { + let zones = value + .zones + .into_iter() + .map(TryInto::try_into) + .collect::>()?; + Ok(Self { + generation: value.generation, + disks: value.disks, + datasets: value.datasets, + zones, + remove_mupdate_override: value.remove_mupdate_override, + host_phase_2: value.host_phase_2, + }) + } +} + +impl TryFrom for OmicronSledConfig { + type Error = external::Error; + fn try_from( + value: inventory::OmicronSledConfig, + ) -> Result { + let zones = value + .zones + .into_iter() + .map(TryInto::try_into) + .collect::>()?; + Ok(Self { + generation: value.generation, + disks: value.disks, + datasets: value.datasets, + zones, + remove_mupdate_override: value.remove_mupdate_override, + host_phase_2: value.host_phase_2, + }) + } +} + +/// Describes one Omicron-managed zone running on a sled +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct OmicronZoneConfig { + pub id: OmicronZoneUuid, + + /// The pool on which we'll place this zone's root filesystem. + /// + /// Note that the root filesystem is transient -- the sled agent is + /// permitted to destroy this dataset each time the zone is initialized. + pub filesystem_pool: Option, + pub zone_type: OmicronZoneType, + // Use `InstallDataset` if this field is not present in a deserialized + // blueprint or ledger. + #[serde(default = "OmicronZoneImageSource::deserialize_default")] + pub image_source: OmicronZoneImageSource, +} + +impl IdOrdItem for OmicronZoneConfig { + type Key<'a> = OmicronZoneUuid; + + fn key(&self) -> Self::Key<'_> { + self.id + } + + id_upcast!(); +} + +impl TryFrom for inventory::OmicronZoneConfig { + type Error = external::Error; + + fn try_from(value: OmicronZoneConfig) -> Result { + Ok(Self { + id: value.id, + filesystem_pool: value.filesystem_pool, + zone_type: value.zone_type.try_into()?, + image_source: value.image_source, + }) + } +} + +impl TryFrom for OmicronZoneConfig { + type Error = external::Error; + + fn try_from( + value: inventory::OmicronZoneConfig, + ) -> Result { + Ok(Self { + id: value.id, + filesystem_pool: value.filesystem_pool, + zone_type: value.zone_type.try_into()?, + image_source: value.image_source, + }) + } +} + +/// Describes the set of Omicron-managed zones running on a sled +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct OmicronZonesConfig { + /// generation number of this configuration + /// + /// This generation number is owned by the control plane (i.e., RSS or + /// Nexus, depending on whether RSS-to-Nexus handoff has happened). It + /// should not be bumped within Sled Agent. + /// + /// Sled Agent rejects attempts to set the configuration to a generation + /// older than the one it's currently running. + pub generation: Generation, + + /// list of running zones + pub zones: Vec, +} + +impl TryFrom for inventory::OmicronZonesConfig { + type Error = external::Error; + + fn try_from(value: OmicronZonesConfig) -> Result { + value + .zones + .into_iter() + .map(TryInto::try_into) + .collect::>() + .map(|zones| inventory::OmicronZonesConfig { + generation: value.generation, + zones, + }) + } +} + +/// Describes what kind of zone this is (i.e., what component is running in it) +/// as well as any type-specific configuration +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum OmicronZoneType { + BoundaryNtp { + address: SocketAddrV6, + ntp_servers: Vec, + dns_servers: Vec, + domain: Option, + /// The service vNIC providing outbound connectivity using OPTE. + nic: NetworkInterface, + /// The SNAT configuration for outbound connections. + snat_cfg: SourceNatConfig, + }, + + /// Type of clickhouse zone used for a single node clickhouse deployment + Clickhouse { + address: SocketAddrV6, + dataset: OmicronZoneDataset, + }, + + /// A zone used to run a Clickhouse Keeper node + /// + /// Keepers are only used in replicated clickhouse setups + ClickhouseKeeper { + address: SocketAddrV6, + dataset: OmicronZoneDataset, + }, + + /// A zone used to run a Clickhouse Server in a replicated deployment + ClickhouseServer { + address: SocketAddrV6, + dataset: OmicronZoneDataset, + }, + + CockroachDb { + address: SocketAddrV6, + dataset: OmicronZoneDataset, + }, + + Crucible { + address: SocketAddrV6, + dataset: OmicronZoneDataset, + }, + CruciblePantry { + address: SocketAddrV6, + }, + ExternalDns { + dataset: OmicronZoneDataset, + /// The address at which the external DNS server API is reachable. + http_address: SocketAddrV6, + /// The address at which the external DNS server is reachable. + dns_address: SocketAddr, + /// The service vNIC providing external connectivity using OPTE. + nic: NetworkInterface, + }, + InternalDns { + dataset: OmicronZoneDataset, + http_address: SocketAddrV6, + dns_address: SocketAddrV6, + /// The addresses in the global zone which should be created + /// + /// For the DNS service, which exists outside the sleds's typical subnet + /// - adding an address in the GZ is necessary to allow inter-zone + /// traffic routing. + gz_address: Ipv6Addr, + + /// The address is also identified with an auxiliary bit of information + /// to ensure that the created global zone address can have a unique + /// name. + gz_address_index: u32, + }, + InternalNtp { + address: SocketAddrV6, + }, + Nexus { + /// The address at which the internal nexus server is reachable. + internal_address: SocketAddrV6, + /// The port at which the internal lockstep server is reachable. This + /// shares the same IP address with `internal_address`. + #[serde(default = "default_nexus_lockstep_port")] + lockstep_port: u16, + /// The address at which the external nexus server is reachable. + external_ip: IpAddr, + /// The service vNIC providing external connectivity using OPTE. + nic: NetworkInterface, + /// Whether Nexus's external endpoint should use TLS + external_tls: bool, + /// External DNS servers Nexus can use to resolve external hosts. + external_dns_servers: Vec, + }, + Oximeter { + address: SocketAddrV6, + }, +} + +const fn default_nexus_lockstep_port() -> u16 { + omicron_common::address::NEXUS_LOCKSTEP_PORT +} + +impl TryFrom for inventory::OmicronZoneType { + type Error = external::Error; + + fn try_from(value: OmicronZoneType) -> Result { + match value { + OmicronZoneType::BoundaryNtp { + address, + ntp_servers, + dns_servers, + domain, + nic, + snat_cfg, + } => Ok(Self::BoundaryNtp { + address, + ntp_servers, + dns_servers, + domain, + nic: nic.try_into()?, + snat_cfg, + }), + OmicronZoneType::Clickhouse { address, dataset } => { + Ok(Self::Clickhouse { address, dataset }) + } + OmicronZoneType::ClickhouseKeeper { address, dataset } => { + Ok(Self::ClickhouseKeeper { address, dataset }) + } + OmicronZoneType::ClickhouseServer { address, dataset } => { + Ok(Self::ClickhouseServer { address, dataset }) + } + OmicronZoneType::CockroachDb { address, dataset } => { + Ok(Self::CockroachDb { address, dataset }) + } + OmicronZoneType::Crucible { address, dataset } => { + Ok(Self::Crucible { address, dataset }) + } + OmicronZoneType::CruciblePantry { address } => { + Ok(Self::CruciblePantry { address }) + } + OmicronZoneType::ExternalDns { + dataset, + http_address, + dns_address, + nic, + } => Ok(Self::ExternalDns { + dataset, + http_address, + dns_address, + nic: nic.try_into()?, + }), + OmicronZoneType::InternalDns { + dataset, + http_address, + dns_address, + gz_address, + gz_address_index, + } => Ok(Self::InternalDns { + dataset, + http_address, + dns_address, + gz_address, + gz_address_index, + }), + OmicronZoneType::InternalNtp { address } => { + Ok(Self::InternalNtp { address }) + } + OmicronZoneType::Nexus { + internal_address, + lockstep_port, + external_ip, + nic, + external_tls, + external_dns_servers, + } => Ok(Self::Nexus { + internal_address, + lockstep_port, + external_ip, + nic: nic.try_into()?, + external_tls, + external_dns_servers, + }), + OmicronZoneType::Oximeter { address } => { + Ok(Self::Oximeter { address }) + } + } + } +} + +impl TryFrom for OmicronZoneType { + type Error = external::Error; + + fn try_from( + value: inventory::OmicronZoneType, + ) -> Result { + match value { + inventory::OmicronZoneType::BoundaryNtp { + address, + ntp_servers, + dns_servers, + domain, + nic, + snat_cfg, + } => Ok(Self::BoundaryNtp { + address, + ntp_servers, + dns_servers, + domain, + nic: nic.try_into()?, + snat_cfg, + }), + inventory::OmicronZoneType::Clickhouse { address, dataset } => { + Ok(Self::Clickhouse { address, dataset }) + } + inventory::OmicronZoneType::ClickhouseKeeper { + address, + dataset, + } => Ok(Self::ClickhouseKeeper { address, dataset }), + inventory::OmicronZoneType::ClickhouseServer { + address, + dataset, + } => Ok(Self::ClickhouseServer { address, dataset }), + inventory::OmicronZoneType::CockroachDb { address, dataset } => { + Ok(Self::CockroachDb { address, dataset }) + } + inventory::OmicronZoneType::Crucible { address, dataset } => { + Ok(Self::Crucible { address, dataset }) + } + inventory::OmicronZoneType::CruciblePantry { address } => { + Ok(Self::CruciblePantry { address }) + } + inventory::OmicronZoneType::ExternalDns { + dataset, + http_address, + dns_address, + nic, + } => Ok(Self::ExternalDns { + dataset, + http_address, + dns_address, + nic: nic.try_into()?, + }), + inventory::OmicronZoneType::InternalDns { + dataset, + http_address, + dns_address, + gz_address, + gz_address_index, + } => Ok(Self::InternalDns { + dataset, + http_address, + dns_address, + gz_address, + gz_address_index, + }), + inventory::OmicronZoneType::InternalNtp { address } => { + Ok(Self::InternalNtp { address }) + } + inventory::OmicronZoneType::Nexus { + internal_address, + lockstep_port, + external_ip, + nic, + external_tls, + external_dns_servers, + } => Ok(Self::Nexus { + internal_address, + lockstep_port, + external_ip, + nic: nic.try_into()?, + external_tls, + external_dns_servers, + }), + inventory::OmicronZoneType::Oximeter { address } => { + Ok(Self::Oximeter { address }) + } + } + } +} + +/// Describes the last attempt made by the sled-agent-config-reconciler to +/// reconcile the current sled config against the actual state of the sled. +#[derive(Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct ConfigReconcilerInventory { + pub last_reconciled_config: OmicronSledConfig, + pub external_disks: + BTreeMap, + pub datasets: BTreeMap, + pub orphaned_datasets: IdOrdMap, + pub zones: BTreeMap, + pub boot_partitions: BootPartitionContents, + /// The result of removing the mupdate override file on disk. + /// + /// `None` if `remove_mupdate_override` was not provided in the sled config. + pub remove_mupdate_override: Option, +} + +impl TryFrom + for ConfigReconcilerInventory +{ + type Error = external::Error; + + fn try_from( + value: inventory::ConfigReconcilerInventory, + ) -> Result { + Ok(Self { + last_reconciled_config: value.last_reconciled_config.try_into()?, + external_disks: value.external_disks, + datasets: value.datasets, + orphaned_datasets: value.orphaned_datasets, + zones: value.zones, + boot_partitions: value.boot_partitions, + remove_mupdate_override: value.remove_mupdate_override, + }) + } +} + +/// Status of the sled-agent-config-reconciler task. +#[derive(Deserialize, Serialize, JsonSchema)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum ConfigReconcilerInventoryStatus { + /// The reconciler task has not yet run for the first time since sled-agent + /// started. + NotYetRun, + /// The reconciler task is actively running. + Running { + config: Box, + started_at: DateTime, + running_for: Duration, + }, + /// The reconciler task is currently idle, but previously did complete a + /// reconciliation attempt. + /// + /// This variant does not include the `OmicronSledConfig` used in the last + /// attempt, because that's always available via + /// [`ConfigReconcilerInventory::last_reconciled_config`]. + Idle { completed_at: DateTime, ran_for: Duration }, +} + +impl TryFrom + for ConfigReconcilerInventoryStatus +{ + type Error = external::Error; + + fn try_from( + value: inventory::ConfigReconcilerInventoryStatus, + ) -> Result { + match value { + inventory::ConfigReconcilerInventoryStatus::NotYetRun => { + Ok(Self::NotYetRun) + } + inventory::ConfigReconcilerInventoryStatus::Running { + config, + started_at, + running_for, + } => Ok(Self::Running { + config: Box::new((*config).try_into()?), + started_at, + running_for, + }), + inventory::ConfigReconcilerInventoryStatus::Idle { + completed_at, + ran_for, + } => Ok(Self::Idle { completed_at, ran_for }), + } + } +} + +/// The body of a request to ensure that a instance and VMM are known to a sled +/// agent. +#[derive(Serialize, Deserialize, JsonSchema)] +pub struct InstanceEnsureBody { + /// The virtual hardware configuration this virtual machine should have when + /// it is started. + pub vmm_spec: VmmSpec, + + /// Information about the sled-local configuration that needs to be + /// established to make the VM's virtual hardware fully functional. + pub local_config: InstanceSledLocalConfig, + + /// The initial VMM runtime state for the VMM being registered. + pub vmm_runtime: VmmRuntimeState, + + /// The ID of the instance for which this VMM is being created. + pub instance_id: InstanceUuid, + + /// The ID of the migration in to this VMM, if this VMM is being + /// ensured is part of a migration in. If this is `None`, the VMM is not + /// being created due to a migration. + pub migration_id: Option, + + /// The address at which this VMM should serve a Propolis server API. + pub propolis_addr: SocketAddr, + + /// Metadata used to track instance statistics. + pub metadata: InstanceMetadata, +} + +impl TryFrom for crate::instance::InstanceEnsureBody { + type Error = external::Error; + + fn try_from(value: InstanceEnsureBody) -> Result { + let local_config = value.local_config.try_into()?; + Ok(Self { + vmm_spec: value.vmm_spec, + local_config, + vmm_runtime: value.vmm_runtime, + instance_id: value.instance_id, + migration_id: value.migration_id, + propolis_addr: value.propolis_addr, + metadata: value.metadata, + }) + } +} + +/// Describes sled-local configuration that a sled-agent must establish to make +/// the instance's virtual hardware fully functional. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct InstanceSledLocalConfig { + pub hostname: Hostname, + pub nics: Vec, + pub source_nat: SourceNatConfig, + /// Zero or more external IP addresses (either floating or ephemeral), + /// provided to an instance to allow inbound connectivity. + pub ephemeral_ip: Option, + pub floating_ips: Vec, + pub firewall_rules: Vec, + pub dhcp_config: DhcpConfig, +} + +impl TryFrom + for crate::instance::InstanceSledLocalConfig +{ + type Error = external::Error; + + fn try_from(value: InstanceSledLocalConfig) -> Result { + let nics = value + .nics + .into_iter() + .map(TryInto::try_into) + .collect::>()?; + let firewall_rules = value + .firewall_rules + .into_iter() + .map(TryInto::try_into) + .collect::>()?; + Ok(Self { + hostname: value.hostname, + nics, + source_nat: value.source_nat, + ephemeral_ip: value.ephemeral_ip, + floating_ips: value.floating_ips, + firewall_rules, + dhcp_config: value.dhcp_config, + }) + } +} + +/// VPC firewall rule after object name resolution has been performed by Nexus +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)] +pub struct ResolvedVpcFirewallRule { + pub status: external::VpcFirewallRuleStatus, + pub direction: external::VpcFirewallRuleDirection, + pub targets: Vec, + pub filter_hosts: Option>, + pub filter_ports: Option>, + pub filter_protocols: Option>, + pub action: external::VpcFirewallRuleAction, + pub priority: external::VpcFirewallRulePriority, +} + +impl TryFrom + for omicron_common::api::internal::shared::ResolvedVpcFirewallRule +{ + type Error = external::Error; + + fn try_from(value: ResolvedVpcFirewallRule) -> Result { + let targets = value + .targets + .into_iter() + .map(TryInto::try_into) + .collect::>()?; + Ok(Self { + status: value.status, + direction: value.direction, + targets, + filter_hosts: value.filter_hosts, + filter_ports: value.filter_ports, + filter_protocols: value.filter_protocols, + action: value.action, + priority: value.priority, + }) + } +} + +/// Update firewall rules for a VPC +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct VpcFirewallRulesEnsureBody { + pub vni: external::Vni, + pub rules: Vec, +} + +impl TryFrom + for crate::firewall_rules::VpcFirewallRulesEnsureBody +{ + type Error = external::Error; + + fn try_from( + value: VpcFirewallRulesEnsureBody, + ) -> Result { + let rules = value + .rules + .into_iter() + .map(TryInto::try_into) + .collect::>()?; + Ok(Self { vni: value.vni, rules }) + } +} diff --git a/sled-agent/types/src/lib.rs b/sled-agent/types/src/lib.rs index 90679b09661..8ea8bd70e9e 100644 --- a/sled-agent/types/src/lib.rs +++ b/sled-agent/types/src/lib.rs @@ -10,6 +10,7 @@ pub mod disk; pub mod early_networking; pub mod firewall_rules; pub mod instance; +pub mod inventory; pub mod probes; pub mod rack_init; pub mod rack_ops; diff --git a/sled-agent/types/src/probes.rs b/sled-agent/types/src/probes/mod.rs similarity index 97% rename from sled-agent/types/src/probes.rs rename to sled-agent/types/src/probes/mod.rs index 21a4b217bfa..203d05665b4 100644 --- a/sled-agent/types/src/probes.rs +++ b/sled-agent/types/src/probes/mod.rs @@ -14,6 +14,9 @@ use serde::Deserialize; use serde::Serialize; use std::net::IpAddr; +/// Version 1 of the probe API types. +pub mod v1; + /// Parameters used to create a probe. #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] pub struct ProbeCreate { diff --git a/sled-agent/types/src/probes/v1.rs b/sled-agent/types/src/probes/v1.rs new file mode 100644 index 00000000000..38284e6f7d8 --- /dev/null +++ b/sled-agent/types/src/probes/v1.rs @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version 1 of types for manipulating networking probe zones. + +use iddqd::IdHashItem; +use iddqd::IdHashMap; +use iddqd::id_upcast; +use omicron_common::api::external; +use omicron_common::api::internal::shared::network_interface::v1::NetworkInterface; +use omicron_uuid_kinds::ProbeUuid; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; + +// Re-export types that haven't changed. +pub use super::ExternalIp; +pub use super::IpKind; + +/// Parameters used to create a probe. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct ProbeCreate { + /// The ID for the probe. + pub id: ProbeUuid, + /// The external IP addresses assigned to the probe. + pub external_ips: Vec, + /// The probe's networking interface. + pub interface: NetworkInterface, +} + +impl IdHashItem for ProbeCreate { + type Key<'a> = ProbeUuid; + + fn key(&self) -> Self::Key<'_> { + self.id + } + + id_upcast!(); +} + +impl TryFrom for super::ProbeCreate { + type Error = external::Error; + + fn try_from(value: ProbeCreate) -> Result { + value.interface.try_into().map(|interface| Self { + id: value.id, + external_ips: value.external_ips, + interface, + }) + } +} + +/// A set of probes that the target sled should run. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct ProbeSet { + /// The exact set of probes to run. + pub probes: IdHashMap, +} + +impl TryFrom for super::ProbeSet { + type Error = external::Error; + + fn try_from(value: ProbeSet) -> Result { + value + .probes + .into_iter() + .map(TryInto::try_into) + .collect::>() + .map(|probes| Self { probes }) + } +} From 307abbdfbaaf56c433be16a6cecb628d4a26206b Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Fri, 14 Nov 2025 21:46:12 +0000 Subject: [PATCH 2/5] Add more OPTE tests, fixup gateway IP computation --- illumos-utils/src/opte/port_manager.rs | 356 ++++++++++++++++++++++++- 1 file changed, 349 insertions(+), 7 deletions(-) diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 4471a16704b..a9fb77360c1 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -123,6 +123,32 @@ pub struct PortCreateParams<'a> { pub dhcp_config: DhcpCfg, } +impl<'a> PortCreateParams<'a> { + fn ensure_no_external_ipv6_addresses(&self) -> Result<(), Error> { + self.ensure_no_external_addresses(IpAddr::is_ipv6) + } + + fn ensure_no_external_ipv4_addresses(&self) -> Result<(), Error> { + self.ensure_no_external_addresses(IpAddr::is_ipv4) + } + + fn ensure_no_external_addresses(&self, f: F) -> Result<(), Error> + where + F: Fn(&IpAddr) -> bool, + { + if self.source_nat.map(|snat| f(&snat.ip)).unwrap_or(false) { + return Err(Error::InvalidPortIpConfig); + } + if self.ephemeral_ip.as_ref().map(&f).unwrap_or(false) { + return Err(Error::InvalidPortIpConfig); + } + if self.floating_ips.iter().any(f) { + return Err(Error::InvalidPortIpConfig); + } + Ok(()) + } +} + impl<'a> TryFrom<&PortCreateParams<'a>> for IpCfg { type Error = Error; @@ -137,9 +163,12 @@ impl<'a> TryFrom<&PortCreateParams<'a>> for IpCfg { // VPC-private configuration for each IP stack. That is, if we have an // IPv4 external Ephemeral IP, we also have an IPv4 VPC-private address. let v4 = match ip_config.ipv4_config() { - None => None, + None => { + params.ensure_no_external_ipv4_addresses()?; + None + } Some(ipv4_config) => { - let gateway_ip = ipv4_config.subnet().first_addr().into(); + let gateway_ip = ipv4_config.subnet().first_host().into(); let vpc_subnet = Ipv4Cidr::from(Ipv4Network::from(*ipv4_config.subnet())); let private_ip = (*ipv4_config.ip()).into(); @@ -162,9 +191,17 @@ impl<'a> TryFrom<&PortCreateParams<'a>> for IpCfg { // if we have an external IPv6 address, we also have a VPC-private IPv6 // address to translate that into. let v6 = match ip_config.ipv6_config() { - None => None, + None => { + params.ensure_no_external_ipv6_addresses()?; + None + } Some(ipv6_config) => { - let gateway_ip = ipv6_config.subnet().first_addr().into(); + let gateway_ip = ipv6_config + .subnet() + .iter() + .nth(1) + .expect("must have at least 2 addresses") + .into(); let vpc_subnet = Ipv6Cidr::from(Ipv6Network::from(*ipv6_config.subnet())); let private_ip = (*ipv6_config.ip()).into(); @@ -1015,12 +1052,13 @@ mod tests { external::{MacAddr, Vni}, internal::shared::{ InternetGatewayRouterTarget, NetworkInterface, - NetworkInterfaceKind, PrivateIpConfig, ResolvedVpcRoute, - ResolvedVpcRouteSet, RouterTarget, RouterVersion, SourceNatConfig, + NetworkInterfaceKind, PrivateIpConfig, PrivateIpv4Config, + PrivateIpv6Config, ResolvedVpcRoute, ResolvedVpcRouteSet, + RouterTarget, RouterVersion, SourceNatConfig, }, }; use omicron_test_utils::dev::test_setup_log; - use oxide_vpc::api::DhcpCfg; + use oxide_vpc::api::{DhcpCfg, IpCfg, Ipv4Cidr, Ipv6Cidr}; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use std::{ collections::HashSet, @@ -1401,4 +1439,308 @@ mod tests { logctx.cleanup_successful(); } + + #[test] + fn ip_cfg_from_ipv4_params() { + let priv_ip = Ipv4Addr::new(172, 30, 2, 5); + let priv_subnet = + Ipv4Net::new(Ipv4Addr::new(172, 30, 2, 0), 24).unwrap(); + let ip_config = + PrivateIpConfig::new_ipv4(priv_ip, priv_subnet).unwrap(); + let mac = "a8:40:25:ff:ff:ff".parse().unwrap(); + let ext_ip = Ipv4Addr::new(10, 151, 2, 169); + let nic = NetworkInterface { + id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Instance { id: Uuid::new_v4() }, + name: "opte0".parse().unwrap(), + ip_config, + mac, + vni: 100.try_into().unwrap(), + primary: true, + slot: 0, + }; + let source_nat = + SourceNatConfig::new(IpAddr::V4(ext_ip), 0, 16383).unwrap(); + let prs = PortCreateParams { + nic: &nic, + source_nat: Some(source_nat), + ephemeral_ip: None, + floating_ips: &[], + firewall_rules: &[], + dhcp_config: DhcpCfg { + hostname: None, + host_domain: None, + domain_search_list: vec![], + dns4_servers: vec![], + dns6_servers: vec![], + }, + }; + let IpCfg::Ipv4(oxide_vpc::api::Ipv4Cfg { + vpc_subnet, + private_ip, + gateway_ip, + external_ips: + oxide_vpc::api::ExternalIpCfg { snat, ephemeral_ip, floating_ips }, + }) = IpCfg::try_from(&prs).unwrap() + else { + panic!("Expected IPv4 config") + }; + + assert_eq!(private_ip, priv_ip.into()); + assert_eq!( + vpc_subnet, + Ipv4Cidr::new( + priv_subnet.network().unwrap().into(), + priv_subnet.width().try_into().unwrap() + ) + ); + assert_eq!(gateway_ip, priv_subnet.first_host().into()); + let oxide_vpc::api::SNatCfg { external_ip, ports } = + snat.expect("SNAT config for this port should be Some(_)"); + assert_eq!(external_ip, ext_ip.into()); + assert_eq!(ports, source_nat.port_range()); + assert!(ephemeral_ip.is_none()); + assert!(floating_ips.is_empty()); + } + + #[test] + fn ip_cfg_from_ipv6_params() { + let priv_ip = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 5); + let priv_subnet = + Ipv6Net::new(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0), 64) + .unwrap(); + let ip_config = + PrivateIpConfig::new_ipv6(priv_ip, priv_subnet).unwrap(); + let mac = "a8:40:25:ff:ff:ff".parse().unwrap(); + let ext_ip = Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1); + let nic = NetworkInterface { + id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Instance { id: Uuid::new_v4() }, + name: "opte0".parse().unwrap(), + ip_config, + mac, + vni: 100.try_into().unwrap(), + primary: true, + slot: 0, + }; + let source_nat = + SourceNatConfig::new(IpAddr::V6(ext_ip), 0, 16383).unwrap(); + let prs = PortCreateParams { + nic: &nic, + source_nat: Some(source_nat), + ephemeral_ip: None, + floating_ips: &[], + firewall_rules: &[], + dhcp_config: DhcpCfg { + hostname: None, + host_domain: None, + domain_search_list: vec![], + dns4_servers: vec![], + dns6_servers: vec![], + }, + }; + let IpCfg::Ipv6(oxide_vpc::api::Ipv6Cfg { + vpc_subnet, + private_ip, + gateway_ip, + external_ips: + oxide_vpc::api::ExternalIpCfg { snat, ephemeral_ip, floating_ips }, + }) = IpCfg::try_from(&prs).unwrap() + else { + panic!("Expected IPv4 config") + }; + + assert_eq!(private_ip, priv_ip.into()); + assert_eq!( + vpc_subnet, + Ipv6Cidr::new( + priv_subnet.first_addr().into(), + priv_subnet.width().try_into().unwrap() + ) + ); + assert_eq!(gateway_ip, priv_subnet.iter().nth(1).unwrap().into()); + let oxide_vpc::api::SNatCfg { external_ip, ports } = + snat.expect("SNAT config for this port should be Some(_)"); + assert_eq!(external_ip, ext_ip.into()); + assert_eq!(ports, source_nat.port_range()); + assert!(ephemeral_ip.is_none()); + assert!(floating_ips.is_empty()); + } + + #[test] + fn ip_cfg_from_dual_stack_params() { + let priv_ipv4 = Ipv4Addr::new(172, 30, 2, 5); + let priv_ipv4_subnet = + Ipv4Net::new(Ipv4Addr::new(172, 30, 2, 0), 24).unwrap(); + let ipv4_config = + PrivateIpv4Config::new(priv_ipv4, priv_ipv4_subnet).unwrap(); + let mac = "a8:40:25:ff:ff:ff".parse().unwrap(); + let ext_ipv4 = Ipv4Addr::new(10, 151, 2, 169); + let priv_ipv6 = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 5); + let priv_ipv6_subnet = + Ipv6Net::new(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0), 64) + .unwrap(); + let ipv6_config = + PrivateIpv6Config::new(priv_ipv6, priv_ipv6_subnet).unwrap(); + let ext_ipv6 = Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1); + let ip_config = + PrivateIpConfig::DualStack { v4: ipv4_config, v6: ipv6_config }; + let nic = NetworkInterface { + id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Instance { id: Uuid::new_v4() }, + name: "opte0".parse().unwrap(), + ip_config, + mac, + vni: 100.try_into().unwrap(), + primary: true, + slot: 0, + }; + + // Ipv4 source NAT, Ipv6 ephemeral + let source_nat = + SourceNatConfig::new(IpAddr::V4(ext_ipv4), 0, 16383).unwrap(); + let prs = PortCreateParams { + nic: &nic, + source_nat: Some(source_nat), + ephemeral_ip: Some(ext_ipv6.into()), + floating_ips: &[], + firewall_rules: &[], + dhcp_config: DhcpCfg { + hostname: None, + host_domain: None, + domain_search_list: vec![], + dns4_servers: vec![], + dns6_servers: vec![], + }, + }; + let IpCfg::DualStack { ipv4, ipv6 } = IpCfg::try_from(&prs).unwrap() + else { + panic!("Expected DualStack config") + }; + + // Check IPv4 configuration + assert_eq!(ipv4.private_ip, priv_ipv4.into()); + assert_eq!( + ipv4.vpc_subnet, + Ipv4Cidr::new( + priv_ipv4_subnet.network().unwrap().into(), + priv_ipv4_subnet.width().try_into().unwrap() + ) + ); + assert_eq!(ipv4.gateway_ip, priv_ipv4_subnet.first_host().into()); + let oxide_vpc::api::SNatCfg { external_ip, ports } = ipv4 + .external_ips + .snat + .expect("SNAT config for this port should be Some(_)"); + assert_eq!(external_ip, ext_ipv4.into()); + assert_eq!(ports, source_nat.port_range()); + assert!(ipv4.external_ips.ephemeral_ip.is_none()); + assert!(ipv4.external_ips.floating_ips.is_empty()); + + // Check IPv6 configuration + assert_eq!(ipv6.private_ip, priv_ipv6.into()); + assert_eq!( + ipv6.vpc_subnet, + Ipv6Cidr::new( + priv_ipv6_subnet.first_addr().into(), + priv_ipv6_subnet.width().try_into().unwrap() + ) + ); + assert_eq!( + ipv6.gateway_ip, + priv_ipv6_subnet.iter().nth(1).unwrap().into() + ); + assert!( + ipv6.external_ips.snat.is_none(), + "Should not have SNAT config for the IPv6 stack" + ); + assert_eq!( + ipv6.external_ips + .ephemeral_ip + .expect("Should have IPv6 ephemeral address"), + ext_ipv6.into(), + ); + assert!(ipv6.external_ips.floating_ips.is_empty()); + } + + #[test] + fn ip_cfg_from_port_params_fails_with_private_ipv4_and_public_ipv6() { + let priv_ip = Ipv4Addr::new(172, 30, 2, 5); + let priv_subnet = + Ipv4Net::new(Ipv4Addr::new(172, 30, 2, 0), 24).unwrap(); + let ip_config = + PrivateIpConfig::new_ipv4(priv_ip, priv_subnet).unwrap(); + let mac = "a8:40:25:ff:ff:ff".parse().unwrap(); + let ext_ip = Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1); + let nic = NetworkInterface { + id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Instance { id: Uuid::new_v4() }, + name: "opte0".parse().unwrap(), + ip_config, + mac, + vni: 100.try_into().unwrap(), + primary: true, + slot: 0, + }; + let source_nat = + SourceNatConfig::new(IpAddr::V6(ext_ip), 0, 16383).unwrap(); + let prs = PortCreateParams { + nic: &nic, + source_nat: Some(source_nat), + ephemeral_ip: None, + floating_ips: &[], + firewall_rules: &[], + dhcp_config: DhcpCfg { + hostname: None, + host_domain: None, + domain_search_list: vec![], + dns4_servers: vec![], + dns6_servers: vec![], + }, + }; + let _ = IpCfg::try_from(&prs).expect_err( + "Should fail to convert with public IPv6 and private IPv4", + ); + } + + #[test] + fn ip_cfg_from_port_params_fails_with_private_ipv6_and_public_ipv4() { + let priv_ip = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 5); + let priv_subnet = + Ipv6Net::new(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0), 64) + .unwrap(); + let ip_config = + PrivateIpConfig::new_ipv6(priv_ip, priv_subnet).unwrap(); + let mac = "a8:40:25:ff:ff:ff".parse().unwrap(); + let ext_ip = Ipv4Addr::new(1, 1, 1, 1); + let nic = NetworkInterface { + id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Instance { id: Uuid::new_v4() }, + name: "opte0".parse().unwrap(), + ip_config, + mac, + vni: 100.try_into().unwrap(), + primary: true, + slot: 0, + }; + let source_nat = + SourceNatConfig::new(IpAddr::V4(ext_ip), 0, 16383).unwrap(); + let prs = PortCreateParams { + nic: &nic, + source_nat: Some(source_nat), + ephemeral_ip: None, + floating_ips: &[], + firewall_rules: &[], + dhcp_config: DhcpCfg { + hostname: None, + host_domain: None, + domain_search_list: vec![], + dns4_servers: vec![], + dns6_servers: vec![], + }, + }; + let _ = IpCfg::try_from(&prs).expect_err( + "Should fail to convert with public IPv4 and private IPv6", + ); + } } From bd60e20533a215e50d6e36d2c6f05521aac4aaa3 Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Fri, 14 Nov 2025 22:11:24 +0000 Subject: [PATCH 3/5] Fix mismerge --- sled-agent/src/sim/server.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 12867b2367b..d1a9b19888c 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -459,7 +459,9 @@ pub async fn run_standalone_server( SocketAddr::V6(a) => a, }, lockstep_port: nexus_lockstep_port, - external_ip: from_ipaddr_to_external_floating_ip(ip), + external_ip: from_ipaddr_to_external_floating_ip( + external_ip, + ), nic: nexus_types::inventory::NetworkInterface { id: Uuid::new_v4(), kind: NetworkInterfaceKind::Service { From d82e117b7618df0d27462d3d4749763541e84170 Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Sat, 15 Nov 2025 01:16:57 +0000 Subject: [PATCH 4/5] Fixup expectorate test --- .../planning/tests/output/planner_nonprovisionable_2_2a.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt index 51308402ef6..341454dd6e0 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt @@ -352,7 +352,7 @@ mismatched zone type: after: Nexus( name: Name( "nexus-5a8e9719-62bd-40be-81b1-20b85970740b", ), - ip: V4( + ip_config: V4( PrivateIpv4Config { ip: 172.30.2.5, subnet: Ipv4Net { From 95a7f8f340f8a09bab9864513881c2298af22644 Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Sat, 15 Nov 2025 23:14:19 +0000 Subject: [PATCH 5/5] small api fixes --- openapi/nexus-lockstep.json | 4 +- ...da02.json => sled-agent-7.0.0-2efbea.json} | 70 +++++++++++++------ openapi/sled-agent/sled-agent-latest.json | 2 +- 3 files changed, 50 insertions(+), 26 deletions(-) rename openapi/sled-agent/{sled-agent-7.0.0-90da02.json => sled-agent-7.0.0-2efbea.json} (99%) diff --git a/openapi/nexus-lockstep.json b/openapi/nexus-lockstep.json index 38b39e24931..cb7f3fe6ec7 100644 --- a/openapi/nexus-lockstep.json +++ b/openapi/nexus-lockstep.json @@ -5607,7 +5607,7 @@ "type": "string", "format": "uuid" }, - "ip": { + "ip_config": { "$ref": "#/components/schemas/PrivateIpConfig" }, "kind": { @@ -5633,7 +5633,7 @@ }, "required": [ "id", - "ip", + "ip_config", "kind", "mac", "name", diff --git a/openapi/sled-agent/sled-agent-7.0.0-90da02.json b/openapi/sled-agent/sled-agent-7.0.0-2efbea.json similarity index 99% rename from openapi/sled-agent/sled-agent-7.0.0-90da02.json rename to openapi/sled-agent/sled-agent-7.0.0-2efbea.json index 94496170835..d56741b78aa 100644 --- a/openapi/sled-agent/sled-agent-7.0.0-90da02.json +++ b/openapi/sled-agent/sled-agent-7.0.0-2efbea.json @@ -4751,24 +4751,6 @@ "minLength": 1, "maxLength": 7 }, - "IdMapDatasetConfig": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/DatasetConfig" - } - }, - "IdMapOmicronPhysicalDiskConfig": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/OmicronPhysicalDiskConfig" - } - }, - "IdMapOmicronZoneConfig": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/OmicronZoneConfig" - } - }, "ImportExportPolicy": { "description": "Define policy relating to the import and export of prefixes from a BGP peer.", "oneOf": [ @@ -5695,7 +5677,7 @@ "type": "string", "format": "uuid" }, - "ip": { + "ip_config": { "$ref": "#/components/schemas/PrivateIpConfig" }, "kind": { @@ -5721,7 +5703,7 @@ }, "required": [ "id", - "ip", + "ip_config", "kind", "mac", "name", @@ -5858,10 +5840,38 @@ "type": "object", "properties": { "datasets": { - "$ref": "#/components/schemas/IdMapDatasetConfig" + "title": "IdOrdMapAsMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/DatasetConfig" + } + ], + "path": "iddqd::IdOrdMap", + "version": "*" + }, + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/DatasetConfig" + } }, "disks": { - "$ref": "#/components/schemas/IdMapOmicronPhysicalDiskConfig" + "title": "IdOrdMapAsMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/OmicronPhysicalDiskConfig" + } + ], + "path": "iddqd::IdOrdMap", + "version": "*" + }, + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/OmicronPhysicalDiskConfig" + } }, "generation": { "$ref": "#/components/schemas/Generation" @@ -5890,7 +5900,21 @@ ] }, "zones": { - "$ref": "#/components/schemas/IdMapOmicronZoneConfig" + "title": "IdOrdMapAsMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/OmicronZoneConfig" + } + ], + "path": "iddqd::IdOrdMap", + "version": "*" + }, + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/OmicronZoneConfig" + } } }, "required": [ diff --git a/openapi/sled-agent/sled-agent-latest.json b/openapi/sled-agent/sled-agent-latest.json index da63ba1eb13..47326f1ec22 120000 --- a/openapi/sled-agent/sled-agent-latest.json +++ b/openapi/sled-agent/sled-agent-latest.json @@ -1 +1 @@ -sled-agent-7.0.0-90da02.json \ No newline at end of file +sled-agent-7.0.0-2efbea.json \ No newline at end of file