diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index a3cbe0a116a..4385e21b45e 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -113,7 +113,14 @@ we'll assign addresses as per RFD 63 as well as incorporating DNS based service discovery. For the purposes of local development today, we specify some hardcoded IPv6 -unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1::/64`: +unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1::/64`. + +If you'd like to modify these values to suit your local network, you can modify +them within the https://github.com/oxidecomputer/omicron/tree/main/smf[`smf/` subdirectory]. +Notably, Nexus is being served from IPv4 address, which may be configured to be +external. By default, it uses a private IPv4 address and no Internet gateway, but may +be configured to use a public-facing IP address with an Internet gateway that may +be set as a default route for the Nexus zone. [options="header"] |=================================================================================================== @@ -121,7 +128,6 @@ unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1: | Sled Agent: Bootstrap | Derived from MAC address of physical data link. | Sled Agent: Dropshot API | `[fd00:1122:3344:0101::1]:12345` | Cockroach DB | `[fd00:1122:3344:0101::2]:32221` -| Nexus: External API | `[fd00:1122:3344:0101::3]:12220` | Nexus: Internal API | `[fd00:1122:3344:0101::3]:12221` | Oximeter | `[fd00:1122:3344:0101::4]:12223` | Clickhouse | `[fd00:1122:3344:0101::5]:8123` @@ -129,6 +135,8 @@ unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1: | Crucible Downstairs 2 | `[fd00:1122:3344:0101::7]:32345` | Crucible Downstairs 3 | `[fd00:1122:3344:0101::8]:32345` | Internal DNS Service | `[fd00:1122:3344:0001::1]:5353` +| Nexus: External API | `192.168.1.20:80` +| Internet Gateway | None, but can be set in `smf/sled-agent/config.toml` |=================================================================================================== Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index a948b9414a9..09c31d4c8ea 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -150,7 +150,7 @@ impl Agent { err, })?; - let etherstub = Dladm::create_etherstub().map_err(|e| { + let etherstub = Dladm::ensure_etherstub().map_err(|e| { BootstrapError::SledError(format!( "Can't access etherstub device: {}", e @@ -158,7 +158,7 @@ impl Agent { })?; let etherstub_vnic = - Dladm::create_etherstub_vnic(ðerstub).map_err(|e| { + Dladm::ensure_etherstub_vnic(ðerstub).map_err(|e| { BootstrapError::SledError(format!( "Can't access etherstub VNIC device: {}", e diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index d67dd088e8a..36cd6c91c41 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -9,7 +9,7 @@ use crate::illumos::dladm::{self, Dladm, PhysicalLink}; use crate::illumos::zpool::ZpoolName; use dropshot::ConfigLogging; use serde::Deserialize; -use std::net::SocketAddr; +use std::net::{Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use uuid::Uuid; @@ -27,6 +27,10 @@ pub struct Config { /// Optional list of zpools to be used as "discovered disks". pub zpools: Option>, + /// Address of the Internet gateway, which is particularly + /// relevant for external-facing services (such as Nexus). + pub gateway_address: Option, + /// The data link on which we infer the bootstrap address. /// /// If unsupplied, we default to the first physical device. diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs index 22251074dc9..da91d421cbd 100644 --- a/sled-agent/src/illumos/dladm.rs +++ b/sled-agent/src/illumos/dladm.rs @@ -124,7 +124,7 @@ pub struct Dladm {} #[cfg_attr(test, mockall::automock, allow(dead_code))] impl Dladm { /// Creates an etherstub, or returns one which already exists. - pub fn create_etherstub() -> Result { + pub fn ensure_etherstub() -> Result { if let Ok(stub) = Self::get_etherstub() { return Ok(stub); } @@ -147,7 +147,7 @@ impl Dladm { /// /// This VNIC is not tracked like [`crate::illumos::vnic::Vnic`], because /// it is expected to exist for the lifetime of the sled. - pub fn create_etherstub_vnic( + pub fn ensure_etherstub_vnic( source: &Etherstub, ) -> Result { if let Ok(vnic) = Self::get_etherstub_vnic() { diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index fee23187688..75316eecfb9 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -5,13 +5,14 @@ //! Utilities to manage running zones. use crate::illumos::addrobj::AddrObject; +use crate::illumos::dladm::Etherstub; use crate::illumos::svc::wait_for_service; use crate::illumos::vnic::{Vnic, VnicAllocator}; use crate::illumos::zone::{AddressRequest, ZONE_PREFIX}; use crate::opte::OptePort; use ipnetwork::IpNetwork; use slog::Logger; -use std::net::Ipv6Addr; +use std::net::{Ipv4Addr, Ipv6Addr}; use std::path::PathBuf; #[cfg(test)] @@ -153,7 +154,6 @@ impl RunningZone { &self, addrtype: AddressRequest, ) -> Result { - info!(self.inner.log, "Adding address: {:?}", addrtype); let name = match addrtype { AddressRequest::Dhcp => "omicron", AddressRequest::Static(net) => match net.ip() { @@ -161,6 +161,15 @@ impl RunningZone { std::net::IpAddr::V6(_) => "omicron6", }, }; + self.ensure_address_with_name(addrtype, name).await + } + + pub async fn ensure_address_with_name( + &self, + addrtype: AddressRequest, + name: &str, + ) -> Result { + info!(self.inner.log, "Adding address: {:?}", addrtype); let addrobj = AddrObject::new(self.inner.control_vnic.name(), name) .map_err(|err| EnsureAddressError::AddrObject { request: addrtype, @@ -172,6 +181,31 @@ impl RunningZone { Ok(network) } + // TODO: Remove once Nexus uses OPTE - external addresses should generally + // be served via OPTE. + pub async fn ensure_external_address_with_name( + &self, + addrtype: AddressRequest, + name: &str, + ) -> Result { + info!(self.inner.log, "Adding address: {:?}", addrtype); + let addrobj = AddrObject::new( + self.inner.physical_nic + .as_ref() + .expect("Cannot allocate external address on zone without physical NIC") + .name(), + name + ) + .map_err(|err| EnsureAddressError::AddrObject { + request: addrtype, + zone: self.inner.name.clone(), + err, + })?; + let network = + Zones::ensure_address(Some(&self.inner.name), &addrobj, addrtype)?; + Ok(network) + } + pub async fn add_default_route( &self, gateway: Ipv6Addr, @@ -187,6 +221,19 @@ impl RunningZone { Ok(()) } + pub async fn add_default_route4( + &self, + gateway: Ipv4Addr, + ) -> Result<(), RunCommandError> { + self.run_cmd(&[ + "/usr/sbin/route", + "add", + "default", + &gateway.to_string(), + ])?; + Ok(()) + } + /// Looks up a running zone based on the `zone_prefix`, if one already exists. /// /// - If the zone was found, is running, and has a network interface, it is @@ -249,6 +296,7 @@ impl RunningZone { // // Re-initialize guest_vnic state by inspecting the zone. opte_ports: vec![], + physical_nic: None, }, }) } @@ -301,6 +349,10 @@ pub struct InstalledZone { // OPTE devices for the guest network interfaces opte_ports: Vec, + + // Physical NIC possibly provisioned to the zone. + // TODO: Remove once Nexus traffic is transmitted over OPTE. + physical_nic: Option, } impl InstalledZone { @@ -325,14 +377,16 @@ impl InstalledZone { zone_name } + #[allow(clippy::too_many_arguments)] pub async fn install( log: &Logger, - vnic_allocator: &VnicAllocator, + vnic_allocator: &VnicAllocator, service_name: &str, unique_name: Option<&str>, datasets: &[zone::Dataset], devices: &[zone::Device], opte_ports: Vec, + physical_nic: Option, ) -> Result { let control_vnic = vnic_allocator.new_control(None).map_err(|err| { InstallZoneError::CreateVnic { @@ -349,6 +403,7 @@ impl InstalledZone { .iter() .map(|port| port.vnic().name().to_string()) .chain(std::iter::once(control_vnic.name().to_string())) + .chain(physical_nic.as_ref().map(|vnic| vnic.name().to_string())) .collect(); Zones::install_omicron_zone( @@ -370,6 +425,7 @@ impl InstalledZone { name: zone_name, control_vnic, opte_ports, + physical_nic, }) } } diff --git a/sled-agent/src/illumos/vnic.rs b/sled-agent/src/illumos/vnic.rs index 85bb32880f8..c9355446ef2 100644 --- a/sled-agent/src/illumos/vnic.rs +++ b/sled-agent/src/illumos/vnic.rs @@ -5,7 +5,7 @@ //! API for controlling a single instance. use crate::illumos::dladm::{ - CreateVnicError, DeleteVnicError, Etherstub, VNIC_PREFIX, + CreateVnicError, DeleteVnicError, VnicSource, VNIC_PREFIX, VNIC_PREFIX_CONTROL, VNIC_PREFIX_GUEST, }; use omicron_common::api::external::MacAddr; @@ -23,13 +23,13 @@ use crate::illumos::dladm::MockDladm as Dladm; /// May be used to allocate runtime-unique IDs for objects /// which have naming constraints - such as VNICs. #[derive(Clone, Debug)] -pub struct VnicAllocator { +pub struct VnicAllocator { value: Arc, scope: String, - data_link: Etherstub, + data_link: DL, } -impl VnicAllocator { +impl VnicAllocator
{ /// Creates a new Vnic name allocator with a particular scope. /// /// The intent with varying scopes is to create non-overlapping @@ -41,11 +41,11 @@ impl VnicAllocator { /// /// VnicAllocator::new("Storage") produces /// - oxControlStorage[NNN] - pub fn new>(scope: S, etherstub: Etherstub) -> Self { + pub fn new>(scope: S, data_link: DL) -> Self { Self { value: Arc::new(AtomicU64::new(0)), scope: scope.as_ref().to_string(), - data_link: etherstub, + data_link, } } @@ -167,6 +167,7 @@ impl Drop for Vnic { #[cfg(test)] mod test { use super::*; + use crate::illumos::dladm::Etherstub; #[test] fn test_allocate() { diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index 4c7ba3baaa7..50dc53e01b0 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -101,8 +101,8 @@ pub struct EnsureAddressError { #[derive(thiserror::Error, Debug)] #[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}. Note to developers: {extra_note}")] pub struct EnsureGzAddressError { - address: Ipv6Addr, - link: EtherstubVnic, + address: IpAddr, + link: String, name: String, #[source] err: anyhow::Error, @@ -580,8 +580,8 @@ impl Zones { Ok(()) }(link.clone(), address, name) .map_err(|err| EnsureGzAddressError { - address, - link, + address: IpAddr::V6(address), + link: link.0.clone(), name: name.to_string(), err, extra_note: diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 79bc5aeeb7b..32e37c4fc1a 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -5,6 +5,7 @@ //! API for controlling a single instance. use crate::common::instance::{Action as InstanceAction, InstanceStates}; +use crate::illumos::dladm::Etherstub; use crate::illumos::running_zone::{ InstalledZone, RunCommandError, RunningZone, }; @@ -204,7 +205,7 @@ struct InstanceInner { propolis_ip: IpAddr, // NIC-related properties - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, // OPTE port related properties underlay_addr: Ipv6Addr, @@ -400,7 +401,7 @@ mockall::mock! { pub fn new( log: Logger, id: Uuid, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, underlay_addr: Ipv6Addr, port_allocator: OptePortAllocator, initial: InstanceHardware, @@ -446,7 +447,7 @@ impl Instance { pub fn new( log: Logger, id: Uuid, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, underlay_addr: Ipv6Addr, port_allocator: OptePortAllocator, initial: InstanceHardware, @@ -534,6 +535,8 @@ impl Instance { zone::Device { name: "/dev/viona".to_string() }, ], opte_ports, + // physical_nic= + None, ) .await?; diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 5f0e9179470..8660ad7cc9e 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -44,7 +44,7 @@ struct InstanceManagerInternal { /// A mapping from a Sled Agent "Instance ID" to ("Propolis ID", [Instance]). instances: Mutex>, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, underlay_addr: Ipv6Addr, port_allocator: OptePortAllocator, } diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index ae4cb40c6e2..6f4b196bc5c 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -303,7 +303,7 @@ impl From for sled_agent_client::types::DatasetEnsureBody { )] #[serde(tag = "type", rename_all = "snake_case")] pub enum ServiceType { - Nexus { internal_address: SocketAddrV6, external_address: SocketAddrV6 }, + Nexus { internal_address: SocketAddrV6, external_address: SocketAddr }, InternalDns { server_address: SocketAddrV6, dns_address: SocketAddrV6 }, Oximeter, } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index adf2731eb9a..fdd6e097eab 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -5,7 +5,8 @@ //! Support for miscellaneous services managed by the sled. use crate::bootstrap::ddm_admin_client::{DdmAdminClient, DdmError}; -use crate::illumos::dladm::{Etherstub, EtherstubVnic}; +use crate::common::underlay; +use crate::illumos::dladm::{Etherstub, EtherstubVnic, PhysicalLink}; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; @@ -24,7 +25,7 @@ use omicron_common::postgres_config::PostgresConfigWithUrl; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::str::FromStr; use tokio::io::AsyncWriteExt; @@ -70,6 +71,12 @@ pub enum Error { #[error("Error contacting ddmd: {0}")] DdmError(#[from] DdmError), + #[error("Failed to access underlay device: {0}")] + Underlay(#[from] underlay::Error), + + #[error("Failed to create Vnic for Nexus: {0}")] + NexusVnicCreation(crate::illumos::dladm::CreateVnicError), + #[error("Failed to add GZ addresses: {message}: {err}")] GzAddress { message: String, @@ -98,13 +105,14 @@ pub fn default_services_config_path() -> PathBuf { } /// Configuration parameters which modify the [`ServiceManager`]'s behavior. -/// -/// These are typically used to make testing easier; production usage -/// should generally prefer to use the defaults. pub struct Config { + /// An optional internet gateway address for external services. + pub gateway_address: Option, + /// The path for the ServiceManager to store information about /// all running services. pub all_svcs_config_path: PathBuf, + /// A function which returns the path the directory holding the /// service's configuration file. pub get_svc_config_dir: Box PathBuf + Send + Sync>, @@ -113,6 +121,7 @@ pub struct Config { impl Default for Config { fn default() -> Self { Self { + gateway_address: None, all_svcs_config_path: default_services_config_path(), get_svc_config_dir: Box::new(|zone_name: &str, svc_name: &str| { PathBuf::from(ZONE_ZFS_DATASET_MOUNTPOINT) @@ -129,7 +138,8 @@ pub struct ServiceManager { log: Logger, config: Config, zones: Mutex>, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, + physical_link_vnic_allocator: VnicAllocator, underlay_vnic: EtherstubVnic, underlay_address: Ipv6Addr, rack_id: Uuid, @@ -154,6 +164,7 @@ impl ServiceManager { underlay_vnic: EtherstubVnic, underlay_address: Ipv6Addr, config: Config, + physical_link: PhysicalLink, rack_id: Uuid, ) -> Result { debug!(log, "Creating new ServiceManager"); @@ -163,6 +174,13 @@ impl ServiceManager { config, zones: Mutex::new(vec![]), vnic_allocator: VnicAllocator::new("Service", etherstub), + physical_link_vnic_allocator: VnicAllocator::new( + "Public", + // NOTE: Right now, we only use a connection to one of the Chelsio + // links. Longer-term, when we we use OPTE, we'll be able to use both + // connections. + physical_link, + ), underlay_vnic, underlay_address, rack_id, @@ -244,6 +262,18 @@ impl ServiceManager { info!(self.log, "Service {} does not yet exist", service.name); } + // TODO: Remove once Nexus traffic is transmitted over OPTE. + let physical_nic = match service.service_type { + ServiceType::Nexus { .. } => { + let vnic = self + .physical_link_vnic_allocator + .new_control(None) + .map_err(|e| Error::NexusVnicCreation(e))?; + Some(vnic) + } + _ => None, + }; + let installed_zone = InstalledZone::install( &self.log, &self.vnic_allocator, @@ -254,8 +284,10 @@ impl ServiceManager { &[], // devices= &[], - // vnics= + // opte_ports= vec![], + // physical_nic= + physical_nic, ) .await?; @@ -305,7 +337,7 @@ impl ServiceManager { // // This is currently being used for the DNS service. // - // TODO: consider limitng the number of GZ addresses which + // TODO: consider limiting the number of GZ addresses which // can be supplied - now that we're actively using it, we // aren't really handling the "many GZ addresses" case, and it // doesn't seem necessary now. @@ -344,13 +376,38 @@ impl ServiceManager { ServiceType::Nexus { internal_address, external_address } => { info!(self.log, "Setting up Nexus service"); + // The address of Nexus' external interface is a special + // case; it may be an IPv4 address. + let addr_request = + AddressRequest::new_static(external_address.ip(), None); + running_zone + .ensure_external_address_with_name( + addr_request, + "public", + ) + .await?; + + if let IpAddr::V4(_public_addr4) = external_address.ip() { + // If requested, create a default route back through + // the internet gateway. + if let Some(ref gateway) = self.config.gateway_address { + running_zone + .add_default_route4(*gateway) + .await + .map_err(|err| Error::ZoneCommand { + intent: "Adding Route".to_string(), + err, + })?; + } + } + // Nexus takes a separate config file for parameters which // cannot be known at packaging time. let deployment_config = NexusDeploymentConfig { id: service.id, rack_id: self.rack_id, dropshot_external: ConfigDropshot { - bind_address: SocketAddr::V6(external_address), + bind_address: external_address, request_body_max_bytes: 1048576, ..Default::default() }, @@ -606,7 +663,7 @@ mod test { svc, zone::MockZones, }; - use std::net::{Ipv6Addr, SocketAddrV6}; + use std::net::Ipv6Addr; use std::os::unix::process::ExitStatusExt; use uuid::Uuid; @@ -635,12 +692,20 @@ mod test { assert_eq!(name, EXPECTED_ZONE_NAME); Ok(()) }); + + // Ensure the address exists + let ensure_address_ctx = MockZones::ensure_address_context(); + ensure_address_ctx.expect().return_once(|_, _, _| { + Ok(ipnetwork::IpNetwork::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 64) + .unwrap()) + }); + // Wait for the networking service. let wait_ctx = svc::wait_for_service_context(); wait_ctx.expect().return_once(|_, _| Ok(())); // Import the manifest, enable the service let execute_ctx = crate::illumos::execute_context(); - execute_ctx.expect().times(3).returning(|_| { + execute_ctx.expect().times(..).returning(|_| { Ok(std::process::Output { status: std::process::ExitStatus::from_raw(0), stdout: vec![], @@ -652,6 +717,7 @@ mod test { Box::new(create_vnic_ctx), Box::new(install_ctx), Box::new(boot_ctx), + Box::new(ensure_address_ctx), Box::new(wait_ctx), Box::new(execute_ctx), ] @@ -665,22 +731,9 @@ mod test { services: vec![ServiceRequest { id, name: SVC_NAME.to_string(), - addresses: vec![], + addresses: vec![Ipv6Addr::LOCALHOST], gz_addresses: vec![], - service_type: ServiceType::Nexus { - internal_address: SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - ), - external_address: SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - ), - }, + service_type: ServiceType::Oximeter, }], }) .await @@ -694,22 +747,9 @@ mod test { services: vec![ServiceRequest { id, name: SVC_NAME.to_string(), - addresses: vec![], + addresses: vec![Ipv6Addr::LOCALHOST], gz_addresses: vec![], - service_type: ServiceType::Nexus { - internal_address: SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - ), - external_address: SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - ), - }, + service_type: ServiceType::Oximeter, }], }) .await @@ -759,6 +799,7 @@ mod test { svc_config_dir.clone() }, ), + ..Default::default() } } } @@ -777,6 +818,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -804,6 +846,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -833,6 +876,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -851,6 +895,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -876,6 +921,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -896,6 +942,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, config, + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index e385d2271ad..f1cd951cb98 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -132,8 +132,8 @@ impl SledAgent { info!(&log, "created sled agent"); let etherstub = - Dladm::create_etherstub().map_err(|e| Error::Etherstub(e))?; - let etherstub_vnic = Dladm::create_etherstub_vnic(ðerstub) + Dladm::ensure_etherstub().map_err(|e| Error::Etherstub(e))?; + let etherstub_vnic = Dladm::ensure_etherstub_vnic(ðerstub) .map_err(|e| Error::EtherstubVnic(e))?; // Before we start creating zones, we need to ensure that the @@ -231,6 +231,7 @@ impl SledAgent { let mut command = std::process::Command::new(PFEXEC); let cmd = command.args(&[ "/usr/sbin/routeadm", + // Needed to access all zones, which are on the underlay. "-e", "ipv6-forwarding", "-u", @@ -261,12 +262,18 @@ impl SledAgent { etherstub.clone(), *sled_address.ip(), ); + + let svc_config = services::Config { + gateway_address: config.gateway_address, + ..Default::default() + }; let services = ServiceManager::new( parent_log.clone(), etherstub.clone(), etherstub_vnic.clone(), *sled_address.ip(), - services::Config::default(), + svc_config, + config.get_link()?, rack_id, ) .await?; diff --git a/sled-agent/src/sp/simulated.rs b/sled-agent/src/sp/simulated.rs index e402654da22..bdf31ffb9f3 100644 --- a/sled-agent/src/sp/simulated.rs +++ b/sled-agent/src/sp/simulated.rs @@ -62,8 +62,8 @@ impl SimulatedSp { // Ensure we have the global zone IP address we need for the SP. let etherstub = - Dladm::create_etherstub().map_err(SpError::CreateEtherstub)?; - let etherstub_vnic = Dladm::create_etherstub_vnic(ðerstub) + Dladm::ensure_etherstub().map_err(SpError::CreateEtherstub)?; + let etherstub_vnic = Dladm::ensure_etherstub_vnic(ðerstub) .map_err(SpError::CreateEtherstubVnic)?; Zones::ensure_has_global_zone_v6_address( etherstub_vnic, diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index a02e68baae9..1e5e5bb4bb9 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -456,7 +456,7 @@ impl DatasetInfo { // Ensures that a zone backing a particular dataset is running. async fn ensure_running_zone( log: &Logger, - vnic_allocator: &VnicAllocator, + vnic_allocator: &VnicAllocator, dataset_info: &DatasetInfo, dataset_name: &DatasetName, do_format: bool, @@ -488,6 +488,7 @@ async fn ensure_running_zone( &[zone::Dataset { name: dataset_name.full() }], &[], vec![], + None, ) .await?; @@ -542,7 +543,7 @@ struct StorageWorker { pools: Arc>>, new_pools_rx: mpsc::Receiver, new_filesystems_rx: mpsc::Receiver, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, underlay_address: Ipv6Addr, } diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index a5f06d1cfc5..36db4ef9c18 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -56,7 +56,8 @@ gz_addresses = [] [request.service.service_type] type = "nexus" internal_address = "[fd00:1122:3344:0101::3]:12221" -external_address = "[fd00:1122:3344:0101::3]:12220" +# NOTE: In the lab, use "172.20.15.226" +external_address = "192.168.1.20:80" # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate Oximeter services. diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index 53ec733e9ec..b99e4db58e4 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -26,6 +26,11 @@ zpools = [ # $ dladm show-phys -p -o LINK # data_link = "igb0" +# Address of Internet gateway +# +# NOTE: In the lab, use "172.20.15.225" +# gateway_address = "192.168.1.1" + [log] level = "info" mode = "file" diff --git a/tools/populate/populate-images.sh b/tools/populate/populate-images.sh index 9f845e09e05..2de2ebbae77 100755 --- a/tools/populate/populate-images.sh +++ b/tools/populate/populate-images.sh @@ -2,7 +2,7 @@ # Populate an Oxide lab host running Omicron with images from server catacomb. set -eu -CATACOMB_TUNNEL="[fd00:1122:3344:101::1]:8080" +CATACOMB_TUNNEL="[fd00:1122:3344:101::1]:54321" res=0 echo "Populating debian" oxide api /images --method POST --input - <