From c20a122c8abd57d77c259a9d4f3ff34f590b31d1 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 13 May 2022 12:26:55 -0400 Subject: [PATCH 01/13] WIP etherstub VNIC allocation --- sled-agent/src/config.rs | 2 +- sled-agent/src/illumos/dladm.rs | 50 +++++++++++++++++++++++++++--- sled-agent/src/illumos/vnic.rs | 12 +++---- sled-agent/src/instance_manager.rs | 6 ++-- sled-agent/src/services.rs | 17 +++++----- sled-agent/src/sled_agent.rs | 12 +++++-- sled-agent/src/storage_manager.rs | 6 ++-- smf/sled-agent/config.toml | 4 ++- 8 files changed, 78 insertions(+), 31 deletions(-) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index cfa6e84927b..d67dd088e8a 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -27,7 +27,7 @@ pub struct Config { /// Optional list of zpools to be used as "discovered disks". pub zpools: Option>, - /// The data link on which to allocate VNICs. + /// The data link on which we infer the bootstrap address. /// /// If unsupplied, we default to the first physical device. pub data_link: Option, diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs index 69e22ca8161..e6b74ba17d5 100644 --- a/sled-agent/src/illumos/dladm.rs +++ b/sled-agent/src/illumos/dladm.rs @@ -21,6 +21,8 @@ pub const VNIC_PREFIX_GUEST: &str = "vopte"; pub const DLADM: &str = "/usr/sbin/dladm"; +pub const ETHERSTUB_NAME: &str = "oxStub0"; + /// Errors returned from [`Dladm::find_physical`]. #[derive(thiserror::Error, Debug)] pub enum FindPhysicalLinkError { @@ -49,7 +51,7 @@ pub enum GetMacError { #[error("Failed to create VNIC {name} on link {link:?}: {err}")] pub struct CreateVnicError { name: String, - link: PhysicalLink, + link: String, #[source] err: ExecutionError, } @@ -75,11 +77,51 @@ pub struct DeleteVnicError { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] pub struct PhysicalLink(pub String); +/// The name of an etherstub +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct Etherstub(pub String); + +pub trait VnicSource { + fn name(&self) -> &str; +} + +impl VnicSource for PhysicalLink { + fn name(&self) -> &str { + &self.0 + } +} + +impl VnicSource for Etherstub { + fn name(&self) -> &str { + &self.0 + } +} + /// Wraps commands for interacting with data links. pub struct Dladm {} #[cfg_attr(test, mockall::automock, allow(dead_code))] impl Dladm { + /// Creates an etherstub, or returns one which already exists. + pub fn create_etherstub() -> Result { + if let Ok(stub) = Self::get_etherstub() { + return Ok(stub); + } + let mut command = std::process::Command::new(PFEXEC); + let cmd = + command.args(&[DLADM, "create-etherstub", "-t", ETHERSTUB_NAME]); + execute(cmd)?; + Ok(Etherstub(ETHERSTUB_NAME.to_string())) + } + + /// Finds an etherstub. + fn get_etherstub() -> Result { + let mut command = std::process::Command::new(PFEXEC); + let cmd = command.args(&[DLADM, "show-etherstub", ETHERSTUB_NAME]); + execute(cmd)?; + Ok(Etherstub(ETHERSTUB_NAME.to_string())) + } + /// Returns the name of the first observed physical data link. pub fn find_physical() -> Result { let mut command = std::process::Command::new(PFEXEC); @@ -136,7 +178,7 @@ impl Dladm { /// * `mac`: An optional unicast MAC address for the newly created NIC. /// * `vlan`: An optional VLAN ID for VLAN tagging. pub fn create_vnic( - physical: &PhysicalLink, + source: &impl VnicSource, vnic_name: &str, mac: Option, vlan: Option, @@ -147,7 +189,7 @@ impl Dladm { "create-vnic".to_string(), "-t".to_string(), "-l".to_string(), - physical.0.to_string(), + source.name().to_string(), ]; if let Some(mac) = mac { @@ -164,7 +206,7 @@ impl Dladm { let cmd = command.args(&args); execute(cmd).map_err(|err| CreateVnicError { name: vnic_name.to_string(), - link: physical.clone(), + link: source.name().to_string(), err, })?; Ok(()) diff --git a/sled-agent/src/illumos/vnic.rs b/sled-agent/src/illumos/vnic.rs index 03aa872fce5..85bb32880f8 100644 --- a/sled-agent/src/illumos/vnic.rs +++ b/sled-agent/src/illumos/vnic.rs @@ -5,7 +5,7 @@ //! API for controlling a single instance. use crate::illumos::dladm::{ - CreateVnicError, DeleteVnicError, PhysicalLink, VNIC_PREFIX, + CreateVnicError, DeleteVnicError, Etherstub, VNIC_PREFIX, VNIC_PREFIX_CONTROL, VNIC_PREFIX_GUEST, }; use omicron_common::api::external::MacAddr; @@ -26,7 +26,7 @@ use crate::illumos::dladm::MockDladm as Dladm; pub struct VnicAllocator { value: Arc, scope: String, - data_link: PhysicalLink, + data_link: Etherstub, } impl VnicAllocator { @@ -41,11 +41,11 @@ impl VnicAllocator { /// /// VnicAllocator::new("Storage") produces /// - oxControlStorage[NNN] - pub fn new>(scope: S, physical_link: PhysicalLink) -> Self { + pub fn new>(scope: S, etherstub: Etherstub) -> Self { Self { value: Arc::new(AtomicU64::new(0)), scope: scope.as_ref().to_string(), - data_link: physical_link, + data_link: etherstub, } } @@ -171,7 +171,7 @@ mod test { #[test] fn test_allocate() { let allocator = - VnicAllocator::new("Foo", PhysicalLink("mylink".to_string())); + VnicAllocator::new("Foo", Etherstub("mystub".to_string())); assert_eq!("oxFoo0", allocator.next()); assert_eq!("oxFoo1", allocator.next()); assert_eq!("oxFoo2", allocator.next()); @@ -180,7 +180,7 @@ mod test { #[test] fn test_allocate_within_scopes() { let allocator = - VnicAllocator::new("Foo", PhysicalLink("mylink".to_string())); + VnicAllocator::new("Foo", Etherstub("mystub".to_string())); assert_eq!("oxFoo0", allocator.next()); let allocator = allocator.new_superscope("Baz"); assert_eq!("oxBazFoo1", allocator.next()); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index ad67e674559..1791d580209 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -4,7 +4,7 @@ //! API for controlling multiple instances on a sled. -use crate::illumos::dladm::PhysicalLink; +use crate::illumos::dladm::Etherstub; use crate::illumos::vnic::VnicAllocator; use crate::nexus::NexusClient; use crate::opte::OptePortAllocator; @@ -54,7 +54,7 @@ impl InstanceManager { pub fn new( log: Logger, nexus_client: Arc, - physical_link: PhysicalLink, + etherstub: Etherstub, underlay_addr: Ipv6Addr, ) -> InstanceManager { InstanceManager { @@ -62,7 +62,7 @@ impl InstanceManager { log: log.new(o!("component" => "InstanceManager")), nexus_client, instances: Mutex::new(BTreeMap::new()), - vnic_allocator: VnicAllocator::new("Instance", physical_link), + vnic_allocator: VnicAllocator::new("Instance", etherstub), underlay_addr, port_allocator: OptePortAllocator::new(), }), diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 69fc0042efb..969701ff9f2 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -4,10 +4,10 @@ //! Support for miscellaneous services managed by the sled. -use crate::illumos::dladm::PhysicalLink; +use crate::illumos::dladm::Etherstub; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; -use crate::illumos::zone::{AddressRequest, Zones}; +use crate::illumos::zone::AddressRequest; use crate::params::{ServiceEnsureBody, ServiceRequest}; use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT}; use slog::Logger; @@ -77,7 +77,6 @@ pub struct ServiceManager { config_path: Option, zones: Mutex>, vnic_allocator: VnicAllocator, - physical_link: PhysicalLink, } impl ServiceManager { @@ -86,13 +85,13 @@ impl ServiceManager { /// /// Args: /// - `log`: The logger - /// - `physical_link`: A physical link on which to allocate datalinks. + /// - `etherstub`: An etherstub on which to allocate VNICs. /// - `config_path`: An optional path to a configuration file to store /// the record of services. By default, [`default_services_config_path`] /// is used. pub async fn new( log: Logger, - physical_link: PhysicalLink, + etherstub: Etherstub, config_path: Option, ) -> Result { debug!(log, "Creating new ServiceManager"); @@ -100,11 +99,7 @@ impl ServiceManager { log: log.new(o!("component" => "ServiceManager")), config_path, zones: Mutex::new(vec![]), - vnic_allocator: VnicAllocator::new( - "Service", - physical_link.clone(), - ), - physical_link, + vnic_allocator: VnicAllocator::new("Service", etherstub), }; let config_path = mgr.services_config_path(); @@ -202,6 +197,7 @@ impl ServiceManager { ); } + /* info!(self.log, "GZ addresses: {:#?}", service.gz_addresses); for addr in &service.gz_addresses { info!( @@ -224,6 +220,7 @@ impl ServiceManager { err, })?; } + */ debug!(self.log, "importing manifest"); diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 4655db63f97..578ce1bffbd 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -38,6 +38,9 @@ pub enum Error { #[error("Physical link not in config, nor found automatically: {0}")] FindPhysicalLink(#[from] crate::illumos::dladm::FindPhysicalLinkError), + #[error("Failed to acquire etherstub: {0}")] + Etherstub(crate::illumos::ExecutionError), + #[error("Failed to lookup VNICs on boot: {0}")] GetVnics(#[from] crate::illumos::dladm::GetVnicError), @@ -117,6 +120,9 @@ impl SledAgent { )); info!(&log, "created sled agent"); + let etherstub = + Dladm::create_etherstub().map_err(|e| Error::Etherstub(e))?; + let data_link = if let Some(link) = config.data_link.clone() { link } else { @@ -199,7 +205,7 @@ impl SledAgent { &parent_log, *id, nexus_client.clone(), - data_link.clone(), + etherstub.clone(), ) .await; if let Some(pools) = &config.zpools { @@ -215,11 +221,11 @@ impl SledAgent { let instances = InstanceManager::new( parent_log.clone(), nexus_client.clone(), - data_link.clone(), + etherstub.clone(), *sled_address.ip(), ); let services = - ServiceManager::new(parent_log.clone(), data_link.clone(), None) + ServiceManager::new(parent_log.clone(), etherstub.clone(), None) .await?; Ok(SledAgent { diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index 8de0e2ec9ce..7ec32833277 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -4,7 +4,7 @@ //! Management of sled-local storage. -use crate::illumos::dladm::PhysicalLink; +use crate::illumos::dladm::Etherstub; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; use crate::illumos::zone::AddressRequest; @@ -889,7 +889,7 @@ impl StorageManager { log: &Logger, sled_id: Uuid, nexus_client: Arc, - physical_link: PhysicalLink, + etherstub: Etherstub, ) -> Self { let log = log.new(o!("component" => "StorageManager")); let pools = Arc::new(Mutex::new(HashMap::new())); @@ -902,7 +902,7 @@ impl StorageManager { pools: pools.clone(), new_pools_rx, new_filesystems_rx, - vnic_allocator: VnicAllocator::new("Storage", physical_link), + vnic_allocator: VnicAllocator::new("Storage", etherstub), }; StorageManager { pools, diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index 45fc6669dfa..3b84899eb1a 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -19,7 +19,9 @@ zpools = [ "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03", ] -# An optional data link on which VNICs should be created. +# An optional data link from which we extract a MAC address. +# This is used as a unique identifier for the bootstrap address. +# # If empty, this will be equivalent to the first result from: # $ dladm show-phys -p -o LINK # data_link = "igb0" From ba5113b32170a731b2beb7a64f5560e3ffb5b454 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 13 May 2022 16:02:56 -0400 Subject: [PATCH 02/13] Working on illumos; time to fix tests --- sled-agent/src/bootstrap/agent.rs | 21 +++++++++++--------- sled-agent/src/illumos/dladm.rs | 32 ++++++++++++++++++++++++++++++- sled-agent/src/illumos/zone.rs | 11 ++++++----- sled-agent/src/services.rs | 11 +++++++---- sled-agent/src/sled_agent.rs | 20 ++++++++++--------- 5 files changed, 67 insertions(+), 28 deletions(-) diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index 9df75b6fb99..76a20df354d 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -138,19 +138,22 @@ impl Agent { "server" => sled_config.id.to_string(), )); - let data_link = if let Some(link) = sled_config.data_link.clone() { - link - } else { - Dladm::find_physical().map_err(|err| { + let etherstub = + Dladm::create_etherstub().map_err(|e| { BootstrapError::SledError(format!( - "Can't access physical link, and none in config: {}", - err + "Can't access etherstub device: {}", e )) - })? - }; + })?; + + let etherstub_vnic = + Dladm::create_etherstub_vnic(ðerstub).map_err(|e| { + BootstrapError::SledError(format!( + "Can't access etherstub VNIC device: {}", e + )) + })?; Zones::ensure_has_global_zone_v6_address( - data_link, + etherstub_vnic, address, "bootstrap6", ) diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs index e6b74ba17d5..4cd7f9a5921 100644 --- a/sled-agent/src/illumos/dladm.rs +++ b/sled-agent/src/illumos/dladm.rs @@ -21,7 +21,8 @@ pub const VNIC_PREFIX_GUEST: &str = "vopte"; pub const DLADM: &str = "/usr/sbin/dladm"; -pub const ETHERSTUB_NAME: &str = "oxStub0"; +pub const ETHERSTUB_NAME: &str = "stub0"; +pub const ETHERSTUB_VNIC_NAME: &str = "underlay0"; /// Errors returned from [`Dladm::find_physical`]. #[derive(thiserror::Error, Debug)] @@ -81,6 +82,10 @@ pub struct PhysicalLink(pub String); #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] pub struct Etherstub(pub String); +/// The name of an etherstub's underlay VNIC +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct EtherstubVnic(pub String); + pub trait VnicSource { fn name(&self) -> &str; } @@ -97,6 +102,12 @@ impl VnicSource for Etherstub { } } +impl VnicSource for EtherstubVnic { + fn name(&self) -> &str { + &self.0 + } +} + /// Wraps commands for interacting with data links. pub struct Dladm {} @@ -122,6 +133,25 @@ impl Dladm { Ok(Etherstub(ETHERSTUB_NAME.to_string())) } + /// Creates a VNIC on top of the etherstub. + /// + /// This VNIC is not tracked like [`crate::illumos::vnic::Vnic`], because + /// it is expected to exist for the lifetime of the sled. + pub fn create_etherstub_vnic(source: &Etherstub) -> Result { + if let Ok(vnic) = Self::get_etherstub_vnic() { + return Ok(vnic); + } + Self::create_vnic(source, ETHERSTUB_VNIC_NAME, None, None)?; + Ok(EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string())) + } + + fn get_etherstub_vnic() -> Result { + let mut command = std::process::Command::new(PFEXEC); + let cmd = command.args(&[DLADM, "show-vnic", ETHERSTUB_VNIC_NAME]); + execute(cmd)?; + Ok(EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string())) + } + /// Returns the name of the first observed physical data link. pub fn find_physical() -> Result { let mut command = std::process::Command::new(PFEXEC); diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index c2ed47cf784..2f0fcf2820a 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -9,8 +9,9 @@ use ipnetwork::IpNetwork; use slog::Logger; use std::net::{IpAddr, Ipv6Addr}; +use omicron_common::address::AZ_PREFIX; use crate::illumos::addrobj::AddrObject; -use crate::illumos::dladm::{PhysicalLink, VNIC_PREFIX_CONTROL}; +use crate::illumos::dladm::{EtherstubVnic, VNIC_PREFIX_CONTROL}; use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::{execute, PFEXEC}; @@ -101,7 +102,7 @@ pub struct EnsureAddressError { #[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}")] pub struct EnsureGzAddressError { address: Ipv6Addr, - link: PhysicalLink, + link: EtherstubVnic, name: String, #[source] err: anyhow::Error, @@ -122,7 +123,7 @@ impl AddressRequest { pub fn new_static(ip: IpAddr, prefix: Option) -> Self { let prefix = prefix.unwrap_or_else(|| match ip { IpAddr::V4(_) => 24, - IpAddr::V6(_) => 64, + IpAddr::V6(_) => AZ_PREFIX, }); let addr = IpNetwork::new(ip, prefix).unwrap(); AddressRequest::Static(addr) @@ -543,13 +544,13 @@ impl Zones { // should remove this function when Sled Agents are provided IPv6 addresses // from RSS. pub fn ensure_has_global_zone_v6_address( - link: PhysicalLink, + link: EtherstubVnic, address: Ipv6Addr, name: &str, ) -> Result<(), EnsureGzAddressError> { // Call the guts of this function within a closure to make it easier // to wrap the error with appropriate context. - |link: PhysicalLink, address, name| -> Result<(), anyhow::Error> { + |link: EtherstubVnic, address, name| -> Result<(), anyhow::Error> { let gz_link_local_addrobj = AddrObject::new(&link.0, "linklocal") .map_err(|err| anyhow!(err))?; Self::ensure_has_link_local_v6_address( diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 969701ff9f2..a2dbd38884c 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -4,11 +4,12 @@ //! Support for miscellaneous services managed by the sled. -use crate::illumos::dladm::Etherstub; +use crate::illumos::dladm::{Etherstub, EtherstubVnic}; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; use crate::illumos::zone::AddressRequest; use crate::params::{ServiceEnsureBody, ServiceRequest}; +use crate::zone::Zones; use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT}; use slog::Logger; use std::collections::HashSet; @@ -77,6 +78,7 @@ pub struct ServiceManager { config_path: Option, zones: Mutex>, vnic_allocator: VnicAllocator, + underlay_vnic: EtherstubVnic, } impl ServiceManager { @@ -86,12 +88,14 @@ impl ServiceManager { /// Args: /// - `log`: The logger /// - `etherstub`: An etherstub on which to allocate VNICs. + /// - `etherstub_vnic`: The underlay's VNIC in the Global Zone. /// - `config_path`: An optional path to a configuration file to store /// the record of services. By default, [`default_services_config_path`] /// is used. pub async fn new( log: Logger, etherstub: Etherstub, + underlay_vnic: EtherstubVnic, config_path: Option, ) -> Result { debug!(log, "Creating new ServiceManager"); @@ -100,6 +104,7 @@ impl ServiceManager { config_path, zones: Mutex::new(vec![]), vnic_allocator: VnicAllocator::new("Service", etherstub), + underlay_vnic, }; let config_path = mgr.services_config_path(); @@ -197,7 +202,6 @@ impl ServiceManager { ); } - /* info!(self.log, "GZ addresses: {:#?}", service.gz_addresses); for addr in &service.gz_addresses { info!( @@ -208,7 +212,7 @@ impl ServiceManager { let addr_name = service.name.replace(&['-', '_'][..], ""); Zones::ensure_has_global_zone_v6_address( - self.physical_link.clone(), + self.underlay_vnic.clone(), *addr, &addr_name, ) @@ -220,7 +224,6 @@ impl ServiceManager { err, })?; } - */ debug!(self.log, "importing manifest"); diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 578ce1bffbd..4057524b4fa 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -41,6 +41,9 @@ pub enum Error { #[error("Failed to acquire etherstub: {0}")] Etherstub(crate::illumos::ExecutionError), + #[error("Failed to acquire etherstub VNIC: {0}")] + EtherstubVnic(crate::illumos::dladm::CreateVnicError), + #[error("Failed to lookup VNICs on boot: {0}")] GetVnics(#[from] crate::illumos::dladm::GetVnicError), @@ -122,12 +125,8 @@ impl SledAgent { let etherstub = Dladm::create_etherstub().map_err(|e| Error::Etherstub(e))?; - - let data_link = if let Some(link) = config.data_link.clone() { - link - } else { - Dladm::find_physical()? - }; + let etherstub_vnic = + Dladm::create_etherstub_vnic(ðerstub).map_err(|e| Error::EtherstubVnic(e))?; // Before we start creating zones, we need to ensure that the // necessary ZFS and Zone resources are ready. @@ -147,7 +146,7 @@ impl SledAgent { // RSS-provided IP address. In the meantime, we use one from the // configuration file. Zones::ensure_has_global_zone_v6_address( - data_link.clone(), + etherstub_vnic.clone(), *sled_address.ip(), "sled6", ) @@ -225,8 +224,11 @@ impl SledAgent { *sled_address.ip(), ); let services = - ServiceManager::new(parent_log.clone(), etherstub.clone(), None) - .await?; + ServiceManager::new( + parent_log.clone(), + etherstub.clone(), + etherstub_vnic.clone(), + None).await?; Ok(SledAgent { id: config.id, From c95b139960d44f39dd913b52701218286dbedd24 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 13 May 2022 16:40:30 -0400 Subject: [PATCH 03/13] Fmt, fix tests --- sled-agent/src/bootstrap/agent.rs | 15 +++++++------ sled-agent/src/illumos/dladm.rs | 34 ++++++------------------------ sled-agent/src/illumos/vnic.rs | 2 +- sled-agent/src/illumos/zone.rs | 2 +- sled-agent/src/instance.rs | 4 ++-- sled-agent/src/instance_manager.rs | 6 +++--- sled-agent/src/services.rs | 28 +++++++++++++----------- sled-agent/src/sled_agent.rs | 17 ++++++++------- 8 files changed, 47 insertions(+), 61 deletions(-) diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index 76a20df354d..2f184c909da 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -138,17 +138,18 @@ impl Agent { "server" => sled_config.id.to_string(), )); - let etherstub = - Dladm::create_etherstub().map_err(|e| { - BootstrapError::SledError(format!( - "Can't access etherstub device: {}", e - )) - })?; + let etherstub = Dladm::create_etherstub().map_err(|e| { + BootstrapError::SledError(format!( + "Can't access etherstub device: {}", + e + )) + })?; let etherstub_vnic = Dladm::create_etherstub_vnic(ðerstub).map_err(|e| { BootstrapError::SledError(format!( - "Can't access etherstub VNIC device: {}", e + "Can't access etherstub VNIC device: {}", + e )) })?; diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs index 4cd7f9a5921..a186367499b 100644 --- a/sled-agent/src/illumos/dladm.rs +++ b/sled-agent/src/illumos/dladm.rs @@ -86,28 +86,6 @@ pub struct Etherstub(pub String); #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] pub struct EtherstubVnic(pub String); -pub trait VnicSource { - fn name(&self) -> &str; -} - -impl VnicSource for PhysicalLink { - fn name(&self) -> &str { - &self.0 - } -} - -impl VnicSource for Etherstub { - fn name(&self) -> &str { - &self.0 - } -} - -impl VnicSource for EtherstubVnic { - fn name(&self) -> &str { - &self.0 - } -} - /// Wraps commands for interacting with data links. pub struct Dladm {} @@ -137,11 +115,13 @@ impl Dladm { /// /// This VNIC is not tracked like [`crate::illumos::vnic::Vnic`], because /// it is expected to exist for the lifetime of the sled. - pub fn create_etherstub_vnic(source: &Etherstub) -> Result { + pub fn create_etherstub_vnic( + source: &Etherstub, + ) -> Result { if let Ok(vnic) = Self::get_etherstub_vnic() { return Ok(vnic); } - Self::create_vnic(source, ETHERSTUB_VNIC_NAME, None, None)?; + Self::create_vnic(&source.0, ETHERSTUB_VNIC_NAME, None, None)?; Ok(EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string())) } @@ -208,7 +188,7 @@ impl Dladm { /// * `mac`: An optional unicast MAC address for the newly created NIC. /// * `vlan`: An optional VLAN ID for VLAN tagging. pub fn create_vnic( - source: &impl VnicSource, + source: &str, vnic_name: &str, mac: Option, vlan: Option, @@ -219,7 +199,7 @@ impl Dladm { "create-vnic".to_string(), "-t".to_string(), "-l".to_string(), - source.name().to_string(), + source.to_string(), ]; if let Some(mac) = mac { @@ -236,7 +216,7 @@ impl Dladm { let cmd = command.args(&args); execute(cmd).map_err(|err| CreateVnicError { name: vnic_name.to_string(), - link: source.name().to_string(), + link: source.to_string(), err, })?; Ok(()) diff --git a/sled-agent/src/illumos/vnic.rs b/sled-agent/src/illumos/vnic.rs index 85bb32880f8..e3c98c18daa 100644 --- a/sled-agent/src/illumos/vnic.rs +++ b/sled-agent/src/illumos/vnic.rs @@ -59,7 +59,7 @@ impl VnicAllocator { let name = allocator.next(); debug_assert!(name.starts_with(VNIC_PREFIX)); debug_assert!(name.starts_with(VNIC_PREFIX_CONTROL)); - Dladm::create_vnic(&self.data_link, &name, mac, None)?; + Dladm::create_vnic(&self.data_link.0, &name, mac, None)?; Ok(Vnic { name, deleted: false, kind: VnicKind::OxideControl }) } diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index 2f0fcf2820a..8bf450ef701 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -9,11 +9,11 @@ use ipnetwork::IpNetwork; use slog::Logger; use std::net::{IpAddr, Ipv6Addr}; -use omicron_common::address::AZ_PREFIX; use crate::illumos::addrobj::AddrObject; use crate::illumos::dladm::{EtherstubVnic, VNIC_PREFIX_CONTROL}; use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::{execute, PFEXEC}; +use omicron_common::address::AZ_PREFIX; const DLADM: &str = "/usr/sbin/dladm"; const IPADM: &str = "/usr/sbin/ipadm"; diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index bb38f84991d..6b93178426b 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -671,7 +671,7 @@ impl Instance { #[cfg(test)] mod test { use super::*; - use crate::illumos::dladm::PhysicalLink; + use crate::illumos::dladm::Etherstub; use crate::mocks::MockNexusClient; use crate::opte::OptePortAllocator; use crate::params::InstanceStateRequested; @@ -742,7 +742,7 @@ mod test { let log = logger(); let vnic_allocator = VnicAllocator::new( "Test".to_string(), - PhysicalLink("mylink".to_string()), + Etherstub("mylink".to_string()), ); let port_allocator = OptePortAllocator::new(); let nexus_client = MockNexusClient::default(); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 1791d580209..69a0c79c354 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -196,7 +196,7 @@ impl Drop for InstanceTicket { #[cfg(test)] mod test { use super::*; - use crate::illumos::dladm::PhysicalLink; + use crate::illumos::dladm::Etherstub; use crate::illumos::{dladm::MockDladm, zone::MockZones}; use crate::instance::MockInstance; use crate::mocks::MockNexusClient; @@ -260,7 +260,7 @@ mod test { let im = InstanceManager::new( log, nexus_client, - PhysicalLink("mylink".to_string()), + Etherstub("mylink".to_string()), std::net::Ipv6Addr::new( 0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, ), @@ -342,7 +342,7 @@ mod test { let im = InstanceManager::new( log, nexus_client, - PhysicalLink("mylink".to_string()), + Etherstub("mylink".to_string()), std::net::Ipv6Addr::new( 0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, ), diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index a2dbd38884c..639e39bf17b 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -480,23 +480,21 @@ impl ServiceManager { mod test { use super::*; use crate::illumos::{ - dladm::MockDladm, dladm::PhysicalLink, svc, zone::MockZones, + dladm::{Etherstub, MockDladm, ETHERSTUB_NAME, ETHERSTUB_VNIC_NAME}, + svc, + zone::MockZones, }; use std::os::unix::process::ExitStatusExt; const SVC_NAME: &str = "my_svc"; const EXPECTED_ZONE_NAME: &str = "oxz_my_svc"; - const EXPECTED_LINK_NAME: &str = "my_link"; // Returns the expectations for a new service to be created. fn expect_new_service() -> Vec> { // Create a VNIC let create_vnic_ctx = MockDladm::create_vnic_context(); create_vnic_ctx.expect().return_once(|physical_link, _, _, _| { - assert_eq!( - physical_link, - &PhysicalLink(EXPECTED_LINK_NAME.to_string()) - ); + assert_eq!(&physical_link, ÐERSTUB_NAME); Ok(()) }); // Install the Omicron Zone @@ -590,7 +588,8 @@ mod test { let config = config_dir.path().join("services.toml"); let mgr = ServiceManager::new( log, - PhysicalLink(EXPECTED_LINK_NAME.to_string()), + Etherstub(ETHERSTUB_NAME.to_string()), + EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Some(config), ) .await @@ -614,7 +613,8 @@ mod test { let config = config_dir.path().join("services.toml"); let mgr = ServiceManager::new( log, - PhysicalLink(EXPECTED_LINK_NAME.to_string()), + Etherstub(ETHERSTUB_NAME.to_string()), + EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Some(config), ) .await @@ -641,7 +641,8 @@ mod test { // down. let mgr = ServiceManager::new( logctx.log.clone(), - PhysicalLink(EXPECTED_LINK_NAME.to_string()), + Etherstub(ETHERSTUB_NAME.to_string()), + EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Some(config.clone()), ) .await @@ -654,7 +655,8 @@ mod test { let _expectations = expect_new_service(); let mgr = ServiceManager::new( logctx.log.clone(), - PhysicalLink(EXPECTED_LINK_NAME.to_string()), + Etherstub(ETHERSTUB_NAME.to_string()), + EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Some(config.clone()), ) .await @@ -678,7 +680,8 @@ mod test { // down. let mgr = ServiceManager::new( logctx.log.clone(), - PhysicalLink(EXPECTED_LINK_NAME.to_string()), + Etherstub(ETHERSTUB_NAME.to_string()), + EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Some(config.clone()), ) .await @@ -693,7 +696,8 @@ mod test { // Observe that the old service is not re-initialized. let mgr = ServiceManager::new( logctx.log.clone(), - PhysicalLink(EXPECTED_LINK_NAME.to_string()), + Etherstub(ETHERSTUB_NAME.to_string()), + EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Some(config.clone()), ) .await diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 4057524b4fa..7710f32656a 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -125,8 +125,8 @@ impl SledAgent { let etherstub = Dladm::create_etherstub().map_err(|e| Error::Etherstub(e))?; - let etherstub_vnic = - Dladm::create_etherstub_vnic(ðerstub).map_err(|e| Error::EtherstubVnic(e))?; + let etherstub_vnic = Dladm::create_etherstub_vnic(ðerstub) + .map_err(|e| Error::EtherstubVnic(e))?; // Before we start creating zones, we need to ensure that the // necessary ZFS and Zone resources are ready. @@ -223,12 +223,13 @@ impl SledAgent { etherstub.clone(), *sled_address.ip(), ); - let services = - ServiceManager::new( - parent_log.clone(), - etherstub.clone(), - etherstub_vnic.clone(), - None).await?; + let services = ServiceManager::new( + parent_log.clone(), + etherstub.clone(), + etherstub_vnic.clone(), + None, + ) + .await?; Ok(SledAgent { id: config.id, From b88c15f732996797a3f020ef7669a5de0785711b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 13 May 2022 16:42:49 -0400 Subject: [PATCH 04/13] Make opte happy too --- sled-agent/src/opte/opte.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sled-agent/src/opte/opte.rs b/sled-agent/src/opte/opte.rs index ad4dd88d463..5a12f487d09 100644 --- a/sled-agent/src/opte/opte.rs +++ b/sled-agent/src/opte/opte.rs @@ -159,7 +159,7 @@ impl OptePortAllocator { let phys = PhysicalLink(name.clone()); let vnic_name = format!("v{}", name); Dladm::create_vnic( - &phys, + &phys.0, &vnic_name, Some(omicron_common::api::external::MacAddr(mac)), None, From 211a95c018177453692b677ad40f43bb7b8ff083 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 13 May 2022 16:52:58 -0400 Subject: [PATCH 05/13] re-add type-safety to vnic creation --- sled-agent/src/illumos/dladm.rs | 27 ++++++++++++++++++++++----- sled-agent/src/illumos/vnic.rs | 2 +- sled-agent/src/opte/opte.rs | 2 +- sled-agent/src/services.rs | 10 ++++++---- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs index a186367499b..19ac155fe2f 100644 --- a/sled-agent/src/illumos/dladm.rs +++ b/sled-agent/src/illumos/dladm.rs @@ -86,6 +86,23 @@ pub struct Etherstub(pub String); #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] pub struct EtherstubVnic(pub String); +/// Identifies that an object may be used to create a VNIC. +pub trait VnicSource { + fn name(&self) -> &str; +} + +impl VnicSource for Etherstub { + fn name(&self) -> &str { + &self.0 + } +} + +impl VnicSource for PhysicalLink { + fn name(&self) -> &str { + &self.0 + } +} + /// Wraps commands for interacting with data links. pub struct Dladm {} @@ -121,7 +138,7 @@ impl Dladm { if let Ok(vnic) = Self::get_etherstub_vnic() { return Ok(vnic); } - Self::create_vnic(&source.0, ETHERSTUB_VNIC_NAME, None, None)?; + Self::create_vnic(source, ETHERSTUB_VNIC_NAME, None, None)?; Ok(EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string())) } @@ -187,8 +204,8 @@ impl Dladm { /// * `vnic_name`: Exact name of the VNIC to be created. /// * `mac`: An optional unicast MAC address for the newly created NIC. /// * `vlan`: An optional VLAN ID for VLAN tagging. - pub fn create_vnic( - source: &str, + pub fn create_vnic( + source: &T, vnic_name: &str, mac: Option, vlan: Option, @@ -199,7 +216,7 @@ impl Dladm { "create-vnic".to_string(), "-t".to_string(), "-l".to_string(), - source.to_string(), + source.name().to_string(), ]; if let Some(mac) = mac { @@ -216,7 +233,7 @@ impl Dladm { let cmd = command.args(&args); execute(cmd).map_err(|err| CreateVnicError { name: vnic_name.to_string(), - link: source.to_string(), + link: source.name().to_string(), err, })?; Ok(()) diff --git a/sled-agent/src/illumos/vnic.rs b/sled-agent/src/illumos/vnic.rs index e3c98c18daa..85bb32880f8 100644 --- a/sled-agent/src/illumos/vnic.rs +++ b/sled-agent/src/illumos/vnic.rs @@ -59,7 +59,7 @@ impl VnicAllocator { let name = allocator.next(); debug_assert!(name.starts_with(VNIC_PREFIX)); debug_assert!(name.starts_with(VNIC_PREFIX_CONTROL)); - Dladm::create_vnic(&self.data_link.0, &name, mac, None)?; + Dladm::create_vnic(&self.data_link, &name, mac, None)?; Ok(Vnic { name, deleted: false, kind: VnicKind::OxideControl }) } diff --git a/sled-agent/src/opte/opte.rs b/sled-agent/src/opte/opte.rs index 5a12f487d09..ad4dd88d463 100644 --- a/sled-agent/src/opte/opte.rs +++ b/sled-agent/src/opte/opte.rs @@ -159,7 +159,7 @@ impl OptePortAllocator { let phys = PhysicalLink(name.clone()); let vnic_name = format!("v{}", name); Dladm::create_vnic( - &phys.0, + &phys, &vnic_name, Some(omicron_common::api::external::MacAddr(mac)), None, diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 639e39bf17b..e1e2559a79f 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -493,10 +493,12 @@ mod test { fn expect_new_service() -> Vec> { // Create a VNIC let create_vnic_ctx = MockDladm::create_vnic_context(); - create_vnic_ctx.expect().return_once(|physical_link, _, _, _| { - assert_eq!(&physical_link, ÐERSTUB_NAME); - Ok(()) - }); + create_vnic_ctx.expect().return_once( + |physical_link: &Etherstub, _, _, _| { + assert_eq!(&physical_link.0, ÐERSTUB_NAME); + Ok(()) + }, + ); // Install the Omicron Zone let install_ctx = MockZones::install_omicron_zone_context(); install_ctx.expect().return_once(|_, name, _, _, _, _| { From 8b129009a4174a76a14958141393ecfe30c7738b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 13 May 2022 16:57:06 -0400 Subject: [PATCH 06/13] cleanup comments --- sled-agent/src/illumos/dladm.rs | 4 ++++ sled-agent/src/services.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs index 19ac155fe2f..2df5edc9586 100644 --- a/sled-agent/src/illumos/dladm.rs +++ b/sled-agent/src/illumos/dladm.rs @@ -19,9 +19,13 @@ pub const VNIC_PREFIX_CONTROL: &str = "oxControl"; // Viona, and thus plumbed directly to guests. pub const VNIC_PREFIX_GUEST: &str = "vopte"; +/// Path to the DLADM command. pub const DLADM: &str = "/usr/sbin/dladm"; +/// The name of the etherstub to be created for the underlay. pub const ETHERSTUB_NAME: &str = "stub0"; + +/// The name of the etherstub VNIC to be created in the global zone. pub const ETHERSTUB_VNIC_NAME: &str = "underlay0"; /// Errors returned from [`Dladm::find_physical`]. diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index e1e2559a79f..a6820936bca 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -88,7 +88,7 @@ impl ServiceManager { /// Args: /// - `log`: The logger /// - `etherstub`: An etherstub on which to allocate VNICs. - /// - `etherstub_vnic`: The underlay's VNIC in the Global Zone. + /// - `underlay_vnic`: The underlay's VNIC in the Global Zone. /// - `config_path`: An optional path to a configuration file to store /// the record of services. By default, [`default_services_config_path`] /// is used. From 5b239a3e40976e610614154649afb05b8724ee3b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 13 May 2022 18:18:47 -0400 Subject: [PATCH 07/13] Fix tests --- sled-agent/tests/integration_tests/multicast.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sled-agent/tests/integration_tests/multicast.rs b/sled-agent/tests/integration_tests/multicast.rs index 45012d087fd..a66eb8d0be6 100644 --- a/sled-agent/tests/integration_tests/multicast.rs +++ b/sled-agent/tests/integration_tests/multicast.rs @@ -40,8 +40,12 @@ async fn test_multicast_bootstrap_address() { // This modifies global state of the target machine, creating // an address named "testbootstrap6", akin to what the bootstrap // agent should do. - let link = dladm::Dladm::find_physical().unwrap(); - let address = bootstrap::agent::bootstrap_address(link.clone()).unwrap(); + let etherstub = dladm::Dladm::create_etherstub().unwrap(); + let link = dladm::Dladm::create_etherstub_vnic(ðerstub).unwrap(); + + let phys_link = dladm::Dladm::find_physical().unwrap(); + let address = + bootstrap::agent::bootstrap_address(phys_link.clone()).unwrap(); let address_name = "testbootstrap6"; let addrobj = AddrObject::new(&link.0, address_name).unwrap(); zone::Zones::ensure_has_global_zone_v6_address( From 942f84306a398bc546f579bf2d89c235e2f5a979 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 16 May 2022 08:11:54 -0400 Subject: [PATCH 08/13] Manual route --- sled-agent/src/illumos/zone.rs | 4 ++-- sled-agent/src/services.rs | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index 8bf450ef701..0610e5312ac 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -13,7 +13,7 @@ use crate::illumos::addrobj::AddrObject; use crate::illumos::dladm::{EtherstubVnic, VNIC_PREFIX_CONTROL}; use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::{execute, PFEXEC}; -use omicron_common::address::AZ_PREFIX; +use omicron_common::address::SLED_PREFIX; const DLADM: &str = "/usr/sbin/dladm"; const IPADM: &str = "/usr/sbin/ipadm"; @@ -123,7 +123,7 @@ impl AddressRequest { pub fn new_static(ip: IpAddr, prefix: Option) -> Self { let prefix = prefix.unwrap_or_else(|| match ip { IpAddr::V4(_) => 24, - IpAddr::V6(_) => AZ_PREFIX, + IpAddr::V6(_) => SLED_PREFIX, }); let addr = IpNetwork::new(ip, prefix).unwrap(); AddressRequest::Static(addr) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index a6820936bca..8db359596ce 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -10,7 +10,7 @@ use crate::illumos::vnic::VnicAllocator; use crate::illumos::zone::AddressRequest; use crate::params::{ServiceEnsureBody, ServiceRequest}; use crate::zone::Zones; -use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT}; +use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT, AZ_PREFIX}; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; @@ -200,6 +200,19 @@ impl ServiceManager { "Ensuring address {} exists - OK", addr.to_string() ); + + // TODO: + // - Verify this works: ✔ + // - Do we need to add it for non-service zone creation?: TODO + // - Improve the error handling: TODO + running_zone.run_cmd(&[ + "/usr/sbin/route", + "add", + "-inet6", + &format!("{}/{}", addr, AZ_PREFIX), + &addr.to_string(), + "-iface", + ]).expect("Failed to add route"); } info!(self.log, "GZ addresses: {:#?}", service.gz_addresses); From c9bf7046bc8bb94a7c2d6b23c56f0c72d2992b56 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 17 May 2022 16:09:22 -0400 Subject: [PATCH 09/13] wip --- sled-agent/src/illumos/running_zone.rs | 17 +++++++++++++++++ sled-agent/src/services.rs | 11 ++--------- sled-agent/src/storage_manager.rs | 2 ++ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index 68a0a28a995..a90ff00e5eb 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -10,7 +10,9 @@ use crate::illumos::vnic::{Vnic, VnicAllocator}; use crate::illumos::zone::{AddressRequest, ZONE_PREFIX}; use crate::opte::OptePort; use ipnetwork::IpNetwork; +use omicron_common::address::AZ_PREFIX; use slog::Logger; +use std::net::{IpAddr, Ipv6Addr}; use std::path::PathBuf; #[cfg(test)] @@ -171,6 +173,21 @@ impl RunningZone { Ok(network) } + pub async fn ensure_route( + &self, + // TODO: Prefer Ipv6Addr + address: IpAddr, + ) -> Result<(), RunCommandError> { + self.run_cmd(&[ + "/usr/sbin/route", + "add", + &format!("{}/{}", address, AZ_PREFIX), + &address.to_string(), + "-iface", + ])?; + Ok(()) + } + /// Looks up a running zone based on the `zone_prefix`, if one already exists. /// /// - If the zone was found, is running, and has a network interface, it is diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 8db359596ce..19c8a06b61c 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -203,16 +203,9 @@ impl ServiceManager { // TODO: // - Verify this works: ✔ - // - Do we need to add it for non-service zone creation?: TODO + // - Do we need to add it for non-service zone creation?: ✔ // - Improve the error handling: TODO - running_zone.run_cmd(&[ - "/usr/sbin/route", - "add", - "-inet6", - &format!("{}/{}", addr, AZ_PREFIX), - &addr.to_string(), - "-iface", - ]).expect("Failed to add route"); + running_zone.ensure_route(IpAddr::V6(*addr)).await.expect("Failed to add route"); } info!(self.log, "GZ addresses: {:#?}", service.gz_addresses); diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index 7ec32833277..cec46eafa97 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -486,6 +486,8 @@ async fn ensure_running_zone( let zone = RunningZone::boot(installed_zone).await?; zone.ensure_address(address_request).await?; + zone.ensure_route(dataset_info.address.ip()).await.expect("Failed to add route"); + dataset_info .start_zone(log, &zone, dataset_info.address, do_format) .await?; From 4362d6777d3faf171fe525ae0d0664e951def0e5 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 17 May 2022 18:01:42 -0400 Subject: [PATCH 10/13] Use 'route' to manually ensure routing works for non-gz zones --- sled-agent/src/illumos/running_zone.rs | 10 ++++---- sled-agent/src/params.rs | 4 +-- sled-agent/src/services.rs | 13 +++++----- sled-agent/src/sled_agent.rs | 4 +-- sled-agent/src/storage_manager.rs | 35 ++++++++++++++++---------- 5 files changed, 38 insertions(+), 28 deletions(-) diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index a90ff00e5eb..b8bd33423ff 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -10,9 +10,8 @@ use crate::illumos::vnic::{Vnic, VnicAllocator}; use crate::illumos::zone::{AddressRequest, ZONE_PREFIX}; use crate::opte::OptePort; use ipnetwork::IpNetwork; -use omicron_common::address::AZ_PREFIX; use slog::Logger; -use std::net::{IpAddr, Ipv6Addr}; +use std::net::Ipv6Addr; use std::path::PathBuf; #[cfg(test)] @@ -175,13 +174,14 @@ impl RunningZone { pub async fn ensure_route( &self, - // TODO: Prefer Ipv6Addr - address: IpAddr, + address: Ipv6Addr, ) -> Result<(), RunCommandError> { self.run_cmd(&[ "/usr/sbin/route", "add", - &format!("{}/{}", address, AZ_PREFIX), + "-inet6", + "default", + "-inet6", &address.to_string(), "-iface", ])?; diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 87fde05d7c6..34ebda537fd 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display, Formatter, Result as FormatResult}; use std::net::IpAddr; use std::net::Ipv6Addr; -use std::net::SocketAddr; +use std::net::{SocketAddr, SocketAddrV6}; use uuid::Uuid; /// Information required to construct a virtual network interface for a guest @@ -218,7 +218,7 @@ pub struct DatasetEnsureBody { // The type of the filesystem. pub dataset_kind: DatasetKind, // The address on which the zone will listen for requests. - pub address: SocketAddr, + pub address: SocketAddrV6, // NOTE: We could insert a UUID here, if we want that to be set by the // caller explicitly? Currently, the lack of a UUID implies that // "at most one dataset type" exists within a zpool. diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 19c8a06b61c..8307eda29ca 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -10,7 +10,7 @@ use crate::illumos::vnic::VnicAllocator; use crate::illumos::zone::AddressRequest; use crate::params::{ServiceEnsureBody, ServiceRequest}; use crate::zone::Zones; -use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT, AZ_PREFIX}; +use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT}; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; @@ -201,11 +201,12 @@ impl ServiceManager { addr.to_string() ); - // TODO: - // - Verify this works: ✔ - // - Do we need to add it for non-service zone creation?: ✔ - // - Improve the error handling: TODO - running_zone.ensure_route(IpAddr::V6(*addr)).await.expect("Failed to add route"); + running_zone.ensure_route(*addr).await.map_err(|err| { + Error::ZoneCommand { + intent: "Adding Route".to_string(), + err, + } + })?; } info!(self.log, "GZ addresses: {:#?}", service.gz_addresses); diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 7710f32656a..d0f14c4f326 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -22,7 +22,7 @@ use omicron_common::api::{ internal::nexus::UpdateArtifact, }; use slog::Logger; -use std::net::{SocketAddr, SocketAddrV6}; +use std::net::SocketAddrV6; use std::sync::Arc; use uuid::Uuid; @@ -261,7 +261,7 @@ impl SledAgent { &self, zpool_uuid: Uuid, dataset_kind: DatasetKind, - address: SocketAddr, + address: SocketAddrV6, ) -> Result<(), Error> { self.storage .upsert_filesystem(zpool_uuid, dataset_kind, address) diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index cec46eafa97..63c0ab0b66e 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -23,7 +23,7 @@ use serde::{Deserialize, Serialize}; use slog::Logger; use std::collections::HashMap; use std::convert::TryFrom; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddrV6}; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; @@ -200,13 +200,17 @@ impl DatasetName { // by the Sled Agent. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] struct DatasetInfo { - address: SocketAddr, + address: SocketAddrV6, kind: DatasetKind, name: DatasetName, } impl DatasetInfo { - fn new(pool: &str, kind: DatasetKind, address: SocketAddr) -> DatasetInfo { + fn new( + pool: &str, + kind: DatasetKind, + address: SocketAddrV6, + ) -> DatasetInfo { match kind { DatasetKind::CockroachDb { .. } => DatasetInfo { name: DatasetName::new(pool, "cockroachdb"), @@ -234,7 +238,7 @@ impl DatasetInfo { &self, log: &Logger, zone: &RunningZone, - address: SocketAddr, + address: SocketAddrV6, do_format: bool, ) -> Result<(), Error> { match self.kind { @@ -303,7 +307,8 @@ impl DatasetInfo { // Await liveness of the cluster. info!(log, "start_zone: awaiting liveness of CRDB"); let check_health = || async { - let http_addr = SocketAddr::new(address.ip(), 8080); + let http_addr = + SocketAddrV6::new(*address.ip(), 8080, 0, 0); reqwest::get(format!("http://{}/health?ready=1", http_addr)) .await .map_err(backoff::BackoffError::transient) @@ -456,8 +461,10 @@ async fn ensure_running_zone( dataset_name: &DatasetName, do_format: bool, ) -> Result { - let address_request = - AddressRequest::new_static(dataset_info.address.ip(), None); + let address_request = AddressRequest::new_static( + IpAddr::V6(*dataset_info.address.ip()), + None, + ); let err = RunningZone::get(log, &dataset_info.zone_prefix(), address_request) @@ -486,7 +493,9 @@ async fn ensure_running_zone( let zone = RunningZone::boot(installed_zone).await?; zone.ensure_address(address_request).await?; - zone.ensure_route(dataset_info.address.ip()).await.expect("Failed to add route"); + zone.ensure_route(*dataset_info.address.ip()) + .await + .map_err(Error::ZoneCommand)?; dataset_info .start_zone(log, &zone, dataset_info.address, do_format) @@ -518,7 +527,7 @@ type NotifyFut = dyn futures::Future< struct NewFilesystemRequest { zpool_id: Uuid, dataset_kind: DatasetKind, - address: SocketAddr, + address: SocketAddrV6, responder: oneshot::Sender>, } @@ -656,7 +665,7 @@ impl StorageWorker { fn add_datasets_notify( &self, nexus_notifications: &mut FuturesOrdered>>, - datasets: Vec<(Uuid, SocketAddr, DatasetKind)>, + datasets: Vec<(Uuid, SocketAddrV6, DatasetKind)>, pool_id: Uuid, ) { let nexus = self.nexus_client.clone(); @@ -766,7 +775,7 @@ impl StorageWorker { &self, pool: &mut Pool, dataset_name: &DatasetName, - ) -> Result<(Uuid, SocketAddr, DatasetKind), Error> { + ) -> Result<(Uuid, SocketAddrV6, DatasetKind), Error> { let name = dataset_name.full(); let id = Zfs::get_oxide_value(&name, "uuid")? .parse::() @@ -945,7 +954,7 @@ impl StorageManager { &self, zpool_id: Uuid, dataset_kind: DatasetKind, - address: SocketAddr, + address: SocketAddrV6, ) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); let request = NewFilesystemRequest { @@ -986,7 +995,7 @@ mod test { #[test] fn serialize_dataset_info() { let dataset_info = DatasetInfo { - address: "127.0.0.1:8080".parse().unwrap(), + address: "[::1]:8080".parse().unwrap(), kind: DatasetKind::Crucible, name: DatasetName::new("pool", "dataset"), }; From 44dc88512cf05133c3e13e2d28bf4d59da4f7b0e Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 31 May 2022 16:17:58 -0400 Subject: [PATCH 11/13] Add manual routes to GZ addresses --- sled-agent/src/illumos/mod.rs | 6 +++-- sled-agent/src/illumos/running_zone.rs | 10 +++----- sled-agent/src/services.rs | 34 +++++++++++++++++++------- sled-agent/src/sled_agent.rs | 18 ++++++++++++++ sled-agent/src/storage_manager.rs | 16 +++++++++--- 5 files changed, 63 insertions(+), 21 deletions(-) diff --git a/sled-agent/src/illumos/mod.rs b/sled-agent/src/illumos/mod.rs index a59521579ad..5baa7bd4421 100644 --- a/sled-agent/src/illumos/mod.rs +++ b/sled-agent/src/illumos/mod.rs @@ -15,7 +15,7 @@ pub mod zfs; pub mod zone; pub mod zpool; -const PFEXEC: &str = "/usr/bin/pfexec"; +pub const PFEXEC: &str = "/usr/bin/pfexec"; #[derive(thiserror::Error, Debug)] pub enum ExecutionError { @@ -23,11 +23,12 @@ pub enum ExecutionError { ExecutionStart { command: String, err: std::io::Error }, #[error( - "Command [{command}] executed and failed with status: {status}. Output: {stderr}" + "Command [{command}] executed and failed with status: {status}. Stdout: {stdout}, Stderr: {stderr}" )] CommandFailure { command: String, status: std::process::ExitStatus, + stdout: String, stderr: String, }, } @@ -63,6 +64,7 @@ mod inner { .collect::>() .join(" "), status: output.status, + stdout: String::from_utf8_lossy(&output.stdout).to_string(), stderr: String::from_utf8_lossy(&output.stderr).to_string(), }); } diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index b8bd33423ff..a58b4c9d185 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -11,7 +11,6 @@ use crate::illumos::zone::{AddressRequest, ZONE_PREFIX}; use crate::opte::OptePort; use ipnetwork::IpNetwork; use slog::Logger; -use std::net::Ipv6Addr; use std::path::PathBuf; #[cfg(test)] @@ -172,18 +171,17 @@ impl RunningZone { Ok(network) } - pub async fn ensure_route( + pub async fn add_route( &self, - address: Ipv6Addr, + destination: ipnetwork::Ipv6Network, ) -> Result<(), RunCommandError> { self.run_cmd(&[ "/usr/sbin/route", "add", "-inet6", - "default", + &format!("{}/{}", destination.network(), destination.prefix()), "-inet6", - &address.to_string(), - "-iface", + &destination.ip().to_string(), ])?; Ok(()) } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index c7e6579748c..0d9a31a5a85 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -10,11 +10,12 @@ use crate::illumos::vnic::VnicAllocator; use crate::illumos::zone::AddressRequest; use crate::params::{ServiceEnsureBody, ServiceRequest}; use crate::zone::Zones; -use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT}; +use ipnetwork::Ipv6Network; +use omicron_common::address::{AZ_PREFIX, DNS_PORT, DNS_SERVER_PORT}; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; -use std::net::IpAddr; +use std::net::{IpAddr, Ipv6Addr}; use std::path::{Path, PathBuf}; use tokio::sync::Mutex; @@ -79,6 +80,7 @@ pub struct ServiceManager { zones: Mutex>, vnic_allocator: VnicAllocator, underlay_vnic: EtherstubVnic, + underlay_address: Ipv6Addr, } impl ServiceManager { @@ -96,6 +98,7 @@ impl ServiceManager { log: Logger, etherstub: Etherstub, underlay_vnic: EtherstubVnic, + underlay_address: Ipv6Addr, config_path: Option, ) -> Result { debug!(log, "Creating new ServiceManager"); @@ -105,6 +108,7 @@ impl ServiceManager { zones: Mutex::new(vec![]), vnic_allocator: VnicAllocator::new("Service", etherstub), underlay_vnic, + underlay_address, }; let config_path = mgr.services_config_path(); @@ -200,13 +204,6 @@ impl ServiceManager { "Ensuring address {} exists - OK", addr.to_string() ); - - running_zone.ensure_route(*addr).await.map_err(|err| { - Error::ZoneCommand { - intent: "Adding Route".to_string(), - err, - } - })?; } info!(self.log, "GZ addresses: {:#?}", service.gz_addresses); @@ -232,6 +229,25 @@ impl ServiceManager { })?; } + let gz_route_subnet = if !service.gz_addresses.is_empty() { + // If this service supplies its own GZ address, add a route. + // + // This is currently being used for the DNS service. + // + // TODO: consider limitng the number of GZ addresses which + // can be supplied - now that we're actively using it, we + // aren't really handling the "many GZ addresses" case, and it + // doesn't seem necessary now. + Ipv6Network::new(service.gz_addresses[0], AZ_PREFIX).unwrap() + } else { + // Otherwise, add a route to the global Zone's sled address for + // everything within the AZ. + Ipv6Network::new(self.underlay_address, AZ_PREFIX).unwrap() + }; + running_zone.add_route(gz_route_subnet).await.map_err(|err| { + Error::ZoneCommand { intent: "Adding Route".to_string(), err } + })?; + debug!(self.log, "importing manifest"); running_zone diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index d0f14c4f326..c0c2ff649c8 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -9,6 +9,7 @@ use crate::illumos::vnic::VnicKind; use crate::illumos::zfs::{ Mountpoint, ZONE_ZFS_DATASET, ZONE_ZFS_DATASET_MOUNTPOINT, }; +use crate::illumos::{execute, PFEXEC}; use crate::instance_manager::InstanceManager; use crate::nexus::NexusClient; use crate::params::{ @@ -38,6 +39,9 @@ pub enum Error { #[error("Physical link not in config, nor found automatically: {0}")] FindPhysicalLink(#[from] crate::illumos::dladm::FindPhysicalLinkError), + #[error("Failed to enable routing: {0}")] + EnablingRouting(crate::illumos::ExecutionError), + #[error("Failed to acquire etherstub: {0}")] Etherstub(crate::illumos::ExecutionError), @@ -200,11 +204,24 @@ impl SledAgent { // https://github.com/oxidecomputer/omicron/issues/725. crate::opte::delete_all_xde_devices(&log)?; + // Ipv6 forwarding must be enabled to route traffic between zones. + // + // This should be a no-op if already enabled. + let mut command = std::process::Command::new(PFEXEC); + let cmd = command.args(&[ + "/usr/sbin/routeadm", + "-e", + "ipv6-forwarding", + "-u", + ]); + execute(cmd).map_err(|e| Error::EnablingRouting(e))?; + let storage = StorageManager::new( &parent_log, *id, nexus_client.clone(), etherstub.clone(), + *sled_address.ip(), ) .await; if let Some(pools) = &config.zpools { @@ -227,6 +244,7 @@ impl SledAgent { parent_log.clone(), etherstub.clone(), etherstub_vnic.clone(), + *sled_address.ip(), None, ) .await?; diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index 63c0ab0b66e..ee364b40310 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -15,7 +15,9 @@ use crate::params::DatasetKind; use futures::stream::FuturesOrdered; use futures::FutureExt; use futures::StreamExt; +use ipnetwork::Ipv6Network; use nexus_client::types::{DatasetPutRequest, ZpoolPutRequest}; +use omicron_common::address::AZ_PREFIX; use omicron_common::api::external::{ByteCount, ByteCountRangeError}; use omicron_common::backoff; use schemars::JsonSchema; @@ -23,7 +25,7 @@ use serde::{Deserialize, Serialize}; use slog::Logger; use std::collections::HashMap; use std::convert::TryFrom; -use std::net::{IpAddr, SocketAddrV6}; +use std::net::{IpAddr, Ipv6Addr, SocketAddrV6}; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; @@ -460,6 +462,7 @@ async fn ensure_running_zone( dataset_info: &DatasetInfo, dataset_name: &DatasetName, do_format: bool, + underlay_address: Ipv6Addr, ) -> Result { let address_request = AddressRequest::new_static( IpAddr::V6(*dataset_info.address.ip()), @@ -493,9 +496,10 @@ async fn ensure_running_zone( let zone = RunningZone::boot(installed_zone).await?; zone.ensure_address(address_request).await?; - zone.ensure_route(*dataset_info.address.ip()) - .await - .map_err(Error::ZoneCommand)?; + + let gz_subnet = + Ipv6Network::new(underlay_address, AZ_PREFIX).unwrap(); + zone.add_route(gz_subnet).await.map_err(Error::ZoneCommand)?; dataset_info .start_zone(log, &zone, dataset_info.address, do_format) @@ -540,6 +544,7 @@ struct StorageWorker { new_pools_rx: mpsc::Receiver, new_filesystems_rx: mpsc::Receiver, vnic_allocator: VnicAllocator, + underlay_address: Ipv6Addr, } impl StorageWorker { @@ -604,6 +609,7 @@ impl StorageWorker { dataset_info, &dataset_name, do_format, + self.underlay_address, ) .await?; @@ -901,6 +907,7 @@ impl StorageManager { sled_id: Uuid, nexus_client: Arc, etherstub: Etherstub, + underlay_address: Ipv6Addr, ) -> Self { let log = log.new(o!("component" => "StorageManager")); let pools = Arc::new(Mutex::new(HashMap::new())); @@ -914,6 +921,7 @@ impl StorageManager { new_pools_rx, new_filesystems_rx, vnic_allocator: VnicAllocator::new("Storage", etherstub), + underlay_address, }; StorageManager { pools, From 8fa90c7dca94c66d12a6c403fbe9955f71a83a15 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 31 May 2022 18:54:10 -0400 Subject: [PATCH 12/13] Add advice for conflicting addresses --- sled-agent/src/illumos/zone.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index 0610e5312ac..cf6ad746090 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -99,13 +99,14 @@ pub struct EnsureAddressError { /// Errors from [`Zones::ensure_has_global_zone_v6_address`]. #[derive(thiserror::Error, Debug)] -#[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}")] +#[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}. Note to developers: {extra_note}")] pub struct EnsureGzAddressError { address: Ipv6Addr, link: EtherstubVnic, name: String, #[source] err: anyhow::Error, + extra_note: String } /// Describes the type of addresses which may be requested from a zone. @@ -583,6 +584,22 @@ impl Zones { link, name: name.to_string(), err, + extra_note: + r#"As of https://github.com/oxidecomputer/omicron/pull/1066, we are changing the + physical device on which Global Zone addresses are allocated. + + Before this PR, we allocated addresses and VNICs directly on a physical link. + After this PR, we are allocating them on etherstubs. + + As a result, however, if your machine previously ran Omicron, it + may have addresses on the physical link which we'd like to + allocate from the etherstub instead. + + This can be fixed with the following commands: + + $ pfexec ipadm delete-addr /bootstrap6 + $ pfexec ipadm delete-addr /sled6 + $ pfexec ipadm delete-addr /internaldns"#.to_string() })?; Ok(()) } From 63cbeca3e7073dd7a596450bcf573a71db4e7006 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 31 May 2022 20:49:33 -0400 Subject: [PATCH 13/13] fix tests --- sled-agent/src/illumos/zone.rs | 2 +- sled-agent/src/services.rs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index cf6ad746090..edd57c8e0cc 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -106,7 +106,7 @@ pub struct EnsureGzAddressError { name: String, #[source] err: anyhow::Error, - extra_note: String + extra_note: String, } /// Describes the type of addresses which may be requested from a zone. diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 0d9a31a5a85..aafd89b221b 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -469,7 +469,7 @@ mod test { wait_ctx.expect().return_once(|_, _| Ok(())); // Import the manifest, enable the service let execute_ctx = crate::illumos::execute_context(); - execute_ctx.expect().times(2).returning(|_| { + execute_ctx.expect().times(3).returning(|_| { Ok(std::process::Output { status: std::process::ExitStatus::from_raw(0), stdout: vec![], @@ -545,6 +545,7 @@ mod test { log, Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), + Ipv6Addr::LOCALHOST, Some(config), ) .await @@ -570,6 +571,7 @@ mod test { log, Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), + Ipv6Addr::LOCALHOST, Some(config), ) .await @@ -598,6 +600,7 @@ mod test { logctx.log.clone(), Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), + Ipv6Addr::LOCALHOST, Some(config.clone()), ) .await @@ -612,6 +615,7 @@ mod test { logctx.log.clone(), Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), + Ipv6Addr::LOCALHOST, Some(config.clone()), ) .await @@ -637,6 +641,7 @@ mod test { logctx.log.clone(), Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), + Ipv6Addr::LOCALHOST, Some(config.clone()), ) .await @@ -653,6 +658,7 @@ mod test { logctx.log.clone(), Etherstub(ETHERSTUB_NAME.to_string()), EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), + Ipv6Addr::LOCALHOST, Some(config.clone()), ) .await