diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 16218589d9b..302ade0597e 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -78,7 +78,7 @@ Transformation Engine (OPTE, the kernel module which provides the private virtual networking to guests). To make this work, OPTE needs to know about the local networking configuration, in this case the MAC address of the local internet gateway. To be able to get into your instances, you _must_ specify this -in the `gateway_mac` field of the config file `smf/sled-agent/config.toml`. +in the `gateway.mac` field of the config file `smf/sled-agent/config-rss.toml`. The value there is correct for the lab environment, so if you're running there, no changes are needed. If you're running elsewhere, you find the value with: @@ -174,7 +174,7 @@ be set as a default route for the Nexus zone. | Crucible Downstairs 3 | `[fd00:1122:3344:0101::8]:32345` | Internal DNS Service | `[fd00:1122:3344:0001::1]:5353` | Nexus: External API | `192.168.1.20:80` -| Internet Gateway | None, but can be set in `smf/sled-agent/config.toml` +| Internet Gateway | None, but can be set in `smf/sled-agent/config-rss.toml` |=================================================================================================== Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index 91f5010af65..2f9eb76bf56 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -277,7 +277,7 @@ impl Agent { self.parent_log.clone(), sled_address, is_scrimlet, - request.rack_id, + request.clone(), ) .await .map_err(|e| { @@ -553,8 +553,12 @@ mod tests { let request = PersistentSledAgentRequest { request: Cow::Owned(SledAgentRequest { id: Uuid::new_v4(), - subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), rack_id: Uuid::new_v4(), + gateway: crate::bootstrap::params::Gateway { + address: None, + mac: MacAddr6::nil(), + }, + subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), }), trust_quorum_share: Some( ShareDistribution { diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs index d778adeab4a..8898ac7a02f 100644 --- a/sled-agent/src/bootstrap/params.rs +++ b/sled-agent/src/bootstrap/params.rs @@ -5,11 +5,35 @@ //! Request types for the bootstrap agent use super::trust_quorum::SerializableShareDistribution; +use macaddr::MacAddr6; use omicron_common::address::{Ipv6Subnet, SLED_PREFIX}; use serde::{Deserialize, Serialize}; +use serde_with::serde_as; +use serde_with::DisplayFromStr; +use serde_with::PickFirst; use std::borrow::Cow; +use std::net::Ipv4Addr; use uuid::Uuid; +/// Information about the internet gateway used for externally-facing services. +#[serde_as] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct Gateway { + /// IP address of the Internet gateway, which is particularly + /// relevant for external-facing services (such as Nexus). + pub address: Option, + + /// MAC address of the internet gateway above. This is used to provide + /// external connectivity into guests, by allowing OPTE to forward traffic + /// destined for the broader network to the gateway. + // This uses the `serde_with` crate's `serde_as` attribute, which tries + // each of the listed serialization types (starting with the default) until + // one succeeds. This supports deserialization from either an array of u8, + // or the display-string representation. + #[serde_as(as = "PickFirst<(_, DisplayFromStr)>")] + pub mac: MacAddr6, +} + /// Configuration information for launching a Sled Agent. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct SledAgentRequest { @@ -19,6 +43,15 @@ pub struct SledAgentRequest { /// Uuid of the rack to which this sled agent belongs. pub rack_id: Uuid, + /// Information about internet gateway to use + // NOTE: This information is currently being configured and sent from RSS, + // but it contains dynamic information that could plausibly change during + // the duration of the sled's lifetime. + // + // Longer-term, it probably makes sense to store this in CRDB and transfer + // it to Sled Agent as part of the request to launch Nexus. + pub gateway: Gateway, + // Note: The order of these fields is load bearing, because we serialize // `SledAgentRequest`s as toml. `subnet` serializes as a TOML table, so it // must come after non-table fields. @@ -109,8 +142,9 @@ mod tests { request: Request::SledAgentRequest( Cow::Owned(SledAgentRequest { id: Uuid::new_v4(), - subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), rack_id: Uuid::new_v4(), + gateway: Gateway { address: None, mac: MacAddr6::nil() }, + subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), }), Some( ShareDistribution { diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 779592c3a01..e1042f3b4bd 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -8,17 +8,11 @@ use crate::common::vlan::VlanID; use crate::illumos::dladm::{self, Dladm, PhysicalLink}; use crate::illumos::zpool::ZpoolName; use dropshot::ConfigLogging; -use macaddr::MacAddr6; use serde::Deserialize; -use serde_with::serde_as; -use serde_with::DisplayFromStr; -use serde_with::PickFirst; -use std::net::Ipv4Addr; use std::path::{Path, PathBuf}; use uuid::Uuid; /// Configuration for a sled agent -#[serde_as] #[derive(Clone, Debug, Deserialize)] pub struct Config { /// Unique id for the sled @@ -30,20 +24,6 @@ pub struct Config { /// Optional list of zpools to be used as "discovered disks". pub zpools: Option>, - /// IP address of the Internet gateway, which is particularly - /// relevant for external-facing services (such as Nexus). - pub gateway_address: Option, - - /// MAC address of the internet gateway above. This is used to provide - /// external connectivity into guests, by allowing OPTE to forward traffic - /// destined for the broader network to the gateway. - // This uses the `serde_with` crate's `serde_as` attribute, which tries - // each of the listed serialization types (starting with the default) until - // one succeeds. This supports deserialization from either an array of u8, - // or the display-string representation. - #[serde_as(as = "PickFirst<(_, DisplayFromStr)>")] - pub gateway_mac: MacAddr6, - /// The data link on which we infer the bootstrap address. /// /// If unsupplied, we default to the first physical device. diff --git a/sled-agent/src/rack_setup/config.rs b/sled-agent/src/rack_setup/config.rs index 1006514b3a9..ee56431a6ee 100644 --- a/sled-agent/src/rack_setup/config.rs +++ b/sled-agent/src/rack_setup/config.rs @@ -4,6 +4,7 @@ //! Interfaces for working with RSS config. +use crate::bootstrap::params::Gateway; use crate::config::ConfigError; use crate::params::{DatasetEnsureBody, ServiceRequest}; use omicron_common::address::{ @@ -36,6 +37,9 @@ pub struct SetupServiceConfig { /// If this value is less than 2, no rack secret will be created on startup; /// this is the typical case for single-server test/development. pub rack_secret_threshold: usize, + + /// Internet gateway information. + pub gateway: Gateway, } /// A request to initialize a sled. @@ -89,6 +93,7 @@ mod test { rack_subnet: "fd00:1122:3344:0100::".parse().unwrap(), requests: vec![], rack_secret_threshold: 0, + gateway: Gateway { address: None, mac: macaddr::MacAddr6::nil() }, }; assert_eq!( diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 45f133a5eb6..7c8c3fda6be 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -356,6 +356,7 @@ impl ServiceInner { id: Uuid::new_v4(), subnet, rack_id, + gateway: config.gateway.clone(), }, services_request: request, }, diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index 78f891712a4..4319f7019af 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -7,6 +7,7 @@ use super::config::Config; use super::http_entrypoints::api as http_api; use super::sled_agent::SledAgent; +use crate::bootstrap::params::SledAgentRequest; use crate::nexus::LazyNexusClient; use omicron_common::backoff::{ internal_service_policy_with_max, retry_notify, BackoffError, @@ -38,7 +39,7 @@ impl Server { log: Logger, addr: SocketAddrV6, is_scrimlet: bool, - rack_id: Uuid, + request: SledAgentRequest, ) -> Result { info!(log, "setting up sled agent server"); @@ -52,7 +53,7 @@ impl Server { log.clone(), lazy_nexus_client.clone(), addr, - rack_id, + request, ) .await .map_err(|e| e.to_string())?; diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index d9040618fae..48b72be5a95 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -4,6 +4,7 @@ //! Sled agent implementation +use crate::bootstrap::params::SledAgentRequest; use crate::config::Config; use crate::illumos::vnic::VnicKind; use crate::illumos::zfs::{ @@ -126,7 +127,7 @@ impl SledAgent { log: Logger, lazy_nexus_client: LazyNexusClient, sled_address: SocketAddrV6, - rack_id: Uuid, + request: SledAgentRequest, ) -> Result { let id = config.id; @@ -254,11 +255,11 @@ impl SledAgent { lazy_nexus_client.clone(), etherstub.clone(), *sled_address.ip(), - config.gateway_mac, + request.gateway.mac, ); let svc_config = services::Config { - gateway_address: config.gateway_address, + gateway_address: request.gateway.address, ..Default::default() }; let services = ServiceManager::new( @@ -268,7 +269,7 @@ impl SledAgent { *sled_address.ip(), svc_config, config.get_link()?, - rack_id, + request.rack_id, ) .await?; diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index fbb5214c556..27b828da7bd 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -11,6 +11,22 @@ rack_subnet = "fd00:1122:3344:0100::" # For values less than 2, no rack secret will be generated. rack_secret_threshold = 1 +[gateway] + +# IP address of Internet gateway +# +# NOTE: In the lab, use "172.20.15.225" +# address = "192.168.1.1" + +# MAC address of the internet gateway in the local network, i.e., of the above +# IP address. +# +# NOTE: This is currently configured for the lab. Developers should override +# this with whatever value they wish to provide inbound connectivity to guests +# in their local network, using the current workaround methods in OPTE. See +# how-to-run.adoc for details on how to determine the value for your network. +mac = "00:0d:b9:54:fe:e4" + [[request]] # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index e3733a8777b..9af1db6f2e2 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -21,20 +21,6 @@ zpools = [ # $ dladm show-phys -p -o LINK # data_link = "igb0" -# IP address of Internet gateway -# -# NOTE: In the lab, use "172.20.15.225" -# gateway_address = "192.168.1.1" - -# MAC address of the internet gateway in the local network, i.e., of the above -# IP address. -# -# NOTE: This is currently configured for the lab. Developers should override -# this with whatever value they wish to provide inbound connectivity to guests -# in their local network, using the current workaround methods in OPTE. See -# how-to-run.adoc for details on how to determine the value for your network. -gateway_mac = "00:0d:b9:54:fe:e4" - [log] level = "info" mode = "file"