From c6f2422cfc9475872dd4514ff0336b7c8e7e8106 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 29 Jul 2022 12:04:28 -0400 Subject: [PATCH 1/4] Remove internal nexus_address from sled agent --- sled-agent/src/config.rs | 4 +--- smf/sled-agent/config.toml | 5 ----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index dce3d124f98..779592c3a01 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -13,7 +13,7 @@ use serde::Deserialize; use serde_with::serde_as; use serde_with::DisplayFromStr; use serde_with::PickFirst; -use std::net::{Ipv4Addr, SocketAddr}; +use std::net::Ipv4Addr; use std::path::{Path, PathBuf}; use uuid::Uuid; @@ -23,8 +23,6 @@ use uuid::Uuid; pub struct Config { /// Unique id for the sled pub id: Uuid, - /// Address of Nexus instance - pub nexus_address: SocketAddr, /// Configuration for the sled agent debug log pub log: ConfigLogging, /// Optional VLAN ID to be used for tagging guest VNICs. diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index 17072d7f865..e3733a8777b 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -2,11 +2,6 @@ id = "fb0f7546-4d46-40ca-9d56-cbb810684ca7" -# TODO: Remove this address - -# Internal address of Nexus -nexus_address = "[fd00:1122:3344:0101::3]:12221" - # A file-backed zpool can be manually created with the following: # # truncate -s 10GB testpool.vdev # # zpool create oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b "$PWD/testpool.vdev" From dbbebf61add046080068f9d08908f8eaa1aad1c3 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 1 Aug 2022 10:54:36 -0400 Subject: [PATCH 2/4] [sled agent] Move gateway configuration out of Sled Agent config, into RSS --- sled-agent/src/bootstrap/agent.rs | 8 +++++-- sled-agent/src/bootstrap/params.rs | 36 +++++++++++++++++++++++++++- sled-agent/src/config.rs | 20 ---------------- sled-agent/src/rack_setup/config.rs | 5 ++++ sled-agent/src/rack_setup/service.rs | 1 + sled-agent/src/server.rs | 5 ++-- sled-agent/src/sled_agent.rs | 9 +++---- smf/sled-agent/config-rss.toml | 14 +++++++++++ smf/sled-agent/config.toml | 14 ----------- 9 files changed, 69 insertions(+), 43 deletions(-) diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index 91f5010af65..2f9eb76bf56 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -277,7 +277,7 @@ impl Agent { self.parent_log.clone(), sled_address, is_scrimlet, - request.rack_id, + request.clone(), ) .await .map_err(|e| { @@ -553,8 +553,12 @@ mod tests { let request = PersistentSledAgentRequest { request: Cow::Owned(SledAgentRequest { id: Uuid::new_v4(), - subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), rack_id: Uuid::new_v4(), + gateway: crate::bootstrap::params::Gateway { + address: None, + mac: MacAddr6::nil(), + }, + subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), }), trust_quorum_share: Some( ShareDistribution { diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs index d778adeab4a..8898ac7a02f 100644 --- a/sled-agent/src/bootstrap/params.rs +++ b/sled-agent/src/bootstrap/params.rs @@ -5,11 +5,35 @@ //! Request types for the bootstrap agent use super::trust_quorum::SerializableShareDistribution; +use macaddr::MacAddr6; use omicron_common::address::{Ipv6Subnet, SLED_PREFIX}; use serde::{Deserialize, Serialize}; +use serde_with::serde_as; +use serde_with::DisplayFromStr; +use serde_with::PickFirst; use std::borrow::Cow; +use std::net::Ipv4Addr; use uuid::Uuid; +/// Information about the internet gateway used for externally-facing services. +#[serde_as] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct Gateway { + /// IP address of the Internet gateway, which is particularly + /// relevant for external-facing services (such as Nexus). + pub address: Option, + + /// MAC address of the internet gateway above. This is used to provide + /// external connectivity into guests, by allowing OPTE to forward traffic + /// destined for the broader network to the gateway. + // This uses the `serde_with` crate's `serde_as` attribute, which tries + // each of the listed serialization types (starting with the default) until + // one succeeds. This supports deserialization from either an array of u8, + // or the display-string representation. + #[serde_as(as = "PickFirst<(_, DisplayFromStr)>")] + pub mac: MacAddr6, +} + /// Configuration information for launching a Sled Agent. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct SledAgentRequest { @@ -19,6 +43,15 @@ pub struct SledAgentRequest { /// Uuid of the rack to which this sled agent belongs. pub rack_id: Uuid, + /// Information about internet gateway to use + // NOTE: This information is currently being configured and sent from RSS, + // but it contains dynamic information that could plausibly change during + // the duration of the sled's lifetime. + // + // Longer-term, it probably makes sense to store this in CRDB and transfer + // it to Sled Agent as part of the request to launch Nexus. + pub gateway: Gateway, + // Note: The order of these fields is load bearing, because we serialize // `SledAgentRequest`s as toml. `subnet` serializes as a TOML table, so it // must come after non-table fields. @@ -109,8 +142,9 @@ mod tests { request: Request::SledAgentRequest( Cow::Owned(SledAgentRequest { id: Uuid::new_v4(), - subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), rack_id: Uuid::new_v4(), + gateway: Gateway { address: None, mac: MacAddr6::nil() }, + subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), }), Some( ShareDistribution { diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 779592c3a01..e1042f3b4bd 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -8,17 +8,11 @@ use crate::common::vlan::VlanID; use crate::illumos::dladm::{self, Dladm, PhysicalLink}; use crate::illumos::zpool::ZpoolName; use dropshot::ConfigLogging; -use macaddr::MacAddr6; use serde::Deserialize; -use serde_with::serde_as; -use serde_with::DisplayFromStr; -use serde_with::PickFirst; -use std::net::Ipv4Addr; use std::path::{Path, PathBuf}; use uuid::Uuid; /// Configuration for a sled agent -#[serde_as] #[derive(Clone, Debug, Deserialize)] pub struct Config { /// Unique id for the sled @@ -30,20 +24,6 @@ pub struct Config { /// Optional list of zpools to be used as "discovered disks". pub zpools: Option>, - /// IP address of the Internet gateway, which is particularly - /// relevant for external-facing services (such as Nexus). - pub gateway_address: Option, - - /// MAC address of the internet gateway above. This is used to provide - /// external connectivity into guests, by allowing OPTE to forward traffic - /// destined for the broader network to the gateway. - // This uses the `serde_with` crate's `serde_as` attribute, which tries - // each of the listed serialization types (starting with the default) until - // one succeeds. This supports deserialization from either an array of u8, - // or the display-string representation. - #[serde_as(as = "PickFirst<(_, DisplayFromStr)>")] - pub gateway_mac: MacAddr6, - /// The data link on which we infer the bootstrap address. /// /// If unsupplied, we default to the first physical device. diff --git a/sled-agent/src/rack_setup/config.rs b/sled-agent/src/rack_setup/config.rs index 1006514b3a9..ee56431a6ee 100644 --- a/sled-agent/src/rack_setup/config.rs +++ b/sled-agent/src/rack_setup/config.rs @@ -4,6 +4,7 @@ //! Interfaces for working with RSS config. +use crate::bootstrap::params::Gateway; use crate::config::ConfigError; use crate::params::{DatasetEnsureBody, ServiceRequest}; use omicron_common::address::{ @@ -36,6 +37,9 @@ pub struct SetupServiceConfig { /// If this value is less than 2, no rack secret will be created on startup; /// this is the typical case for single-server test/development. pub rack_secret_threshold: usize, + + /// Internet gateway information. + pub gateway: Gateway, } /// A request to initialize a sled. @@ -89,6 +93,7 @@ mod test { rack_subnet: "fd00:1122:3344:0100::".parse().unwrap(), requests: vec![], rack_secret_threshold: 0, + gateway: Gateway { address: None, mac: macaddr::MacAddr6::nil() }, }; assert_eq!( diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 45f133a5eb6..7c8c3fda6be 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -356,6 +356,7 @@ impl ServiceInner { id: Uuid::new_v4(), subnet, rack_id, + gateway: config.gateway.clone(), }, services_request: request, }, diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index 78f891712a4..4319f7019af 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -7,6 +7,7 @@ use super::config::Config; use super::http_entrypoints::api as http_api; use super::sled_agent::SledAgent; +use crate::bootstrap::params::SledAgentRequest; use crate::nexus::LazyNexusClient; use omicron_common::backoff::{ internal_service_policy_with_max, retry_notify, BackoffError, @@ -38,7 +39,7 @@ impl Server { log: Logger, addr: SocketAddrV6, is_scrimlet: bool, - rack_id: Uuid, + request: SledAgentRequest, ) -> Result { info!(log, "setting up sled agent server"); @@ -52,7 +53,7 @@ impl Server { log.clone(), lazy_nexus_client.clone(), addr, - rack_id, + request, ) .await .map_err(|e| e.to_string())?; diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index d9040618fae..48b72be5a95 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -4,6 +4,7 @@ //! Sled agent implementation +use crate::bootstrap::params::SledAgentRequest; use crate::config::Config; use crate::illumos::vnic::VnicKind; use crate::illumos::zfs::{ @@ -126,7 +127,7 @@ impl SledAgent { log: Logger, lazy_nexus_client: LazyNexusClient, sled_address: SocketAddrV6, - rack_id: Uuid, + request: SledAgentRequest, ) -> Result { let id = config.id; @@ -254,11 +255,11 @@ impl SledAgent { lazy_nexus_client.clone(), etherstub.clone(), *sled_address.ip(), - config.gateway_mac, + request.gateway.mac, ); let svc_config = services::Config { - gateway_address: config.gateway_address, + gateway_address: request.gateway.address, ..Default::default() }; let services = ServiceManager::new( @@ -268,7 +269,7 @@ impl SledAgent { *sled_address.ip(), svc_config, config.get_link()?, - rack_id, + request.rack_id, ) .await?; diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index fbb5214c556..251741d5663 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -11,6 +11,20 @@ rack_subnet = "fd00:1122:3344:0100::" # For values less than 2, no rack secret will be generated. rack_secret_threshold = 1 +# IP address of Internet gateway +# +# NOTE: In the lab, use "172.20.15.225" +# gateway_address = "192.168.1.1" + +# MAC address of the internet gateway in the local network, i.e., of the above +# IP address. +# +# NOTE: This is currently configured for the lab. Developers should override +# this with whatever value they wish to provide inbound connectivity to guests +# in their local network, using the current workaround methods in OPTE. See +# how-to-run.adoc for details on how to determine the value for your network. +gateway_mac = "00:0d:b9:54:fe:e4" + [[request]] # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index e3733a8777b..9af1db6f2e2 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -21,20 +21,6 @@ zpools = [ # $ dladm show-phys -p -o LINK # data_link = "igb0" -# IP address of Internet gateway -# -# NOTE: In the lab, use "172.20.15.225" -# gateway_address = "192.168.1.1" - -# MAC address of the internet gateway in the local network, i.e., of the above -# IP address. -# -# NOTE: This is currently configured for the lab. Developers should override -# this with whatever value they wish to provide inbound connectivity to guests -# in their local network, using the current workaround methods in OPTE. See -# how-to-run.adoc for details on how to determine the value for your network. -gateway_mac = "00:0d:b9:54:fe:e4" - [log] level = "info" mode = "file" From 9ab7570db54860b171e5aeab3094aba29200b1da Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 1 Aug 2022 10:57:58 -0400 Subject: [PATCH 3/4] Update docs --- docs/how-to-run.adoc | 4 ++-- sled-agent/src/config.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 16218589d9b..8323dfe9106 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -78,7 +78,7 @@ Transformation Engine (OPTE, the kernel module which provides the private virtual networking to guests). To make this work, OPTE needs to know about the local networking configuration, in this case the MAC address of the local internet gateway. To be able to get into your instances, you _must_ specify this -in the `gateway_mac` field of the config file `smf/sled-agent/config.toml`. +in the `gateway_mac` field of the config file `smf/sled-agent/config-rss.toml`. The value there is correct for the lab environment, so if you're running there, no changes are needed. If you're running elsewhere, you find the value with: @@ -174,7 +174,7 @@ be set as a default route for the Nexus zone. | Crucible Downstairs 3 | `[fd00:1122:3344:0101::8]:32345` | Internal DNS Service | `[fd00:1122:3344:0001::1]:5353` | Nexus: External API | `192.168.1.20:80` -| Internet Gateway | None, but can be set in `smf/sled-agent/config.toml` +| Internet Gateway | None, but can be set in `smf/sled-agent/config-rss.toml` |=================================================================================================== Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 553ff5fb75f..e1042f3b4bd 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -9,7 +9,6 @@ use crate::illumos::dladm::{self, Dladm, PhysicalLink}; use crate::illumos::zpool::ZpoolName; use dropshot::ConfigLogging; use serde::Deserialize; -use std::net::Ipv4Addr; use std::path::{Path, PathBuf}; use uuid::Uuid; From 24eec2aba6474bad371d799cfdeed292d89e7205 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 1 Aug 2022 11:07:56 -0400 Subject: [PATCH 4/4] use the right names --- docs/how-to-run.adoc | 2 +- smf/sled-agent/config-rss.toml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 8323dfe9106..302ade0597e 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -78,7 +78,7 @@ Transformation Engine (OPTE, the kernel module which provides the private virtual networking to guests). To make this work, OPTE needs to know about the local networking configuration, in this case the MAC address of the local internet gateway. To be able to get into your instances, you _must_ specify this -in the `gateway_mac` field of the config file `smf/sled-agent/config-rss.toml`. +in the `gateway.mac` field of the config file `smf/sled-agent/config-rss.toml`. The value there is correct for the lab environment, so if you're running there, no changes are needed. If you're running elsewhere, you find the value with: diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index 251741d5663..27b828da7bd 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -11,10 +11,12 @@ rack_subnet = "fd00:1122:3344:0100::" # For values less than 2, no rack secret will be generated. rack_secret_threshold = 1 +[gateway] + # IP address of Internet gateway # # NOTE: In the lab, use "172.20.15.225" -# gateway_address = "192.168.1.1" +# address = "192.168.1.1" # MAC address of the internet gateway in the local network, i.e., of the above # IP address. @@ -23,7 +25,7 @@ rack_secret_threshold = 1 # this with whatever value they wish to provide inbound connectivity to guests # in their local network, using the current workaround methods in OPTE. See # how-to-run.adoc for details on how to determine the value for your network. -gateway_mac = "00:0d:b9:54:fe:e4" +mac = "00:0d:b9:54:fe:e4" [[request]]