From a0d50a6d77073fc42deeab5d825b779601633b8d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 29 Jun 2022 23:58:32 -0400 Subject: [PATCH 01/18] [sled-agent] Launch nexus on an IPv4 address. Routing still broken. --- sled-agent/src/params.rs | 2 +- sled-agent/src/services.rs | 56 +++++++++++++--------------------- sled-agent/src/sled_agent.rs | 4 +++ smf/sled-agent/config-rss.toml | 2 +- 4 files changed, 27 insertions(+), 37 deletions(-) diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 4752caad940..d5428358acd 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -290,7 +290,7 @@ impl From for sled_agent_client::types::DatasetEnsureBody { )] #[serde(tag = "type", rename_all = "snake_case")] pub enum ServiceType { - Nexus { internal_address: SocketAddrV6, external_address: SocketAddrV6 }, + Nexus { internal_address: SocketAddrV6, external_address: SocketAddr }, InternalDns { server_address: SocketAddrV6, dns_address: SocketAddrV6 }, Oximeter, } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index be11bfb2a6d..ec2733718b1 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -230,7 +230,7 @@ impl ServiceManager { &[], // devices= &[], - // vnics= + // opte_ports= vec![], ) .await?; @@ -277,7 +277,7 @@ impl ServiceManager { // // This is currently being used for the DNS service. // - // TODO: consider limitng the number of GZ addresses which + // TODO: consider limiting the number of GZ addresses which // can be supplied - now that we're actively using it, we // aren't really handling the "many GZ addresses" case, and it // doesn't seem necessary now. @@ -315,6 +315,10 @@ impl ServiceManager { match service.service_type { ServiceType::Nexus { internal_address, external_address } => { info!(self.log, "Setting up Nexus service"); + // The address of Nexus' external interface is a special + // case; it may be an IPv4 address. + let addr_request = AddressRequest::new_static(external_address.ip(), None); + running_zone.ensure_address(addr_request).await?; // Nexus takes a separate config file for parameters which // cannot be known at packaging time. @@ -322,7 +326,7 @@ impl ServiceManager { id: service.id, rack_id: self.rack_id, dropshot_external: ConfigDropshot { - bind_address: SocketAddr::V6(external_address), + bind_address: external_address, request_body_max_bytes: 1048576, ..Default::default() }, @@ -578,7 +582,7 @@ mod test { svc, zone::MockZones, }; - use std::net::{Ipv6Addr, SocketAddrV6}; + use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6}; use std::os::unix::process::ExitStatusExt; use uuid::Uuid; @@ -607,12 +611,19 @@ mod test { assert_eq!(name, EXPECTED_ZONE_NAME); Ok(()) }); + + // Ensure the address exists + let ensure_address_ctx = MockZones::ensure_address_context(); + ensure_address_ctx.expect().return_once(|_, _, _| { + Ok(ipnetwork::IpNetwork::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 64).unwrap()) + }); + // Wait for the networking service. let wait_ctx = svc::wait_for_service_context(); wait_ctx.expect().return_once(|_, _| Ok(())); // Import the manifest, enable the service let execute_ctx = crate::illumos::execute_context(); - execute_ctx.expect().times(3).returning(|_| { + execute_ctx.expect().times(..).returning(|_| { Ok(std::process::Output { status: std::process::ExitStatus::from_raw(0), stdout: vec![], @@ -624,6 +635,7 @@ mod test { Box::new(create_vnic_ctx), Box::new(install_ctx), Box::new(boot_ctx), + Box::new(ensure_address_ctx), Box::new(wait_ctx), Box::new(execute_ctx), ] @@ -637,22 +649,9 @@ mod test { services: vec![ServiceRequest { id, name: SVC_NAME.to_string(), - addresses: vec![], + addresses: vec![Ipv6Addr::LOCALHOST], gz_addresses: vec![], - service_type: ServiceType::Nexus { - internal_address: SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - ), - external_address: SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - ), - }, + service_type: ServiceType::Oximeter, }], }) .await @@ -666,22 +665,9 @@ mod test { services: vec![ServiceRequest { id, name: SVC_NAME.to_string(), - addresses: vec![], + addresses: vec![Ipv6Addr::LOCALHOST], gz_addresses: vec![], - service_type: ServiceType::Nexus { - internal_address: SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - ), - external_address: SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - ), - }, + service_type: ServiceType::Oximeter, }], }) .await diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index e385d2271ad..d09bf076fb5 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -231,6 +231,10 @@ impl SledAgent { let mut command = std::process::Command::new(PFEXEC); let cmd = command.args(&[ "/usr/sbin/routeadm", + // Needed to access Nexus's external interface. + "-e", + "ipv4-forwarding", + // Needed to access all other zones, which are on the underlay. "-e", "ipv6-forwarding", "-u", diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index a5f06d1cfc5..2ffbd38e1a0 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -56,7 +56,7 @@ gz_addresses = [] [request.service.service_type] type = "nexus" internal_address = "[fd00:1122:3344:0101::3]:12221" -external_address = "[fd00:1122:3344:0101::3]:12220" +external_address = "172.20.15.224:12220" # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate Oximeter services. From a1ec4cd85bee09dfc08090f384b2d0c7cb278d84 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 00:42:11 -0400 Subject: [PATCH 02/18] Update the dbwipe commands for parity with dbinit --- common/src/sql/dbwipe.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/sql/dbwipe.sql b/common/src/sql/dbwipe.sql index 40460bacb47..dcf7a78f4af 100644 --- a/common/src/sql/dbwipe.sql +++ b/common/src/sql/dbwipe.sql @@ -9,4 +9,5 @@ * initialization code and dbwipe.sql. */ DROP DATABASE IF EXISTS omicron; +ALTER DEFAULT PRIVILEGES FOR ROLE root REVOKE ALL ON TABLES FROM omicron; DROP USER IF EXISTS omicron; From a32cde77fb5790d7814c723495835996b868f4e1 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 01:42:21 -0400 Subject: [PATCH 03/18] now we're cooking with ipv4 on the underlay --- sled-agent/src/illumos/running_zone.rs | 15 +++++++++- sled-agent/src/illumos/zone.rs | 40 ++++++++++++++++++++++++-- sled-agent/src/services.rs | 10 +++++++ sled-agent/src/sled_agent.rs | 34 +++++++++++++++++++++- 4 files changed, 94 insertions(+), 5 deletions(-) diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index fee23187688..fc8eff18894 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -11,7 +11,7 @@ use crate::illumos::zone::{AddressRequest, ZONE_PREFIX}; use crate::opte::OptePort; use ipnetwork::IpNetwork; use slog::Logger; -use std::net::Ipv6Addr; +use std::net::{Ipv4Addr, Ipv6Addr}; use std::path::PathBuf; #[cfg(test)] @@ -187,6 +187,19 @@ impl RunningZone { Ok(()) } + pub async fn add_default_route4( + &self, + gateway: Ipv4Addr, + ) -> Result<(), RunCommandError> { + self.run_cmd(&[ + "/usr/sbin/route", + "add", + "default", + &gateway.to_string(), + ])?; + Ok(()) + } + /// Looks up a running zone based on the `zone_prefix`, if one already exists. /// /// - If the zone was found, is running, and has a network interface, it is diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index 4c7ba3baaa7..9833f601a01 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -7,7 +7,7 @@ use anyhow::anyhow; use ipnetwork::IpNetwork; use slog::Logger; -use std::net::{IpAddr, Ipv6Addr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use crate::illumos::addrobj::AddrObject; use crate::illumos::dladm::{EtherstubVnic, VNIC_PREFIX_CONTROL}; @@ -101,7 +101,7 @@ pub struct EnsureAddressError { #[derive(thiserror::Error, Debug)] #[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}. Note to developers: {extra_note}")] pub struct EnsureGzAddressError { - address: Ipv6Addr, + address: IpAddr, link: EtherstubVnic, name: String, #[source] @@ -541,6 +541,40 @@ impl Zones { Ok(()) } + // TODO: Remove once Nexus traffic is transmitted over OPTE. + pub fn ensure_has_global_zone_v4_address( + link: EtherstubVnic, + address: Ipv4Addr, + name: &str, + ) -> Result<(), EnsureGzAddressError> { + // Call the guts of this function within a closure to make it easier + // to wrap the error with appropriate context. + |link: EtherstubVnic, address, name| -> Result<(), anyhow::Error> { + // Ensure that a static IPv4 address has been allocated + // to the Global Zone. Without this, we don't have a way + // to route to IP addresses that we want to create in + // the non-GZ. + Self::ensure_address( + None, + &AddrObject::new(&link.0, name).unwrap(), + AddressRequest::new_static( + IpAddr::V4(address), + None, + ), + ) + .map_err(|err| anyhow!(err))?; + Ok(()) + }(link.clone(), address, name) + .map_err(|err| EnsureGzAddressError { + address: IpAddr::V4(address), + link, + name: name.to_string(), + err, + extra_note: "".to_string(), + })?; + Ok(()) + } + // TODO(https://github.com/oxidecomputer/omicron/issues/821): We // should remove this function when Sled Agents are provided IPv6 addresses // from RSS. @@ -580,7 +614,7 @@ impl Zones { Ok(()) }(link.clone(), address, name) .map_err(|err| EnsureGzAddressError { - address, + address: IpAddr::V6(address), link, name: name.to_string(), err, diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index a9e13bbb42f..2aded5945d6 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -343,6 +343,16 @@ impl ServiceManager { match service.service_type { ServiceType::Nexus { internal_address, external_address } => { info!(self.log, "Setting up Nexus service"); + + // TODO: Remove once Nexus traffic is transmitted over OPTE. + let gateway4 = crate::sled_agent::get_gz_ipv4(gateway); + running_zone.add_default_route4(gateway4).await.map_err(|err| { + Error::ZoneCommand { intent: "Adding Route".to_string(), err } + })?; + + // TODO: Do i need to add a route from the GZ to this zone + // too? + // The address of Nexus' external interface is a special // case; it may be an IPv4 address. let addr_request = AddressRequest::new_static(external_address.ip(), None); diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index d09bf076fb5..4e6d088f75f 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -25,7 +25,7 @@ use omicron_common::api::{ internal::nexus::UpdateArtifact, }; use slog::Logger; -use std::net::SocketAddrV6; +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -90,6 +90,30 @@ impl From for omicron_common::api::external::Error { } } +// TODO: Remove once Nexus traffic is transmitted over OPTE. +// This is a hack necessary for routing IPv4 traffic to zones hosting external +// services. +// +// Here's the deal: For a sled IPv6 addresses, different portions of the +// prefix act like indices: +// +// fd00:1122:3344:0101::/64 +// | | ^^ Sled +// | ^^ Rack +// ^^^^^^^^^^^^^ AZ +// +// These IPv6 addresses are allocated uniquely to sleds. However, for +// our short-term hackish purposes, we'd like to allocate IPv4 addresses +// on the underlay too, so we can do the routing we need to do. +// +// We'd like this to be as lightweight as possible. Rather than +// explicitly provisioning anything in nexus, we piggyback on the +// existing IPv6 address. By grabbing the "sled octet" from the IPv6 +// address, we get a rack-wide unique identifier. +pub fn get_gz_ipv4(ip6: Ipv6Addr) -> Ipv4Addr { + Ipv4Addr::new(10, 0, ip6.octets()[7], 1) +} + /// Describes an executing Sled Agent object. /// /// Contains both a connection to the Nexus, as well as managed instances. @@ -160,6 +184,14 @@ impl SledAgent { ) .map_err(|err| Error::SledSubnet { err })?; + // TODO: Remove once Nexus traffic is transmitted over OPTE. + Zones::ensure_has_global_zone_v4_address( + etherstub_vnic.clone(), + get_gz_ipv4(*sled_address.ip()), + "sled4", + ) + .map_err(|err| Error::SledSubnet { err })?; + // Initialize the xde kernel driver with the underlay devices. crate::opte::initialize_xde_driver(&log)?; From 91bf43e613265b71a732c332c2b732ed184a220d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 02:04:34 -0400 Subject: [PATCH 04/18] create pub/private ipv4s for nexus, start setting up routes --- sled-agent/src/illumos/running_zone.rs | 10 +++++++- sled-agent/src/services.rs | 33 ++++++++++++++++++-------- sled-agent/src/sled_agent.rs | 9 +++++++ 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index fc8eff18894..65bc6e2db0a 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -153,7 +153,6 @@ impl RunningZone { &self, addrtype: AddressRequest, ) -> Result { - info!(self.inner.log, "Adding address: {:?}", addrtype); let name = match addrtype { AddressRequest::Dhcp => "omicron", AddressRequest::Static(net) => match net.ip() { @@ -161,6 +160,15 @@ impl RunningZone { std::net::IpAddr::V6(_) => "omicron6", }, }; + self.ensure_address_with_name(addrtype, name).await + } + + pub async fn ensure_address_with_name( + &self, + addrtype: AddressRequest, + name: &str, + ) -> Result { + info!(self.inner.log, "Adding address: {:?}", addrtype); let addrobj = AddrObject::new(self.inner.control_vnic.name(), name) .map_err(|err| EnsureAddressError::AddrObject { request: addrtype, diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 2aded5945d6..944fbfd3b18 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -344,19 +344,32 @@ impl ServiceManager { ServiceType::Nexus { internal_address, external_address } => { info!(self.log, "Setting up Nexus service"); - // TODO: Remove once Nexus traffic is transmitted over OPTE. - let gateway4 = crate::sled_agent::get_gz_ipv4(gateway); - running_zone.add_default_route4(gateway4).await.map_err(|err| { - Error::ZoneCommand { intent: "Adding Route".to_string(), err } - })?; - - // TODO: Do i need to add a route from the GZ to this zone - // too? - // The address of Nexus' external interface is a special // case; it may be an IPv4 address. let addr_request = AddressRequest::new_static(external_address.ip(), None); - running_zone.ensure_address(addr_request).await?; + running_zone.ensure_address_with_name(addr_request, "public").await?; + + // TODO: Remove once Nexus traffic is transmitted over OPTE. + match external_address.ip() { + IpAddr::V4(_) => { + // Create an address which is routable from the GZ's + // Ipv4 address. + let private_addr4 = crate::sled_agent::get_nexus_private_ipv4(self.underlay_address); + let addr_request = AddressRequest::new_static(IpAddr::V4(private_addr4), None); + running_zone.ensure_address_with_name(addr_request, "private").await?; + + // Create a default route back through the GZ's + // underlay. + let gateway4 = crate::sled_agent::get_gz_ipv4(self.underlay_address); + running_zone.add_default_route4(gateway4).await.map_err(|err| { + Error::ZoneCommand { intent: "Adding Route".to_string(), err } + })?; + + // TODO: Do i need to add a route from the GZ to this zone + // too? + } + _ => (), + } // Nexus takes a separate config file for parameters which // cannot be known at packaging time. diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 4e6d088f75f..03e590569c0 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -114,6 +114,15 @@ pub fn get_gz_ipv4(ip6: Ipv6Addr) -> Ipv4Addr { Ipv4Addr::new(10, 0, ip6.octets()[7], 1) } +// TODO: Remove once Nexus traffic is transmitted over OPTE. +// +// Input: The sled underlay IPv6 address. +// Output: A private IPv4 address Nexus should use for routing +// to / from the GZ. +pub fn get_nexus_private_ipv4(ip6: Ipv6Addr) -> Ipv4Addr { + Ipv4Addr::new(10, 0, ip6.octets()[7], 2) +} + /// Describes an executing Sled Agent object. /// /// Contains both a connection to the Nexus, as well as managed instances. From 809140d3b30f69ebd55b512787950ce0877e98fe Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 02:14:58 -0400 Subject: [PATCH 05/18] route from GZ to non-gz explicitly added --- sled-agent/src/illumos/zone.rs | 5 +-- sled-agent/src/services.rs | 63 ++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index 9833f601a01..a631fb2d878 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -557,10 +557,7 @@ impl Zones { Self::ensure_address( None, &AddrObject::new(&link.0, name).unwrap(), - AddressRequest::new_static( - IpAddr::V4(address), - None, - ), + AddressRequest::new_static(IpAddr::V4(address), None), ) .map_err(|err| anyhow!(err))?; Ok(()) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 944fbfd3b18..018c4846b0c 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -10,6 +10,7 @@ use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::zone::AddressRequest; +use crate::illumos::{execute, PFEXEC}; use crate::params::{ServiceEnsureBody, ServiceRequest, ServiceType}; use crate::zone::Zones; use dropshot::ConfigDropshot; @@ -58,6 +59,9 @@ pub enum Error { err: crate::illumos::running_zone::RunCommandError, }, + #[error("Installing GZ route to IPv4 address: {0}")] + GzIpv4Route(crate::illumos::ExecutionError), + #[error("Failed to boot zone: {0}")] ZoneBoot(#[from] crate::illumos::running_zone::BootError), @@ -346,27 +350,57 @@ impl ServiceManager { // The address of Nexus' external interface is a special // case; it may be an IPv4 address. - let addr_request = AddressRequest::new_static(external_address.ip(), None); - running_zone.ensure_address_with_name(addr_request, "public").await?; + let addr_request = + AddressRequest::new_static(external_address.ip(), None); + running_zone + .ensure_address_with_name(addr_request, "public") + .await?; // TODO: Remove once Nexus traffic is transmitted over OPTE. match external_address.ip() { - IpAddr::V4(_) => { + IpAddr::V4(public_addr4) => { // Create an address which is routable from the GZ's // Ipv4 address. - let private_addr4 = crate::sled_agent::get_nexus_private_ipv4(self.underlay_address); - let addr_request = AddressRequest::new_static(IpAddr::V4(private_addr4), None); - running_zone.ensure_address_with_name(addr_request, "private").await?; + let private_addr4 = + crate::sled_agent::get_nexus_private_ipv4( + self.underlay_address, + ); + let addr_request = AddressRequest::new_static( + IpAddr::V4(private_addr4), + None, + ); + running_zone + .ensure_address_with_name( + addr_request, + "private", + ) + .await?; // Create a default route back through the GZ's // underlay. - let gateway4 = crate::sled_agent::get_gz_ipv4(self.underlay_address); - running_zone.add_default_route4(gateway4).await.map_err(|err| { - Error::ZoneCommand { intent: "Adding Route".to_string(), err } - })?; - - // TODO: Do i need to add a route from the GZ to this zone - // too? + let gateway4 = crate::sled_agent::get_gz_ipv4( + self.underlay_address, + ); + running_zone + .add_default_route4(gateway4) + .await + .map_err(|err| Error::ZoneCommand { + intent: "Adding Route".to_string(), + err, + })?; + + // Add a route from the GZ to this zone, using the + // private address as a gateway. + let mut command = + std::process::Command::new(PFEXEC); + let cmd = command.args(&[ + "/usr/sbin/route", + "add", + &public_addr4.to_string(), + &private_addr4.to_string(), + ]); + execute(cmd) + .map_err(|err| Error::GzIpv4Route(err))?; } _ => (), } @@ -666,7 +700,8 @@ mod test { // Ensure the address exists let ensure_address_ctx = MockZones::ensure_address_context(); ensure_address_ctx.expect().return_once(|_, _, _| { - Ok(ipnetwork::IpNetwork::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 64).unwrap()) + Ok(ipnetwork::IpNetwork::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 64) + .unwrap()) }); // Wait for the networking service. From 26afcdb68acb9e5156a4c43cebfddb0ece48506d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 02:29:47 -0400 Subject: [PATCH 06/18] error handling --- sled-agent/src/services.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 018c4846b0c..33342a35d2e 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -399,7 +399,26 @@ impl ServiceManager { &public_addr4.to_string(), &private_addr4.to_string(), ]); + execute(cmd) + .map(|_| ()) + .or_else(|err| { + // If the command failed because the entry + // already exists in the global zone, we're + // good to continue. + match err { + crate::illumos::ExecutionError::CommandFailure { + ref stdout, + .. + } => { + if stdout.contains("entry exists") { + return Ok(()); + } + } + _ => (), + } + return Err(err); + }) .map_err(|err| Error::GzIpv4Route(err))?; } _ => (), From 06ca2fc55404718ad756e0c272ffe1225bea6de2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 10:13:06 -0400 Subject: [PATCH 07/18] Do a silly thing --- common/src/sql/dbwipe.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/src/sql/dbwipe.sql b/common/src/sql/dbwipe.sql index dcf7a78f4af..148417ff255 100644 --- a/common/src/sql/dbwipe.sql +++ b/common/src/sql/dbwipe.sql @@ -9,5 +9,11 @@ * initialization code and dbwipe.sql. */ DROP DATABASE IF EXISTS omicron; + +/* + * This is silly, but we throw an error if the user was already deleted. + * Create the user so we can always delete it. + */ +CREATE USER IF NOT EXISTS omicron; ALTER DEFAULT PRIVILEGES FOR ROLE root REVOKE ALL ON TABLES FROM omicron; DROP USER IF EXISTS omicron; From 1d0b4c2afe116253e014e4a40c89b162c096e251 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 10:17:42 -0400 Subject: [PATCH 08/18] Create the DB too --- common/src/sql/dbwipe.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/src/sql/dbwipe.sql b/common/src/sql/dbwipe.sql index 148417ff255..92f7b089dd8 100644 --- a/common/src/sql/dbwipe.sql +++ b/common/src/sql/dbwipe.sql @@ -8,12 +8,15 @@ * NOTE: the database and user names MUST be kept in sync with the * initialization code and dbwipe.sql. */ -DROP DATABASE IF EXISTS omicron; /* * This is silly, but we throw an error if the user was already deleted. * Create the user so we can always delete it. */ +CREATE DATABASE IF NOT EXISTS omicron; CREATE USER IF NOT EXISTS omicron; + ALTER DEFAULT PRIVILEGES FOR ROLE root REVOKE ALL ON TABLES FROM omicron; + +DROP DATABASE IF EXISTS omicron; DROP USER IF EXISTS omicron; From 2f76eaf18746efb2dcf1d8b0d0060ababd13154f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 10:21:48 -0400 Subject: [PATCH 09/18] warnings --- sled-agent/src/services.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 33342a35d2e..9878e5dcdb9 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -686,7 +686,7 @@ mod test { svc, zone::MockZones, }; - use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6}; + use std::net::Ipv6Addr; use std::os::unix::process::ExitStatusExt; use uuid::Uuid; From 4611e5d425e94848c9ed578904ca92bc1e94d359 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 10:42:24 -0400 Subject: [PATCH 10/18] 224 -> 226 --- smf/sled-agent/config-rss.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index 2ffbd38e1a0..4e39d6debd1 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -56,7 +56,7 @@ gz_addresses = [] [request.service.service_type] type = "nexus" internal_address = "[fd00:1122:3344:0101::3]:12221" -external_address = "172.20.15.224:12220" +external_address = "172.20.15.226:12220" # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate Oximeter services. From 325069106c1bdb360416c166d3105d54375a9511 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jun 2022 11:26:48 -0400 Subject: [PATCH 11/18] Update port too --- smf/sled-agent/config-rss.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index 4e39d6debd1..156112959f4 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -56,7 +56,7 @@ gz_addresses = [] [request.service.service_type] type = "nexus" internal_address = "[fd00:1122:3344:0101::3]:12221" -external_address = "172.20.15.226:12220" +external_address = "172.20.15.226:80" # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate Oximeter services. From d7baa2726d0db1db64d8e86ec541827a5147363b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 6 Jul 2022 13:13:58 -0400 Subject: [PATCH 12/18] Allocate VNIC for Nexus, default route to config-based gateway --- sled-agent/src/bootstrap/agent.rs | 4 +- sled-agent/src/config.rs | 6 +- sled-agent/src/illumos/dladm.rs | 4 +- sled-agent/src/illumos/running_zone.rs | 42 +++++++- sled-agent/src/illumos/vnic.rs | 13 +-- sled-agent/src/illumos/zone.rs | 12 +-- sled-agent/src/instance.rs | 9 +- sled-agent/src/instance_manager.rs | 2 +- sled-agent/src/services.rs | 137 ++++++++++++------------- sled-agent/src/sled_agent.rs | 60 ++++------- sled-agent/src/sp/simulated.rs | 4 +- sled-agent/src/storage_manager.rs | 5 +- smf/sled-agent/config-rss.toml | 3 +- smf/sled-agent/config.toml | 5 + 14 files changed, 164 insertions(+), 142 deletions(-) diff --git a/sled-agent/src/bootstrap/agent.rs b/sled-agent/src/bootstrap/agent.rs index a948b9414a9..09c31d4c8ea 100644 --- a/sled-agent/src/bootstrap/agent.rs +++ b/sled-agent/src/bootstrap/agent.rs @@ -150,7 +150,7 @@ impl Agent { err, })?; - let etherstub = Dladm::create_etherstub().map_err(|e| { + let etherstub = Dladm::ensure_etherstub().map_err(|e| { BootstrapError::SledError(format!( "Can't access etherstub device: {}", e @@ -158,7 +158,7 @@ impl Agent { })?; let etherstub_vnic = - Dladm::create_etherstub_vnic(ðerstub).map_err(|e| { + Dladm::ensure_etherstub_vnic(ðerstub).map_err(|e| { BootstrapError::SledError(format!( "Can't access etherstub VNIC device: {}", e diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index d67dd088e8a..36cd6c91c41 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -9,7 +9,7 @@ use crate::illumos::dladm::{self, Dladm, PhysicalLink}; use crate::illumos::zpool::ZpoolName; use dropshot::ConfigLogging; use serde::Deserialize; -use std::net::SocketAddr; +use std::net::{Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use uuid::Uuid; @@ -27,6 +27,10 @@ pub struct Config { /// Optional list of zpools to be used as "discovered disks". pub zpools: Option>, + /// Address of the Internet gateway, which is particularly + /// relevant for external-facing services (such as Nexus). + pub gateway_address: Option, + /// The data link on which we infer the bootstrap address. /// /// If unsupplied, we default to the first physical device. diff --git a/sled-agent/src/illumos/dladm.rs b/sled-agent/src/illumos/dladm.rs index 2df5edc9586..4eb509f7b08 100644 --- a/sled-agent/src/illumos/dladm.rs +++ b/sled-agent/src/illumos/dladm.rs @@ -113,7 +113,7 @@ pub struct Dladm {} #[cfg_attr(test, mockall::automock, allow(dead_code))] impl Dladm { /// Creates an etherstub, or returns one which already exists. - pub fn create_etherstub() -> Result { + pub fn ensure_etherstub() -> Result { if let Ok(stub) = Self::get_etherstub() { return Ok(stub); } @@ -136,7 +136,7 @@ impl Dladm { /// /// This VNIC is not tracked like [`crate::illumos::vnic::Vnic`], because /// it is expected to exist for the lifetime of the sled. - pub fn create_etherstub_vnic( + pub fn ensure_etherstub_vnic( source: &Etherstub, ) -> Result { if let Ok(vnic) = Self::get_etherstub_vnic() { diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index 65bc6e2db0a..760596324ba 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -5,6 +5,7 @@ //! Utilities to manage running zones. use crate::illumos::addrobj::AddrObject; +use crate::illumos::dladm::Etherstub; use crate::illumos::svc::wait_for_service; use crate::illumos::vnic::{Vnic, VnicAllocator}; use crate::illumos::zone::{AddressRequest, ZONE_PREFIX}; @@ -180,6 +181,31 @@ impl RunningZone { Ok(network) } + // TODO: Remove once Nexus uses OPTE - external addresses should generally + // be served via OPTE. + pub async fn ensure_external_address_with_name( + &self, + addrtype: AddressRequest, + name: &str, + ) -> Result { + info!(self.inner.log, "Adding address: {:?}", addrtype); + let addrobj = AddrObject::new( + self.inner.physical_vnic + .as_ref() + .expect("Cannot allocate external address on zone without physical VNIC") + .name(), + name + ) + .map_err(|err| EnsureAddressError::AddrObject { + request: addrtype, + zone: self.inner.name.clone(), + err, + })?; + let network = + Zones::ensure_address(Some(&self.inner.name), &addrobj, addrtype)?; + Ok(network) + } + pub async fn add_default_route( &self, gateway: Ipv6Addr, @@ -270,6 +296,7 @@ impl RunningZone { // // Re-initialize guest_vnic state by inspecting the zone. opte_ports: vec![], + physical_vnic: None, }, }) } @@ -322,6 +349,10 @@ pub struct InstalledZone { // OPTE devices for the guest network interfaces opte_ports: Vec, + + // Physical VNIC possibly provisioned to the zone. + // TODO: Remove once Nexus traffic is transmitted over OPTE. + physical_vnic: Option, } impl InstalledZone { @@ -348,12 +379,13 @@ impl InstalledZone { pub async fn install( log: &Logger, - vnic_allocator: &VnicAllocator, + vnic_allocator: &VnicAllocator, service_name: &str, unique_name: Option<&str>, datasets: &[zone::Dataset], devices: &[zone::Device], opte_ports: Vec, + physical_vnic: Option, ) -> Result { let control_vnic = vnic_allocator.new_control(None).map_err(|err| { InstallZoneError::CreateVnic { @@ -366,10 +398,17 @@ impl InstalledZone { let zone_image_path = PathBuf::from(&format!("/opt/oxide/{}.tar.gz", service_name)); + let phys_iter = if let Some(ref vnic) = physical_vnic { + vec![vnic.name().to_string()].into_iter() + } else { + vec![].into_iter() + }; + let net_device_names: Vec = opte_ports .iter() .map(|port| port.vnic().name().to_string()) .chain(std::iter::once(control_vnic.name().to_string())) + .chain(phys_iter) .collect(); Zones::install_omicron_zone( @@ -391,6 +430,7 @@ impl InstalledZone { name: zone_name, control_vnic, opte_ports, + physical_vnic, }) } } diff --git a/sled-agent/src/illumos/vnic.rs b/sled-agent/src/illumos/vnic.rs index 85bb32880f8..c9355446ef2 100644 --- a/sled-agent/src/illumos/vnic.rs +++ b/sled-agent/src/illumos/vnic.rs @@ -5,7 +5,7 @@ //! API for controlling a single instance. use crate::illumos::dladm::{ - CreateVnicError, DeleteVnicError, Etherstub, VNIC_PREFIX, + CreateVnicError, DeleteVnicError, VnicSource, VNIC_PREFIX, VNIC_PREFIX_CONTROL, VNIC_PREFIX_GUEST, }; use omicron_common::api::external::MacAddr; @@ -23,13 +23,13 @@ use crate::illumos::dladm::MockDladm as Dladm; /// May be used to allocate runtime-unique IDs for objects /// which have naming constraints - such as VNICs. #[derive(Clone, Debug)] -pub struct VnicAllocator { +pub struct VnicAllocator { value: Arc, scope: String, - data_link: Etherstub, + data_link: DL, } -impl VnicAllocator { +impl VnicAllocator
{ /// Creates a new Vnic name allocator with a particular scope. /// /// The intent with varying scopes is to create non-overlapping @@ -41,11 +41,11 @@ impl VnicAllocator { /// /// VnicAllocator::new("Storage") produces /// - oxControlStorage[NNN] - pub fn new>(scope: S, etherstub: Etherstub) -> Self { + pub fn new>(scope: S, data_link: DL) -> Self { Self { value: Arc::new(AtomicU64::new(0)), scope: scope.as_ref().to_string(), - data_link: etherstub, + data_link, } } @@ -167,6 +167,7 @@ impl Drop for Vnic { #[cfg(test)] mod test { use super::*; + use crate::illumos::dladm::Etherstub; #[test] fn test_allocate() { diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index a631fb2d878..fcefbb50f34 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -10,7 +10,7 @@ use slog::Logger; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use crate::illumos::addrobj::AddrObject; -use crate::illumos::dladm::{EtherstubVnic, VNIC_PREFIX_CONTROL}; +use crate::illumos::dladm::{EtherstubVnic, PhysicalLink, VNIC_PREFIX_CONTROL}; use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::{execute, PFEXEC}; use omicron_common::address::SLED_PREFIX; @@ -102,7 +102,7 @@ pub struct EnsureAddressError { #[error("Failed to create address {address} with name {name} in the GZ on {link:?}: {err}. Note to developers: {extra_note}")] pub struct EnsureGzAddressError { address: IpAddr, - link: EtherstubVnic, + link: String, name: String, #[source] err: anyhow::Error, @@ -543,13 +543,13 @@ impl Zones { // TODO: Remove once Nexus traffic is transmitted over OPTE. pub fn ensure_has_global_zone_v4_address( - link: EtherstubVnic, + link: PhysicalLink, address: Ipv4Addr, name: &str, ) -> Result<(), EnsureGzAddressError> { // Call the guts of this function within a closure to make it easier // to wrap the error with appropriate context. - |link: EtherstubVnic, address, name| -> Result<(), anyhow::Error> { + |link: PhysicalLink, address, name| -> Result<(), anyhow::Error> { // Ensure that a static IPv4 address has been allocated // to the Global Zone. Without this, we don't have a way // to route to IP addresses that we want to create in @@ -564,7 +564,7 @@ impl Zones { }(link.clone(), address, name) .map_err(|err| EnsureGzAddressError { address: IpAddr::V4(address), - link, + link: link.0.clone(), name: name.to_string(), err, extra_note: "".to_string(), @@ -612,7 +612,7 @@ impl Zones { }(link.clone(), address, name) .map_err(|err| EnsureGzAddressError { address: IpAddr::V6(address), - link, + link: link.0.clone(), name: name.to_string(), err, extra_note: diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 79bc5aeeb7b..9f63fe2e457 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -5,6 +5,7 @@ //! API for controlling a single instance. use crate::common::instance::{Action as InstanceAction, InstanceStates}; +use crate::illumos::dladm::Etherstub; use crate::illumos::running_zone::{ InstalledZone, RunCommandError, RunningZone, }; @@ -204,7 +205,7 @@ struct InstanceInner { propolis_ip: IpAddr, // NIC-related properties - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, // OPTE port related properties underlay_addr: Ipv6Addr, @@ -400,7 +401,7 @@ mockall::mock! { pub fn new( log: Logger, id: Uuid, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, underlay_addr: Ipv6Addr, port_allocator: OptePortAllocator, initial: InstanceHardware, @@ -446,7 +447,7 @@ impl Instance { pub fn new( log: Logger, id: Uuid, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, underlay_addr: Ipv6Addr, port_allocator: OptePortAllocator, initial: InstanceHardware, @@ -534,6 +535,8 @@ impl Instance { zone::Device { name: "/dev/viona".to_string() }, ], opte_ports, + // physical_vnic= + None, ) .await?; diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 5f0e9179470..8660ad7cc9e 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -44,7 +44,7 @@ struct InstanceManagerInternal { /// A mapping from a Sled Agent "Instance ID" to ("Propolis ID", [Instance]). instances: Mutex>, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, underlay_addr: Ipv6Addr, port_allocator: OptePortAllocator, } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 9878e5dcdb9..c72873b4ead 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -5,12 +5,12 @@ //! Support for miscellaneous services managed by the sled. use crate::bootstrap::ddm_admin_client::{DdmAdminClient, DdmError}; -use crate::illumos::dladm::{Etherstub, EtherstubVnic}; +use crate::common::underlay; +use crate::illumos::dladm::{Etherstub, EtherstubVnic, PhysicalLink}; use crate::illumos::running_zone::{InstalledZone, RunningZone}; use crate::illumos::vnic::VnicAllocator; use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::zone::AddressRequest; -use crate::illumos::{execute, PFEXEC}; use crate::params::{ServiceEnsureBody, ServiceRequest, ServiceType}; use crate::zone::Zones; use dropshot::ConfigDropshot; @@ -25,7 +25,7 @@ use omicron_common::postgres_config::PostgresConfigWithUrl; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::str::FromStr; use tokio::io::AsyncWriteExt; @@ -59,9 +59,6 @@ pub enum Error { err: crate::illumos::running_zone::RunCommandError, }, - #[error("Installing GZ route to IPv4 address: {0}")] - GzIpv4Route(crate::illumos::ExecutionError), - #[error("Failed to boot zone: {0}")] ZoneBoot(#[from] crate::illumos::running_zone::BootError), @@ -74,6 +71,12 @@ pub enum Error { #[error("Error contacting ddmd: {0}")] DdmError(#[from] DdmError), + #[error("Failed to access underlay device: {0}")] + Underlay(#[from] underlay::Error), + + #[error("Failed to create Vnic for Nexus: {0}")] + NexusVnicCreation(crate::illumos::dladm::CreateVnicError), + #[error("Failed to add GZ addresses: {message}: {err}")] GzAddress { message: String, @@ -102,13 +105,14 @@ pub fn default_services_config_path() -> PathBuf { } /// Configuration parameters which modify the [`ServiceManager`]'s behavior. -/// -/// These are typically used to make testing easier; production usage -/// should generally prefer to use the defaults. pub struct Config { + /// An optional internet gateway address for external services. + pub gateway_address: Option, + /// The path for the ServiceManager to store information about /// all running services. pub all_svcs_config_path: PathBuf, + /// A function which returns the path the directory holding the /// service's configuration file. pub get_svc_config_dir: Box PathBuf + Send + Sync>, @@ -117,6 +121,7 @@ pub struct Config { impl Default for Config { fn default() -> Self { Self { + gateway_address: None, all_svcs_config_path: default_services_config_path(), get_svc_config_dir: Box::new(|zone_name: &str, svc_name: &str| { PathBuf::from(ZONE_ZFS_DATASET_MOUNTPOINT) @@ -133,7 +138,8 @@ pub struct ServiceManager { log: Logger, config: Config, zones: Mutex>, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, + physical_link_vnic_allocator: VnicAllocator, underlay_vnic: EtherstubVnic, underlay_address: Ipv6Addr, rack_id: Uuid, @@ -158,6 +164,7 @@ impl ServiceManager { underlay_vnic: EtherstubVnic, underlay_address: Ipv6Addr, config: Config, + physical_link: PhysicalLink, rack_id: Uuid, ) -> Result { debug!(log, "Creating new ServiceManager"); @@ -167,6 +174,13 @@ impl ServiceManager { config, zones: Mutex::new(vec![]), vnic_allocator: VnicAllocator::new("Service", etherstub), + physical_link_vnic_allocator: VnicAllocator::new( + "Public", + // NOTE: Right now, we only use a connection to one of the Chelsio + // links. Longer-term, when we we use OPTE, we'll be able to use both + // connections. + physical_link, + ), underlay_vnic, underlay_address, rack_id, @@ -248,6 +262,18 @@ impl ServiceManager { info!(self.log, "Service {} does not yet exist", service.name); } + // TODO: Remove once Nexus traffic is transmitted over OPTE. + let physical_vnic = match service.service_type { + ServiceType::Nexus { .. } => { + let vnic = self + .physical_link_vnic_allocator + .new_control(None) + .map_err(|e| Error::NexusVnicCreation(e))?; + Some(vnic) + } + _ => None, + }; + let installed_zone = InstalledZone::install( &self.log, &self.vnic_allocator, @@ -260,6 +286,8 @@ impl ServiceManager { &[], // opte_ports= vec![], + // physical_vnic= + physical_vnic, ) .await?; @@ -353,73 +381,27 @@ impl ServiceManager { let addr_request = AddressRequest::new_static(external_address.ip(), None); running_zone - .ensure_address_with_name(addr_request, "public") + .ensure_external_address_with_name( + addr_request, + "public", + ) .await?; - // TODO: Remove once Nexus traffic is transmitted over OPTE. match external_address.ip() { - IpAddr::V4(public_addr4) => { - // Create an address which is routable from the GZ's - // Ipv4 address. - let private_addr4 = - crate::sled_agent::get_nexus_private_ipv4( - self.underlay_address, - ); - let addr_request = AddressRequest::new_static( - IpAddr::V4(private_addr4), - None, - ); - running_zone - .ensure_address_with_name( - addr_request, - "private", - ) - .await?; - - // Create a default route back through the GZ's - // underlay. - let gateway4 = crate::sled_agent::get_gz_ipv4( - self.underlay_address, - ); - running_zone - .add_default_route4(gateway4) - .await - .map_err(|err| Error::ZoneCommand { - intent: "Adding Route".to_string(), - err, - })?; - - // Add a route from the GZ to this zone, using the - // private address as a gateway. - let mut command = - std::process::Command::new(PFEXEC); - let cmd = command.args(&[ - "/usr/sbin/route", - "add", - &public_addr4.to_string(), - &private_addr4.to_string(), - ]); - - execute(cmd) - .map(|_| ()) - .or_else(|err| { - // If the command failed because the entry - // already exists in the global zone, we're - // good to continue. - match err { - crate::illumos::ExecutionError::CommandFailure { - ref stdout, - .. - } => { - if stdout.contains("entry exists") { - return Ok(()); - } - } - _ => (), - } - return Err(err); - }) - .map_err(|err| Error::GzIpv4Route(err))?; + IpAddr::V4(_public_addr4) => { + // If requested, create a default route back through + // the internet gateway. + if let Some(ref gateway) = + self.config.gateway_address + { + running_zone + .add_default_route4(*gateway) + .await + .map_err(|err| Error::ZoneCommand { + intent: "Adding Route".to_string(), + err, + })?; + } } _ => (), } @@ -822,6 +804,7 @@ mod test { svc_config_dir.clone() }, ), + ..Default::default() } } } @@ -840,6 +823,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -867,6 +851,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -896,6 +881,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -914,6 +900,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -939,6 +926,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, test_config.make_config(), + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await @@ -959,6 +947,7 @@ mod test { EtherstubVnic(ETHERSTUB_VNIC_NAME.to_string()), Ipv6Addr::LOCALHOST, config, + PhysicalLink("link".to_string()), Uuid::new_v4(), ) .await diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 03e590569c0..b5bc6b1ff1d 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -25,7 +25,7 @@ use omicron_common::api::{ internal::nexus::UpdateArtifact, }; use slog::Logger; -use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV6}; +use std::net::SocketAddrV6; use std::sync::Arc; use uuid::Uuid; @@ -90,39 +90,6 @@ impl From for omicron_common::api::external::Error { } } -// TODO: Remove once Nexus traffic is transmitted over OPTE. -// This is a hack necessary for routing IPv4 traffic to zones hosting external -// services. -// -// Here's the deal: For a sled IPv6 addresses, different portions of the -// prefix act like indices: -// -// fd00:1122:3344:0101::/64 -// | | ^^ Sled -// | ^^ Rack -// ^^^^^^^^^^^^^ AZ -// -// These IPv6 addresses are allocated uniquely to sleds. However, for -// our short-term hackish purposes, we'd like to allocate IPv4 addresses -// on the underlay too, so we can do the routing we need to do. -// -// We'd like this to be as lightweight as possible. Rather than -// explicitly provisioning anything in nexus, we piggyback on the -// existing IPv6 address. By grabbing the "sled octet" from the IPv6 -// address, we get a rack-wide unique identifier. -pub fn get_gz_ipv4(ip6: Ipv6Addr) -> Ipv4Addr { - Ipv4Addr::new(10, 0, ip6.octets()[7], 1) -} - -// TODO: Remove once Nexus traffic is transmitted over OPTE. -// -// Input: The sled underlay IPv6 address. -// Output: A private IPv4 address Nexus should use for routing -// to / from the GZ. -pub fn get_nexus_private_ipv4(ip6: Ipv6Addr) -> Ipv4Addr { - Ipv4Addr::new(10, 0, ip6.octets()[7], 2) -} - /// Describes an executing Sled Agent object. /// /// Contains both a connection to the Nexus, as well as managed instances. @@ -165,8 +132,8 @@ impl SledAgent { info!(&log, "created sled agent"); let etherstub = - Dladm::create_etherstub().map_err(|e| Error::Etherstub(e))?; - let etherstub_vnic = Dladm::create_etherstub_vnic(ðerstub) + Dladm::ensure_etherstub().map_err(|e| Error::Etherstub(e))?; + let etherstub_vnic = Dladm::ensure_etherstub_vnic(ðerstub) .map_err(|e| Error::EtherstubVnic(e))?; // Before we start creating zones, we need to ensure that the @@ -193,13 +160,18 @@ impl SledAgent { ) .map_err(|err| Error::SledSubnet { err })?; - // TODO: Remove once Nexus traffic is transmitted over OPTE. + // TODO: I'd like a more automated way of doing this, but where should + // this address come from? It seems a bit odd to provide it at + // config-time, since presumably this needs to be in the same range + // as the user-provided IP pools. + /* Zones::ensure_has_global_zone_v4_address( - etherstub_vnic.clone(), - get_gz_ipv4(*sled_address.ip()), - "sled4", + config.get_link()?, + "172.20.15.238".parse().unwrap(), + "sledv4", ) .map_err(|err| Error::SledSubnet { err })?; + */ // Initialize the xde kernel driver with the underlay devices. crate::opte::initialize_xde_driver(&log)?; @@ -306,12 +278,18 @@ impl SledAgent { etherstub.clone(), *sled_address.ip(), ); + + let svc_config = services::Config { + gateway_address: config.gateway_address, + ..Default::default() + }; let services = ServiceManager::new( parent_log.clone(), etherstub.clone(), etherstub_vnic.clone(), *sled_address.ip(), - services::Config::default(), + svc_config, + config.get_link()?, rack_id, ) .await?; diff --git a/sled-agent/src/sp/simulated.rs b/sled-agent/src/sp/simulated.rs index e402654da22..bdf31ffb9f3 100644 --- a/sled-agent/src/sp/simulated.rs +++ b/sled-agent/src/sp/simulated.rs @@ -62,8 +62,8 @@ impl SimulatedSp { // Ensure we have the global zone IP address we need for the SP. let etherstub = - Dladm::create_etherstub().map_err(SpError::CreateEtherstub)?; - let etherstub_vnic = Dladm::create_etherstub_vnic(ðerstub) + Dladm::ensure_etherstub().map_err(SpError::CreateEtherstub)?; + let etherstub_vnic = Dladm::ensure_etherstub_vnic(ðerstub) .map_err(SpError::CreateEtherstubVnic)?; Zones::ensure_has_global_zone_v6_address( etherstub_vnic, diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index a02e68baae9..1e5e5bb4bb9 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -456,7 +456,7 @@ impl DatasetInfo { // Ensures that a zone backing a particular dataset is running. async fn ensure_running_zone( log: &Logger, - vnic_allocator: &VnicAllocator, + vnic_allocator: &VnicAllocator, dataset_info: &DatasetInfo, dataset_name: &DatasetName, do_format: bool, @@ -488,6 +488,7 @@ async fn ensure_running_zone( &[zone::Dataset { name: dataset_name.full() }], &[], vec![], + None, ) .await?; @@ -542,7 +543,7 @@ struct StorageWorker { pools: Arc>>, new_pools_rx: mpsc::Receiver, new_filesystems_rx: mpsc::Receiver, - vnic_allocator: VnicAllocator, + vnic_allocator: VnicAllocator, underlay_address: Ipv6Addr, } diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index 156112959f4..36db4ef9c18 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -56,7 +56,8 @@ gz_addresses = [] [request.service.service_type] type = "nexus" internal_address = "[fd00:1122:3344:0101::3]:12221" -external_address = "172.20.15.226:80" +# NOTE: In the lab, use "172.20.15.226" +external_address = "192.168.1.20:80" # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate Oximeter services. diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index 53ec733e9ec..03d4a614e37 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -26,6 +26,11 @@ zpools = [ # $ dladm show-phys -p -o LINK # data_link = "igb0" +# Address of Internet gateway +# +# NOTE: In the lab, use "172.20.15.225" +gateway_address = "192.168.1.1" + [log] level = "info" mode = "file" From f5fbdd975b25e8101261416c42ab1b1232ca1084 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 6 Jul 2022 13:29:48 -0400 Subject: [PATCH 13/18] Clippy, dead code --- sled-agent/src/illumos/running_zone.rs | 1 + sled-agent/src/illumos/zone.rs | 35 ++------------------------ sled-agent/src/sled_agent.rs | 13 ---------- 3 files changed, 3 insertions(+), 46 deletions(-) diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index 760596324ba..5bd28dc618e 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -377,6 +377,7 @@ impl InstalledZone { zone_name } + #[allow(clippy::too_many_arguments)] pub async fn install( log: &Logger, vnic_allocator: &VnicAllocator, diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index fcefbb50f34..50dc53e01b0 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -7,10 +7,10 @@ use anyhow::anyhow; use ipnetwork::IpNetwork; use slog::Logger; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::{IpAddr, Ipv6Addr}; use crate::illumos::addrobj::AddrObject; -use crate::illumos::dladm::{EtherstubVnic, PhysicalLink, VNIC_PREFIX_CONTROL}; +use crate::illumos::dladm::{EtherstubVnic, VNIC_PREFIX_CONTROL}; use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::{execute, PFEXEC}; use omicron_common::address::SLED_PREFIX; @@ -541,37 +541,6 @@ impl Zones { Ok(()) } - // TODO: Remove once Nexus traffic is transmitted over OPTE. - pub fn ensure_has_global_zone_v4_address( - link: PhysicalLink, - address: Ipv4Addr, - name: &str, - ) -> Result<(), EnsureGzAddressError> { - // Call the guts of this function within a closure to make it easier - // to wrap the error with appropriate context. - |link: PhysicalLink, address, name| -> Result<(), anyhow::Error> { - // Ensure that a static IPv4 address has been allocated - // to the Global Zone. Without this, we don't have a way - // to route to IP addresses that we want to create in - // the non-GZ. - Self::ensure_address( - None, - &AddrObject::new(&link.0, name).unwrap(), - AddressRequest::new_static(IpAddr::V4(address), None), - ) - .map_err(|err| anyhow!(err))?; - Ok(()) - }(link.clone(), address, name) - .map_err(|err| EnsureGzAddressError { - address: IpAddr::V4(address), - link: link.0.clone(), - name: name.to_string(), - err, - extra_note: "".to_string(), - })?; - Ok(()) - } - // TODO(https://github.com/oxidecomputer/omicron/issues/821): We // should remove this function when Sled Agents are provided IPv6 addresses // from RSS. diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index b5bc6b1ff1d..63925dc6578 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -160,19 +160,6 @@ impl SledAgent { ) .map_err(|err| Error::SledSubnet { err })?; - // TODO: I'd like a more automated way of doing this, but where should - // this address come from? It seems a bit odd to provide it at - // config-time, since presumably this needs to be in the same range - // as the user-provided IP pools. - /* - Zones::ensure_has_global_zone_v4_address( - config.get_link()?, - "172.20.15.238".parse().unwrap(), - "sledv4", - ) - .map_err(|err| Error::SledSubnet { err })?; - */ - // Initialize the xde kernel driver with the underlay devices. crate::opte::initialize_xde_driver(&log)?; From e548459266e112da64c605e5cffc6cd2691baa6f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 7 Jul 2022 09:12:35 -0400 Subject: [PATCH 14/18] review feedback --- sled-agent/src/illumos/running_zone.rs | 22 ++++++---------- sled-agent/src/instance.rs | 2 +- sled-agent/src/services.rs | 35 ++++++++++++-------------- tools/populate/populate-images.sh | 2 +- 4 files changed, 26 insertions(+), 35 deletions(-) diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index 5bd28dc618e..75316eecfb9 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -190,9 +190,9 @@ impl RunningZone { ) -> Result { info!(self.inner.log, "Adding address: {:?}", addrtype); let addrobj = AddrObject::new( - self.inner.physical_vnic + self.inner.physical_nic .as_ref() - .expect("Cannot allocate external address on zone without physical VNIC") + .expect("Cannot allocate external address on zone without physical NIC") .name(), name ) @@ -296,7 +296,7 @@ impl RunningZone { // // Re-initialize guest_vnic state by inspecting the zone. opte_ports: vec![], - physical_vnic: None, + physical_nic: None, }, }) } @@ -350,9 +350,9 @@ pub struct InstalledZone { // OPTE devices for the guest network interfaces opte_ports: Vec, - // Physical VNIC possibly provisioned to the zone. + // Physical NIC possibly provisioned to the zone. // TODO: Remove once Nexus traffic is transmitted over OPTE. - physical_vnic: Option, + physical_nic: Option, } impl InstalledZone { @@ -386,7 +386,7 @@ impl InstalledZone { datasets: &[zone::Dataset], devices: &[zone::Device], opte_ports: Vec, - physical_vnic: Option, + physical_nic: Option, ) -> Result { let control_vnic = vnic_allocator.new_control(None).map_err(|err| { InstallZoneError::CreateVnic { @@ -399,17 +399,11 @@ impl InstalledZone { let zone_image_path = PathBuf::from(&format!("/opt/oxide/{}.tar.gz", service_name)); - let phys_iter = if let Some(ref vnic) = physical_vnic { - vec![vnic.name().to_string()].into_iter() - } else { - vec![].into_iter() - }; - let net_device_names: Vec = opte_ports .iter() .map(|port| port.vnic().name().to_string()) .chain(std::iter::once(control_vnic.name().to_string())) - .chain(phys_iter) + .chain(physical_nic.as_ref().map(|vnic| vnic.name().to_string())) .collect(); Zones::install_omicron_zone( @@ -431,7 +425,7 @@ impl InstalledZone { name: zone_name, control_vnic, opte_ports, - physical_vnic, + physical_nic, }) } } diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 9f63fe2e457..32e37c4fc1a 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -535,7 +535,7 @@ impl Instance { zone::Device { name: "/dev/viona".to_string() }, ], opte_ports, - // physical_vnic= + // physical_nic= None, ) .await?; diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index c72873b4ead..bdc1ff37ffd 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -263,7 +263,7 @@ impl ServiceManager { } // TODO: Remove once Nexus traffic is transmitted over OPTE. - let physical_vnic = match service.service_type { + let physical_nic = match service.service_type { ServiceType::Nexus { .. } => { let vnic = self .physical_link_vnic_allocator @@ -286,8 +286,8 @@ impl ServiceManager { &[], // opte_ports= vec![], - // physical_vnic= - physical_vnic, + // physical_nic= + physical_nic, ) .await?; @@ -387,23 +387,20 @@ impl ServiceManager { ) .await?; - match external_address.ip() { - IpAddr::V4(_public_addr4) => { - // If requested, create a default route back through - // the internet gateway. - if let Some(ref gateway) = - self.config.gateway_address - { - running_zone - .add_default_route4(*gateway) - .await - .map_err(|err| Error::ZoneCommand { - intent: "Adding Route".to_string(), - err, - })?; - } + if let IpAddr::V4(_public_addr4) = external_address.ip() { + // If requested, create a default route back through + // the internet gateway. + if let Some(ref gateway) = + self.config.gateway_address + { + running_zone + .add_default_route4(*gateway) + .await + .map_err(|err| Error::ZoneCommand { + intent: "Adding Route".to_string(), + err, + })?; } - _ => (), } // Nexus takes a separate config file for parameters which diff --git a/tools/populate/populate-images.sh b/tools/populate/populate-images.sh index 9f845e09e05..2de2ebbae77 100755 --- a/tools/populate/populate-images.sh +++ b/tools/populate/populate-images.sh @@ -2,7 +2,7 @@ # Populate an Oxide lab host running Omicron with images from server catacomb. set -eu -CATACOMB_TUNNEL="[fd00:1122:3344:101::1]:8080" +CATACOMB_TUNNEL="[fd00:1122:3344:101::1]:54321" res=0 echo "Populating debian" oxide api /images --method POST --input - < Date: Thu, 7 Jul 2022 09:21:11 -0400 Subject: [PATCH 15/18] no more ipv4 forwarding --- sled-agent/src/sled_agent.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 63925dc6578..f1cd951cb98 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -231,10 +231,7 @@ impl SledAgent { let mut command = std::process::Command::new(PFEXEC); let cmd = command.args(&[ "/usr/sbin/routeadm", - // Needed to access Nexus's external interface. - "-e", - "ipv4-forwarding", - // Needed to access all other zones, which are on the underlay. + // Needed to access all zones, which are on the underlay. "-e", "ipv6-forwarding", "-u", From 3826dbbe03ec247fbc7df8f12d940af2bc1f3a05 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 7 Jul 2022 09:27:13 -0400 Subject: [PATCH 16/18] update docs --- docs/how-to-run.adoc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index a3cbe0a116a..cdfda854583 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -113,7 +113,10 @@ we'll assign addresses as per RFD 63 as well as incorporating DNS based service discovery. For the purposes of local development today, we specify some hardcoded IPv6 -unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1::/64`: +unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1::/64`. + +If you'd like to modify these values to suit your local network, you can modify +them within the https://github.com/oxidecomputer/omicron/tree/main/smf[`smf/` subdirectory]. [options="header"] |=================================================================================================== @@ -121,7 +124,6 @@ unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1: | Sled Agent: Bootstrap | Derived from MAC address of physical data link. | Sled Agent: Dropshot API | `[fd00:1122:3344:0101::1]:12345` | Cockroach DB | `[fd00:1122:3344:0101::2]:32221` -| Nexus: External API | `[fd00:1122:3344:0101::3]:12220` | Nexus: Internal API | `[fd00:1122:3344:0101::3]:12221` | Oximeter | `[fd00:1122:3344:0101::4]:12223` | Clickhouse | `[fd00:1122:3344:0101::5]:8123` @@ -129,6 +131,8 @@ unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1: | Crucible Downstairs 2 | `[fd00:1122:3344:0101::7]:32345` | Crucible Downstairs 3 | `[fd00:1122:3344:0101::8]:32345` | Internal DNS Service | `[fd00:1122:3344:0001::1]:5353` +| Nexus: External API | `192.168.1.20:80` +| Internet Gateway | `192.168.1.1` |=================================================================================================== Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other From e118c2aa7ffdfd7594ac600c18f090e984262ec2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 7 Jul 2022 09:28:28 -0400 Subject: [PATCH 17/18] fmt --- sled-agent/src/services.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index bdc1ff37ffd..fdd6e097eab 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -390,9 +390,7 @@ impl ServiceManager { if let IpAddr::V4(_public_addr4) = external_address.ip() { // If requested, create a default route back through // the internet gateway. - if let Some(ref gateway) = - self.config.gateway_address - { + if let Some(ref gateway) = self.config.gateway_address { running_zone .add_default_route4(*gateway) .await From 68263c63fe3575461158fedd47eeaeab89604c22 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 7 Jul 2022 09:37:25 -0400 Subject: [PATCH 18/18] No default gateway, update docs --- docs/how-to-run.adoc | 6 +++++- smf/sled-agent/config.toml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index cdfda854583..4385e21b45e 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -117,6 +117,10 @@ unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1: If you'd like to modify these values to suit your local network, you can modify them within the https://github.com/oxidecomputer/omicron/tree/main/smf[`smf/` subdirectory]. +Notably, Nexus is being served from IPv4 address, which may be configured to be +external. By default, it uses a private IPv4 address and no Internet gateway, but may +be configured to use a public-facing IP address with an Internet gateway that may +be set as a default route for the Nexus zone. [options="header"] |=================================================================================================== @@ -132,7 +136,7 @@ them within the https://github.com/oxidecomputer/omicron/tree/main/smf[`smf/` su | Crucible Downstairs 3 | `[fd00:1122:3344:0101::8]:32345` | Internal DNS Service | `[fd00:1122:3344:0001::1]:5353` | Nexus: External API | `192.168.1.20:80` -| Internet Gateway | `192.168.1.1` +| Internet Gateway | None, but can be set in `smf/sled-agent/config.toml` |=================================================================================================== Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index 03d4a614e37..b99e4db58e4 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -29,7 +29,7 @@ zpools = [ # Address of Internet gateway # # NOTE: In the lab, use "172.20.15.225" -gateway_address = "192.168.1.1" +# gateway_address = "192.168.1.1" [log] level = "info"