From d5bda04ce625a651cc9c6810b13414068e072f27 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 2 Jun 2022 17:20:49 -0400 Subject: [PATCH 1/7] enable IPv6 routing in non-global zones --- sled-agent/src/services.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index aafd89b221b..808a076b078 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -248,6 +248,9 @@ impl ServiceManager { Error::ZoneCommand { intent: "Adding Route".to_string(), err } })?; + // TODO: Related to + // https://github.com/oxidecomputer/omicron/pull/1124 , should we + // avoid importing this manifest? debug!(self.log, "importing manifest"); running_zone @@ -267,6 +270,18 @@ impl ServiceManager { let smf_name = format!("svc:/system/illumos/{}", service.name); let default_smf_name = format!("{}:default", smf_name); + running_zone + .run_cmd(&[ + "/usr/sbin/routeadm", + "-e", + "ipv6-routing", + "-u", + ]) + .map_err(|err| Error::ZoneCommand { + intent: "enabling IPv6 routing".to_string(), + err, + })?; + match service.name.as_str() { "internal-dns" => { info!(self.log, "Setting up internal-dns service"); From b424a1ed37d1bf18051eac94c7a598a7506fcb6d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 2 Jun 2022 17:23:07 -0400 Subject: [PATCH 2/7] comment --- sled-agent/src/services.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 808a076b078..984ab7a2fe2 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -270,6 +270,10 @@ impl ServiceManager { let smf_name = format!("svc:/system/illumos/{}", service.name); let default_smf_name = format!("{}:default", smf_name); + // Ensure the IPv6 traffic can be routed to this non-global zone. + // + // This is particularly important for accessing Nexus' external + // interface from off-device. running_zone .run_cmd(&[ "/usr/sbin/routeadm", From 2a2a061ca3f8a513c8134ea2280a186fd56a1bdc Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 2 Jun 2022 17:25:20 -0400 Subject: [PATCH 3/7] fmt --- sled-agent/src/services.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 984ab7a2fe2..adb2a15907e 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -275,12 +275,7 @@ impl ServiceManager { // This is particularly important for accessing Nexus' external // interface from off-device. running_zone - .run_cmd(&[ - "/usr/sbin/routeadm", - "-e", - "ipv6-routing", - "-u", - ]) + .run_cmd(&["/usr/sbin/routeadm", "-e", "ipv6-routing", "-u"]) .map_err(|err| Error::ZoneCommand { intent: "enabling IPv6 routing".to_string(), err, From 0c22350ae76e20bb8381dfa1a49cd51a858e30c4 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 2 Jun 2022 23:11:08 -0400 Subject: [PATCH 4/7] enable forwarding too --- sled-agent/src/services.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index adb2a15907e..2502ce1571a 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -274,6 +274,12 @@ impl ServiceManager { // // This is particularly important for accessing Nexus' external // interface from off-device. + running_zone + .run_cmd(&["/usr/sbin/routeadm", "-e", "ipv6-forwarding"]) + .map_err(|err| Error::ZoneCommand { + intent: "enabling IPv6 forwarding".to_string(), + err, + })?; running_zone .run_cmd(&["/usr/sbin/routeadm", "-e", "ipv6-routing", "-u"]) .map_err(|err| Error::ZoneCommand { From 65ae20711c041ca1c3ba18462b5c74717059e21d Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 2 Jun 2022 23:12:18 -0400 Subject: [PATCH 5/7] tests --- sled-agent/src/services.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 2502ce1571a..2f0e41d6641 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -489,7 +489,7 @@ mod test { wait_ctx.expect().return_once(|_, _| Ok(())); // Import the manifest, enable the service let execute_ctx = crate::illumos::execute_context(); - execute_ctx.expect().times(3).returning(|_| { + execute_ctx.expect().times(5).returning(|_| { Ok(std::process::Output { status: std::process::ExitStatus::from_raw(0), stdout: vec![], From c8229473df844d21ba79698b3d43b833197db508 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 3 Jun 2022 12:48:24 -0400 Subject: [PATCH 6/7] do it in one --- sled-agent/src/services.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 2f0e41d6641..52cbc636c1c 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -275,15 +275,16 @@ impl ServiceManager { // This is particularly important for accessing Nexus' external // interface from off-device. running_zone - .run_cmd(&["/usr/sbin/routeadm", "-e", "ipv6-forwarding"]) - .map_err(|err| Error::ZoneCommand { - intent: "enabling IPv6 forwarding".to_string(), - err, - })?; - running_zone - .run_cmd(&["/usr/sbin/routeadm", "-e", "ipv6-routing", "-u"]) + .run_cmd(&[ + "/usr/sbin/routeadm", + "-e", + "ipv6-forwarding", + "-e", + "ipv6-routing", + "-u", + ]) .map_err(|err| Error::ZoneCommand { - intent: "enabling IPv6 routing".to_string(), + intent: "enabling IPv6 forwarding & routing".to_string(), err, })?; @@ -489,7 +490,7 @@ mod test { wait_ctx.expect().return_once(|_, _| Ok(())); // Import the manifest, enable the service let execute_ctx = crate::illumos::execute_context(); - execute_ctx.expect().times(5).returning(|_| { + execute_ctx.expect().times(4).returning(|_| { Ok(std::process::Output { status: std::process::ExitStatus::from_raw(0), stdout: vec![], From a3061090ba7b706d1adeada333bc63099d6ae0f9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 3 Jun 2022 14:23:02 -0400 Subject: [PATCH 7/7] Don't enable forwarding/routing - use default routes instead --- sled-agent/src/illumos/running_zone.rs | 9 ++++--- sled-agent/src/services.rs | 34 ++++++-------------------- sled-agent/src/storage_manager.rs | 9 +++---- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/sled-agent/src/illumos/running_zone.rs b/sled-agent/src/illumos/running_zone.rs index a58b4c9d185..fee23187688 100644 --- a/sled-agent/src/illumos/running_zone.rs +++ b/sled-agent/src/illumos/running_zone.rs @@ -11,6 +11,7 @@ use crate::illumos::zone::{AddressRequest, ZONE_PREFIX}; use crate::opte::OptePort; use ipnetwork::IpNetwork; use slog::Logger; +use std::net::Ipv6Addr; use std::path::PathBuf; #[cfg(test)] @@ -171,17 +172,17 @@ impl RunningZone { Ok(network) } - pub async fn add_route( + pub async fn add_default_route( &self, - destination: ipnetwork::Ipv6Network, + gateway: Ipv6Addr, ) -> Result<(), RunCommandError> { self.run_cmd(&[ "/usr/sbin/route", "add", "-inet6", - &format!("{}/{}", destination.network(), destination.prefix()), + "default", "-inet6", - &destination.ip().to_string(), + &gateway.to_string(), ])?; Ok(()) } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 52cbc636c1c..72444a79b17 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -10,8 +10,7 @@ use crate::illumos::vnic::VnicAllocator; use crate::illumos::zone::AddressRequest; use crate::params::{ServiceEnsureBody, ServiceRequest}; use crate::zone::Zones; -use ipnetwork::Ipv6Network; -use omicron_common::address::{AZ_PREFIX, DNS_PORT, DNS_SERVER_PORT}; +use omicron_common::address::{DNS_PORT, DNS_SERVER_PORT}; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; @@ -229,7 +228,7 @@ impl ServiceManager { })?; } - let gz_route_subnet = if !service.gz_addresses.is_empty() { + let gateway = if !service.gz_addresses.is_empty() { // If this service supplies its own GZ address, add a route. // // This is currently being used for the DNS service. @@ -238,13 +237,12 @@ impl ServiceManager { // can be supplied - now that we're actively using it, we // aren't really handling the "many GZ addresses" case, and it // doesn't seem necessary now. - Ipv6Network::new(service.gz_addresses[0], AZ_PREFIX).unwrap() + service.gz_addresses[0] } else { - // Otherwise, add a route to the global Zone's sled address for - // everything within the AZ. - Ipv6Network::new(self.underlay_address, AZ_PREFIX).unwrap() + self.underlay_address }; - running_zone.add_route(gz_route_subnet).await.map_err(|err| { + + running_zone.add_default_route(gateway).await.map_err(|err| { Error::ZoneCommand { intent: "Adding Route".to_string(), err } })?; @@ -270,24 +268,6 @@ impl ServiceManager { let smf_name = format!("svc:/system/illumos/{}", service.name); let default_smf_name = format!("{}:default", smf_name); - // Ensure the IPv6 traffic can be routed to this non-global zone. - // - // This is particularly important for accessing Nexus' external - // interface from off-device. - running_zone - .run_cmd(&[ - "/usr/sbin/routeadm", - "-e", - "ipv6-forwarding", - "-e", - "ipv6-routing", - "-u", - ]) - .map_err(|err| Error::ZoneCommand { - intent: "enabling IPv6 forwarding & routing".to_string(), - err, - })?; - match service.name.as_str() { "internal-dns" => { info!(self.log, "Setting up internal-dns service"); @@ -490,7 +470,7 @@ mod test { wait_ctx.expect().return_once(|_, _| Ok(())); // Import the manifest, enable the service let execute_ctx = crate::illumos::execute_context(); - execute_ctx.expect().times(4).returning(|_| { + execute_ctx.expect().times(3).returning(|_| { Ok(std::process::Output { status: std::process::ExitStatus::from_raw(0), stdout: vec![], diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index ee364b40310..a02e68baae9 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -15,9 +15,7 @@ use crate::params::DatasetKind; use futures::stream::FuturesOrdered; use futures::FutureExt; use futures::StreamExt; -use ipnetwork::Ipv6Network; use nexus_client::types::{DatasetPutRequest, ZpoolPutRequest}; -use omicron_common::address::AZ_PREFIX; use omicron_common::api::external::{ByteCount, ByteCountRangeError}; use omicron_common::backoff; use schemars::JsonSchema; @@ -497,9 +495,10 @@ async fn ensure_running_zone( zone.ensure_address(address_request).await?; - let gz_subnet = - Ipv6Network::new(underlay_address, AZ_PREFIX).unwrap(); - zone.add_route(gz_subnet).await.map_err(Error::ZoneCommand)?; + let gateway = underlay_address; + zone.add_default_route(gateway) + .await + .map_err(Error::ZoneCommand)?; dataset_info .start_zone(log, &zone, dataset_info.address, do_format)