From 7b0d6df33fb5f37d0091248d4cc0dd3840824888 Mon Sep 17 00:00:00 2001 From: Nils Nieuwejaar Date: Sat, 2 Jul 2022 15:39:17 +0000 Subject: [PATCH 1/4] Add support for installing and launching the dendrite service zone --- common/src/address.rs | 1 + internal-dns-client/src/names.rs | 6 ++++ nexus/src/db/model/service_kind.rs | 4 +++ nexus/src/internal_api/params.rs | 3 ++ openapi/sled-agent.json | 26 +++++++++++++++++ package-manifest.toml | 17 +++++++++++ sled-agent/src/params.rs | 46 +++++++++++++++++++++++++++++- sled-agent/src/services.rs | 15 ++++++++++ smf/sled-agent/config-rss.toml | 9 ++++++ 9 files changed, 126 insertions(+), 1 deletion(-) diff --git a/common/src/address.rs b/common/src/address.rs index 6013932d47a..7b22e8f3ec9 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -35,6 +35,7 @@ pub const SLED_AGENT_PORT: u16 = 12345; pub const PROPOLIS_PORT: u16 = 12400; pub const CLICKHOUSE_PORT: u16 = 8123; pub const OXIMETER_PORT: u16 = 12223; +pub const DENDRITE_PORT: u16 = 12224; pub const NEXUS_INTERNAL_PORT: u16 = 12221; diff --git a/internal-dns-client/src/names.rs b/internal-dns-client/src/names.rs index 1b633f915e1..f3a81ed45f2 100644 --- a/internal-dns-client/src/names.rs +++ b/internal-dns-client/src/names.rs @@ -17,6 +17,7 @@ pub enum ServiceName { InternalDNS, Nexus, Oximeter, + Dendrite, } impl fmt::Display for ServiceName { @@ -27,6 +28,7 @@ impl fmt::Display for ServiceName { ServiceName::InternalDNS => write!(f, "internalDNS"), ServiceName::Nexus => write!(f, "nexus"), ServiceName::Oximeter => write!(f, "oximeter"), + ServiceName::Dendrite => write!(f, "dendrite"), } } } @@ -122,6 +124,10 @@ mod test { SRV::Service(ServiceName::Oximeter).to_string(), "_oximeter._tcp.control-plane.oxide.internal", ); + assert_eq!( + SRV::Service(ServiceName::Dendrite).to_string(), + "_dendrite._tcp.control-plane.oxide.internal", + ); } #[test] diff --git a/nexus/src/db/model/service_kind.rs b/nexus/src/db/model/service_kind.rs index f66532e64c0..7b5197cd0ad 100644 --- a/nexus/src/db/model/service_kind.rs +++ b/nexus/src/db/model/service_kind.rs @@ -20,6 +20,7 @@ impl_enum_type!( InternalDNS => b"internal_dns" Nexus => b"nexus" Oximeter => b"oximeter" + Dendrite => b"dendrite" ); impl From for ServiceKind { @@ -32,6 +33,9 @@ impl From for ServiceKind { internal_api::params::ServiceKind::Oximeter => { ServiceKind::Oximeter } + internal_api::params::ServiceKind::Dendrite => { + ServiceKind::Dendrite + } } } } diff --git a/nexus/src/internal_api/params.rs b/nexus/src/internal_api/params.rs index 8b83138c2b5..4e111cd06dd 100644 --- a/nexus/src/internal_api/params.rs +++ b/nexus/src/internal_api/params.rs @@ -106,6 +106,7 @@ pub enum ServiceKind { InternalDNS, Nexus, Oximeter, + Dendrite, } impl fmt::Display for ServiceKind { @@ -115,6 +116,7 @@ impl fmt::Display for ServiceKind { InternalDNS => "internal_dns", Nexus => "nexus", Oximeter => "oximeter", + Dendrite => "dendrite", }; write!(f, "{}", s) } @@ -129,6 +131,7 @@ impl FromStr for ServiceKind { "nexus" => Ok(Nexus), "oximeter" => Ok(Oximeter), "internal_dns" => Ok(InternalDNS), + "dendrite" => Ok(Dendrite), _ => Err(Self::Err::InternalError { internal_message: format!("Unknown service kind: {}", s), }), diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 8dc890d68b8..34796f6abe8 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -403,6 +403,14 @@ } ] }, + "DendriteAsic": { + "type": "string", + "enum": [ + "tofino_asic", + "tofino_stub", + "soft_npu" + ] + }, "DiskEnsureBody": { "description": "Sent from to a sled agent to establish the runtime state of a Disk", "type": "object", @@ -1206,6 +1214,24 @@ "required": [ "type" ] + }, + { + "type": "object", + "properties": { + "asic": { + "$ref": "#/components/schemas/DendriteAsic" + }, + "type": { + "type": "string", + "enum": [ + "dendrite" + ] + } + }, + "required": [ + "asic", + "type" + ] } ] }, diff --git a/package-manifest.toml b/package-manifest.toml index d987cb0b21a..8684bd39229 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -124,3 +124,20 @@ commit = "2be097ddd1d3fd8e7f56bc0a4bfd696253b11454" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt sha256 = "94218915ec6fed75dcc81d736bd5e9ef62c9eb651a67a943b297d94d6b390941" + +[external_package.dendrite] +service_name = "dendrite" +zone = true +[external_package.dendrite.source] +# To manually override the package source: +# +# 1. Build the zone image manually +# 1a. cd +# 1b. cargo build --features= +# 1c. cargo xtask dist -o +# 2. Copy the output zone image from dendrite/out to omicron/out +# 3. Use type = "manual" instead of the "prebuilt" +type = "prebuilt" +repo = "dendrite" +commit = "4f3f9c6617d21164117664650b6fb843bbc28b39" +sha256 = "8348bde61569ff27df50baa99afdbd0b7dfc91d23a2b46bba42a6d4b98c420c2" diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index ae4cb40c6e2..d3649b981bb 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use internal_dns_client::names::{BackendName, ServiceName, AAAA, SRV}; -use omicron_common::address::OXIMETER_PORT; +use omicron_common::address::{DENDRITE_PORT, OXIMETER_PORT}; use omicron_common::api::external; use omicron_common::api::internal::nexus::{ DiskRuntimeState, InstanceRuntimeState, @@ -199,6 +199,44 @@ pub struct InstanceSerialConsoleData { pub last_byte_offset: u64, } +// The type of networking 'ASIC' the Dendrite service is expected to manage +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Copy, Hash, +)] +#[serde(rename_all = "snake_case")] +pub enum DendriteAsic { + #[serde(rename = "tofino_asic")] + TofinoAsic, + #[serde(rename = "tofino_stub")] + TofinoStub, + #[serde(rename = "softnpu")] + SoftNpu, +} + +impl std::fmt::Display for DendriteAsic { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + DendriteAsic::TofinoAsic => "tofino_asic", + DendriteAsic::TofinoStub => "tofino_stub", + DendriteAsic::SoftNpu => "softnpu", + } + ) + } +} + +impl From for sled_agent_client::types::DendriteAsic { + fn from(a: DendriteAsic) -> Self { + match a { + DendriteAsic::TofinoAsic => Self::TofinoAsic, + DendriteAsic::TofinoStub => Self::TofinoStub, + DendriteAsic::SoftNpu => Self::SoftNpu, + } + } +} + /// The type of a dataset, and an auxiliary information necessary /// to successfully launch a zone managing the associated data. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] @@ -306,6 +344,7 @@ pub enum ServiceType { Nexus { internal_address: SocketAddrV6, external_address: SocketAddrV6 }, InternalDns { server_address: SocketAddrV6, dns_address: SocketAddrV6 }, Oximeter, + Dendrite { asic: DendriteAsic }, } impl From for sled_agent_client::types::ServiceType { @@ -325,6 +364,7 @@ impl From for sled_agent_client::types::ServiceType { } } St::Oximeter => AutoSt::Oximeter, + St::Dendrite { asic } => AutoSt::Dendrite { asic: asic.into() }, } } } @@ -367,6 +407,7 @@ impl ServiceRequest { } ServiceType::Nexus { .. } => SRV::Service(ServiceName::Nexus), ServiceType::Oximeter => SRV::Service(ServiceName::Oximeter), + ServiceType::Dendrite { .. } => SRV::Service(ServiceName::Dendrite), } } @@ -377,6 +418,9 @@ impl ServiceRequest { ServiceType::Oximeter => { SocketAddrV6::new(self.addresses[0], OXIMETER_PORT, 0, 0) } + ServiceType::Dendrite { .. } => { + SocketAddrV6::new(self.addresses[0], DENDRITE_PORT, 0, 0) + } } } } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index adf2731eb9a..52f456eae19 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -507,6 +507,21 @@ impl ServiceManager { err, })?; } + ServiceType::Dendrite { asic } => { + info!(self.log, "Setting up dendrite service"); + running_zone + .run_cmd(&[ + crate::illumos::zone::SVCCFG, + "-s", + &smf_name, + "setprop", + &format!("config/asic={}", asic), + ]) + .map_err(|err| Error::ZoneCommand { + intent: "set dendrite asic type".to_string(), + err, + })?; + } } debug!(self.log, "enabling service"); diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index a5f06d1cfc5..33a2ee53b2b 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -67,3 +67,12 @@ addresses = [ "fd00:1122:3344:0101::4" ] gz_addresses = [] [request.service.service_type] type = "oximeter" + +[[request.service]] +id = "a0fe5ebc-9261-6f77-acc1-972481755789" +name = "dendrite" +addresses = [ "fd00:1122:3344:0101::9" ] +gz_addresses = [] +[request.service.service_type] +type = "dendrite" +asic = "tofino_stub" From 52253f16c3a6a4375a02b04e601c93d9848e8a1c Mon Sep 17 00:00:00 2001 From: Nils Nieuwejaar Date: Sat, 2 Jul 2022 19:06:56 +0000 Subject: [PATCH 2/4] fix rust->json naming --- openapi/sled-agent.json | 2 +- sled-agent/src/params.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 34796f6abe8..021508f72fe 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -408,7 +408,7 @@ "enum": [ "tofino_asic", "tofino_stub", - "soft_npu" + "softnpu" ] }, "DiskEnsureBody": { diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index d3649b981bb..f2d946d3d65 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -203,7 +203,6 @@ pub struct InstanceSerialConsoleData { #[derive( Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Copy, Hash, )] -#[serde(rename_all = "snake_case")] pub enum DendriteAsic { #[serde(rename = "tofino_asic")] TofinoAsic, From 502b147b29c607949490f7cc8ccfaf4534cbd12b Mon Sep 17 00:00:00 2001 From: Nils Nieuwejaar Date: Sun, 3 Jul 2022 15:12:24 +0000 Subject: [PATCH 3/4] update openapo json --- openapi/nexus-internal.json | 3 ++- sled-agent/src/params.rs | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 743b4107589..549ac7600ea 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -1741,7 +1741,8 @@ "enum": [ "internal_d_n_s", "nexus", - "oximeter" + "oximeter", + "dendrite" ] }, "ServicePutRequest": { diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index f2d946d3d65..1d0558905e9 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -203,13 +203,11 @@ pub struct InstanceSerialConsoleData { #[derive( Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Copy, Hash, )] +#[serde(rename_all = "snake_case")] pub enum DendriteAsic { - #[serde(rename = "tofino_asic")] TofinoAsic, - #[serde(rename = "tofino_stub")] TofinoStub, - #[serde(rename = "softnpu")] - SoftNpu, + Softnpu, } impl std::fmt::Display for DendriteAsic { @@ -220,7 +218,7 @@ impl std::fmt::Display for DendriteAsic { match self { DendriteAsic::TofinoAsic => "tofino_asic", DendriteAsic::TofinoStub => "tofino_stub", - DendriteAsic::SoftNpu => "softnpu", + DendriteAsic::Softnpu => "softnpu", } ) } @@ -231,7 +229,7 @@ impl From for sled_agent_client::types::DendriteAsic { match a { DendriteAsic::TofinoAsic => Self::TofinoAsic, DendriteAsic::TofinoStub => Self::TofinoStub, - DendriteAsic::SoftNpu => Self::SoftNpu, + DendriteAsic::Softnpu => Self::Softnpu, } } } From 9d2191ffc64b72f4b727606af8ea2e91f927230f Mon Sep 17 00:00:00 2001 From: Nils Nieuwejaar Date: Wed, 6 Jul 2022 14:15:46 +0000 Subject: [PATCH 4/4] refresh dendrite service config before enabling it --- package-manifest.toml | 4 ++-- sled-agent/src/services.rs | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/package-manifest.toml b/package-manifest.toml index 8684bd39229..b6450ed25b9 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -139,5 +139,5 @@ zone = true # 3. Use type = "manual" instead of the "prebuilt" type = "prebuilt" repo = "dendrite" -commit = "4f3f9c6617d21164117664650b6fb843bbc28b39" -sha256 = "8348bde61569ff27df50baa99afdbd0b7dfc91d23a2b46bba42a6d4b98c420c2" +commit = "e752af977c167fa64625231a74bedcbfa5457008" +sha256 = "ddb7c271f4d4b68332adc69b32f90e728f07b1395e01cffd7e71230c2c648c63" diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 52f456eae19..7fefa81b313 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -521,6 +521,21 @@ impl ServiceManager { intent: "set dendrite asic type".to_string(), err, })?; + + running_zone + .run_cmd(&[ + crate::illumos::zone::SVCCFG, + "-s", + &default_smf_name, + "refresh", + ]) + .map_err(|err| Error::ZoneCommand { + intent: format!( + "Refresh SMF manifest {}", + default_smf_name + ), + err, + })?; } }