From 95e35d2b7af12cbf8273b9b603bd3a26e344b5ed Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 5 Oct 2022 11:53:26 -0400 Subject: [PATCH 1/4] MGS: Take server ID/address as command line args --- gateway/Cargo.toml | 4 ++ gateway/examples/config.toml | 8 --- gateway/src/bin/gateway.rs | 51 ----------------- gateway/src/bin/mgs.rs | 62 +++++++++++++++++++++ gateway/src/config.rs | 6 +- gateway/src/lib.rs | 37 ++++++++++-- gateway/tests/config.test.toml | 13 ----- gateway/tests/integration_tests/commands.rs | 16 +++--- gateway/tests/integration_tests/setup.rs | 20 +++++-- 9 files changed, 119 insertions(+), 98 deletions(-) delete mode 100644 gateway/src/bin/gateway.rs create mode 100644 gateway/src/bin/mgs.rs diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 108a62ce374..d395102f771 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -41,3 +41,7 @@ serde_json = "1.0" sp-sim = { path = "../sp-sim" } subprocess = "0.2.9" tokio-tungstenite = "0.17" + +[[bin]] +name = "mgs" +doc = false diff --git a/gateway/examples/config.toml b/gateway/examples/config.toml index c3e6bbeb559..cc61efc02a3 100644 --- a/gateway/examples/config.toml +++ b/gateway/examples/config.toml @@ -2,9 +2,6 @@ # Oxide API: example configuration file # -# Identifier for this instance of MGS -id = "8afcb12d-f625-4df9-bdf2-f495c3bbd323" - [switch] # which vsc port is connected to our local sidecar SP (i.e., the SP that acts as # our contact to the ignition controller) @@ -79,11 +76,6 @@ bulk_request_max_millis = 60_000 bulk_request_page_millis = 1_000 bulk_request_retain_grace_period_millis = 10_000 -[dropshot] -# IP address and TCP port on which to listen for the external API -bind_address = "127.0.0.1:12222" -request_body_max_bytes = 134217728 # 128 MiB - [log] # Show log messages of this level and more severe level = "debug" diff --git a/gateway/src/bin/gateway.rs b/gateway/src/bin/gateway.rs deleted file mode 100644 index 2a4b679fad2..00000000000 --- a/gateway/src/bin/gateway.rs +++ /dev/null @@ -1,51 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Executable program to run gateway, the management gateway service - -use clap::Parser; -use omicron_common::cmd::{fatal, CmdError}; -use omicron_gateway::{run_openapi, run_server, Config}; -use std::path::PathBuf; - -#[derive(Debug, Parser)] -#[clap(name = "gateway", about = "See README.adoc for more information")] -struct Args { - #[clap( - short = 'O', - long = "openapi", - help = "Print the external OpenAPI Spec document and exit", - action - )] - openapi: bool, - - #[clap( - name = "CONFIG_FILE_PATH", - action, - required_unless_present = "openapi" - )] - config_file_path: Option, -} - -#[tokio::main] -async fn main() { - if let Err(cmd_error) = do_run().await { - fatal(cmd_error); - } -} - -async fn do_run() -> Result<(), CmdError> { - let args = Args::parse(); - - if args.openapi { - run_openapi().map_err(CmdError::Failure) - } else { - // `.unwrap()` here is fine because our clap config requires - // `config_file_path` to be passed if `openapi` is not. - let config = Config::from_file(args.config_file_path.unwrap()) - .map_err(|e| CmdError::Failure(e.to_string()))?; - - run_server(config).await.map_err(CmdError::Failure) - } -} diff --git a/gateway/src/bin/mgs.rs b/gateway/src/bin/mgs.rs new file mode 100644 index 00000000000..0f07038f2b9 --- /dev/null +++ b/gateway/src/bin/mgs.rs @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Executable program to run gateway, the management gateway service + +use clap::Parser; +use omicron_common::cmd::{fatal, CmdError}; +use omicron_gateway::{run_openapi, run_server, Config, MgsArguments}; +use std::net::SocketAddrV6; +use std::path::PathBuf; +use uuid::Uuid; + +#[derive(Debug, Parser)] +#[clap(name = "gateway", about = "See README.adoc for more information")] +enum Args { + /// Print the external OpenAPI Spec document and exit + #[clap( + help = "Print the external OpenAPI Spec document and exit", + action + )] + Openapi, + + /// Start an MGS server + Run { + #[clap(name = "CONFIG_FILE_PATH", action)] + config_file_path: PathBuf, + + #[clap(short, long, action)] + id: Uuid, + + #[clap(short, long, action)] + address: SocketAddrV6, + }, +} + +#[tokio::main] +async fn main() { + if let Err(cmd_error) = do_run().await { + fatal(cmd_error); + } +} + +async fn do_run() -> Result<(), CmdError> { + let args = Args::parse(); + + match args { + Args::Openapi => run_openapi().map_err(CmdError::Failure), + Args::Run { config_file_path, id, address } => { + let config = Config::from_file(&config_file_path).map_err(|e| { + CmdError::Failure(format!( + "failed to parse {}: {}", + config_file_path.display(), + e + )) + })?; + + let args = MgsArguments { id, address }; + run_server(config, args).await.map_err(CmdError::Failure) + } + } +} diff --git a/gateway/src/config.rs b/gateway/src/config.rs index 806fe910acb..96a86f799d2 100644 --- a/gateway/src/config.rs +++ b/gateway/src/config.rs @@ -5,7 +5,7 @@ //! Interfaces for parsing configuration files and working with a gateway server //! configuration -use dropshot::{ConfigDropshot, ConfigLogging}; +use dropshot::ConfigLogging; use gateway_sp_comms::SwitchConfig; use serde::{Deserialize, Serialize}; use std::path::Path; @@ -36,12 +36,8 @@ pub struct Timeouts { /// Configuration for a gateway server #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Config { - /// Identifier for this instance of MGS - pub id: uuid::Uuid, /// Various timeouts pub timeouts: Timeouts, - /// Dropshot configuration for API server - pub dropshot: ConfigDropshot, /// Configuration of the management switch. pub switch: SwitchConfig, /// Server-wide logging configuration. diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 3c5835bea24..ff800595173 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -13,11 +13,19 @@ pub mod http_entrypoints; // TODO pub only for testing - is this right? pub use config::Config; pub use context::ServerContext; -use slog::{debug, error, info, o, warn, Logger}; +use dropshot::ConfigDropshot; +use slog::debug; +use slog::error; +use slog::info; +use slog::o; +use slog::warn; +use slog::Logger; +use std::net::SocketAddr; +use std::net::SocketAddrV6; use std::sync::Arc; use uuid::Uuid; -/// Run the OpenAPI generator for the API, which emits the OpenAPI spec +/// Run the OpenAPI generator for the API; which emits the OpenAPI spec /// to stdout. pub fn run_openapi() -> Result<(), String> { http_entrypoints::api() @@ -29,6 +37,11 @@ pub fn run_openapi() -> Result<(), String> { .map_err(|e| e.to_string()) } +pub struct MgsArguments { + pub id: Uuid, + pub address: SocketAddrV6, +} + pub struct Server { /// shared state used by API request handlers pub apictx: Arc, @@ -40,10 +53,11 @@ impl Server { /// Start a gateway server. pub async fn start( config: Config, + args: MgsArguments, _rack_id: Uuid, log: &Logger, ) -> Result { - let log = log.new(o!("name" => config.id.to_string())); + let log = log.new(o!("name" => args.id.to_string())); info!(log, "setting up gateway server"); match gateway_sp_comms::register_probes() { @@ -59,8 +73,16 @@ impl Server { format!("initializing server context: {}", error) })?; + let dropshot = ConfigDropshot { + bind_address: SocketAddr::V6(args.address), + // We need to be able to accept the largest single update blob we + // expect to be given, which at the moment is probably a host phase + // 1 image? (32 MiB raw, plus overhead for HTTP encoding) + request_body_max_bytes: 128 << 20, + ..Default::default() + }; let http_server_starter = dropshot::HttpServerStarter::new( - &config.dropshot, + &dropshot, http_entrypoints::api(), Arc::clone(&apictx), &log.new(o!("component" => "dropshot")), @@ -92,7 +114,10 @@ impl Server { } /// Run an instance of the [Server]. -pub async fn run_server(config: Config) -> Result<(), String> { +pub async fn run_server( + config: Config, + args: MgsArguments, +) -> Result<(), String> { use slog::Drain; let (drain, registration) = slog_dtrace::with_drain( config @@ -109,7 +134,7 @@ pub async fn run_server(config: Config) -> Result<(), String> { debug!(log, "registered DTrace probes"); } let rack_id = Uuid::new_v4(); - let server = Server::start(config, rack_id, &log).await?; + let server = Server::start(config, args, rack_id, &log).await?; // server.register_as_producer().await; server.wait_for_finish().await } diff --git a/gateway/tests/config.test.toml b/gateway/tests/config.test.toml index 9ab338e1114..8db65186a5e 100644 --- a/gateway/tests/config.test.toml +++ b/gateway/tests/config.test.toml @@ -2,10 +2,6 @@ # Oxide API: example configuration file # -# Identifier for this instance of MGS -# NOTE: The test suite always overrides this. -id = "8afcb12d-f625-4df9-bdf2-f495c3bbd323" - [switch] # which vsc port is connected to our local sidecar SP (i.e., the SP that acts as # our contact to the ignition controller) @@ -87,15 +83,6 @@ bulk_request_max_millis = 40_000 bulk_request_page_millis = 2_000 bulk_request_retain_grace_period_millis = 10_000 -# -# NOTE: for the test suite, the port MUST be 0 (in order to bind to any -# available port) because the test suite will be running many servers -# concurrently. -# -[dropshot] -bind_address = "127.0.0.1:0" -request_body_max_bytes = 1048576 - # # NOTE: for the test suite, if mode = "file", the file path MUST be the sentinel # string "UNUSED". The actual path will be generated by the test suite for each diff --git a/gateway/tests/integration_tests/commands.rs b/gateway/tests/integration_tests/commands.rs index d2eddcdf4fb..5b1b32199cc 100644 --- a/gateway/tests/integration_tests/commands.rs +++ b/gateway/tests/integration_tests/commands.rs @@ -11,21 +11,19 @@ use omicron_test_utils::dev::test_cmds::{ use openapiv3::OpenAPI; use subprocess::Exec; -// name of gateway executable -const CMD_GATEWAY: &str = env!("CARGO_BIN_EXE_gateway"); +// name of mgs executable +const CMD_MGS: &str = env!("CARGO_BIN_EXE_mgs"); -fn path_to_gateway() -> PathBuf { - path_to_executable(CMD_GATEWAY) +fn path_to_mgs() -> PathBuf { + path_to_executable(CMD_MGS) } #[test] -fn test_gateway_openapi_sled() { - let exec = Exec::cmd(path_to_gateway()) - .arg("--openapi") - .arg("examples/config.toml"); +fn test_mgs_openapi_sled() { + let exec = Exec::cmd(path_to_mgs()).arg("openapi"); let (exit_status, stdout_text, stderr_text) = run_command(exec); assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text); - assert_contents("tests/output/cmd-gateway-openapi-stderr", &stderr_text); + assert_contents("tests/output/cmd-mgs-openapi-stderr", &stderr_text); let spec: OpenAPI = serde_json::from_str(&stdout_text) .expect("stdout was not valid OpenAPI"); diff --git a/gateway/tests/integration_tests/setup.rs b/gateway/tests/integration_tests/setup.rs index f9223f5ff45..dbeba7d0092 100644 --- a/gateway/tests/integration_tests/setup.rs +++ b/gateway/tests/integration_tests/setup.rs @@ -8,6 +8,7 @@ use dropshot::test_util::ClientTestContext; use dropshot::test_util::LogContext; use gateway_messages::SpPort; use gateway_sp_comms::SpType; +use omicron_gateway::MgsArguments; use omicron_test_utils::dev::poll; use omicron_test_utils::dev::poll::CondCheckError; use slog::o; @@ -40,10 +41,9 @@ impl GatewayTestContext { pub fn load_test_config() -> (omicron_gateway::Config, sp_sim::Config) { let server_config_file_path = Path::new("tests/config.test.toml"); - let mut server_config = + let server_config = omicron_gateway::Config::from_file(server_config_file_path) .expect("failed to load config.test.toml"); - server_config.id = Uuid::new_v4(); let sp_sim_config_file_path = Path::new("tests/sp_sim_config.test.toml"); let sp_sim_config = sp_sim::Config::from_file(sp_sim_config_file_path) @@ -120,10 +120,18 @@ pub async fn test_setup_with_config( // Start gateway server let rack_id = Uuid::parse_str(RACK_UUID).unwrap(); - let server = - omicron_gateway::Server::start(server_config.clone(), rack_id, log) - .await - .unwrap(); + let args = MgsArguments { + id: Uuid::new_v4(), + address: "[::1]:0".parse().unwrap(), + }; + let server = omicron_gateway::Server::start( + server_config.clone(), + args, + rack_id, + log, + ) + .await + .unwrap(); // Build a list of all SPs defined in our config let mut all_sp_ids = Vec::new(); From 13c1350907d59a7fef5b3c3935baa124e1d47d5b Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 5 Oct 2022 15:33:12 -0400 Subject: [PATCH 2/4] Add MGS zone to package-manifest --- package-manifest.toml | 9 +++++ smf/mgs/config.toml | 82 +++++++++++++++++++++++++++++++++++++++++++ smf/mgs/manifest.xml | 41 ++++++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 smf/mgs/config.toml create mode 100644 smf/mgs/manifest.xml diff --git a/package-manifest.toml b/package-manifest.toml index 80d1e20dbcf..43160cf51c9 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -83,6 +83,15 @@ zone = true from = "smf/internal-dns" to = "/var/svc/manifest/site/internal-dns" +[package.omicron-gateway] +rust.binary_names = ["mgs"] +rust.release = true +service_name = "mgs" +zone = true +[[package.omicron-gateway.paths]] +from = "smf/mgs" +to = "/var/svc/manifest/site/mgs" + # Packages not built within Omicron, but which must be imported. # Refer to diff --git a/smf/mgs/config.toml b/smf/mgs/config.toml new file mode 100644 index 00000000000..f9ec64dfe3f --- /dev/null +++ b/smf/mgs/config.toml @@ -0,0 +1,82 @@ +# +# Oxide API: example configuration file +# + +[switch] +# which vsc port is connected to our local sidecar SP (i.e., the SP that acts as +# our contact to the ignition controller) +local_ignition_controller_port = 0 + +# when sending UDP RPC packets to an SP, how many total attempts do we make +# before giving up? +rpc_max_attempts = 15 + +# sleep time between UDP RPC resends (up to `rpc_max_attempts`) +rpc_per_attempt_timeout_millis = 2000 + +[switch.location] +# possible locations where MGS could be running; these names appear in logs and +# are used in the remainder of the `[switch.*]` configuration to define port +# mappings +names = ["switch0", "switch1"] + +# `[[switch.location.determination]]` is a list of switch ports we should +# contact in order to determine our location; each port defines a subset of +# `[switch.location.names]` which are the possible location(s) of this MGS +# instance if the message was received on the given SP port. When MGS starts, it +# will send a discovery message on each port listed in this section, collect the +# responses, and determine its location via the intersection of the names listed +# below (for all ports which returned a successful response). This process can +# fail if too few SPs respond (leaving us with 2 or more possible locations) or +# if there is a miscabling that results in an unsolvable system (e.g., +# determination 0 reports "switch0" and determination 1 reports "switch1"). +[[switch.location.determination]] +switch_port = 1 +sp_port_1 = ["switch0"] +sp_port_2 = ["switch1"] + +[[switch.location.determination]] +switch_port = 2 +sp_port_1 = ["switch0"] +sp_port_2 = ["switch1"] + +# `[[switch.port.*]]` defines the local data link address (in RFD 250 terms, the +# interface configured to use VLAN tag assigned to the given port) and the +# logical ID of the remote SP ("sled 7", "switch 1", etc.), which must have an +# entry for each member of `[switch.location]` above. +# +# TODO This needs to be replaced with information relevant to real vlan +# datalinks. For now we pick a bunch of arbitrary addresses and ports that don't +# mean anything unless someone spins up simulated SPs on those ports. +[switch.port.0] +data_link_addr = "[::]:33200" +multicast_addr = "[ff15:0:1de::0]:33300" +[switch.port.0.location] +switch0 = ["switch", 0] +switch1 = ["switch", 1] + +[switch.port.1] +data_link_addr = "[::]:33201" +multicast_addr = "[ff15:0:1de::1]:33310" +[switch.port.1.location] +switch0 = ["sled", 0] +switch1 = ["sled", 1] + +[switch.port.2] +data_link_addr = "[::]:33202" +multicast_addr = "[ff15:0:1de::2]:33320" +[switch.port.2.location] +switch0 = ["sled", 1] +switch1 = ["sled", 0] + +[timeouts] +bulk_request_default_millis = 5_000 +bulk_request_max_millis = 60_000 +bulk_request_page_millis = 1_000 +bulk_request_retain_grace_period_millis = 10_000 + +[log] +level = "info" +mode = "file" +path = "/dev/stdout" +if_exists = "append" diff --git a/smf/mgs/manifest.xml b/smf/mgs/manifest.xml new file mode 100644 index 00000000000..44efe84812a --- /dev/null +++ b/smf/mgs/manifest.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From f2284fa1caeb0199fba7d13b69b813e1c3841e25 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Thu, 6 Oct 2022 11:53:00 -0400 Subject: [PATCH 3/4] cargo fmt --- gateway/src/bin/mgs.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gateway/src/bin/mgs.rs b/gateway/src/bin/mgs.rs index 0f07038f2b9..66dfc9728da 100644 --- a/gateway/src/bin/mgs.rs +++ b/gateway/src/bin/mgs.rs @@ -15,10 +15,7 @@ use uuid::Uuid; #[clap(name = "gateway", about = "See README.adoc for more information")] enum Args { /// Print the external OpenAPI Spec document and exit - #[clap( - help = "Print the external OpenAPI Spec document and exit", - action - )] + #[clap(help = "Print the external OpenAPI Spec document and exit", action)] Openapi, /// Start an MGS server From de57d1efc2bace8ca2914e2b3f74d11e567558f9 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Thu, 6 Oct 2022 12:04:48 -0400 Subject: [PATCH 4/4] fix clap help --- gateway/src/bin/mgs.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/gateway/src/bin/mgs.rs b/gateway/src/bin/mgs.rs index 66dfc9728da..cdae9f2ef16 100644 --- a/gateway/src/bin/mgs.rs +++ b/gateway/src/bin/mgs.rs @@ -15,7 +15,6 @@ use uuid::Uuid; #[clap(name = "gateway", about = "See README.adoc for more information")] enum Args { /// Print the external OpenAPI Spec document and exit - #[clap(help = "Print the external OpenAPI Spec document and exit", action)] Openapi, /// Start an MGS server