[sled-agent] Launch from bootstrap agent, derive bootstrap addresses, allocate sled /64s

common/src/api/external/mod.rs

@@ -1072,6 +1072,12 @@ impl std::fmt::Display for Ipv6Net {
     }
 }
 
+impl From<ipnetwork::Ipv6Network> for Ipv6Net {
+    fn from(n: ipnetwork::Ipv6Network) -> Ipv6Net {
+        Self(n)
+    }
+}
+
 impl JsonSchema for Ipv6Net {
     fn schema_name() -> String {
         "Ipv6Net".to_string()

docs/how-to-run.adoc

@@ -113,19 +113,21 @@ we'll assign addresses as per RFD 63 as well as incorporating DNS based
 service discovery.
 
 For the purposes of local development today, we specify some hardcoded IPv6
-unique local addresses in `fd00:1de::/16`:
+unique local addresses in the subnet of the first Sled Agent: `fd00:1122:3344:1::/64`:
 
 [options="header"]
 |===================================================================================================
-| Service                    | Endpoint                                                             
-| Sled Agent: Bootstrap      | `[::]:12346`                                                         
-| Sled Agent: Dropshot API   | `[fd00:1de::]:12345`                                                 
-| Cockroach DB               | `[fd00:1de::5]:32221`                                                
-| Oximeter                   | `[fd00:1de::6]:12223`                                                
-| Nexus: External API        | `[fd00:1de::7]:12220`                                                
-| Nexus: Internal API        | `[fd00:1de::7]:12221`                                                
-| Clickhouse                 | `[fd00:1de::8]:8123`                                                 
-| Crucible Downstairs        | `[fd00:1de::9]:32345`, `[fd00:1de::10]:32345`, `[fd00:1de::11]:32345`
+| Service                    | Endpoint
+| Sled Agent: Bootstrap      | Derived from MAC address of physical data link.
+| Sled Agent: Dropshot API   | `[fd00:1122:3344:1::1]:12345`
+| Cockroach DB               | `[fd00:1122:3344:1::2]:32221`
+| Nexus: External API        | `[fd00:1122:3344:1::3]:12220`
+| Nexus: Internal API        | `[fd00:1122:3344:1::3]:12221`
+| Oximeter                   | `[fd00:1122:3344:1::4]:12223`
+| Clickhouse                 | `[fd00:1122:3344:1::5]:8123`
+| Crucible Downstairs 1      | `[fd00:1122:3344:1::6]:32345`
+| Crucible Downstairs 2      | `[fd00:1122:3344:1::7]:32345`
+| Crucible Downstairs 3      | `[fd00:1122:3344:1::8]:32345`
 |===================================================================================================
 
 Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other

openapi/bootstrap-agent.json

@@ -12,7 +12,7 @@
   "paths": {
     "/request_share": {
       "get": {
-        "operationId": "api_request_share",
+        "operationId": "request_share",
         "requestBody": {
           "content": {
             "application/json": {
@@ -42,6 +42,39 @@
           }
         }
       }
+    },
+    "/start_sled": {
+      "put": {
+        "operationId": "start_sled",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/SledAgentRequest"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/SledAgentResponse"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
     }
   },
   "components": {
@@ -77,6 +110,13 @@
           "request_id"
         ]
       },
+      "Ipv6Net": {
+        "title": "An IPv6 subnet",
+        "description": "An IPv6 subnet, including prefix and subnet mask",
+        "type": "string",
+        "pattern": "^(fd|FD)[0-9a-fA-F]{2}:((([0-9a-fA-F]{1,4}\\:){6}[0-9a-fA-F]{1,4})|(([0-9a-fA-F]{1,4}:){1,6}:))/(6[4-9]|[7-9][0-9]|1[0-1][0-9]|12[0-6])$",
+        "maxLength": 43
+      },
       "ShareRequest": {
         "description": "Identity signed by local RoT and Oxide certificate chain.",
         "type": "object",
@@ -110,6 +150,44 @@
         "required": [
           "shared_secret"
         ]
+      },
+      "SledAgentRequest": {
+        "description": "Configuration information for launching a Sled Agent.",
+        "type": "object",
+        "properties": {
+          "subnet": {
+            "description": "Portion of the IP space to be managed by the Sled Agent.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/SledSubnet"
+              }
+            ]
+          }
+        },
+        "required": [
+          "subnet"
+        ]
+      },
+      "SledAgentResponse": {
+        "description": "Describes the Sled Agent running on the device.",
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": [
+          "id"
+        ]
+      },
+      "SledSubnet": {
+        "description": "Represents subnets belonging to Sleds.\n\nThis is a thin wrapper around the [`Ipv6Net`] type - which may be accessed by [`AsRef<Ipv6Net>`] - which adds additional validation that this is a /64 subnet with an expected prefix.",
+        "allOf": [
+          {
+            "$ref": "#/components/schemas/Ipv6Net"
+          }
+        ]
       }
     }
   }

sled-agent/Cargo.toml

@@ -17,6 +17,7 @@ crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev
 dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] }
 futures = "0.3.21"
 ipnetwork = "0.18"
+macaddr = { version = "1.0.1", features = [ "serde_std" ] }
 nexus-client = { path = "../nexus-client" }
 omicron-common = { path = "../common" }
 p256 = "0.9.0"

sled-agent/src/bin/sled-agent.rs

@@ -4,19 +4,19 @@
 
 //! Executable program to run the sled agent
 
-#![feature(async_closure)]
-
 use dropshot::ConfigDropshot;
 use dropshot::ConfigLogging;
 use dropshot::ConfigLoggingLevel;
 use omicron_common::api::external::Error;
 use omicron_common::cmd::fatal;
 use omicron_common::cmd::CmdError;
 use omicron_sled_agent::bootstrap::{
-    config::Config as BootstrapConfig, server as bootstrap_server,
+    agent::bootstrap_address, config::Config as BootstrapConfig,
+    server as bootstrap_server,
 };
 use omicron_sled_agent::rack_setup::config::SetupServiceConfig as RssConfig;
 use omicron_sled_agent::{config::Config as SledConfig, server as sled_server};
+use std::net::SocketAddr;
 use std::path::PathBuf;
 use structopt::StructOpt;
 
@@ -81,10 +81,8 @@ async fn do_run() -> Result<(), CmdError> {
             }
         },
         Args::Run { config_path } => {
-            let mut config = SledConfig::from_file(&config_path)
+            let config = SledConfig::from_file(&config_path)
                 .map_err(|e| CmdError::Failure(e.to_string()))?;
-            config.dropshot.request_body_max_bytes = 1024 * 1024;
-            let config = config;
 
             // - Sled agent starts with the normal config file - typically
             // called "config.toml".
@@ -113,11 +111,18 @@ async fn do_run() -> Result<(), CmdError> {
                 None
             };
 
+            // Derive the bootstrap address from the data link's MAC address.
+            let link = config
+                .get_link()
+                .map_err(|e| CmdError::Failure(e.to_string()))?;
+            let bootstrap_address = bootstrap_address(link)
+                .map_err(|e| CmdError::Failure(e.to_string()))?;
+
             // Configure and run the Bootstrap server.
             let bootstrap_config = BootstrapConfig {
                 id: config.id,
                 dropshot: ConfigDropshot {
-                    bind_address: config.bootstrap_address,
+                    bind_address: SocketAddr::V6(bootstrap_address),
                     request_body_max_bytes: 1024 * 1024,
                     ..Default::default()
                 },
@@ -126,32 +131,21 @@ async fn do_run() -> Result<(), CmdError> {
                 },
                 rss_config,
             };
-            let run_bootstrap = async move || -> Result<(), CmdError> {
-                bootstrap_server::Server::start(&bootstrap_config)
-                    .await
-                    .map_err(CmdError::Failure)?
-                    .wait_for_finish()
-                    .await
-                    .map_err(CmdError::Failure)
-            };
 
-            let run_sled_server = async move || -> Result<(), CmdError> {
-                sled_server::Server::start(&config)
-                    .await
-                    .map_err(CmdError::Failure)?
-                    .wait_for_finish()
-                    .await
-                    .map_err(CmdError::Failure)
-            };
+            // TODO: It's a little silly to pass the config this way - namely,
+            // that we construct the bootstrap config from `config`, but then
+            // pass it separately just so the sled agent can ingest it later on.
+            bootstrap_server::Server::start(
+                *bootstrap_address.ip(),
+                bootstrap_config,
+                config,
+            )
+            .await
+            .map_err(CmdError::Failure)?
+            .wait_for_finish()
+            .await
+            .map_err(CmdError::Failure)?;
 
-            tokio::select! {
-                Err(e) = run_bootstrap() => {
-                    eprintln!("Boot server exited unexpectedly: {:?}", e);
-                },
-                Err(e) = run_sled_server() => {
-                    eprintln!("Sled server exited unexpectedly: {:?}", e);
-                },
-            }
             Ok(())
         }
     }