From 256ced2d5a7f843473a1dad85bd27baac274164b Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Tue, 29 Mar 2022 05:24:46 -0700 Subject: [PATCH 1/2] Slightly change IPv6 prefix used and update README with address mappings. --- README.adoc | 27 +++++++++++++++++++++++++++ sled-agent/src/illumos/zone.rs | 5 +---- smf/nexus/config.toml | 8 ++++---- smf/oximeter/config.toml | 6 +++--- smf/sled-agent/config-rss.toml | 18 +++++++++--------- smf/sled-agent/config.toml | 12 ++---------- 6 files changed, 46 insertions(+), 30 deletions(-) diff --git a/README.adoc b/README.adoc index 844af21ad0b..75de8463321 100644 --- a/README.adoc +++ b/README.adoc @@ -437,6 +437,33 @@ executed: $ sudo ./target/debug/omicron-package uninstall ---- +=== Test Environment + +When we deploy, we're effectively creating a number of different zones +for all the components that make up Omicron (Nexus, Clickhouse, Crucible, etc). +Since all these services run in different zones they cannot communicate with +each other (and Sled Agent in the global zone) via `localhost`. In practise, +we'll assign addresses as per RFD 63 as well as incorporating DNS based +service discovery. + +For the purposes of local development today, we specify some hardcoded IPv6 +unique local addresses in `fd00:1de::/16`: + +[options="header"] +|=================================================================================================== +| Service | Endpoint +| Sled Agent: Bootstrap | `[::]:12346` +| Sled Agent: Dropshot API | `[fd00:1de::]:12345` +| Cockroach DB | `[fd00:1de::5]:32221` +| Oximeter | `[fd00:1de::6]:12223` +| Nexus: External API | `[fd00:1de::7]:12220` +| Nexus: Internal API | `[fd00:1de::7]:12221` +| Clickhouse | `[fd00:1de::8]:8123` +| Crucible Downstairs | `[fd00:1de::9]:32345`, `[fd00:1de::10]:32345`, `[fd00:1de::11]:32345` +|=================================================================================================== + +Note that Sled Agent runs in the global zone and is the one responsible for bringing up all the other +other services and allocating them with vNICs and IPv6 addresses. == Configuration reference diff --git a/sled-agent/src/illumos/zone.rs b/sled-agent/src/illumos/zone.rs index d62911aa50e..021ab98a2dd 100644 --- a/sled-agent/src/illumos/zone.rs +++ b/sled-agent/src/illumos/zone.rs @@ -428,10 +428,7 @@ impl Zones { Self::ensure_address( None, &gz_link_local_addrobj.on_same_interface("v6route")?, - AddressRequest::new_static( - "fd00:1234::".parse().unwrap(), - Some(16), - ), + AddressRequest::new_static("fd00:1de::".parse().unwrap(), Some(16)), )?; Ok(()) } diff --git a/smf/nexus/config.toml b/smf/nexus/config.toml index 257e8991acd..02f663f7c94 100644 --- a/smf/nexus/config.toml +++ b/smf/nexus/config.toml @@ -18,15 +18,15 @@ schemes_external = ["spoof", "session_cookie"] [database] # URL for connecting to the database -url = "postgresql://root@[fd00:1234::5]:32221/omicron?sslmode=disable" +url = "postgresql://root@[fd00:1de::5]:32221/omicron?sslmode=disable" [dropshot_external] # IP address and TCP port on which to listen for the external API -bind_address = "192.168.1.123:12220" +bind_address = "[fd00:1de::7]:12220" [dropshot_internal] # IP address and TCP port on which to listen for the internal API -bind_address = "[fd00:1234::7]:12221" +bind_address = "[fd00:1de::7]:12221" [log] # Show log messages of this level and more severe @@ -42,4 +42,4 @@ mode = "stderr-terminal" # Configuration for interacting with the timeseries database [timeseries_db] -address = "[fd00:1234::8]:8123" +address = "[fd00:1de::8]:8123" diff --git a/smf/oximeter/config.toml b/smf/oximeter/config.toml index b628f1840a1..78eb1027842 100644 --- a/smf/oximeter/config.toml +++ b/smf/oximeter/config.toml @@ -2,10 +2,10 @@ id = "1da65e5b-210c-4859-a7d7-200c1e659972" # Internal address of nexus -nexus_address = "[fd00:1234::7]:12221" +nexus_address = "[fd00:1de::7]:12221" [db] -address = "[fd00:1234::8]:8123" +address = "[fd00:1de::8]:8123" batch_size = 1000 batch_interval = 5 # In seconds @@ -14,4 +14,4 @@ level = "debug" mode = "stderr-terminal" [dropshot] -bind_address = "[fd00:1234::6]:12223" +bind_address = "[fd00:1de::6]:12223" diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index 8c3e5d976b4..c9fd9ff43dc 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -1,40 +1,40 @@ # RSS (Rack Setup Service) "stand-in" configuration. [[request]] -sled_address = "192.168.1.201:12345" +sled_address = "[fd00:1de::]:12345" [[request.partition]] zpool_uuid = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1234::9]:32345" +address = "[fd00:1de::9]:32345" partition_kind.type = "crucible" [[request.partition]] zpool_uuid = "e4b4dc87-ab46-49fb-a4b4-d361ae214c03" -address = "[fd00:1234::10]:32345" +address = "[fd00:1de::10]:32345" partition_kind.type = "crucible" [[request.partition]] zpool_uuid = "f4b4dc87-ab46-49fb-a4b4-d361ae214c03" -address = "[fd00:1234::11]:32345" +address = "[fd00:1de::11]:32345" partition_kind.type = "crucible" [[request.partition]] zpool_uuid = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1234::5]:32221" +address = "[fd00:1de::5]:32221" partition_kind.type = "cockroach_db" partition_kind.all_addresses = [ - "[fd00:1234::5]:32221", + "[fd00:1de::5]:32221", ] [[request.partition]] zpool_uuid = "d462a7f7-b628-40fe-80ff-4e4189e2d62b" -address = "[fd00:1234::8]:8123" +address = "[fd00:1de::8]:8123" partition_kind.type = "clickhouse" [[request.service]] name = "nexus" -addresses = [ "[fd00:1234::7]:12221", "192.168.1.123:12220" ] +addresses = [ "[fd00:1de::7]:12220", "[fd00:1de::7]:12221" ] [[request.service]] name = "oximeter" -addresses = [ "[fd00:1234::6]:12223" ] +addresses = [ "[fd00:1de::6]:12223" ] diff --git a/smf/sled-agent/config.toml b/smf/sled-agent/config.toml index 5aae3b8ee7c..2aaddae943f 100644 --- a/smf/sled-agent/config.toml +++ b/smf/sled-agent/config.toml @@ -4,7 +4,7 @@ id = "fb0f7546-4d46-40ca-9d56-cbb810684ca7" bootstrap_address = "[::]:12346" # Internal address of Nexus -nexus_address = "[fd00:1234::7]:12221" +nexus_address = "[fd00:1de::7]:12221" # A file-backed zpool can be manually created with the following: # $ truncate -s 10GB testpool.vdev @@ -25,16 +25,8 @@ zpools = [ # With the usage of non-global zones, we no longer can use localhost addresses, # as Nexus (within a Zone) is effectively "on a different machine" from the # sled agent. -# -# This address may be allocated with the following command: -# -# pfexec ipadm create-addr -t -T static -a 192.168.1.201 $(dladm show-phys -p -o LINK)/sled4 -# | | | -# Please | | -# Create a temporary static address | -# On whatever physical datalink I have [dropshot] -bind_address = "192.168.1.201:12345" +bind_address = "[fd00:1de::]:12345" [log] level = "info" From 4d2e12d7d551c7c9b8162f4338282f4a218a187a Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Wed, 30 Mar 2022 10:13:33 -0700 Subject: [PATCH 2/2] Update README.adoc Co-authored-by: Sean Klein --- README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index 75de8463321..ef6e5d40f65 100644 --- a/README.adoc +++ b/README.adoc @@ -442,7 +442,7 @@ $ sudo ./target/debug/omicron-package uninstall When we deploy, we're effectively creating a number of different zones for all the components that make up Omicron (Nexus, Clickhouse, Crucible, etc). Since all these services run in different zones they cannot communicate with -each other (and Sled Agent in the global zone) via `localhost`. In practise, +each other (and Sled Agent in the global zone) via `localhost`. In practice, we'll assign addresses as per RFD 63 as well as incorporating DNS based service discovery.