Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
91059b2
first cut at "add sled" blueprint maker
davepacheco Jan 4, 2024
95a7cd0
Merge branch 'main' into dap/update-control-2
davepacheco Jan 5, 2024
9c7a40d
WIP: make BlueprintBuilder more general because planning logic will g…
davepacheco Jan 5, 2024
7f7235e
refactor towards incremental plans
davepacheco Jan 6, 2024
0fb4e95
when planning, we want to be using the parent blueprint and not the c…
davepacheco Jan 8, 2024
a46520d
WIP: adding internal APIs to be able to play with this stuff
davepacheco Jan 8, 2024
e9bbb5c
WIP: flesh out those internal APIs
davepacheco Jan 8, 2024
155c0be
fetch sleds and zpools for planning
davepacheco Jan 9, 2024
cf3aa30
omdb support for these APIs
davepacheco Jan 9, 2024
52c7d9f
sled agent client could use more common networking types
davepacheco Jan 10, 2024
8e4ef98
print out more about blueprints, plus some cleanup
davepacheco Jan 10, 2024
9a8ec62
add blueprint authz
davepacheco Jan 10, 2024
9670e8b
no more table scan
davepacheco Jan 10, 2024
2646426
remove XXX for issue filed
davepacheco Jan 10, 2024
8f0b344
have simulated sled agent simulate storage too
davepacheco Jan 10, 2024
ef1fa8e
general cleanup -- fix lots of XXXs
davepacheco Jan 10, 2024
a1d40ca
follow-up to sled-agent-client changes
davepacheco Jan 10, 2024
c8f7685
clean up various XXXs
davepacheco Jan 10, 2024
d4b9813
blueprint builder could be idempotent; comment about internal DNS
davepacheco Jan 11, 2024
faa9dde
the case I was worried about should not be a problem
davepacheco Jan 11, 2024
b5a9d47
clippy backlog
davepacheco Jan 11, 2024
2897254
Merge branch 'main' into dap/update-control-2
davepacheco Jan 11, 2024
f50ee0f
use new IP address range
davepacheco Jan 11, 2024
a2abb97
test run + fixups
davepacheco Jan 11, 2024
f0b0451
remove one XXX
davepacheco Jan 11, 2024
b1f2fb9
omdb nits, regenerate spec
davepacheco Jan 11, 2024
52fc677
remove XXXs now tracked elsewhere
davepacheco Jan 11, 2024
d8ad0ee
nit
davepacheco Jan 11, 2024
c39c75d
add diff, simplify some types
davepacheco Jan 12, 2024
81842f6
cleanup, programmatic diff, fix up test
davepacheco Jan 12, 2024
c0da2e5
rustfmt
davepacheco Jan 13, 2024
1342d9f
Merge branch 'main' into dap/update-control-2
davepacheco Jan 13, 2024
58f7e39
support diff collection and blueprint and use that in the test
davepacheco Jan 16, 2024
ce76d0f
add test for ip_allocator
davepacheco Jan 16, 2024
0ec1d8a
add basic tests for blueprint builder
davepacheco Jan 17, 2024
b49794a
Merge branch 'main' into dap/update-control-2
davepacheco Jan 17, 2024
c58c7ed
review feedback
davepacheco Jan 17, 2024
06825c0
make user specify collection
davepacheco Jan 17, 2024
701d2c7
review feedback
davepacheco Jan 17, 2024
2e17346
Merge branch 'main' into dap/update-control-2
davepacheco Jan 17, 2024
80bec1a
stray comment
davepacheco Jan 18, 2024
402ea0c
Merge branch 'main' into dap/update-control-2
davepacheco Jan 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ members = [
"nexus/db-model",
"nexus/db-queries",
"nexus/defaults",
"nexus/deployment",
"nexus/inventory",
"nexus/test-interface",
"nexus/test-utils-macros",
Expand Down Expand Up @@ -114,6 +115,7 @@ default-members = [
"nexus/db-model",
"nexus/db-queries",
"nexus/defaults",
"nexus/deployment",
"nexus/inventory",
"nexus/types",
"oximeter/collector",
Expand Down Expand Up @@ -228,6 +230,7 @@ installinator-artifact-client = { path = "clients/installinator-artifact-client"
installinator-common = { path = "installinator-common" }
internal-dns = { path = "internal-dns" }
ipcc = { path = "ipcc" }
ipnet = "2.9"
ipnetwork = { version = "0.20", features = ["schemars"] }
itertools = "0.12.0"
key-manager = { path = "key-manager" }
Expand All @@ -244,6 +247,7 @@ nexus-client = { path = "clients/nexus-client" }
nexus-db-model = { path = "nexus/db-model" }
nexus-db-queries = { path = "nexus/db-queries" }
nexus-defaults = { path = "nexus/defaults" }
nexus-deployment = { path = "nexus/deployment" }
nexus-inventory = { path = "nexus/inventory" }
omicron-certificates = { path = "certificates" }
omicron-passwords = { path = "passwords" }
Expand Down
1 change: 1 addition & 0 deletions clients/nexus-client/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ license = "MPL-2.0"
chrono.workspace = true
futures.workspace = true
ipnetwork.workspace = true
nexus-types.workspace = true
omicron-common.workspace = true
omicron-passwords.workspace = true
progenitor.workspace = true
Expand Down
21 changes: 8 additions & 13 deletions clients/nexus-client/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ progenitor::generate_api!(
slog::debug!(log, "client response"; "result" => ?result);
}),
replace = {
// It's kind of unfortunate to pull in such a complex and unstable type
// as "blueprint" this way, but we have really useful functionality
// (e.g., diff'ing) that's implemented on our local type.
Blueprint = nexus_types::deployment::Blueprint,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ahl installed some healthy fear of replace in me a while back - how painful would it be to impl From for blueprints instead of replaceing?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I share yours and Adam's reluctance about replace. In this case, the functionality being shared (diff'ing blueprints) is pretty non-trivial and I think it would be fairly painful to impl the requisite Froms: I think it would require implementing From for Blueprint, and OmicronZonesConfig, which in turn requires OmicronZoneConfig, then OmicronZoneType (a big enum with a bunch of properties in many variants), OmicronZoneDataset, SledRole, ZpoolName, NetworkInterface, NetworkInterfaceKind, and SourceNatConfig. (I'm not positive, but I think that's right.)

I think better approaches here might be to have Progenitor validate that your replace type really is compatible with the spec, or alternatively if there were a way to automatically generate From impls between two types that are identical (which would presumably break -- correctly -- if they weren't identical). It doesn't feel sustainable to me to either duplicate complex functionality on multiple types or hand-roll so many complex From impls. What do you think?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I actually had the opposite reaction, and didn't think it was unfortunate to use replace at all. However, clearly I am missing what is problematic here.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think what sucks about using replace is that you lose the ability to notice when your local type doesn't match the spec. In turn that means we lose all of the benefit of Rust's strong static typing across the OpenAPI boundary.

Concretely: in the normal case where Progenitor generates the types for you, say we pull in a new version of the spec with a new required field. Progenitor's type will have that new field in it. So if we had a code path that creates an instance of that type (i.e., to make the request) it will now fail to compile. That's good: if we didn't know it before, we now know we've made an incompatible change to the spec. If we did know it, now's when we update our code to fill in the new value.

If on the other hand we use replace to supply the type, when we pull in the updated spec, everything still compiles -- there's no reason it wouldn't. Instead, that code path will fail at runtime when the server rejects the request because it's missing the required field. That really sucks.

Similarly on the output side, if we use replace to supply our own type, we don't find out at compile time when our type is incompatible. A particularly bad case would be an enum, where the server might add a new variant (which one might even consider a non-breaking change). But if the client sees it, it will fail to deserialize the response. By contrast, if we use the Progenitor-generated type, we'll fail to build if, say, we try to match on the enum and don't include the new variant.

In summary, using Progenitor types gets a lot of the benefit of Rust across the OpenAPI boundary. Using "replace" bypasses all of it (for that type). The downside of Progenitor types is that when there's non-trivial functionality on the original type, you either have to impl that functionality twice (sucks) or convert the type to your own type (okay for simple types in a few places, but kind of sucks in the large). For simple, stable types like Name I think the benefits of "replace" probably outweigh the risks. For complex types that are likely to change, I'm less sure (but still feel what I said earlier about this case).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think better approaches here might be to have Progenitor validate that your replace type really is compatible with the spec

This feels like it would be huge - it would allow use to use replace fearlessly, even on big / complex / volatile types like this one, I think?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think better approaches here might be to have Progenitor validate that your replace type really is compatible with the spec

This feels like it would be huge - it would allow use to use replace fearlessly, even on big / complex / volatile types like this one, I think?

Potentially, yeah. I spoke with Adam about this a bit. It's not clear how we could do this at compile time, but I imagine we might be able to do it at test-time.

Generation = omicron_common::api::external::Generation,
Ipv4Network = ipnetwork::Ipv4Network,
Ipv6Network = ipnetwork::Ipv6Network,
IpNetwork = ipnetwork::IpNetwork,
Expand Down Expand Up @@ -91,7 +96,7 @@ impl From<omicron_common::api::internal::nexus::InstanceRuntimeState>
) -> Self {
Self {
dst_propolis_id: s.dst_propolis_id,
gen: s.gen.into(),
gen: s.gen,
migration_id: s.migration_id,
propolis_id: s.propolis_id,
time_updated: s.time_updated,
Expand All @@ -103,11 +108,7 @@ impl From<omicron_common::api::internal::nexus::VmmRuntimeState>
for types::VmmRuntimeState
{
fn from(s: omicron_common::api::internal::nexus::VmmRuntimeState) -> Self {
Self {
gen: s.gen.into(),
state: s.state.into(),
time_updated: s.time_updated,
}
Self { gen: s.gen, state: s.state.into(), time_updated: s.time_updated }
}
}

Expand Down Expand Up @@ -145,19 +146,13 @@ impl From<omicron_common::api::external::InstanceState>
}
}

impl From<omicron_common::api::external::Generation> for types::Generation {
fn from(s: omicron_common::api::external::Generation) -> Self {
Self(i64::from(&s) as u64)
}
}

impl From<omicron_common::api::internal::nexus::DiskRuntimeState>
for types::DiskRuntimeState
{
fn from(s: omicron_common::api::internal::nexus::DiskRuntimeState) -> Self {
Self {
disk_state: s.disk_state.into(),
gen: s.gen.into(),
gen: s.gen,
time_updated: s.time_updated,
}
}
Expand Down
60 changes: 24 additions & 36 deletions clients/sled-agent-client/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,15 @@ progenitor::generate_api!(
replace = {
ByteCount = omicron_common::api::external::ByteCount,
Generation = omicron_common::api::external::Generation,
MacAddr = omicron_common::api::external::MacAddr,
Name = omicron_common::api::external::Name,
SwitchLocation = omicron_common::api::external::SwitchLocation,
Ipv6Network = ipnetwork::Ipv6Network,
IpNetwork = ipnetwork::IpNetwork,
PortFec = omicron_common::api::internal::shared::PortFec,
PortSpeed = omicron_common::api::internal::shared::PortSpeed,
SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig,
Vni = omicron_common::api::external::Vni,
}
);

Expand Down Expand Up @@ -65,6 +68,24 @@ impl types::OmicronZoneType {
types::OmicronZoneType::Oximeter { .. } => "oximeter",
}
}

/// Identifies whether this is an NTP zone
pub fn is_ntp(&self) -> bool {
match self {
types::OmicronZoneType::BoundaryNtp { .. }
| types::OmicronZoneType::InternalNtp { .. } => true,

types::OmicronZoneType::Clickhouse { .. }
| types::OmicronZoneType::ClickhouseKeeper { .. }
| types::OmicronZoneType::CockroachDb { .. }
| types::OmicronZoneType::Crucible { .. }
| types::OmicronZoneType::CruciblePantry { .. }
| types::OmicronZoneType::ExternalDns { .. }
| types::OmicronZoneType::InternalDns { .. }
| types::OmicronZoneType::Nexus { .. }
| types::OmicronZoneType::Oximeter { .. } => false,
}
}
}

impl omicron_common::api::external::ClientError for types::Error {
Expand Down Expand Up @@ -243,31 +264,6 @@ impl From<types::DiskState> for omicron_common::api::external::DiskState {
}
}

impl From<omicron_common::api::external::Vni> for types::Vni {
fn from(v: omicron_common::api::external::Vni) -> Self {
Self(u32::from(v))
}
}

impl From<types::Vni> for omicron_common::api::external::Vni {
fn from(s: types::Vni) -> Self {
Self::try_from(s.0).unwrap()
}
}

impl From<omicron_common::api::external::MacAddr> for types::MacAddr {
fn from(s: omicron_common::api::external::MacAddr) -> Self {
Self::try_from(s.0.to_string())
.unwrap_or_else(|e| panic!("{}: {}", s.0, e))
}
}

impl From<types::MacAddr> for omicron_common::api::external::MacAddr {
fn from(s: types::MacAddr) -> Self {
s.parse().unwrap()
}
}

impl From<omicron_common::api::external::Ipv4Net> for types::Ipv4Net {
fn from(n: omicron_common::api::external::Ipv4Net) -> Self {
Self::try_from(n.to_string()).unwrap_or_else(|e| panic!("{}: {}", n, e))
Expand Down Expand Up @@ -424,7 +420,7 @@ impl From<omicron_common::api::internal::nexus::HostIdentifier>
use omicron_common::api::internal::nexus::HostIdentifier::*;
match s {
Ip(net) => Self::Ip(net.into()),
Vpc(vni) => Self::Vpc(vni.into()),
Vpc(vni) => Self::Vpc(vni),
}
}
}
Expand Down Expand Up @@ -505,23 +501,15 @@ impl From<omicron_common::api::internal::shared::NetworkInterface>
kind: s.kind.into(),
name: s.name,
ip: s.ip,
mac: s.mac.into(),
mac: s.mac,
subnet: s.subnet.into(),
vni: s.vni.into(),
vni: s.vni,
primary: s.primary,
slot: s.slot,
}
}
}

impl From<omicron_common::api::internal::shared::SourceNatConfig>
for types::SourceNatConfig
{
fn from(s: omicron_common::api::internal::shared::SourceNatConfig) -> Self {
Self { ip: s.ip, first_port: s.first_port, last_port: s.last_port }
}
}

/// Exposes additional [`Client`] interfaces for use by the test suite. These
/// are bonus endpoints, not generated in the real client.
#[async_trait]
Expand Down
22 changes: 22 additions & 0 deletions common/src/address.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,15 @@ pub const RSS_RESERVED_ADDRESSES: u16 = 32;
// The maximum number of addresses per sled reserved for control plane services.
pub const CP_SERVICES_RESERVED_ADDRESSES: u16 = 0xFFFF;

// Number of addresses reserved (by the Nexus deployment planner) for allocation
// by the sled itself. This is currently used for the first two addresses of
// the sled subnet, which are used for the sled global zone and the switch zone,
// if any. Note that RSS does not honor this yet (in fact, per the above
// RSS_RESERVED_ADDRESSES, it will _only_ choose from this range). And
// historically, systems did not have this reservation at all. So it's not safe
// to assume that addresses in this subnet are available.
pub const SLED_RESERVED_ADDRESSES: u16 = 32;

/// Wraps an [`Ipv6Network`] with a compile-time prefix length.
#[derive(Debug, Clone, Copy, JsonSchema, Serialize, Hash, PartialEq, Eq)]
#[schemars(rename = "Ipv6Subnet")]
Expand Down Expand Up @@ -279,6 +288,19 @@ impl ReservedRackSubnet {
}
}

/// Return the list of DNS servers for the rack, given any address in the AZ
/// subnet
pub fn get_internal_dns_server_addresses(addr: Ipv6Addr) -> Vec<IpAddr> {
let az_subnet = Ipv6Subnet::<AZ_PREFIX>::new(addr);
let reserved_rack_subnet = ReservedRackSubnet::new(az_subnet);
let dns_subnets =
&reserved_rack_subnet.get_dns_subnets()[0..DNS_REDUNDANCY];
dns_subnets
.iter()
.map(|dns_subnet| IpAddr::from(dns_subnet.dns_address().ip()))
.collect()
}

const SLED_AGENT_ADDRESS_INDEX: usize = 1;
const SWITCH_ZONE_ADDRESS_INDEX: usize = 2;

Expand Down
1 change: 1 addition & 0 deletions common/src/api/external/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,7 @@ pub enum ResourceType {
BackgroundTask,
BgpConfig,
BgpAnnounceSet,
Blueprint,
Fleet,
Silo,
SiloUser,
Expand Down
1 change: 1 addition & 0 deletions dev-tools/omdb/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ serde.workspace = true
serde_json.workspace = true
sled-agent-client.workspace = true
slog.workspace = true
slog-error-chain.workspace = true
strum.workspace = true
tabled.workspace = true
textwrap.workspace = true
Expand Down
Loading