From 3be20009088f7d376a5a3bb8ed242175adcdb801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C5=A1a=20Tomi=C4=87?= Date: Fri, 9 Feb 2024 10:00:21 +0100 Subject: [PATCH] feat(registry-dump): Interpret vec data in registry structs as corresponding types (#175) * feat(registry-dump): Interpret vec data in registry structs as corresponding types - Instead of expecting from the user to interpret Vec as principals (somehow), we can do this in the code directly, and show principals as principals - Make the dump more in line with what ic-admin shows (in ic-admin get-topology) - Show the node reward table as well - Use native JSON serialize for the dump, instead of manually crafting the wrapper struct - Use the same dir that the registry sync uses, to avoid the slow re-downloading the entire mainnet registry on every execution * clippy and some missing changes --- Cargo.Bazel.lock | 393 +----------------- Cargo.lock | 79 +--- rs/cli/Cargo.toml | 14 +- rs/cli/src/main.rs | 2 +- rs/cli/src/registry_dump.rs | 279 +++++++++++-- rs/decentralization/src/nakamoto/mod.rs | 1 + .../multiservice-discovery/src/definition.rs | 34 +- 7 files changed, 293 insertions(+), 509 deletions(-) diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index f61606d8..b82b6958 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "2bda21c3422f1e2a634bfe6e3d48e0041bd5dbb73f9d63b191e5393c2e338c19", + "checksum": "9911caa648b8619f3b434bb29fac026c8711419ff79e05c956d0ed4cb5e151a4", "crates": { "actix-codec 0.5.2": { "name": "actix-codec", @@ -11548,6 +11548,10 @@ "id": "ic-interfaces-registry 0.9.0", "target": "ic_interfaces_registry" }, + { + "id": "ic-nns-common 0.9.0", + "target": "ic_nns_common" + }, { "id": "ic-nns-constants 0.9.0", "target": "ic_nns_constants" @@ -11560,10 +11564,18 @@ "id": "ic-protobuf 0.9.0", "target": "ic_protobuf" }, + { + "id": "ic-registry-keys 0.9.0", + "target": "ic_registry_keys" + }, { "id": "ic-registry-local-registry 0.9.0", "target": "ic_registry_local_registry" }, + { + "id": "ic-registry-subnet-type 0.9.0", + "target": "ic_registry_subnet_type" + }, { "id": "ic-sys 0.9.0", "target": "ic_sys" @@ -11584,10 +11596,18 @@ "id": "pretty_env_logger 0.5.0", "target": "pretty_env_logger" }, + { + "id": "prost 0.12.3", + "target": "prost" + }, { "id": "regex 1.10.3", "target": "regex" }, + { + "id": "registry-canister 0.9.0", + "target": "registry_canister" + }, { "id": "reqwest 0.11.24", "target": "reqwest" @@ -11604,10 +11624,6 @@ "id": "sha2 0.10.8", "target": "sha2" }, - { - "id": "shellexpand 3.1.0", - "target": "shellexpand" - }, { "id": "socket2 0.5.5", "target": "socket2" @@ -11628,10 +11644,6 @@ "id": "tabular 0.2.0", "target": "tabular" }, - { - "id": "tempdir 0.3.7", - "target": "tempdir" - }, { "id": "tokio 1.36.0", "target": "tokio" @@ -11639,10 +11651,6 @@ { "id": "url 2.5.0", "target": "url" - }, - { - "id": "uuid 1.7.0", - "target": "uuid" } ], "selects": {} @@ -13386,36 +13394,6 @@ }, "license": "MIT/Apache-2.0" }, - "fuchsia-cprng 0.1.1": { - "name": "fuchsia-cprng", - "version": "0.1.1", - "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/fuchsia-cprng/0.1.1/download", - "sha256": "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - } - }, - "targets": [ - { - "Library": { - "crate_name": "fuchsia_cprng", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "fuchsia_cprng", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "edition": "2018", - "version": "0.1.1" - }, - "license": null - }, "funty 2.0.0": { "name": "funty", "version": "2.0.0", @@ -28157,29 +28135,23 @@ ], "crate_features": { "common": [ + "elf", + "errno", "general", "ioctl", "no_std" ], "selects": { "aarch64-unknown-linux-gnu": [ - "elf", - "errno", "std" ], "arm-unknown-linux-gnueabi": [ - "elf", - "errno", "std" ], "armv7-unknown-linux-gnueabi": [ - "elf", - "errno", "std" ], "i686-unknown-linux-gnu": [ - "elf", - "errno", "std" ], "powerpc-unknown-linux-gnu": [ @@ -28189,8 +28161,6 @@ "std" ], "x86_64-unknown-linux-gnu": [ - "elf", - "errno", "std" ] } @@ -35321,77 +35291,6 @@ }, "license": "MIT" }, - "rand 0.4.6": { - "name": "rand", - "version": "0.4.6", - "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/rand/0.4.6/download", - "sha256": "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" - } - }, - "targets": [ - { - "Library": { - "crate_name": "rand", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "rand", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "crate_features": { - "common": [ - "default", - "libc", - "std" - ], - "selects": {} - }, - "deps": { - "common": [], - "selects": { - "cfg(target_env = \"sgx\")": [ - { - "id": "rand_core 0.3.1", - "target": "rand_core" - }, - { - "id": "rdrand 0.4.0", - "target": "rdrand" - } - ], - "cfg(target_os = \"fuchsia\")": [ - { - "id": "fuchsia-cprng 0.1.1", - "target": "fuchsia_cprng" - } - ], - "cfg(unix)": [ - { - "id": "libc 0.2.153", - "target": "libc" - } - ], - "cfg(windows)": [ - { - "id": "winapi 0.3.9", - "target": "winapi" - } - ] - } - }, - "edition": "2015", - "version": "0.4.6" - }, - "license": "MIT/Apache-2.0" - }, "rand 0.7.3": { "name": "rand", "version": "0.7.3", @@ -35628,75 +35527,6 @@ }, "license": "MIT OR Apache-2.0" }, - "rand_core 0.3.1": { - "name": "rand_core", - "version": "0.3.1", - "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/rand_core/0.3.1/download", - "sha256": "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" - } - }, - "targets": [ - { - "Library": { - "crate_name": "rand_core", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "rand_core", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "deps": { - "common": [ - { - "id": "rand_core 0.4.2", - "target": "rand_core" - } - ], - "selects": {} - }, - "edition": "2015", - "version": "0.3.1" - }, - "license": "MIT/Apache-2.0" - }, - "rand_core 0.4.2": { - "name": "rand_core", - "version": "0.4.2", - "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/rand_core/0.4.2/download", - "sha256": "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - } - }, - "targets": [ - { - "Library": { - "crate_name": "rand_core", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "rand_core", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "edition": "2015", - "version": "0.4.2" - }, - "license": "MIT/Apache-2.0" - }, "rand_core 0.5.1": { "name": "rand_core", "version": "0.5.1", @@ -35899,45 +35729,6 @@ }, "license": "MIT/Apache-2.0" }, - "rdrand 0.4.0": { - "name": "rdrand", - "version": "0.4.0", - "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/rdrand/0.4.0/download", - "sha256": "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" - } - }, - "targets": [ - { - "Library": { - "crate_name": "rdrand", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "rdrand", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "deps": { - "common": [ - { - "id": "rand_core 0.3.1", - "target": "rand_core" - } - ], - "selects": {} - }, - "edition": "2015", - "version": "0.4.0" - }, - "license": "ISC" - }, "redox_syscall 0.4.1": { "name": "redox_syscall", "version": "0.4.1", @@ -36561,47 +36352,6 @@ }, "license": null }, - "remove_dir_all 0.5.3": { - "name": "remove_dir_all", - "version": "0.5.3", - "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/remove_dir_all/0.5.3/download", - "sha256": "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" - } - }, - "targets": [ - { - "Library": { - "crate_name": "remove_dir_all", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "remove_dir_all", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "deps": { - "common": [], - "selects": { - "cfg(windows)": [ - { - "id": "winapi 0.3.9", - "target": "winapi" - } - ] - } - }, - "edition": "2015", - "version": "0.5.3" - }, - "license": "MIT/Apache-2.0" - }, "rend 0.4.2": { "name": "rend", "version": "0.4.2", @@ -39875,54 +39625,6 @@ }, "license": "MIT/Apache-2.0" }, - "shellexpand 3.1.0": { - "name": "shellexpand", - "version": "3.1.0", - "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/shellexpand/3.1.0/download", - "sha256": "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" - } - }, - "targets": [ - { - "Library": { - "crate_name": "shellexpand", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "shellexpand", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "crate_features": { - "common": [ - "base-0", - "default", - "dirs", - "tilde" - ], - "selects": {} - }, - "deps": { - "common": [ - { - "id": "dirs 5.0.1", - "target": "dirs" - } - ], - "selects": {} - }, - "edition": "2018", - "version": "3.1.0" - }, - "license": "MIT/Apache-2.0" - }, "signal-hook-registry 1.4.1": { "name": "signal-hook-registry", "version": "1.4.1", @@ -42213,49 +41915,6 @@ }, "license": "MIT/Apache-2.0" }, - "tempdir 0.3.7": { - "name": "tempdir", - "version": "0.3.7", - "repository": { - "Http": { - "url": "https://crates.io/api/v1/crates/tempdir/0.3.7/download", - "sha256": "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" - } - }, - "targets": [ - { - "Library": { - "crate_name": "tempdir", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "tempdir", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "deps": { - "common": [ - { - "id": "rand 0.4.6", - "target": "rand" - }, - { - "id": "remove_dir_all 0.5.3", - "target": "remove_dir_all" - } - ], - "selects": {} - }, - "edition": "2015", - "version": "0.3.7" - }, - "license": "MIT/Apache-2.0" - }, "tempfile 3.9.0": { "name": "tempfile", "version": "3.9.0", @@ -46186,7 +45845,6 @@ "objbase", "processenv", "processthreadsapi", - "profileapi", "shlobj", "std", "sysinfoapi", @@ -50440,7 +50098,6 @@ "wasm32-unknown-unknown", "wasm32-wasi" ], - "cfg(target_env = \"sgx\")": [], "cfg(target_family = \"unix\")": [ "aarch64-apple-darwin", "aarch64-apple-ios", @@ -50476,10 +50133,6 @@ "i686-unknown-freebsd", "x86_64-unknown-freebsd" ], - "cfg(target_os = \"fuchsia\")": [ - "aarch64-fuchsia", - "x86_64-fuchsia" - ], "cfg(target_os = \"haiku\")": [], "cfg(target_os = \"hermit\")": [], "cfg(target_os = \"ios\")": [ diff --git a/Cargo.lock b/Cargo.lock index 79b38f59..bdc50eb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2295,31 +2295,33 @@ dependencies = [ "ic-interfaces-registry", "ic-management-backend", "ic-management-types", + "ic-nns-common", "ic-nns-constants", "ic-nns-governance", "ic-protobuf", + "ic-registry-keys", "ic-registry-local-registry", + "ic-registry-subnet-type", "ic-sys", "itertools 0.12.1", "keyring", "log", "pretty_env_logger", + "prost", "regex", + "registry-canister", "reqwest", "serde", "serde_json", "sha2 0.10.8", - "shellexpand", "socket2 0.5.5", "spinners", "strum 0.25.0", "tabled", "tabular", - "tempdir", "tempfile", "tokio", "url", - "uuid", "wiremock", ] @@ -2661,12 +2663,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "funty" version = "2.0.0" @@ -7034,19 +7030,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" -[[package]] -name = "rand" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" -dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", -] - [[package]] name = "rand" version = "0.7.3" @@ -7091,21 +7074,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.5.1" @@ -7148,15 +7116,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "977b1e897f9d764566891689e642653e5ed90c6895106acd005eb4c1d0203991" -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -7270,15 +7229,6 @@ dependencies = [ "url", ] -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - [[package]] name = "rend" version = "0.4.2" @@ -7903,15 +7853,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" -[[package]] -name = "shellexpand" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" -dependencies = [ - "dirs", -] - [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -8361,16 +8302,6 @@ dependencies = [ "xattr", ] -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -dependencies = [ - "rand 0.4.6", - "remove_dir_all", -] - [[package]] name = "tempfile" version = "3.9.0" diff --git a/rs/cli/Cargo.toml b/rs/cli/Cargo.toml index e6ac745c..f0ec3631 100644 --- a/rs/cli/Cargo.toml +++ b/rs/cli/Cargo.toml @@ -27,16 +27,24 @@ futures = { workspace = true } ic-base-types = { workspace = true } ic-canister-client = { workspace = true } ic-canisters = { workspace = true } +ic-interfaces-registry = { workspace = true } ic-management-backend = { workspace = true } ic-management-types = { workspace = true } +ic-nns-common = { workspace = true } ic-nns-constants = { workspace = true } ic-nns-governance = { workspace = true } +ic-protobuf = { workspace = true } +ic-registry-keys = { workspace = true } +ic-registry-local-registry = { workspace = true } +ic-registry-subnet-type = { workspace = true } ic-sys = { workspace = true } itertools = { workspace = true } keyring = { workspace = true } log = { workspace = true } pretty_env_logger = { workspace = true } +prost = { workspace = true } regex = { workspace = true } +registry-canister = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -48,12 +56,6 @@ tabled = { workspace = true } tabular = { workspace = true } tokio = { workspace = true } url = { workspace = true } -tempdir = "0.3" -uuid = "1.7.0" -shellexpand = "3.1.0" -ic-registry-local-registry = { workspace = true } -ic-protobuf = { workspace = true } -ic-interfaces-registry = { workspace = true } [dev-dependencies] tempfile = "3.3.0" diff --git a/rs/cli/src/main.rs b/rs/cli/src/main.rs index 78737e03..095b10bf 100644 --- a/rs/cli/src/main.rs +++ b/rs/cli/src/main.rs @@ -286,7 +286,7 @@ async fn main() -> Result<(), anyhow::Error> { }, cli::Commands::DumpRegistry { version, path } => { - registry_dump::dump_registry(path, version, cli_opts.network).await + registry_dump::dump_registry(path, cli_opts.network, version).await } } }) diff --git a/rs/cli/src/registry_dump.rs b/rs/cli/src/registry_dump.rs index a110655a..00300bd5 100644 --- a/rs/cli/src/registry_dump.rs +++ b/rs/cli/src/registry_dump.rs @@ -1,32 +1,27 @@ -use std::{path::PathBuf, str::FromStr, time::Duration}; +use std::{collections::BTreeMap, path::PathBuf, str::FromStr, time::Duration}; use anyhow::Error; -use ic_base_types::RegistryVersion; +use ic_base_types::{PrincipalId, RegistryVersion}; use ic_interfaces_registry::RegistryClient; use ic_management_backend::registry::{local_registry_path, sync_local_store, RegistryFamilyEntries}; use ic_management_types::Network; use ic_protobuf::registry::{ - dc::v1::DataCenterRecord, node::v1::NodeRecord, node_operator::v1::NodeOperatorRecord, subnet::v1::SubnetRecord, + dc::v1::DataCenterRecord, + node::v1::{ConnectionEndpoint, IPv4InterfaceConfig, NodeRecord}, + node_operator::v1::NodeOperatorRecord, + subnet::v1::{EcdsaConfig, GossipConfig as GossipConfigProto, SubnetFeatures, SubnetRecord as SubnetRecordProto}, }; +use ic_registry_keys::NODE_REWARDS_TABLE_KEY; use ic_registry_local_registry::LocalRegistry; -use uuid::Uuid; - -pub async fn dump_registry(path: &Option, version: &i64, network: Network) -> Result<(), Error> { - let (path, should_dispose) = match path { - Some(p) => (p.clone(), false), - None => { - let uuid = Uuid::new_v4(); - let binding = format!("~/tmp/{}", uuid); - let local_temp = shellexpand::tilde(&binding); - ( - PathBuf::from_str(&local_temp).map_err(|e| anyhow::anyhow!("Couldn't create path: {:?}", e))?, - true, - ) - } - }; - - std::env::set_var("LOCAL_REGISTRY_PATH", path.clone()); +use ic_registry_subnet_type::SubnetType; +use itertools::Itertools; +use registry_canister::mutations::common::decode_registry_value; +use serde::Serialize; +pub async fn dump_registry(path: &Option, network: Network, version: &i64) -> Result<(), Error> { + if let Some(path) = path { + std::env::set_var("LOCAL_REGISTRY_PATH", path) + } sync_local_store(network.clone()).await?; let local_registry = LocalRegistry::new(local_registry_path(network), Duration::from_secs(10)) @@ -41,34 +36,238 @@ pub async fn dump_registry(path: &Option, version: &i64, network: Netwo } }; + let node_operators = get_node_operators(&local_registry, version)?; + + let dcs = get_data_centers(&local_registry, version)?; + + let subnets = get_subnets(&local_registry, version)?; + + let nodes = get_nodes(&local_registry, version, &node_operators, &subnets)?; + + let node_rewards_table = get_node_rewards_table(&local_registry, version); + + #[derive(Serialize)] + struct RegistryDump { + nodes: Vec, + subnets: Vec, + dcs: Vec, + node_operators: Vec, + node_rewards_table: NodeRewardsTableFlattened, + } + println!( + "{}", + serde_json::to_string(&RegistryDump { + nodes, + subnets, + dcs, + node_operators: node_operators.values().cloned().collect_vec(), + node_rewards_table + })? + ); + + Ok(()) +} + +fn get_nodes( + local_registry: &LocalRegistry, + version: RegistryVersion, + node_operators: &BTreeMap, + subnets: &[SubnetRecord], +) -> Result, Error> { let nodes = local_registry .get_family_entries_of_version::(version) - .map_err(|e| anyhow::anyhow!("Couldn't get data centers: {:?}", e))?; - let nodes = serde_json::to_string(&nodes).map_err(|e| anyhow::anyhow!("Couldn't convert to JSON: {:?}", e))?; + .map_err(|e| anyhow::anyhow!("Couldn't get nodes: {:?}", e))? + .into_iter() + .map(|(k, (_, record))| { + let node_operator_id = + PrincipalId::try_from(&record.node_operator_id).expect("Couldn't parse principal id"); + NodeDetails { + node_id: PrincipalId::from_str(&k).expect("Couldn't parse principal id"), + xnet: record.xnet, + http: record.http, + node_operator_id, + chip_id: record.chip_id, + hostos_version_id: record.hostos_version_id, + public_ipv4_config: record.public_ipv4_config, + node_provider_id: node_operators + .get(&node_operator_id) + .expect("Couldn't find node provider for node operator") + .node_provider_principal_id, + subnet_id: subnets + .iter() + .find(|subnet| subnet.membership.contains(&k)) + .map(|subnet| subnet.subnet_id), + dc_id: node_operators + .get(&node_operator_id) + .expect("Couldn't find node provider for node operator") + .dc_id + .clone(), + } + }) + .collect::>(); + Ok(nodes) +} - let subnets = local_registry - .get_family_entries_of_version::(version) - .map_err(|e| anyhow::anyhow!("Couldn't get data centers: {:?}", e))?; - let subnets = serde_json::to_string(&subnets).map_err(|e| anyhow::anyhow!("Couldn't convert to JSON: {:?}", e))?; +fn get_subnets(local_registry: &LocalRegistry, version: RegistryVersion) -> Result, Error> { + Ok(local_registry + .get_family_entries_of_version::(version) + .map_err(|e| anyhow::anyhow!("Couldn't get subnets: {:?}", e))? + .into_iter() + .map(|(subnet_id, (_, record))| SubnetRecord { + subnet_id: PrincipalId::from_str(&subnet_id).expect("Couldn't parse principal id"), + membership: record + .membership + .iter() + .map(|n| { + PrincipalId::try_from(&n[..]) + .expect("could not create PrincipalId from membership entry") + .to_string() + }) + .collect(), + nodes: Default::default(), + max_ingress_bytes_per_message: record.max_ingress_bytes_per_message, + max_ingress_messages_per_block: record.max_ingress_messages_per_block, + max_block_payload_size: record.max_block_payload_size, + unit_delay_millis: record.unit_delay_millis, + initial_notary_delay_millis: record.initial_notary_delay_millis, + replica_version_id: record.replica_version_id, + dkg_interval_length: record.dkg_interval_length, + gossip_config: record.gossip_config, + start_as_nns: record.start_as_nns, + subnet_type: SubnetType::try_from(record.subnet_type).unwrap(), + max_instructions_per_message: record.max_instructions_per_message, + max_instructions_per_round: record.max_instructions_per_round, + max_instructions_per_install_code: record.max_instructions_per_install_code, + features: record.features.clone().unwrap_or_default(), + max_number_of_canisters: record.max_number_of_canisters, + ssh_readonly_access: record.ssh_readonly_access, + ssh_backup_access: record.ssh_backup_access, + ecdsa_config: record.ecdsa_config, + }) + .collect::>()) +} - let dcs = local_registry +fn get_data_centers(local_registry: &LocalRegistry, version: RegistryVersion) -> Result, Error> { + Ok(local_registry .get_family_entries_of_version::(version) - .map_err(|e| anyhow::anyhow!("Couldn't get data centers: {:?}", e))?; - let dcs = serde_json::to_string(&dcs).map_err(|e| anyhow::anyhow!("Couldn't convert to JSON: {:?}", e))?; + .map_err(|e| anyhow::anyhow!("Couldn't get data centers: {:?}", e))? + .into_iter() + .map(|(_, (_, record))| record) + .collect()) +} +fn get_node_operators( + local_registry: &LocalRegistry, + version: RegistryVersion, +) -> Result, Error> { let node_operators = local_registry .get_family_entries_of_version::(version) - .map_err(|e| anyhow::anyhow!("Couldn't get data centers: {:?}", e))?; - let node_operators = - serde_json::to_string(&node_operators).map_err(|e| anyhow::anyhow!("Couldn't convert to JSON: {:?}", e))?; + .map_err(|e| anyhow::anyhow!("Couldn't get node operators: {:?}", e))? + .into_iter() + .map(|(k, (_, record))| { + let node_operator_principal_id = PrincipalId::from_str(&k).expect("Couldn't parse principal id"); + ( + node_operator_principal_id, + NodeOperator { + node_operator_principal_id, + node_allowance: record.node_allowance, + node_provider_principal_id: PrincipalId::try_from(record.node_provider_principal_id) + .expect("Couldn't parse principal id"), + dc_id: record.dc_id, + rewardable_nodes: record.rewardable_nodes, + ipv6: record.ipv6, + }, + ) + }) + .collect::>(); + Ok(node_operators) +} - println!( - "{{ \"nodes\": {}, \"subnets\": {}, \"dcs\": {}, \"node_operators\": {} }}", - nodes, subnets, dcs, node_operators - ); +fn get_node_rewards_table(local_registry: &LocalRegistry, version: RegistryVersion) -> NodeRewardsTableFlattened { + let rewards_table_bytes = local_registry + .get_value(NODE_REWARDS_TABLE_KEY, version) + .expect("Failed to get Node Rewards Table") + .expect("Failed to get Node Rewards Table"); - if should_dispose { - std::fs::remove_dir_all(path).map_err(|e| anyhow::anyhow!("Error removing created dir: {:?}", e))? - } - Ok(()) + decode_registry_value::(rewards_table_bytes) +} + +#[derive(Serialize, Clone)] +struct NodeDetails { + node_id: PrincipalId, + xnet: Option, + http: Option, + node_operator_id: PrincipalId, + chip_id: Option>, + hostos_version_id: Option, + public_ipv4_config: Option, + subnet_id: Option, + dc_id: String, + node_provider_id: PrincipalId, +} + +/// User-friendly representation of a SubnetRecord. For instance, +/// the `membership` field is a `Vec` to pretty-print the node IDs. +#[derive(Default, Serialize, Clone)] +struct SubnetRecord { + subnet_id: PrincipalId, + membership: Vec, + nodes: BTreeMap, + max_ingress_bytes_per_message: u64, + max_ingress_messages_per_block: u64, + max_block_payload_size: u64, + unit_delay_millis: u64, + initial_notary_delay_millis: u64, + replica_version_id: String, + dkg_interval_length: u64, + gossip_config: Option, + start_as_nns: bool, + subnet_type: SubnetType, + max_instructions_per_message: u64, + max_instructions_per_round: u64, + max_instructions_per_install_code: u64, + features: SubnetFeatures, + max_number_of_canisters: u64, + ssh_readonly_access: Vec, + ssh_backup_access: Vec, + ecdsa_config: Option, +} + +#[derive(Clone, Serialize)] +struct NodeOperator { + node_operator_principal_id: PrincipalId, + node_allowance: u64, + node_provider_principal_id: PrincipalId, + dc_id: String, + rewardable_nodes: std::collections::BTreeMap, + ipv6: Option, +} + +// We re-create the rewards structs here in order to convert the output of get-rewards-table into the format +// that can also be parsed by propose-to-update-node-rewards-table. +// This is a bit of a hack, but it's the easiest way to get the desired output. +// A more proper way would be to adjust the upstream structs to flatten the "rates" and "table" fields +// directly, but this breaks some of the candid encoding and decoding and also some of the tests. +// Make sure to keep these structs in sync with the upstream ones. +#[derive(serde::Serialize, PartialEq, ::prost::Message)] +pub struct NodeRewardRateFlattened { + #[prost(uint64, tag = "1")] + pub xdr_permyriad_per_node_per_month: u64, + #[prost(int32, optional, tag = "2")] + #[serde(skip_serializing_if = "Option::is_none")] + pub reward_coefficient_percent: Option, +} + +#[derive(serde::Serialize, PartialEq, ::prost::Message)] +pub struct NodeRewardRatesFlattened { + #[prost(btree_map = "string, message", tag = "1")] + #[serde(flatten)] + pub rates: BTreeMap, +} + +#[derive(serde::Serialize, PartialEq, ::prost::Message)] +pub struct NodeRewardsTableFlattened { + #[prost(btree_map = "string, message", tag = "1")] + #[serde(flatten)] + pub table: BTreeMap, } diff --git a/rs/decentralization/src/nakamoto/mod.rs b/rs/decentralization/src/nakamoto/mod.rs index fcd93c5c..a5180bef 100644 --- a/rs/decentralization/src/nakamoto/mod.rs +++ b/rs/decentralization/src/nakamoto/mod.rs @@ -338,6 +338,7 @@ impl Ord for NakamotoScore { impl PartialOrd for NakamotoScore { /// By default, the higher value will take the precedence + #[allow(clippy::non_canonical_partial_ord_impl)] fn partial_cmp(&self, other: &Self) -> Option { // Prefer higher score across all features let mut cmp = self.score_min().partial_cmp(&other.score_min()); diff --git a/rs/ic-observability/multiservice-discovery/src/definition.rs b/rs/ic-observability/multiservice-discovery/src/definition.rs index 23deb928..70e390dd 100644 --- a/rs/ic-observability/multiservice-discovery/src/definition.rs +++ b/rs/ic-observability/multiservice-discovery/src/definition.rs @@ -45,13 +45,13 @@ pub struct FSDefinition { impl From for FSDefinition { fn from(definition: Definition) -> Self { Self { - nns_urls: definition.nns_urls, + nns_urls: definition.nns_urls, registry_path: definition.registry_path, name: definition.name, public_key: definition.public_key, poll_interval: definition.poll_interval, registry_query_timeout: definition.registry_query_timeout, - boundary_nodes: definition.boundary_nodes + boundary_nodes: definition.boundary_nodes, } } } @@ -72,13 +72,13 @@ pub struct Definition { impl From for Definition { fn from(fs_definition: FSDefinition) -> Self { Definition::new( - fs_definition.nns_urls, - fs_definition.registry_path, - fs_definition.name, - make_logger(), - fs_definition.public_key, - fs_definition.poll_interval, - fs_definition.registry_query_timeout + fs_definition.nns_urls, + fs_definition.registry_path, + fs_definition.name, + make_logger(), + fs_definition.public_key, + fs_definition.poll_interval, + fs_definition.registry_query_timeout, ) } } @@ -458,17 +458,15 @@ impl DefinitionsSupervisor { std::fs::OpenOptions::new() .create(true) .write(true) - .open(&networks_state_file.as_path()) + .open(networks_state_file.as_path()) .and_then(|mut file| { let fs_def: Vec = existing - .values() - .cloned() - .into_iter() - .map(|running_def| running_def.definition.into()) - .collect::>(); - - file.write_all(serde_json::to_string(&fs_def)?.as_bytes()) - .map(|_| file) + .values() + .cloned() + .map(|running_def| running_def.definition.into()) + .collect::>(); + + file.write_all(serde_json::to_string(&fs_def)?.as_bytes()).map(|_| file) }) .and_then(|mut file| file.flush()) })?;