From 6e6a3772f766bf0c52e01151fd47fed733f27fff Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic <73236646+NikolaMilosa@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:58:18 +0100 Subject: [PATCH] feat(rollout-controller): Implementing action taking and refactoring tests (#249) * transformed cli to be a lib aswell * added deps to rollout controller * started work on action taking * finishing action taking * adding checking of blessed version * reducing test verbosity * moving feat build tests in the same module * add a partially executed step test * renaming * renaming * remove macro calls * adding better calculation of start of release --- Cargo.Bazel.lock | 13 +- Cargo.lock | 1 + rs/cli/BUILD.bazel | 18 +- rs/cli/Cargo.toml | 8 + rs/cli/src/cli.rs | 42 +- rs/cli/src/general.rs | 4 +- rs/cli/src/ic_admin.rs | 17 +- rs/cli/src/lib.rs | 25 + rs/cli/src/main.rs | 31 +- rs/cli/src/runner.rs | 4 +- rs/ic-management-types/src/lib.rs | 6 + rs/rollout-controller/BUILD.bazel | 3 +- rs/rollout-controller/Cargo.toml | 1 + rs/rollout-controller/src/actions/mod.rs | 180 +++ rs/rollout-controller/src/calculation/mod.rs | 62 +- .../src/calculation/stage_checks.rs | 1427 ++++------------- rs/rollout-controller/src/main.rs | 48 +- 17 files changed, 709 insertions(+), 1181 deletions(-) create mode 100644 rs/cli/src/lib.rs create mode 100644 rs/rollout-controller/src/actions/mod.rs diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index a7f9b6cc..969dbe3b 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "683ef224e0a517f71f7b62868192a1d7a020d93d122f65348365ef7029c150ee", + "checksum": "3e505218ac8b9878195c82edc086ac1c9c01901eaf619e2cc955439273525d79", "crates": { "actix-codec 0.5.2": { "name": "actix-codec", @@ -11519,6 +11519,15 @@ "version": "0.3.0", "repository": null, "targets": [ + { + "Library": { + "crate_name": "dre", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, { "BuildScript": { "crate_name": "build_script_build", @@ -11529,7 +11538,7 @@ } } ], - "library_target_name": null, + "library_target_name": "dre", "common_attrs": { "compile_data_glob": [ "**" diff --git a/Cargo.lock b/Cargo.lock index 498d358b..758c380b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7414,6 +7414,7 @@ dependencies = [ "clap 4.4.18", "crossbeam", "crossbeam-channel", + "dre", "humantime", "humantime-serde", "ic-base-types", diff --git a/rs/cli/BUILD.bazel b/rs/cli/BUILD.bazel index 91019a95..62395cfd 100644 --- a/rs/cli/BUILD.bazel +++ b/rs/cli/BUILD.bazel @@ -1,5 +1,5 @@ load("@crate_index_dre//:defs.bzl", "aliases", "all_crate_deps") -load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test") +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test", "rust_library") DEPS = [ "//rs/ic-canisters", @@ -8,6 +8,8 @@ DEPS = [ "//rs/ic-management-backend:ic-management-backend-lib", ] +package(default_visibility = ["//visibility:public"]) + rust_binary( name = "dre", srcs = glob(["src/**/*.rs"]), @@ -16,6 +18,20 @@ rust_binary( proc_macro_deps = all_crate_deps( proc_macro = True, ), + deps = all_crate_deps( + normal = True, + ) + DEPS + ["//rs/cli:dre-lib"], +) + +rust_library( + name = "dre-lib", + srcs = glob(["src/**/*.rs"]), + aliases = aliases(), + compile_data = glob(["config/**/*"]), + crate_name = "dre", + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), deps = all_crate_deps( normal = True, ) + DEPS, diff --git a/rs/cli/Cargo.toml b/rs/cli/Cargo.toml index 1eefa6b8..6cfac3ec 100644 --- a/rs/cli/Cargo.toml +++ b/rs/cli/Cargo.toml @@ -62,3 +62,11 @@ tempfile = "3.10.0" [dev-dependencies] wiremock = "0.6.0" + +[[bin]] +name = "dre" +path = "src/main.rs" + +[lib] +name = "dre" +path = "src/lib.rs" diff --git a/rs/cli/src/cli.rs b/rs/cli/src/cli.rs index 11b9dcc9..2ac0b987 100644 --- a/rs/cli/src/cli.rs +++ b/rs/cli/src/cli.rs @@ -9,45 +9,45 @@ use log::error; use crate::detect_neuron::{detect_hsm_auth, detect_neuron, Auth, Neuron}; // For more info about the version setup, look at https://docs.rs/clap/latest/clap/struct.Command.html#method.version -#[derive(Parser, Clone)] +#[derive(Parser, Clone, Default)] #[clap(about, version = env!("CARGO_PKG_VERSION"), author)] pub struct Opts { #[clap(long, env = "HSM_PIN", global = true)] - pub(crate) hsm_pin: Option, + pub hsm_pin: Option, #[clap(long, value_parser=maybe_hex::, env = "HSM_SLOT", global = true)] - pub(crate) hsm_slot: Option, + pub hsm_slot: Option, #[clap(long, env = "HSM_KEY_ID", global = true)] - pub(crate) hsm_key_id: Option, + pub hsm_key_id: Option, #[clap(long, env = "PRIVATE_KEY_PEM", global = true)] - pub(crate) private_key_pem: Option, + pub private_key_pem: Option, #[clap(long, env = "NEURON_ID", global = true)] - pub(crate) neuron_id: Option, + pub neuron_id: Option, #[clap(long, env = "IC_ADMIN", global = true)] - pub(crate) ic_admin: Option, + pub ic_admin: Option, #[clap(long, env = "DEV", global = true)] - pub(crate) dev: bool, + pub dev: bool, // Skip the confirmation prompt #[clap(short, long, env = "YES", global = true, conflicts_with = "simulate")] - pub(crate) yes: bool, + pub yes: bool, // Simulate submission of the proposal, but do not actually submit it. #[clap(long, aliases = ["dry-run", "dryrun", "no"], global = true, conflicts_with = "yes")] - pub(crate) simulate: bool, + pub simulate: bool, #[clap(long, env = "VERBOSE", global = true)] - pub(crate) verbose: bool, + pub verbose: bool, // Specify the target network: "mainnet" (default), "staging", or NNS URL #[clap(long, env = "NETWORK", default_value = "mainnet")] - pub(crate) network: Network, + pub network: Network, #[clap(subcommand)] - pub(crate) subcommand: Commands, + pub subcommand: Commands, } #[derive(Subcommand, Clone)] -pub(crate) enum Commands { +pub enum Commands { // Convert a DER file to a Principal DerToPrincipal { /// Path to the DER file @@ -155,7 +155,13 @@ pub(crate) enum Commands { }, } -pub(crate) mod subnet { +impl Default for Commands { + fn default() -> Self { + Commands::Get { args: vec![] } + } +} + +pub mod subnet { use super::*; use ic_base_types::PrincipalId; @@ -269,7 +275,7 @@ pub(crate) mod subnet { } } -pub(crate) mod version { +pub mod version { use super::*; #[derive(Subcommand, Clone)] @@ -322,7 +328,7 @@ pub(crate) mod version { } } -pub(crate) mod hostos { +pub mod hostos { use crate::operations::hostos_rollout::{NodeAssignment, NodeOwner}; use super::*; @@ -367,7 +373,7 @@ pub(crate) mod hostos { } } -pub(crate) mod nodes { +pub mod nodes { use super::*; #[derive(Parser, Clone)] diff --git a/rs/cli/src/general.rs b/rs/cli/src/general.rs index 491cc5c7..ba7ea5b4 100644 --- a/rs/cli/src/general.rs +++ b/rs/cli/src/general.rs @@ -17,7 +17,7 @@ use url::Url; use crate::detect_neuron::{Auth, Neuron}; -pub(crate) async fn vote_on_proposals( +pub async fn vote_on_proposals( neuron: &Neuron, nns_url: &Url, accepted_proposers: &[u64], @@ -89,7 +89,7 @@ pub(crate) async fn vote_on_proposals( Ok(()) } -pub(crate) async fn get_node_metrics_history( +pub async fn get_node_metrics_history( wallet: CanisterId, subnets: Vec, start_at_nanos: u64, diff --git a/rs/cli/src/ic_admin.rs b/rs/cli/src/ic_admin.rs index 5e182c46..3d79695a 100644 --- a/rs/cli/src/ic_admin.rs +++ b/rs/cli/src/ic_admin.rs @@ -190,12 +190,7 @@ impl IcAdminWrapper { ); } - pub(crate) fn propose_run( - &self, - cmd: ProposeCommand, - opts: ProposeOptions, - simulate: bool, - ) -> anyhow::Result { + pub fn propose_run(&self, cmd: ProposeCommand, opts: ProposeOptions, simulate: bool) -> anyhow::Result { let exec = |cli: &IcAdminWrapper, cmd: ProposeCommand, opts: ProposeOptions, add_dryrun_arg: bool| { if let Some(summary) = opts.clone().summary { let summary_count = summary.chars().count(); @@ -311,7 +306,7 @@ impl IcAdminWrapper { } } - pub(crate) fn run(&self, command: &str, args: &[String], with_auth: bool) -> anyhow::Result { + pub fn run(&self, command: &str, args: &[String], with_auth: bool) -> anyhow::Result { let ic_admin_args = [&[command.to_string()], args].concat(); self._run_ic_admin_with_args(&ic_admin_args, with_auth) } @@ -346,7 +341,7 @@ impl IcAdminWrapper { } /// Run an `ic-admin get-*` command directly, and without an HSM - pub(crate) fn run_passthrough_get(&self, args: &[String]) -> anyhow::Result<()> { + pub fn run_passthrough_get(&self, args: &[String]) -> anyhow::Result<()> { if args.is_empty() { println!("List of available ic-admin 'get' sub-commands:\n"); for subcmd in self.grep_subcommands(r"\s+get-(.+?)\s") { @@ -378,7 +373,7 @@ impl IcAdminWrapper { } /// Run an `ic-admin propose-to-*` command directly - pub(crate) fn run_passthrough_propose(&self, args: &[String], simulate: bool) -> anyhow::Result<()> { + pub fn run_passthrough_propose(&self, args: &[String], simulate: bool) -> anyhow::Result<()> { if args.is_empty() { println!("List of available ic-admin 'propose' sub-commands:\n"); for subcmd in self.grep_subcommands(r"\s+propose-to-(.+?)\s") { @@ -572,7 +567,7 @@ impl IcAdminWrapper { Ok((update_urls, expected_hash)) } - pub(crate) async fn prepare_to_propose_to_update_elected_versions( + pub async fn prepare_to_propose_to_update_elected_versions( release_artifact: &Artifact, version: &String, release_tag: &String, @@ -910,7 +905,7 @@ must be identical, and must match the SHA256 from the payload of the NNS proposa #[derive(Display, Clone)] #[strum(serialize_all = "kebab-case")] -pub(crate) enum ProposeCommand { +pub enum ProposeCommand { ChangeSubnetMembership { subnet_id: PrincipalId, node_ids_add: Vec, diff --git a/rs/cli/src/lib.rs b/rs/cli/src/lib.rs new file mode 100644 index 00000000..0195888b --- /dev/null +++ b/rs/cli/src/lib.rs @@ -0,0 +1,25 @@ +pub mod cli; +pub mod clients; +pub(crate) mod defaults; +pub mod detect_neuron; +pub mod general; +pub mod ic_admin; +pub mod operations; +pub mod ops_subnet_node_replace; +pub mod registry_dump; +pub mod runner; + +/// Get a localhost socket address with random, unused port. +pub fn local_unused_port() -> u16 { + let addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); + let socket = socket2::Socket::new( + socket2::Domain::IPV4, + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + ) + .unwrap(); + socket.bind(&addr.into()).unwrap(); + socket.set_reuse_address(true).unwrap(); + let tcp = std::net::TcpListener::from(socket); + tcp.local_addr().unwrap().port() +} diff --git a/rs/cli/src/main.rs b/rs/cli/src/main.rs index d35119df..09cd0cee 100644 --- a/rs/cli/src/main.rs +++ b/rs/cli/src/main.rs @@ -1,8 +1,9 @@ -use crate::general::{get_node_metrics_history, vote_on_proposals}; use crate::ic_admin::IcAdminWrapper; -use crate::operations::hostos_rollout::{NodeGroupUpdate, NumberOfNodes}; use clap::{error::ErrorKind, CommandFactory, Parser}; use dotenv::dotenv; +use dre::general::{get_node_metrics_history, vote_on_proposals}; +use dre::operations::hostos_rollout::{NodeGroupUpdate, NumberOfNodes}; +use dre::{cli, ic_admin, local_unused_port, registry_dump, runner}; use ic_base_types::CanisterId; use ic_canisters::governance::governance_canister_version; use ic_management_backend::endpoints; @@ -14,17 +15,6 @@ use std::str::FromStr; use std::sync::mpsc; use std::thread; -mod cli; -mod clients; -pub(crate) mod defaults; -mod detect_neuron; -mod general; -mod ic_admin; -mod operations; -mod ops_subnet_node_replace; -mod registry_dump; -mod runner; - const STAGING_NEURON_ID: u64 = 49; #[tokio::main] @@ -398,21 +388,6 @@ fn parse_min_nakamoto_coefficients( }) } -/// Get a localhost socket address with random, unused port. -fn local_unused_port() -> u16 { - let addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); - let socket = socket2::Socket::new( - socket2::Domain::IPV4, - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - ) - .unwrap(); - socket.bind(&addr.into()).unwrap(); - socket.set_reuse_address(true).unwrap(); - let tcp = std::net::TcpListener::from(socket); - tcp.local_addr().unwrap().port() -} - fn init_logger() { match std::env::var("RUST_LOG") { Ok(val) => std::env::set_var("LOG_LEVEL", val), diff --git a/rs/cli/src/runner.rs b/rs/cli/src/runner.rs index ae8941d5..921486d0 100644 --- a/rs/cli/src/runner.rs +++ b/rs/cli/src/runner.rs @@ -210,7 +210,7 @@ impl Runner { }) } - pub(crate) async fn prepare_versions_to_retire( + pub async fn prepare_versions_to_retire( &self, release_artifact: &Artifact, edit_summary: bool, @@ -282,7 +282,7 @@ impl Runner { }) } - pub(crate) async fn hostos_rollout_nodes( + pub async fn hostos_rollout_nodes( &self, node_group: NodeGroupUpdate, version: &String, diff --git a/rs/ic-management-types/src/lib.rs b/rs/ic-management-types/src/lib.rs index 3d50257b..e594ba64 100644 --- a/rs/ic-management-types/src/lib.rs +++ b/rs/ic-management-types/src/lib.rs @@ -569,6 +569,12 @@ pub enum Network { Url(url::Url), } +impl Default for Network { + fn default() -> Self { + Network::Mainnet + } +} + impl Debug for Network { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self) diff --git a/rs/rollout-controller/BUILD.bazel b/rs/rollout-controller/BUILD.bazel index 0c0ecf6d..a93ee412 100644 --- a/rs/rollout-controller/BUILD.bazel +++ b/rs/rollout-controller/BUILD.bazel @@ -6,7 +6,8 @@ load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test") DEPS = [ "//rs/ic-observability/service-discovery", "//rs/ic-management-types", - "//rs/ic-management-backend:ic-management-backend-lib",] + "//rs/ic-management-backend:ic-management-backend-lib", + "//rs/cli:dre-lib"] rust_binary( diff --git a/rs/rollout-controller/Cargo.toml b/rs/rollout-controller/Cargo.toml index cfcd557c..29b3c962 100644 --- a/rs/rollout-controller/Cargo.toml +++ b/rs/rollout-controller/Cargo.toml @@ -38,6 +38,7 @@ candid = { workspace = true } ic-base-types = { workspace = true } pretty_assertions = { workspace = true } itertools = { workspace = true } +dre = { path = "../cli"} [dev-dependencies] rstest = "0.18.2" diff --git a/rs/rollout-controller/src/actions/mod.rs b/rs/rollout-controller/src/actions/mod.rs new file mode 100644 index 00000000..bdd66f3d --- /dev/null +++ b/rs/rollout-controller/src/actions/mod.rs @@ -0,0 +1,180 @@ +use std::time::Duration; + +use dre::{ + cli::Opts, + ic_admin::{IcAdminWrapper, ProposeCommand, ProposeOptions}, +}; +use ic_base_types::PrincipalId; +use ic_management_types::Network; +use slog::{info, Logger}; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum SubnetAction { + Noop { + subnet_short: String, + }, + Baking { + subnet_short: String, + remaining: Duration, + }, + PendingProposal { + subnet_short: String, + proposal_id: u64, + }, + PlaceProposal { + is_unassigned: bool, + subnet_principal: PrincipalId, + version: String, + }, + WaitForNextWeek { + subnet_short: String, + }, +} + +impl SubnetAction { + fn print(&self) -> String { + match self { + SubnetAction::Noop { subnet_short } => format!("Noop for subnet '{}'", subnet_short), + SubnetAction::Baking { + subnet_short, + remaining, + } => { + let humantime = humantime::format_duration(*remaining); + format!("Subnet '{}' is pending to bake for {}", subnet_short, humantime) + } + SubnetAction::PendingProposal { + subnet_short, + proposal_id, + } => format!( + "Subnet '{}' has a pending proposal with id '{}' that has to be voted on", + subnet_short, proposal_id + ), + SubnetAction::PlaceProposal { + is_unassigned, + subnet_principal, + version, + } => format!( + "Placing proposal for '{}' to upgrade to version '{}'", + match is_unassigned { + true => "unassigned nodes".to_string(), + false => subnet_principal.to_string(), + }, + version + ), + SubnetAction::WaitForNextWeek { subnet_short } => { + format!("Waiting for next week to place proposal for '{}'", subnet_short) + } + } + } +} + +impl<'a> SubnetAction { + fn execute(&self, executor: &'a ActionExecutor, blessed_replica_versions: &'a [String]) -> anyhow::Result<()> { + if let Some(logger) = executor.logger { + info!(logger, "Subnet action: {}", self.print()) + } + if let SubnetAction::PlaceProposal { + is_unassigned, + subnet_principal, + version, + } = self + { + if !blessed_replica_versions.contains(version) { + return Err(anyhow::anyhow!("Replica version '{}' is not blessed.", version)); + } + let principal_string = subnet_principal.to_string(); + + let proposal = match is_unassigned { + true => ProposeCommand::UpdateUnassignedNodes { + replica_version: version.to_string(), + }, + false => ProposeCommand::UpdateSubnetReplicaVersion { + subnet: *subnet_principal, + version: version.to_string(), + }, + }; + + let opts = ProposeOptions { + title: Some(format!( + "Update subnet {} to replica version {}", + principal_string.split_once('-').expect("Should contain '-'").0, + version.split_at(8).0 + )), + summary: Some(format!( + "Update subnet {} to replica version {}", + principal_string, version + )), + ..Default::default() + }; + + executor.ic_admin.propose_run(proposal, opts, executor.simulate)?; + } + + Ok(()) + } +} + +pub struct ActionExecutor<'a> { + ic_admin: IcAdminWrapper, + simulate: bool, + logger: Option<&'a Logger>, +} + +impl<'a> ActionExecutor<'a> { + pub async fn new( + neuron_id: u64, + private_key_pem: String, + network: Network, + simulate: bool, + logger: Option<&'a Logger>, + ) -> anyhow::Result { + Ok(Self { + ic_admin: dre::cli::Cli::from_opts( + &Opts { + neuron_id: Some(neuron_id), + private_key_pem: Some(private_key_pem), + yes: true, + network, + ..Default::default() + }, + true, + ) + .await? + .into(), + simulate, + logger, + }) + } + + pub async fn test(network: Network, logger: Option<&'a Logger>) -> anyhow::Result { + Ok(Self { + ic_admin: dre::cli::Cli::from_opts( + &Opts { + yes: true, + network, + ..Default::default() + }, + false, + ) + .await? + .into(), + simulate: true, + logger, + }) + } + + pub fn execute(&self, actions: &[SubnetAction], blessed_replica_versions: &[String]) -> anyhow::Result<()> { + if let Some(logger) = self.logger { + info!(logger, "Executing following actions: {:?}", actions) + } + + for (i, action) in actions.iter().enumerate() { + if let Some(logger) = self.logger { + info!(logger, "Executing action {}: {:?}", i, action) + } + action.execute(self, blessed_replica_versions)?; + } + + Ok(()) + } +} diff --git a/rs/rollout-controller/src/calculation/mod.rs b/rs/rollout-controller/src/calculation/mod.rs index 99dd6a2e..da0e613f 100644 --- a/rs/rollout-controller/src/calculation/mod.rs +++ b/rs/rollout-controller/src/calculation/mod.rs @@ -1,15 +1,16 @@ use std::{collections::BTreeMap, time::Duration}; use crate::calculation::should_proceed::should_proceed; -use chrono::{Local, NaiveDate, NaiveDateTime}; +use chrono::{Local, NaiveDate, TimeDelta}; use ic_management_backend::registry::RegistryState; use ic_management_types::Subnet; +use itertools::Itertools; use prometheus_http_query::Client; -use regex::Regex; use serde::Deserialize; use slog::{info, Logger}; -use self::stage_checks::{check_stages, SubnetAction}; +use self::stage_checks::{check_stages, desired_rollout_release_version}; +use crate::actions::SubnetAction; mod should_proceed; mod stage_checks; @@ -39,30 +40,13 @@ pub struct Stage { update_unassigned_nodes: bool, } -#[derive(Deserialize, Clone, Default, Eq, PartialEq, Hash)] +#[derive(Deserialize, Clone, Default, Eq, PartialEq, Hash, Debug)] pub struct Release { pub rc_name: String, pub versions: Vec, } -impl Release { - pub fn date(&self) -> NaiveDateTime { - let regex = Regex::new(r"rc--(?P\d{4}-\d{2}-\d{2}_\d{2}-\d{2})").unwrap(); - - NaiveDateTime::parse_from_str( - regex - .captures(&self.rc_name) - .expect("should have format with date") - .name("datetime") - .expect("should match group datetime") - .as_str(), - "%Y-%m-%d_%H-%M", - ) - .expect("should be valid date") - } -} - -#[derive(Deserialize, Clone, Default, Eq, PartialEq, Hash)] +#[derive(Deserialize, Clone, Default, Eq, PartialEq, Hash, Debug)] pub struct Version { pub version: String, pub name: String, @@ -108,6 +92,31 @@ pub async fn calculate_progress<'a>( last_bake_status.insert(subnet.to_string(), last_update); } + let subnets = registry_state.subnets().into_values().collect::>(); + let desired_versions = desired_rollout_release_version(&subnets, &index.releases); + let concatenated_versions = desired_versions + .release + .versions + .iter() + .map(|v| v.version.clone()) + .join("|"); + + let result = prometheus_client.query(format!(r#" + time() - first_over_time((timestamp(group(ic_replica_info{{ic_active_version=~"{concatenated_versions}"}})))[14d:1d]) + "#)).get().await?; + + let since_start = match result.data().clone().into_vector().into_iter().last() { + Some(data) => match data.iter().last() { + Some(data) => data.sample().value(), + None => { + return Err(anyhow::anyhow!( + "There should be data regarding start of releases in response vector" + )) + } + }, + None => return Err(anyhow::anyhow!("There should be data regarding start of releases")), + }; + let subnet_update_proposals = registry_state.open_subnet_upgrade_proposals().await?; let unassigned_nodes_version = registry_state.get_unassigned_nodes_replica_version().await?; let unassigned_nodes_proposals = registry_state.open_upgrade_unassigned_nodes_proposals().await?; @@ -119,8 +128,15 @@ pub async fn calculate_progress<'a>( index, Some(&logger), &unassigned_nodes_version, - ®istry_state.subnets().into_values().collect::>(), + &subnets, Local::now().date_naive(), + Local::now() + .checked_sub_signed( + TimeDelta::try_seconds(since_start as i64).expect("Should be able to convert to seconds"), + ) + .expect("Should be able to sub from now") + .date_naive(), + desired_versions, )?; Ok(actions) diff --git a/rs/rollout-controller/src/calculation/stage_checks.rs b/rs/rollout-controller/src/calculation/stage_checks.rs index 2a918f06..6f1818be 100644 --- a/rs/rollout-controller/src/calculation/stage_checks.rs +++ b/rs/rollout-controller/src/calculation/stage_checks.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, time::Duration}; +use crate::actions::SubnetAction; use chrono::{Datelike, Days, NaiveDate, Weekday}; use humantime::format_duration; use ic_base_types::PrincipalId; @@ -10,29 +11,9 @@ use slog::{debug, info, Logger}; use super::{Index, Stage}; -#[derive(Debug)] -pub enum SubnetAction { - Noop { - subnet_short: String, - }, - Baking { - subnet_short: String, - remaining: Duration, - }, - PendingProposal { - subnet_short: String, - proposal_id: u64, - }, - PlaceProposal { - is_unassigned: bool, - subnet_principal: String, - version: String, - }, - WaitForNextWeek { - subnet_short: String, - }, -} - +/// For the set of inputs, generate a vector of `SubnetAction`'s for an arbitrary stage. +/// All produced actions are always related to the same stage of an index rollout. +/// To find out more take a look at the e2e tests pub fn check_stages<'a>( last_bake_status: &'a BTreeMap, subnet_update_proposals: &'a [SubnetUpdateProposal], @@ -42,15 +23,15 @@ pub fn check_stages<'a>( unassigned_version: &'a String, subnets: &'a [Subnet], now: NaiveDate, + start_of_release: NaiveDate, + desired_versions: DesiredReleaseVersion, ) -> anyhow::Result> { - let desired_versions = desired_rollout_release_version(subnets.to_vec(), index.releases); for (i, stage) in index.rollout.stages.iter().enumerate() { if let Some(logger) = logger { info!(logger, "Checking stage {}", i) } - let start_of_release = desired_versions.release.date(); - if stage.wait_for_next_week && !week_passed(start_of_release.date(), now) { + if stage.wait_for_next_week && !week_passed(start_of_release, now) { let actions = stage .subnets .iter() @@ -142,11 +123,11 @@ fn check_stage<'a>( }) { None => stage_actions.push(SubnetAction::PlaceProposal { is_unassigned: true, - subnet_principal: "".to_string(), + subnet_principal: PrincipalId::new_anonymous(), version: desired_versions.unassigned_nodes.version, }), Some(proposal) => stage_actions.push(SubnetAction::PendingProposal { - subnet_short: "unassigned-version".to_string(), + subnet_short: PrincipalId::new_anonymous().to_string(), proposal_id: proposal.info.id, }), } @@ -231,7 +212,7 @@ fn check_stage<'a>( // If subnet is not on desired version and there is no open proposal submit it stage_actions.push(SubnetAction::PlaceProposal { is_unassigned: false, - subnet_principal: subnet.principal.to_string(), + subnet_principal: subnet.principal, version: desired_version.version.clone(), }) } @@ -239,16 +220,16 @@ fn check_stage<'a>( Ok(stage_actions) } -#[derive(Clone)] -struct DesiredReleaseVersion { - subnets: BTreeMap, - unassigned_nodes: crate::calculation::Version, - release: crate::calculation::Release, +#[derive(Clone, Debug)] +pub struct DesiredReleaseVersion { + pub subnets: BTreeMap, + pub unassigned_nodes: crate::calculation::Version, + pub release: crate::calculation::Release, } -fn desired_rollout_release_version( - subnets: Vec, - releases: Vec, +pub fn desired_rollout_release_version<'a>( + subnets: &'a [Subnet], + releases: &'a [crate::calculation::Release], ) -> DesiredReleaseVersion { let subnets_releases = subnets .iter() @@ -531,6 +512,35 @@ mod test { use super::*; + pub(super) fn subnet(id: u64, version: &str) -> Subnet { + Subnet { + principal: PrincipalId::new_subnet_test_id(id), + replica_version: version.to_string(), + metadata: SubnetMetadata { + name: format!("{id}"), + ..Default::default() + }, + ..Default::default() + } + } + + pub(super) fn release(name: &str, versions: Vec<(&str, Vec)>) -> Release { + Release { + rc_name: name.to_string(), + versions: versions + .iter() + .map(|(v, subnets)| Version { + version: v.to_string(), + subnets: subnets + .iter() + .map(|id| PrincipalId::new_subnet_test_id(*id).to_string()) + .collect(), + ..Default::default() + }) + .collect(), + } + } + #[test] fn desired_version_test_cases() { struct TestCase { @@ -540,35 +550,6 @@ mod test { want: BTreeMap, } - fn subnet(id: u64, version: &str) -> Subnet { - Subnet { - principal: PrincipalId::new_subnet_test_id(id), - replica_version: version.to_string(), - metadata: SubnetMetadata { - name: format!("{id}"), - ..Default::default() - }, - ..Default::default() - } - } - - fn release(name: &str, versions: Vec<(&str, Vec)>) -> Release { - Release { - rc_name: name.to_string(), - versions: versions - .iter() - .map(|(v, subnets)| Version { - version: v.to_string(), - subnets: subnets - .iter() - .map(|id| PrincipalId::new_subnet_test_id(*id).to_string()) - .collect(), - ..Default::default() - }) - .collect(), - } - } - for tc in vec![ TestCase { name: "all versions on the newest version already", @@ -632,7 +613,7 @@ mod test { .collect(), }, ] { - let desired_release = desired_rollout_release_version(tc.subnets, tc.releases); + let desired_release = desired_rollout_release_version(&tc.subnets, &tc.releases); assert_eq!( tc.want .into_iter() @@ -652,11 +633,10 @@ mod test { // E2E tests for decision making process for happy path without feature builds #[cfg(test)] -mod check_stages_tests_no_feature_builds { - use std::str::FromStr; +#[allow(dead_code)] +mod check_stages_tests { - use candid::Principal; - use check_stages_tests_no_feature_builds::get_open_proposal_for_subnet_tests::craft_executed_proposals; + use check_stages_tests::test::subnet; use ic_base_types::PrincipalId; use ic_management_backend::proposal::ProposalInfoInternal; use registry_canister::mutations::{ @@ -664,7 +644,9 @@ mod check_stages_tests_no_feature_builds { do_update_unassigned_nodes_config::UpdateUnassignedNodesConfigPayload, }; - use crate::calculation::{Index, Release, Rollout, Version}; + use crate::calculation::{Index, Rollout}; + + use self::test::release; use super::*; @@ -683,24 +665,24 @@ mod check_stages_tests_no_feature_builds { /// pause: false // Tested in `should_proceed.rs` module /// skip_days: [] // Tested in `should_proceed.rs` module /// stages: - /// - subnets: [io67a] + /// - subnets: [1] /// bake_time: 8h - /// - subnets: [shefu, uzr34] + /// - subnets: [2, 3] /// bake_time: 4h /// - update_unassigned_nodes: true - /// - subnets: [pjljw] + /// - subnets: [4] /// wait_for_next_week: true /// bake_time: 4h /// releases: /// - rc_name: rc--2024-02-21_23-01 /// versions: - /// - version: 2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f + /// - version: b /// name: rc--2024-02-21_23-01 /// release_notes_ready: /// subnets: [] // empty because its a regular build /// - rc_name: rc--2024-02-14_23-01 /// versions: - /// - version: 85bd56a70e55b2cea75cae6405ae11243e5fdad8 + /// - version: a /// name: rc--2024-02-14_23-01 /// release_notes_ready: /// subnets: [] // empty because its a regular build @@ -710,1075 +692,344 @@ mod check_stages_tests_no_feature_builds { pause: false, skip_days: vec![], stages: vec![ - Stage { - subnets: vec!["io67a".to_string()], - bake_time: humantime::parse_duration("8h").expect("Should be able to parse."), - ..Default::default() - }, - Stage { - subnets: vec!["shefu".to_string(), "uzr34".to_string()], - bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), - ..Default::default() - }, - Stage { - update_unassigned_nodes: true, - ..Default::default() - }, - Stage { - subnets: vec!["pjljw".to_string()], - bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), - wait_for_next_week: true, - ..Default::default() - }, + stage(&[1], "8h"), + stage(&[2, 3], "4h"), + stage_unassigned(), + stage_next_week(&[4], "4h"), ], }, releases: vec![ - Release { - rc_name: "rc--2024-02-21_23-01".to_string(), - versions: vec![Version { - name: "rc--2024-02-21_23-01".to_string(), - version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - ..Default::default() - }], - }, - Release { - rc_name: "rc--2024-02-14_23-01".to_string(), - versions: vec![Version { - name: "rc--2024-02-14_23-01".to_string(), - version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - ..Default::default() - }], - }, + release("rc--2024-02-21_23-01", vec![("b", vec![])]), + release("rc--2024-02-14_23-01", vec![("a", vec![])]), ], } } - pub(super) fn craft_subnets() -> Vec { - vec![ - Subnet { - principal: PrincipalId( - Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") - .expect("Should be able to create a principal"), - ), - replica_version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - ..Default::default() - }, - Subnet { - principal: PrincipalId( - Principal::from_str("shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe") - .expect("Should be able to create a principal"), - ), - replica_version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - ..Default::default() - }, - Subnet { - principal: PrincipalId( - Principal::from_str("uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe") - .expect("Should be able to create a principal"), - ), - replica_version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - ..Default::default() - }, - Subnet { - principal: PrincipalId( - Principal::from_str("pjljw-kztyl-46ud4-ofrj6-nzkhm-3n4nt-wi3jt-ypmav-ijqkt-gjf66-uae") - .expect("Should be able to create a principal"), - ), - replica_version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - ..Default::default() - }, - ] + fn stage(subnet_ids: &[u64], bake_time: &'static str) -> Stage { + Stage { + bake_time: humantime::parse_duration(bake_time).expect("Should be able to parse."), + subnets: subnet_ids.iter().map(|id| principal(*id).to_string()).collect_vec(), + ..Default::default() + } } - - pub(super) fn replace_versions(subnets: &mut Vec, tuples: &[(&str, &str)]) { - for (id, ver) in tuples { - if let Some(subnet) = subnets.iter_mut().find(|s| s.principal.to_string().contains(id)) { - subnet.replica_version = ver.to_string(); - } + fn stage_unassigned() -> Stage { + Stage { + update_unassigned_nodes: true, + ..Default::default() } } + fn stage_next_week(subnet_ids: &[u64], bake_time: &'static str) -> Stage { + let mut stage = stage(subnet_ids, bake_time); + stage.wait_for_next_week = true; + stage + } - /// Use-Case 1: Beginning of a new rollout - /// - /// `last_bake_status` - empty, because no subnets have the version - /// `subnet_update_proposals` - can be empty but doesn't have to be. For e.g. if its Monday it is possible to have an open proposal for NNS - /// But it is for a different version (one from last week) - /// `unassigned_nodes_proposals` - empty - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-21` - #[test] - fn test_use_case_1() { - let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); - let last_bake_status = BTreeMap::new(); - let subnet_update_proposals = Vec::new(); - let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); - let unassigned_nodes_proposals = vec![]; - let subnets = &craft_subnets(); - let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposals, - index, - None, - &unassigned_version, - subnets, - now, - ); - - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - - assert_eq!(actions.len(), 1); - for action in actions { - match action { - SubnetAction::PlaceProposal { - is_unassigned, - subnet_principal, - version, - } => { - assert_eq!(is_unassigned, false); - assert_eq!(version, current_version); - assert!(subnet_principal.starts_with("io67a")) - } - // Fail the test - _ => assert!(false), - } - } + pub(super) fn craft_subnets() -> Vec { + vec![subnet(1, "a"), subnet(2, "a"), subnet(3, "a"), subnet(4, "a")] } - /// Use case 2: First batch is submitted but the proposal wasn't executed - /// - /// `last_bake_status` - empty, because no subnets have the version - /// `subnet_update_proposals` - contains proposals from the first stage - /// `unassigned_nodes_proposals` - empty - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-21` - #[test] - fn test_use_case_2() { - let index = craft_index_state(); - let last_bake_status = BTreeMap::new(); - let subnet_principal = Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") - .expect("Should be possible to create principal"); - let subnet_update_proposals = vec![SubnetUpdateProposal { - info: ProposalInfoInternal { - executed: false, - executed_timestamp_seconds: 0, - proposal_timestamp_seconds: 0, - id: 1, - }, - payload: UpdateSubnetReplicaVersionPayload { - subnet_id: PrincipalId(subnet_principal.clone()), - replica_version_id: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - }, - }]; - let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); - let unassigned_nodes_proposals = vec![]; - let subnets = &craft_subnets(); - let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposals, - index, - None, - &unassigned_version, - subnets, - now, - ); + struct TestCase { + name: &'static str, + index: Index, + subnets: Vec, + subnet_update_proposals: Vec, + unassigned_node_proposals: Vec, + now: NaiveDate, + last_bake_status: BTreeMap, + unassigned_node_version: String, + expect_outcome_success: bool, + expect_actions: Vec, + release_start: NaiveDate, + } - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - println!("{:#?}", actions); - assert_eq!(actions.len(), 1); - for action in actions { - match action { - SubnetAction::PendingProposal { - subnet_short, - proposal_id, - } => { - assert_eq!(proposal_id, 1); - assert!(subnet_principal.to_string().starts_with(&subnet_short)) - } - // Just fail - _ => assert!(false), + impl Default for TestCase { + fn default() -> Self { + Self { + name: Default::default(), + index: craft_index_state(), + subnets: craft_subnets(), + subnet_update_proposals: Default::default(), + unassigned_node_proposals: Default::default(), + now: NaiveDate::parse_from_str("2024-02-26", "%Y-%m-%d").expect("Should parse date"), + expect_actions: Default::default(), + last_bake_status: Default::default(), + unassigned_node_version: Default::default(), + expect_outcome_success: true, + release_start: NaiveDate::parse_from_str("2024-02-26", "%Y-%m-%d").expect("Should parse date"), } } } - /// Use case 3: First batch is submitted the proposal was executed and the subnet is baking - /// - /// `last_bake_status` - contains the status for the first subnet - /// `subnet_update_proposals` - contains proposals from the first stage - /// `unassigned_nodes_proposals` - empty - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-21` - #[test] - fn test_use_case_3() { - let index = craft_index_state(); - let last_bake_status = [( - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - humantime::parse_duration("3h"), - )] - .iter() - .map(|(id, duration)| { - ( - id.to_string(), - duration.clone().expect("Should parse duration").as_secs_f64(), - ) - }) - .collect(); - let subnet_principal = Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") - .expect("Should be possible to create principal"); - let subnet_update_proposals = vec![SubnetUpdateProposal { - info: ProposalInfoInternal { - executed: true, - executed_timestamp_seconds: 0, - proposal_timestamp_seconds: 0, - id: 1, - }, - payload: UpdateSubnetReplicaVersionPayload { - subnet_id: PrincipalId(subnet_principal.clone()), - replica_version_id: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - }, - }]; - let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); - let unassigned_nodes_proposals = vec![]; - let mut subnets = craft_subnets(); - replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); - let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposals, - index, - None, - &unassigned_version, - &subnets, - now, - ); - - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - println!("{:#?}", actions); - assert_eq!(actions.len(), 1); - for action in actions { - match action { - SubnetAction::Baking { - subnet_short, - remaining, - } => { - assert!(subnet_principal.to_string().starts_with(&subnet_short)); - assert!(remaining.eq(&humantime::parse_duration("5h").expect("Should parse duration"))) - } - // Just fail - _ => assert!(false), - } + impl TestCase { + pub fn new(name: &'static str) -> Self { + let mut case = Self::default(); + case.name = name; + case } - } - /// Use case 4: First batch is submitted the proposal was executed and the subnet is baked, placing proposal for next stage - /// - /// `last_bake_status` - contains the status for the first subnet - /// `subnet_update_proposals` - contains proposals from the first stage - /// `unassigned_nodes_proposals` - empty - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-21` - #[test] - fn test_use_case_4() { - let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); - let last_bake_status = [( - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - humantime::parse_duration("9h"), - )] - .iter() - .map(|(id, duration)| { - ( - id.to_string(), - duration.clone().expect("Should parse duration").as_secs_f64(), - ) - }) - .collect(); - let subnet_principal = Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") - .expect("Should be possible to create principal"); - let subnet_update_proposals = vec![SubnetUpdateProposal { - info: ProposalInfoInternal { - executed: true, - executed_timestamp_seconds: 0, - proposal_timestamp_seconds: 0, - id: 1, - }, - payload: UpdateSubnetReplicaVersionPayload { - subnet_id: PrincipalId(subnet_principal.clone()), - replica_version_id: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - }, - }]; - let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); - let unassigned_nodes_proposals = vec![]; - let mut subnets = craft_subnets(); - replace_versions(&mut subnets, &[("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f")]); - let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposals, - index, - None, - &unassigned_version, - &subnets, - now, - ); + pub fn with_index(mut self, index: Index) -> Self { + self.index = index; + self + } - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - println!("{:#?}", actions); - assert_eq!(actions.len(), 2); - let subnets = vec![ - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - ]; - for action in actions { - match action { - SubnetAction::PlaceProposal { - is_unassigned, - subnet_principal, - version, - } => { - assert_eq!(is_unassigned, false); - assert_eq!(version, current_version); - assert!(subnets.contains(&subnet_principal.as_str())) - } - // Just fail - _ => assert!(false), - } + pub fn with_subnets(mut self, subnets: &[Subnet]) -> Self { + self.subnets = subnets.to_vec(); + self } - } - /// Use case 5: Updating unassigned nodes - /// - /// `last_bake_status` - contains the status for all subnets before unassigned nodes - /// `subnet_update_proposals` - contains proposals from previous two stages - /// `unassigned_nodes_proposals` - empty - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-21` - #[test] - fn test_use_case_5() { - let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); - let last_bake_status = [ - ( - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - humantime::parse_duration("9h"), - ), - ( - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - humantime::parse_duration("5h"), - ), - ( - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - humantime::parse_duration("5h"), - ), - ] - .iter() - .map(|(id, duration)| { - ( - id.to_string(), - duration.clone().expect("Should parse duration").as_secs_f64(), - ) - }) - .collect(); - let subnet_update_proposals = craft_executed_proposals( - &[ - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - ], - ¤t_version, - ); - let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); - let unassigned_nodes_proposals = vec![]; - let mut subnets = craft_subnets(); - replace_versions( - &mut subnets, - &[ - ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ], - ); - let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposals, - index, - None, - &unassigned_version, - &subnets, - now, - ); + pub fn with_subnet_update_proposals(mut self, subnet_update_proposals: &[(u64, bool, &'static str)]) -> Self { + self.subnet_update_proposals = subnet_update_proposals + .iter() + .map(|(id, executed, version)| { + if *executed { + if let Some(subnet) = self + .subnets + .iter_mut() + .find(|subnet| subnet.principal.eq(&principal(*id))) + { + subnet.replica_version = version.to_string() + } + } + subnet_update_proposal(*id, *executed, version) + }) + .collect(); + self + } - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - println!("{:#?}", actions); - assert_eq!(actions.len(), 1); - for action in actions { - match action { - SubnetAction::PlaceProposal { - is_unassigned, - subnet_principal: _, - version, - } => { - assert!(is_unassigned); - assert_eq!(version, current_version); - } - // Just fail - _ => assert!(false), - } + pub fn with_unassigned_node_proposals( + mut self, + unassigned_node_update_proposals: &[(bool, &'static str)], + ) -> Self { + self.unassigned_node_proposals = unassigned_node_update_proposals + .iter() + .map(|(executed, v)| { + if *executed { + self.unassigned_node_version = v.to_string() + } + UpdateUnassignedNodesProposal { + info: ProposalInfoInternal { + executed: *executed, + executed_timestamp_seconds: 0, + id: 0, + proposal_timestamp_seconds: 0, + }, + payload: UpdateUnassignedNodesConfigPayload { + replica_version: Some(v.to_string()), + ssh_readonly_access: None, + }, + } + }) + .collect(); + self } - } - /// Use case 6: Proposal sent for updating unassigned nodes but it is not executed - /// - /// `last_bake_status` - contains the status for all subnets before unassigned nodes - /// `subnet_update_proposals` - contains proposals from previous two stages - /// `unassigned_nodes_proposals` - contains open proposal for unassigned nodes - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-21` - #[test] - fn test_use_case_6() { - let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); - let last_bake_status = [ - ( - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - humantime::parse_duration("9h"), - ), - ( - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - humantime::parse_duration("5h"), - ), - ( - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - humantime::parse_duration("5h"), - ), - ] - .iter() - .map(|(id, duration)| { - ( - id.to_string(), - duration.clone().expect("Should parse duration").as_secs_f64(), - ) - }) - .collect(); - let subnet_update_proposals = craft_executed_proposals( - &[ - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - ], - ¤t_version, - ); - let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); - let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { - info: ProposalInfoInternal { - executed: false, - executed_timestamp_seconds: 0, - id: 5, - proposal_timestamp_seconds: 0, - }, - payload: UpdateUnassignedNodesConfigPayload { - ssh_readonly_access: None, - replica_version: Some(current_version.clone()), - }, - }]; - let mut subnets = craft_subnets(); - replace_versions( - &mut subnets, - &[ - ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ], - ); - let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposal, - index, - None, - &unassigned_version, - &subnets, - now, - ); + pub fn with_now(mut self, now: &'static str) -> Self { + self.now = NaiveDate::parse_from_str(now, "%Y-%m-%d").expect("Should parse date"); + self + } - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - println!("{:#?}", actions); - assert_eq!(actions.len(), 1); - for action in actions { - match action { - SubnetAction::PendingProposal { - proposal_id, - subnet_short, - } => { - assert_eq!(proposal_id, 5); - assert_eq!(subnet_short, "unassigned-version"); - } - // Just fail - _ => assert!(false), - } + pub fn expect_actions(mut self, expect_actions: &[SubnetAction]) -> Self { + self.expect_actions = expect_actions.to_vec(); + self } - } - /// Use case 7: Executed update unassigned nodes, waiting for next week - /// - /// `last_bake_status` - contains the status for all subnets before unassigned nodes - /// `subnet_update_proposals` - contains proposals from previous two stages - /// `unassigned_nodes_proposals` - contains executed proposal for unassigned nodes - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-24` - #[test] - fn test_use_case_7() { - let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); - let last_bake_status = [ - ( - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - humantime::parse_duration("9h"), - ), - ( - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - humantime::parse_duration("5h"), - ), - ( - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - humantime::parse_duration("5h"), - ), - ] - .iter() - .map(|(id, duration)| { - ( - id.to_string(), - duration.clone().expect("Should parse duration").as_secs_f64(), - ) - }) - .collect(); - let subnet_update_proposals = craft_executed_proposals( - &[ - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - ], - ¤t_version, - ); - let unassigned_version = current_version.clone(); - let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { - info: ProposalInfoInternal { - executed: true, - executed_timestamp_seconds: 0, - id: 5, - proposal_timestamp_seconds: 0, - }, - payload: UpdateUnassignedNodesConfigPayload { - ssh_readonly_access: None, - replica_version: Some(current_version.clone()), - }, - }]; - let mut subnets = craft_subnets(); - replace_versions( - &mut subnets, - &[ - ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ], - ); - let now = NaiveDate::parse_from_str("2024-02-24", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposal, - index, - None, - &unassigned_version, - &subnets, - now, - ); + pub fn with_last_bake_status(mut self, last_bake_status: &[(u64, &'static str)]) -> Self { + self.last_bake_status = last_bake_status + .iter() + .map(|(id, duration)| { + ( + principal(*id).to_string(), + humantime::parse_duration(duration) + .expect("Should be able to parse duration for test") + .as_secs_f64(), + ) + }) + .collect(); + self + } - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - println!("{:#?}", actions); - assert_eq!(actions.len(), 1); - for action in actions { - match action { - SubnetAction::WaitForNextWeek { subnet_short } => { - assert_eq!(subnet_short, "pjljw"); - } - // Just fail - _ => assert!(false), - } + pub fn with_unassigned_node_version(mut self, unassigned_node_version: &'static str) -> Self { + self.unassigned_node_version = unassigned_node_version.to_string(); + self } - } - /// Use case 8: Next monday came, should place proposal for updating the last subnet - /// - /// `last_bake_status` - contains the status for all subnets before unassigned nodes - /// `subnet_update_proposals` - contains proposals from previous two stages - /// `unassigned_nodes_proposals` - contains executed proposal for unassigned nodes - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-26` - #[test] - fn test_use_case_8() { - let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); - let last_bake_status = [ - ( - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - humantime::parse_duration("9h"), - ), - ( - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - humantime::parse_duration("5h"), - ), - ( - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - humantime::parse_duration("5h"), - ), - ] - .iter() - .map(|(id, duration)| { - ( - id.to_string(), - duration.clone().expect("Should parse duration").as_secs_f64(), - ) - }) - .collect(); - let subnet_update_proposals = craft_executed_proposals( - &[ - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - ], - ¤t_version, - ); - let unassigned_version = current_version.clone(); - let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { - info: ProposalInfoInternal { - executed: true, - executed_timestamp_seconds: 0, - id: 5, - proposal_timestamp_seconds: 0, - }, - payload: UpdateUnassignedNodesConfigPayload { - ssh_readonly_access: None, - replica_version: Some(current_version.clone()), - }, - }]; - let mut subnets = craft_subnets(); - replace_versions( - &mut subnets, - &[ - ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ], - ); - let now = NaiveDate::parse_from_str("2024-02-28", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposal, - index, - None, - &unassigned_version, - &subnets, - now, - ); + pub fn expect_outcome_success(mut self, expect_outcome_success: bool) -> Self { + self.expect_outcome_success = expect_outcome_success; + self + } - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - println!("{:#?}", actions); - assert_eq!(actions.len(), 1); - for action in actions { - match action { - SubnetAction::PlaceProposal { - is_unassigned, - subnet_principal, - version, - } => { - assert!(subnet_principal.starts_with("pjljw")); - assert_eq!(is_unassigned, false); - assert_eq!(version, current_version) - } - // Just fail - _ => assert!(false), - } + pub fn with_release_start(mut self, release_start: &'static str) -> Self { + self.release_start = NaiveDate::parse_from_str(release_start, "%Y-%m-%d").expect("Should parse date"); + self } } - /// Use case 9: Next monday came, proposal for last subnet executed and bake time passed. Rollout finished - /// - /// `last_bake_status` - contains the status for all subnets before unassigned nodes - /// `subnet_update_proposals` - contains proposals from previous two stages - /// `unassigned_nodes_proposals` - contains executed proposal for unassigned nodes - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-26` - #[test] - fn test_use_case_9() { - let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); - let last_bake_status = [ - ( - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - humantime::parse_duration("9h"), - ), - ( - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - humantime::parse_duration("5h"), - ), - ( - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - humantime::parse_duration("5h"), - ), - ( - "pjljw-kztyl-46ud4-ofrj6-nzkhm-3n4nt-wi3jt-ypmav-ijqkt-gjf66-uae", - humantime::parse_duration("5h"), - ), - ] - .iter() - .map(|(id, duration)| { - ( - id.to_string(), - duration.clone().expect("Should parse duration").as_secs_f64(), - ) - }) - .collect(); - let subnet_update_proposals = craft_executed_proposals( - &[ - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - "pjljw-kztyl-46ud4-ofrj6-nzkhm-3n4nt-wi3jt-ypmav-ijqkt-gjf66-uae", - ], - ¤t_version, - ); - let unassigned_version = current_version.clone(); - let unassigned_nodes_proposal = vec![UpdateUnassignedNodesProposal { + fn principal(id: u64) -> PrincipalId { + PrincipalId::new_subnet_test_id(id) + } + + fn subnet_update_proposal(subnet_id: u64, executed: bool, version: &'static str) -> SubnetUpdateProposal { + SubnetUpdateProposal { info: ProposalInfoInternal { - executed: true, + executed, executed_timestamp_seconds: 0, - id: 5, proposal_timestamp_seconds: 0, + id: subnet_id, }, - payload: UpdateUnassignedNodesConfigPayload { - ssh_readonly_access: None, - replica_version: Some(current_version.clone()), + payload: UpdateSubnetReplicaVersionPayload { + replica_version_id: version.to_string(), + subnet_id: principal(subnet_id), }, - }]; - let mut subnets = craft_subnets(); - replace_versions( - &mut subnets, - &[ - ("io67a", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("shefu", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("uzr34", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ("pjljw", "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f"), - ], - ); - let now = NaiveDate::parse_from_str("2024-02-28", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposal, - index, - None, - &unassigned_version, - &subnets, - now, - ); - - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - assert_eq!(actions.len(), 0); + } } -} -// E2E tests for decision making process for happy path with feature builds -#[cfg(test)] -mod check_stages_tests_feature_builds { - use std::str::FromStr; + #[test] + fn test_use_cases_no_feature_builds() { + let tests = vec![ + TestCase::new("Beginning of a new rollout").expect_actions(&[SubnetAction::PlaceProposal { + is_unassigned: false, + subnet_principal: principal(1), + version: "b".to_string(), + },]), + TestCase::new("First batch is submitted but the proposal wasn't executed") + .with_subnet_update_proposals(&[(1, false, "b"),]) + .expect_actions(&[SubnetAction::PendingProposal { + subnet_short: principal(1).to_string(), + proposal_id: 1, + }]), + TestCase::new("First batch is submitted the proposal was executed and the subnet is baking") + .with_subnet_update_proposals(&[(1, true, "b"),]) + .with_last_bake_status(&[(1, "3h")]) + .expect_actions(&[SubnetAction::Baking { + subnet_short: principal(1).to_string(), + remaining: humantime::parse_duration("5h").expect("Should parse duration"), + }]), + TestCase::new("First batch is submitted the proposal was executed and the subnet is baked, placing proposal for next stage") + .with_subnet_update_proposals(&[(1, true, "b")]) + .with_last_bake_status(&[(1, "9h")]) + .expect_actions(&[SubnetAction::PlaceProposal { is_unassigned: false, subnet_principal: principal(2), version: "b".to_string() }, SubnetAction::PlaceProposal { is_unassigned: false, subnet_principal: principal(3), version: "b".to_string() }]), + TestCase::new("Updating unassigned nodes") + .with_subnet_update_proposals(&[(1, true, "b"), (2, true, "b"), (3, true, "b")]) + .with_last_bake_status(&[(1, "9h"), (2, "5h"), (3, "5h")]) + .with_unassigned_node_version("a") + .expect_actions(&[SubnetAction::PlaceProposal { is_unassigned: true, subnet_principal: PrincipalId::new_anonymous(), version: "b".to_string()}]), + TestCase::new("Proposal sent for updating unassigned nodes but it is not executed") + .with_subnet_update_proposals(&[(1, true, "b"), (2, true, "b"), (3, true, "b")]) + .with_last_bake_status(&[(1, "9h"), (2, "5h"), (3, "5h")]) + .with_unassigned_node_version("a") + .with_unassigned_node_proposals(&[(false, "b")]) + .expect_actions(&[SubnetAction::PendingProposal { subnet_short: PrincipalId::new_anonymous().to_string(), proposal_id: 0 }]), + TestCase::new("Executed update unassigned nodes, waiting for next week") + .with_subnet_update_proposals(&[(1, true, "b"), (2, true, "b"), (3, true, "b")]) + .with_last_bake_status(&[(1, "9h"), (2, "5h"), (3, "5h")]) + .with_unassigned_node_proposals(&[(true, "b")]) + .with_now("2024-03-03") + .expect_actions(&[SubnetAction::WaitForNextWeek { subnet_short: principal(4).to_string() }]), + TestCase::new("Next monday came, should place proposal for updating the last subnet") + .with_subnet_update_proposals(&[(1, true, "b"), (2, true, "b"), (3, true, "b")]) + .with_last_bake_status(&[(1, "9h"), (2, "5h"), (3, "5h")]) + .with_unassigned_node_proposals(&[(true, "b")]) + .with_now("2024-03-04") + .expect_actions(&[SubnetAction::PlaceProposal { is_unassigned: false, subnet_principal: principal(4), version: "b".to_string() }]), + TestCase::new("Next monday came, proposal for last subnet executed and bake time passed. Rollout finished") + .with_subnet_update_proposals(&[(1, true, "b"), (2, true, "b"), (3, true, "b"), (4, true, "b")]) + .with_last_bake_status(&[(1, "9h"), (2, "5h"), (3, "5h"), (4, "5h")]) + .with_unassigned_node_proposals(&[(true, "b")]) + .with_now("2024-03-04") + .expect_actions(&vec![]), + TestCase::new("Partially executed step, a subnet is baking but the other doesn't have a submitted proposal") + .with_subnet_update_proposals(&[(1, true, "b"), (2, true, "b")]) + .with_last_bake_status(&[(1, "9h"), (2, "3h")]) + .expect_actions(&[SubnetAction::Baking { subnet_short: principal(2).to_string(), remaining: humantime::parse_duration("1h").expect("Should parse duration") }, SubnetAction::PlaceProposal { is_unassigned: false, subnet_principal: principal(3), version: "b".to_string() }]) + ]; - use candid::Principal; - use check_stages_tests_feature_builds::check_stages_tests_no_feature_builds::{craft_subnets, replace_versions}; - use ic_base_types::PrincipalId; - use ic_management_backend::proposal::ProposalInfoInternal; - use registry_canister::mutations::do_update_subnet_replica::UpdateSubnetReplicaVersionPayload; + for test in tests { + let desired_versions = desired_rollout_release_version(&test.subnets, &test.index.releases); + let maybe_actions = check_stages( + &test.last_bake_status, + &test.subnet_update_proposals, + &test.unassigned_node_proposals, + test.index, + None, + &test.unassigned_node_version, + &test.subnets, + test.now, + test.release_start, + desired_versions, + ); - use crate::calculation::{Index, Release, Rollout, Version}; + assert_eq!( + maybe_actions.is_ok(), + test.expect_outcome_success, + "test case '{}' failed", + test.name + ); + if !test.expect_outcome_success { + continue; + } - use super::*; + let actions = maybe_actions.unwrap(); + assert_eq!(actions, test.expect_actions, "test case '{}' failed", test.name) + } + } - /// Part two => Feature builds - /// `last_bake_status` - can be defined depending on the use case - /// `subnet_update_proposals` - can be defined depending on the use case - /// `unassigned_nodes_update_proposals` - can be defined depending on the use case - /// `index` - has to be defined - /// `logger` - can be defined, but won't be because these are only tests - /// `subnets` - has to be defined - /// `now` - has to be defined - /// - /// For all use cases we will use the following setup - /// rollout: - /// pause: false // Tested in `should_proceed.rs` module - /// skip_days: [] // Tested in `should_proceed.rs` module - /// stages: - /// - subnets: [io67a] - /// bake_time: 8h - /// - subnets: [shefu, uzr34] - /// bake_time: 4h - /// - update_unassigned_nodes: true - /// - subnets: [pjljw] - /// wait_for_next_week: true - /// bake_time: 4h - /// releases: - /// - rc_name: rc--2024-02-21_23-01 - /// versions: - /// - version: 2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f - /// name: rc--2024-02-21_23-01 - /// release_notes_ready: - /// subnets: [] - /// - version: 76521ef765e86187c43f7d6a02e63332a6556c8c - /// name: rc--2024-02-21_23-01-feat - /// release_notes_ready: - /// subnets: - /// - shefu - /// - io67a - /// - rc_name: rc--2024-02-14_23-01 - /// versions: - /// - version: 85bd56a70e55b2cea75cae6405ae11243e5fdad8 - /// name: rc--2024-02-14_23-01 - /// release_notes_ready: - /// subnets: [] // empty because its a regular build - fn craft_index_state() -> Index { - Index { + #[test] + fn test_use_cases_with_feature_builds() { + let index_with_features = Index { rollout: Rollout { pause: false, skip_days: vec![], stages: vec![ - Stage { - subnets: vec!["io67a".to_string()], - bake_time: humantime::parse_duration("8h").expect("Should be able to parse."), - ..Default::default() - }, - Stage { - subnets: vec!["shefu".to_string(), "uzr34".to_string()], - bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), - ..Default::default() - }, - Stage { - update_unassigned_nodes: true, - ..Default::default() - }, - Stage { - subnets: vec!["pjljw".to_string()], - bake_time: humantime::parse_duration("4h").expect("Should be able to parse."), - wait_for_next_week: true, - ..Default::default() - }, + stage(&vec![1], "8h"), + stage(&vec![2, 3], "4h"), + stage_unassigned(), + stage_next_week(&vec![4], "4h"), ], }, releases: vec![ - Release { - rc_name: "rc--2024-02-21_23-01".to_string(), - versions: vec![ - Version { - name: "rc--2024-02-21_23-01".to_string(), - version: "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(), - ..Default::default() - }, - Version { - name: "rc--2024-02-21_23-01-feat".to_string(), - version: "76521ef765e86187c43f7d6a02e63332a6556c8c".to_string(), - subnets: ["io67a", "shefu"].iter().map(|f| f.to_string()).collect(), - ..Default::default() - }, - ], - }, - Release { - rc_name: "rc--2024-02-14_23-01".to_string(), - versions: vec![Version { - name: "rc--2024-02-14_23-01".to_string(), - version: "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(), - ..Default::default() - }], - }, + release("rc--2024-02-21_23-01", vec![("b", vec![]), ("b.feat", vec![1, 2])]), + release("rc--2024-02-14_23-01", vec![("a", vec![])]), ], - } - } - - /// Use-Case 1: Beginning of a new rollout - /// - /// `last_bake_status` - empty, because no subnets have the version - /// `subnet_update_proposals` - can be empty but doesn't have to be. For e.g. if its Monday it is possible to have an open proposal for NNS - /// But it is for a different version (one from last week) - /// `unassigned_nodes_proposals` - empty - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-21` - #[test] - fn test_use_case_1() { - let index = craft_index_state(); - let last_bake_status = BTreeMap::new(); - let subnet_update_proposals = Vec::new(); - let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); - let unassigned_nodes_proposals = vec![]; - let subnets = &craft_subnets(); - let feature = index - .releases - .get(0) - .expect("Should be at least one") - .versions - .get(1) - .expect("Should be set to be the second version being rolled out"); - // TODO: replace in index - let mut current_release_feature_spec = BTreeMap::new(); - current_release_feature_spec.insert(feature.version.clone(), feature.subnets.clone()); - let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposals, - index.clone(), - None, - &unassigned_version, - subnets, - now, - ); + }; + let tests = vec![TestCase::new("Beginning of a new rollout") + .with_index(index_with_features.clone()) + .expect_actions(&[SubnetAction::PlaceProposal { + is_unassigned: false, + subnet_principal: principal(1), + version: "b.feat".to_string(), + }]), TestCase::new("First batch is submitted the proposal was executed and the subnet is baked, placing proposal for next stage") + .with_index(index_with_features) + .with_last_bake_status(&[(1, "9h")]) + .with_subnet_update_proposals(&[(1, true, "b.feat")]) + .expect_actions(&[SubnetAction::PlaceProposal { is_unassigned: false, subnet_principal: principal(2), version: "b.feat".to_string() }, SubnetAction::PlaceProposal { is_unassigned: false, subnet_principal: principal(3), version: "b".to_string() }])]; + + for test in tests { + let desired_versions = desired_rollout_release_version(&test.subnets, &test.index.releases); + let maybe_actions = check_stages( + &test.last_bake_status, + &test.subnet_update_proposals, + &test.unassigned_node_proposals, + test.index, + None, + &test.unassigned_node_version, + &test.subnets, + test.now, + test.release_start, + desired_versions, + ); - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - - assert_eq!(actions.len(), 1); - for action in actions { - match action { - SubnetAction::PlaceProposal { - is_unassigned, - subnet_principal, - version, - } => { - assert_eq!(is_unassigned, false); - assert_eq!(version, feature.version); - assert!(subnet_principal.starts_with("io67a")) - } - // Fail the test - _ => assert!(false), + assert_eq!( + maybe_actions.is_ok(), + test.expect_outcome_success, + "test case '{}' failed", + test.name + ); + if !test.expect_outcome_success { + continue; } - } - } - /// Use case 2: First batch is submitted the proposal was executed and the subnet is baked, placing proposal for next stage - /// - /// `last_bake_status` - contains the status for the first subnet - /// `subnet_update_proposals` - contains proposals from the first stage - /// `unassigned_nodes_proposals` - empty - /// `subnets` - can be seen in `craft_index_state` - /// `now` - same `2024-02-21` - #[test] - fn test_use_case_2() { - let index = craft_index_state(); - let current_version = "2e921c9adfc71f3edc96a9eb5d85fc742e7d8a9f".to_string(); - let last_bake_status = [( - "io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe", - humantime::parse_duration("9h"), - )] - .iter() - .map(|(id, duration)| { - ( - id.to_string(), - duration.clone().expect("Should parse duration").as_secs_f64(), - ) - }) - .collect(); - let subnet_principal = Principal::from_str("io67a-2jmkw-zup3h-snbwi-g6a5n-rm5dn-b6png-lvdpl-nqnto-yih6l-gqe") - .expect("Should be possible to create principal"); - let subnet_update_proposals = vec![SubnetUpdateProposal { - info: ProposalInfoInternal { - executed: true, - executed_timestamp_seconds: 0, - proposal_timestamp_seconds: 0, - id: 1, - }, - payload: UpdateSubnetReplicaVersionPayload { - subnet_id: PrincipalId(subnet_principal.clone()), - replica_version_id: "76521ef765e86187c43f7d6a02e63332a6556c8c".to_string(), - }, - }]; - let unassigned_version = "85bd56a70e55b2cea75cae6405ae11243e5fdad8".to_string(); - let unassigned_nodes_proposals = vec![]; - let mut subnets = craft_subnets(); - replace_versions(&mut subnets, &[("io67a", "76521ef765e86187c43f7d6a02e63332a6556c8c")]); - let feature = index - .releases - .get(0) - .expect("Should be at least one") - .versions - .get(1) - .expect("Should be set to be the second version being rolled out"); - // TODO: replace in index - let mut current_release_feature_spec = BTreeMap::new(); - current_release_feature_spec.insert(feature.version.clone(), feature.subnets.clone()); - let now = NaiveDate::parse_from_str("2024-02-21", "%Y-%m-%d").expect("Should parse date"); - - let maybe_actions = check_stages( - &last_bake_status, - &subnet_update_proposals, - &unassigned_nodes_proposals, - index.clone(), - None, - &unassigned_version, - &subnets, - now, - ); - - assert!(maybe_actions.is_ok()); - let actions = maybe_actions.unwrap(); - println!("{:#?}", actions); - assert_eq!(actions.len(), 2); - let subnets = vec![ - "shefu-t3kr5-t5q3w-mqmdq-jabyv-vyvtf-cyyey-3kmo4-toyln-emubw-4qe", - "uzr34-akd3s-xrdag-3ql62-ocgoh-ld2ao-tamcv-54e7j-krwgb-2gm4z-oqe", - ]; - for action in actions { - match action { - SubnetAction::PlaceProposal { - is_unassigned, - subnet_principal, - version, - } => { - assert_eq!(is_unassigned, false); - if subnet_principal.starts_with("shefu") { - assert_eq!(version, feature.version); - } else { - assert_eq!(version, current_version); - } - assert!(subnets.contains(&subnet_principal.as_str())) - } - // Just fail - _ => assert!(false), - } + let actions = maybe_actions.unwrap(); + assert_eq!(actions, test.expect_actions, "test case '{}' failed", test.name) } } } diff --git a/rs/rollout-controller/src/main.rs b/rs/rollout-controller/src/main.rs index bc2e319e..c018190f 100644 --- a/rs/rollout-controller/src/main.rs +++ b/rs/rollout-controller/src/main.rs @@ -10,8 +10,9 @@ use tokio::select; use tokio_util::sync::CancellationToken; use url::Url; -use crate::{calculation::calculate_progress, registry_wrappers::sync_wrap}; +use crate::{actions::ActionExecutor, calculation::calculate_progress, registry_wrappers::sync_wrap}; +mod actions; mod calculation; mod fetching; mod registry_wrappers; @@ -50,6 +51,11 @@ async fn main() -> anyhow::Result<()> { let fetcher = fetching::resolve(args.subcommand, logger.clone()).await?; + let executor = match args.private_key_pem { + Some(path) => ActionExecutor::new(args.neuron_id, path, args.network.clone(), false, Some(&logger)).await?, + None => ActionExecutor::test(args.network.clone(), Some(&logger)).await?, + }; + let mut interval = tokio::time::interval(args.poll_interval); let mut should_sleep = false; loop { @@ -93,6 +99,16 @@ async fn main() -> anyhow::Result<()> { } }; + // Get blessed replica versions for later + let blessed_versions = match registry_state.get_blessed_replica_versions().await { + Ok(versions) => versions, + Err(e) => { + warn!(logger, "{:?}", e); + should_sleep = false; + continue; + } + }; + // Calculate what should be done info!(logger, "Calculating the progress of the current release"); let actions = match calculate_progress(&logger, index, &client, registry_state).await { @@ -105,12 +121,15 @@ async fn main() -> anyhow::Result<()> { info!(logger, "Calculating completed"); if actions.is_empty() { - info!(logger, "No actions needed, sleeping"); - continue; + info!(logger, "Rollout completed"); + token.cancel(); + break; } info!(logger, "Calculated actions: {:#?}", actions); - // Apply changes - token.cancel(); + match executor.execute(&actions, &blessed_versions) { + Ok(()) => info!(logger, "Actions taken successfully"), + Err(e) => warn!(logger, "{:?}", e), + }; } info!(logger, "Shutdown complete"); shutdown_handle.await.unwrap(); @@ -190,6 +209,25 @@ If not specified it will take following based on 'Network' values: )] victoria_url: Option, + #[clap( + long = "private-key-pem", + help = r#" +Path to private key pem file that will be used to submit proposals. +If not specified will run in dry-run mode. + "# + )] + private_key_pem: Option, + + #[clap( + long = "neuron-id", + help = r#" +Neuron id that corresponds to the key that is in private key pem. +By default is 0. + "#, + default_value = "0" + )] + neuron_id: u64, + #[clap(subcommand)] pub(crate) subcommand: Commands, }