Skip to content

Commit

Permalink
Merge branch 'alic/fix-upgrade-test2' into 'master'
Browse files Browse the repository at this point in the history
fix(consensus): [ICSUP-3714] Change upgrade cycle in upgrade_downgrade test (fixed)

Fix of reverted MR: https://gitlab.com/dfinity-lab/public/ic/-/merge_requests/15563

I had to get the proper tip-of-branch replica version from the env variable, instead of hardcoding to `0000000000000000000000000000000000000000`. 

See merge request dfinity-lab/public/ic!15723
  • Loading branch information
dist1ll committed Oct 27, 2023
2 parents 98ae73d + cd9fac4 commit de9f7c1
Showing 1 changed file with 30 additions and 33 deletions.
63 changes: 30 additions & 33 deletions rs/tests/src/orchestrator/upgrade_downgrade.rs
@@ -1,11 +1,11 @@
/* tag::catalog[]
Title:: Upgradability to/from oldest prod replica version.
Title:: Upgradability from/to the mainnet replica version.
Goal:: Ensure the upgradability of the branch version against the oldest used replica version
Runbook::
. Setup an IC with 4-nodes NNS and 4-nodes app subnet using the code from the branch.
. Downgrade each type of subnet to the mainnet version and back.
. Setup an IC with 4-nodes NNS and 4-nodes app subnet using the mainnet replica version.
. Upgrade each type of subnet to the branch version, and downgrade again.
. During both upgrades simulate a disconnected node and make sure it catches up.
Success:: Upgrades work into both directions for all subnet types.
Expand Down Expand Up @@ -52,6 +52,7 @@ pub const UP_DOWNGRADE_PER_TEST_TIMEOUT: Duration = Duration::from_secs(15 * 60)

pub fn config(env: TestEnv) {
InternetComputer::new()
.with_mainnet_config()
.add_subnet(
Subnet::new(SubnetType::System)
.add_nodes(SUBNET_SIZE)
Expand All @@ -68,17 +69,17 @@ pub fn config(env: TestEnv) {
install_nns_and_check_progress(env.topology_snapshot());
}

// Tests a downgrade of the nns subnet to the mainnet version and an upgrade back to the branch version
// Tests an upgrade of the NNS subnet to the branch version and a downgrade back to the mainnet version
pub fn upgrade_downgrade_nns_subnet(env: TestEnv) {
upgrade_downgrade(env, SubnetType::System);
}

// Tests a downgrade of the app subnet to the mainnet version and an upgrade back to the branch version
// Tests an upgrade of the app subnet to the branch version and a downgrade back to the mainnet version
pub fn upgrade_downgrade_app_subnet(env: TestEnv) {
upgrade_downgrade(env, SubnetType::Application);
}

// Downgrades a subnet to $TARGET_VERSION and back to branch version
// Upgrades to the branch version, and back to mainnet NNS version.
fn upgrade_downgrade(env: TestEnv, subnet_type: SubnetType) {
let logger = env.logger();

Expand Down Expand Up @@ -123,23 +124,13 @@ fn upgrade_downgrade(env: TestEnv, subnet_type: SubnetType) {
let key = enable_ecdsa_signing_on_subnet(&nns_node, &nns_canister, subnet_id, &logger);
run_ecdsa_signature_test(&nns_canister, &logger, key);

let original_branch_version = get_assigned_replica_version(&nns_node).unwrap();
// We have to upgrade to `<VERSION>-test` because the original version is stored without the
// download URL in the registry.
let branch_version = format!("{}-test", original_branch_version);

// Check that the two versions do not initially match, which could hide failures.
assert!(mainnet_version != original_branch_version);
let original_branch_version = env
.read_dependency_from_env_to_string("ENV_DEPS__IC_VERSION_FILE")
.expect("tip-of-branch IC version");

// Bless both replica versions
block_on(bless_public_replica_version(
&nns_node,
&mainnet_version,
UpdateImageType::Image,
UpdateImageType::Image,
&logger,
));
let branch_version = format!("{}-test", original_branch_version);

// Bless branch version (mainnet is already blessed)
let sha256 = env.get_ic_os_update_img_test_sha256().unwrap();
let upgrade_url = env.get_ic_os_update_img_test_url().unwrap();
block_on(bless_replica_version(
Expand All @@ -152,23 +143,23 @@ fn upgrade_downgrade(env: TestEnv, subnet_type: SubnetType) {
));
info!(&logger, "Blessed all versions");

downgrade_upgrade_roundtrip(
upgrade_downgrade_roundtrip(
env,
&nns_node,
&mainnet_version,
&branch_version,
&mainnet_version,
subnet_type,
&nns_canister,
key,
);
}

// Downgrades and upgrades a subnet with one faulty node.
fn downgrade_upgrade_roundtrip(
// Upgrades and downgrades a subnet with one faulty node.
fn upgrade_downgrade_roundtrip(
env: TestEnv,
nns_node: &IcNodeSnapshot,
target_version: &str,
branch_version: &str,
upgrade_version: &str,
downgrade_version: &str,
subnet_type: SubnetType,
nns_canister: &MessageCanister,
key: VerifyingKey,
Expand Down Expand Up @@ -225,15 +216,15 @@ fn downgrade_upgrade_roundtrip(

stop_node(&logger, &faulty_node);

info!(logger, "Upgrade to version {}", target_version);
upgrade_to(nns_node, subnet_id, &subnet_node, target_version, &logger);
info!(logger, "Upgrade to version {}", upgrade_version);
upgrade_to(nns_node, subnet_id, &subnet_node, upgrade_version, &logger);

// Killing redundant nodes should not prevent the `faulty_node` downgrading to mainnet version and catching up after restarting.
for redundant_node in &redundant_nodes {
stop_node(&logger, redundant_node);
}
start_node(&logger, &faulty_node);
assert_assigned_replica_version(&faulty_node, target_version, env.logger());
assert_assigned_replica_version(&faulty_node, upgrade_version, env.logger());

assert!(can_read_msg(
&logger,
Expand Down Expand Up @@ -265,8 +256,14 @@ fn downgrade_upgrade_roundtrip(

stop_node(&logger, &faulty_node);

info!(logger, "Downgrade to version {}", branch_version);
upgrade_to(nns_node, subnet_id, &subnet_node, branch_version, &logger);
info!(logger, "Downgrade to version {}", downgrade_version);
upgrade_to(
nns_node,
subnet_id,
&subnet_node,
downgrade_version,
&logger,
);

let msg_3 = "hello world after upgrade!";
let can_id_3 = store_message(
Expand All @@ -279,7 +276,7 @@ fn downgrade_upgrade_roundtrip(
stop_node(&logger, redundant_node);
}
start_node(&logger, &faulty_node);
assert_assigned_replica_version(&faulty_node, branch_version, env.logger());
assert_assigned_replica_version(&faulty_node, downgrade_version, env.logger());

for (can_id, msg) in &[(can_id, msg), (can_id_2, msg_2), (can_id_3, msg_3)] {
assert!(can_read_msg_with_retries(
Expand Down

0 comments on commit de9f7c1

Please sign in to comment.