test: automatic integration heartbeat (#4068)

Co-authored-by: dandanlen <3168260+dandanlen@users.noreply.github.com>

state-chain/cf-integration-tests/src/account.rs

@@ -39,7 +39,7 @@ fn account_deletion_removes_relevant_storage_items() {
 			.unwrap();
 		assert!(Reputations::<Runtime>::get(backup_node.clone()).online_credits > 0);
 
-		let elon_vanity_name = "ElonShibMoonInu";
+		let elon_vanity_name = "ElonShibaMoonInu";
 		network::Cli::set_vanity_name(&backup_node, elon_vanity_name);
 		let vanity_names = VanityNames::<Runtime>::get();
 		assert_eq!(*vanity_names.get(&backup_node).unwrap(), elon_vanity_name.as_bytes().to_vec());

state-chain/cf-integration-tests/src/authorities.rs

@@ -68,17 +68,13 @@ fn authority_rotates_with_correct_sequence() {
 
 			// Skip the first authority rotation, as key handover is guaranteed to succeed
 			// when rotating for the first time.
-			testnet.move_forward_blocks(EPOCH_BLOCKS);
-			testnet.submit_heartbeat_all_engines();
-
-			testnet.move_forward_blocks(VAULT_ROTATION_BLOCKS);
+			testnet.move_to_the_next_epoch();
 
 			assert!(matches!(Validator::current_rotation_phase(), RotationPhase::Idle));
 			assert_eq!(AllVaults::status(), AsyncResult::Ready(VaultStatus::RotationComplete));
 			assert_eq!(GENESIS_EPOCH + 1, Validator::epoch_index());
 
-			testnet.move_forward_blocks(EPOCH_BLOCKS);
-			testnet.submit_heartbeat_all_engines();
+			testnet.move_to_the_end_of_epoch();
 
 			// Start the Authority and Vault rotation
 			// idle -> Keygen
@@ -194,17 +190,7 @@ fn genesis_nodes_rotated_out_accumulate_rewards_correctly() {
 				fund_authorities_and_join_auction(MAX_AUTHORITIES);
 
 			// Start an auction
-			testnet.move_to_next_epoch();
-			testnet.submit_heartbeat_all_engines();
-			testnet.move_forward_blocks(1);
-
-			assert_eq!(
-				GENESIS_EPOCH,
-				Validator::epoch_index(),
-				"We should still be in the genesis epoch"
-			);
-
-			testnet.move_forward_blocks(VAULT_ROTATION_BLOCKS);
+			testnet.move_to_the_next_epoch();
 			assert_eq!(GENESIS_EPOCH + 1, Validator::epoch_index(), "We should be in a new epoch");
 
 			// assert list of authorities as being the new nodes
@@ -268,8 +254,7 @@ fn authority_rotation_can_succeed_after_aborted_by_safe_mode() {
 			let (mut testnet, _, _) = fund_authorities_and_join_auction(MAX_AUTHORITIES);
 
 			// Resolve Auction
-			testnet.move_to_next_epoch();
-			testnet.submit_heartbeat_all_engines();
+			testnet.move_to_the_end_of_epoch();
 
 			// Run until key gen is completed.
 			testnet.move_forward_blocks(4);
@@ -303,7 +288,6 @@ fn authority_rotation_can_succeed_after_aborted_by_safe_mode() {
 			));
 
 			// Authority rotation should be successful.
-			testnet.submit_heartbeat_all_engines();
 			testnet.move_forward_blocks(VAULT_ROTATION_BLOCKS);
 			assert_eq!(GENESIS_EPOCH + 1, Validator::epoch_index(), "We should be in a new epoch");
 		});
@@ -321,8 +305,7 @@ fn authority_rotation_cannot_be_aborted_after_key_handover_but_stalls_on_safe_mo
 			let (mut testnet, _, _) = fund_authorities_and_join_auction(MAX_AUTHORITIES);
 
 			// Resolve Auction
-			testnet.move_to_next_epoch();
-			testnet.submit_heartbeat_all_engines();
+			testnet.move_to_the_end_of_epoch();
 
 			// Run until key handover starts
 			testnet.move_forward_blocks(5);
@@ -371,13 +354,10 @@ fn authority_rotation_can_recover_after_keygen_fails() {
 		.execute_with(|| {
 			let (mut testnet, _, backup_nodes) = fund_authorities_and_join_auction(MAX_AUTHORITIES);
 
-			backup_nodes.iter().for_each(|validator| {
-				testnet.set_active(validator, false);
-			});
+			testnet.set_active_all_nodes(false);
 
 			// Begin the rotation, but make Keygen fail.
-			testnet.move_to_next_epoch();
-			testnet.submit_heartbeat_all_engines();
+			testnet.move_to_the_end_of_epoch();
 
 			testnet.move_forward_blocks(1);
 			assert!(matches!(
@@ -404,10 +384,8 @@ fn authority_rotation_can_recover_after_keygen_fails() {
 			});
 
 			// Authority rotation can recover and succeed.
-			backup_nodes.iter().for_each(|validator| {
-				testnet.set_active(validator, true);
-			});
-			testnet.submit_heartbeat_all_engines();
+			testnet.set_active_all_nodes(true);
+
 			testnet.move_forward_blocks(VAULT_ROTATION_BLOCKS + 1);
 			assert_eq!(GENESIS_EPOCH + 1, Validator::epoch_index(), "We should be in a new epoch");
 		});
@@ -424,21 +402,17 @@ fn authority_rotation_can_recover_after_key_handover_fails() {
 		.execute_with(|| {
 			let (mut testnet, _, backup_nodes) = fund_authorities_and_join_auction(MAX_AUTHORITIES);
 			// Rotate authority at least once to ensure epoch keys are set.
-			testnet.move_to_next_epoch();
-			testnet.submit_heartbeat_all_engines();
-			testnet.move_forward_blocks(VAULT_ROTATION_BLOCKS);
+			testnet.move_to_the_next_epoch();
 			assert_eq!(GENESIS_EPOCH + 1, Validator::epoch_index(), "We should be in a new epoch");
 
 			// Begin the second rotation.
-			testnet.move_forward_blocks(EPOCH_BLOCKS);
-			testnet.submit_heartbeat_all_engines();
+			testnet.move_to_the_end_of_epoch();
 			testnet.move_forward_blocks(4);
 
 			// Make Key Handover fail. Only Bitcoin vault can fail during Key Handover.
 			// Ethereum and Polkadot do not need to wait for Key Handover.
-			backup_nodes.iter().for_each(|validator| {
-				testnet.set_active(validator, false);
-			});
+			testnet.set_active_all_nodes(false);
+
 			testnet.move_forward_blocks(1);
 			backup_nodes.iter().for_each(|validator| {
 				assert_ok!(BitcoinVault::report_key_handover_outcome(
@@ -476,10 +450,8 @@ fn authority_rotation_can_recover_after_key_handover_fails() {
 
 			// Key handovers are retried after failure.
 			// Authority rotation can recover and succeed.
-			backup_nodes.iter().for_each(|validator| {
-				testnet.set_active(validator, true);
-			});
-			testnet.submit_heartbeat_all_engines();
+			testnet.set_active_all_nodes(true);
+
 			testnet.move_forward_blocks(VAULT_ROTATION_BLOCKS);
 			assert_eq!(GENESIS_EPOCH + 2, Validator::epoch_index(), "We should be in a new epoch");
 		});

state-chain/cf-integration-tests/src/funding.rs

@@ -81,17 +81,14 @@ fn cannot_redeem_funds_out_of_redemption_period() {
 			assert_eq!(1, Validator::epoch_index(), "We should still be in the first epoch");
 
 			// Move to new epoch
-			testnet.move_to_next_epoch();
-			testnet.submit_heartbeat_all_engines();
+			testnet.move_to_the_next_epoch();
 			// TODO: figure out how to avoid this.
 			<pallet_cf_reputation::Pallet<Runtime> as OffenceReporter>::forgive_all(
 				Offence::MissedAuthorshipSlot,
 			);
 			<pallet_cf_reputation::Pallet<Runtime> as OffenceReporter>::forgive_all(
 				Offence::GrandpaEquivocation,
 			);
-			// Run things to a successful vault rotation
-			testnet.move_forward_blocks(VAULT_ROTATION_BLOCKS);
 
 			assert_eq!(
 				2,

state-chain/cf-integration-tests/src/network.rs

@@ -2,7 +2,7 @@ use super::*;
 
 use crate::threshold_signing::{BtcThresholdSigner, DotThresholdSigner, EthThresholdSigner};
 
-use cf_primitives::{AccountRole, EpochIndex, FlipBalance, TxId, GENESIS_EPOCH};
+use cf_primitives::{AccountRole, BlockNumber, EpochIndex, FlipBalance, TxId, GENESIS_EPOCH};
 use cf_traits::{AccountRoleRegistry, EpochInfo, VaultRotator};
 use chainflip_node::test_account_from_seed;
 use codec::Encode;
@@ -139,7 +139,12 @@ impl Cli {
 // Engine monitoring contract
 pub struct Engine {
 	pub node_id: NodeId,
+	// Automatically responds to events and responds with "OK".
 	pub live: bool,
+	// Automatically submits heartbeat to keep alive.
+	pub auto_submit_heartbeat: bool,
+	pub last_heartbeat: BlockNumber,
+
 	// conveniently creates a threshold "signature" (not really)
 	// all engines have the same one, so they create the same sig
 	pub eth_threshold_signer: Rc<RefCell<EthThresholdSigner>>,
@@ -160,6 +165,8 @@ impl Engine {
 			eth_threshold_signer,
 			dot_threshold_signer,
 			btc_threshold_signer,
+			auto_submit_heartbeat: true,
+			last_heartbeat: Default::default(),
 		}
 	}
 
@@ -434,7 +441,6 @@ pub struct Network {
 }
 
 thread_local! {
-	// TODO: USE THIS IN WHEN HANDLING EVENTS INSTEAD OF DISPATCHING CALLS DIRECTLY
 	static PENDING_EXTRINSICS: RefCell<VecDeque<(state_chain_runtime::RuntimeCall, RuntimeOrigin)>> = RefCell::default();
 	static TIMESTAMP: RefCell<u64> = RefCell::new(SLOT_DURATION);
 }
@@ -505,6 +511,16 @@ impl Network {
 			.collect()
 	}
 
+	pub fn set_active_all_nodes(&mut self, active: bool) {
+		self.engines.iter_mut().for_each(|(_, e)| e.live = active);
+	}
+
+	pub fn set_auto_heartbeat_all_nodes(&mut self, auto_heartbeat: bool) {
+		self.engines
+			.iter_mut()
+			.for_each(|(_, e)| e.auto_submit_heartbeat = auto_heartbeat);
+	}
+
 	// Create a network which includes the authorities in genesis of number of nodes
 	// and return a network and sorted list of nodes within
 	pub fn create(
@@ -517,6 +533,7 @@ impl Network {
 		for node in existing_nodes {
 			network.add_engine(node);
 			setup_peer_mapping(node);
+			assert_ok!(Reputation::heartbeat(RuntimeOrigin::signed(node.clone())));
 		}
 
 		// Create the backup nodes
@@ -552,18 +569,42 @@ impl Network {
 		);
 	}
 
-	pub fn move_to_next_epoch(&mut self) {
-		let blocks_per_epoch = Validator::blocks_per_epoch();
-		let current_block_number = System::block_number();
-		self.move_forward_blocks(blocks_per_epoch - (current_block_number % blocks_per_epoch));
+	/// Move to the next epoch, to the block after the completion of Authority rotation.
+	pub fn move_to_the_next_epoch(&mut self) {
+		self.move_to_the_end_of_epoch();
+		self.move_forward_blocks(VAULT_ROTATION_BLOCKS);
 	}
 
-	pub fn submit_heartbeat_all_engines(&self) {
-		for engine in self.engines.values() {
-			let _result = Reputation::heartbeat(RuntimeOrigin::signed(engine.node_id.clone()));
+	/// Move to the last block of the epoch - next block will start Authority rotation
+	pub fn move_to_the_end_of_epoch(&mut self) {
+		let current_block = System::block_number();
+		let target = Validator::current_epoch_started_at() + Validator::blocks_per_epoch();
+		if target > current_block {
+			self.move_forward_blocks(target - current_block - 1)
 		}
 	}
 
+	// Submits heartbeat for keep alive.
+	// If `force_update`, submit heartbeat unconditionally.
+	// else, submit according to auto-heartbeat setting and current block_number.
+	pub fn submit_heartbeat_all_engines(&mut self, force_update: bool) {
+		let current_block = System::block_number();
+		self.engines.iter_mut().for_each(|(_, engine)| {
+			// only validator roles are allowed to submit heartbeat.
+			if AccountRoles::has_account_role(&engine.node_id, AccountRole::Validator) &&
+				match force_update {
+					true => true,
+					false =>
+						engine.auto_submit_heartbeat &&
+							engine.last_heartbeat + Validator::blocks_per_epoch() - 1 <=
+								current_block,
+				} {
+				assert_ok!(Reputation::heartbeat(RuntimeOrigin::signed(engine.node_id.clone())));
+				engine.last_heartbeat = current_block;
+			}
+		});
+	}
+
 	pub fn move_forward_blocks(&mut self, n: u32) {
 		let start_block = System::block_number() + 1;
 		for block_number in start_block..(start_block + n) {
@@ -606,6 +647,7 @@ impl Network {
 				.unwrap()
 				.dispatch_bypass_filter(RuntimeOrigin::none()));
 
+			self.submit_heartbeat_all_engines(false);
 			dispatch_all_pending_extrinsics();
 
 			// Provide very large weight to ensure all on_idle processing can occur

state-chain/cf-integration-tests/src/new_epoch.rs

@@ -44,9 +44,10 @@ fn auction_repeats_after_failure_because_of_liveness() {
 				testnet.set_active(node, false);
 				pallet_cf_reputation::LastHeartbeat::<Runtime>::remove(node);
 			}
+			testnet.set_auto_heartbeat_all_nodes(false);
 
 			// Run to the next epoch to start the auction
-			testnet.move_to_next_epoch();
+			testnet.move_to_the_end_of_epoch();
 
 			assert!(
 				matches!(Validator::current_rotation_phase(), RotationPhase::Idle),
@@ -63,14 +64,12 @@ fn auction_repeats_after_failure_because_of_liveness() {
 				Validator::current_rotation_phase(),
 			);
 
-			for node in &offline_nodes {
-				testnet.set_active(node, true);
-			}
+			testnet.set_active_all_nodes(true);
 
 			// Submit a heartbeat, for all the nodes. Given we were waiting for the nodes to
 			// come online to start the rotation, the rotation ought to start on the next
 			// block
-			testnet.submit_heartbeat_all_engines();
+			testnet.submit_heartbeat_all_engines(true);
 			testnet.move_forward_blocks(1);
 
 			assert_eq!(GENESIS_EPOCH, Validator::epoch_index());
@@ -153,8 +152,7 @@ fn epoch_rotates() {
 				network::Cli::start_bidding(node);
 			}
 
-			testnet.move_to_next_epoch();
-			testnet.submit_heartbeat_all_engines();
+			testnet.move_to_the_end_of_epoch();
 			testnet.move_forward_blocks(1);
 
 			assert!(matches!(

state-chain/cf-integration-tests/src/threshold_signing.rs

@@ -86,8 +86,8 @@ where
 	KeyComponents: KeyUtils<SigVerification = SigVerification, AggKey = AggKey> + Clone,
 {
 	pub fn sign_with_key(&self, key: AggKey, message: &KeyComponents::Message) -> SigVerification {
-		let curr_key = self.key_components.agg_key();
-		if key == curr_key {
+		let current_key = self.key_components.agg_key();
+		if key == current_key {
 			return self.key_components.sign(message)
 		}
 		let next_key = self.proposed_key_components.as_ref().unwrap().agg_key();