From f95087bc508eda09ceecd7b8cb4a113b109cef7f Mon Sep 17 00:00:00 2001 From: Michael Birch Date: Tue, 23 Apr 2024 06:06:49 -0400 Subject: [PATCH] Feat(stateless-validation): Rewards for stateless validators (#11121) The goal of this PR is to give rewards to stateless validators (validators which do not produce blocks or chunks but do validate chunks by producing endorsements). This means updating `EpochInfoAggregator` to track the expected number of endorsements for each validator as well as how many were actually produced. For the MVP we are not actually tracking endorsements directly, instead we assume that if a chunk was included in a block then all stateless validators for that shard should be rewarded (because there must have been a sufficient number of endorsements for the block producer to include the chunk). However, this logic will need to be iterated on in future versions of the stateless validation protocol. Additionally, the reward calculator required updating to use the new endorsement stats as part of the uptime calculation which forms the basis of the rewards. Both of those changes are relatively minor. The technical snag in all of this is that the `EpochInfoAggregator` and other structures related to validator stats are persisted in the node state via borsh serialization. ~To avoid a large database migration, I have made this update backwards compatible with the old format without the endorsement stats. The logic for this is in `core/primitives/src/types/chunk_validator_stats.rs`. The legacy variant is distinguished from the new one in the serialization by serializing the one's compliment of the `u64` values for the new variant. During deserialization we check if the first `u64` is larger than 2^32 - 1 or not. If it is larger then we assume the value is actually a one's compliment and proceed with the new variant deserialization, and otherwise we do the legacy deserialization.~ ~This means that the serialized values can be interpreted improperly if 2^32 (4.29 billion) or more chunks are produced by a single validator during one epoch. I do not expect this to be an issue, but it is a limitation worth documenting.~ @Longarithm told me that the DB migration is actually fine here because it's not that much data. So the DB migration of the `EpochValidatorInfo` column is also implemented. --- chain/chain/src/runtime/tests.rs | 59 ++++- chain/epoch-manager/src/lib.rs | 116 +++++++--- chain/epoch-manager/src/reward_calculator.rs | 202 +++++++++++++++--- chain/epoch-manager/src/tests/mod.rs | 50 ++--- .../epoch-manager/src/tests/random_epochs.rs | 4 +- chain/epoch-manager/src/types.rs | 45 +++- core/primitives/src/types.rs | 6 +- .../src/types/chunk_validator_stats.rs | 105 +++++++++ core/primitives/src/views.rs | 12 ++ core/store/src/metadata.rs | 2 +- core/store/src/migrations.rs | 119 ++++++++++- nearcore/src/migrations.rs | 1 + 12 files changed, 615 insertions(+), 106 deletions(-) create mode 100644 core/primitives/src/types/chunk_validator_stats.rs diff --git a/chain/chain/src/runtime/tests.rs b/chain/chain/src/runtime/tests.rs index 79ef34ffb52..c5a3b8414ab 100644 --- a/chain/chain/src/runtime/tests.rs +++ b/chain/chain/src/runtime/tests.rs @@ -788,13 +788,28 @@ fn test_get_validator_info() { let staking_transaction = stake(1, &signer, &block_producers[0], 0); let mut expected_blocks = [0, 0]; let mut expected_chunks = [0, 0]; + let mut expected_endorsements = [0, 0]; let update_validator_stats = - |env: &mut TestEnv, expected_blocks: &mut [u64], expected_chunks: &mut [u64]| { + |env: &mut TestEnv, + expected_blocks: &mut [u64; 2], + expected_chunks: &mut [u64; 2], + expected_endorsements: &mut [u64; 2]| { let epoch_id = env.head.epoch_id.clone(); let height = env.head.height; let em = env.runtime.epoch_manager.read(); let bp = em.get_block_producer_info(&epoch_id, height).unwrap(); let cp = em.get_chunk_producer_info(&epoch_id, height, 0).unwrap(); + let stateless_validators = + em.get_chunk_validator_assignments(&epoch_id, 0, height).ok(); + + if let Some(vs) = stateless_validators { + if vs.contains(&validators[0]) { + expected_endorsements[0] += 1; + } + if vs.contains(&validators[1]) { + expected_endorsements[1] += 1; + } + } if bp.account_id() == "test1" { expected_blocks[0] += 1; @@ -809,13 +824,23 @@ fn test_get_validator_info() { } }; env.step_default(vec![staking_transaction]); - update_validator_stats(&mut env, &mut expected_blocks, &mut expected_chunks); + update_validator_stats( + &mut env, + &mut expected_blocks, + &mut expected_chunks, + &mut expected_endorsements, + ); assert!(env .epoch_manager .get_validator_info(ValidatorInfoIdentifier::EpochId(env.head.epoch_id.clone())) .is_err()); env.step_default(vec![]); - update_validator_stats(&mut env, &mut expected_blocks, &mut expected_chunks); + update_validator_stats( + &mut env, + &mut expected_blocks, + &mut expected_chunks, + &mut expected_endorsements, + ); let mut current_epoch_validator_info = vec![ CurrentEpochValidatorInfo { account_id: "test1".parse().unwrap(), @@ -829,6 +854,10 @@ fn test_get_validator_info() { num_expected_chunks: expected_chunks[0], num_produced_chunks_per_shard: vec![expected_chunks[0]], num_expected_chunks_per_shard: vec![expected_chunks[0]], + num_produced_endorsements: expected_endorsements[0], + num_expected_endorsements: expected_endorsements[0], + num_expected_endorsements_per_shard: vec![expected_endorsements[0]], + num_produced_endorsements_per_shard: vec![expected_endorsements[0]], }, CurrentEpochValidatorInfo { account_id: "test2".parse().unwrap(), @@ -842,6 +871,10 @@ fn test_get_validator_info() { num_expected_chunks: expected_chunks[1], num_produced_chunks_per_shard: vec![expected_chunks[1]], num_expected_chunks_per_shard: vec![expected_chunks[1]], + num_produced_endorsements: expected_endorsements[1], + num_expected_endorsements: expected_endorsements[1], + num_expected_endorsements_per_shard: vec![expected_endorsements[1]], + num_produced_endorsements_per_shard: vec![expected_endorsements[1]], }, ]; let next_epoch_validator_info = vec![ @@ -882,8 +915,14 @@ fn test_get_validator_info() { ); expected_blocks = [0, 0]; expected_chunks = [0, 0]; + expected_endorsements = [0, 0]; env.step_default(vec![]); - update_validator_stats(&mut env, &mut expected_blocks, &mut expected_chunks); + update_validator_stats( + &mut env, + &mut expected_blocks, + &mut expected_chunks, + &mut expected_endorsements, + ); let response = env .epoch_manager .get_validator_info(ValidatorInfoIdentifier::BlockHash(env.head.last_block_hash)) @@ -895,12 +934,24 @@ fn test_get_validator_info() { current_epoch_validator_info[0].num_expected_chunks = expected_chunks[0]; current_epoch_validator_info[0].num_produced_chunks_per_shard = vec![expected_chunks[0]]; current_epoch_validator_info[0].num_expected_chunks_per_shard = vec![expected_chunks[0]]; + current_epoch_validator_info[0].num_produced_endorsements = expected_endorsements[0]; + current_epoch_validator_info[0].num_expected_endorsements = expected_endorsements[0]; + current_epoch_validator_info[0].num_produced_endorsements_per_shard = + vec![expected_endorsements[0]]; + current_epoch_validator_info[0].num_expected_endorsements_per_shard = + vec![expected_endorsements[0]]; current_epoch_validator_info[1].num_produced_blocks = expected_blocks[1]; current_epoch_validator_info[1].num_expected_blocks = expected_blocks[1]; current_epoch_validator_info[1].num_produced_chunks = expected_chunks[1]; current_epoch_validator_info[1].num_expected_chunks = expected_chunks[1]; current_epoch_validator_info[1].num_produced_chunks_per_shard = vec![expected_chunks[1]]; current_epoch_validator_info[1].num_expected_chunks_per_shard = vec![expected_chunks[1]]; + current_epoch_validator_info[1].num_produced_endorsements = expected_endorsements[1]; + current_epoch_validator_info[1].num_expected_endorsements = expected_endorsements[1]; + current_epoch_validator_info[1].num_produced_endorsements_per_shard = + vec![expected_endorsements[1]]; + current_epoch_validator_info[1].num_expected_endorsements_per_shard = + vec![expected_endorsements[1]]; assert_eq!(response.current_validators, current_epoch_validator_info); assert_eq!( response.next_validators, diff --git a/chain/epoch-manager/src/lib.rs b/chain/epoch-manager/src/lib.rs index 7f73fd1b583..06532f4069e 100644 --- a/chain/epoch-manager/src/lib.rs +++ b/chain/epoch-manager/src/lib.rs @@ -17,8 +17,8 @@ use near_primitives::shard_layout::ShardLayout; use near_primitives::stateless_validation::ChunkValidatorAssignments; use near_primitives::types::validator_stake::ValidatorStake; use near_primitives::types::{ - AccountId, ApprovalStake, Balance, BlockChunkValidatorStats, BlockHeight, EpochId, - EpochInfoProvider, NumBlocks, NumSeats, ShardId, ValidatorId, ValidatorInfoIdentifier, + AccountId, ApprovalStake, Balance, BlockChunkValidatorStats, BlockHeight, ChunkValidatorStats, + EpochId, EpochInfoProvider, NumSeats, ShardId, ValidatorId, ValidatorInfoIdentifier, ValidatorKickoutReason, ValidatorStats, }; use near_primitives::version::{ProtocolVersion, UPGRADABILITY_FIX_PROTOCOL_VERSION}; @@ -399,22 +399,22 @@ impl EpochManager { .iter() .map(|(account, stats)| { let production_ratio = - if stats.block_stats.expected == 0 && stats.chunk_stats.expected == 0 { + if stats.block_stats.expected == 0 && stats.chunk_stats.expected() == 0 { Rational64::from_integer(1) } else if stats.block_stats.expected == 0 { Rational64::new( - stats.chunk_stats.produced as i64, - stats.chunk_stats.expected as i64, + stats.chunk_stats.produced() as i64, + stats.chunk_stats.expected() as i64, ) - } else if stats.chunk_stats.expected == 0 { + } else if stats.chunk_stats.expected() == 0 { Rational64::new( stats.block_stats.produced as i64, stats.block_stats.expected as i64, ) } else { (Rational64::new( - stats.chunk_stats.produced as i64, - stats.chunk_stats.expected as i64, + stats.chunk_stats.produced() as i64, + stats.chunk_stats.expected() as i64, ) + Rational64::new( stats.block_stats.produced as i64, stats.block_stats.expected as i64, @@ -463,7 +463,7 @@ impl EpochManager { config: &EpochConfig, epoch_info: &EpochInfo, block_validator_tracker: &HashMap, - chunk_validator_tracker: &HashMap>, + chunk_validator_tracker: &HashMap>, slashed: &HashMap, prev_validator_kickout: &HashMap, ) -> (HashMap, HashMap) @@ -484,11 +484,11 @@ impl EpochManager { .get(&(i as u64)) .unwrap_or(&ValidatorStats { expected: 0, produced: 0 }) .clone(); - let mut chunk_stats = ValidatorStats { produced: 0, expected: 0 }; + let mut chunk_stats = ChunkValidatorStats::default(); for (_, tracker) in chunk_validator_tracker.iter() { if let Some(stat) = tracker.get(&(i as u64)) { - chunk_stats.expected += stat.expected; - chunk_stats.produced += stat.produced; + *chunk_stats.expected_mut() += stat.expected(); + *chunk_stats.produced_mut() += stat.produced(); } } total_stake += v.stake(); @@ -530,13 +530,13 @@ impl EpochManager { }, ); } - if stats.chunk_stats.produced * 100 - < u64::from(chunk_producer_kickout_threshold) * stats.chunk_stats.expected + if stats.chunk_stats.produced() * 100 + < u64::from(chunk_producer_kickout_threshold) * stats.chunk_stats.expected() { validator_kickout.entry(account_id.clone()).or_insert_with(|| { ValidatorKickoutReason::NotEnoughChunks { - produced: stats.chunk_stats.produced, - expected: stats.chunk_stats.expected, + produced: stats.chunk_stats.produced(), + expected: stats.chunk_stats.expected(), } }); } @@ -1346,7 +1346,10 @@ impl EpochManager { .get(info.account_id()) .unwrap_or(&BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 0, expected: 0 }, - chunk_stats: ValidatorStats { produced: 0, expected: 0 }, + chunk_stats: ChunkValidatorStats { + production: ValidatorStats { produced: 0, expected: 0 }, + endorsement: ValidatorStats { produced: 0, expected: 0 }, + }, }); let mut shards = validator_to_shard[validator_id] .iter() @@ -1365,8 +1368,19 @@ impl EpochManager { shards, num_produced_blocks: validator_stats.block_stats.produced, num_expected_blocks: validator_stats.block_stats.expected, - num_produced_chunks: validator_stats.chunk_stats.produced, - num_expected_chunks: validator_stats.chunk_stats.expected, + num_produced_chunks: validator_stats.chunk_stats.produced(), + num_expected_chunks: validator_stats.chunk_stats.expected(), + num_produced_endorsements: validator_stats + .chunk_stats + .endorsement_stats() + .produced, + num_expected_endorsements: validator_stats + .chunk_stats + .endorsement_stats() + .expected, + // Same TODO as above for `num_produced_chunks_per_shard` + num_produced_endorsements_per_shard: Vec::new(), + num_expected_endorsements_per_shard: Vec::new(), }) }) .collect::, EpochError>>()?; @@ -1390,19 +1404,29 @@ impl EpochManager { .unwrap_or(&ValidatorStats { produced: 0, expected: 0 }) .clone(); - let mut chunks_produced_by_shard: HashMap = + let mut chunks_stats_by_shard: HashMap = HashMap::new(); - let mut chunks_expected_by_shard: HashMap = - HashMap::new(); - let mut chunk_stats = ValidatorStats { produced: 0, expected: 0 }; + let mut chunk_stats = ChunkValidatorStats::default(); for (shard, tracker) in aggregator.shard_tracker.iter() { if let Some(stats) = tracker.get(&(validator_id as u64)) { - chunk_stats.produced += stats.produced; - chunk_stats.expected += stats.expected; - *chunks_produced_by_shard.entry(*shard).or_insert(0) += - stats.produced; - *chunks_expected_by_shard.entry(*shard).or_insert(0) += - stats.expected; + let produced = stats.produced(); + let expected = stats.expected(); + let endorsement_stats = stats.endorsement_stats(); + + *chunk_stats.produced_mut() += produced; + *chunk_stats.expected_mut() += expected; + chunk_stats.endorsement_stats_mut().produced += + endorsement_stats.produced; + chunk_stats.endorsement_stats_mut().expected += + endorsement_stats.expected; + + let shard_stats = chunks_stats_by_shard.entry(*shard).or_default(); + *shard_stats.produced_mut() += produced; + *shard_stats.expected_mut() += expected; + shard_stats.endorsement_stats_mut().produced += + endorsement_stats.produced; + shard_stats.endorsement_stats_mut().expected += + endorsement_stats.expected; } } let mut shards = validator_to_shard[validator_id] @@ -1419,15 +1443,41 @@ impl EpochManager { shards: shards.clone(), num_produced_blocks: block_stats.produced, num_expected_blocks: block_stats.expected, - num_produced_chunks: chunk_stats.produced, - num_expected_chunks: chunk_stats.expected, + num_produced_chunks: chunk_stats.produced(), + num_expected_chunks: chunk_stats.expected(), num_produced_chunks_per_shard: shards .iter() - .map(|shard| *chunks_produced_by_shard.entry(*shard).or_default()) + .map(|shard| { + chunks_stats_by_shard + .get(shard) + .map_or(0, |stats| stats.produced()) + }) .collect(), num_expected_chunks_per_shard: shards .iter() - .map(|shard| *chunks_expected_by_shard.entry(*shard).or_default()) + .map(|shard| { + chunks_stats_by_shard + .get(shard) + .map_or(0, |stats| stats.expected()) + }) + .collect(), + num_produced_endorsements: chunk_stats.endorsement_stats().produced, + num_expected_endorsements: chunk_stats.endorsement_stats().expected, + num_produced_endorsements_per_shard: shards + .iter() + .map(|shard| { + chunks_stats_by_shard + .get(shard) + .map_or(0, |stats| stats.endorsement_stats().produced) + }) + .collect(), + num_expected_endorsements_per_shard: shards + .iter() + .map(|shard| { + chunks_stats_by_shard + .get(shard) + .map_or(0, |stats| stats.endorsement_stats().expected) + }) .collect(), }) }) diff --git a/chain/epoch-manager/src/reward_calculator.rs b/chain/epoch-manager/src/reward_calculator.rs index 865e8bb8bea..76665a2eea2 100644 --- a/chain/epoch-manager/src/reward_calculator.rs +++ b/chain/epoch-manager/src/reward_calculator.rs @@ -88,23 +88,77 @@ impl RewardCalculator { let mut epoch_actual_reward = epoch_protocol_treasury; let total_stake: Balance = validator_stake.values().sum(); for (account_id, stats) in validator_block_chunk_stats { - // Uptime is an average of block produced / expected and chunk produced / expected. + // Uptime is an average of block produced / expected, chunk produced / expected, + // and chunk endorsed produced / expected. + + let expected_blocks = stats.block_stats.expected; + let expected_chunks = stats.chunk_stats.expected(); + let expected_endorsements = stats.chunk_stats.endorsement_stats().expected; + let (average_produced_numer, average_produced_denom) = - if stats.block_stats.expected == 0 && stats.chunk_stats.expected == 0 { - (U256::from(0), U256::from(1)) - } else if stats.block_stats.expected == 0 { - (U256::from(stats.chunk_stats.produced), U256::from(stats.chunk_stats.expected)) - } else if stats.chunk_stats.expected == 0 { - (U256::from(stats.block_stats.produced), U256::from(stats.block_stats.expected)) - } else { - ( - U256::from( - stats.block_stats.produced * stats.chunk_stats.expected - + stats.chunk_stats.produced * stats.block_stats.expected, - ), - U256::from(2 * stats.chunk_stats.expected * stats.block_stats.expected), - ) + match (expected_blocks, expected_chunks, expected_endorsements) { + // Validator was not expected to do anything + (0, 0, 0) => (U256::from(0), U256::from(1)), + // Validator was a stateless validator only (not expected to produce anything) + (0, 0, expected_endorsements) => { + let endorsement_stats = stats.chunk_stats.endorsement_stats(); + (U256::from(endorsement_stats.produced), U256::from(expected_endorsements)) + } + // Validator was a chunk-only producer + (0, expected_chunks, 0) => { + (U256::from(stats.chunk_stats.produced()), U256::from(expected_chunks)) + } + // Validator was only a block producer + (expected_blocks, 0, 0) => { + (U256::from(stats.block_stats.produced), U256::from(expected_blocks)) + } + // Validator produced blocks and chunks, but not endorsements + (expected_blocks, expected_chunks, 0) => { + let numer = U256::from( + stats.block_stats.produced * expected_chunks + + stats.chunk_stats.produced() * expected_blocks, + ); + let denom = U256::from(2 * expected_chunks * expected_blocks); + (numer, denom) + } + // Validator produced chunks and endorsements, but not blocks + (0, expected_chunks, expected_endorsements) => { + let endorsement_stats = stats.chunk_stats.endorsement_stats(); + let numer = U256::from( + endorsement_stats.produced * expected_chunks + + stats.chunk_stats.produced() * expected_endorsements, + ); + let denom = U256::from(2 * expected_chunks * expected_endorsements); + (numer, denom) + } + // Validator produced blocks and endorsements, but not chunks + (expected_blocks, 0, expected_endorsements) => { + let endorsement_stats = stats.chunk_stats.endorsement_stats(); + let numer = U256::from( + endorsement_stats.produced * expected_blocks + + stats.block_stats.produced * expected_endorsements, + ); + let denom = U256::from(2 * expected_blocks * expected_endorsements); + (numer, denom) + } + // Validator did all the things + (expected_blocks, expected_chunks, expected_endorsements) => { + let produced_blocks = stats.block_stats.produced; + let produced_chunks = stats.chunk_stats.produced(); + let produced_endorsements = stats.chunk_stats.endorsement_stats().produced; + + let numer = U256::from( + produced_blocks * expected_chunks * expected_endorsements + + produced_chunks * expected_blocks * expected_endorsements + + produced_endorsements * expected_blocks * expected_chunks, + ); + let denom = U256::from( + 3 * expected_chunks * expected_blocks * expected_endorsements, + ); + (numer, denom) + } }; + let online_min_numer = U256::from(*self.online_min_threshold.numer() as u64); let online_min_denom = U256::from(*self.online_min_threshold.denom() as u64); // If average of produced blocks below online min threshold, validator gets 0 reward. @@ -113,13 +167,14 @@ impl RewardCalculator { let reward = if average_produced_numer * online_min_denom < online_min_numer * average_produced_denom || (chunk_only_producers_enabled - && stats.chunk_stats.expected == 0 - && stats.block_stats.expected == 0) + && expected_chunks == 0 + && expected_blocks == 0 + && expected_endorsements == 0) // This is for backwards compatibility. In 2021 December, after we changed to 4 shards, // mainnet was ran without SynchronizeBlockChunkProduction for some time and it's // possible that some validators have expected blocks or chunks to be zero. || (!chunk_only_producers_enabled - && (stats.chunk_stats.expected == 0 || stats.block_stats.expected == 0)) + && (expected_chunks == 0 || expected_blocks == 0)) { 0 } else { @@ -153,7 +208,7 @@ impl RewardCalculator { #[cfg(test)] mod tests { use super::*; - use near_primitives::types::{BlockChunkValidatorStats, ValidatorStats}; + use near_primitives::types::{BlockChunkValidatorStats, ChunkValidatorStats, ValidatorStats}; use near_primitives::version::PROTOCOL_VERSION; use num_rational::Ratio; use std::collections::HashMap; @@ -176,14 +231,14 @@ mod tests { "test1".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 0, expected: 0 }, - chunk_stats: ValidatorStats { produced: 0, expected: 0 }, + chunk_stats: ChunkValidatorStats::default(), }, ), ( "test2".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 0, expected: 1 }, - chunk_stats: ValidatorStats { produced: 0, expected: 1 }, + chunk_stats: ChunkValidatorStats::new_with_production(0, 1), }, ), ]); @@ -227,21 +282,21 @@ mod tests { "test1".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 945, expected: 1000 }, - chunk_stats: ValidatorStats { produced: 945, expected: 1000 }, + chunk_stats: ChunkValidatorStats::new_with_production(945, 1000), }, ), ( "test2".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 999, expected: 1000 }, - chunk_stats: ValidatorStats { produced: 999, expected: 1000 }, + chunk_stats: ChunkValidatorStats::new_with_production(999, 1000), }, ), ( "test3".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 850, expected: 1000 }, - chunk_stats: ValidatorStats { produced: 850, expected: 1000 }, + chunk_stats: ChunkValidatorStats::new_with_production(850, 1000), }, ), ]); @@ -292,7 +347,7 @@ mod tests { "test1".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 945, expected: 1000 }, - chunk_stats: ValidatorStats { produced: 945, expected: 1000 }, + chunk_stats: ChunkValidatorStats::new_with_production(945, 1000), }, ), // chunk only producer @@ -300,7 +355,7 @@ mod tests { "test2".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 0, expected: 0 }, - chunk_stats: ValidatorStats { produced: 999, expected: 1000 }, + chunk_stats: ChunkValidatorStats::new_with_production(999, 1000), }, ), // block only producer (not implemented right now, just for testing) @@ -308,7 +363,7 @@ mod tests { "test3".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 945, expected: 1000 }, - chunk_stats: ValidatorStats { produced: 0, expected: 0 }, + chunk_stats: ChunkValidatorStats::default(), }, ), // a validator that expected blocks and chunks are both 0 (this could occur with very @@ -317,7 +372,7 @@ mod tests { "test4".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 0, expected: 0 }, - chunk_stats: ValidatorStats { produced: 0, expected: 0 }, + chunk_stats: ChunkValidatorStats::default(), }, ), ]); @@ -353,6 +408,92 @@ mod tests { } } + // Test rewards when some validators are only responsible for endorsements + #[test] + fn test_reward_stateless_validation() { + let epoch_length = 1000; + let reward_calculator = RewardCalculator { + max_inflation_rate: Ratio::new(1, 100), + num_blocks_per_year: 1000, + epoch_length, + protocol_reward_rate: Ratio::new(0, 10), + protocol_treasury_account: "near".parse().unwrap(), + online_min_threshold: Ratio::new(9, 10), + online_max_threshold: Ratio::new(99, 100), + num_seconds_per_year: 1000, + }; + let validator_block_chunk_stats = HashMap::from([ + // Blocks, chunks, endorsements + ( + "test1".parse().unwrap(), + BlockChunkValidatorStats { + block_stats: ValidatorStats { produced: 945, expected: 1000 }, + chunk_stats: ChunkValidatorStats { + production: ValidatorStats { produced: 944, expected: 1000 }, + endorsement: ValidatorStats { produced: 946, expected: 1000 }, + }, + }, + ), + // Chunks and endorsements + ( + "test2".parse().unwrap(), + BlockChunkValidatorStats { + block_stats: ValidatorStats { produced: 0, expected: 0 }, + chunk_stats: ChunkValidatorStats { + production: ValidatorStats { produced: 998, expected: 1000 }, + endorsement: ValidatorStats { produced: 1000, expected: 1000 }, + }, + }, + ), + // Blocks and endorsements + ( + "test3".parse().unwrap(), + BlockChunkValidatorStats { + block_stats: ValidatorStats { produced: 940, expected: 1000 }, + chunk_stats: ChunkValidatorStats::new_with_endorsement(950, 1000), + }, + ), + // Endorsements only + ( + "test4".parse().unwrap(), + BlockChunkValidatorStats { + block_stats: ValidatorStats { produced: 0, expected: 0 }, + chunk_stats: ChunkValidatorStats::new_with_endorsement(1000, 1000), + }, + ), + ]); + let validator_stake = HashMap::from([ + ("test1".parse().unwrap(), 500_000), + ("test2".parse().unwrap(), 500_000), + ("test3".parse().unwrap(), 500_000), + ("test4".parse().unwrap(), 500_000), + ]); + let total_supply = 1_000_000_000; + let result = reward_calculator.calculate_reward( + validator_block_chunk_stats, + &validator_stake, + total_supply, + PROTOCOL_VERSION, + PROTOCOL_VERSION, + epoch_length * NUM_NS_IN_SECOND, + ); + // Total reward is 10_000_000. Divided by 4 equal stake validators - each gets 2_500_000. + // test1 with 94.5% online gets 50% because of linear between (0.99-0.9) online. + { + assert_eq!( + result.0, + HashMap::from([ + ("near".parse().unwrap(), 0), + ("test1".parse().unwrap(), 1_250_000u128), + ("test2".parse().unwrap(), 2_500_000u128), + ("test3".parse().unwrap(), 1_250_000u128), + ("test4".parse().unwrap(), 2_500_000u128) + ]) + ); + assert_eq!(result.1, 7_500_000u128); + } + } + /// Test that under an extreme setting (total supply 100b, epoch length half a day), /// reward calculation will not overflow. #[test] @@ -373,7 +514,10 @@ mod tests { "test".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 43200, expected: 43200 }, - chunk_stats: ValidatorStats { produced: 345600, expected: 345600 }, + chunk_stats: ChunkValidatorStats { + production: ValidatorStats { produced: 345600, expected: 345600 }, + endorsement: ValidatorStats { produced: 345600, expected: 345600 }, + }, }, )]); let validator_stake = HashMap::from([("test".parse().unwrap(), 500_000 * 10_u128.pow(24))]); diff --git a/chain/epoch-manager/src/tests/mod.rs b/chain/epoch-manager/src/tests/mod.rs index 4bd4cf05539..f88c07f1a88 100644 --- a/chain/epoch-manager/src/tests/mod.rs +++ b/chain/epoch-manager/src/tests/mod.rs @@ -729,7 +729,7 @@ fn test_validator_reward_one_validator() { "test2".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 1, expected: 1 }, - chunk_stats: ValidatorStats { produced: 1, expected: 1 }, + chunk_stats: ChunkValidatorStats::new_with_production(1, 1), }, ); let mut validator_stakes = HashMap::new(); @@ -819,14 +819,14 @@ fn test_validator_reward_weight_by_stake() { "test1".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 1, expected: 1 }, - chunk_stats: ValidatorStats { produced: 1, expected: 1 }, + chunk_stats: ChunkValidatorStats::new_with_production(1, 1), }, ); validator_online_ratio.insert( "test2".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 1, expected: 1 }, - chunk_stats: ValidatorStats { produced: 1, expected: 1 }, + chunk_stats: ChunkValidatorStats::new_with_production(1, 1), }, ); let mut validators_stakes = HashMap::new(); @@ -942,7 +942,7 @@ fn test_reward_multiple_shards() { "test2".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 1, expected: 1 }, - chunk_stats: ValidatorStats { produced: 1, expected: 1 }, + chunk_stats: ChunkValidatorStats::new_with_production(1, 1), }, ); let mut validators_stakes = HashMap::new(); @@ -2591,18 +2591,18 @@ fn test_validator_kickout_sanity() { ( 0, HashMap::from([ - (0, ValidatorStats { produced: 100, expected: 100 }), - (1, ValidatorStats { produced: 80, expected: 100 }), - (2, ValidatorStats { produced: 70, expected: 100 }), + (0, ChunkValidatorStats::new_with_production(100, 100)), + (1, ChunkValidatorStats::new_with_production(80, 100)), + (2, ChunkValidatorStats::new_with_production(70, 100)), ]), ), ( 1, HashMap::from([ - (0, ValidatorStats { produced: 70, expected: 100 }), - (1, ValidatorStats { produced: 79, expected: 100 }), - (3, ValidatorStats { produced: 100, expected: 100 }), - (4, ValidatorStats { produced: 100, expected: 100 }), + (0, ChunkValidatorStats::new_with_production(70, 100)), + (1, ChunkValidatorStats::new_with_production(79, 100)), + (3, ChunkValidatorStats::new_with_production(100, 100)), + (4, ChunkValidatorStats::new_with_production(100, 100)), ]), ), ]), @@ -2624,35 +2624,35 @@ fn test_validator_kickout_sanity() { "test0".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 100, expected: 100 }, - chunk_stats: ValidatorStats { produced: 170, expected: 200 } + chunk_stats: ChunkValidatorStats::new_with_production(170, 200), } ), ( "test1".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 90, expected: 100 }, - chunk_stats: ValidatorStats { produced: 159, expected: 200 } + chunk_stats: ChunkValidatorStats::new_with_production(159, 200), } ), ( "test2".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 100, expected: 100 }, - chunk_stats: ValidatorStats { produced: 70, expected: 100 } + chunk_stats: ChunkValidatorStats::new_with_production(70, 100), } ), ( "test3".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 89, expected: 100 }, - chunk_stats: ValidatorStats { produced: 100, expected: 100 } + chunk_stats: ChunkValidatorStats::new_with_production(100, 100), } ), ( "test4".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 0, expected: 0 }, - chunk_stats: ValidatorStats { produced: 100, expected: 100 } + chunk_stats: ChunkValidatorStats::new_with_production(100, 100), } ), ]) @@ -2696,15 +2696,15 @@ fn test_max_kickout_stake_ratio() { ( 0, HashMap::from([ - (0, ValidatorStats { produced: 0, expected: 100 }), - (1, ValidatorStats { produced: 0, expected: 100 }), + (0, ChunkValidatorStats::new_with_production(0, 100)), + (1, ChunkValidatorStats::new_with_production(0, 100)), ]), ), ( 1, HashMap::from([ - (2, ValidatorStats { produced: 100, expected: 100 }), - (4, ValidatorStats { produced: 50, expected: 100 }), + (2, ChunkValidatorStats::new_with_production(100, 100)), + (4, ChunkValidatorStats::new_with_production(50, 100)), ]), ), ]); @@ -2735,35 +2735,35 @@ fn test_max_kickout_stake_ratio() { "test0".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 50, expected: 100 }, - chunk_stats: ValidatorStats { produced: 0, expected: 100 }, + chunk_stats: ChunkValidatorStats::new_with_production(0, 100), }, ), ( "test1".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 70, expected: 100 }, - chunk_stats: ValidatorStats { produced: 0, expected: 100 }, + chunk_stats: ChunkValidatorStats::new_with_production(0, 100), }, ), ( "test2".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 70, expected: 100 }, - chunk_stats: ValidatorStats { produced: 100, expected: 100 }, + chunk_stats: ChunkValidatorStats::new_with_production(100, 100), }, ), ( "test3".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 0, expected: 0 }, - chunk_stats: ValidatorStats { produced: 0, expected: 0 }, + chunk_stats: ChunkValidatorStats::default(), }, ), ( "test4".parse().unwrap(), BlockChunkValidatorStats { block_stats: ValidatorStats { produced: 0, expected: 0 }, - chunk_stats: ValidatorStats { produced: 50, expected: 100 }, + chunk_stats: ChunkValidatorStats::new_with_production(50, 100), }, ), ]); diff --git a/chain/epoch-manager/src/tests/random_epochs.rs b/chain/epoch-manager/src/tests/random_epochs.rs index 534a419e975..05f16e0848d 100644 --- a/chain/epoch-manager/src/tests/random_epochs.rs +++ b/chain/epoch-manager/src/tests/random_epochs.rs @@ -347,14 +347,14 @@ fn verify_block_stats( .get(&shard_id) .unwrap() .values() - .map(|value| value.produced) + .map(|value| value.produced()) .sum::(); let sum_expected = aggregator .shard_tracker .get(&shard_id) .unwrap() .values() - .map(|value| value.expected) + .map(|value| value.expected()) .sum::(); assert_eq!(sum_produced, blocks_in_epoch); assert_eq!(sum_expected, blocks_in_epoch_expected); diff --git a/chain/epoch-manager/src/types.rs b/chain/epoch-manager/src/types.rs index 8ded84fb03f..968ac7ecde8 100644 --- a/chain/epoch-manager/src/types.rs +++ b/chain/epoch-manager/src/types.rs @@ -6,7 +6,8 @@ use near_primitives::epoch_manager::epoch_info::EpochInfo; use near_primitives::hash::CryptoHash; use near_primitives::types::validator_stake::ValidatorStake; use near_primitives::types::{ - AccountId, Balance, BlockHeight, EpochId, ShardId, ValidatorId, ValidatorStats, + AccountId, Balance, BlockHeight, ChunkValidatorStats, EpochId, ShardId, ValidatorId, + ValidatorStats, }; use near_primitives::version::ProtocolVersion; use std::collections::{BTreeMap, HashMap}; @@ -59,7 +60,7 @@ pub struct EpochInfoAggregator { /// Map from validator index to (num_blocks_produced, num_blocks_expected) so far in the given epoch. pub block_tracker: HashMap, /// For each shard, a map of validator id to (num_chunks_produced, num_chunks_expected) so far in the given epoch. - pub shard_tracker: HashMap>, + pub shard_tracker: HashMap>, /// Latest protocol version that each validator supports. pub version_tracker: HashMap, /// All proposals in this epoch up to this block. @@ -133,8 +134,12 @@ impl EpochInfoAggregator { } // Step 2: update shard tracker + + // Note: a possible optimization is to access the epoch_manager cache of this value. + let chunk_validator_assignment = epoch_info.sample_chunk_validators(prev_block_height + 1); + for (i, mask) in block_info.chunk_mask().iter().enumerate() { - let chunk_validator_id = EpochManager::chunk_producer_from_info( + let chunk_producer_id = EpochManager::chunk_producer_from_info( epoch_info, prev_block_height + 1, i as ShardId, @@ -142,21 +147,41 @@ impl EpochInfoAggregator { .unwrap(); let tracker = self.shard_tracker.entry(i as ShardId).or_insert_with(HashMap::new); tracker - .entry(chunk_validator_id) + .entry(chunk_producer_id) .and_modify(|stats| { if *mask { - stats.produced += 1; + *stats.produced_mut() += 1; } else { debug!( target: "epoch_tracker", - chunk_validator = ?epoch_info.validator_account_id(chunk_validator_id), + chunk_validator = ?epoch_info.validator_account_id(chunk_producer_id), shard_id = i, block_height = prev_block_height + 1, "Missed chunk"); } - stats.expected += 1; + *stats.expected_mut() += 1; }) - .or_insert(ValidatorStats { produced: u64::from(*mask), expected: 1 }); + .or_insert_with(|| ChunkValidatorStats::new_with_production(u64::from(*mask), 1)); + + let chunk_validators = chunk_validator_assignment + .get(i) + .map_or::<&[(u64, u128)], _>(&[], Vec::as_slice) + .iter() + .map(|(id, _)| *id); + for chunk_validator_id in chunk_validators { + tracker + .entry(chunk_validator_id) + .and_modify(|stats| { + let endorsement_stats = stats.endorsement_stats_mut(); + if *mask { + endorsement_stats.produced += 1; + } + endorsement_stats.expected += 1; + }) + .or_insert_with(|| { + ChunkValidatorStats::new_with_endorsement(u64::from(*mask), 1) + }); + } } // Step 3: update version tracker @@ -264,8 +289,8 @@ impl EpochInfoAggregator { for (chunk_producer_id, stat) in stats.iter() { e.entry(*chunk_producer_id) .and_modify(|entry| { - entry.expected += stat.expected; - entry.produced += stat.produced; + *entry.expected_mut() += stat.expected(); + *entry.produced_mut() += stat.produced(); }) .or_insert_with(|| stat.clone()); } diff --git a/core/primitives/src/types.rs b/core/primitives/src/types.rs index 2481844aad0..c5b84717799 100644 --- a/core/primitives/src/types.rs +++ b/core/primitives/src/types.rs @@ -14,6 +14,10 @@ use serde_with::base64::Base64; use serde_with::serde_as; use std::sync::Arc; +mod chunk_validator_stats; + +pub use chunk_validator_stats::ChunkValidatorStats; + /// Hash used by to store state root. pub type StateRoot = CryptoHash; @@ -915,7 +919,7 @@ pub struct ValidatorStats { #[derive(Debug, BorshSerialize, BorshDeserialize, PartialEq, Eq)] pub struct BlockChunkValidatorStats { pub block_stats: ValidatorStats, - pub chunk_stats: ValidatorStats, + pub chunk_stats: ChunkValidatorStats, } #[derive(serde::Deserialize, Debug, arbitrary::Arbitrary, PartialEq, Eq)] diff --git a/core/primitives/src/types/chunk_validator_stats.rs b/core/primitives/src/types/chunk_validator_stats.rs new file mode 100644 index 00000000000..0879b393247 --- /dev/null +++ b/core/primitives/src/types/chunk_validator_stats.rs @@ -0,0 +1,105 @@ +use { + super::{NumBlocks, ValidatorStats}, + borsh::{self, BorshDeserialize, BorshSerialize}, +}; + +/// An extension to `ValidatorStats` which also tracks endorsements +/// coming from stateless validators. +#[derive(Default, BorshSerialize, BorshDeserialize, Clone, Debug, PartialEq, Eq)] +pub struct ChunkValidatorStats { + pub production: ValidatorStats, + pub endorsement: ValidatorStats, +} + +impl ChunkValidatorStats { + pub const fn new_with_production(produced: u64, expected: u64) -> Self { + ChunkValidatorStats { + production: ValidatorStats { produced, expected }, + endorsement: ValidatorStats { produced: 0, expected: 0 }, + } + } + + pub const fn new_with_endorsement(produced: u64, expected: u64) -> Self { + ChunkValidatorStats { + production: ValidatorStats { produced: 0, expected: 0 }, + endorsement: ValidatorStats { produced, expected }, + } + } + + pub fn produced(&self) -> NumBlocks { + self.production.produced + } + + pub fn expected(&self) -> NumBlocks { + self.production.expected + } + + pub fn produced_mut(&mut self) -> &mut NumBlocks { + &mut self.production.produced + } + + pub fn expected_mut(&mut self) -> &mut NumBlocks { + &mut self.production.expected + } + + pub fn endorsement_stats(&self) -> &ValidatorStats { + &self.endorsement + } + + pub fn endorsement_stats_mut(&mut self) -> &mut ValidatorStats { + &mut self.endorsement + } +} + +#[test] +fn test_mutability() { + let mut stats = ChunkValidatorStats::new_with_production(0, 0); + + *stats.expected_mut() += 1; + assert_eq!(stats, ChunkValidatorStats::new_with_production(0, 1)); + + *stats.produced_mut() += 1; + assert_eq!(stats, ChunkValidatorStats::new_with_production(1, 1)); + + let endorsement_stats = stats.endorsement_stats_mut(); + endorsement_stats.produced += 10; + endorsement_stats.expected += 10; + + assert_eq!( + stats, + ChunkValidatorStats { + production: ValidatorStats { produced: 1, expected: 1 }, + endorsement: ValidatorStats { produced: 10, expected: 10 } + } + ); + + *stats.expected_mut() += 1; + assert_eq!( + stats, + ChunkValidatorStats { + production: ValidatorStats { produced: 1, expected: 2 }, + endorsement: ValidatorStats { produced: 10, expected: 10 } + } + ); + + *stats.produced_mut() += 1; + assert_eq!( + stats, + ChunkValidatorStats { + production: ValidatorStats { produced: 2, expected: 2 }, + endorsement: ValidatorStats { produced: 10, expected: 10 } + } + ); + + let endorsement_stats = stats.endorsement_stats_mut(); + endorsement_stats.produced += 10; + endorsement_stats.expected += 10; + + assert_eq!( + stats, + ChunkValidatorStats { + production: ValidatorStats { produced: 2, expected: 2 }, + endorsement: ValidatorStats { produced: 20, expected: 20 } + } + ); +} diff --git a/core/primitives/src/views.rs b/core/primitives/src/views.rs index e365a19d461..9c782e15a08 100644 --- a/core/primitives/src/views.rs +++ b/core/primitives/src/views.rs @@ -2109,6 +2109,18 @@ pub struct CurrentEpochValidatorInfo { pub num_produced_chunks_per_shard: Vec, #[serde(default)] pub num_expected_chunks_per_shard: Vec, + #[serde(default, skip_serializing_if = "num_blocks_is_zero")] + pub num_produced_endorsements: NumBlocks, + #[serde(default, skip_serializing_if = "num_blocks_is_zero")] + pub num_expected_endorsements: NumBlocks, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub num_produced_endorsements_per_shard: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub num_expected_endorsements_per_shard: Vec, +} + +fn num_blocks_is_zero(n: &NumBlocks) -> bool { + n == &0 } #[derive( diff --git a/core/store/src/metadata.rs b/core/store/src/metadata.rs index 30a25a20ef3..dbbc0932fae 100644 --- a/core/store/src/metadata.rs +++ b/core/store/src/metadata.rs @@ -2,7 +2,7 @@ pub type DbVersion = u32; /// Current version of the database. -pub const DB_VERSION: DbVersion = 38; +pub const DB_VERSION: DbVersion = 39; /// Database version at which point DbKind was introduced. const DB_VERSION_WITH_KIND: DbVersion = 34; diff --git a/core/store/src/migrations.rs b/core/store/src/migrations.rs index a43a9b3e630..484fc962b25 100644 --- a/core/store/src/migrations.rs +++ b/core/store/src/migrations.rs @@ -1,10 +1,19 @@ use crate::metadata::DbKind; use crate::{DBCol, Store, StoreUpdate}; use borsh::{BorshDeserialize, BorshSerialize}; +use near_primitives::epoch_manager::epoch_info::EpochSummary; +use near_primitives::epoch_manager::AGGREGATOR_KEY; +use near_primitives::hash::CryptoHash; use near_primitives::state::FlatStateValue; use near_primitives::transaction::{ExecutionOutcomeWithIdAndProof, ExecutionOutcomeWithProof}; +use near_primitives::types::{ + validator_stake::ValidatorStake, AccountId, EpochId, ShardId, ValidatorId, + ValidatorKickoutReason, ValidatorStats, +}; +use near_primitives::types::{BlockChunkValidatorStats, ChunkValidatorStats}; use near_primitives::utils::get_outcome_id_block_hash; -use std::collections::HashMap; +use near_primitives::version::ProtocolVersion; +use std::collections::{BTreeMap, HashMap}; use tracing::info; pub struct BatchedStoreUpdate<'a> { @@ -229,3 +238,111 @@ pub fn migrate_37_to_38(store: &Store) -> anyhow::Result<()> { update.commit()?; Ok(()) } + +/// Migrates the database from version 38 to 39. +/// +/// Rewrites Epoch summary to include endorsement stats. +pub fn migrate_38_to_39(store: &Store) -> anyhow::Result<()> { + #[derive(BorshSerialize, BorshDeserialize)] + struct EpochInfoAggregator { + /// Map from validator index to (num_blocks_produced, num_blocks_expected) so far in the given epoch. + pub block_tracker: HashMap, + /// For each shard, a map of validator id to (num_chunks_produced, num_chunks_expected) so far in the given epoch. + pub shard_tracker: HashMap>, + /// Latest protocol version that each validator supports. + pub version_tracker: HashMap, + /// All proposals in this epoch up to this block. + pub all_proposals: BTreeMap, + /// Id of the epoch that this aggregator is in. + pub epoch_id: EpochId, + /// Last block hash recorded. + pub last_block_hash: CryptoHash, + } + + type LegacyEpochInfoAggregator = EpochInfoAggregator; + type NewEpochInfoAggregator = EpochInfoAggregator; + + #[derive(BorshDeserialize)] + struct LegacyBlockChunkValidatorStats { + pub block_stats: ValidatorStats, + pub chunk_stats: ValidatorStats, + } + + #[derive(BorshDeserialize)] + struct LegacyEpochSummary { + pub prev_epoch_last_block_hash: CryptoHash, + /// Proposals from the epoch, only the latest one per account + pub all_proposals: Vec, + /// Kickout set, includes slashed + pub validator_kickout: HashMap, + /// Only for validators who met the threshold and didn't get slashed + pub validator_block_chunk_stats: HashMap, + /// Protocol version for next epoch. + pub next_version: ProtocolVersion, + } + + let mut update = store.store_update(); + + // Update EpochInfoAggregator + let maybe_legacy_aggregator: Option = + store.get_ser(DBCol::EpochInfo, AGGREGATOR_KEY)?; + if let Some(legacy_aggregator) = maybe_legacy_aggregator { + let new_aggregator = NewEpochInfoAggregator { + block_tracker: legacy_aggregator.block_tracker, + shard_tracker: legacy_aggregator + .shard_tracker + .into_iter() + .map(|(shard_id, legacy_stats)| { + let new_stats = legacy_stats + .into_iter() + .map(|(validator_id, stats)| { + ( + validator_id, + ChunkValidatorStats::new_with_production( + stats.produced, + stats.expected, + ), + ) + }) + .collect(); + (shard_id, new_stats) + }) + .collect(), + version_tracker: legacy_aggregator.version_tracker, + all_proposals: legacy_aggregator.all_proposals, + epoch_id: legacy_aggregator.epoch_id, + last_block_hash: legacy_aggregator.last_block_hash, + }; + update.set_ser(DBCol::EpochInfo, AGGREGATOR_KEY, &new_aggregator)?; + } + + // Update EpochSummary + for result in store.iter(DBCol::EpochValidatorInfo) { + let (key, old_value) = result?; + let legacy_summary = LegacyEpochSummary::try_from_slice(&old_value)?; + let new_value = EpochSummary { + prev_epoch_last_block_hash: legacy_summary.prev_epoch_last_block_hash, + all_proposals: legacy_summary.all_proposals, + validator_kickout: legacy_summary.validator_kickout, + validator_block_chunk_stats: legacy_summary + .validator_block_chunk_stats + .into_iter() + .map(|(account_id, stats)| { + let new_stats = BlockChunkValidatorStats { + block_stats: stats.block_stats, + chunk_stats: ChunkValidatorStats::new_with_production( + stats.chunk_stats.produced, + stats.chunk_stats.expected, + ), + }; + (account_id, new_stats) + }) + .collect(), + next_version: legacy_summary.next_version, + }; + update.set(DBCol::EpochValidatorInfo, &key, &borsh::to_vec(&new_value)?); + } + + update.commit()?; + Ok(()) +} diff --git a/nearcore/src/migrations.rs b/nearcore/src/migrations.rs index a63803fc0f0..bfccff49025 100644 --- a/nearcore/src/migrations.rs +++ b/nearcore/src/migrations.rs @@ -87,6 +87,7 @@ impl<'a> near_store::StoreMigrator for Migrator<'a> { } 36 => near_store::migrations::migrate_36_to_37(store), 37 => near_store::migrations::migrate_37_to_38(store), + 38 => near_store::migrations::migrate_38_to_39(store), DB_VERSION.. => unreachable!(), } }