Restaking service + validator RPC improvement (#1962)

* Adding restaking tool. Modify RPC for validators to show produced/expected. Added author to Chunk and Block

Validators RPC shows shards/pk, network info RPC shows peer ids for known validators. Make stake naming consistent

Print which test is runned before run it (#1889)

* Print which test is runned before run it

* adding a timeout

Add transaction to tx rpc (#1865)

* Add transaction to tx rpc

* address comments

* fix

Run test as non root in docker and check binary exist before mount to docker (#1891)

* Run test as non root in docker, check is file before mount to docker

Fix stake return for double sign slashing (#1877)

Addressing comments & fixing fmt

Improve naming for stats in the epoch manager. Fix some comments in restaking service

Add test for restaked

* Fixes

Cargo.toml

@@ -33,6 +33,7 @@ members = [
     "genesis-tools/genesis-csv-to-json",
     "genesis-tools/genesis-populate",
     "genesis-tools/keypair-generator",
+    "tools/restaked"
 ]
 
 [dev-dependencies]

chain/chain/src/chain.rs

@@ -65,6 +65,9 @@ pub const TX_ROUTING_HEIGHT_HORIZON: BlockHeightDelta = 4;
 /// The multiplier of the stake percentage when computing block weight
 pub const WEIGHT_MULTIPLIER: u128 = 1_000_000_000;
 
+/// Private constant for 1 NEAR (copy from near/config.rs) used for reporting.
+const NEAR_BASE: Balance = 1_000_000_000_000_000_000_000_000;
+
 /// Block economics config taken from genesis config
 pub struct BlockEconomicsConfig {
     pub gas_price_adjustment_rate: u8,
@@ -372,7 +375,7 @@ impl Chain {
         for bp in bps.iter() {
             arr.append(&mut hash(bp.account_id.as_bytes()).into());
             arr.append(&mut hash(bp.public_key.try_to_vec()?.as_ref()).into());
-            arr.append(&mut hash(bp.amount.try_to_vec()?.as_ref()).into());
+            arr.append(&mut hash(bp.stake.try_to_vec()?.as_ref()).into());
         }
 
         Ok(hash(&arr))
@@ -911,15 +914,13 @@ impl Chain {
                     }
                     None => {}
                 }
-                // Sum validator balances in full NEARs (divided by 10**18)
+                // Sum validator balances in full NEARs (divided by 10**24)
                 let sum = block
                     .header
                     .inner_rest
                     .validator_proposals
                     .iter()
-                    .map(|validator_stake| {
-                        (validator_stake.amount / 1_000_000_000_000_000_000) as i64
-                    })
+                    .map(|validator_stake| (validator_stake.stake / NEAR_BASE) as i64)
                     .sum::<i64>();
                 near_metrics::set_gauge(&metrics::VALIDATOR_AMOUNT_STAKED, sum);
 

chain/chain/src/finality.rs

@@ -193,7 +193,7 @@ impl FinalityGadget {
         let mut stake_surrounding_no_quorum = 0 as Balance;
 
         let account_id_to_stake =
-            stakes.iter().map(|x| (&x.account_id, x.amount)).collect::<HashMap<_, _>>();
+            stakes.iter().map(|x| (&x.account_id, x.stake)).collect::<HashMap<_, _>>();
         assert!(account_id_to_stake.len() == stakes.len());
         let threshold = account_id_to_stake.values().sum::<u128>() * 2u128 / 3u128;
 

chain/chain/src/test_utils.rs

@@ -3,17 +3,11 @@ use std::collections::{HashMap, HashSet};
 use std::convert::TryFrom;
 use std::sync::{Arc, RwLock};
 
-use serde::Serialize;
-
 use borsh::{BorshDeserialize, BorshSerialize};
 use chrono::Utc;
 use log::debug;
+use serde::Serialize;
 
-use crate::chain::WEIGHT_MULTIPLIER;
-use crate::error::{Error, ErrorKind};
-use crate::store::ChainStoreAccess;
-use crate::types::{ApplyTransactionResult, BlockHeader, RuntimeAdapter};
-use crate::{Chain, ChainGenesis};
 use near_crypto::{InMemorySigner, KeyType, PublicKey, SecretKey, Signature, Signer};
 use near_pool::types::PoolIterator;
 use near_primitives::account::{AccessKey, Account};
@@ -30,7 +24,7 @@ use near_primitives::transaction::{
 };
 use near_primitives::types::{
     AccountId, Balance, BlockHeight, EpochId, Gas, Nonce, NumBlocks, ShardId, StateRoot,
-    StateRootNode, ValidatorStake,
+    StateRootNode, ValidatorStake, ValidatorStats,
 };
 use near_primitives::views::{
     AccessKeyInfoView, AccessKeyList, EpochValidatorInfo, QueryResponse, QueryResponseKind,
@@ -40,6 +34,12 @@ use near_store::{
     ColBlockHeader, PartialStorage, Store, StoreUpdate, Trie, TrieChanges, WrappedTrieChanges,
 };
 
+use crate::chain::{Chain, ChainGenesis, WEIGHT_MULTIPLIER};
+use crate::error::{Error, ErrorKind};
+use crate::store::ChainStoreAccess;
+use crate::types::ApplyTransactionResult;
+use crate::{BlockHeader, RuntimeAdapter};
+
 #[derive(
     BorshSerialize, BorshDeserialize, Serialize, Hash, PartialEq, Eq, Ord, PartialOrd, Clone, Debug,
 )]
@@ -140,7 +140,7 @@ impl KeyValueRuntime {
                             account_id: account_id.clone(),
                             public_key: SecretKey::from_seed(KeyType::ED25519, account_id)
                                 .public_key(),
-                            amount: 1_000_000,
+                            stake: 1_000_000,
                         })
                         .collect()
                 })
@@ -325,13 +325,13 @@ impl RuntimeAdapter for KeyValueRuntime {
         Ok(validators[offset + delta].account_id.clone())
     }
 
-    fn get_num_missing_blocks(
+    fn get_num_validator_blocks(
         &self,
         _epoch_id: &EpochId,
         _last_known_block_hash: &CryptoHash,
         _account_id: &AccountId,
-    ) -> Result<u64, Error> {
-        Ok(0)
+    ) -> Result<ValidatorStats, Error> {
+        Ok(ValidatorStats { produced: 0, expected: 0 })
     }
 
     fn num_shards(&self) -> ShardId {

chain/chain/src/types.rs

@@ -1,28 +1,29 @@
+use std::cmp::{max, Ordering};
 use std::collections::HashMap;
 
 use borsh::{BorshDeserialize, BorshSerialize};
+use serde::Serialize;
+
 use near_crypto::Signature;
+use near_pool::types::PoolIterator;
 use near_primitives::block::{Approval, WeightAndScore};
 pub use near_primitives::block::{Block, BlockHeader, Weight};
 use near_primitives::challenge::{ChallengesResult, SlashedValidator};
+use near_primitives::errors::InvalidTxError;
 use near_primitives::hash::{hash, CryptoHash};
 use near_primitives::merkle::{merklize, MerklePath};
 use near_primitives::receipt::Receipt;
 use near_primitives::sharding::{ReceiptProof, ShardChunk, ShardChunkHeader};
 use near_primitives::transaction::{ExecutionOutcomeWithId, SignedTransaction};
 use near_primitives::types::{
     AccountId, Balance, BlockHeight, EpochId, Gas, MerkleHash, ShardId, StateRoot, StateRootNode,
-    ValidatorStake,
+    ValidatorStake, ValidatorStats,
 };
 use near_primitives::views::{EpochValidatorInfo, QueryResponse};
 use near_store::{PartialStorage, StoreUpdate, WrappedTrieChanges};
-use serde::Serialize;
 
 use crate::chain::WEIGHT_MULTIPLIER;
 use crate::error::Error;
-use near_pool::types::PoolIterator;
-use near_primitives::errors::InvalidTxError;
-use std::cmp::{max, Ordering};
 
 #[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, Serialize)]
 pub struct ReceiptResponse(pub CryptoHash, pub Vec<Receipt>);
@@ -225,12 +226,12 @@ pub trait RuntimeAdapter: Send + Sync {
     ) -> Result<(ValidatorStake, bool), Error>;
 
     /// Number of missed blocks for given block producer.
-    fn get_num_missing_blocks(
+    fn get_num_validator_blocks(
         &self,
         epoch_id: &EpochId,
         last_known_block_hash: &CryptoHash,
         account_id: &AccountId,
-    ) -> Result<u64, Error>;
+    ) -> Result<ValidatorStats, Error>;
 
     /// Get current number of shards.
     fn num_shards(&self) -> ShardId;
@@ -456,8 +457,8 @@ impl dyn RuntimeAdapter {
         let mut total_stake = 0;
 
         for bp in block_producers {
-            account_to_stake.insert(bp.0.account_id, bp.0.amount);
-            total_stake += bp.0.amount;
+            account_to_stake.insert(bp.0.account_id, bp.0.stake);
+            total_stake += bp.0.stake;
         }
 
         for approval in approvals {
@@ -555,12 +556,13 @@ mod tests {
 
     use near_crypto::{InMemorySigner, KeyType, Signer};
     use near_primitives::block::genesis_chunks;
-
-    use super::*;
-    use crate::Chain;
     use near_primitives::merkle::verify_path;
     use near_primitives::transaction::{ExecutionOutcome, ExecutionStatus};
 
+    use crate::Chain;
+
+    use super::*;
+
     #[test]
     fn test_block_produce() {
         let num_shards = 32;

chain/chain/tests/finality.rs

@@ -24,7 +24,7 @@ fn compute_quorums_slow(
     let mut all_heights_and_hashes = vec![];
 
     let account_id_to_stake =
-        stakes.iter().map(|x| (&x.account_id, x.amount)).collect::<HashMap<_, _>>();
+        stakes.iter().map(|x| (&x.account_id, x.stake)).collect::<HashMap<_, _>>();
     assert!(account_id_to_stake.len() == stakes.len());
     let threshold = account_id_to_stake.values().sum::<u128>() * 2u128 / 3u128;
 
@@ -300,7 +300,7 @@ fn test_finality_basic() {
 fn test_finality_weight() {
     let (mut chain, _, signer) = setup();
     let mut stakes = gen_stakes(4);
-    stakes[0].amount = 8;
+    stakes[0].stake = 8;
 
     let genesis_block = chain.get_block(&chain.genesis().hash()).unwrap().clone();
 

chain/client/src/client_actor.rs

@@ -508,11 +508,12 @@ impl ClientActor {
             }
         } else {
             let num_blocks_missing = if head.epoch_id == epoch_id {
-                self.client.runtime_adapter.get_num_missing_blocks(
+                let validator_stats = self.client.runtime_adapter.get_num_validator_blocks(
                     &head.epoch_id,
                     &head.last_block_hash,
                     &next_block_producer_account,
-                )?
+                )?;
+                validator_stats.expected - validator_stats.produced
             } else {
                 0
             };

chain/client/src/info.rs

@@ -156,7 +156,7 @@ fn try_sign_json(
 fn display_sync_status(sync_status: &SyncStatus, head: &Tip) -> String {
     match sync_status {
         SyncStatus::AwaitingPeers => format!("#{:>8} Waiting for peers", head.height),
-        SyncStatus::NoSync => format!("#{:>8} {}", head.height, head.last_block_hash),
+        SyncStatus::NoSync => format!("#{:>8} {:>44}", head.height, head.last_block_hash),
         SyncStatus::HeaderSync { current_height, highest_height } => {
             let percent = if *highest_height == 0 {
                 0

chain/client/src/types.rs

@@ -9,12 +9,12 @@ use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 
 use near_crypto::{InMemorySigner, Signer};
-use near_network::types::AccountOrPeerIdOrHash;
+use near_network::types::{AccountOrPeerIdOrHash, KnownProducer};
 use near_network::PeerInfo;
 use near_primitives::hash::CryptoHash;
 use near_primitives::sharding::ChunkHash;
 use near_primitives::types::{
-    AccountId, BlockHeight, BlockHeightDelta, NumBlocks, NumSeats, ShardId, Version,
+    AccountId, BlockHeight, BlockHeightDelta, MaybeBlockId, NumBlocks, NumSeats, ShardId, Version,
 };
 use near_primitives::utils::generate_random_string;
 use near_primitives::views::{
@@ -343,10 +343,8 @@ impl Message for GetNetworkInfo {
     type Result = Result<NetworkInfoResponse, String>;
 }
 
-pub enum GetGasPrice {
-    Height(BlockHeight),
-    Hash(CryptoHash),
-    None,
+pub struct GetGasPrice {
+    pub block_id: MaybeBlockId,
 }
 
 impl Message for GetGasPrice {
@@ -361,7 +359,7 @@ pub struct NetworkInfoResponse {
     pub sent_bytes_per_sec: u64,
     pub received_bytes_per_sec: u64,
     /// Accounts of known block and chunk producers from routing table.
-    pub known_producers: Vec<AccountId>,
+    pub known_producers: Vec<KnownProducer>,
 }
 
 /// Status of given transaction including all the subsequent receipts.
@@ -375,7 +373,7 @@ impl Message for TxStatus {
 }
 
 pub struct GetValidatorInfo {
-    pub last_block_hash: CryptoHash,
+    pub block_id: MaybeBlockId,
 }
 
 impl Message for GetValidatorInfo {

chain/client/src/view_client.rs

@@ -7,7 +7,7 @@ use actix::{Actor, Context, Handler};
 use log::{error, warn};
 
 use near_chain::{Chain, ChainGenesis, ChainStoreAccess, ErrorKind, RuntimeAdapter};
-use near_primitives::types::AccountId;
+use near_primitives::types::{AccountId, BlockId, MaybeBlockId};
 use near_primitives::views::{
     BlockView, ChunkView, EpochValidatorInfo, FinalExecutionOutcomeView, FinalExecutionStatus,
     GasPriceView, LightClientBlockView, QueryResponse,
@@ -75,6 +75,17 @@ impl ViewClientActor {
         })
     }
 
+    fn maybe_block_id_to_block_hash(
+        &mut self,
+        block_id: MaybeBlockId,
+    ) -> Result<CryptoHash, near_chain::Error> {
+        match block_id {
+            None => Ok(self.chain.head()?.last_block_hash),
+            Some(BlockId::Height(height)) => Ok(self.chain.get_header_by_height(height)?.hash()),
+            Some(BlockId::Hash(block_hash)) => Ok(block_hash),
+        }
+    }
+
     fn need_request<K: Hash + Eq + Clone>(key: K, cache: &mut SizedCache<K, Instant>) -> bool {
         let now = Instant::now();
         let need_request = match cache.cache_get(&key) {
@@ -281,7 +292,14 @@ impl Handler<GetBlock> for ViewClientActor {
             GetBlock::Height(height) => self.chain.get_block_by_height(height).map(Clone::clone),
             GetBlock::Hash(hash) => self.chain.get_block(&hash).map(Clone::clone),
         }
-        .map(|block| block.into())
+        .and_then(|block| {
+            self.runtime_adapter
+                .get_block_producer(
+                    &block.header.inner_lite.epoch_id,
+                    block.header.inner_lite.height,
+                )
+                .map(|author| BlockView::from_author_block(author, block))
+        })
         .map_err(|err| err.to_string())
     }
 }
@@ -317,7 +335,20 @@ impl Handler<GetChunk> for ViewClientActor {
                 })
             }
         }
-        .map(|chunk| chunk.into())
+        .and_then(|chunk| {
+            self.chain
+                .get_block_by_height(chunk.header.height_included)
+                .map(|block| (block.header.inner_lite.epoch_id.clone(), chunk))
+        })
+        .and_then(|(epoch_id, chunk)| {
+            self.runtime_adapter
+                .get_chunk_producer(
+                    &epoch_id,
+                    chunk.header.inner.height_created,
+                    chunk.header.inner.shard_id,
+                )
+                .map(|author| ChunkView::from_author_chunk(author, chunk))
+        })
         .map_err(|err| err.to_string())
     }
 }
@@ -334,7 +365,9 @@ impl Handler<GetValidatorInfo> for ViewClientActor {
     type Result = Result<EpochValidatorInfo, String>;
 
     fn handle(&mut self, msg: GetValidatorInfo, _: &mut Context<Self>) -> Self::Result {
-        self.runtime_adapter.get_validator_info(&msg.last_block_hash).map_err(|e| e.to_string())
+        self.maybe_block_id_to_block_hash(msg.block_id)
+            .and_then(|block_hash| self.runtime_adapter.get_validator_info(&block_hash))
+            .map_err(|err| err.to_string())
     }
 }
 
@@ -547,11 +580,9 @@ impl Handler<GetGasPrice> for ViewClientActor {
     type Result = Result<GasPriceView, String>;
 
     fn handle(&mut self, msg: GetGasPrice, _ctx: &mut Self::Context) -> Self::Result {
-        let header = match msg {
-            GetGasPrice::None => self.chain.head_header(),
-            GetGasPrice::Height(height) => self.chain.get_header_by_height(height),
-            GetGasPrice::Hash(block_hash) => self.chain.get_block_header(&block_hash),
-        };
+        let header = self
+            .maybe_block_id_to_block_hash(msg.block_id)
+            .and_then(|block_hash| self.chain.get_block_header(&block_hash));
         header
             .map(|b| GasPriceView { gas_price: b.inner_rest.gas_price })
             .map_err(|e| e.to_string())