Skip to content
Permalink
Browse files

Merklize execution outcomes (#1627)

* Merklize the ExecutionOutcome and add the root to the ChunkHeader and BlockHeader

* Store outcomes with proofs and return proofs when outcomes are queried
  • Loading branch information
ilblackdragon authored and SkidanovAlex committed Nov 3, 2019
1 parent 9315a50 commit 7214e61418bf2c26f7096c3765afc93889d2b538
@@ -17,7 +17,7 @@ use near_primitives::receipt::Receipt;
use near_primitives::sharding::{
ChunkHash, ChunkHashHeight, ReceiptProof, ShardChunk, ShardChunkHeader, ShardProof,
};
use near_primitives::transaction::{ExecutionOutcome, ExecutionStatus};
use near_primitives::transaction::{ExecutionOutcome, ExecutionOutcomeWithProof, ExecutionStatus};
use near_primitives::types::{
AccountId, Balance, BlockExtra, BlockIndex, ChunkExtra, EpochId, Gas, ShardId,
};
@@ -33,9 +33,10 @@ use crate::error::{Error, ErrorKind};
use crate::metrics;
use crate::store::{ChainStore, ChainStoreAccess, ChainStoreUpdate, ShardInfo, StateSyncInfo};
use crate::types::{
AcceptedBlock, Block, BlockHeader, BlockStatus, Provenance, ReceiptList, ReceiptProofResponse,
ReceiptResponse, RootProof, RuntimeAdapter, ShardStateSyncResponseHeader,
ShardStateSyncResponsePart, StateHeaderKey, Tip, ValidatorSignatureVerificationResult,
AcceptedBlock, ApplyTransactionResult, Block, BlockHeader, BlockStatus, Provenance,
ReceiptList, ReceiptProofResponse, ReceiptResponse, RootProof, RuntimeAdapter,
ShardStateSyncResponseHeader, ShardStateSyncResponsePart, StateHeaderKey, Tip,
ValidatorSignatureVerificationResult,
};
use crate::validate::{validate_challenge, validate_chunk_proofs, validate_chunk_with_chunk_extra};

@@ -271,6 +272,7 @@ impl Chain {
chunk_header.inner.shard_id,
ChunkExtra::new(
state_root,
CryptoHash::default(),
vec![],
0,
chain_genesis.gas_limit,
@@ -1402,13 +1404,17 @@ impl Chain {
&mut self,
hash: &CryptoHash,
) -> Result<ExecutionOutcomeView, String> {
match self.get_transaction_result(hash) {
match self.get_execution_outcome(hash) {
Ok(result) => Ok(result.clone().into()),
Err(err) => match err.kind() {
ErrorKind::DBNotFoundErr(_) => {
Ok(ExecutionOutcome { status: ExecutionStatus::Unknown, ..Default::default() }
.into())
ErrorKind::DBNotFoundErr(_) => Ok(ExecutionOutcomeWithProof {
outcome: ExecutionOutcome {
status: ExecutionStatus::Unknown,
..Default::default()
},
..Default::default()
}
.into()),
_ => Err(err.to_string()),
},
}
@@ -1577,11 +1583,11 @@ impl Chain {

/// Get transaction result for given hash of transaction.
#[inline]
pub fn get_transaction_result(
pub fn get_execution_outcome(
&mut self,
hash: &CryptoHash,
) -> Result<&ExecutionOutcome, Error> {
self.store.get_transaction_result(hash)
) -> Result<&ExecutionOutcomeWithProof, Error> {
self.store.get_execution_outcome(hash)
}

/// Returns underlying ChainStore.
@@ -1941,13 +1947,17 @@ impl<'a> ChainUpdate<'a> {
)
.map_err(|e| ErrorKind::Other(e.to_string()))?;

let (outcome_root, outcome_paths) =
ApplyTransactionResult::compute_outcomes_proof(&apply_result.outcomes);

self.chain_store_update.save_trie_changes(apply_result.trie_changes);
// Save state root after applying transactions.
self.chain_store_update.save_chunk_extra(
&block.hash(),
shard_id,
ChunkExtra::new(
&apply_result.new_root,
outcome_root,
apply_result.validator_proposals,
apply_result.total_gas_burnt,
gas_limit,
@@ -1970,10 +1980,8 @@ impl<'a> ChainUpdate<'a> {
outgoing_receipts,
);
// Save receipt and transaction results.
for tx_result in apply_result.transaction_results {
self.chain_store_update
.save_transaction_result(&tx_result.id, tx_result.outcome);
}
self.chain_store_update
.save_outcomes_with_proofs(apply_result.outcomes, outcome_paths);
} else {
let mut new_extra = self
.chain_store_update
@@ -2482,11 +2490,15 @@ impl<'a> ChainUpdate<'a> {
&block_header.inner.challenges_result,
)?;

let (outcome_root, outcome_proofs) =
ApplyTransactionResult::compute_outcomes_proof(&apply_result.outcomes);

self.chain_store_update.save_chunk(&chunk.chunk_hash, chunk.clone());

self.chain_store_update.save_trie_changes(apply_result.trie_changes);
let chunk_extra = ChunkExtra::new(
&apply_result.new_root,
outcome_root,
apply_result.validator_proposals,
apply_result.total_gas_burnt,
gas_limit,
@@ -2507,9 +2519,7 @@ impl<'a> ChainUpdate<'a> {
outgoing_receipts,
);
// Saving transaction results.
for tx_result in apply_result.transaction_results {
self.chain_store_update.save_transaction_result(&tx_result.id, tx_result.outcome);
}
self.chain_store_update.save_outcomes_with_proofs(apply_result.outcomes, outcome_proofs);
// Saving all incoming receipts.
for receipt_proof_response in incoming_receipts_proofs {
self.chain_store_update.save_incoming_receipt(
@@ -64,6 +64,9 @@ pub enum ErrorKind {
/// Invalid receipts proof.
#[fail(display = "Invalid Receipts Proof")]
InvalidReceiptsProof,
/// Invalid outcomes proof.
#[fail(display = "Invalid Outcomes Proof")]
InvalidOutcomesProof,
/// Invalid state payload on state sync.
#[fail(display = "Invalid State Payload")]
InvalidStatePayload,
@@ -182,6 +185,7 @@ impl Error {
| ErrorKind::InvalidStateRoot
| ErrorKind::InvalidTxRoot
| ErrorKind::InvalidChunkReceiptsRoot
| ErrorKind::InvalidOutcomesProof
| ErrorKind::InvalidChunkHeadersRoot
| ErrorKind::InvalidChunkTxRoot
| ErrorKind::InvalidReceiptsProof
@@ -13,7 +13,7 @@ use near_primitives::receipt::Receipt;
use near_primitives::sharding::{
ChunkHash, ChunkOnePart, EncodedShardChunk, ReceiptProof, ShardChunk, ShardChunkHeader,
};
use near_primitives::transaction::ExecutionOutcome;
use near_primitives::transaction::{ExecutionOutcomeWithId, ExecutionOutcomeWithProof};
use near_primitives::types::{BlockExtra, BlockIndex, ChunkExtra, ShardId};
use near_primitives::utils::{index_to_bytes, to_timestamp};
use near_store::{
@@ -27,6 +27,7 @@ use crate::byzantine_assert;
use crate::error::{Error, ErrorKind};
use crate::types::{Block, BlockHeader, LatestKnown, ReceiptProofResponse, ReceiptResponse, Tip};
use near_primitives::errors::InvalidTxError;
use near_primitives::merkle::MerklePath;

const HEAD_KEY: &[u8; 4] = b"HEAD";
const TAIL_KEY: &[u8; 4] = b"TAIL";
@@ -245,8 +246,11 @@ pub trait ChainStoreAccess {
hash: &CryptoHash,
shard_id: ShardId,
) -> Result<&Vec<ReceiptProof>, Error>;
/// Returns transaction result for given tx hash.
fn get_transaction_result(&mut self, hash: &CryptoHash) -> Result<&ExecutionOutcome, Error>;
/// Returns transaction and receipt outcome for given hash.
fn get_execution_outcome(
&mut self,
hash: &CryptoHash,
) -> Result<&ExecutionOutcomeWithProof, Error>;
/// Returns whether the block with the given hash was challenged
fn is_block_challenged(&mut self, hash: &CryptoHash) -> Result<bool, Error>;

@@ -293,7 +297,7 @@ pub struct ChainStore {
/// Cache with incoming receipts.
incoming_receipts: SizedCache<Vec<u8>, Vec<ReceiptProof>>,
/// Cache transaction statuses.
transaction_results: SizedCache<Vec<u8>, ExecutionOutcome>,
outcomes: SizedCache<Vec<u8>, ExecutionOutcomeWithProof>,
/// Invalid chunks.
invalid_chunks: SizedCache<Vec<u8>, EncodedShardChunk>,
}
@@ -322,7 +326,7 @@ impl ChainStore {
block_hash_per_height: SizedCache::with_size(CACHE_SIZE),
outgoing_receipts: SizedCache::with_size(CACHE_SIZE),
incoming_receipts: SizedCache::with_size(CACHE_SIZE),
transaction_results: SizedCache::with_size(CACHE_SIZE),
outcomes: SizedCache::with_size(CACHE_SIZE),
invalid_chunks: SizedCache::with_size(CACHE_SIZE),
}
}
@@ -609,12 +613,15 @@ impl ChainStoreAccess for ChainStore {
)
}

fn get_transaction_result(&mut self, hash: &CryptoHash) -> Result<&ExecutionOutcome, Error> {
fn get_execution_outcome(
&mut self,
hash: &CryptoHash,
) -> Result<&ExecutionOutcomeWithProof, Error> {
option_to_not_found(
read_with_cache(
&*self.store,
COL_TRANSACTION_RESULT,
&mut self.transaction_results,
&mut self.outcomes,
hash.as_ref(),
),
&format!("TRANSACTION: {}", hash),
@@ -679,7 +686,7 @@ pub struct ChainStoreUpdate<'a, T> {
block_index: HashMap<BlockIndex, Option<CryptoHash>>,
outgoing_receipts: HashMap<(CryptoHash, ShardId), Vec<Receipt>>,
incoming_receipts: HashMap<(CryptoHash, ShardId), Vec<ReceiptProof>>,
transaction_results: HashMap<CryptoHash, ExecutionOutcome>,
outcomes: HashMap<CryptoHash, ExecutionOutcomeWithProof>,
head: Option<Tip>,
tail: Option<Tip>,
header_head: Option<Tip>,
@@ -711,7 +718,7 @@ impl<'a, T: ChainStoreAccess> ChainStoreUpdate<'a, T> {
chunk_one_parts: HashMap::default(),
outgoing_receipts: HashMap::default(),
incoming_receipts: HashMap::default(),
transaction_results: HashMap::default(),
outcomes: HashMap::default(),
head: None,
tail: None,
header_head: None,
@@ -899,8 +906,11 @@ impl<'a, T: ChainStoreAccess> ChainStoreAccess for ChainStoreUpdate<'a, T> {
}
}

fn get_transaction_result(&mut self, hash: &CryptoHash) -> Result<&ExecutionOutcome, Error> {
self.chain_store.get_transaction_result(hash)
fn get_execution_outcome(
&mut self,
hash: &CryptoHash,
) -> Result<&ExecutionOutcomeWithProof, Error> {
self.chain_store.get_execution_outcome(hash)
}

fn get_chunk(&mut self, chunk_hash: &ChunkHash) -> Result<&ShardChunk, Error> {
@@ -1111,8 +1121,17 @@ impl<'a, T: ChainStoreAccess> ChainStoreUpdate<'a, T> {
self.incoming_receipts.insert((*hash, shard_id), receipt_proof);
}

pub fn save_transaction_result(&mut self, hash: &CryptoHash, result: ExecutionOutcome) {
self.transaction_results.insert(*hash, result);
pub fn save_outcomes_with_proofs(
&mut self,
outcomes: Vec<ExecutionOutcomeWithId>,
proofs: Vec<MerklePath>,
) {
for (outcome_with_id, proof) in outcomes.into_iter().zip(proofs.into_iter()) {
self.outcomes.insert(
outcome_with_id.id,
ExecutionOutcomeWithProof { outcome: outcome_with_id.outcome, proof },
);
}
}

/// Starts a sub-ChainUpdate with atomic commit/rollback of all operations done
@@ -1257,8 +1276,8 @@ impl<'a, T: ChainStoreAccess> ChainStoreUpdate<'a, T> {
&receipt,
)?;
}
for (hash, tx_result) in self.transaction_results.drain() {
store_update.set_ser(COL_TRANSACTION_RESULT, hash.as_ref(), &tx_result)?;
for (hash, outcome) in self.outcomes.drain() {
store_update.set_ser(COL_TRANSACTION_RESULT, hash.as_ref(), &outcome)?;
}
for trie_changes in self.trie_changes {
trie_changes
@@ -649,7 +649,7 @@ impl RuntimeAdapter for KeyValueRuntime {
TrieChanges::empty(state_root.hash),
),
new_root: new_state_root,
transaction_results: tx_results,
outcomes: tx_results,
receipt_result: new_receipts,
validator_proposals: vec![],
total_gas_burnt: 0,
@@ -7,12 +7,12 @@ pub use near_primitives::block::{Block, BlockHeader, Weight};
use near_primitives::challenge::ChallengesResult;
use near_primitives::errors::RuntimeError;
use near_primitives::hash::{hash, CryptoHash};
use near_primitives::merkle::MerklePath;
use near_primitives::merkle::{merklize, MerklePath};
use near_primitives::receipt::Receipt;
use near_primitives::sharding::{ReceiptProof, ShardChunk, ShardChunkHeader};
use near_primitives::transaction::{ExecutionOutcomeWithId, SignedTransaction};
use near_primitives::types::{
AccountId, Balance, BlockIndex, EpochId, Gas, ShardId, StateRoot, ValidatorStake,
AccountId, Balance, BlockIndex, EpochId, Gas, MerkleHash, ShardId, StateRoot, ValidatorStake,
};
use near_primitives::views::QueryResponse;
use near_store::{PartialStorage, StoreUpdate, WrappedTrieChanges};
@@ -108,7 +108,7 @@ impl ValidatorSignatureVerificationResult {
pub struct ApplyTransactionResult {
pub trie_changes: WrappedTrieChanges,
pub new_root: StateRoot,
pub transaction_results: Vec<ExecutionOutcomeWithId>,
pub outcomes: Vec<ExecutionOutcomeWithId>,
pub receipt_result: ReceiptResult,
pub validator_proposals: Vec<ValidatorStake>,
pub total_gas_burnt: Gas,
@@ -118,6 +118,19 @@ pub struct ApplyTransactionResult {
pub proof: Option<PartialStorage>,
}

impl ApplyTransactionResult {
/// Returns root and paths for all the outcomes in the result.
pub fn compute_outcomes_proof(
outcomes: &[ExecutionOutcomeWithId],
) -> (MerkleHash, Vec<MerklePath>) {
let mut result = vec![];
for outcome_with_id in outcomes.iter() {
result.extend(outcome_with_id.outcome.to_hashes());
}
merklize(&result)
}
}

/// Bridge between the chain and the runtime.
/// Main function is to update state given transactions.
/// Additionally handles validators and block weight computation.
@@ -11,7 +11,7 @@ use near_primitives::types::{AccountId, ChunkExtra, EpochId};
use near_store::PartialStorage;

use crate::byzantine_assert;
use crate::types::ValidatorSignatureVerificationResult;
use crate::types::{ApplyTransactionResult, ValidatorSignatureVerificationResult};
use crate::{ChainStore, Error, ErrorKind, RuntimeAdapter};

/// Verifies that chunk's proofs in the header match the body.
@@ -60,6 +60,10 @@ pub fn validate_chunk_with_chunk_extra(
return Err(ErrorKind::InvalidStateRoot.into());
}

if prev_chunk_extra.outcome_root != chunk_header.inner.outcome_root {
return Err(ErrorKind::InvalidOutcomesProof.into());
}

if prev_chunk_extra.validator_proposals != chunk_header.inner.validator_proposals {
return Err(ErrorKind::InvalidValidatorProposals.into());
}
@@ -228,7 +232,9 @@ fn validate_chunk_state_challenge(
&ChallengesResult::default(),
)
.map_err(|_| Error::from(ErrorKind::MaliciousChallenge))?;
let outcome_root = ApplyTransactionResult::compute_outcomes_proof(&result.outcomes).0;
if result.new_root != chunk_state.chunk_header.inner.prev_state_root
|| outcome_root != chunk_state.chunk_header.inner.outcome_root
|| result.validator_proposals != chunk_state.chunk_header.inner.validator_proposals
|| result.total_gas_burnt != chunk_state.chunk_header.inner.gas_used
|| result.total_rent_paid != chunk_state.chunk_header.inner.rent_paid
@@ -907,6 +907,7 @@ impl ShardsManager {
&mut self,
prev_block_hash: CryptoHash,
prev_state_root: StateRoot,
outcome_root: CryptoHash,
height: u64,
shard_id: ShardId,
gas_used: Gas,
@@ -926,6 +927,7 @@ impl ShardsManager {
EncodedShardChunk::new(
prev_block_hash,
prev_state_root,
outcome_root,
height,
shard_id,
total_parts,
@@ -418,6 +418,7 @@ impl Client {
let (encoded_chunk, merkle_paths) = self.shards_mgr.create_encoded_shard_chunk(
prev_block_hash,
chunk_extra.state_root,
chunk_extra.outcome_root,
next_height,
shard_id,
chunk_extra.gas_used,
@@ -894,7 +895,7 @@ impl Client {
return NetworkClientResponses::TxStatus(res);
}
let me = self.block_producer.as_ref().map(|bp| &bp.account_id);
let has_tx_result = match self.chain.get_transaction_result(&tx_hash) {
let has_tx_result = match self.chain.get_execution_outcome(&tx_hash) {
Ok(_) => true,
Err(e) => match e.kind() {
ErrorKind::DBNotFoundErr(_) => false,
@@ -206,6 +206,7 @@ fn test_verify_chunk_invalid_state_challenge() {
.create_encoded_shard_chunk(
last_block.hash(),
StateRoot { hash: CryptoHash::default(), num_parts: 1 },
CryptoHash::default(),
last_block.header.inner.height + 1,
0,
0,

0 comments on commit 7214e61

Please sign in to comment.
You can’t perform that action at this time.