Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Mostly StateRoot #1444

Merged
merged 5 commits into from Oct 11, 2019
Merged

Mostly StateRoot #1444

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
74b0717
download strategy + minors
Kouprin Oct 10, 2019
93cff67
StateRoot struct
Kouprin Oct 10, 2019
1e625b9
MerkleProof -> Vec<u8> (any proof)
Kouprin Oct 11, 2019
263aae5
StateRoot -> &StateRoot in query
Kouprin Oct 11, 2019
5d89737
put sync_state_nodes_multishard into expensive tests
Kouprin Oct 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
32 changes: 9 additions & 23 deletions chain/chain/src/chain.rs
View file
Open in desktop
Expand Up @@ -183,10 +183,9 @@ impl Chain {
let mut store = ChainStore::new(store);

// Get runtime initial state and create genesis block out of it.
let (state_store_update, state_roots, state_num_parts) = runtime_adapter.genesis_state();
let (state_store_update, state_roots) = runtime_adapter.genesis_state();
let genesis = Block::genesis(
state_roots.clone(),
state_num_parts.clone(),
chain_genesis.time,
runtime_adapter.num_shards(),
chain_genesis.gas_limit,
Expand Down Expand Up @@ -242,20 +241,12 @@ impl Chain {
store_update.save_block_header(genesis.header.clone());
store_update.save_block(genesis.clone());

for (chunk_header, (state_root, state_num_parts)) in
genesis.chunks.iter().zip(state_roots.iter().zip(state_num_parts.iter()))
for (chunk_header, state_root) in genesis.chunks.iter().zip(state_roots.iter())
{
store_update.save_chunk_extra(
&genesis.hash(),
chunk_header.inner.shard_id,
ChunkExtra::new(
state_root,
*state_num_parts,
vec![],
0,
chain_genesis.gas_limit,
0,
),
ChunkExtra::new(state_root, vec![], 0, chain_genesis.gas_limit, 0),
);
}

Expand Down Expand Up @@ -947,18 +938,17 @@ impl Chain {
)
.into());
}
let state_root = sync_prev_block.chunks[shard_id as usize].inner.prev_state_root;
let state_num_parts = sync_prev_block.chunks[shard_id as usize].inner.prev_state_num_parts;
let state_root = sync_prev_block.chunks[shard_id as usize].inner.prev_state_root.clone();

if part_id >= state_num_parts {
if part_id >= state_root.num_parts {
return Err(ErrorKind::Other(
"get_state_response_part fail: part_id out of bound".to_string(),
)
.into());
}
let (state_part, proof) = self
.runtime_adapter
.obtain_state_part(shard_id, part_id, state_root, state_num_parts)
.obtain_state_part(shard_id, part_id, &state_root)
.map_err(|err| ErrorKind::Other(err.to_string()))?;

Ok(ShardStateSyncResponsePart { state_part, proof })
Expand Down Expand Up @@ -1141,8 +1131,7 @@ impl Chain {
) -> Result<(), Error> {
let shard_state_header = self.get_received_state_header(shard_id, sync_hash)?;
let ShardStateSyncResponseHeader { chunk, .. } = shard_state_header;
let state_root = chunk.header.inner.prev_state_root;
let _state_num_parts = chunk.header.inner.prev_state_num_parts;
let state_root = &chunk.header.inner.prev_state_root;
self.runtime_adapter
.accept_state_part(state_root, &part.state_part, &part.proof)
.map_err(|_| ErrorKind::InvalidStatePayload)?;
Expand All @@ -1163,13 +1152,12 @@ impl Chain {
incoming_receipts_proofs,
root_proofs: _,
} = shard_state_header;
let state_root = chunk.header.inner.prev_state_root;
let state_num_parts = chunk.header.inner.prev_state_num_parts;
let state_root = &chunk.header.inner.prev_state_root;

let block_header = self.get_header_by_height(chunk.header.height_included)?.clone();

// Applying chunk is started here.
self.runtime_adapter.confirm_state(state_root, state_num_parts)?;
self.runtime_adapter.confirm_state(&state_root)?;

// Getting actual incoming receipts.
let mut receipt_proof_response: Vec<ReceiptProofResponse> = vec![];
Expand Down Expand Up @@ -1207,7 +1195,6 @@ impl Chain {
chain_store_update.save_trie_changes(apply_result.trie_changes);
let chunk_extra = ChunkExtra::new(
&apply_result.new_root,
apply_result.new_num_parts,
apply_result.validator_proposals,
apply_result.total_gas_burnt,
gas_limit,
Expand Down Expand Up @@ -1765,7 +1752,6 @@ impl<'a> ChainUpdate<'a> {
shard_id,
ChunkExtra::new(
&apply_result.new_root,
apply_result.new_num_parts,
apply_result.validator_proposals,
apply_result.total_gas_burnt,
gas_limit,
Expand Down
88 changes: 40 additions & 48 deletions chain/chain/src/test_utils.rs
View file
Open in desktop
Expand Up @@ -20,7 +20,8 @@ use near_primitives::transaction::{
TransferAction,
};
use near_primitives::types::{
AccountId, Balance, BlockIndex, EpochId, Gas, Nonce, ShardId, StateRoot, ValidatorStake,
AccountId, Balance, BlockIndex, EpochId, Gas, MerkleHash, Nonce, ShardId, StateRoot,
ValidatorStake,
};
use near_primitives::views::QueryResponse;
use near_store::test_utils::create_test_store;
Expand Down Expand Up @@ -58,8 +59,7 @@ pub struct KeyValueRuntime {
num_shards: ShardId,

// A mapping state_root => {account id => amounts}, for transactions and receipts
state: RwLock<HashMap<StateRoot, KVState>>,
state_num_parts: RwLock<HashMap<StateRoot, u64>>,
state: RwLock<HashMap<MerkleHash, KVState>>,
state_parts: RwLock<HashMap<CryptoHash, StatePart>>,
state_proofs: RwLock<HashMap<CryptoHash, MerklePath>>,

Expand Down Expand Up @@ -111,15 +111,14 @@ impl KeyValueRuntime {

let mut state = HashMap::new();
state.insert(
StateRoot::default(),
MerkleHash::default(),
KVState {
amounts: initial_amounts,
receipt_nonces: HashSet::default(),
tx_nonces: HashSet::default(),
},
);
// TODO initializing for StateRoot::default()?
let state_num_parts = HashMap::new();
// TODO MOO initializing for StateRoot::default()?
let state_parts = HashMap::new();
let state_proofs = HashMap::new();
KeyValueRuntime {
Expand All @@ -142,7 +141,6 @@ impl KeyValueRuntime {
validator_groups,
num_shards,
state: RwLock::new(state),
state_num_parts: RwLock::new(state_num_parts),
state_parts: RwLock::new(state_parts),
state_proofs: RwLock::new(state_proofs),
hash_to_epoch: RwLock::new(HashMap::new()),
Expand All @@ -152,8 +150,8 @@ impl KeyValueRuntime {
}
}

pub fn get_root(&self) -> StateRoot {
self.root
pub fn get_root(&self) -> CryptoHash {
self.root.hash
}

fn get_prev_height(
Expand Down Expand Up @@ -245,11 +243,15 @@ impl KeyValueRuntime {
}

impl RuntimeAdapter for KeyValueRuntime {
fn genesis_state(&self) -> (StoreUpdate, Vec<StateRoot>, Vec<u64>) {
fn genesis_state(&self) -> (StoreUpdate, Vec<StateRoot>) {
(
self.store.store_update(),
((0..self.num_shards()).map(|_| StateRoot::default()).collect()),
vec![17; self.num_shards() as usize], // TODO MOO
((0..self.num_shards())
.map(|_| StateRoot {
hash: CryptoHash::default(),
num_parts: DEFAULT_STATE_NUM_PARTS, /* TODO MOO */
})
.collect()),
)
}

Expand Down Expand Up @@ -490,8 +492,7 @@ impl RuntimeAdapter for KeyValueRuntime {
assert!(!generate_storage_proof);
let mut tx_results = vec![];

//println!("HERE! {:?}", state_root); // TODO MOO
let mut state = self.state.read().unwrap().get(state_root).cloned().unwrap();
let mut state = self.state.read().unwrap().get(&state_root.hash).cloned().unwrap();

let mut balance_transfers = vec![];

Expand Down Expand Up @@ -635,31 +636,23 @@ impl RuntimeAdapter for KeyValueRuntime {
let part = StatePart { shard_id, part_id: i as u64, data: data[begin..end].to_vec() };
parts.push(part);
}
let (new_state_root, proofs) = merklize(&parts);

self.state.write().unwrap().insert(new_state_root, state);
/*println!( TODO MOO
"QQQ {:?} datalen={:?} {:?} {:?}",
shard_id,
data.len(),
new_state_root,
state_num_parts
);*/
self.state_num_parts.write().unwrap().insert(new_state_root, state_num_parts as u64);
let (state_hash, proofs) = merklize(&parts);
let new_state_root = StateRoot { hash: state_hash, num_parts: state_num_parts as u64 };

self.state.write().unwrap().insert(new_state_root.hash, state);
for i in 0..state_num_parts {
let key = hash(&StatePartKey(i as u64, new_state_root).try_to_vec().unwrap());
assert!(verify_path(new_state_root, &proofs[i], &parts[i]));
let key = hash(&StatePartKey(i as u64, new_state_root.clone()).try_to_vec().unwrap());
assert!(verify_path(new_state_root.hash, &proofs[i], &parts[i]));
self.state_parts.write().unwrap().insert(key, parts[i].clone());
self.state_proofs.write().unwrap().insert(key, proofs[i].clone());
}

Ok(ApplyTransactionResult {
trie_changes: WrappedTrieChanges::new(
self.trie.clone(),
TrieChanges::empty(*state_root),
TrieChanges::empty(state_root.hash),
),
new_root: new_state_root,
new_num_parts: state_num_parts as u64,
transaction_results: tx_results,
receipt_result: new_receipts,
validator_proposals: vec![],
Expand All @@ -671,7 +664,7 @@ impl RuntimeAdapter for KeyValueRuntime {

fn query(
&self,
state_root: StateRoot,
state_root: &StateRoot,
_height: BlockIndex,
_block_timestamp: u64,
_block_hash: &CryptoHash,
Expand All @@ -686,7 +679,7 @@ impl RuntimeAdapter for KeyValueRuntime {
.state
.read()
.unwrap()
.get(&state_root)
.get(&state_root.hash)
.map_or_else(|| 0, |state| *state.amounts.get(&account_id2).unwrap_or(&0)),
locked: 0,
code_hash: CryptoHash::default(),
Expand All @@ -701,41 +694,41 @@ impl RuntimeAdapter for KeyValueRuntime {
&self,
shard_id: ShardId,
part_id: u64,
state_root: StateRoot,
state_num_parts: u64,
) -> Result<(StatePart, MerklePath), Box<dyn std::error::Error>> {
if part_id >= state_num_parts {
state_root: &StateRoot,
) -> Result<(StatePart, Vec<u8>), Box<dyn std::error::Error>> {
if part_id >= state_root.num_parts {
return Err("Invalid part_id in obtain_state_part".to_string().into());
}
if shard_id >= self.num_shards() {
return Err("Invalid shard_id in obtain_state_part".to_string().into());
}
let key = hash(&StatePartKey(part_id, state_root).try_to_vec().unwrap());
let key = hash(&StatePartKey(part_id, state_root.clone()).try_to_vec().unwrap());
let part = self.state_parts.read().unwrap().get(&key).unwrap().clone();
let proof = self.state_proofs.read().unwrap().get(&key).unwrap().clone();
assert!(verify_path(state_root, &proof, &part));
Ok((part, proof))
assert!(verify_path(state_root.hash, &proof, &part));
Ok((part, proof.try_to_vec()?))
}

fn accept_state_part(
&self,
state_root: StateRoot,
state_root: &StateRoot,
part: &StatePart,
proof: &MerklePath,
proof: &Vec<u8>,
) -> Result<(), Box<dyn std::error::Error>> {
if !verify_path(state_root, proof, part) {
let merkle_proof = MerklePath::try_from_slice(&proof)?;
if !verify_path(state_root.hash, &merkle_proof, part) {
return Err("set_shard_state failed: invalid StatePart".into());
}
let key = hash(&StatePartKey(part.part_id, state_root).try_to_vec().unwrap());
let key = hash(&StatePartKey(part.part_id, state_root.clone()).try_to_vec().unwrap());
self.state_parts.write().unwrap().insert(key, part.clone());
self.state_proofs.write().unwrap().insert(key, proof.to_vec());
self.state_proofs.write().unwrap().insert(key, merkle_proof);
Ok(())
}

fn confirm_state(&self, state_root: StateRoot, state_num_parts: u64) -> Result<bool, Error> {
fn confirm_state(&self, state_root: &StateRoot) -> Result<bool, Error> {
let mut data = vec![];
for i in 0..state_num_parts as usize {
let key = hash(&StatePartKey(i as u64, state_root).try_to_vec().unwrap());
for i in 0..state_root.num_parts as usize {
let key = hash(&StatePartKey(i as u64, state_root.clone()).try_to_vec().unwrap());
match self.state_parts.read().unwrap().get(&key) {
Some(part) => {
data.push(part.data.clone());
Expand All @@ -749,8 +742,7 @@ impl RuntimeAdapter for KeyValueRuntime {
}
let data_flatten: Vec<u8> = data.iter().flatten().cloned().collect();
let state = KVState::try_from_slice(&data_flatten).unwrap();
// println!("HERE RE! {:?} {:?}", state_root, state); // TODO MOO
self.state.write().unwrap().insert(state_root, state);
self.state.write().unwrap().insert(state_root.hash, state);
Ok(true)
}

Expand Down
21 changes: 9 additions & 12 deletions chain/chain/src/types.rs
View file
Open in desktop
Expand Up @@ -102,7 +102,6 @@ pub enum ValidatorSignatureVerificationResult {
pub struct ApplyTransactionResult {
pub trie_changes: WrappedTrieChanges,
pub new_root: StateRoot,
pub new_num_parts: u64,
pub transaction_results: Vec<ExecutionOutcomeWithId>,
pub receipt_result: ReceiptResult,
pub validator_proposals: Vec<ValidatorStake>,
Expand All @@ -117,7 +116,7 @@ pub struct ApplyTransactionResult {
pub trait RuntimeAdapter: Send + Sync {
/// Initialize state to genesis state and returns StoreUpdate, state root and initial validators.
/// StoreUpdate can be discarded if the chain past the genesis.
fn genesis_state(&self) -> (StoreUpdate, Vec<StateRoot>, Vec<u64>);
fn genesis_state(&self) -> (StoreUpdate, Vec<StateRoot>);

/// Verify block producer validity and return weight of given block for fork choice rule.
fn compute_block_weight(
Expand Down Expand Up @@ -312,7 +311,7 @@ pub trait RuntimeAdapter: Send + Sync {
/// Query runtime with given `path` and `data`.
fn query(
&self,
state_root: StateRoot,
state_root: &StateRoot,
height: BlockIndex,
block_timestamp: u64,
block_hash: &CryptoHash,
Expand All @@ -325,9 +324,8 @@ pub trait RuntimeAdapter: Send + Sync {
&self,
shard_id: ShardId,
part_id: u64,
state_root: StateRoot,
state_num_parts: u64,
) -> Result<(StatePart, MerklePath), Box<dyn std::error::Error>>;
state_root: &StateRoot,
) -> Result<(StatePart, Vec<u8>), Box<dyn std::error::Error>>;

/// Set state part that expected to be given state root with provided data.
/// Returns error if:
Expand All @@ -336,14 +334,14 @@ pub trait RuntimeAdapter: Send + Sync {
/// 3. The resulting part doesn't match the expected one.
fn accept_state_part(
&self,
state_root: StateRoot,
state_root: &StateRoot,
part: &StatePart,
proof: &MerklePath,
proof: &Vec<u8>,
) -> Result<(), Box<dyn std::error::Error>>;

/// Should be executed after accepting all the parts.
/// Returns `true` if state is set successfully.
fn confirm_state(&self, state_root: StateRoot, num_parts: u64) -> Result<bool, Error>;
fn confirm_state(&self, state_root: &StateRoot) -> Result<bool, Error>;

/// Build receipts hashes.
fn build_receipts_hashes(&self, receipts: &Vec<Receipt>) -> Result<Vec<CryptoHash>, Error> {
Expand Down Expand Up @@ -466,7 +464,7 @@ pub struct ShardStateSyncResponseHeader {
#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)]
pub struct ShardStateSyncResponsePart {
pub state_part: StatePart,
pub proof: MerklePath,
pub proof: Vec<u8>,
}

#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)]
Expand All @@ -489,8 +487,7 @@ mod tests {
fn test_block_produce() {
let num_shards = 32;
let genesis = Block::genesis(
vec![StateRoot::default()],
vec![9], /* TODO MOO */
vec![StateRoot { hash: CryptoHash::default(), num_parts: 9 /* TODO MOO */ }],
Utc::now(),
num_shards,
1_000_000,
Expand Down