Fork choice and metadata framework for Engine

ethcore/light/src/client/header_chain.rs

@@ -446,6 +446,7 @@ impl HeaderChain {
 		let raw = header.encoded().into_inner();
 		transaction.put_vec(self.col, &hash[..], raw);
 
+		// TODO: For engines when required, use cryptoeconomic guarantees.
 		let (best_num, is_new_best) = {
 			let cur_best = self.best_block.read();
 			if cur_best.total_difficulty < total_difficulty {

ethcore/src/blockchain/blockchain.rs

@@ -896,7 +896,7 @@ impl BlockChain {
 	/// Inserts the block into backing cache database.
 	/// Expects the block to be valid and already verified.
 	/// If the block is already known, does nothing.
-	pub fn insert_block(&self, batch: &mut DBTransaction, bytes: &[u8], receipts: Vec<Receipt>) -> ImportRoute {
+	pub fn insert_block(&self, batch: &mut DBTransaction, bytes: &[u8], receipts: Vec<Receipt>, metadata: U256, is_new_best: bool) -> ImportRoute {
 		// create views onto rlp
 		let block = BlockView::new(bytes);
 		let header = block.header_view();
@@ -915,7 +915,7 @@ impl BlockChain {
 		batch.put(db::COL_HEADERS, &hash, &compressed_header);
 		batch.put(db::COL_BODIES, &hash, &compressed_body);
 
-		let info = self.block_info(&header);
+		let info = self.block_info(&header, metadata, is_new_best);
 
 		if let BlockLocation::BranchBecomingCanonChain(ref d) = info.location {
 			info!(target: "reorg", "Reorg to {} ({} {} {})",
@@ -940,17 +940,16 @@ impl BlockChain {
 	}
 
 	/// Get inserted block info which is critical to prepare extras updates.
-	fn block_info(&self, header: &HeaderView) -> BlockInfo {
+	fn block_info(&self, header: &HeaderView, metadata: U256, is_new_best: bool) -> BlockInfo {
 		let hash = header.hash();
 		let number = header.number();
 		let parent_hash = header.parent_hash();
 		let parent_details = self.block_details(&parent_hash).unwrap_or_else(|| panic!("Invalid parent hash: {:?}", parent_hash));
-		let is_new_best = parent_details.total_difficulty + header.difficulty() > self.best_block_total_difficulty();
 
 		BlockInfo {
 			hash: hash,
 			number: number,
-			total_difficulty: parent_details.total_difficulty + header.difficulty(),
+			total_difficulty: metadata,
 			location: if is_new_best {
 				// on new best block we need to make sure that all ancestors
 				// are moved to "canon chain"
@@ -1416,7 +1415,7 @@ mod tests {
 	use std::sync::Arc;
 	use rustc_hex::FromHex;
 	use hash::keccak;
-	use kvdb::KeyValueDB;
+	use kvdb::{KeyValueDB, DBTransaction};
 	use kvdb_memorydb;
 	use ethereum_types::*;
 	use receipt::{Receipt, TransactionOutcome};
@@ -1439,6 +1438,33 @@ mod tests {
 		BlockChain::new(Config::default(), genesis, db)
 	}
 
+	fn insert_block(db: &Arc<KeyValueDB>, bc: &BlockChain, bytes: &[u8], receipts: Vec<Receipt>) -> ImportRoute {
+		insert_block_commit(db, bc, bytes, receipts, true)
+	}
+
+	fn insert_block_commit(db: &Arc<KeyValueDB>, bc: &BlockChain, bytes: &[u8], receipts: Vec<Receipt>, commit: bool) -> ImportRoute {
+		let mut batch = db.transaction();
+		let res = insert_block_batch(&mut batch, bc, bytes, receipts);
+		db.write(batch).unwrap();
+		if commit {
+			bc.commit();
+		}
+		res
+	}
+
+	fn insert_block_batch(batch: &mut DBTransaction, bc: &BlockChain, bytes: &[u8], receipts: Vec<Receipt>) -> ImportRoute {
+		use views::BlockView;
+
+		let block = BlockView::new(bytes);
+		let header = block.header_view();
+		let parent_hash = header.parent_hash();
+		let parent_details = bc.block_details(&parent_hash).unwrap_or_else(|| panic!("Invalid parent hash: {:?}", parent_hash));
+		let block_total_difficulty = parent_details.total_difficulty + header.difficulty();
+		let is_new_best = block_total_difficulty > bc.best_block_total_difficulty();
+
+		bc.insert_block(batch, bytes, receipts, block_total_difficulty, is_new_best)
+	}
+
 	#[test]
 	fn should_cache_best_block() {
 		// given
@@ -1450,8 +1476,7 @@ mod tests {
 		assert_eq!(bc.best_block_number(), 0);
 
 		// when
-		let mut batch = db.transaction();
-		bc.insert_block(&mut batch, &first.last().encoded(), vec![]);
+		insert_block_commit(&db, &bc, &first.last().encoded(), vec![], false);
 		assert_eq!(bc.best_block_number(), 0);
 		bc.commit();
 		// NOTE no db.write here (we want to check if best block is cached)
@@ -1481,7 +1506,7 @@ mod tests {
 		assert_eq!(bc.block_details(&genesis_hash).unwrap().children, vec![]);
 
 		let mut batch = db.transaction();
-		bc.insert_block(&mut batch, &first.encoded(), vec![]);
+		insert_block_batch(&mut batch, &bc, &first.encoded(), vec![]);
 		db.write(batch).unwrap();
 		bc.commit();
 
@@ -1507,7 +1532,7 @@ mod tests {
 		let mut batch = db.transaction();
 		for block in generator {
 			block_hashes.push(block.hash());
-			bc.insert_block(&mut batch, &block.encoded(), vec![]);
+			insert_block_batch(&mut batch, &bc, &block.encoded(), vec![]);
 			bc.commit();
 		}
 		db.write(batch).unwrap();
@@ -1547,14 +1572,10 @@ mod tests {
 		let db = new_db();
 		let bc = new_chain(&genesis.last().encoded(), db.clone());
 
-		let mut batch = db.transaction();
 		for b in generator {
-			bc.insert_block(&mut batch, &b.encoded(), vec![]);
-			bc.commit();
+			insert_block(&db, &bc, &b.encoded(), vec![]);
 		}
 
-		db.write(batch).unwrap();
-
 		assert_eq!(uncle_headers, bc.find_uncle_headers(&b4a_hash, 3).unwrap());
 		// TODO: insert block that already includes one of them as an uncle to check it's not allowed.
 	}
@@ -1588,9 +1609,9 @@ mod tests {
 		let bc = new_chain(&genesis.last().encoded(), db.clone());
 
 		let mut batch = db.transaction();
-		let _ = bc.insert_block(&mut batch, &b1a.last().encoded(), vec![]);
+		let _ = insert_block_batch(&mut batch, &bc, &b1a.last().encoded(), vec![]);
 		bc.commit();
-		let _ = bc.insert_block(&mut batch, &b1b.last().encoded(), vec![]);
+		let _ = insert_block_batch(&mut batch, &bc, &b1b.last().encoded(), vec![]);
 		bc.commit();
 		db.write(batch).unwrap();
 
@@ -1602,7 +1623,7 @@ mod tests {
 
 		// now let's make forked chain the canon chain
 		let mut batch = db.transaction();
-		let _ = bc.insert_block(&mut batch, &b2.last().encoded(), vec![]);
+		let _ = insert_block_batch(&mut batch, &bc, &b2.last().encoded(), vec![]);
 		bc.commit();
 		db.write(batch).unwrap();
 
@@ -1663,9 +1684,9 @@ mod tests {
 		let bc = new_chain(&genesis.last().encoded(), db.clone());
 
 		let mut batch = db.transaction();
-		let _ = bc.insert_block(&mut batch, &b1a.last().encoded(), vec![]);
+		let _ = insert_block_batch(&mut batch, &bc, &b1a.last().encoded(), vec![]);
 		bc.commit();
-		let _ = bc.insert_block(&mut batch, &b1b.last().encoded(), vec![]);
+		let _ = insert_block_batch(&mut batch, &bc, &b1b.last().encoded(), vec![]);
 		bc.commit();
 		db.write(batch).unwrap();
 
@@ -1681,7 +1702,7 @@ mod tests {
 
 		// now let's make forked chain the canon chain
 		let mut batch = db.transaction();
-		let _ = bc.insert_block(&mut batch, &b2.last().encoded(), vec![]);
+		let _ = insert_block_batch(&mut batch, &bc, &b2.last().encoded(), vec![]);
 		bc.commit();
 		db.write(batch).unwrap();
 
@@ -1721,16 +1742,16 @@ mod tests {
 		let bc = new_chain(&genesis.last().encoded(), db.clone());
 
 		let mut batch = db.transaction();
-		let ir1 = bc.insert_block(&mut batch, &b1.last().encoded(), vec![]);
+		let ir1 = insert_block_batch(&mut batch, &bc, &b1.last().encoded(), vec![]);
 		bc.commit();
-		let ir2 = bc.insert_block(&mut batch, &b2.last().encoded(), vec![]);
+		let ir2 = insert_block_batch(&mut batch, &bc, &b2.last().encoded(), vec![]);
 		bc.commit();
-		let ir3b = bc.insert_block(&mut batch, &b3b.last().encoded(), vec![]);
+		let ir3b = insert_block_batch(&mut batch, &bc, &b3b.last().encoded(), vec![]);
 		bc.commit();
 		db.write(batch).unwrap();
 		assert_eq!(bc.block_hash(3).unwrap(), b3b_hash);
 		let mut batch = db.transaction();
-		let ir3a = bc.insert_block(&mut batch, &b3a.last().encoded(), vec![]);
+		let ir3a = insert_block_batch(&mut batch, &bc, &b3a.last().encoded(), vec![]);
 		bc.commit();
 		db.write(batch).unwrap();
 
@@ -1835,7 +1856,7 @@ mod tests {
 			let bc = new_chain(&genesis.last().encoded(), db.clone());
 			assert_eq!(bc.best_block_hash(), genesis_hash);
 			let mut batch = db.transaction();
-			bc.insert_block(&mut batch, &first.last().encoded(), vec![]);
+			insert_block_batch(&mut batch, &bc, &first.last().encoded(), vec![]);
 			db.write(batch).unwrap();
 			bc.commit();
 			assert_eq!(bc.best_block_hash(), first_hash);
@@ -1894,7 +1915,7 @@ mod tests {
 		let db = new_db();
 		let bc = new_chain(&genesis, db.clone());
 		let mut batch =db.transaction();
-		bc.insert_block(&mut batch, &b1, vec![]);
+		insert_block_batch(&mut batch, &bc, &b1, vec![]);
 		db.write(batch).unwrap();
 		bc.commit();
 
@@ -1905,14 +1926,6 @@ mod tests {
 		}
 	}
 
-	fn insert_block(db: &Arc<KeyValueDB>, bc: &BlockChain, bytes: &[u8], receipts: Vec<Receipt>) -> ImportRoute {
-		let mut batch = db.transaction();
-		let res = bc.insert_block(&mut batch, bytes, receipts);
-		db.write(batch).unwrap();
-		bc.commit();
-		res
-	}
-
 	#[test]
 	fn test_logs() {
 		let t1 = Transaction {
@@ -2158,12 +2171,12 @@ mod tests {
 			let mut batch = db.transaction();
 			// create a longer fork
 			for block in generator {
-				bc.insert_block(&mut batch, &block.encoded(), vec![]);
+				insert_block_batch(&mut batch, &bc, &block.encoded(), vec![]);
 				bc.commit();
 			}
 
 			assert_eq!(bc.best_block_number(), 5);
-			bc.insert_block(&mut batch, &uncle.last().encoded(), vec![]);
+			insert_block_batch(&mut batch, &bc, &uncle.last().encoded(), vec![]);
 			db.write(batch).unwrap();
 			bc.commit();
 		}
@@ -2190,7 +2203,7 @@ mod tests {
 			// create a longer fork
 			for (i, block) in generator.into_iter().enumerate() {
 
-				bc.insert_block(&mut batch, &block.encoded(), vec![]);
+				insert_block_batch(&mut batch, &bc, &block.encoded(), vec![]);
 				bc.insert_epoch_transition(&mut batch, i as u64, EpochTransition {
 					block_hash: block.hash(),
 					block_number: i as u64 + 1,
@@ -2201,7 +2214,7 @@ mod tests {
 
 			assert_eq!(bc.best_block_number(), 5);
 
-			bc.insert_block(&mut batch, &uncle.last().encoded(), vec![]);
+			insert_block_batch(&mut batch, &bc, &uncle.last().encoded(), vec![]);
 			bc.insert_epoch_transition(&mut batch, 999, EpochTransition {
 				block_hash: uncle.last().hash(),
 				block_number: 1,
@@ -2251,23 +2264,15 @@ mod tests {
 		// and a non-canonical fork of 8 from genesis.
 		let fork_hash = {
 			for block in fork_generator {
-				let mut batch = db.transaction();
-
-				bc.insert_block(&mut batch, &block.encoded(), vec![]);
-				bc.commit();
-				db.write(batch).unwrap();
+				insert_block(&db, &bc, &block.encoded(), vec![]);
 			}
 
 			assert_eq!(bc.best_block_number(), 7);
 			bc.chain_info().best_block_hash
 		};
 
 		for block in next_generator {
-			let mut batch = db.transaction();
-			bc.insert_block(&mut batch, &block.encoded(), vec![]);
-			bc.commit();
-
-			db.write(batch).unwrap();
+			insert_block(&db, &bc, &block.encoded(), vec![]);
 		}
 
 		assert_eq!(bc.best_block_number(), 10);

ethcore/src/client/client.rs

@@ -505,6 +505,8 @@ impl Importer {
 	//
 	// The header passed is from the original block data and is sealed.
 	fn commit_block<B>(&self, block: B, header: &Header, block_data: &[u8], client: &Client) -> ImportRoute where B: IsBlock + Drain {
+		use std::ops::Deref;
+
 		let hash = &header.hash();
 		let number = header.number();
 		let parent = header.parent_hash();
@@ -538,7 +540,9 @@ impl Importer {
 		);
 
 		state.journal_under(&mut batch, number, hash).expect("DB commit failed");
-		let route = chain.insert_block(&mut batch, block_data, receipts.clone());
+		let block_metadata = client.engine.generate_metadata(block_data, chain.deref().deref());
+		let is_new_best = client.engine.is_new_best(block_data, &block_metadata, &chain.best_block_total_difficulty(), chain.deref().deref());
+		let route = chain.insert_block(&mut batch, block_data, receipts.clone(), block_metadata, is_new_best);
 
 		client.tracedb.read().import(&mut batch, TraceImportRequest {
 			traces: traces.into(),
@@ -2253,6 +2257,7 @@ mod tests {
 		use std::sync::Arc;
 		use std::sync::atomic::{AtomicBool, Ordering};
 		use kvdb::DBTransaction;
+		use ethereum_types::U256;
 
 		let client = generate_dummy_client(0);
 		let genesis = client.chain_info().best_block_hash;
@@ -2265,7 +2270,7 @@ mod tests {
 			let another_client = client.clone();
 			thread::spawn(move || {
 				let mut batch = DBTransaction::new();
-				another_client.chain.read().insert_block(&mut batch, &new_block, Vec::new());
+				another_client.chain.read().insert_block(&mut batch, &new_block, Vec::new(), U256::zero(), true);
 				go_thread.store(true, Ordering::SeqCst);
 			});
 			go

ethcore/src/engines/authority_round/mod.rs

@@ -1318,6 +1318,14 @@ impl Engine<EthereumMachine> for AuthorityRound {
 			Some(Box::new(::snapshot::PoaSnapshot))
 		}
 	}
+
+	fn generate_metadata(&self, bytes: &[u8], provider: &<EthereumMachine as ::parity_machine::Machine>::BlockProvider) -> <EthereumMachine as ::parity_machine::Machine>::BlockMetadata {
+		super::total_difficulty_generate_metadata(bytes, provider)
+	}
+
+	fn is_new_best(&self, bytes: &[u8], block_metadata: &<EthereumMachine as ::parity_machine::Machine>::BlockMetadata, best_block_metadata: &<EthereumMachine as ::parity_machine::Machine>::BlockMetadata, provider: &<EthereumMachine as ::parity_machine::Machine>::BlockProvider) -> bool {
+		super::total_difficulty_is_new_best(bytes, *block_metadata, *best_block_metadata, provider)
+	}
 }
 
 #[cfg(test)]

ethcore/src/engines/basic_authority.rs

@@ -191,6 +191,14 @@ impl Engine<EthereumMachine> for BasicAuthority {
 	fn snapshot_components(&self) -> Option<Box<::snapshot::SnapshotComponents>> {
 		None
 	}
+
+	fn generate_metadata(&self, bytes: &[u8], provider: &<EthereumMachine as ::parity_machine::Machine>::BlockProvider) -> <EthereumMachine as ::parity_machine::Machine>::BlockMetadata {
+		super::total_difficulty_generate_metadata(bytes, provider)
+	}
+
+	fn is_new_best(&self, bytes: &[u8], block_metadata: &<EthereumMachine as ::parity_machine::Machine>::BlockMetadata, best_block_metadata: &<EthereumMachine as ::parity_machine::Machine>::BlockMetadata, provider: &<EthereumMachine as ::parity_machine::Machine>::BlockProvider) -> bool {
+		super::total_difficulty_is_new_best(bytes, *block_metadata, *best_block_metadata, provider)
+	}
 }
 
 #[cfg(test)]

ethcore/src/engines/instant_seal.rs

@@ -32,7 +32,7 @@ impl<M> InstantSeal<M> {
 	}
 }
 
-impl<M: Machine> Engine<M> for InstantSeal<M>
+impl<M: Machine<BlockMetadata=::ethereum_types::U256, BlockProvider=::blockchain::BlockProvider>> Engine<M> for InstantSeal<M>
 	where M::LiveBlock: Transactions
 {
 	fn name(&self) -> &str {
@@ -61,6 +61,14 @@ impl<M: Machine> Engine<M> for InstantSeal<M>
 	fn is_timestamp_valid(&self, header_timestamp: u64, parent_timestamp: u64) -> bool {
 		header_timestamp >= parent_timestamp
 	}
+
+	fn generate_metadata(&self, bytes: &[u8], provider: &<::machine::EthereumMachine as ::parity_machine::Machine>::BlockProvider) -> <::machine::EthereumMachine as ::parity_machine::Machine>::BlockMetadata {
+		super::total_difficulty_generate_metadata(bytes, provider)
+	}
+
+	fn is_new_best(&self, bytes: &[u8], block_metadata: &<::machine::EthereumMachine as ::parity_machine::Machine>::BlockMetadata, best_block_metadata: &<::machine::EthereumMachine as ::parity_machine::Machine>::BlockMetadata, provider: &<::machine::EthereumMachine as ::parity_machine::Machine>::BlockProvider) -> bool {
+		super::total_difficulty_is_new_best(bytes, *block_metadata, *best_block_metadata, provider)
+	}
 }
 
 #[cfg(test)]