Update to latest substrate-master and polkadot v0.3 (paritytech#195)

* Rebuild runtime

* Remove invalid value from chainspec (paritytech#68)

* service: use grandpa block import for locally sealed aura blocks (paritytech#85)

* bump version to v0.3.1

* Update lock file.

* limit number of transactions when building blocks (paritytech#91)

* Update to latest Substrate

* Bump to 0.3.2

* Actually bump.

* v0.3.2 (paritytech#98)

* bump substrate version

* fix polkadot-collator

* point to alexander-backports of substrate

* bump version

* cli: fix node shutdown (paritytech#100)

* update to latest substrate, change to v0.3.4

* update to latest substrate, bump version to 0.3.5

* v0.3.6

* try to build on every v0.3 commit and update alexander-backports

* bump to v0.3.7

* bump to 0.3.8

* Bump to 0.3.9: network and pruning improvements

* Bump to 0.3.10: reduce network bandwidth usage

* Use libp2p-kad 0.3.2 (paritytech#122)

* Bump libp2p-identify to 0.3.1 (paritytech#123)

* Bump to 0.3.12 (paritytech#127)

* Update Substrate again (paritytech#128)

* update substrate and bump version to v0.3.13

* bump version to v0.3.14: fix --reserved-nodes

* add a manually curated grandpa module (paritytech#136)

* updating v0.3 to use substrate v0.10 (paritytech#146)

* updating to latest substrate v0.10

* better handling of outer poll

* nit

* fix tests

* remove comment

* reduce indentation

* use self.poll

* bring oneshot into scope

* spaces

* wrap

* remove match

* wrap

* Update primitives/Cargo.toml

Co-Authored-By: gterzian <2792687+gterzian@users.noreply.github.com>

* Update runtime/wasm/Cargo.toml

Co-Authored-By: gterzian <2792687+gterzian@users.noreply.github.com>

* Update runtime/wasm/Cargo.toml

Co-Authored-By: gterzian <2792687+gterzian@users.noreply.github.com>

* Update test-parachains/adder/collator/src/main.rs

Co-Authored-By: gterzian <2792687+gterzian@users.noreply.github.com>

* indent

* add paranthese

* config: fix wrong ip for alexander bootnode (paritytech#161)

* fix curated-grandpa and rebuild wasm (paritytech#162)

* [v0.3] Integrates new gossip system into Polkadot (paritytech#166)

* new gossip validation in network

* integrate new gossip into service

* network: guard validation network future under exit signal (paritytech#168)

* bump version to v0.3.15: substrate v0.10

* [v0.3] update to substrate master (paritytech#175)

* update to substrate master

* fix test

* service: fix telemetry endpoints on alexander chainspec (paritytech#169) (paritytech#178)

* Update v0.3 to latest Substrate master (paritytech#177)

* update substrate v0.3 to latest master

* bump spec version

* update to latest master: remove fees module

* update runtime blobs

* bump version to 0.3.16

* replace sr25519 accountid with anysigner

* bump version to v0.3.17

* Some PoC-3 GRANDPA tweaks (paritytech#181)

* call on_finalise after triggering curated_grandpa change

* make grandpa rounds shorter for faster finalization

* use authorities when calculating duty roster (paritytech#185)

* [v0.3] Update to substrate master (paritytech#183)

* update to latest substrate master

* bump version to 0.3.18

* update to latest substrate master

* bump spec version

* update runtime wasm blobs

* remove current_offline_slash from chain spec

* update to substrate master: bump version to v0.3.19 (paritytech#188)

* update to substrate master: bump version to v0.3.19

libp2p network improvements

* network: replace NodeIndex with PeerId

* network: fix tests

* polkadot v0.3.20 (paritytech#190)

* update to substrate master: bump version to 0.3.20

* runtime: add offchain worker trait

* runtime: rebuild wasm blobs

* bump spec version (paritytech#191)

* Fix compilation

* Update version to 0.4.0

* Switch to use `polkadot-master` branch from substrate

* Remove unused struct

* Remove `grandpa::SyncedAuthorities` from `OnSessionChange`

.gitlab-ci.yml

@@ -235,7 +235,7 @@ publish-s3-release:
     - kubectl get nodes -l node=polkadot
       -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range @.status.addresses[?(@.type=="ExternalIP")]}{.address}{"\n"}{end}'
     - echo "# polkadots' nodes"
-    - kubectl -n polkadot get pods 
+    - kubectl -n polkadot get pods
       -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.nodeName}{"\n"}{end}'
 
 

Cargo.toml

@@ -4,7 +4,7 @@ path = "src/main.rs"
 
 [package]
 name = "polkadot"
-version = "0.3.0"
+version = "0.4.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 build = "build.rs"
 

availability-store/Cargo.toml

@@ -9,7 +9,7 @@ polkadot-primitives = { path = "../primitives" }
 parking_lot = "0.7.1"
 log = "0.4.6"
 parity-codec = "3.0"
-substrate-primitives = { git = "https://github.com/paritytech/substrate" }
+substrate-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 kvdb = { git = "https://github.com/paritytech/parity-common", rev="616b40150ded71f57f650067fcbc5c99d7c343e6" }
 kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="616b40150ded71f57f650067fcbc5c99d7c343e6" }
 kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="616b40150ded71f57f650067fcbc5c99d7c343e6" }

cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polkadot-cli"
-version = "0.3.0"
+version = "0.4.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 description = "Polkadot node implementation in Rust."
 
@@ -9,5 +9,5 @@ log = "0.4.6"
 tokio = "0.1.7"
 futures = "0.1.17"
 exit-future = "0.1"
-substrate-cli = { git = "https://github.com/paritytech/substrate" }
+substrate-cli = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 polkadot-service = { path = "../service" }

cli/src/lib.rs

@@ -36,14 +36,13 @@ use chain_spec::ChainSpec;
 use futures::Future;
 use tokio::runtime::Runtime;
 use service::Service as BareService;
-use cli::NoCustom;
 
 pub use service::{
 	Components as ServiceComponents, PolkadotService, CustomConfiguration, ServiceFactory, Factory,
 	ProvideRuntimeApi, CoreApi, ParachainHost,
 };
 
-pub use cli::{VersionInfo, IntoExit};
+pub use cli::{VersionInfo, IntoExit, NoCustom};
 pub use cli::error;
 pub use tokio::runtime::TaskExecutor;
 

collator/Cargo.toml

@@ -6,9 +6,9 @@ description = "Collator node implementation"
 
 [dependencies]
 futures = "0.1.17"
-substrate-client = { git = "https://github.com/paritytech/substrate" }
+substrate-client = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 parity-codec = "3.0"
-substrate-primitives = { git = "https://github.com/paritytech/substrate" }
+substrate-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 polkadot-runtime = { path = "../runtime", version = "0.1" }
 polkadot-primitives = { path = "../primitives", version = "0.1" }
 polkadot-cli = { path = "../cli" }
@@ -18,4 +18,4 @@ log = "0.4"
 tokio = "0.1.7"
 
 [dev-dependencies]
-substrate-keyring = { git = "https://github.com/paritytech/substrate" }
+substrate-keyring = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }

collator/src/lib.rs

@@ -259,14 +259,15 @@ impl<P, E> Worker for CollationNode<P, E> where
 				match known_oracle.block_status(&BlockId::hash(*block_hash)) {
 					Err(_) | Ok(BlockStatus::Unknown) | Ok(BlockStatus::Queued) => None,
 					Ok(BlockStatus::KnownBad) => Some(Known::Bad),
-					Ok(BlockStatus::InChain) => match known_oracle.leaves() {
-						Err(_) => None,
-						Ok(leaves) => if leaves.contains(block_hash) {
-							Some(Known::Leaf)
-						} else {
-							Some(Known::Old)
-						},
-					}
+					Ok(BlockStatus::InChainWithState) | Ok(BlockStatus::InChainPruned) =>
+						match known_oracle.leaves() {
+							Err(_) => None,
+							Ok(leaves) => if leaves.contains(block_hash) {
+								Some(Known::Leaf)
+							} else {
+								Some(Known::Old)
+							},
+						}
 				}
 			},
 		);
@@ -481,3 +482,4 @@ mod tests {
 		assert_eq!(collation.receipt.egress_queue_roots, vec![(a, root_a), (b, root_b)]);
 	}
 }
+

erasure-coding/Cargo.toml

@@ -8,5 +8,5 @@ edition = "2018"
 polkadot-primitives = { path = "../primitives" }
 reed-solomon-erasure = { git = "https://github.com/paritytech/reed-solomon-erasure" }
 parity-codec = "3.0"
-substrate-primitives = { git = "https://github.com/paritytech/substrate" }
-substrate-trie = { git = "https://github.com/paritytech/substrate" }
+substrate-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+substrate-trie = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }

executor/Cargo.toml

@@ -5,6 +5,6 @@ authors = ["Parity Technologies <admin@parity.io>"]
 description = "Polkadot node implementation in Rust."
 
 [dependencies]
-substrate-executor = { git = "https://github.com/paritytech/substrate" }
-substrate-primitives = { git = "https://github.com/paritytech/substrate" }
+substrate-executor = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+substrate-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 polkadot-runtime = { path = "../runtime" }

network/Cargo.toml

@@ -12,15 +12,15 @@ polkadot-validation = { path = "../validation" }
 polkadot-primitives = { path = "../primitives" }
 parity-codec = "3.0"
 parity-codec-derive = "3.0"
-substrate-network = { git = "https://github.com/paritytech/substrate" }
-substrate-primitives = { git = "https://github.com/paritytech/substrate" }
-sr-primitives = { git = "https://github.com/paritytech/substrate" }
+substrate-network = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+substrate-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+sr-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
 futures = "0.1"
 tokio = "0.1.7"
 log = "0.4"
 slice-group-by = "0.2.2"
 exit-future = "0.1.4"
 
 [dev-dependencies]
-substrate-client = { git = "https://github.com/paritytech/substrate" }
-substrate-keyring = { git = "https://github.com/paritytech/substrate" }
+substrate-client = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
+substrate-keyring = { git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }

network/src/collator_pool.rs

@@ -16,8 +16,8 @@
 
 //! Bridge between the network and consensus service for getting collations to it.
 
-use polkadot_primitives::{parachain::CollatorId, Hash};
-use polkadot_primitives::parachain::{Id as ParaId, Collation};
+use polkadot_primitives::Hash;
+use polkadot_primitives::parachain::{CollatorId, Id as ParaId, Collation};
 use futures::sync::oneshot;
 
 use std::collections::hash_map::{HashMap, Entry};

network/src/lib.rs

@@ -55,9 +55,9 @@ pub mod gossip;
 
 use codec::{Decode, Encode};
 use futures::sync::oneshot;
-use polkadot_primitives::{Block, SessionKey, Hash, Header, parachain::CollatorId};
-use polkadot_primitives::parachain::{Id as ParaId, BlockData, CandidateReceipt, Collation};
-use substrate_network::{NodeIndex, RequestId, Context, Severity};
+use polkadot_primitives::{Block, SessionKey, Hash, Header};
+use polkadot_primitives::parachain::{Id as ParaId, CollatorId, BlockData, CandidateReceipt, Collation};
+use substrate_network::{PeerId, RequestId, Context, Severity};
 use substrate_network::{message, generic_message};
 use substrate_network::specialization::NetworkSpecialization as Specialization;
 use substrate_network::StatusMessage as GenericFullStatus;
@@ -156,21 +156,21 @@ pub enum Message {
 	Collation(Hash, Collation),
 }
 
-fn send_polkadot_message(ctx: &mut Context<Block>, to: NodeIndex, message: Message) {
+fn send_polkadot_message(ctx: &mut Context<Block>, to: PeerId, message: Message) {
 	trace!(target: "p_net", "Sending polkadot message to {}: {:?}", to, message);
 	let encoded = message.encode();
 	ctx.send_message(to, generic_message::Message::ChainSpecific(encoded))
 }
 
 /// Polkadot protocol attachment for substrate.
 pub struct PolkadotProtocol {
-	peers: HashMap<NodeIndex, PeerInfo>,
+	peers: HashMap<PeerId, PeerInfo>,
 	collating_for: Option<(CollatorId, ParaId)>,
 	collators: CollatorPool,
-	validators: HashMap<SessionKey, NodeIndex>,
+	validators: HashMap<SessionKey, PeerId>,
 	local_collations: LocalCollations<Collation>,
 	live_validation_sessions: LiveValidationSessions,
-	in_flight: HashMap<(RequestId, NodeIndex), BlockDataRequest>,
+	in_flight: HashMap<(RequestId, PeerId), BlockDataRequest>,
 	pending: Vec<BlockDataRequest>,
 	extrinsic_store: Option<::av_store::Store>,
 	next_req_id: u64,
@@ -225,7 +225,7 @@ impl PolkadotProtocol {
 			{
 				peer_data.collator_state.send_key(new_local.clone(), |msg| send_polkadot_message(
 					ctx,
-					*id,
+					id.clone(),
 					msg
 				));
 			}
@@ -257,7 +257,7 @@ impl PolkadotProtocol {
 				}
 				Err(Some(known_keys)) => {
 					let next_peer = known_keys.iter()
-						.filter_map(|x| validator_keys.get(x).map(|id| (x.clone(), *id)))
+						.filter_map(|x| validator_keys.get(x).map(|id| (x.clone(), id.clone())))
 						.find(|&(ref key, _)| pending.attempted_peers.insert(key.clone()))
 						.map(|(_, id)| id);
 
@@ -268,7 +268,7 @@ impl PolkadotProtocol {
 
 						send_polkadot_message(
 							ctx,
-							who,
+							who.clone(),
 							Message::RequestBlockData(req_id, parent, c_hash),
 						);
 
@@ -290,7 +290,7 @@ impl PolkadotProtocol {
 		self.pending = new_pending;
 	}
 
-	fn on_polkadot_message(&mut self, ctx: &mut Context<Block>, who: NodeIndex, msg: Message) {
+	fn on_polkadot_message(&mut self, ctx: &mut Context<Block>, who: PeerId, msg: Message) {
 		trace!(target: "p_net", "Polkadot message from {}: {:?}", who, msg);
 		match msg {
 			Message::SessionKey(key) => self.on_session_key(ctx, who, key),
@@ -313,7 +313,7 @@ impl PolkadotProtocol {
 		}
 	}
 
-	fn on_session_key(&mut self, ctx: &mut Context<Block>, who: NodeIndex, key: SessionKey) {
+	fn on_session_key(&mut self, ctx: &mut Context<Block>, who: PeerId, key: SessionKey) {
 		{
 			let info = match self.peers.get_mut(&who) {
 				Some(peer) => peer,
@@ -343,7 +343,7 @@ impl PolkadotProtocol {
 			for (relay_parent, collation) in new_collations {
 				send_polkadot_message(
 					ctx,
-					who,
+					who.clone(),
 					Message::Collation(relay_parent, collation),
 				)
 			}
@@ -354,8 +354,8 @@ impl PolkadotProtocol {
 		self.dispatch_pending_requests(ctx);
 	}
 
-	fn on_block_data(&mut self, ctx: &mut Context<Block>, who: NodeIndex, req_id: RequestId, data: Option<BlockData>) {
-		match self.in_flight.remove(&(req_id, who)) {
+	fn on_block_data(&mut self, ctx: &mut Context<Block>, who: PeerId, req_id: RequestId, data: Option<BlockData>) {
+		match self.in_flight.remove(&(req_id, who.clone())) {
 			Some(req) => {
 				if let Some(data) = data {
 					if data.hash() == req.block_data_hash {
@@ -372,7 +372,7 @@ impl PolkadotProtocol {
 	}
 
 	// when a validator sends us (a collator) a new role.
-	fn on_new_role(&mut self, ctx: &mut Context<Block>, who: NodeIndex, role: Role) {
+	fn on_new_role(&mut self, ctx: &mut Context<Block>, who: PeerId, role: Role) {
 		let info = match self.peers.get_mut(&who) {
 			Some(peer) => peer,
 			None => {
@@ -400,7 +400,7 @@ impl PolkadotProtocol {
 				debug!(target: "p_net", "Broadcasting collation on relay parent {:?}", relay_parent);
 				send_polkadot_message(
 					ctx,
-					who,
+					who.clone(),
 					Message::Collation(relay_parent, collation),
 				)
 			}
@@ -413,7 +413,7 @@ impl Specialization<Block> for PolkadotProtocol {
 		Status { collating_for: self.collating_for.clone() }.encode()
 	}
 
-	fn on_connect(&mut self, ctx: &mut Context<Block>, who: NodeIndex, status: FullStatus) {
+	fn on_connect(&mut self, ctx: &mut Context<Block>, who: PeerId, status: FullStatus) {
 		let local_status = match Status::decode(&mut &status.chain_status[..]) {
 			Some(status) => status,
 			None => {
@@ -440,7 +440,7 @@ impl Specialization<Block> for PolkadotProtocol {
 
 			peer_info.collator_state.set_role(collator_role, |msg| send_polkadot_message(
 				ctx,
-				who,
+				who.clone(),
 				msg,
 			));
 		}
@@ -450,7 +450,7 @@ impl Specialization<Block> for PolkadotProtocol {
 			for local_session_key in self.live_validation_sessions.recent_keys() {
 				peer_info.collator_state.send_key(local_session_key.clone(), |msg| send_polkadot_message(
 					ctx,
-					who,
+					who.clone(),
 					msg,
 				));
 			}
@@ -460,7 +460,7 @@ impl Specialization<Block> for PolkadotProtocol {
 		self.dispatch_pending_requests(ctx);
 	}
 
-	fn on_disconnect(&mut self, ctx: &mut Context<Block>, who: NodeIndex) {
+	fn on_disconnect(&mut self, ctx: &mut Context<Block>, who: PeerId) {
 		if let Some(info) = self.peers.remove(&who) {
 			if let Some((acc_id, _)) = info.collating_for {
 				let new_primary = self.collators.on_disconnect(acc_id)
@@ -469,7 +469,7 @@ impl Specialization<Block> for PolkadotProtocol {
 				if let Some((new_primary, primary_info)) = new_primary {
 					primary_info.collator_state.set_role(Role::Primary, |msg| send_polkadot_message(
 						ctx,
-						new_primary,
+						new_primary.clone(),
 						msg,
 					));
 				}
@@ -502,7 +502,7 @@ impl Specialization<Block> for PolkadotProtocol {
 		}
 	}
 
-	fn on_message(&mut self, ctx: &mut Context<Block>, who: NodeIndex, message: &mut Option<message::Message<Block>>) {
+	fn on_message(&mut self, ctx: &mut Context<Block>, who: PeerId, message: &mut Option<message::Message<Block>>) {
 		match message.take() {
 			Some(generic_message::Message::ChainSpecific(raw)) => {
 				match Message::decode(&mut raw.as_slice()) {
@@ -532,7 +532,7 @@ impl Specialization<Block> for PolkadotProtocol {
 				Action::NewRole(account_id, role) => if let Some((collator, info)) = self.collator_peer(account_id) {
 					info.collator_state.set_role(role, |msg| send_polkadot_message(
 						ctx,
-						collator,
+						collator.clone(),
 						msg,
 					))
 				},
@@ -548,7 +548,7 @@ impl Specialization<Block> for PolkadotProtocol {
 
 impl PolkadotProtocol {
 	// we received a collation from a peer
-	fn on_collation(&mut self, ctx: &mut Context<Block>, from: NodeIndex, relay_parent: Hash, collation: Collation) {
+	fn on_collation(&mut self, ctx: &mut Context<Block>, from: PeerId, relay_parent: Hash, collation: Collation) {
 		let collation_para = collation.receipt.parachain_index;
 		let collated_acc = collation.receipt.collator.clone();
 
@@ -577,7 +577,7 @@ impl PolkadotProtocol {
 	}
 
 	// get connected peer with given account ID for collation.
-	fn collator_peer(&mut self, collator_id: CollatorId) -> Option<(NodeIndex, &mut PeerInfo)> {
+	fn collator_peer(&mut self, collator_id: CollatorId) -> Option<(PeerId, &mut PeerInfo)> {
 		let check_info = |info: &PeerInfo| info
 			.collating_for
 			.as_ref()
@@ -586,7 +586,7 @@ impl PolkadotProtocol {
 		self.peers
 			.iter_mut()
 			.filter(|&(_, ref info)| check_info(&**info))
-			.map(|(who, info)| (*who, info))
+			.map(|(who, info)| (who.clone(), info))
 			.next()
 	}
 
@@ -616,7 +616,7 @@ impl PolkadotProtocol {
 					debug!(target: "p_net", "Sending local collation to {:?}", primary);
 					send_polkadot_message(
 						ctx,
-						*who,
+						who.clone(),
 						Message::Collation(relay_parent, cloned_collation),
 					)
 				},