Add try-runtime with storage migrations of the encointer-pallets

Cargo.toml

@@ -10,20 +10,20 @@ members = [
 panic = "unwind"
 
 #only while debugging/developping
-# [patch."https://github.com/encointer/pallets"]
-# encointer-balances-tx-payment = { path = "../pallets/balances-tx-payment" }
-# #encointer-balances-tx-payment-rpc = { path = "../pallets/balances-tx-payment/rpc" }
-# #encointer-balances-tx-payment-rpc-runtime-api = { path = "../pallets/balances-tx-payment/rpc/runtime-api" }
-# encointer-ceremonies-assignment = { path = "../pallets/ceremonies/assignment" }
-# encointer-primitives = { path = "../pallets/primitives" }
-# pallet-encointer-ceremonies = { path = "../pallets/ceremonies" }
-# pallet-encointer-ceremonies-rpc = { path = "../pallets/ceremonies/rpc" }
-# pallet-encointer-ceremonies-rpc-runtime-api = { path = "../pallets/ceremonies/rpc/runtime-api" }
-# pallet-encointer-communities = { path = "../pallets/communities" }
-# pallet-encointer-communities-rpc = { path = "../pallets/communities/rpc" }
-# pallet-encointer-communities-rpc-runtime-api = { path = "../pallets/communities/rpc/runtime-api" }
-# pallet-encointer-balances = { path = "../pallets/balances" }
-# pallet-encointer-scheduler = { path = "../pallets/scheduler" }
-# pallet-encointer-bazaar = { path = "../pallets/bazaar" }
-# pallet-encointer-bazaar-rpc = { path = "../pallets/bazaar/rpc" }
-# pallet-encointer-bazaar-rpc-runtime-api = { path = "../pallets/bazaar/rpc/runtime-api" }
+#[patch."https://github.com/encointer/pallets"]
+#encointer-balances-tx-payment = { path = "../encointer-pallets/balances-tx-payment" }
+#encointer-balances-tx-payment-rpc = { path = "../encointer-pallets/balances-tx-payment/rpc" }
+#encointer-balances-tx-payment-rpc-runtime-api = { path = "../encointer-pallets/balances-tx-payment/rpc/runtime-api" }
+#encointer-ceremonies-assignment = { path = "../encointer-pallets/ceremonies/assignment" }
+#encointer-primitives = { path = "../encointer-pallets/primitives" }
+#pallet-encointer-ceremonies = { path = "../encointer-pallets/ceremonies" }
+#pallet-encointer-ceremonies-rpc = { path = "../encointer-pallets/ceremonies/rpc" }
+#pallet-encointer-ceremonies-rpc-runtime-api = { path = "../encointer-pallets/ceremonies/rpc/runtime-api" }
+#pallet-encointer-communities = { path = "../encointer-pallets/communities" }
+#pallet-encointer-communities-rpc = { path = "../encointer-pallets/communities/rpc" }
+#pallet-encointer-communities-rpc-runtime-api = { path = "../encointer-pallets/communities/rpc/runtime-api" }
+#pallet-encointer-balances = { path = "../encointer-pallets/balances" }
+#pallet-encointer-scheduler = { path = "../encointer-pallets/scheduler" }
+#pallet-encointer-bazaar = { path = "../encointer-pallets/bazaar" }
+#pallet-encointer-bazaar-rpc = { path = "../encointer-pallets/bazaar/rpc" }
+#pallet-encointer-bazaar-rpc-runtime-api = { path = "../encointer-pallets/bazaar/rpc/runtime-api" }

README.md

@@ -136,6 +136,23 @@ encointer-collator \
 * The relay chain and the collator need to be about equally recent. This might require frequent rebasing of this repository on the `rococo-v1` branch.
 * Sanity check: The genesis state is printed when starting the collator. Make sure it matches the one from the `genesis.state` file.
 
+## Test state migrations on live data
+Compile the node with
+
+```bash
+cargo build --release --features try-runtime
+```
+
+Then run the state migrations with live data from the encointer-kusama parachain
+
+```bash
+./target/release/encointer-collator try-runtime \
+  --chain encointer-kusama-local \
+  --runtime ./target/release/wbuild/encointer-runtime/encointer_runtime.wasm \
+  on-runtime-upgrade --checks=all \
+  live --uri wss://kusama.api.encointer.org:443
+```
+
 ## Benchmark the runtimes
 In `./scripts` we have two scripts for benchmarking the runtimes.
 

polkadot-parachains/Cargo.toml

@@ -70,7 +70,7 @@ sp-consensus-aura = { git = "https://github.com/paritytech/substrate", branch =
 sc-sysinfo = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.37" }
 sp-serializer = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.37" }
 substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.37" }
-try-runtime-cli = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.37" }
+try-runtime-cli = { git = "https://github.com/paritytech/substrate", optional = true , branch = "polkadot-v0.9.37" }
 
 # RPC related dependencies
 sc-transaction-pool-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.37" }
@@ -112,3 +112,7 @@ runtime-benchmarks = [
 	'launch-runtime/runtime-benchmarks',
 	'parachain-runtime/runtime-benchmarks',
 ]
+try-runtime = [
+	"parachain-runtime/try-runtime",
+	"try-runtime-cli/try-runtime",
+]

polkadot-parachains/encointer-runtime/Cargo.toml

@@ -42,6 +42,8 @@ sp-block-builder = { git = "https://github.com/paritytech/substrate", default-fe
 sp-transaction-pool = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.37" }
 sp-inherents = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.37" }
 sp-consensus-aura = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.37" }
+frame-try-runtime = { git = "https://github.com/paritytech/substrate.git", default-features = false,branch = "polkadot-v0.9.37", optional = true }
+try-runtime-cli = { git = "https://github.com/paritytech/substrate.git", default-features = false,branch = "polkadot-v0.9.37", optional = true }
 
 frame-support = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.37" }
 frame-executive = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.37" }
@@ -178,4 +180,37 @@ std = [
 	"pallet-encointer-scheduler/std",
 #	"pallet-encointer-personhood-oracle/std",
 #	"pallet-encointer-sybil-gate-template/std",
+	"frame-try-runtime/std",	
 ]
+
+
+try-runtime = [
+	"frame-try-runtime/try-runtime",
+	"frame-system/try-runtime",
+	"frame-executive/try-runtime",
+	"cumulus-pallet-parachain-system/try-runtime",
+	"pallet-randomness-collective-flip/try-runtime",
+	"pallet-timestamp/try-runtime",
+	"parachain-info/try-runtime",
+	"pallet-balances/try-runtime",
+	"pallet-transaction-payment/try-runtime",
+	"pallet-asset-tx-payment/try-runtime",
+	"pallet-aura/try-runtime",
+	"cumulus-pallet-aura-ext/try-runtime",
+	"cumulus-pallet-xcmp-queue/try-runtime",
+	"pallet-xcm/try-runtime",
+	"cumulus-pallet-xcm/try-runtime",
+	"cumulus-pallet-dmp-queue/try-runtime",
+	"pallet-utility/try-runtime",
+	"pallet-treasury/try-runtime",
+	"pallet-proxy/try-runtime",
+	"pallet-scheduler/try-runtime",
+	"pallet-collective/try-runtime",
+	"pallet-membership/try-runtime",
+	"pallet-encointer-scheduler/try-runtime",
+	"pallet-encointer-ceremonies/try-runtime",
+	"pallet-encointer-communities/try-runtime",
+	"pallet-encointer-balances/try-runtime",
+	"pallet-encointer-bazaar/try-runtime"
+]
+

polkadot-parachains/encointer-runtime/src/lib.rs

@@ -89,6 +89,7 @@ pub use encointer_primitives::{
 	communities::{CommunityIdentifier, Location},
 	scheduler::CeremonyPhaseType,
 };
+use sp_core::ConstU32;
 
 // XCM imports
 // Polkadot imports
@@ -454,13 +455,18 @@ impl pallet_encointer_ceremonies::Config for Runtime {
 	type MeetupNewbieLimitDivider = MeetupNewbieLimitDivider;
 	type CeremonyMaster = MoreThanHalfCouncil;
 	type WeightInfo = weights::pallet_encointer_ceremonies::WeightInfo<Runtime>;
+	type MaxAttestations = ConstU32<100>;
 }
 
 impl pallet_encointer_communities::Config for Runtime {
 	type RuntimeEvent = RuntimeEvent;
 	type CommunityMaster = MoreThanHalfCouncil;
 	type TrustableForNonDestructiveAction = MoreThanHalfCouncil;
 	type WeightInfo = weights::pallet_encointer_communities::WeightInfo<Runtime>;
+	type MaxCommunityIdentifiers = ConstU32<10000>;
+	type MaxBootstrappers = ConstU32<10000>;
+	type MaxLocationsPerGeohash = ConstU32<10000>;
+	type MaxCommunityIdentifiersPerGeohash = ConstU32<10000>;
 }
 
 impl pallet_encointer_balances::Config for Runtime {
@@ -629,7 +635,10 @@ pub type Executive = frame_executive::Executive<
 	frame_system::ChainContext<Runtime>,
 	Runtime,
 	AllPalletsWithSystem,
-	(),
+	(
+		pallet_encointer_communities::migrations::v1::Migration<Runtime>,
+		pallet_encointer_ceremonies::migrations::v1::Migration<Runtime>,
+	),
 >;
 
 #[cfg(feature = "runtime-benchmarks")]
@@ -804,14 +813,23 @@ impl_runtime_apis! {
 
 	#[cfg(feature = "try-runtime")]
 	impl frame_try_runtime::TryRuntime<Block> for Runtime {
-		fn on_runtime_upgrade() -> (Weight, Weight) {
-			log::info!("try-runtime::on_runtime_upgrade statemine.");
-			let weight = Executive::try_runtime_upgrade().unwrap();
+		fn on_runtime_upgrade(checks: frame_try_runtime::UpgradeCheckSelect) -> (Weight, Weight) {
+			// NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to
+			// have a backtrace here. If any of the pre/post migration checks fail, we shall stop
+			// right here and right now.
+			let weight = Executive::try_runtime_upgrade(checks).unwrap();
 			(weight, RuntimeBlockWeights::get().max_block)
 		}
 
-		fn execute_block_no_check(block: Block) -> Weight {
-			Executive::execute_block_no_check(block)
+		fn execute_block(
+			block: Block,
+			state_root_check: bool,
+			signature_check: bool,
+			select: frame_try_runtime::TryStateSelect
+		) -> Weight {
+			// NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to
+			// have a backtrace here.
+			Executive::try_execute_block(block, state_root_check, signature_check, select).expect("execute-block failed")
 		}
 	}
 

polkadot-parachains/launch-runtime/Cargo.toml

@@ -30,6 +30,7 @@ sp-transaction-pool = { git = "https://github.com/paritytech/substrate", default
 sp-inherents = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.37" }
 sp-consensus-aura = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.37" }
 
+try-runtime-cli = { git = "https://github.com/paritytech/substrate.git",default-features = false, branch = "polkadot-v0.9.37", optional = true }
 frame-try-runtime = { git = "https://github.com/paritytech/substrate", default-features = false, optional = true, branch = "polkadot-v0.9.37" }
 frame-support = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.37" }
 frame-executive = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.37" }
@@ -142,9 +143,12 @@ std = [
 	"sp-consensus-aura/std",
 	# encointer
 	"runtime-common/std",
+	"frame-try-runtime/std",
 ]
 
+
 try-runtime = [
+	"frame-try-runtime/try-runtime",
 	"cumulus-pallet-aura-ext/try-runtime",
 	"cumulus-pallet-dmp-queue/try-runtime",
 	"cumulus-pallet-parachain-system/try-runtime",
@@ -159,4 +163,11 @@ try-runtime = [
 	"pallet-transaction-payment/try-runtime",
 	"pallet-xcm/try-runtime",
 	"parachain-info/try-runtime",
+	"frame-try-runtime/try-runtime",
+	"frame-support/try-runtime",
+	"pallet-balances/try-runtime",
+	"pallet-randomness-collective-flip/try-runtime",
+	"pallet-timestamp/try-runtime",
+	"pallet-transaction-payment/try-runtime",
+	"pallet-utility/try-runtime"
 ]

polkadot-parachains/src/command.rs

@@ -374,30 +374,33 @@ pub fn run() -> Result<()> {
 		},
 		#[cfg(feature = "try-runtime")]
 		Some(Subcommand::TryRuntime(cmd)) => {
-			if cfg!(feature = "try-runtime") {
-				// grab the task manager.
-				let runner = cli.create_runner(cmd)?;
-				let registry = &runner.config().prometheus_config.as_ref().map(|cfg| &cfg.registry);
-				let task_manager =
-					TaskManager::new(runner.config().tokio_handle.clone(), *registry)
-						.map_err(|e| format!("Error: {:?}", e))?;
-
-				if runner.config().chain_spec.is_launch() {
-					runner.async_run(|config| {
-						Ok((cmd.run::<Block, LaunchParachainRuntimeExecutor>(config), task_manager))
-					})
-				} else if runner.config().chain_spec.is_encointer() {
-					runner.async_run(|config| {
-						Ok((
-							cmd.run::<Block, EncointerParachainRuntimeExecutor>(config),
-							task_manager,
-						))
-					})
-				} else {
-					Err("Chain doesn't support try-runtime".into())
-				}
+			// grab the task manager.
+			let runner = cli.create_runner(cmd)?;
+			let registry = &runner.config().prometheus_config.as_ref().map(|cfg| &cfg.registry);
+			let task_manager =
+				sc_service::TaskManager::new(runner.config().tokio_handle.clone(), *registry)
+					.map_err(|e| format!("Error: {:?}", e))?;
+			use sc_executor::{sp_wasm_interface::ExtendedHostFunctions, NativeExecutionDispatch};
+			type HostFunctionsOf<E> = ExtendedHostFunctions<
+				sp_io::SubstrateHostFunctions,
+				<E as NativeExecutionDispatch>::ExtendHostFunctions,
+			>;
+			if runner.config().chain_spec.is_launch() {
+				runner.async_run(|_| {
+					Ok((
+						cmd.run::<Block, HostFunctionsOf<LaunchParachainRuntimeExecutor>>(),
+						task_manager,
+					))
+				})
+			} else if runner.config().chain_spec.is_encointer() {
+				runner.async_run(|_| {
+					Ok((
+						cmd.run::<Block, HostFunctionsOf<EncointerParachainRuntimeExecutor>>(),
+						task_manager,
+					))
+				})
 			} else {
-				Err("Try-runtime must be enabled by `--features try-runtime`.".into())
+				Err("Chain doesn't support try-runtime".into())
 			}
 		},
 		#[cfg(not(feature = "try-runtime"))]

polkadot-parachains/src/rpc.rs

@@ -80,24 +80,17 @@ where
 
 	module.merge(System::new(client.clone(), pool, deny_unsafe).into_rpc())?;
 	module.merge(TransactionPayment::new(client.clone()).into_rpc())?;
-	module.merge(BazaarRpc::new(client.clone(), deny_unsafe).into_rpc())?;
+	module.merge(BazaarRpc::new(client.clone()).into_rpc())?;
 
 	match backend.offchain_storage() {
 		Some(storage) => {
 			module.merge(
-				CommunitiesRpc::new(
-					client.clone(),
-					storage.clone(),
-					offchain_indexing_enabled,
-					deny_unsafe,
-				)
-				.into_rpc(),
-			)?;
-
-			module.merge(
-				CeremoniesRpc::new(client, deny_unsafe, storage, offchain_indexing_enabled)
+				CommunitiesRpc::new(client.clone(), storage.clone(), offchain_indexing_enabled)
 					.into_rpc(),
 			)?;
+
+			module
+				.merge(CeremoniesRpc::new(client, storage, offchain_indexing_enabled).into_rpc())?;
 		},
 		None => log::warn!(
 			"Offchain caching disabled, due to lack of offchain storage support in backend. \n