Remove-rate-limit (#393)

Refactor packet sequence handling in query methods (#394) * Refactor packet sequence handling in query methods Replaced `HashSet` with `HashMap` for efficient and accurate packet sequence handling in `query_send_packets` and `query_received_packets` methods. The change optimizes the query process by ensuring that the latest block events are tracked correctly. * add a comment Ensure target block headers are within votes ancestries (#396) * Ensure target block headers are within votes ancestries Added necessary logic to fetch and include the target block header manually in the event it is not found within the returned justification's votes ancestries. This resolves potential issues where the target block header may not be immediately available within the ancestries of the justification. * add the target only if the ancestries are not empty chore: upgrade to polkadot-v0.9.43 upgrade hash-db more hash-db sp_consensus_grandpa rename fungibles::Transfer becomes Mutate paritytech/substrate#12951 update toolchain keystore updates benchmarks compiling compilation errors comment cargo fmt propagate dependencies more updates updates nasty trick to make it compile more fmt build network rm rootevent rm Root rm more rootevent life is hard replace codec fix codegen update polkadot-launch upgrade types to match version release-v9.10037.1 (#399) * upgrade types to match version release-v9.10037.1 * fmt change Keystore for MemoryStore change preservation parameter in transfer function from Preservation::Protect to Preservation::Expendable rm logger fix unused import bullish change wsPort for rpcPort on parachain-launch/config fix ports udeps is happy make docker-compose work fine Create config.json Signed-off-by: dzmitry-lahoda <dzmitry@lahoda.pro> fixed zombie chain id Signed-off-by: dzmitry-lahoda <dzmitry@lahoda.pro> more like PL config Signed-off-by: dzmitry-lahoda <dzmitry@lahoda.pro> script to ZN start Signed-off-by: dzmitry-lahoda <dzmitry@lahoda.pro> added zombienet guide Signed-off-by: dzmitry-lahoda <dzmitry@lahoda.pro> add zombienet to ci fixing workflow to enable nix installation merge install nix with installation for zombienet nix before zombienet change version and add nix-build remove old nativ polka launch, added logs from it to ZN fixing ci nix install using other action test nix first br propagate runtime-benchmarks docker thing is that need to disable broadcast discovery, to avoid connecting to wrong relay (similar to docker network isolation) fixing nix ci docker isntall proper ports probe 100% same nix install as in composable repo on official GH runners runs try nix-quick-install-actio install manually get {ZN, process-comopse} as binaries (ditching NIX) more fixes wrong pwd fixed ports and run cluster in background clean rm nodejs installation fix port of second network oh go devs no dashes get some logs chmod everybody chmod polkadot sleep before checking status of parachains add BenchmarkHelper on mock::Test query PARA_HOST fix typo in script reduced amount of blocks to wait, tested locally - ports are super ok and make READY status wheriam ports no docker ... no netstat given same set of product, one cooks bad and one well debug netsat very high sleep just to debug nohup sleep after checks changing step Wait until parachains are ready don't run process compose remove the nohup fixing lines 132 and 133 revert v2 of actions unify steps relative path avoid installing parachain-node feature flag BenchmarkHelper and propagate runtime-benchmarks feature flag BenchmarkHelper and propagate runtime-benchmarks only one relaychain to rule them all update url of static server remove unsafe_cast_to_jsonrpsee_client update the timestamp fix build. Revert "fix build." This reverts commit 8cfb09d. Revert "update the timestamp" This reverts commit 2059cf3. final fix for extrinsic tests remove unsafe_cast_to_jsonrpsee_client compilation issue fix keystore fix weight decoding key is compact u32 increase timeouts update ports on zombienet output stack trace on fail debug
ComposableFi · Sep 15, 2023 · f24d60d · f24d60d
1 parent 750a0f9
commit f24d60d
Show file tree

Hide file tree

Showing 45 changed files with 13,784 additions and 23,604 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -34,6 +34,8 @@ jobs:
           apt update -y
           apt install git -y
           git --version
+          whoami
+          uname -a          
 
       - uses: actions/checkout@v2
         with:
@@ -55,7 +57,10 @@ jobs:
             libclang-dev \
             pkg-config \
             libssl-dev \
-            unzip
+            unzip \
+            net-tools
+          
+          netstat -ltup | grep LISTEN
 
           # Get Rust
           curl https://sh.rustup.rs -sSf | bash -s -- -y
@@ -70,13 +75,17 @@ jobs:
       - name: Rustup show
         run: $HOME/.cargo/bin/rustup show
 
-      - name: Install Node.js
+      - name: Install zombienet
         run: |
-          curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
-          apt-get install -y nodejs
-          npm install --global yarn
-          node --version
-          yarn --version
+          curl -L https://github.com/paritytech/zombienet/releases/download/v1.3.67/zombienet-linux-x64 -o /usr/bin/zombienet && \
+          chmod +x /usr/bin/zombienet
+
+      - name: Install process compose
+        run: |
+            curl -L https://github.com/F1bonacc1/process-compose/releases/download/v0.60.0/process-compose_Linux_x86_64.tar.gz -o /tmp/process-compose_Linux_x86_64.tar.gz && \
+            tar xvfz /tmp/process-compose_Linux_x86_64.tar.gz && \
+            mv process-compose /usr/bin/process-compose && \
+            chmod +x /usr/bin/process-compose
 
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
@@ -86,100 +95,27 @@ jobs:
       - name: Install latest nextest release
         uses: taiki-e/install-action@nextest
 
-      - name: Install Docker
+      - name: Install polkadot binary
         run: |
-          apt-get install -y \
-            ca-certificates \
-            gnupg \
-            lsb-release
-
-          mkdir -p /etc/apt/keyrings
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-
-          echo \
-            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-            $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
-
-          apt-get update
-          apt-get install -y \
-            docker-ce \
-            docker-ce-cli \
-            containerd.io \
-            docker-compose-plugin
-
-          docker --version
-
-      - name: Install docker-compose
-        run: |
-          curl -L "https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-$(uname -s)-$(uname -m)" -o ./docker-compose
-          chmod +x ./docker-compose
-
-      - name: Store network gateway IP
-        run: |
-          # GATEWAY_IP equals host ip where this docker container was ran, this ip has access to Docker port mapping.
-          GATEWAY_IP=$(ip r l | grep default | awk '{print $3}')
-          echo "GATEWAY_IP=$GATEWAY_IP" >> $GITHUB_ENV
-          echo "RELAY_HOST=$GATEWAY_IP" >> $GITHUB_ENV
-          echo "PARA_HOST=$GATEWAY_IP" >> $GITHUB_ENV
-
-      - name: Run local parachain cluster
-        run: |
-          DOCKER_BUILDKIT=0 docker build --platform linux/amd64 -f scripts/parachain.Dockerfile . -t parachain-node:latest
-
-          mkdir $BUILD_DIR/tmp
-          yarn --cwd ./scripts/parachain-launch install
-          TMPDIR=$BUILD_DIR/tmp OUTPUT=$BUILD_DIR yarn --cwd ./scripts/parachain-launch dev
-          for file in dev-2000-2000.json \
-            dev-2001-2001.json \
-            docker-compose.yml \
-            parachain-2000.Dockerfile \
-            parachain-2001.Dockerfile \
-            relaychain.Dockerfile \
-            rococo-local.json; do
-            cp "$BUILD_DIR/$file" ./scripts/parachain-launch/
-          done
-          sed -i 's/9944:9933/9144:9933/g' ./scripts/parachain-launch/docker-compose.yml
+          curl -L https://github.com/paritytech/polkadot/releases/download/v0.9.43/polkadot -o /usr/bin/polkadot && \
+          chmod +x /usr/bin/polkadot
 
-          ./docker-compose -f scripts/parachain-launch/docker-compose.yml up -d --build
-          ./scripts/wait_for_tcp_port_opening.sh $RELAY_HOST 9944
-          ./scripts/wait_for_tcp_port_opening.sh $PARA_HOST 9188
-
-      - name: Check that subxt-generated is up to date
+      - name: Install parachain node
         run: |
-          RELAY_HOST=ws://$GATEWAY_IP:9944 PARA_HOST=ws://$GATEWAY_IP:9188 ./scripts/generate-subxt.sh
-
-      - name: Run all unit tests
+          mkdir -p ./target/release/
+          curl -L http://34.70.93.30/parachain-node -o ./target/release/parachain-node
+          cp ./target/release/parachain-node /usr/bin/parachain-node
+          chmod +x /usr/bin/parachain-node
+  
+      - name: Run local parachain cluster & Tests
         run: |
+          export PARA_HOST=127.0.0.1
+          cd scripts/zombienet
+          process-compose up -f process-compose.yml -t=false &
+          sleep 100
+          netstat -ltup | grep LISTEN
+          RUST_BACKTRACE=full METRICS_PORT=39988 ./wait-for-parachains-network.sh
+          RUST_BACKTRACE=full METRICS_PORT=39998 ./wait-for-parachains-network.sh 
           SKIP_WASM_BUILD=1 cargo +nightly nextest run --locked --release --workspace --exclude ibc-derive --exclude hyperspace-testsuite
-
-#      - name: Run simnode tests
-#        run: |
-#          cargo +nightly test -p simnode --locked --release
-
-      - name: Run hyperspace integration tests
-        run: |
-          cargo +nightly test -p hyperspace-testsuite --locked --release
-
-      - name: Terminate local parachain cluster
-        run: |
-          ./docker-compose -f scripts/parachain-launch/docker-compose.yml down -v
-
-  test-solidity:
-    runs-on:
-      - self-hosted
-      - sre
-    defaults:
-      run:
-        shell: bash
-        working-directory: contracts/ethereum
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: recursive
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 16
-      - name: Install dependencies
-        run: npm install
-      - name: run unit test
-        run: npm run test
+          RUST_BACKTRACE=full cargo +nightly test -p hyperspace-testsuite --locked --debug
+          process-compose down -f process-compose.yml
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/algorithms/beefy/prover/src/helpers.rs b/algorithms/beefy/prover/src/helpers.rs
@@ -66,8 +66,7 @@ pub async fn fetch_timestamp_extrinsic_with_proof<T: Config>(
 		))
 	})?;
 
-	let extrinsics = block.block.extrinsics.into_iter().map(|e| e.0.encode()).collect::<Vec<_>>();
-
+	let extrinsics = block.block.extrinsics.into_iter().map(|e| e.0).collect::<Vec<_>>();
 	let (ext, proof) = {
 		if extrinsics.is_empty() {
 			return Err(From::from("Block has no extrinsics".to_string()))
@@ -80,15 +79,14 @@ pub async fn fetch_timestamp_extrinsic_with_proof<T: Config>(
 			let mut root = Default::default();
 			let mut trie =
 				<TrieDBMutBuilder<sp_trie::LayoutV0<BlakeTwo256>>>::new(&mut db, &mut root).build();
-
 			for (i, ext) in extrinsics.into_iter().enumerate() {
-				let key = codec::Compact(i as u32).encode();
+				let key = codec::Compact(i as u64).encode();
 				trie.insert(&key, &ext)?;
 			}
 			*trie.root()
 		};
-
 		let key = codec::Compact::<u32>(0u32).encode();
+
 		let extrinsic_proof =
 			generate_trie_proof::<sp_trie::LayoutV0<BlakeTwo256>, _, _, _>(&db, root, vec![&key])?;
 

diff --git a/algorithms/beefy/verifier/src/lib.rs b/algorithms/beefy/verifier/src/lib.rs
@@ -222,7 +222,7 @@ where
 		// Verify timestamp extrinsic
 		// Timestamp extrinsic should be the first inherent and hence the first extrinsic
 		// https://github.com/paritytech/substrate/blob/d602397a0bbb24b5d627795b797259a44a5e29e9/primitives/trie/src/lib.rs#L99-L101
-		let timestamp_ext_key = codec::Compact(0u32).encode();
+		let timestamp_ext_key = codec::Compact(0u64).encode();
 		sp_trie::verify_trie_proof::<LayoutV0<H::BlakeTwo256>, _, _, _>(
 			&decoded_para_header.extrinsics_root,
 			&&*parachain_header.extrinsic_proof,

diff --git a/algorithms/grandpa/prover/src/lib.rs b/algorithms/grandpa/prover/src/lib.rs
@@ -19,9 +19,7 @@
 
 use anyhow::anyhow;
 pub use beefy_prover;
-use beefy_prover::helpers::{
-	fetch_timestamp_extrinsic_with_proof, unsafe_arc_cast, TimeStampExtWithProof,
-};
+use beefy_prover::helpers::{fetch_timestamp_extrinsic_with_proof, TimeStampExtWithProof};
 use codec::{Decode, Encode};
 use finality_grandpa_rpc::GrandpaApiClient;
 use jsonrpsee::{async_client::Client, tracing::log, ws_client::WsClientBuilder};
@@ -264,11 +262,7 @@ where
 		} else {
 			let encoded = GrandpaApiClient::<JustificationNotification, H256, u32>::prove_finality(
 				// we cast between the same type but different crate versions.
-				&*unsafe {
-					unsafe_arc_cast::<_, jsonrpsee_ws_client::WsClient>(
-						self.relay_ws_client.clone(),
-					)
-				},
+				&*self.relay_ws_client.clone(),
 				latest_finalized_height,
 			)
 			.await?

diff --git a/algorithms/grandpa/verifier/src/lib.rs b/algorithms/grandpa/verifier/src/lib.rs
@@ -128,7 +128,7 @@ where
 		para_heights.push(parachain_header.number().clone().into());
 		// Timestamp extrinsic should be the first inherent and hence the first extrinsic
 		// https://github.com/paritytech/substrate/blob/d602397a0bbb24b5d627795b797259a44a5e29e9/primitives/trie/src/lib.rs#L99-L101
-		let key = codec::Compact(0u32).encode();
+		let key = codec::Compact(0u64).encode();
 		// verify extrinsic proof for timestamp extrinsic
 		sp_trie::verify_trie_proof::<LayoutV0<Host::BlakeTwo256>, _, _, _>(
 			parachain_header.extrinsics_root(),

diff --git a/algorithms/grandpa/verifier/src/tests.rs b/algorithms/grandpa/verifier/src/tests.rs
@@ -92,10 +92,7 @@ async fn follow_grandpa_justifications() {
 		let next_relay_height = client_state.latest_relay_height + 1;
 
 		let encoded = finality_grandpa_rpc::GrandpaApiClient::<JustificationNotification, H256, u32>::prove_finality(
-			// we cast between the same type but different crate versions.
-			&*unsafe {
-				unsafe_arc_cast::<_, jsonrpsee_ws_client::WsClient>(prover.relay_ws_client.clone())
-			},
+				&*prover.relay_ws_client.clone(),
 			next_relay_height,
 		)
 			.await
@@ -127,9 +124,6 @@ async fn follow_grandpa_justifications() {
 			justification.commit.target_hash, justification.commit.target_number
 		);
 
-		dbg!(&client_state.latest_para_height);
-		dbg!(&header_numbers);
-
 		let proof = prover
 			.query_finalized_parachain_headers_with_proof::<SubstrateHeader<u32, BlakeTwo256>>(
 				client_state.latest_relay_height,

diff --git a/contracts/pallet-ibc/Cargo.toml b/contracts/pallet-ibc/Cargo.toml
@@ -3,35 +3,49 @@ edition = "2021"
 homepage = "https://composable.finance"
 name = "pallet-ibc"
 version = "0.0.1"
-authors = ["Seun Lanlege <seunlanlege@gmail.com>", "David Salami <wizdave97@gmail.com>"]
+authors = [
+	"Seun Lanlege <seunlanlege@gmail.com>",
+	"David Salami <wizdave97@gmail.com>",
+]
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
 
 [package.metadata.cargo-udeps.ignore]
-normal = ["frame-benchmarking", "pallet-ibc-ping", "balances", "pallet-assets"]
+normal = [
+	"frame-benchmarking",
+	"pallet-ibc-ping",
+	"balances",
+	"pallet-assets",
+	"simple-iavl",
+]
 
 [dependencies]
 # crates.io
 log = { version = "0.4.0", default-features = false }
 serde = { version = "1.0.136", default-features = false, features = ["derive"] }
-scale-info = { version = "2.1.1", default-features = false, features = ["derive"] }
+scale-info = { version = "2.1.1", default-features = false, features = [
+	"derive",
+] }
 serde_json = { version = "1.0.45", default-features = false }
 sha2 = { version = "0.10.2", default-features = false }
-derive_more = { version = "0.99.17", default-features = false, features = ["from", "display"]  }
+derive_more = { version = "0.99.17", default-features = false, features = [
+	"from",
+	"display",
+] }
 # substrate
 frame-support = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false }
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false }
 parachain-info = { git = "https://github.com/paritytech/cumulus", branch = "polkadot-v0.9.43", default-features = false }
-cumulus-primitives-core  = { git = "https://github.com/paritytech/cumulus", branch = "polkadot-v0.9.43", default-features = false }
+cumulus-primitives-core = { git = "https://github.com/paritytech/cumulus", branch = "polkadot-v0.9.43", default-features = false }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false }
 sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false }
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false }
 sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false }
 sp-trie = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false }
 # ibc
 ibc-proto = { path = "../../ibc/proto", default-features = false }
-tendermint = { git = "https://github.com/informalsystems/tendermint-rs", rev = "e81f7bf23d63ffbcd242381d1ce5e35da3515ff1", default-features = false } # cannot be defined as optional in workspace
+tendermint = { git = "https://github.com/informalsystems/tendermint-rs", rev = "e81f7bf23d63ffbcd242381d1ce5e35da3515ff1", default-features = false }       # cannot be defined as optional in workspace
 tendermint-proto = { git = "https://github.com/informalsystems/tendermint-rs", rev = "e81f7bf23d63ffbcd242381d1ce5e35da3515ff1", default-features = false }
 ics23 = { git = "https://github.com/cosmos/ics23", rev = "74ce807b7be39a7e0afb4e2efb8e28a57965f57b", default-features = false }
 
@@ -60,7 +74,9 @@ sp-consensus-slots = { git = "https://github.com/paritytech/substrate", branch =
 sp-consensus-aura = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false, optional = true }
 simple-iavl = { path = "simple-iavl", default-features = false }
 sp-consensus-grandpa = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false, optional = true }
-finality-grandpa = { version = "0.16.2", features = ["derive-codec"], default-features = false, optional = true }
+finality-grandpa = { version = "0.16.2", features = [
+	"derive-codec",
+], default-features = false, optional = true }
 
 [dependencies.ibc]
 path = "../../ibc/modules"
@@ -97,7 +113,7 @@ prost = { version = "0.11" }
 sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43" }
 tendermint = { git = "https://github.com/informalsystems/tendermint-rs", rev = "e81f7bf23d63ffbcd242381d1ce5e35da3515ff1", default-features = false }
 pallet-ibc-ping = { path = "ping", default-features = false }
-sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43"}
+sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43" }
 env_logger = "0.10.0"
 
 [dev-dependencies.ibc]
@@ -153,27 +169,34 @@ std = [
 	"sp-consensus-aura?/std",
 	"sp-consensus-slots?/std",
 	"sp-keystore/std",
-	"tendermint/std"
+	"tendermint/std",
 ]
 
 runtime-benchmarks = [
-  "frame-benchmarking",
-  "pallet-ibc-ping",
-  "frame-system/runtime-benchmarks",
-  "frame-support/runtime-benchmarks",
-  "hex-literal",
-  "pallet-timestamp/runtime-benchmarks",
-  "pallet-aura",
-  "pallet-membership",
-  "pallet-assets",
-  "balances",
-  "sp-consensus-slots",
-  "sp-consensus-aura/serde",
-  "sp-consensus-aura/std",
-  "ibc-primitives/runtime-benchmarks",
-  "sp-consensus-grandpa",
-  "finality-grandpa",
-  "pallet-ibc-ping/runtime-benchmarks"
+	"frame-benchmarking",
+	"pallet-ibc-ping",
+	"frame-system/runtime-benchmarks",
+	"frame-support/runtime-benchmarks",
+	"hex-literal",
+	"pallet-assets/runtime-benchmarks",
+	"pallet-timestamp/runtime-benchmarks",
+	"pallet-aura",
+	"pallet-membership",
+	"pallet-assets",
+	"balances",
+	"sp-consensus-slots",
+	"sp-consensus-aura/serde",
+	"sp-consensus-aura/std",
+	"ibc-primitives/runtime-benchmarks",
+	"sp-consensus-grandpa",
+	"finality-grandpa",
+	"pallet-ibc-ping/runtime-benchmarks",
+	"frame-benchmarking?/runtime-benchmarks",
+	"orml-tokens/runtime-benchmarks",
+	"pallet-assets?/runtime-benchmarks",
+	"balances?/runtime-benchmarks",
+	"pallet-membership?/runtime-benchmarks",
+	"sp-runtime/runtime-benchmarks"
 ]
 
 try-runtime = ["frame-support/try-runtime"]