From 7f3e8fdf33411417dfd91d378cc952e7ed54c68e Mon Sep 17 00:00:00 2001 From: Dengjianping Date: Sun, 9 Jul 2023 00:01:20 +0800 Subject: [PATCH 1/9] [no ci]Remove unused packages Signed-off-by: Dengjianping --- .github/workflows/metadata_diff.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index 89f6e5edc..5174efd53 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -75,6 +75,17 @@ jobs: sudo apt update sudo apt install -y pkg-config libssl-dev protobuf-compiler protoc --version + df -h + sudo apt remove -y '^dotnet-.*' + sudo apt remove -y '^mongodb-.*' + sudo apt remove -y '^mysql-.*' + sudo apt remove -y '^postgresql-*' + sudo apt remove -y google-cloud-sdk google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri + sudo apt autoremove -y + sudo apt clean + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + df -h curl -s https://sh.rustup.rs -sSf | sh -s -- -y source ${HOME}/.cargo/env rustup toolchain install nightly-2023-03-03 From e8434d5ee42dc51de02e345c6cb0cf6a1b3c40f1 Mon Sep 17 00:00:00 2001 From: Dengjianping Date: Sun, 9 Jul 2023 17:37:25 +0800 Subject: [PATCH 2/9] [no ci]Switch to 22.04 Signed-off-by: Dengjianping --- .github/workflows/metadata_diff.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index 5174efd53..33b1f520f 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -20,7 +20,7 @@ env: jobs: start-checks: timeout-minutes: 120 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: runner-label: ${{ steps.start-self-hosted-runner.outputs.runner-label }} aws-region: ${{ steps.start-self-hosted-runner.outputs.aws-region }} @@ -75,7 +75,6 @@ jobs: sudo apt update sudo apt install -y pkg-config libssl-dev protobuf-compiler protoc --version - df -h sudo apt remove -y '^dotnet-.*' sudo apt remove -y '^mongodb-.*' sudo apt remove -y '^mysql-.*' @@ -85,7 +84,6 @@ jobs: sudo apt clean sudo rm -rf /usr/local/lib/android sudo rm -rf /usr/share/dotnet - df -h curl -s https://sh.rustup.rs -sSf | sh -s -- -y source ${HOME}/.cargo/env rustup toolchain install nightly-2023-03-03 From 43347506639821fe9822e34efa88c95f6d625726 Mon Sep 17 00:00:00 2001 From: Dengjianping Date: Mon, 10 Jul 2023 00:25:40 +0800 Subject: [PATCH 3/9] [no ci]Replace docker image Signed-off-by: Dengjianping --- .github/workflows/metadata_diff.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index 33b1f520f..33799e589 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -123,11 +123,11 @@ jobs: echo "Chain: $CHAIN" >> output.txt echo "----------------------------------------------------------------------" >> output.txt - name: Pull polkadot-js-tools image - run: docker pull wilwade/polkadot-js-tools + run: docker pull jacogr/polkadot-js-tools - name: Compare the metadata run: | - CMD="docker run --pull always --network host wilwade/polkadot-js-tools metadata ws://localhost:9944 ws://localhost:9955" + CMD="docker run --pull always --network host jacogr/polkadot-js-tools metadata ws://localhost:9944 ws://localhost:9955" echo -e "Running:\n$CMD" $CMD >> output.txt sed -z -i 's/\n\n/\n/g' output.txt From 479711e8d31258f284c06e3e1f1fcf2467377e0a Mon Sep 17 00:00:00 2001 From: Dengjianping Date: Mon, 10 Jul 2023 11:01:06 +0800 Subject: [PATCH 4/9] [no ci]Compare with mainnet instead of local node, and support manta runtime Signed-off-by: Dengjianping --- .github/workflows/metadata_diff.yml | 35 ++++++++++++++++------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index 33799e589..779559b81 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -2,9 +2,9 @@ name: Transaction Version Update Check on: workflow_dispatch: inputs: - reference_binary: - description: The link to the reference binary, which should be the previous release. - required: true + endpoint: + description: The endpoint to the calamari or manta mainnet. + required: false chain: description: The name of the chain under test. Usually, you would pass a local chain default: manta-local @@ -16,7 +16,7 @@ env: AWS_IMAGE_SEARCH_PATTERN: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-* AWS_IMAGE_SEARCH_OWNERS: '["099720109477"]' CHAIN: ${{github.event.inputs.chain}} - REF_BINARY: ${{github.event.inputs.reference_binary}} + ENDPOINT: ${{github.event.inputs.endpoint}} jobs: start-checks: timeout-minutes: 120 @@ -101,19 +101,17 @@ jobs: chmod +x target/release/manta - name: stop sccache server run: sccache --stop-server || true - - name: Fetch and chmod old release binary - run: | - curl -L -o $HOME/manta-old $REF_BINARY - chmod +x $HOME/manta-old - ls -ahl $HOME/ - - name: Start old node - run: | - echo Running on $CHAIN - $HOME/manta-old --chain=$CHAIN --port 30444 --base-path ./temp1 --ws-port 9944 -- --chain kusama & - - name: Start new node + - name: Start latest manta node run: | echo Running on $CHAIN - target/release/manta --chain=$CHAIN --port 30555 --base-path ./temp2 --ws-port 9955 -- --chain kusama & + RELAYCHAIN=kusama + ENDPOINT=wss://calamari.systems + if [[ "$CHAIN" == *"manta"* ]]; then + ENDPOINT=wss://ws.manta.systems + RELAYCHAIN=polkadot + echo "It's going to diff runtime metadata for $CHAIN with endpoint $ENDPOINT." + fi; + target/release/manta --chain=$CHAIN --port 30555 --base-path ./temp1 --ws-port 9955 -- --chain $RELAYCHAIN & - name: Prepare output run: | VERSION=$(./target/release/manta --version) @@ -127,7 +125,12 @@ jobs: - name: Compare the metadata run: | - CMD="docker run --pull always --network host jacogr/polkadot-js-tools metadata ws://localhost:9944 ws://localhost:9955" + ENDPOINT=wss://calamari.systems + if [[ "$CHAIN" == *"manta"* ]]; then + ENDPOINT=wss://ws.manta.systems + fi; + echo $ENDPOINT + CMD="docker run --pull always --network host jacogr/polkadot-js-tools metadata $ENDPOINT ws://localhost:9955" echo -e "Running:\n$CMD" $CMD >> output.txt sed -z -i 's/\n\n/\n/g' output.txt From 81d496a61c83dc0387e8e001cd96189815cf2dd0 Mon Sep 17 00:00:00 2001 From: Dengjianping Date: Mon, 10 Jul 2023 12:57:22 +0800 Subject: [PATCH 5/9] [no ci]Use source project instead of docker image Signed-off-by: Dengjianping --- .github/workflows/metadata_diff.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index 779559b81..e7a15a222 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -20,7 +20,10 @@ env: jobs: start-checks: timeout-minutes: 120 - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 + strategy: + matrix: + node-version: [16.x] outputs: runner-label: ${{ steps.start-self-hosted-runner.outputs.runner-label }} aws-region: ${{ steps.start-self-hosted-runner.outputs.aws-region }} @@ -39,6 +42,7 @@ jobs: aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }} aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }} - uses: actions/checkout@v2 + - uses: borales/actions-yarn@v3.0.0 - name: install sccache env: SCCACHE_RELEASE_URL: https://github.com/mozilla/sccache/releases/download @@ -107,9 +111,9 @@ jobs: RELAYCHAIN=kusama ENDPOINT=wss://calamari.systems if [[ "$CHAIN" == *"manta"* ]]; then - ENDPOINT=wss://ws.manta.systems - RELAYCHAIN=polkadot - echo "It's going to diff runtime metadata for $CHAIN with endpoint $ENDPOINT." + ENDPOINT=wss://ws.manta.systems + RELAYCHAIN=polkadot + echo "It's going to diff runtime metadata for $CHAIN with endpoint $ENDPOINT." fi; target/release/manta --chain=$CHAIN --port 30555 --base-path ./temp1 --ws-port 9955 -- --chain $RELAYCHAIN & - name: Prepare output @@ -120,17 +124,20 @@ jobs: echo "Target version: $VERSION" >> output.txt echo "Chain: $CHAIN" >> output.txt echo "----------------------------------------------------------------------" >> output.txt - - name: Pull polkadot-js-tools image - run: docker pull jacogr/polkadot-js-tools - + - uses: actions/checkout@v2 + with: + repository: polkadot-js/tools + path: tools - name: Compare the metadata run: | ENDPOINT=wss://calamari.systems if [[ "$CHAIN" == *"manta"* ]]; then - ENDPOINT=wss://ws.manta.systems + ENDPOINT=wss://ws.manta.systems fi; echo $ENDPOINT - CMD="docker run --pull always --network host jacogr/polkadot-js-tools metadata $ENDPOINT ws://localhost:9955" + cd ${{ github.workspace }}/tools + yarn + CMD="yarn run:metadata $ENDPOINT ws://localhost:9955" echo -e "Running:\n$CMD" $CMD >> output.txt sed -z -i 's/\n\n/\n/g' output.txt From 1bc1dcf4af9d1529f59ecbffdfd34c07cfa1bca5 Mon Sep 17 00:00:00 2001 From: Dengjianping Date: Mon, 10 Jul 2023 13:04:25 +0800 Subject: [PATCH 6/9] [no ci]Fix yaml Signed-off-by: Dengjianping --- .github/workflows/metadata_diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index e7a15a222..3aff5df55 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -124,7 +124,7 @@ jobs: echo "Target version: $VERSION" >> output.txt echo "Chain: $CHAIN" >> output.txt echo "----------------------------------------------------------------------" >> output.txt - - uses: actions/checkout@v2 + - uses: actions/checkout@v2 with: repository: polkadot-js/tools path: tools From b21a1f3eaec8a455e2364a95984d3cb42fae2064 Mon Sep 17 00:00:00 2001 From: Dengjianping Date: Thu, 20 Jul 2023 11:27:17 +0800 Subject: [PATCH 7/9] [no ci]Use pm2 to manage manta node Signed-off-by: Dengjianping --- .github/workflows/metadata_diff.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index 3aff5df55..3eb61289d 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -105,6 +105,8 @@ jobs: chmod +x target/release/manta - name: stop sccache server run: sccache --stop-server || true + - name: Install pm2 + run: npm install -g pm2 - name: Start latest manta node run: | echo Running on $CHAIN @@ -115,7 +117,7 @@ jobs: RELAYCHAIN=polkadot echo "It's going to diff runtime metadata for $CHAIN with endpoint $ENDPOINT." fi; - target/release/manta --chain=$CHAIN --port 30555 --base-path ./temp1 --ws-port 9955 -- --chain $RELAYCHAIN & + pm2 start --name diff-metadata "target/release/manta --chain=$CHAIN --port 30555 --base-path ./temp1 --ws-port 9955 -- --chain $RELAYCHAIN" - name: Prepare output run: | VERSION=$(./target/release/manta --version) From c4da4096d827af0da65e7fbd0572695fedb3f5b0 Mon Sep 17 00:00:00 2001 From: Dengjianping Date: Fri, 21 Jul 2023 15:38:06 +0800 Subject: [PATCH 8/9] [no ci]Use polkadot-launch to start manta node Signed-off-by: Dengjianping --- .github/workflows/metadata_diff.yml | 57 ++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index 3eb61289d..6e62e2a73 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -4,10 +4,11 @@ on: inputs: endpoint: description: The endpoint to the calamari or manta mainnet. + default: wss://ws.manta.systems required: false chain: - description: The name of the chain under test. Usually, you would pass a local chain - default: manta-local + description: The name of the chain under test. Usually, you would pass a dev chain + default: manta-dev required: true env: AWS_REGION: us-east-1 @@ -102,25 +103,45 @@ jobs: run: | source ${HOME}/.cargo/env RUSTC_BOOTSTRAP=1 cargo build --release - chmod +x target/release/manta + cp target/release/manta $HOME/manta + chmod +x $HOME/manta + - name: Fetch and chmod old release binary + run: | + curl -L -o $HOME/polkadot https://github.com/paritytech/polkadot/releases/download/v0.9.42/polkadot + chmod +x $HOME/polkadot + ls -ahl $HOME/ - name: stop sccache server run: sccache --stop-server || true - name: Install pm2 run: npm install -g pm2 - - name: Start latest manta node + - uses: actions/checkout@v2 + with: + repository: paritytech/polkadot-launch + path: polkadot-launch + - name: create launch config run: | - echo Running on $CHAIN - RELAYCHAIN=kusama - ENDPOINT=wss://calamari.systems - if [[ "$CHAIN" == *"manta"* ]]; then - ENDPOINT=wss://ws.manta.systems - RELAYCHAIN=polkadot - echo "It's going to diff runtime metadata for $CHAIN with endpoint $ENDPOINT." - fi; - pm2 start --name diff-metadata "target/release/manta --chain=$CHAIN --port 30555 --base-path ./temp1 --ws-port 9955 -- --chain $RELAYCHAIN" + cd ${{ github.workspace }}/polkadot-launch + cat ${{ github.workspace }}/.github/resources/config-for-integration-test.json | \ + jq \ + --arg relaychain_bin $HOME/polkadot \ + --arg parachains_bin $HOME/manta \ + --arg parachains_spec $CHAIN \ + '.relaychain.bin = $relaychain_bin + | .parachains[0].bin = $parachains_bin + | .parachains[].chain = $parachains_spec + ' > ${{ github.workspace }}/.github/resources/diff-metadata.json + - name: launch testnet + run: | + cd ${{ github.workspace }}/polkadot-launch + yarn install + yarn build + pm2 start dist/cli.js --name polkadot-launch \ + --output ${{ github.workspace }}/stdout.log \ + --error ${{ github.workspace }}/stderr.log \ + -- ${{ github.workspace }}/.github/resources/diff-metadata.json - name: Prepare output run: | - VERSION=$(./target/release/manta --version) + VERSION=$($HOME/manta --version) echo "Metadata comparison:" >> output.txt echo "Date: $(date)" >> output.txt echo "Target version: $VERSION" >> output.txt @@ -132,6 +153,8 @@ jobs: path: tools - name: Compare the metadata run: | + cat ${{ github.workspace }}/stdout.log + cat ${{ github.workspace }}/stderr.log ENDPOINT=wss://calamari.systems if [[ "$CHAIN" == *"manta"* ]]; then ENDPOINT=wss://ws.manta.systems @@ -139,13 +162,13 @@ jobs: echo $ENDPOINT cd ${{ github.workspace }}/tools yarn - CMD="yarn run:metadata $ENDPOINT ws://localhost:9955" + CMD="yarn run:metadata $ENDPOINT ws://127.0.0.1:9921" echo -e "Running:\n$CMD" $CMD >> output.txt sed -z -i 's/\n\n/\n/g' output.txt cat output.txt | egrep -n -i '' - chmod +x ./scripts/github/extrinsic-ordering-filter.sh - SUMMARY=$(./scripts/github/extrinsic-ordering-filter.sh output.txt) + chmod +x ${{ github.workspace }}/scripts/github/extrinsic-ordering-filter.sh + SUMMARY=$(${{ github.workspace }}/scripts/github/extrinsic-ordering-filter.sh output.txt) echo -e $SUMMARY echo -e $SUMMARY >> output.txt - name: Show result From b88d53c8f58e753f48f24e78d611ddc8e85bceaf Mon Sep 17 00:00:00 2001 From: Dengjianping Date: Mon, 24 Jul 2023 20:46:30 +0800 Subject: [PATCH 9/9] [no ci]Fix comments Signed-off-by: Dengjianping --- .github/workflows/integration_test_calamari.yml | 2 +- .github/workflows/integration_test_manta.yml | 2 +- .github/workflows/metadata_diff.yml | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration_test_calamari.yml b/.github/workflows/integration_test_calamari.yml index 3f531b3c4..0f40555ad 100644 --- a/.github/workflows/integration_test_calamari.yml +++ b/.github/workflows/integration_test_calamari.yml @@ -111,7 +111,7 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'A-calamari' || github.ref == 'refs/heads/manta') needs: [build-node-current, start-calamari-integration-tester] runs-on: ${{ needs.start-calamari-integration-tester.outputs.runner-label }} - timeout-minutes: 180 + timeout-minutes: 240 strategy: fail-fast: false matrix: diff --git a/.github/workflows/integration_test_manta.yml b/.github/workflows/integration_test_manta.yml index d9b0e5d54..de30df97a 100644 --- a/.github/workflows/integration_test_manta.yml +++ b/.github/workflows/integration_test_manta.yml @@ -111,7 +111,7 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'A-manta' || github.ref == 'refs/heads/manta') needs: [build-node-current, start-manta-integration-tester] runs-on: ${{ needs.start-manta-integration-tester.outputs.runner-label }} - timeout-minutes: 180 + timeout-minutes: 240 strategy: fail-fast: false matrix: diff --git a/.github/workflows/metadata_diff.yml b/.github/workflows/metadata_diff.yml index 6e62e2a73..5dcf582e8 100644 --- a/.github/workflows/metadata_diff.yml +++ b/.github/workflows/metadata_diff.yml @@ -153,12 +153,6 @@ jobs: path: tools - name: Compare the metadata run: | - cat ${{ github.workspace }}/stdout.log - cat ${{ github.workspace }}/stderr.log - ENDPOINT=wss://calamari.systems - if [[ "$CHAIN" == *"manta"* ]]; then - ENDPOINT=wss://ws.manta.systems - fi; echo $ENDPOINT cd ${{ github.workspace }}/tools yarn