From 1291d57a74887055df7b4802790410832cfbcb85 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 23 Dec 2020 10:44:38 +0100 Subject: [PATCH 01/10] feat: docker init script for Stacks 1.0 and Stacks 2.0 migrated state --- import-verification/.dockerignore | 1 + import-verification/Dockerfile | 72 ++++++++++++ import-verification/run.sh | 5 + import-verification/test/index.ts | 63 ++++++++++ import-verification/test/package-lock.json | 130 +++++++++++++++++++++ import-verification/test/package.json | 19 +++ import-verification/test/tsconfig.json | 9 ++ 7 files changed, 299 insertions(+) create mode 100644 import-verification/.dockerignore create mode 100644 import-verification/Dockerfile create mode 100755 import-verification/run.sh create mode 100644 import-verification/test/index.ts create mode 100644 import-verification/test/package-lock.json create mode 100644 import-verification/test/package.json create mode 100644 import-verification/test/tsconfig.json diff --git a/import-verification/.dockerignore b/import-verification/.dockerignore new file mode 100644 index 00000000000..600e365ec83 --- /dev/null +++ b/import-verification/.dockerignore @@ -0,0 +1 @@ +**/node_modules \ No newline at end of file diff --git a/import-verification/Dockerfile b/import-verification/Dockerfile new file mode 100644 index 00000000000..02126533aa2 --- /dev/null +++ b/import-verification/Dockerfile @@ -0,0 +1,72 @@ +FROM rust:buster + +### Install Stacks 2.0 +RUN git clone --depth 1 --branch v24.2.1.0-xenon https://github.com/blockstack/stacks-blockchain.git /stacks2.0 +WORKDIR /stacks2.0/testnet/stacks-node +RUN cargo fetch +RUN cargo build --release +RUN cp /stacks2.0/target/release/stacks-node /bin/stacks-node +RUN stacks-node version + +### Install Node.js +RUN apt-get update +RUN curl -sL https://deb.nodesource.com/setup_15.x | bash - +RUN apt-get install -y nodejs +RUN node --version + +### Install Stacks 1.0 +RUN git clone --depth 1 --branch v1-migration https://github.com/blockstack/stacks-blockchain.git /stacks1.0 +RUN python --version +RUN apt-get install -y python-setuptools python-pip rng-tools libgmp3-dev +RUN pip install pyparsing +WORKDIR /stacks1.0 +RUN python ./setup.py build +RUN python ./setup.py install +# RUN python ./bin/blockstack-core version +RUN blockstack-core version + +### Sync Stacks 1.0 chain +RUN blockstack-core fast_sync --working-dir /stacks1.0-chain +# use sqlite cli to set get_v2_upgrade_threshold_block +RUN apt-get install -y sqlite3 +RUN sqlite3 /stacks1.0-chain/blockstack-server.db 'UPDATE v2_upgrade_signal SET threshold_block_id = 1 WHERE id = 1' +RUN sqlite3 /stacks1.0-chain/blockstack-server.db 'UPDATE v2_upgrade_signal SET import_block_id = 1 WHERE id = 1' +RUN blockstack-core fast_sync_snapshot 0 /stacks1.0-snapshot --working-dir /stacks1.0-chain > fast_sync_snapshot.log +# Fast-sync-snapshot completed at block 662399 with consensus hash bf81462f58d511cb2ab9d9752d745af8 + +# RUN blockstack-core start --working-dir /stacks1.0-export + +RUN cat fast_sync_snapshot.log | grep "consensus hash" | tail -1 | sed "s/.*at block \(.*\) with consensus hash \(.*\).*/\1/" > export_block +RUN cat fast_sync_snapshot.log | grep "consensus hash" | tail -1 | sed "s/.*at block \(.*\) with consensus hash \(.*\).*/\2/" > consensus_hash + +RUN echo "Block $(cat export_block) hash $(cat consensus_hash)" + +# RUN cat fast_sync_snapshot.log | grep "consensus hash" | tail -1 | sed -e 's/.*block \([^"]*\)with.*/\1/' > export_block +# RUN curl -s https://core.blockstack.org/v1/blockchains/bitcoin/consensus/$(cat export_block) | jq '.consensus_hash' -r > consensus_hash + +# RUN sed 's/.*block//; s/with.*//' +# RUN sed -e 's/.*block\(.*\)with.*/\1/' +# RUN sed -e 's/.*block\([^"]*\)with.*/\1/' + +# RUN curl https://core.blockstack.org/v1/blockchains/bitcoin/consensus/662399 +# RUN curl https://core.blockstack.org/v1/blockchains/bitcoin/consensus-block/bf81462f58d511cb2ab9d9752d745af8 +# RUN curl http://localhost:6270/v1/info + +RUN rm -rf /stacks1.0-chain + +# RUN blockstack-core export_migration_json /stacks1.0-snapshot $(cat export_block) $(cat consensus_hash) --working-dir /stacks1.0-export +RUN blockstack-core export_migration_json /stacks1.0-snapshot $(cat export_block) $(cat consensus_hash) /stacks1.0-export --working-dir /stacks1.0-chain +RUN ls -al /stacks1.0-export +RUN cp /stacks1.0-export/chainstate.txt /stacks2.0/stx-genesis/chainstate.txt +RUN cp /stacks1.0-export/chainstate.txt.sha256 /stacks2.0/stx-genesis/chainstate.txt.sha256 + +WORKDIR /stacks2.0/testnet/stacks-node +RUN cargo build --release +RUN cp /stacks2.0/target/release/stacks-node /bin/stacks-node + +# RUN blockstack-core start --working-dir /stacks1.0-chain && sleep 10 && curl --max-time 2 --retry 200 http://localhost:6270/v1/info + +WORKDIR /test +COPY test ./ +RUN npm i +RUN npm test \ No newline at end of file diff --git a/import-verification/run.sh b/import-verification/run.sh new file mode 100755 index 00000000000..281be7ae6be --- /dev/null +++ b/import-verification/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd "$(dirname "$(realpath "$0")")" + +DOCKER_BUILDKIT=1 BUILDKIT_PROGRESS=plain docker build . \ No newline at end of file diff --git a/import-verification/test/index.ts b/import-verification/test/index.ts new file mode 100644 index 00000000000..3b49ffa474a --- /dev/null +++ b/import-verification/test/index.ts @@ -0,0 +1,63 @@ +import { ChildProcess, execFile, spawn } from 'child_process'; +import fetch from 'node-fetch'; + +// ## Get account balance [GET /v1/accounts/{address}/{tokenType}/balance] + + +async function main() { + console.log('Starting Stacks 1.0 node...'); + const stacksNode1Proc = spawn('blockstack-core', ['start', '--foreground', '--working-dir', '/stacks1.0-chain'], { stdio: 'inherit' }); + const stacksNode1Exit = waitProcessExit(stacksNode1Proc); + console.log('Waiting for Stacks 1.0 RPC init...'); + await awaitHttpGetSuccess('http://localhost:6270/v1/info'); + console.log('Stacks 1.0 RPC online'); + + console.log('Starting Stacks 2.0 node...'); + const stacksNode2Proc = spawn('stacks-node', ['mocknet'], { stdio: 'inherit' }); + const stacksNode2Exit = waitProcessExit(stacksNode2Proc); + console.log('Waiting for Stacks 1.0 RPC init...'); + await awaitHttpGetSuccess('http://localhost:20443/v2/info'); + console.log('Stacks 2.0 RPC online'); + + await Promise.race([stacksNode1Exit, stacksNode2Exit]); +} + +main().catch(error => { + console.error(error); +}); + +async function waitProcessExit(proc: ChildProcess): Promise { + return await new Promise((resolve, reject) => { + proc.on('exit', code => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`${proc.spawnfile} exited with code ${code}`)); + } + }); + }); +} + +async function timeout(ms: number) { + await new Promise(res => setTimeout(res, ms)); +} + +async function awaitHttpGetSuccess(endpoint: string, waitTime = 5 * 60 * 1000, retryDelay = 2500) { + const startTime = Date.now(); + let fetchError: Error | undefined; + while (Date.now() - startTime < waitTime) { + try { + await fetch(endpoint); + return; + } catch (error) { + fetchError = error; + console.log(`Testing connection to ${endpoint}...`); + await timeout(retryDelay); + } + } + if (fetchError) { + throw fetchError; + } else { + throw new Error(`Timeout waiting for request to ${endpoint}`); + } +} \ No newline at end of file diff --git a/import-verification/test/package-lock.json b/import-verification/test/package-lock.json new file mode 100644 index 00000000000..da4f0e7ef68 --- /dev/null +++ b/import-verification/test/package-lock.json @@ -0,0 +1,130 @@ +{ + "name": "@blockstack/test", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@types/node": { + "version": "14.14.14", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.14.tgz", + "integrity": "sha512-UHnOPWVWV1z+VV8k6L1HhG7UbGBgIdghqF3l9Ny9ApPghbjICXkUJSd/b9gOgQfjM1r+37cipdw/HJ3F6ICEnQ==" + }, + "@types/node-fetch": { + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz", + "integrity": "sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw==", + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, + "arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==" + }, + "form-data": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", + "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, + "make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==" + }, + "mime-db": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==" + }, + "mime-types": { + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", + "requires": { + "mime-db": "1.44.0" + } + }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + }, + "source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "ts-node": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.1.1.tgz", + "integrity": "sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==", + "requires": { + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "source-map-support": "^0.5.17", + "yn": "3.1.1" + } + }, + "typescript": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.3.tgz", + "integrity": "sha512-B3ZIOf1IKeH2ixgHhj6la6xdwR9QrLC5d1VKeCSY4tvkqhF2eqd9O7txNlS0PO3GrBAFIdr3L1ndNwteUbZLYg==" + }, + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==" + } + } +} diff --git a/import-verification/test/package.json b/import-verification/test/package.json new file mode 100644 index 00000000000..d0c45cd846b --- /dev/null +++ b/import-verification/test/package.json @@ -0,0 +1,19 @@ +{ + "name": "@blockstack/test", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "ts-node index.ts" + }, + "keywords": [], + "author": "Matthew Little", + "license": "ISC", + "dependencies": { + "@types/node": "^14.14.14", + "@types/node-fetch": "^2.5.7", + "node-fetch": "^2.6.1", + "ts-node": "^9.1.1", + "typescript": "^4.1.3" + } +} diff --git a/import-verification/test/tsconfig.json b/import-verification/test/tsconfig.json new file mode 100644 index 00000000000..72dbbc8d2c7 --- /dev/null +++ b/import-verification/test/tsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "strict": true, + "skipLibCheck": true, + "noEmit": true, + } +} From ddd07d92fad068b1634487d8696129bc53804e04 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 23 Dec 2020 14:15:45 +0100 Subject: [PATCH 02/10] feat: balances tested against Stacks 1.0 and Stacks 2.0, passing --- import-verification/Dockerfile | 41 +++++----- import-verification/test/index.ts | 87 +++++++++++++++++++--- import-verification/test/package-lock.json | 58 +++++++++++++++ import-verification/test/package.json | 1 + 4 files changed, 156 insertions(+), 31 deletions(-) diff --git a/import-verification/Dockerfile b/import-verification/Dockerfile index 02126533aa2..876f06b9fd3 100644 --- a/import-verification/Dockerfile +++ b/import-verification/Dockerfile @@ -22,36 +22,24 @@ RUN pip install pyparsing WORKDIR /stacks1.0 RUN python ./setup.py build RUN python ./setup.py install -# RUN python ./bin/blockstack-core version RUN blockstack-core version ### Sync Stacks 1.0 chain RUN blockstack-core fast_sync --working-dir /stacks1.0-chain -# use sqlite cli to set get_v2_upgrade_threshold_block + +# Use sqlite cli to mark the chain as exported/frozen so Stacks 1.0 does not process new transactions RUN apt-get install -y sqlite3 RUN sqlite3 /stacks1.0-chain/blockstack-server.db 'UPDATE v2_upgrade_signal SET threshold_block_id = 1 WHERE id = 1' RUN sqlite3 /stacks1.0-chain/blockstack-server.db 'UPDATE v2_upgrade_signal SET import_block_id = 1 WHERE id = 1' -RUN blockstack-core fast_sync_snapshot 0 /stacks1.0-snapshot --working-dir /stacks1.0-chain > fast_sync_snapshot.log -# Fast-sync-snapshot completed at block 662399 with consensus hash bf81462f58d511cb2ab9d9752d745af8 -# RUN blockstack-core start --working-dir /stacks1.0-export +# Perform fast sync snapshot +RUN blockstack-core fast_sync_snapshot 0 /stacks1.0-snapshot --working-dir /stacks1.0-chain > fast_sync_snapshot.log +# Extract the snapshotted block height and consensus hash RUN cat fast_sync_snapshot.log | grep "consensus hash" | tail -1 | sed "s/.*at block \(.*\) with consensus hash \(.*\).*/\1/" > export_block RUN cat fast_sync_snapshot.log | grep "consensus hash" | tail -1 | sed "s/.*at block \(.*\) with consensus hash \(.*\).*/\2/" > consensus_hash - RUN echo "Block $(cat export_block) hash $(cat consensus_hash)" -# RUN cat fast_sync_snapshot.log | grep "consensus hash" | tail -1 | sed -e 's/.*block \([^"]*\)with.*/\1/' > export_block -# RUN curl -s https://core.blockstack.org/v1/blockchains/bitcoin/consensus/$(cat export_block) | jq '.consensus_hash' -r > consensus_hash - -# RUN sed 's/.*block//; s/with.*//' -# RUN sed -e 's/.*block\(.*\)with.*/\1/' -# RUN sed -e 's/.*block\([^"]*\)with.*/\1/' - -# RUN curl https://core.blockstack.org/v1/blockchains/bitcoin/consensus/662399 -# RUN curl https://core.blockstack.org/v1/blockchains/bitcoin/consensus-block/bf81462f58d511cb2ab9d9752d745af8 -# RUN curl http://localhost:6270/v1/info - RUN rm -rf /stacks1.0-chain # RUN blockstack-core export_migration_json /stacks1.0-snapshot $(cat export_block) $(cat consensus_hash) --working-dir /stacks1.0-export @@ -64,9 +52,22 @@ WORKDIR /stacks2.0/testnet/stacks-node RUN cargo build --release RUN cp /stacks2.0/target/release/stacks-node /bin/stacks-node -# RUN blockstack-core start --working-dir /stacks1.0-chain && sleep 10 && curl --max-time 2 --retry 200 http://localhost:6270/v1/info - WORKDIR /test + +# Dump 1000 high activity / balance addresses +RUN echo "select address, (cast(credit_value as integer) - cast(debit_value as integer)) as amount from ( \ + select * \ + from accounts \ + where type = \"STACKS\" \ + group by address \ + having block_id = max(block_id) and vtxindex = max(vtxindex) \ + order by block_id DESC, vtxindex DESC \ + ) amounts \ + order by amount DESC, address \ + limit 1000" | sqlite3 /stacks1.0-chain/blockstack-server.db > check_addrs.txt +RUN cat check_addrs.txt + +# Run the js test script COPY test ./ RUN npm i -RUN npm test \ No newline at end of file +RUN npm test diff --git a/import-verification/test/index.ts b/import-verification/test/index.ts index 3b49ffa474a..c664bf8c2bd 100644 --- a/import-verification/test/index.ts +++ b/import-verification/test/index.ts @@ -1,38 +1,103 @@ -import { ChildProcess, execFile, spawn } from 'child_process'; +import { ChildProcess, spawn } from 'child_process'; +import * as fs from 'fs'; import fetch from 'node-fetch'; - -// ## Get account balance [GET /v1/accounts/{address}/{tokenType}/balance] - +import * as c32check from 'c32check'; async function main() { + const addresses = fs.readFileSync('check_addrs.txt', { encoding: 'ascii' }).split('\n'); + const accounts: {stxAddr: string, testnetAddr: string; amount: string}[] = []; + let i = 0; + for (const line of addresses) { + const [addr, amount] = line.split('|'); + try { + const stxAddr = c32check.b58ToC32(addr); + const testnetAddr = getTestnetAddress(stxAddr); + console.log(`${stxAddr} / ${testnetAddr}: ${amount}`); + accounts.push({stxAddr, testnetAddr, amount}); + } catch (error) { + console.log(`Skipping check for placeholder: ${addr}`); + } + i++; + // TODO: only checking 50 right now because Stacks 2.0 account queries are very slow. + if (i > 50) { + break; + } + } + console.log('Starting Stacks 1.0 node...'); const stacksNode1Proc = spawn('blockstack-core', ['start', '--foreground', '--working-dir', '/stacks1.0-chain'], { stdio: 'inherit' }); const stacksNode1Exit = waitProcessExit(stacksNode1Proc); console.log('Waiting for Stacks 1.0 RPC init...'); - await awaitHttpGetSuccess('http://localhost:6270/v1/info'); + await waitHttpGetSuccess('http://localhost:6270/v1/info'); console.log('Stacks 1.0 RPC online'); + for (const account of accounts) { + const res: {balance: string} = await (await fetch(`http://localhost:6270/v1/accounts/${account.stxAddr}/STACKS/balance`)).json(); + console.log(`got: ${res.balance}, expected ${account.amount}`); + if (res.balance !== account.amount) { + throw new Error(`Unexpected Stacks 1.0 balance for ${account.stxAddr}. Expected ${account.amount} got ${res.balance}`); + } + console.log(`Stacks 2.0 has expected balance ${res.balance} for ${account.stxAddr}`); + } + + console.log('Shutting down Stacks 1.0 node...'); + stacksNode1Proc.kill('SIGKILL'); + await stacksNode1Exit; + console.log('Starting Stacks 2.0 node...'); const stacksNode2Proc = spawn('stacks-node', ['mocknet'], { stdio: 'inherit' }); const stacksNode2Exit = waitProcessExit(stacksNode2Proc); console.log('Waiting for Stacks 1.0 RPC init...'); - await awaitHttpGetSuccess('http://localhost:20443/v2/info'); + await waitHttpGetSuccess('http://localhost:20443/v2/info'); console.log('Stacks 2.0 RPC online'); - await Promise.race([stacksNode1Exit, stacksNode2Exit]); + while (true) { + console.log('Checking for Stacks 2.0 node block initialized...') + const res: {stacks_tip_height: number} = await (await fetch('http://localhost:20443/v2/info')).json(); + if (res.stacks_tip_height > 0) { + break; + } + await timeout(1500); + } + + for (const account of accounts) { + const res: {balance: string} = await (await fetch(`http://localhost:20443/v2/accounts/${account.testnetAddr}`)).json(); + const balance = BigInt(res.balance).toString(); + if (balance !== account.amount) { + throw new Error(`Unexpected Stacks 2.0 balance for ${account.testnetAddr}. Expected ${account.amount} got ${balance}`); + } + console.log(`Stacks 2.0 has expected balance ${balance} for ${account.testnetAddr}`); + } + + console.log('Shutting down Stacks 2.0 node...'); + stacksNode2Proc.kill('SIGKILL'); + await stacksNode2Exit; } main().catch(error => { console.error(error); }); +function getTestnetAddress(mainnetAddress: string): string { + const [version, hash160] = c32check.c32addressDecode(mainnetAddress); + let testnetVersion = 0; + if (version === c32check.versions.mainnet.p2pkh) { + testnetVersion = c32check.versions.testnet.p2pkh; + } else if (version === c32check.versions.mainnet.p2sh) { + testnetVersion = c32check.versions.testnet.p2sh; + } else { + throw new Error(`Unexpected address version: ${version}`); + } + return c32check.c32address(testnetVersion, hash160); +} + async function waitProcessExit(proc: ChildProcess): Promise { return await new Promise((resolve, reject) => { - proc.on('exit', code => { - if (code === 0) { + proc.on('exit', (code, signal) => { + if (code === 0 || signal === 'SIGKILL') { resolve(); } else { - reject(new Error(`${proc.spawnfile} exited with code ${code}`)); + reject(new Error(`${proc.spawnfile} exited with code ${code} signal ${signal}`)); } }); }); @@ -42,7 +107,7 @@ async function timeout(ms: number) { await new Promise(res => setTimeout(res, ms)); } -async function awaitHttpGetSuccess(endpoint: string, waitTime = 5 * 60 * 1000, retryDelay = 2500) { +async function waitHttpGetSuccess(endpoint: string, waitTime = 5 * 60 * 1000, retryDelay = 2500) { const startTime = Date.now(); let fetchError: Error | undefined; while (Date.now() - startTime < waitTime) { diff --git a/import-verification/test/package-lock.json b/import-verification/test/package-lock.json index da4f0e7ef68..be60471c20e 100644 --- a/import-verification/test/package-lock.json +++ b/import-verification/test/package-lock.json @@ -28,11 +28,43 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "base-x": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.8.tgz", + "integrity": "sha512-Rl/1AWP4J/zRrk54hhlxH4drNxPJXYUaKffODVI53/dAsV4t9fBxyxYKAVPU1XBHxYwOWP9h9H0hM2MVw4YfJA==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, + "c32check": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/c32check/-/c32check-1.1.2.tgz", + "integrity": "sha512-YgmbvOQ9HfoH7ptW80JP6WJdgoHJFGqFjxaFYvwD+bU5i3dJ44a1LI0yxdiA2n/tVKq9W92tYcFjTP5hGlvhcg==", + "requires": { + "base-x": "^3.0.8", + "buffer": "^5.6.0", + "cross-sha256": "^1.1.2" + } + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -46,6 +78,22 @@ "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" }, + "cross-sha256": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cross-sha256/-/cross-sha256-1.1.2.tgz", + "integrity": "sha512-ZMGqJvPZQY/hmFvTJyM4LGVZIvEqD58GrCWA28goaDdo6wGzjgxWKEDxVfahkNCF/ryxBNfHe3Ql/BMSwPPbcg==", + "requires": { + "@types/node": "^8.0.0", + "buffer": "^5.6.0" + }, + "dependencies": { + "@types/node": { + "version": "8.10.66", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.66.tgz", + "integrity": "sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw==" + } + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -66,6 +114,11 @@ "mime-types": "^2.1.12" } }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -89,6 +142,11 @@ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/import-verification/test/package.json b/import-verification/test/package.json index d0c45cd846b..529fd21b93c 100644 --- a/import-verification/test/package.json +++ b/import-verification/test/package.json @@ -12,6 +12,7 @@ "dependencies": { "@types/node": "^14.14.14", "@types/node-fetch": "^2.5.7", + "c32check": "^1.1.2", "node-fetch": "^2.6.1", "ts-node": "^9.1.1", "typescript": "^4.1.3" From 3473e1db596a7aa6d542769f71f870dfe5030d4b Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 23 Dec 2020 15:08:18 +0100 Subject: [PATCH 03/10] chore: cleanup dockerfile steps --- import-verification/Dockerfile | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/import-verification/Dockerfile b/import-verification/Dockerfile index 876f06b9fd3..c0fff233b5e 100644 --- a/import-verification/Dockerfile +++ b/import-verification/Dockerfile @@ -1,19 +1,16 @@ FROM rust:buster -### Install Stacks 2.0 -RUN git clone --depth 1 --branch v24.2.1.0-xenon https://github.com/blockstack/stacks-blockchain.git /stacks2.0 -WORKDIR /stacks2.0/testnet/stacks-node -RUN cargo fetch -RUN cargo build --release -RUN cp /stacks2.0/target/release/stacks-node /bin/stacks-node -RUN stacks-node version - ### Install Node.js RUN apt-get update RUN curl -sL https://deb.nodesource.com/setup_15.x | bash - RUN apt-get install -y nodejs RUN node --version +### Checkout Stacks 2.0 src +RUN git clone --depth 1 --branch v24.2.1.0-xenon https://github.com/blockstack/stacks-blockchain.git /stacks2.0 +WORKDIR /stacks2.0/testnet/stacks-node +RUN cargo fetch + ### Install Stacks 1.0 RUN git clone --depth 1 --branch v1-migration https://github.com/blockstack/stacks-blockchain.git /stacks1.0 RUN python --version @@ -40,21 +37,21 @@ RUN cat fast_sync_snapshot.log | grep "consensus hash" | tail -1 | sed "s/.*at b RUN cat fast_sync_snapshot.log | grep "consensus hash" | tail -1 | sed "s/.*at block \(.*\) with consensus hash \(.*\).*/\2/" > consensus_hash RUN echo "Block $(cat export_block) hash $(cat consensus_hash)" -RUN rm -rf /stacks1.0-chain - -# RUN blockstack-core export_migration_json /stacks1.0-snapshot $(cat export_block) $(cat consensus_hash) --working-dir /stacks1.0-export +# Generate a chainstate export from the snapshot RUN blockstack-core export_migration_json /stacks1.0-snapshot $(cat export_block) $(cat consensus_hash) /stacks1.0-export --working-dir /stacks1.0-chain -RUN ls -al /stacks1.0-export + +# Copy exported data into Stacks 2.0 src RUN cp /stacks1.0-export/chainstate.txt /stacks2.0/stx-genesis/chainstate.txt RUN cp /stacks1.0-export/chainstate.txt.sha256 /stacks2.0/stx-genesis/chainstate.txt.sha256 +# Build Stacks 2.0 with exported data WORKDIR /stacks2.0/testnet/stacks-node RUN cargo build --release RUN cp /stacks2.0/target/release/stacks-node /bin/stacks-node - -WORKDIR /test +RUN stacks-node version # Dump 1000 high activity / balance addresses +WORKDIR /test RUN echo "select address, (cast(credit_value as integer) - cast(debit_value as integer)) as amount from ( \ select * \ from accounts \ From 74bd51b3723149ec44f6d17d66a919b89f6e2320 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 23 Dec 2020 15:35:09 +0100 Subject: [PATCH 04/10] chore: rename folder to `migration-verification` --- {import-verification => migration-verification}/.dockerignore | 0 {import-verification => migration-verification}/Dockerfile | 0 {import-verification => migration-verification}/run.sh | 0 {import-verification => migration-verification}/test/index.ts | 0 .../test/package-lock.json | 0 {import-verification => migration-verification}/test/package.json | 0 .../test/tsconfig.json | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename {import-verification => migration-verification}/.dockerignore (100%) rename {import-verification => migration-verification}/Dockerfile (100%) rename {import-verification => migration-verification}/run.sh (100%) rename {import-verification => migration-verification}/test/index.ts (100%) rename {import-verification => migration-verification}/test/package-lock.json (100%) rename {import-verification => migration-verification}/test/package.json (100%) rename {import-verification => migration-verification}/test/tsconfig.json (100%) diff --git a/import-verification/.dockerignore b/migration-verification/.dockerignore similarity index 100% rename from import-verification/.dockerignore rename to migration-verification/.dockerignore diff --git a/import-verification/Dockerfile b/migration-verification/Dockerfile similarity index 100% rename from import-verification/Dockerfile rename to migration-verification/Dockerfile diff --git a/import-verification/run.sh b/migration-verification/run.sh similarity index 100% rename from import-verification/run.sh rename to migration-verification/run.sh diff --git a/import-verification/test/index.ts b/migration-verification/test/index.ts similarity index 100% rename from import-verification/test/index.ts rename to migration-verification/test/index.ts diff --git a/import-verification/test/package-lock.json b/migration-verification/test/package-lock.json similarity index 100% rename from import-verification/test/package-lock.json rename to migration-verification/test/package-lock.json diff --git a/import-verification/test/package.json b/migration-verification/test/package.json similarity index 100% rename from import-verification/test/package.json rename to migration-verification/test/package.json diff --git a/import-verification/test/tsconfig.json b/migration-verification/test/tsconfig.json similarity index 100% rename from import-verification/test/tsconfig.json rename to migration-verification/test/tsconfig.json From e028ffd998ab3511afbd80cdc2aab567d72e7045 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 23 Dec 2020 15:51:46 +0100 Subject: [PATCH 05/10] chore: remove limit on amount of addresses tested --- migration-verification/test/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/migration-verification/test/index.ts b/migration-verification/test/index.ts index c664bf8c2bd..727bc069791 100644 --- a/migration-verification/test/index.ts +++ b/migration-verification/test/index.ts @@ -18,10 +18,13 @@ async function main() { console.log(`Skipping check for placeholder: ${addr}`); } i++; - // TODO: only checking 50 right now because Stacks 2.0 account queries are very slow. + // Uncomment to limit the amount of address tested during dev. + // The Stacks 2.0 account queries are very slow, several minutes per 100 account queries. + /* if (i > 50) { break; } + */ } console.log('Starting Stacks 1.0 node...'); From 5007dc64a99ce36e38eb1ce79e14ba6e9b294d96 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 29 Dec 2020 19:17:46 +0100 Subject: [PATCH 06/10] feat: validate lockup schedules between Stacks 1.0 db and Stacks 2.0 contract map data --- migration-verification/Dockerfile | 7 + migration-verification/test/index.ts | 152 ++++- migration-verification/test/package-lock.json | 588 +++++++++++++++++- migration-verification/test/package.json | 1 + 4 files changed, 723 insertions(+), 25 deletions(-) diff --git a/migration-verification/Dockerfile b/migration-verification/Dockerfile index c0fff233b5e..657b86bb52a 100644 --- a/migration-verification/Dockerfile +++ b/migration-verification/Dockerfile @@ -64,6 +64,13 @@ RUN echo "select address, (cast(credit_value as integer) - cast(debit_value as i limit 1000" | sqlite3 /stacks1.0-chain/blockstack-server.db > check_addrs.txt RUN cat check_addrs.txt +RUN echo "\ + SELECT address, vesting_value, block_id FROM account_vesting \ + WHERE address IN (SELECT address FROM account_vesting ORDER BY RANDOM() LIMIT 20) \ + ORDER BY address, block_id \ + " | sqlite3 /stacks1.0-chain/blockstack-server.db > check_lockups.txt +RUN cat check_lockups.txt + # Run the js test script COPY test ./ RUN npm i diff --git a/migration-verification/test/index.ts b/migration-verification/test/index.ts index 727bc069791..381f05298a3 100644 --- a/migration-verification/test/index.ts +++ b/migration-verification/test/index.ts @@ -2,20 +2,60 @@ import { ChildProcess, spawn } from 'child_process'; import * as fs from 'fs'; import fetch from 'node-fetch'; import * as c32check from 'c32check'; +import * as stxTx from '@stacks/transactions'; + +interface LockupSchedule { + stxAddr: string; testnetAddr: string; amount: string; height: number; +} async function main() { + + // Get the Stacks 1.0 block height of when the export was triggered. + const exportBlockHeight = parseInt(fs.readFileSync('/stacks1.0/export_block', {encoding: 'ascii'})); + console.log(`Export block height: ${exportBlockHeight}`); + + // Parse the sample of account lockups from Stacks 1.0. + const lockups = fs.readFileSync('check_lockups.txt', { encoding: 'ascii'}).split('\n'); + const schedules: LockupSchedule[] = []; + const lockupMap = new Map(); + for (const line of lockups) { + const [addr, amount, block] = line.split('|'); + const blockHeight = parseInt(block); + if (blockHeight < exportBlockHeight) { + // Ignore schedules that have unlocked since the export block height. + continue; + } + try { + const stxAddr = c32check.b58ToC32(addr); + const testnetAddr = getTestnetAddress(stxAddr); + // Get the expected Stacks 2.0 block height. + const stacks2Height = blockHeight - exportBlockHeight; + const schedule: LockupSchedule = {stxAddr, testnetAddr, amount, height: stacks2Height}; + schedules.push(schedule); + const blockSchedules = lockupMap.get(stacks2Height) ?? []; + blockSchedules.push(schedule); + lockupMap.set(stacks2Height, blockSchedules); + } catch (error) { + console.log(`Skipping check for placeholder lockup: ${addr}`); + } + } + console.log(`Validating lockup schedules:\n${JSON.stringify(schedules)}`); + + const expectedHeights = new Set([...schedules].sort((a, b) => a.height - b.height).map(s => s.height)); + console.log(`Checking lockup schedules at heights: ${[...expectedHeights].join(', ')}`); + + // Parse the sample of address balances from Stacks 1.0. const addresses = fs.readFileSync('check_addrs.txt', { encoding: 'ascii' }).split('\n'); - const accounts: {stxAddr: string, testnetAddr: string; amount: string}[] = []; + const accounts: {stxAddr: string; testnetAddr: string; amount: string}[] = []; let i = 0; for (const line of addresses) { const [addr, amount] = line.split('|'); try { const stxAddr = c32check.b58ToC32(addr); const testnetAddr = getTestnetAddress(stxAddr); - console.log(`${stxAddr} / ${testnetAddr}: ${amount}`); accounts.push({stxAddr, testnetAddr, amount}); } catch (error) { - console.log(`Skipping check for placeholder: ${addr}`); + console.log(`Skipping check for placeholder balance: ${addr}`); } i++; // Uncomment to limit the amount of address tested during dev. @@ -27,33 +67,17 @@ async function main() { */ } - console.log('Starting Stacks 1.0 node...'); - const stacksNode1Proc = spawn('blockstack-core', ['start', '--foreground', '--working-dir', '/stacks1.0-chain'], { stdio: 'inherit' }); - const stacksNode1Exit = waitProcessExit(stacksNode1Proc); - console.log('Waiting for Stacks 1.0 RPC init...'); - await waitHttpGetSuccess('http://localhost:6270/v1/info'); - console.log('Stacks 1.0 RPC online'); - - for (const account of accounts) { - const res: {balance: string} = await (await fetch(`http://localhost:6270/v1/accounts/${account.stxAddr}/STACKS/balance`)).json(); - console.log(`got: ${res.balance}, expected ${account.amount}`); - if (res.balance !== account.amount) { - throw new Error(`Unexpected Stacks 1.0 balance for ${account.stxAddr}. Expected ${account.amount} got ${res.balance}`); - } - console.log(`Stacks 2.0 has expected balance ${res.balance} for ${account.stxAddr}`); - } - - console.log('Shutting down Stacks 1.0 node...'); - stacksNode1Proc.kill('SIGKILL'); - await stacksNode1Exit; - + // Start the Stacks 2.0 node process console.log('Starting Stacks 2.0 node...'); const stacksNode2Proc = spawn('stacks-node', ['mocknet'], { stdio: 'inherit' }); const stacksNode2Exit = waitProcessExit(stacksNode2Proc); + + // Wait until the Stacks 2.0 RPC server is responsive. console.log('Waiting for Stacks 1.0 RPC init...'); await waitHttpGetSuccess('http://localhost:20443/v2/info'); console.log('Stacks 2.0 RPC online'); + // Wait until the Stacks 2.0 node has mined the first block, otherwise RPC queries fail. while (true) { console.log('Checking for Stacks 2.0 node block initialized...') const res: {stacks_tip_height: number} = await (await fetch('http://localhost:20443/v2/info')).json(); @@ -63,8 +87,48 @@ async function main() { await timeout(1500); } + // Query the Stacks 2.0 lockup contract, ensuring the exported Stacks 1.0 lockups match. + for (let [blockHeight, lockupSchedule] of lockupMap) { + // Fetch the lockup schedules for the current block height. + const queryUrl = "http://localhost:20443/v2/map_entry/ST000000000000000000002AMW42H/lockup/lockups?proof=0"; + const clarityCv = stxTx.uintCV(blockHeight); + const serialized = '0x' + stxTx.serializeCV(clarityCv).toString('hex'); + const res = await fetch(queryUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: `"${serialized}"` + }); + const resData: {data: string} = await res.json(); + + // Deserialize the Clarity value response into regular objects. + const clarityVal = stxTx.deserializeCV(Buffer.from(resData.data.substr(2), 'hex')); + if (clarityVal.type !== stxTx.ClarityType.OptionalSome) { + throw new Error(`Expected lockup schedules at block height ${blockHeight}`) + } + const contractSchedules: LockupSchedule[] = []; + const clarityList = (clarityVal.value as any).list; + for (const tupleVal of clarityList) { + const amount = tupleVal.data['amount'].value.toString(); + const recipient = tupleVal.data['recipient']; + const testnetAddr = c32check.c32address(recipient.address.version, recipient.address.hash160); + const stxAddr = getMainnetAddress(testnetAddr); + contractSchedules.push({testnetAddr, stxAddr, amount, height: blockHeight}); + } + + // Ensure each Stacks 1.0 schedule exists in the Stacks 2.0 lookup result. + for (const stacks1Schedule of lockupSchedule) { + const found = contractSchedules.find(s => s.amount === stacks1Schedule.amount && s.stxAddr === stacks1Schedule.stxAddr); + if (!found) { + throw new Error(`Could not find schedule in Stacks 2.0: ${blockHeight} ${stacks1Schedule.stxAddr} ${stacks1Schedule.amount}`); + } + } + console.log(`Lockups okay at height ${blockHeight} for ${lockupSchedule.length} schedules`); + } + console.log(`Stacks 2.0 lockups OKAY`); + + // Query the Stacks 2.0 accounts, ensuring the exported Stacks 1.0 balances match. for (const account of accounts) { - const res: {balance: string} = await (await fetch(`http://localhost:20443/v2/accounts/${account.testnetAddr}`)).json(); + const res: {balance: string} = await (await fetch(`http://localhost:20443/v2/accounts/${account.testnetAddr}?proof=0`)).json(); const balance = BigInt(res.balance).toString(); if (balance !== account.amount) { throw new Error(`Unexpected Stacks 2.0 balance for ${account.testnetAddr}. Expected ${account.amount} got ${balance}`); @@ -72,15 +136,55 @@ async function main() { console.log(`Stacks 2.0 has expected balance ${balance} for ${account.testnetAddr}`); } + // Shutdown the Stacks 2.0 node. console.log('Shutting down Stacks 2.0 node...'); stacksNode2Proc.kill('SIGKILL'); await stacksNode2Exit; + + // Start the Stacks 1.0 node process. + console.log('Starting Stacks 1.0 node...'); + const stacksNode1Proc = spawn('blockstack-core', ['start', '--foreground', '--working-dir', '/stacks1.0-chain'], { stdio: 'inherit' }); + const stacksNode1Exit = waitProcessExit(stacksNode1Proc); + console.log('Waiting for Stacks 1.0 RPC init...'); + + // Wait until the Stacks 1.0 RPC server is responsive. + await waitHttpGetSuccess('http://localhost:6270/v1/info'); + console.log('Stacks 1.0 RPC online'); + + // Validate the balance samples previously exported from sqlite match the Stacks 1.0 account view. + for (const account of accounts) { + const res: {balance: string} = await (await fetch(`http://localhost:6270/v1/accounts/${account.stxAddr}/STACKS/balance`)).json(); + console.log(`got: ${res.balance}, expected ${account.amount}`); + if (res.balance !== account.amount) { + throw new Error(`Unexpected Stacks 1.0 balance for ${account.stxAddr}. Expected ${account.amount} got ${res.balance}`); + } + console.log(`Stacks 1.0 has expected balance ${res.balance} for ${account.stxAddr}`); + } + + // Shutdown the Stacks 1.0 node. + console.log('Shutting down Stacks 1.0 node...'); + stacksNode1Proc.kill('SIGKILL'); + await stacksNode1Exit; } main().catch(error => { console.error(error); + process.exit(1); }); +function getMainnetAddress(testnetAddress: string): string { + const [version, hash160] = c32check.c32addressDecode(testnetAddress); + let ver = 0; + if (version === c32check.versions.testnet.p2pkh) { + ver = c32check.versions.mainnet.p2pkh; + } else if (version === c32check.versions.testnet.p2sh) { + ver = c32check.versions.mainnet.p2sh; + } else { + throw new Error(`Unexpected address version: ${version}`); + } + return c32check.c32address(ver, hash160); + } + function getTestnetAddress(mainnetAddress: string): string { const [version, hash160] = c32check.c32addressDecode(mainnetAddress); let testnetVersion = 0; diff --git a/migration-verification/test/package-lock.json b/migration-verification/test/package-lock.json index be60471c20e..229cd5e1435 100644 --- a/migration-verification/test/package-lock.json +++ b/migration-verification/test/package-lock.json @@ -1,9 +1,481 @@ { "name": "@blockstack/test", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "name": "@blockstack/test", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@stacks/transactions": "^1.0.0-beta.20", + "@types/node": "^14.14.14", + "@types/node-fetch": "^2.5.7", + "c32check": "^1.1.2", + "node-fetch": "^2.6.1", + "ts-node": "^9.1.1", + "typescript": "^4.1.3" + } + }, + "node_modules/@stacks/common": { + "version": "1.0.0-beta.20", + "resolved": "https://registry.npmjs.org/@stacks/common/-/common-1.0.0-beta.20.tgz", + "integrity": "sha512-Je1MEO+63A44HZE0JzjQM5giSrNgY3bep2jaGAq8if5uIV9qKiFu7rZD9jQVc7lw3gmt+/S2zuvaM8K6Qs8ZtA==", + "dependencies": { + "cross-fetch": "^3.0.5" + } + }, + "node_modules/@stacks/network": { + "version": "1.0.0-beta.20", + "resolved": "https://registry.npmjs.org/@stacks/network/-/network-1.0.0-beta.20.tgz", + "integrity": "sha512-wHGnX1D0k9N0UJzZcQ9JmvDmLOmQPTPEKz1w/8Daa9ok7HYl/Fs8Fu1CPd+q97o5+8myLGIhidsysro0ojEGOA==", + "dependencies": { + "@stacks/common": "^1.0.0-beta.20" + } + }, + "node_modules/@stacks/transactions": { + "version": "1.0.0-beta.20", + "resolved": "https://registry.npmjs.org/@stacks/transactions/-/transactions-1.0.0-beta.20.tgz", + "integrity": "sha512-tp0MC700zjktzNlYpZ76/ukMFMgUjhEqdNIUMTBVJAwsu2EUqkrHk7kcshzpSYsx6elJZr2XhRgBXXrE0sQNTA==", + "dependencies": { + "@stacks/common": "^1.0.0-beta.20", + "@stacks/network": "^1.0.0-beta.20", + "@types/bn.js": "^4.11.6", + "@types/elliptic": "^6.4.12", + "@types/randombytes": "^2.0.0", + "@types/sha.js": "^2.4.0", + "bn.js": "^4.11.9", + "c32check": "^1.1.1", + "cross-fetch": "^3.0.5", + "elliptic": "^6.5.3", + "lodash": "^4.17.20", + "randombytes": "^2.1.0", + "ripemd160-min": "^0.0.6", + "sha.js": "^2.4.11", + "smart-buffer": "^4.1.0" + } + }, + "node_modules/@types/bn.js": { + "version": "4.11.6", + "resolved": "https://registry.npmjs.org/@types/bn.js/-/bn.js-4.11.6.tgz", + "integrity": "sha512-pqr857jrp2kPuO9uRjZ3PwnJTjoQy+fcdxvBTvHm6dkmEL9q+hDD/2j/0ELOBPtPnS8LjCX0gI9nbl8lVkadpg==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/elliptic": { + "version": "6.4.12", + "resolved": "https://registry.npmjs.org/@types/elliptic/-/elliptic-6.4.12.tgz", + "integrity": "sha512-gP1KsqoouLJGH6IJa28x7PXb3cRqh83X8HCLezd2dF+XcAIMKYv53KV+9Zn6QA561E120uOqZBQ+Jy/cl+fviw==", + "dependencies": { + "@types/bn.js": "*" + } + }, + "node_modules/@types/node": { + "version": "14.14.14", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.14.tgz", + "integrity": "sha512-UHnOPWVWV1z+VV8k6L1HhG7UbGBgIdghqF3l9Ny9ApPghbjICXkUJSd/b9gOgQfjM1r+37cipdw/HJ3F6ICEnQ==" + }, + "node_modules/@types/node-fetch": { + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz", + "integrity": "sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw==", + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, + "node_modules/@types/randombytes": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/randombytes/-/randombytes-2.0.0.tgz", + "integrity": "sha512-bz8PhAVlwN72vqefzxa14DKNT8jK/mV66CSjwdVQM/k3Th3EPKfUtdMniwZgMedQTFuywAsfjnZsg+pEnltaMA==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/sha.js": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@types/sha.js/-/sha.js-2.4.0.tgz", + "integrity": "sha512-amxKgPy6WJTKuw8mpUwjX2BSxuBtBmZfRwIUDIuPJKNwGN8CWDli8JTg5ONTWOtcTkHIstvT7oAhhYXqEjStHQ==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "node_modules/base-x": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.8.tgz", + "integrity": "sha512-Rl/1AWP4J/zRrk54hhlxH4drNxPJXYUaKffODVI53/dAsV4t9fBxyxYKAVPU1XBHxYwOWP9h9H0hM2MVw4YfJA==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, + "node_modules/c32check": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/c32check/-/c32check-1.1.2.tgz", + "integrity": "sha512-YgmbvOQ9HfoH7ptW80JP6WJdgoHJFGqFjxaFYvwD+bU5i3dJ44a1LI0yxdiA2n/tVKq9W92tYcFjTP5hGlvhcg==", + "dependencies": { + "base-x": "^3.0.8", + "buffer": "^5.6.0", + "cross-sha256": "^1.1.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" + }, + "node_modules/cross-fetch": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.0.6.tgz", + "integrity": "sha512-KBPUbqgFjzWlVcURG+Svp9TlhA5uliYtiNx/0r8nv0pdypeQCRJ9IaSIc3q/x3q8t3F75cHuwxVql1HFGHCNJQ==", + "dependencies": { + "node-fetch": "2.6.1" + } + }, + "node_modules/cross-sha256": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cross-sha256/-/cross-sha256-1.1.2.tgz", + "integrity": "sha512-ZMGqJvPZQY/hmFvTJyM4LGVZIvEqD58GrCWA28goaDdo6wGzjgxWKEDxVfahkNCF/ryxBNfHe3Ql/BMSwPPbcg==", + "dependencies": { + "@types/node": "^8.0.0", + "buffer": "^5.6.0" + } + }, + "node_modules/cross-sha256/node_modules/@types/node": { + "version": "8.10.66", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.66.tgz", + "integrity": "sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw==" + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/elliptic": { + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", + "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", + "dependencies": { + "bn.js": "^4.4.0", + "brorand": "^1.0.1", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.0" + } + }, + "node_modules/form-data": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", + "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "dependencies": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "node_modules/hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "dependencies": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==" + }, + "node_modules/mime-db": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", + "dependencies": { + "mime-db": "1.44.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "node_modules/minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=" + }, + "node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/ripemd160-min": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/ripemd160-min/-/ripemd160-min-0.0.6.tgz", + "integrity": "sha512-+GcJgQivhs6S9qvLogusiTcS9kQUfgR75whKuy5jIhuiOfQuJ8fjqxV6EGD5duH1Y/FawFUMtMhyeq3Fbnib8A==", + "engines": { + "node": ">=8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "node_modules/sha.js": { + "version": "2.4.11", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "dependencies": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + }, + "bin": { + "sha.js": "bin.js" + } + }, + "node_modules/smart-buffer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.1.0.tgz", + "integrity": "sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/ts-node": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.1.1.tgz", + "integrity": "sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==", + "dependencies": { + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "source-map-support": "^0.5.17", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/typescript": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.3.tgz", + "integrity": "sha512-B3ZIOf1IKeH2ixgHhj6la6xdwR9QrLC5d1VKeCSY4tvkqhF2eqd9O7txNlS0PO3GrBAFIdr3L1ndNwteUbZLYg==", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "engines": { + "node": ">=6" + } + } + }, "dependencies": { + "@stacks/common": { + "version": "1.0.0-beta.20", + "resolved": "https://registry.npmjs.org/@stacks/common/-/common-1.0.0-beta.20.tgz", + "integrity": "sha512-Je1MEO+63A44HZE0JzjQM5giSrNgY3bep2jaGAq8if5uIV9qKiFu7rZD9jQVc7lw3gmt+/S2zuvaM8K6Qs8ZtA==", + "requires": { + "cross-fetch": "^3.0.5" + } + }, + "@stacks/network": { + "version": "1.0.0-beta.20", + "resolved": "https://registry.npmjs.org/@stacks/network/-/network-1.0.0-beta.20.tgz", + "integrity": "sha512-wHGnX1D0k9N0UJzZcQ9JmvDmLOmQPTPEKz1w/8Daa9ok7HYl/Fs8Fu1CPd+q97o5+8myLGIhidsysro0ojEGOA==", + "requires": { + "@stacks/common": "^1.0.0-beta.20" + } + }, + "@stacks/transactions": { + "version": "1.0.0-beta.20", + "resolved": "https://registry.npmjs.org/@stacks/transactions/-/transactions-1.0.0-beta.20.tgz", + "integrity": "sha512-tp0MC700zjktzNlYpZ76/ukMFMgUjhEqdNIUMTBVJAwsu2EUqkrHk7kcshzpSYsx6elJZr2XhRgBXXrE0sQNTA==", + "requires": { + "@stacks/common": "^1.0.0-beta.20", + "@stacks/network": "^1.0.0-beta.20", + "@types/bn.js": "^4.11.6", + "@types/elliptic": "^6.4.12", + "@types/randombytes": "^2.0.0", + "@types/sha.js": "^2.4.0", + "bn.js": "^4.11.9", + "c32check": "^1.1.1", + "cross-fetch": "^3.0.5", + "elliptic": "^6.5.3", + "lodash": "^4.17.20", + "randombytes": "^2.1.0", + "ripemd160-min": "^0.0.6", + "sha.js": "^2.4.11", + "smart-buffer": "^4.1.0" + } + }, + "@types/bn.js": { + "version": "4.11.6", + "resolved": "https://registry.npmjs.org/@types/bn.js/-/bn.js-4.11.6.tgz", + "integrity": "sha512-pqr857jrp2kPuO9uRjZ3PwnJTjoQy+fcdxvBTvHm6dkmEL9q+hDD/2j/0ELOBPtPnS8LjCX0gI9nbl8lVkadpg==", + "requires": { + "@types/node": "*" + } + }, + "@types/elliptic": { + "version": "6.4.12", + "resolved": "https://registry.npmjs.org/@types/elliptic/-/elliptic-6.4.12.tgz", + "integrity": "sha512-gP1KsqoouLJGH6IJa28x7PXb3cRqh83X8HCLezd2dF+XcAIMKYv53KV+9Zn6QA561E120uOqZBQ+Jy/cl+fviw==", + "requires": { + "@types/bn.js": "*" + } + }, "@types/node": { "version": "14.14.14", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.14.tgz", @@ -18,6 +490,22 @@ "form-data": "^3.0.0" } }, + "@types/randombytes": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/randombytes/-/randombytes-2.0.0.tgz", + "integrity": "sha512-bz8PhAVlwN72vqefzxa14DKNT8jK/mV66CSjwdVQM/k3Th3EPKfUtdMniwZgMedQTFuywAsfjnZsg+pEnltaMA==", + "requires": { + "@types/node": "*" + } + }, + "@types/sha.js": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@types/sha.js/-/sha.js-2.4.0.tgz", + "integrity": "sha512-amxKgPy6WJTKuw8mpUwjX2BSxuBtBmZfRwIUDIuPJKNwGN8CWDli8JTg5ONTWOtcTkHIstvT7oAhhYXqEjStHQ==", + "requires": { + "@types/node": "*" + } + }, "arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", @@ -41,6 +529,16 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, + "bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" + }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -78,6 +576,14 @@ "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" }, + "cross-fetch": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.0.6.tgz", + "integrity": "sha512-KBPUbqgFjzWlVcURG+Svp9TlhA5uliYtiNx/0r8nv0pdypeQCRJ9IaSIc3q/x3q8t3F75cHuwxVql1HFGHCNJQ==", + "requires": { + "node-fetch": "2.6.1" + } + }, "cross-sha256": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/cross-sha256/-/cross-sha256-1.1.2.tgz", @@ -104,6 +610,20 @@ "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==" }, + "elliptic": { + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", + "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", + "requires": { + "bn.js": "^4.4.0", + "brorand": "^1.0.1", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.0" + } + }, "form-data": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", @@ -114,11 +634,40 @@ "mime-types": "^2.1.12" } }, + "hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "requires": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "requires": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, "make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -137,16 +686,53 @@ "mime-db": "1.44.0" } }, + "minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=" + }, "node-fetch": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "requires": { + "safe-buffer": "^5.1.0" + } + }, + "ripemd160-min": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/ripemd160-min/-/ripemd160-min-0.0.6.tgz", + "integrity": "sha512-+GcJgQivhs6S9qvLogusiTcS9kQUfgR75whKuy5jIhuiOfQuJ8fjqxV6EGD5duH1Y/FawFUMtMhyeq3Fbnib8A==" + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, + "sha.js": { + "version": "2.4.11", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "smart-buffer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.1.0.tgz", + "integrity": "sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw==" + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/migration-verification/test/package.json b/migration-verification/test/package.json index 529fd21b93c..6f21a89cc6f 100644 --- a/migration-verification/test/package.json +++ b/migration-verification/test/package.json @@ -10,6 +10,7 @@ "author": "Matthew Little", "license": "ISC", "dependencies": { + "@stacks/transactions": "^1.0.0-beta.20", "@types/node": "^14.14.14", "@types/node-fetch": "^2.5.7", "c32check": "^1.1.2", From c82864c140b50c1d0ecf5426db3d0fc05a681ec7 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 29 Dec 2020 19:33:19 +0100 Subject: [PATCH 07/10] chore: increase amount of lockup schedules sampled and tested --- migration-verification/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/migration-verification/Dockerfile b/migration-verification/Dockerfile index 657b86bb52a..9639d7881f8 100644 --- a/migration-verification/Dockerfile +++ b/migration-verification/Dockerfile @@ -64,9 +64,10 @@ RUN echo "select address, (cast(credit_value as integer) - cast(debit_value as i limit 1000" | sqlite3 /stacks1.0-chain/blockstack-server.db > check_addrs.txt RUN cat check_addrs.txt +# Dump ~1000 randomly sampled vesting schedules RUN echo "\ SELECT address, vesting_value, block_id FROM account_vesting \ - WHERE address IN (SELECT address FROM account_vesting ORDER BY RANDOM() LIMIT 20) \ + WHERE address IN (SELECT address FROM account_vesting ORDER BY RANDOM() LIMIT 35) \ ORDER BY address, block_id \ " | sqlite3 /stacks1.0-chain/blockstack-server.db > check_lockups.txt RUN cat check_lockups.txt From b23c97dba0145218dd63f6f51e7768f517d6f198 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 30 Dec 2020 15:23:28 +0100 Subject: [PATCH 08/10] fix: use BASH_SOURCE instead of realpath --- migration-verification/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration-verification/run.sh b/migration-verification/run.sh index 281be7ae6be..d2017c6dc56 100755 --- a/migration-verification/run.sh +++ b/migration-verification/run.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd "$(dirname "$(realpath "$0")")" +cd "$(dirname "${BASH_SOURCE[0]}")" DOCKER_BUILDKIT=1 BUILDKIT_PROGRESS=plain docker build . \ No newline at end of file From bee8f2664f87982a5723f6ebf09addfb1fe3f7c6 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 30 Dec 2020 15:43:46 +0100 Subject: [PATCH 09/10] feat: use docker arg to specify Stacks 2.0 branch --- migration-verification/Dockerfile | 3 ++- migration-verification/README.md | 29 +++++++++++++++++++++++++++++ migration-verification/run.sh | 5 ----- 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 migration-verification/README.md delete mode 100755 migration-verification/run.sh diff --git a/migration-verification/Dockerfile b/migration-verification/Dockerfile index 9639d7881f8..03490f053b4 100644 --- a/migration-verification/Dockerfile +++ b/migration-verification/Dockerfile @@ -7,7 +7,8 @@ RUN apt-get install -y nodejs RUN node --version ### Checkout Stacks 2.0 src -RUN git clone --depth 1 --branch v24.2.1.0-xenon https://github.com/blockstack/stacks-blockchain.git /stacks2.0 +ARG STACKS_V2_BRANCH +RUN git clone --depth 1 --branch $STACKS_V2_BRANCH https://github.com/blockstack/stacks-blockchain.git /stacks2.0 WORKDIR /stacks2.0/testnet/stacks-node RUN cargo fetch diff --git a/migration-verification/README.md b/migration-verification/README.md new file mode 100644 index 00000000000..67d6e24ec44 --- /dev/null +++ b/migration-verification/README.md @@ -0,0 +1,29 @@ +This directory contains a Dockerfile that performs automated validation of the migration process from Stacks 1.0 to Stacks 2.0. + +A sampling of STX balances and lockup schedules are tested. + +The following steps are automatically performed: +1. Checkout and install Stacks 1.0. +2. Run a Stacks 1.0 fast-sync to get caught up to the latest chain state (as of the latest hosted snapshot). +3. Trigger a fast-sync-dump similar to how it will be triggered from the name threshold. +4. Perform the chainstate export step from the fast-sync-dump. +5. Checkout the Stacks 2.0 source, and copy over the newly exported chainstate.txt, and build. +6. Query the Stacks 1.0 db for 1000 address balances, and ~1000 lockup schedules. +7. Spin up both a Stacks 1.0 and Stacks 2.0 node, and validate the address balances match using the account RPC endpoints: + * Stacks 1.0: `/v1/accounts/{address}/STACKS/balance` + * Stacks 2.0: `/v2/accounts/{address-in-testnet-format}` +8. Validate lockup schedules in Stacks 2.0 match the samples dumped from the Stacks 1.0, using a contract map lookup: + * `/v2/map_entry/ST000000000000000000002AMW42H/lockup/lockups` + + + +### Running +This is a resources intensive process and can take upwards of an hour. + +Ensure Docker is allocated at least 70GB disk size and 4GB memory. + +Run the docker build: +```shell +cd migration-verification +DOCKER_BUILDKIT=1 BUILDKIT_PROGRESS=plain docker build --build-arg STACKS_V2_BRANCH= . +``` diff --git a/migration-verification/run.sh b/migration-verification/run.sh deleted file mode 100755 index d2017c6dc56..00000000000 --- a/migration-verification/run.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cd "$(dirname "${BASH_SOURCE[0]}")" - -DOCKER_BUILDKIT=1 BUILDKIT_PROGRESS=plain docker build . \ No newline at end of file From 80222b38ffaf69aa1d4968ca42958a4d38c39b0e Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Wed, 30 Dec 2020 15:51:46 +0100 Subject: [PATCH 10/10] fix: log message typo --- migration-verification/test/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration-verification/test/index.ts b/migration-verification/test/index.ts index 381f05298a3..5e4dc8aaacf 100644 --- a/migration-verification/test/index.ts +++ b/migration-verification/test/index.ts @@ -73,7 +73,7 @@ async function main() { const stacksNode2Exit = waitProcessExit(stacksNode2Proc); // Wait until the Stacks 2.0 RPC server is responsive. - console.log('Waiting for Stacks 1.0 RPC init...'); + console.log('Waiting for Stacks 2.0 RPC init...'); await waitHttpGetSuccess('http://localhost:20443/v2/info'); console.log('Stacks 2.0 RPC online');