From 6c2f1029197c0c2d9863aac9f9daaa47450bf6f2 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Tue, 3 Jul 2018 12:31:06 +0200 Subject: [PATCH 1/9] Remove unneeded setting of genesis hash --- packages/client-chains/src/genesis/index.ts | 3 --- packages/client-p2p/src/handler/index.ts | 6 ++++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/client-chains/src/genesis/index.ts b/packages/client-chains/src/genesis/index.ts index 96ab92a5..9794c514 100644 --- a/packages/client-chains/src/genesis/index.ts +++ b/packages/client-chains/src/genesis/index.ts @@ -16,8 +16,5 @@ export default function initGenesis (self: ChainState, initialState: ChainGenesi self.blockDb.bestNumber.set(0); self.blockDb.block.set(genesis.block, genesis.headerHash); - self.stateDb.system.blockHashAt.set(genesis.headerHash, 0); - self.stateDb.db.commit(); - return genesis; } diff --git a/packages/client-p2p/src/handler/index.ts b/packages/client-p2p/src/handler/index.ts index 9e5ad0f3..51897067 100644 --- a/packages/client-p2p/src/handler/index.ts +++ b/packages/client-p2p/src/handler/index.ts @@ -16,13 +16,15 @@ type Message = { message: MessageInterface }; -const HANDLERS: Array = [ +const handlers: Array = [ blockAnnounce, blockRequest, blockResponse, status ]; export default function onPeerMessage (self: P2pState): void { self.peers.on('message', ({ peer, message }: Message): void => { - const handler = HANDLERS.find((handler) => handler.type === message.type); + const handler = handlers.find(({ type }) => + type === message.type + ); if (!handler) { self.l.error(`Unhandled message type=${message.type}`); From 847ac7f761197aade339ee9d8ad0c58182fb7b4e Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Tue, 3 Jul 2018 14:57:50 +0200 Subject: [PATCH 2/9] Add basic in-memory hashDB (wrong package atm) --- package.json | 2 +- packages/client-runtime/package.json | 1 + packages/client-wasm/src/create/exports.ts | 4 +- yarn.lock | 60 ++++++++++++++++++++-- 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index c2937bca..017ad3d3 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "test": "jest --coverage", "start": "node packages/client/scripts/polkadot.js --db-path ./tmp/start-1", "start2": "node packages/client/scripts/polkadot.js --db-path ./tmp/start-2 --p2p-nodes /ip4/127.0.0.1/tcp/39933/ipfs/QmfUiXCYtrKotHgDbP4Kc74NUi2LxckEiAdkK1SMSosLaz --p2p-port 39934 --rpc-port 9934", - "start-rust": "node packages/client/scripts/polkadot.js --db-path ./tmp/start-rust --p2p-nodes /ip4/127.0.0.1/tcp/30333/ipfs/QmbggqmmTM1irJkKbxi9TX8WyHy4c2f2HFGVhQHhR7orBH /ip4/127.0.0.1/tcp/39933/ipfs/QmfUiXCYtrKotHgDbP4Kc74NUi2LxckEiAdkK1SMSosLaz --p2p-port 39935 --rpc-port 9935" + "start-rust": "node packages/client/scripts/polkadot.js --db-path ./tmp/start-rust --p2p-nodes /ip4/127.0.0.1/tcp/30333/ipfs/QmX1nafodGEN1m2ueX26UvqLVXhvR2A1g8HdFBVD7GHgjJ --p2p-port 39935 --rpc-port 9935" }, "devDependencies": { "@polkadot/dev": "^0.20.9", diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 1422120f..b6d3ea5b 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -35,6 +35,7 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", + "@polkadot/trie-db": "^0.26.10", "@polkadot/util": "^0.26.10", "@polkadot/util-crypto": "^0.26.10", "@polkadot/util-triedb": "^0.26.10" diff --git a/packages/client-wasm/src/create/exports.ts b/packages/client-wasm/src/create/exports.ts index 147889f8..2d26a650 100644 --- a/packages/client-wasm/src/create/exports.ts +++ b/packages/client-wasm/src/create/exports.ts @@ -4,7 +4,7 @@ import { WasmExtraImports, WasmInstanceExports } from '../types'; -import blake2AsU8a from '@polkadot/util-crypto/blake2/asU8a'; +import xxhash from '@polkadot/util-crypto/xxhash/xxhash64/asRaw'; import createImports from './imports'; @@ -20,7 +20,7 @@ const DEFAULT_TABLE: WebAssembly.TableDescriptor = { const cache: Cache = {}; export default function createExports (bytecode: Uint8Array, imports?: WasmExtraImports, memory?: WebAssembly.Memory | null, forceCreate: boolean = false): WasmInstanceExports { - const codeHash = blake2AsU8a(bytecode.subarray(0, 2048), 0).toString(); + const codeHash = xxhash(bytecode.subarray(0, 2048), 0); // NOTE compilation is quite resource intensive, here we bypass the actual Uint8Array -> Module compilation when we already have this module bytecode in our cache if (!cache[codeHash] || forceCreate) { diff --git a/yarn.lock b/yarn.lock index 6d4b4efe..5941897a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -794,6 +794,25 @@ "@polkadot/util-crypto" "^0.26.10" "@polkadot/util-keyring" "^0.26.10" +"@polkadot/trie-db@^0.26.10": + version "0.26.10" + resolved "https://registry.yarnpkg.com/@polkadot/trie-db/-/trie-db-0.26.10.tgz#4d4e70e62d13b997a8e3e02e9a3c9983656d1cea" + dependencies: + "@babel/runtime" "^7.0.0-beta.51" + "@polkadot/trie-hash" "^0.26.10" + "@polkadot/util" "^0.26.10" + "@polkadot/util-crypto" "^0.26.10" + "@polkadot/util-rlp" "^0.26.10" + "@types/async" "^2.0.49" + "@types/semaphore" "^1.1.0" + async "^1.4.2" + encoding-down "^5.0.3" + level-ws "0.1.0" + levelup "^3.0.1" + memdown "^3.0.0" + readable-stream "^2.2.8" + semaphore ">=1.0.1" + "@polkadot/trie-hash@^0.26.10": version "0.26.10" resolved "https://registry.yarnpkg.com/@polkadot/trie-hash/-/trie-hash-0.26.10.tgz#72a98dd18809eaef322feb54b677120e54918bec" @@ -872,6 +891,10 @@ dependencies: "@types/node" "*" +"@types/async@^2.0.49": + version "2.0.49" + resolved "https://registry.yarnpkg.com/@types/async/-/async-2.0.49.tgz#92e33d13f74c895cb9a7f38ba97db8431ed14bc0" + "@types/base-x@*": version "1.0.29" resolved "https://registry.yarnpkg.com/@types/base-x/-/base-x-1.0.29.tgz#8a2d73e4a5c3121757a5f8870cfa4509655186b6" @@ -1024,6 +1047,10 @@ version "1.2.2" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.2.tgz#fa8e1ad1d474688a757140c91de6dace6f4abc8d" +"@types/semaphore@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@types/semaphore/-/semaphore-1.1.0.tgz#a2f8db75e6429ae2703bbbe7d288f0edc67894b2" + "@types/semver-compare@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@types/semver-compare/-/semver-compare-1.0.0.tgz#b6e39ec7d0d3f03e5e74ec654bfbbfbc3bbe1b19" @@ -1069,7 +1096,7 @@ abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" -abstract-leveldown@~5.0.0: +abstract-leveldown@^5.0.0, abstract-leveldown@~5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-5.0.0.tgz#f7128e1f86ccabf7d2893077ce5d06d798e386c6" dependencies: @@ -1310,7 +1337,7 @@ async-each@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d" -async@^1.4.0, async@^1.5.0, async@~1.5.2: +async@^1.4.0, async@^1.4.2, async@^1.5.0, async@~1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" @@ -2632,6 +2659,16 @@ elliptic@^6.2.3: minimalistic-assert "^1.0.0" minimalistic-crypto-utils "^1.0.0" +encoding-down@^5.0.3: + version "5.0.4" + resolved "https://registry.yarnpkg.com/encoding-down/-/encoding-down-5.0.4.tgz#1e477da8e9e9d0f7c8293d320044f8b2cd8e9614" + dependencies: + abstract-leveldown "^5.0.0" + inherits "^2.0.3" + level-codec "^9.0.0" + level-errors "^2.0.0" + xtend "^4.0.1" + encoding@^0.1.11: version "0.1.12" resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb" @@ -4773,7 +4810,11 @@ lerna@^2.11.0: write-pkg "^3.1.0" yargs "^8.0.2" -level-errors@~2.0.0: +level-codec@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/level-codec/-/level-codec-9.0.0.tgz#2d3a0e835c4aa8339ec63de3f5a37480b74a5f87" + +level-errors@^2.0.0, level-errors@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/level-errors/-/level-errors-2.0.0.tgz#2de5b566b62eef92f99e19be74397fbc512563fa" dependencies: @@ -4787,6 +4828,13 @@ level-iterator-stream@~2.0.0: readable-stream "^2.0.5" xtend "^4.0.0" +level-ws@0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/level-ws/-/level-ws-0.1.0.tgz#9728f02f3e7f34607f05010c1e4a83242e90b0b2" + dependencies: + readable-stream "^2.2.8" + xtend "^4.0.0" + leveldown@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/leveldown/-/leveldown-4.0.1.tgz#7bc3df93c9fa574feb39ce45a0c4073aa948cfef" @@ -6490,7 +6538,7 @@ readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable string_decoder "~1.0.3" util-deprecate "~1.0.1" -readable-stream@^2.2.9, readable-stream@^2.3.6: +readable-stream@^2.2.8, readable-stream@^2.2.9, readable-stream@^2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" dependencies: @@ -6904,6 +6952,10 @@ secp256k1@^3.3.0: nan "^2.2.1" safe-buffer "^5.1.0" +semaphore@>=1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/semaphore/-/semaphore-1.1.0.tgz#aaad8b86b20fe8e9b32b16dc2ee682a8cd26a8aa" + semver-compare@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" From ea5644e0898d5721c68656fdf4b3e2ecfc618095 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Tue, 3 Jul 2018 23:07:48 +0200 Subject: [PATCH 3/9] Use sync-ified trie-db --- packages/client-chains/package.json | 14 +- .../src/chains/chain-dev.spec.js | 7 +- packages/client-chains/src/genesis/block.ts | 2 - packages/client-chains/src/genesis/state.ts | 6 +- packages/client-chains/src/index.spec.js | 7 +- packages/client-chains/src/index.ts | 4 +- packages/client-chains/src/state.ts | 4 +- packages/client-chains/src/types.d.ts | 1 - packages/client-db-chain/package.json | 10 +- packages/client-db-chain/src/block/index.ts | 4 +- packages/client-db-chain/src/db/arrayU8a.ts | 6 +- packages/client-db-chain/src/db/base.ts | 6 +- packages/client-db-chain/src/db/bn.ts | 4 +- packages/client-db-chain/src/db/bool.ts | 4 +- packages/client-db-chain/src/db/u8a.ts | 4 +- .../src/state/accountIndexOf.spec.js | 2 +- .../client-db-chain/src/state/balance.spec.js | 2 +- .../src/state/blockHash.spec.js | 4 +- packages/client-db-chain/src/state/index.ts | 2 +- packages/client-db-chain/src/types.d.ts | 16 +- packages/client-p2p-messages/package.json | 6 +- packages/client-p2p/package.json | 4 +- packages/client-rpc/package.json | 2 +- packages/client-runtime/package.json | 7 +- .../src/environment/db/clear.spec.js | 27 -- .../src/environment/db/clear.ts | 9 - .../src/environment/db/commit.spec.js | 39 --- .../src/environment/db/commit.ts | 16 -- .../src/environment/db/del.spec.js | 29 --- .../client-runtime/src/environment/db/del.ts | 12 - .../src/environment/db/get.spec.js | 33 --- .../client-runtime/src/environment/db/get.ts | 17 -- .../src/environment/db/index.ts | 42 --- .../src/environment/db/pairs.spec.js | 67 ----- .../src/environment/db/pairs.ts | 25 -- .../src/environment/db/set.spec.js | 36 --- .../client-runtime/src/environment/db/set.ts | 13 - .../src/environment/db/types.d.ts | 18 -- .../client-runtime/src/environment/index.ts | 7 +- packages/client-runtime/src/index.ts | 2 +- .../src/storage/allocated.spec.js | 9 +- packages/client-runtime/src/storage/data.ts | 2 +- packages/client-runtime/src/storage/get.ts | 2 +- .../client-runtime/src/storage/root.spec.js | 12 +- .../client-runtime/src/storage/set.spec.js | 4 +- packages/client-runtime/src/storage/trie.ts | 6 +- packages/client-runtime/src/types.d.ts | 6 +- packages/client-telemetry/package.json | 9 +- packages/client-wasm/package.json | 4 +- .../src/executor/executeBlock.spec.js | 5 +- .../src/executor/generateBlock.spec.js | 5 +- .../client-wasm/src/executor/generateBlock.ts | 4 +- .../client-wasm/src/executor/importBlock.ts | 2 + .../src/executor/initialiseBlock.spec.js | 5 +- packages/client/package.json | 7 +- packages/client/src/index.ts | 5 +- yarn.lock | 243 ++++++++---------- 57 files changed, 210 insertions(+), 640 deletions(-) delete mode 100644 packages/client-runtime/src/environment/db/clear.spec.js delete mode 100644 packages/client-runtime/src/environment/db/clear.ts delete mode 100644 packages/client-runtime/src/environment/db/commit.spec.js delete mode 100644 packages/client-runtime/src/environment/db/commit.ts delete mode 100644 packages/client-runtime/src/environment/db/del.spec.js delete mode 100644 packages/client-runtime/src/environment/db/del.ts delete mode 100644 packages/client-runtime/src/environment/db/get.spec.js delete mode 100644 packages/client-runtime/src/environment/db/get.ts delete mode 100644 packages/client-runtime/src/environment/db/index.ts delete mode 100644 packages/client-runtime/src/environment/db/pairs.spec.js delete mode 100644 packages/client-runtime/src/environment/db/pairs.ts delete mode 100644 packages/client-runtime/src/environment/db/set.spec.js delete mode 100644 packages/client-runtime/src/environment/db/set.ts delete mode 100644 packages/client-runtime/src/environment/db/types.d.ts diff --git a/packages/client-chains/package.json b/packages/client-chains/package.json index 5e307d6a..1fe8e33e 100644 --- a/packages/client-chains/package.json +++ b/packages/client-chains/package.json @@ -37,16 +37,16 @@ "@polkadot/client-db-chain": "^0.13.3", "@polkadot/client-runtime": "^0.13.3", "@polkadot/client-wasm": "^0.13.3", - "@polkadot/storage": "^0.26.10", - "@polkadot/util": "^0.26.10", - "@polkadot/util-crypto": "^0.26.10", - "@polkadot/util-keyring": "^0.26.10" + "@polkadot/storage": "^0.26.13", + "@polkadot/util": "^0.26.13", + "@polkadot/util-crypto": "^0.26.13", + "@polkadot/util-keyring": "^0.26.13" }, "devDependencies": { "@polkadot/client": "^0.13.3", "@polkadot/client-db": "^0.13.3", - "@polkadot/extrinsics": "^0.26.10", - "@polkadot/primitives": "^0.26.10", - "@polkadot/storage": "^0.26.10" + "@polkadot/extrinsics": "^0.26.13", + "@polkadot/primitives": "^0.26.13", + "@polkadot/storage": "^0.26.13" } } diff --git a/packages/client-chains/src/chains/chain-dev.spec.js b/packages/client-chains/src/chains/chain-dev.spec.js index fd34eafe..c712e579 100644 --- a/packages/client-chains/src/chains/chain-dev.spec.js +++ b/packages/client-chains/src/chains/chain-dev.spec.js @@ -2,7 +2,8 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import memDb from '@polkadot/util-triedb/temp'; +import HashDb from '@polkadot/client-db/Hash'; +import MemoryDb from '@polkadot/client-db/Memory'; import init from '../index'; @@ -10,8 +11,8 @@ describe('genesis', () => { let genesis; beforeEach(() => { - const stateDb = memDb(); - const blockDb = memDb(); + const stateDb = new MemoryDb(); + const blockDb = new HashDb(); genesis = init({ chain: 'dev' }, stateDb, blockDb).genesis; }); diff --git a/packages/client-chains/src/genesis/block.ts b/packages/client-chains/src/genesis/block.ts index b51cbdb2..077d61a7 100644 --- a/packages/client-chains/src/genesis/block.ts +++ b/packages/client-chains/src/genesis/block.ts @@ -21,7 +21,6 @@ export default function genesisBlock ({ stateDb: { db } }: ChainState): ChainGen throw new Error('Unable to retrieve genesis code'); } - const codeHash = blake2Asu8a(code, 256); const block = createBlock({ header: { stateRoot: db.trieRoot(), @@ -34,7 +33,6 @@ export default function genesisBlock ({ stateDb: { db } }: ChainState): ChainGen return { block: encodeBlock(block), code, - codeHash, header: block.header, headerHash }; diff --git a/packages/client-chains/src/genesis/state.ts b/packages/client-chains/src/genesis/state.ts index e89a1f25..684e7d86 100644 --- a/packages/client-chains/src/genesis/state.ts +++ b/packages/client-chains/src/genesis/state.ts @@ -7,9 +7,11 @@ import { ChainState, ChainGenesisState } from '../types'; import hexToU8a from '@polkadot/util/hex/toU8a'; export default function genesisState ({ stateDb: { db } }: ChainState, initial: ChainGenesisState): void { + // db.checkpoint(); + Object.keys(initial).forEach((key) => - db.set(hexToU8a(key), hexToU8a(initial[key])) + db.put(hexToU8a(key), hexToU8a(initial[key])) ); - db.commit(); + // db.commit(); } diff --git a/packages/client-chains/src/index.spec.js b/packages/client-chains/src/index.spec.js index 64c95714..001631ea 100644 --- a/packages/client-chains/src/index.spec.js +++ b/packages/client-chains/src/index.spec.js @@ -2,7 +2,8 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import memoryDb from '@polkadot/util-triedb/temp'; +import HashDb from '@polkadot/client-db/Hash'; +import MemoryDb from '@polkadot/client-db/Memory'; import createChain from './index'; @@ -15,8 +16,8 @@ describe('client-chains', () => { config = { chain: 'dev' }; - blockDb = memoryDb(); - stateDb = memoryDb(); + blockDb = new HashDb(); + stateDb = new MemoryDb(); }); it('instantiates a known chain', () => { diff --git a/packages/client-chains/src/index.ts b/packages/client-chains/src/index.ts index 2d11e78c..44cabeed 100644 --- a/packages/client-chains/src/index.ts +++ b/packages/client-chains/src/index.ts @@ -3,7 +3,7 @@ // of the ISC license. See the LICENSE file for details. import { Config } from '@polkadot/client/types'; -import { TrieDb } from '@polkadot/util-triedb/types'; +import { BaseDb, TrieDb } from '@polkadot/client-db/types'; import { ChainInterface } from './types'; import createExecutor from '@polkadot/client-wasm/index'; @@ -12,7 +12,7 @@ import loadChain from './load'; import createGenesis from './genesis'; import createState from './state'; -export default function chains (config: Config, baseStateDb: TrieDb, baseBlockDb: TrieDb): ChainInterface { +export default function chains (config: Config, baseStateDb: TrieDb, baseBlockDb: BaseDb): ChainInterface { const initial = loadChain(config.chain); const self = createState(config, baseStateDb, baseBlockDb); const genesis = createGenesis(self, initial); diff --git a/packages/client-chains/src/state.ts b/packages/client-chains/src/state.ts index 76568ce6..09e51726 100644 --- a/packages/client-chains/src/state.ts +++ b/packages/client-chains/src/state.ts @@ -3,7 +3,7 @@ // of the ISC license. See the LICENSE file for details. import { Config } from '@polkadot/client/types'; -import { TrieDb } from '@polkadot/util-triedb/types'; +import { BaseDb, TrieDb } from '@polkadot/client-db/types'; import { ChainState } from './types'; import createBlockDb from '@polkadot/client-db-chain/block'; @@ -11,7 +11,7 @@ import createStateDb from '@polkadot/client-db-chain/state'; import createRuntime from '@polkadot/client-runtime/index'; import logger from '@polkadot/util/logger'; -export default function state (config: Config, baseStateDb: TrieDb, baseBlockDb: TrieDb): ChainState { +export default function state (config: Config, baseStateDb: TrieDb, baseBlockDb: BaseDb): ChainState { const l = logger(`chain-${config.chain}`); const runtime = createRuntime(baseStateDb); const blockDb = createBlockDb(baseBlockDb); diff --git a/packages/client-chains/src/types.d.ts b/packages/client-chains/src/types.d.ts index ac7c2edb..b8f74b32 100644 --- a/packages/client-chains/src/types.d.ts +++ b/packages/client-chains/src/types.d.ts @@ -19,7 +19,6 @@ export type ChainGenesisState = { export type ChainGenesis = { block: Uint8Array, code: Uint8Array, - codeHash: Uint8Array, header: Header, headerHash: Uint8Array }; diff --git a/packages/client-db-chain/package.json b/packages/client-db-chain/package.json index 6071f837..c6e848ff 100644 --- a/packages/client-db-chain/package.json +++ b/packages/client-db-chain/package.json @@ -35,12 +35,12 @@ "dependencies": { "@babel/runtime": "^7.0.0-beta.51", "@polkadot/client-db": "^0.13.3", - "@polkadot/storage": "^0.26.10", - "@polkadot/trie-hash": "^0.26.10", - "@polkadot/util": "^0.26.10", - "@polkadot/util-crypto": "^0.26.10" + "@polkadot/storage": "^0.26.13", + "@polkadot/trie-hash": "^0.26.13", + "@polkadot/util": "^0.26.13", + "@polkadot/util-crypto": "^0.26.13" }, "devDependencies": { - "@polkadot/storage": "^0.26.10" + "@polkadot/storage": "^0.26.13" } } diff --git a/packages/client-db-chain/src/block/index.ts b/packages/client-db-chain/src/block/index.ts index 040167cd..16b6e35a 100644 --- a/packages/client-db-chain/src/block/index.ts +++ b/packages/client-db-chain/src/block/index.ts @@ -2,14 +2,14 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import { TrieDb } from '@polkadot/util-triedb/types'; +import { BaseDb } from '@polkadot/client-db/types'; import { BlockDb } from '../types'; import createBn from '../db/bn'; import createU8a from '../db/u8a'; import keys from './keys'; -export default function blockDb (db: TrieDb): BlockDb { +export default function blockDb (db: BaseDb): BlockDb { return { db, bestHash: createU8a(db, keys.public.bestHash), diff --git a/packages/client-db-chain/src/db/arrayU8a.ts b/packages/client-db-chain/src/db/arrayU8a.ts index 687bb419..ada59f89 100644 --- a/packages/client-db-chain/src/db/arrayU8a.ts +++ b/packages/client-db-chain/src/db/arrayU8a.ts @@ -4,7 +4,7 @@ import { SectionItem } from '@polkadot/params/types'; import { Storage$Key$Value } from '@polkadot/storage/types'; -import { TrieDb } from '@polkadot/util-triedb/types'; +import { BaseDb } from '@polkadot/client-db/types'; import { StorageMethod$ArrayU8a } from '../types'; import bnToU8a from '@polkadot/util/bn/toU8a'; @@ -13,7 +13,7 @@ import u8aToBn from '@polkadot/util/u8a/toBn'; import creator from '../key'; -export default function decodeArrayU8a (db: TrieDb, key: SectionItem): StorageMethod$ArrayU8a { +export default function decodeArrayU8a (db: BaseDb, key: SectionItem): StorageMethod$ArrayU8a { const createKey = creator(key); return { @@ -37,7 +37,7 @@ export default function decodeArrayU8a (db: TrieDb, key: SectionItem): St return result; }, set: (value: Array, ...keyParams: Array): void => - db.set(createKey(keyParams), u8aConcat( + db.put(createKey(keyParams), u8aConcat( bnToU8a(value.length, 32, true), u8aConcat.apply(null, value)) ), diff --git a/packages/client-db-chain/src/db/base.ts b/packages/client-db-chain/src/db/base.ts index a188106e..7de19df8 100644 --- a/packages/client-db-chain/src/db/base.ts +++ b/packages/client-db-chain/src/db/base.ts @@ -2,7 +2,7 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import { TrieDb } from '@polkadot/util-triedb/types'; +import { BaseDb } from '@polkadot/client-db/types'; type Base = { del (key: Uint8Array): void, @@ -13,7 +13,7 @@ type Base = { type Subscribers = Array<(value: T, raw: Uint8Array) => void>; -export default function base (db: TrieDb): Base { +export default function base (db: BaseDb): Base { const subscribers: Subscribers = []; return { @@ -22,7 +22,7 @@ export default function base (db: TrieDb): Base { get: (key: Uint8Array): Uint8Array => db.get(key) || new Uint8Array([]), set: (key: Uint8Array, value: T, raw: Uint8Array): void => { - db.set(key, raw); + db.put(key, raw); subscribers.forEach((subscriber) => subscriber(value, raw) ); diff --git a/packages/client-db-chain/src/db/bn.ts b/packages/client-db-chain/src/db/bn.ts index 3bc1832e..cf443e68 100644 --- a/packages/client-db-chain/src/db/bn.ts +++ b/packages/client-db-chain/src/db/bn.ts @@ -5,7 +5,7 @@ import BN from 'bn.js'; import { SectionItem } from '@polkadot/params/types'; import { Storage$Key$Value } from '@polkadot/storage/types'; -import { TrieDb } from '@polkadot/util-triedb/types'; +import { BaseDb } from '@polkadot/client-db/types'; import { StorageMethod$Bn } from '../types'; import bnToU8a from '@polkadot/util/bn/toU8a'; @@ -14,7 +14,7 @@ import u8aToBn from '@polkadot/util/u8a/toBn'; import createBase from './base'; import creator from '../key'; -export default function decodeBn (db: TrieDb, key: SectionItem, bitLength: 32 | 64 | 128): StorageMethod$Bn { +export default function decodeBn (db: BaseDb, key: SectionItem, bitLength: 32 | 64 | 128): StorageMethod$Bn { const createKey = creator(key); const base = createBase(db); diff --git a/packages/client-db-chain/src/db/bool.ts b/packages/client-db-chain/src/db/bool.ts index 1318c673..945d10f3 100644 --- a/packages/client-db-chain/src/db/bool.ts +++ b/packages/client-db-chain/src/db/bool.ts @@ -4,13 +4,13 @@ import { SectionItem } from '@polkadot/params/types'; import { Storage$Key$Value } from '@polkadot/storage/types'; -import { TrieDb } from '@polkadot/util-triedb/types'; +import { BaseDb } from '@polkadot/client-db/types'; import { StorageMethod$Bool } from '../types'; import createBase from './base'; import creator from '../key'; -export default function decodeBool (db: TrieDb, key: SectionItem): StorageMethod$Bool { +export default function decodeBool (db: BaseDb, key: SectionItem): StorageMethod$Bool { const createKey = creator(key); const base = createBase(db); diff --git a/packages/client-db-chain/src/db/u8a.ts b/packages/client-db-chain/src/db/u8a.ts index 58c13cf6..f025c90b 100644 --- a/packages/client-db-chain/src/db/u8a.ts +++ b/packages/client-db-chain/src/db/u8a.ts @@ -4,13 +4,13 @@ import { SectionItem } from '@polkadot/params/types'; import { Storage$Key$Value } from '@polkadot/storage/types'; -import { TrieDb } from '@polkadot/util-triedb/types'; +import { BaseDb } from '@polkadot/client-db/types'; import { StorageMethod$U8a } from '../types'; import creator from '../key'; import createBase from './base'; -export default function decodeU8a (db: TrieDb, key: SectionItem): StorageMethod$U8a { +export default function decodeU8a (db: BaseDb, key: SectionItem): StorageMethod$U8a { const createKey = creator(key); const base = createBase(db); diff --git a/packages/client-db-chain/src/state/accountIndexOf.spec.js b/packages/client-db-chain/src/state/accountIndexOf.spec.js index c149d572..50394d38 100644 --- a/packages/client-db-chain/src/state/accountIndexOf.spec.js +++ b/packages/client-db-chain/src/state/accountIndexOf.spec.js @@ -24,7 +24,7 @@ describe('accountIndexOf', () => { return store[u8aToHex(key)] || new Uint8Array([]); }, - set: (key, value) => { + put: (key, value) => { console.log('setting', u8aToHex(key), value); store[u8aToHex(key)] = value; diff --git a/packages/client-db-chain/src/state/balance.spec.js b/packages/client-db-chain/src/state/balance.spec.js index 6fd29790..ac562af7 100644 --- a/packages/client-db-chain/src/state/balance.spec.js +++ b/packages/client-db-chain/src/state/balance.spec.js @@ -54,7 +54,7 @@ describe('balance', () => { staking = db({ get: (key) => store[u8aToHex(key)] || new Uint8Array([]), - set: (key, value) => { + put: (key, value) => { store[u8aToHex(key)] = value; } }).staking; diff --git a/packages/client-db-chain/src/state/blockHash.spec.js b/packages/client-db-chain/src/state/blockHash.spec.js index 571f7d81..7f1880b4 100644 --- a/packages/client-db-chain/src/state/blockHash.spec.js +++ b/packages/client-db-chain/src/state/blockHash.spec.js @@ -24,7 +24,7 @@ describe('blockHash', () => { return store[u8aToHex(key)] || new Uint8Array([]); }, - set: (value, key) => { + put: (value, key) => { console.log('setting', u8aToHex(key), value); store[u8aToHex(key)] = value; @@ -45,7 +45,7 @@ describe('blockHash', () => { system = db({ get: (key) => store[u8aToHex(key)] || new Uint8Array([]), - set: (key, value) => { + put: (key, value) => { store[u8aToHex(key)] = value; } }).system; diff --git a/packages/client-db-chain/src/state/index.ts b/packages/client-db-chain/src/state/index.ts index 8a5813af..f064019e 100644 --- a/packages/client-db-chain/src/state/index.ts +++ b/packages/client-db-chain/src/state/index.ts @@ -4,7 +4,7 @@ import { Section } from '@polkadot/params/types'; import { Storages } from '@polkadot/storage/types'; -import { TrieDb } from '@polkadot/util-triedb/types'; +import { TrieDb } from '@polkadot/client-db/types'; import { StateDb } from '../types'; import storage from '@polkadot/storage'; diff --git a/packages/client-db-chain/src/types.d.ts b/packages/client-db-chain/src/types.d.ts index 5dd21b28..6c10aba6 100644 --- a/packages/client-db-chain/src/types.d.ts +++ b/packages/client-db-chain/src/types.d.ts @@ -3,7 +3,7 @@ // of the ISC license. See the LICENSE file for details. import BN from 'bn.js'; -import { TrieDb } from '@polkadot/util-triedb/types'; +import { BaseDb, TrieDb } from '@polkadot/client-db/types'; import { Storage$Key$Value } from '@polkadot/storage/types'; export type StorageMethod = { @@ -27,16 +27,13 @@ export type StorageMethod$ArrayU8a = StorageMethod, Array = O & { - db: TrieDb -} - -export type BlockDb = WrappedDb<{ +export type BlockDb = { + db: BaseDb, bestHash: StorageMethod$U8a, bestNumber: StorageMethod$Bn, block: StorageMethod$U8a, header: StorageMethod$U8a -}>; +}; export type StateDb$Consensus = { authorityAt: StorageMethod$Account, @@ -100,7 +97,8 @@ export type StateDb$Timestamp = { didUpdate: StorageMethod$Bool }; -export type StateDb = WrappedDb<{ +export type StateDb = { + db: TrieDb, consensus: StateDb$Consensus, council: StateDb$Council, councilVoting: StateDb$CouncilVoting, @@ -111,4 +109,4 @@ export type StateDb = WrappedDb<{ staking: StateDb$Staking, system: StateDb$System, timestamp: StateDb$Timestamp -}>; +}; diff --git a/packages/client-p2p-messages/package.json b/packages/client-p2p-messages/package.json index f82e18e4..62adbc33 100644 --- a/packages/client-p2p-messages/package.json +++ b/packages/client-p2p-messages/package.json @@ -35,8 +35,8 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", - "@polkadot/jsonrpc": "^0.26.10", - "@polkadot/primitives": "^0.26.10", - "@polkadot/util": "^0.26.10" + "@polkadot/jsonrpc": "^0.26.13", + "@polkadot/primitives": "^0.26.13", + "@polkadot/util": "^0.26.13" } } diff --git a/packages/client-p2p/package.json b/packages/client-p2p/package.json index 01266abc..54919ff5 100644 --- a/packages/client-p2p/package.json +++ b/packages/client-p2p/package.json @@ -36,8 +36,8 @@ "dependencies": { "@babel/runtime": "^7.0.0-beta.51", "@polkadot/client-p2p-messages": "^0.13.3", - "@polkadot/primitives": "^0.26.10", - "@polkadot/util": "^0.26.10", + "@polkadot/primitives": "^0.26.13", + "@polkadot/util": "^0.26.13", "eventemitter3": "^2.0.3", "libp2p": "^0.22.0", "libp2p-kad-dht": "^0.10.0", diff --git a/packages/client-rpc/package.json b/packages/client-rpc/package.json index c193f413..fe4ed352 100644 --- a/packages/client-rpc/package.json +++ b/packages/client-rpc/package.json @@ -39,7 +39,7 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", - "@polkadot/util": "^0.26.10", + "@polkadot/util": "^0.26.13", "@types/co-body": "^0.0.3", "@types/koa": "^2.0.46", "@types/koa-route": "^3.2.4", diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index b6d3ea5b..3563ef45 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -35,9 +35,8 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", - "@polkadot/trie-db": "^0.26.10", - "@polkadot/util": "^0.26.10", - "@polkadot/util-crypto": "^0.26.10", - "@polkadot/util-triedb": "^0.26.10" + "@polkadot/client-db": "^0.13.3", + "@polkadot/util": "^0.26.13", + "@polkadot/util-crypto": "^0.26.13" } } diff --git a/packages/client-runtime/src/environment/db/clear.spec.js b/packages/client-runtime/src/environment/db/clear.spec.js deleted file mode 100644 index bb92cc48..00000000 --- a/packages/client-runtime/src/environment/db/clear.spec.js +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import index from './index'; - -describe('clear', () => { - let backend; - let int; - - beforeEach(() => { - backend = { - get: jest.fn(() => new Uint8Array([3])) - }; - - int = index(backend); - }); - - it('clears all set values', () => { - int.set(new Uint8Array([1]), new Uint8Array([2])); - int.clear(); - - expect( - int.isEmpty() - ).toEqual(true); - }); -}); diff --git a/packages/client-runtime/src/environment/db/clear.ts b/packages/client-runtime/src/environment/db/clear.ts deleted file mode 100644 index e65ebcb3..00000000 --- a/packages/client-runtime/src/environment/db/clear.ts +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import { DbState } from './types'; - -export default function clear (self: DbState): void { - self.pending = {}; -} diff --git a/packages/client-runtime/src/environment/db/commit.spec.js b/packages/client-runtime/src/environment/db/commit.spec.js deleted file mode 100644 index a3e30f09..00000000 --- a/packages/client-runtime/src/environment/db/commit.spec.js +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import index from './index'; - -describe('commit', () => { - let backend; - let int; - - beforeEach(() => { - backend = { - commit: jest.fn(), - pairs: jest.fn(() => []) - }; - - int = index(backend); - }); - - it('clears storage after commit', () => { - int.set(new Uint8Array([1]), new Uint8Array([2])); - int.commit(); - - expect( - int.isEmpty() - ).toEqual(true); - }); - - it('calls backend commit', () => { - int.set(new Uint8Array([1]), new Uint8Array([2])); - int.commit(); - - expect( - backend.commit - ).toHaveBeenCalledWith([ - { k: new Uint8Array([1]), v: new Uint8Array([2]) } - ]); - }); -}); diff --git a/packages/client-runtime/src/environment/db/commit.ts b/packages/client-runtime/src/environment/db/commit.ts deleted file mode 100644 index cd9d4c41..00000000 --- a/packages/client-runtime/src/environment/db/commit.ts +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import { Trie$Pairs } from '@polkadot/trie-hash/types'; -import { DbState } from './types'; - -import clear from './clear'; -import pairs from './pairs'; - -export default function commit (self: DbState, values: Trie$Pairs): void { - self.backend.commit( - pairs(self).concat(values) - ); - clear(self); -} diff --git a/packages/client-runtime/src/environment/db/del.spec.js b/packages/client-runtime/src/environment/db/del.spec.js deleted file mode 100644 index 5096a761..00000000 --- a/packages/client-runtime/src/environment/db/del.spec.js +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import index from './index'; - -describe('del', () => { - let int; - - beforeEach(() => { - int = index({}); - }); - - it('removes a set value', () => { - int.set(new Uint8Array([1]), new Uint8Array([2])); - - expect( - int.get(new Uint8Array([1])) - ).toEqual( - new Uint8Array([2]) - ); - - int.del(new Uint8Array([1])); - - expect( - int.get(new Uint8Array([1])) - ).toBeNull(); - }); -}); diff --git a/packages/client-runtime/src/environment/db/del.ts b/packages/client-runtime/src/environment/db/del.ts deleted file mode 100644 index 8ba91a05..00000000 --- a/packages/client-runtime/src/environment/db/del.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import { DbState } from './types'; - -export default function del ({ pending }: DbState, k: Uint8Array): void { - pending[k.toString()] = { - k: k.slice(), - v: null - }; -} diff --git a/packages/client-runtime/src/environment/db/get.spec.js b/packages/client-runtime/src/environment/db/get.spec.js deleted file mode 100644 index f0eb47f7..00000000 --- a/packages/client-runtime/src/environment/db/get.spec.js +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import index from './index'; - -describe('get', () => { - let int; - - beforeEach(() => { - int = index({ - get: jest.fn(() => new Uint8Array([3, 4, 5])) - }); - }); - - it('retrieves a value', () => { - int.set(new Uint8Array([1]), new Uint8Array([2, 3, 4])); - - expect( - int.get(new Uint8Array([1])) - ).toEqual( - new Uint8Array([2, 3, 4]) - ); - }); - - it('retrieves a value from backed storage', () => { - expect( - int.get(new Uint8Array([1])) - ).toEqual( - new Uint8Array([3, 4, 5]) - ); - }); -}); diff --git a/packages/client-runtime/src/environment/db/get.ts b/packages/client-runtime/src/environment/db/get.ts deleted file mode 100644 index e6ae91c7..00000000 --- a/packages/client-runtime/src/environment/db/get.ts +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import { DbState } from './types'; - -export default function get ({ backend, pending }: DbState, k: Uint8Array): Uint8Array | null { - const value = pending[k.toString()]; - - if (value) { - return value.v - ? value.v.slice() - : null; - } - - return backend.get(k) || null; -} diff --git a/packages/client-runtime/src/environment/db/index.ts b/packages/client-runtime/src/environment/db/index.ts deleted file mode 100644 index d33850c0..00000000 --- a/packages/client-runtime/src/environment/db/index.ts +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import { TrieDb } from '@polkadot/util-triedb/types'; -import { Trie$Pairs } from '@polkadot/trie-hash/types'; -import { DbState } from './types'; - -import trieRoot from '@polkadot/trie-hash/root'; - -import clear from './clear'; -import commit from './commit'; -import del from './del'; -import get from './get'; -import pairs from './pairs'; -import set from './set'; - -export default function envDb (backend: TrieDb): TrieDb { - const self: DbState = { - backend, - pending: {} - }; - - return { - clear: (): void => - clear(self), - commit: (values: Trie$Pairs = []): void => - commit(self, values), - del: (k: Uint8Array): void => - del(self, k), - isEmpty: (): boolean => - Object.keys(self.pending).length === 0, - get: (k: Uint8Array): Uint8Array | null => - get(self, k), - pairs: (): Trie$Pairs => - pairs(self), - set: (k: Uint8Array, v: Uint8Array): void => - set(self, k, v), - trieRoot: (): Uint8Array => - trieRoot(pairs(self)) - }; -} diff --git a/packages/client-runtime/src/environment/db/pairs.spec.js b/packages/client-runtime/src/environment/db/pairs.spec.js deleted file mode 100644 index a4d42879..00000000 --- a/packages/client-runtime/src/environment/db/pairs.spec.js +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import index from './index'; - -describe('pairs', () => { - let backend; - - beforeEach(() => { - backend = { - pairs: jest.fn(() => [ - { k: new Uint8Array([1]), v: new Uint8Array([0]) }, - { k: new Uint8Array([3]), v: new Uint8Array([3]) }, - { k: new Uint8Array([4]), v: new Uint8Array([4]) } - ]) - }; - }); - - it('retrieves pending and backend pairs', () => { - const int = index(backend); - - int.set(new Uint8Array([1]), new Uint8Array([1])); - int.set(new Uint8Array([2]), new Uint8Array([2])); - - expect( - int.pairs() - ).toEqual([ - { k: new Uint8Array([1]), v: new Uint8Array([0]) }, - { k: new Uint8Array([3]), v: new Uint8Array([3]) }, - { k: new Uint8Array([4]), v: new Uint8Array([4]) }, - { k: new Uint8Array([1]), v: new Uint8Array([1]) }, - { k: new Uint8Array([2]), v: new Uint8Array([2]) } - ]); - }); - - it('retrieves pending only', () => { - const int = index(); - - int.set(new Uint8Array([1]), new Uint8Array([1])); - int.set(new Uint8Array([2]), new Uint8Array([2])); - - expect( - int.pairs() - ).toEqual([ - { k: new Uint8Array([1]), v: new Uint8Array([1]) }, - { k: new Uint8Array([2]), v: new Uint8Array([2]) } - ]); - }); - - it('retrieves, ignoring deleted pairs', () => { - const int = index(backend); - - int.set(new Uint8Array([1]), new Uint8Array([1])); - int.set(new Uint8Array([2]), new Uint8Array([2])); - int.del(new Uint8Array([4])); - - expect( - int.pairs() - ).toEqual([ - { k: new Uint8Array([1]), v: new Uint8Array([0]) }, - { k: new Uint8Array([3]), v: new Uint8Array([3]) }, - { k: new Uint8Array([1]), v: new Uint8Array([1]) }, - { k: new Uint8Array([2]), v: new Uint8Array([2]) } - ]); - }); -}); diff --git a/packages/client-runtime/src/environment/db/pairs.ts b/packages/client-runtime/src/environment/db/pairs.ts deleted file mode 100644 index 552cb157..00000000 --- a/packages/client-runtime/src/environment/db/pairs.ts +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import { Trie$Pairs } from '@polkadot/trie-hash/types'; -import { DbState } from './types'; - -export default function pairs ({ backend, pending }: DbState): Trie$Pairs { - const backendPairs = backend - ? backend.pairs() - : []; - const pendingKeys = Object.keys(pending); - const deletedKeys = pendingKeys.filter((k) => - pending[k].v === null - ); - const pendingPairs = Object.values(pending).filter(({ v }) => - v !== null - ); - - return backendPairs - .filter(({ k }) => - deletedKeys.indexOf(k.toString()) === -1 - ) - .concat(pendingPairs as Trie$Pairs); -} diff --git a/packages/client-runtime/src/environment/db/set.spec.js b/packages/client-runtime/src/environment/db/set.spec.js deleted file mode 100644 index b0058686..00000000 --- a/packages/client-runtime/src/environment/db/set.spec.js +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import index from './index'; - -describe('set', () => { - let backend; - let int; - - beforeEach(() => { - backend = { - pairs: () => [] - }; - - int = index(backend); - }); - - it('sets the value', () => { - int.set(new Uint8Array([1]), new Uint8Array([2])); - - expect( - int.isEmpty() - ).toEqual(false); - }); - - it('sets the value (retrievable)', () => { - int.set(new Uint8Array([1]), new Uint8Array([2])); - - expect( - int.pairs() - ).toEqual([ - { k: new Uint8Array([1]), v: new Uint8Array([2]) } - ]); - }); -}); diff --git a/packages/client-runtime/src/environment/db/set.ts b/packages/client-runtime/src/environment/db/set.ts deleted file mode 100644 index 912156fb..00000000 --- a/packages/client-runtime/src/environment/db/set.ts +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import { Trie$Pair } from '@polkadot/trie-hash/types'; -import { DbState } from './types'; - -export default function set ({ pending }: DbState, k: Uint8Array, v: Uint8Array): void { - pending[k.toString()] = ({ - k: k.slice(), - v: v.slice() - } as Trie$Pair); -} diff --git a/packages/client-runtime/src/environment/db/types.d.ts b/packages/client-runtime/src/environment/db/types.d.ts deleted file mode 100644 index 0cefe4fe..00000000 --- a/packages/client-runtime/src/environment/db/types.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-runtime authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import { TrieDb } from '@polkadot/util-triedb/types'; -import { Trie$Pair } from '@polkadot/trie-hash/types'; - -type Trie$Pair$Null = { - k: Uint8Array, - v: Uint8Array | null -}; - -export type DbState = { - backend: TrieDb, - pending: { - [index: string]: Trie$Pair | Trie$Pair$Null - } -}; diff --git a/packages/client-runtime/src/environment/index.ts b/packages/client-runtime/src/environment/index.ts index 09c47ace..3ae58c06 100644 --- a/packages/client-runtime/src/environment/index.ts +++ b/packages/client-runtime/src/environment/index.ts @@ -2,20 +2,19 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import { TrieDb } from '@polkadot/util-triedb/types'; +import { TrieDb } from '@polkadot/client-db/types'; import { RuntimeEnv } from '../types'; import logger from '@polkadot/util/logger'; -import envDb from './db'; import envHeap from './heap'; const l = logger('runtime'); -export default function environment (stateDb: TrieDb): RuntimeEnv { +export default function environment (db: TrieDb): RuntimeEnv { return { l, - db: envDb(stateDb), + db, heap: envHeap() }; } diff --git a/packages/client-runtime/src/index.ts b/packages/client-runtime/src/index.ts index 43b84605..7f7dd3ae 100644 --- a/packages/client-runtime/src/index.ts +++ b/packages/client-runtime/src/index.ts @@ -2,7 +2,7 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import { TrieDb } from '@polkadot/util-triedb/types'; +import { TrieDb } from '@polkadot/client-db/types'; import { RuntimeInterface, RuntimeInterface$Exports } from './types'; import createChain from './chain'; diff --git a/packages/client-runtime/src/storage/allocated.spec.js b/packages/client-runtime/src/storage/allocated.spec.js index b2049cdf..248f3860 100644 --- a/packages/client-runtime/src/storage/allocated.spec.js +++ b/packages/client-runtime/src/storage/allocated.spec.js @@ -3,10 +3,10 @@ // of the ISC license. See the LICENSE file for details. /* eslint camelcase: 0 */ +import HashDb from '@polkadot/client-db/Hash'; import logger from '@polkadot/util/logger'; import u8aFromString from '@polkadot/util/u8a/fromString'; -import envDb from '../environment/db'; import envHeap from '../environment/heap'; import index from './index'; @@ -20,9 +20,8 @@ describe('get_allocated_storage', () => { beforeEach(() => { heap = envHeap(); heap.setWasmMemory({ buffer: new Uint8Array(1024 * 1024) }); - db = envDb({ - get: () => null - }); + + db = new HashDb(); get_allocated_storage = index({ l, heap, db }).get_allocated_storage; }); @@ -31,7 +30,7 @@ describe('get_allocated_storage', () => { const key = u8aFromString('key'); const value = u8aFromString('some value'); - db.set(key, value); + db.put(key, value); const keyPtr = heap.allocate(key.length); const lenPtr = heap.allocate(4); diff --git a/packages/client-runtime/src/storage/data.ts b/packages/client-runtime/src/storage/data.ts index 856f5422..3e000079 100644 --- a/packages/client-runtime/src/storage/data.ts +++ b/packages/client-runtime/src/storage/data.ts @@ -61,7 +61,7 @@ export default function data ({ l, heap, db }: RuntimeEnv): RuntimeInterface$Sto l.debug(() => ['set_storage', [keyPtr, keyLength, dataPtr, dataLength], '<-', u8aToHex(key), '=', u8aToHex(data)]); - db.set(key, data); + db.put(key, data); }) }; } diff --git a/packages/client-runtime/src/storage/get.ts b/packages/client-runtime/src/storage/get.ts index 2b2ba3d2..6b0203ae 100644 --- a/packages/client-runtime/src/storage/get.ts +++ b/packages/client-runtime/src/storage/get.ts @@ -2,7 +2,7 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import { TrieDb } from '@polkadot/util-triedb/types'; +import { TrieDb } from '@polkadot/client-db/types'; export default function get (db: TrieDb, key: Uint8Array, offset: number, maxLength: number): Uint8Array | null { const data = db.get(key); diff --git a/packages/client-runtime/src/storage/root.spec.js b/packages/client-runtime/src/storage/root.spec.js index 2d1be449..eef164db 100644 --- a/packages/client-runtime/src/storage/root.spec.js +++ b/packages/client-runtime/src/storage/root.spec.js @@ -3,6 +3,7 @@ // of the ISC license. See the LICENSE file for details. /* eslint camelcase: 0 */ +import MemoryDb from '@polkadot/client-db/Memory'; import logger from '@polkadot/util/logger'; import hexToU8a from '@polkadot/util/hex/toU8a'; import u8aFromString from '@polkadot/util/u8a/fromString'; @@ -21,17 +22,16 @@ describe('storage_root', () => { set: jest.fn() }; - db = {}; + db = new MemoryDb(); storage_root = index({ l, heap, db }).storage_root; }); it('creates a basic storage root', () => { - db.pairs = () => ([ - { k: u8aFromString('doe'), v: u8aFromString('reindeer') }, - { k: u8aFromString('dog'), v: u8aFromString('puppy') }, - { k: u8aFromString('dogglesworth'), v: u8aFromString('cat') } - ]); + db.put(u8aFromString('doe'), u8aFromString('reindeer')); + db.put(u8aFromString('dog'), u8aFromString('puppy')); + db.put(u8aFromString('dogglesworth'), u8aFromString('cat')); + storage_root(5); expect( diff --git a/packages/client-runtime/src/storage/set.spec.js b/packages/client-runtime/src/storage/set.spec.js index 2ebfa635..40873c9d 100644 --- a/packages/client-runtime/src/storage/set.spec.js +++ b/packages/client-runtime/src/storage/set.spec.js @@ -22,7 +22,7 @@ describe('set_storage', () => { }; db = { - set: jest.fn() + put: jest.fn() }; set_storage = index({ l, heap, db }).set_storage; @@ -32,7 +32,7 @@ describe('set_storage', () => { set_storage(0, 3, 3, 5); expect( - db.set + db.put ).toHaveBeenCalledWith( new Uint8Array([0x53, 0x61, 0x79]), new Uint8Array([72, 101, 108, 108, 111]) diff --git a/packages/client-runtime/src/storage/trie.ts b/packages/client-runtime/src/storage/trie.ts index 4124ceb9..8db86fac 100644 --- a/packages/client-runtime/src/storage/trie.ts +++ b/packages/client-runtime/src/storage/trie.ts @@ -4,7 +4,6 @@ import { RuntimeEnv, RuntimeInterface$Storage$Trie, Pointer } from '../types'; -import trieRoot from '@polkadot/trie-hash/root'; import trieRootOrdered from '@polkadot/trie-hash/rootOrdered'; import u8aToHex from '@polkadot/util/u8a/toHex'; @@ -30,10 +29,9 @@ export default function storage ({ l, heap, db }: RuntimeEnv): RuntimeInterface$ }), storage_root: (resultPtr: Pointer): void => instrument('storage_root', (): void => { - const pairs = db.pairs(); - const root = trieRoot(pairs); + const root = db.trieRoot(); - l.debug(() => ['storage_root', [resultPtr], '<-', pairs.length, '->', u8aToHex(root)]); + l.debug(() => ['storage_root', [resultPtr], '->', u8aToHex(root)]); heap.set(resultPtr, root); }) diff --git a/packages/client-runtime/src/types.d.ts b/packages/client-runtime/src/types.d.ts index fb1d0d43..7382535e 100644 --- a/packages/client-runtime/src/types.d.ts +++ b/packages/client-runtime/src/types.d.ts @@ -4,7 +4,7 @@ /// -import { TrieDb } from '@polkadot/util-triedb/types'; +import { TrieDb } from '@polkadot/client-db/types'; import { Logger } from '@polkadot/util/types'; import { SizeUsed } from './environment/heap/types'; @@ -38,8 +38,8 @@ export type RuntimeEnv$Heap = { export type RuntimeEnv = { heap: RuntimeEnv$Heap, - l: Logger, - db: TrieDb + db: TrieDb, + l: Logger }; export type RuntimeInterface$Chain = { diff --git a/packages/client-telemetry/package.json b/packages/client-telemetry/package.json index 30f5e22c..ac2f2ec8 100644 --- a/packages/client-telemetry/package.json +++ b/packages/client-telemetry/package.json @@ -35,14 +35,7 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", - "@polkadot/util": "^0.26.10", - "@polkadot/util-triedb": "^0.26.10", - "@types/mkdirp": "^0.5.2", - "mkdirp": "^0.5.1", + "@polkadot/util": "^0.26.13", "websocket": "^1.0.26" - }, - "devDependencies": { - "@polkadot/client": "^0.13.3", - "rimraf": "^2.6.2" } } diff --git a/packages/client-wasm/package.json b/packages/client-wasm/package.json index bc8e5161..fc6c1531 100644 --- a/packages/client-wasm/package.json +++ b/packages/client-wasm/package.json @@ -36,8 +36,8 @@ "dependencies": { "@babel/runtime": "^7.0.0-beta.51", "@polkadot/client-runtime": "^0.13.3", - "@polkadot/primitives": "^0.26.10", - "@polkadot/util": "^0.26.10", + "@polkadot/primitives": "^0.26.13", + "@polkadot/util": "^0.26.13", "@polkadot/wasm-bin": "paritytech/polkadot-wasm-bin#5ab4fc6083501b26e5fd979b93c35e2f9010b324", "@types/webassembly-js-api": "^0.0.1", "yargs": "^11.0.0" diff --git a/packages/client-wasm/src/executor/executeBlock.spec.js b/packages/client-wasm/src/executor/executeBlock.spec.js index d5d57724..1e2b29a9 100644 --- a/packages/client-wasm/src/executor/executeBlock.spec.js +++ b/packages/client-wasm/src/executor/executeBlock.spec.js @@ -2,7 +2,8 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import memoryDb from '@polkadot/util-triedb/temp'; +import HashDb from '@polkadot/client-db/Hash'; +import MemoryDb from '@polkadot/client-db/Memory'; import init from '@polkadot/client-chains'; @@ -18,7 +19,7 @@ describe('executeBlock', () => { wasm: {} }; - chain = init(config, memoryDb(), memoryDb()); + chain = init(config, new MemoryDb(), new HashDb()); }); it('executes an actual block', () => { diff --git a/packages/client-wasm/src/executor/generateBlock.spec.js b/packages/client-wasm/src/executor/generateBlock.spec.js index 36ebf4c5..5abb8922 100644 --- a/packages/client-wasm/src/executor/generateBlock.spec.js +++ b/packages/client-wasm/src/executor/generateBlock.spec.js @@ -2,7 +2,8 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import memoryDb from '@polkadot/util-triedb/temp'; +import HashDb from '@polkadot/client-db/Hash'; +import MemoryDb from '@polkadot/client-db/Memory'; import methods from '@polkadot/extrinsics'; import encodeUnchecked from '@polkadot/extrinsics/codec/encode/unchecked'; import u8aConcat from '@polkadot/util/u8a/concat'; @@ -39,7 +40,7 @@ describe('generateBlock', () => { wasm: {} }; - chain = init(config, memoryDb(), memoryDb()); + chain = init(config, new MemoryDb(), new HashDb()); }); it('generates a basic block (empty)', () => { diff --git a/packages/client-wasm/src/executor/generateBlock.ts b/packages/client-wasm/src/executor/generateBlock.ts index 7011be30..79651b13 100644 --- a/packages/client-wasm/src/executor/generateBlock.ts +++ b/packages/client-wasm/src/executor/generateBlock.ts @@ -22,6 +22,8 @@ export default function generateBlock (self: ExecutorState, _extrinsics: Array `Generating block #${number.toString()}`); + self.stateDb.db.checkpoint(); + const extrinsics = withInherent(self, timestamp, _extrinsics); const header = createHeader({ number, @@ -45,7 +47,7 @@ export default function generateBlock (self: ExecutorState, _extrinsics: Array `Block #${number.toString()} generated (${Date.now() - start}ms)`); diff --git a/packages/client-wasm/src/executor/importBlock.ts b/packages/client-wasm/src/executor/importBlock.ts index 7ed74bc3..6cf3c168 100644 --- a/packages/client-wasm/src/executor/importBlock.ts +++ b/packages/client-wasm/src/executor/importBlock.ts @@ -14,6 +14,8 @@ export default function importBlock (self: ExecutorState, block: Uint8Array): Ex self.l.debug(() => 'Importing block'); + self.stateDb.db.checkpoint(); + executeBlock(self, block); self.stateDb.db.commit(); diff --git a/packages/client-wasm/src/executor/initialiseBlock.spec.js b/packages/client-wasm/src/executor/initialiseBlock.spec.js index 90e573dc..0b4156cf 100644 --- a/packages/client-wasm/src/executor/initialiseBlock.spec.js +++ b/packages/client-wasm/src/executor/initialiseBlock.spec.js @@ -4,7 +4,8 @@ import createHeader from '@polkadot/primitives/create/header'; import encodeHeader from '@polkadot/primitives/codec/header/encode'; -import memoryDb from '@polkadot/util-triedb/temp'; +import HashDb from '@polkadot/client-db/Hash'; +import MemoryDb from '@polkadot/client-db/Memory'; import init from '@polkadot/client-chains'; @@ -17,7 +18,7 @@ describe('initialiseBlock', () => { wasm: {} }; - chain = init(config, memoryDb(), memoryDb()); + chain = init(config, new MemoryDb(), new HashDb()); }); it('initialises a block', () => { diff --git a/packages/client/package.json b/packages/client/package.json index 63e2dda8..7ee59673 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -44,10 +44,9 @@ "@polkadot/client-rpc": "^0.13.3", "@polkadot/client-telemetry": "^0.13.3", "@polkadot/client-wasm": "^0.13.3", - "@polkadot/jsonrpc": "^0.26.10", - "@polkadot/primitives": "^0.26.10", - "@polkadot/util": "^0.26.10", - "@polkadot/util-triedb": "^0.26.10", + "@polkadot/jsonrpc": "^0.26.13", + "@polkadot/primitives": "^0.26.13", + "@polkadot/util": "^0.26.13", "@types/package-json": "^4.0.1", "@types/semver-compare": "^1.0.0", "@types/yargs": "^11.0.0", diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index fd837d27..e339e125 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -5,10 +5,11 @@ import './license'; import createChain from '@polkadot/client-chains/index'; -import memoryDb from '@polkadot/util-triedb/temp'; import createP2p from '@polkadot/client-p2p/index'; import telemetry from '@polkadot/client-telemetry/index'; import logger from '@polkadot/util/logger'; +import HashDb from '@polkadot/client-db/Hash'; +import MemoryDb from '@polkadot/client-db/Memory'; import * as clientId from './clientId'; import cli from './cli'; @@ -24,7 +25,7 @@ const config = cli(); l.log(`Running version ${clientId.version} (${verStatus})`); l.log(`Initialising for roles=${config.roles.join(',')} on chain=${config.chain}`); - const chain = createChain(config, memoryDb(), memoryDb()); + const chain = createChain(config, new MemoryDb(), new HashDb()); telemetry.init(config, chain); diff --git a/yarn.lock b/yarn.lock index 5941897a..c080d1d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -759,11 +759,20 @@ "@polkadot/primitives" "^0.26.10" "@polkadot/util" "^0.26.10" -"@polkadot/jsonrpc@^0.26.10": - version "0.26.10" - resolved "https://registry.yarnpkg.com/@polkadot/jsonrpc/-/jsonrpc-0.26.10.tgz#c0cf8beb332ad072294e3ab2d1b658b42950b490" +"@polkadot/extrinsics@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/extrinsics/-/extrinsics-0.26.13.tgz#b6358536c0125a98e3b4659ba9771d13021dba73" dependencies: - "@polkadot/params" "^0.26.10" + "@babel/runtime" "^7.0.0-beta.51" + "@polkadot/params" "^0.26.13" + "@polkadot/primitives" "^0.26.13" + "@polkadot/util" "^0.26.13" + +"@polkadot/jsonrpc@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/jsonrpc/-/jsonrpc-0.26.13.tgz#53709aee56834818422e8c0d1a869dc03528cce8" + dependencies: + "@polkadot/params" "^0.26.13" babel-runtime "^6.26.0" "@polkadot/params@^0.26.10": @@ -775,6 +784,15 @@ "@polkadot/primitives" "^0.26.10" "@polkadot/util" "^0.26.10" +"@polkadot/params@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/params/-/params-0.26.13.tgz#ae0fef6446c5ed2f7700d218060603e6a795b689" + dependencies: + "@babel/runtime" "^7.0.0-beta.51" + "@polkadot/extrinsics" "^0.26.13" + "@polkadot/primitives" "^0.26.13" + "@polkadot/util" "^0.26.13" + "@polkadot/primitives@^0.26.10": version "0.26.10" resolved "https://registry.yarnpkg.com/@polkadot/primitives/-/primitives-0.26.10.tgz#e1a5823433ecabc5b15a1e31fbe378b7d4ea6e55" @@ -783,6 +801,14 @@ "@polkadot/trie-hash" "^0.26.10" "@polkadot/util" "^0.26.10" +"@polkadot/primitives@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/primitives/-/primitives-0.26.13.tgz#592d1a92865b7dd0fe57ffa12755f068c33e16df" + dependencies: + "@babel/runtime" "^7.0.0-beta.51" + "@polkadot/trie-hash" "^0.26.13" + "@polkadot/util" "^0.26.13" + "@polkadot/storage@^0.26.10": version "0.26.10" resolved "https://registry.yarnpkg.com/@polkadot/storage/-/storage-0.26.10.tgz#8d69490011e916c951fa566d411355f26bb7b082" @@ -794,15 +820,26 @@ "@polkadot/util-crypto" "^0.26.10" "@polkadot/util-keyring" "^0.26.10" -"@polkadot/trie-db@^0.26.10": - version "0.26.10" - resolved "https://registry.yarnpkg.com/@polkadot/trie-db/-/trie-db-0.26.10.tgz#4d4e70e62d13b997a8e3e02e9a3c9983656d1cea" +"@polkadot/storage@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/storage/-/storage-0.26.13.tgz#774be216cfc15ef345fc9dc35b78a376e32a5849" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/trie-hash" "^0.26.10" - "@polkadot/util" "^0.26.10" - "@polkadot/util-crypto" "^0.26.10" - "@polkadot/util-rlp" "^0.26.10" + "@polkadot/params" "^0.26.13" + "@polkadot/primitives" "^0.26.13" + "@polkadot/util" "^0.26.13" + "@polkadot/util-crypto" "^0.26.13" + "@polkadot/util-keyring" "^0.26.13" + +"@polkadot/trie-db@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/trie-db/-/trie-db-0.26.13.tgz#b3d6d3fc4cfd58db3ae9aea0c3df68ec414bb68d" + dependencies: + "@babel/runtime" "^7.0.0-beta.51" + "@polkadot/trie-hash" "^0.26.13" + "@polkadot/util" "^0.26.13" + "@polkadot/util-crypto" "^0.26.13" + "@polkadot/util-rlp" "^0.26.13" "@types/async" "^2.0.49" "@types/semaphore" "^1.1.0" async "^1.4.2" @@ -822,6 +859,15 @@ "@polkadot/util-crypto" "^0.26.10" "@polkadot/util-rlp" "^0.26.10" +"@polkadot/trie-hash@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/trie-hash/-/trie-hash-0.26.13.tgz#f47cfa4c9ede895342244e85076279a874e99451" + dependencies: + "@babel/runtime" "^7.0.0-beta.51" + "@polkadot/util" "^0.26.13" + "@polkadot/util-crypto" "^0.26.13" + "@polkadot/util-rlp" "^0.26.13" + "@polkadot/ts@^0.1.8": version "0.1.8" resolved "https://registry.yarnpkg.com/@polkadot/ts/-/ts-0.1.8.tgz#04e936fca0b104b8e0c4902bbfcc75ff5d58b98c" @@ -837,6 +883,17 @@ tweetnacl "^1.0.0" xxhashjs "^0.2.2" +"@polkadot/util-crypto@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/util-crypto/-/util-crypto-0.26.13.tgz#b882c79fce24d50586687ef72a30dfa916650b23" + dependencies: + "@babel/runtime" "^7.0.0-beta.51" + "@polkadot/util" "^0.26.13" + blakejs "^1.1.0" + js-sha3 "^0.7.0" + tweetnacl "^1.0.0" + xxhashjs "^0.2.2" + "@polkadot/util-keyring@^0.26.10": version "0.26.10" resolved "https://registry.yarnpkg.com/@polkadot/util-keyring/-/util-keyring-0.26.10.tgz#42b26231a3216b92cc6576397af59edbaa2db7ef" @@ -847,6 +904,16 @@ "@types/bs58" "^3.0.30" bs58 "^4.0.1" +"@polkadot/util-keyring@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/util-keyring/-/util-keyring-0.26.13.tgz#b15e36eda74f5faa0bf923a071a18d20eba35c34" + dependencies: + "@babel/runtime" "^7.0.0-beta.51" + "@polkadot/util" "^0.26.13" + "@polkadot/util-crypto" "^0.26.13" + "@types/bs58" "^3.0.30" + bs58 "^4.0.1" + "@polkadot/util-rlp@^0.26.10": version "0.26.10" resolved "https://registry.yarnpkg.com/@polkadot/util-rlp/-/util-rlp-0.26.10.tgz#85599e297e6348b2fafd8d2935855dcfaaf139c7" @@ -854,18 +921,12 @@ "@babel/runtime" "^7.0.0-beta.51" "@polkadot/util" "^0.26.10" -"@polkadot/util-triedb@^0.26.10": - version "0.26.10" - resolved "https://registry.yarnpkg.com/@polkadot/util-triedb/-/util-triedb-0.26.10.tgz#f78046d0283dda7ff3e9acdce18f46d48a66f187" +"@polkadot/util-rlp@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/util-rlp/-/util-rlp-0.26.13.tgz#0f346ba5b1b41e2232d06523a4b4a0addeed4026" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/trie-hash" "^0.26.10" - "@polkadot/util" "^0.26.10" - "@polkadot/util-crypto" "^0.26.10" - "@polkadot/util-rlp" "^0.26.10" - leveldown "^4.0.1" - levelup "^3.0.1" - memdown "^3.0.0" + "@polkadot/util" "^0.26.13" "@polkadot/util@^0.26.10": version "0.26.10" @@ -881,6 +942,20 @@ deasync "^0.1.13" ip-regex "^2.1.0" +"@polkadot/util@^0.26.13": + version "0.26.13" + resolved "https://registry.yarnpkg.com/@polkadot/util/-/util-0.26.13.tgz#bca07792761f231b427f8bce0aaeccac8882e707" + dependencies: + "@babel/runtime" "^7.0.0-beta.51" + "@types/bn.js" "^4.11.1" + "@types/deasync" "^0.1.0" + "@types/ip-regex" "^2.0.0" + "@types/xxhashjs" "^0.1.1" + bn.js "^4.11.8" + chalk "^2.4.1" + deasync "^0.1.13" + ip-regex "^2.1.0" + "@polkadot/wasm-bin@paritytech/polkadot-wasm-bin#5ab4fc6083501b26e5fd979b93c35e2f9010b324": version "20180601.165414.0" resolved "https://codeload.github.com/paritytech/polkadot-wasm-bin/tar.gz/5ab4fc6083501b26e5fd979b93c35e2f9010b324" @@ -1593,7 +1668,7 @@ binary-extensions@^1.0.0: version "1.11.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.11.0.tgz#46aa1751fb6a2f93ee5e689bb1087d4b14c6c205" -bindings@^1.2.1, bindings@~1.3.0: +bindings@^1.2.1: version "1.3.0" resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.3.0.tgz#b346f6ecf6a95f5a815c5839fc7cdb22502f1ed7" @@ -1607,12 +1682,6 @@ bip66@^1.1.3: dependencies: safe-buffer "^5.0.1" -bl@^1.0.0: - version "1.2.1" - resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.1.tgz#cac328f7bee45730d404b692203fcb590e172d5e" - dependencies: - readable-stream "^2.0.5" - blakejs@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/blakejs/-/blakejs-1.1.0.tgz#69df92ef953aa88ca51a32df6ab1c54a155fc7a5" @@ -2442,12 +2511,6 @@ decode-uri-component@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" -decompress-response@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3" - dependencies: - mimic-response "^1.0.0" - dedent@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c" @@ -2547,7 +2610,7 @@ detect-indent@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-5.0.0.tgz#3871cc0a6a002e8c3e5b3cf7f336264675f06b9d" -detect-libc@^1.0.2, detect-libc@^1.0.3: +detect-libc@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" @@ -2965,10 +3028,6 @@ expand-range@^1.8.1: dependencies: fill-range "^2.1.0" -expand-template@^1.0.2: - version "1.1.0" - resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-1.1.0.tgz#e09efba977bf98f9ee0ed25abd0c692e02aec3fc" - expect@^23.1.0: version "23.1.0" resolved "https://registry.yarnpkg.com/expect/-/expect-23.1.0.tgz#bfdfd57a2a20170d875999ee9787cc71f01c205f" @@ -3036,10 +3095,6 @@ fast-deep-equal@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" -fast-future@~1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/fast-future/-/fast-future-1.0.2.tgz#8435a9aaa02d79248d17d704e76259301d99280a" - fast-json-stable-stringify@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" @@ -3360,10 +3415,6 @@ gitconfiglocal@^1.0.0: dependencies: ini "^1.3.2" -github-from-package@0.0.0: - version "0.0.0" - resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" - glob-base@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/glob-base/-/glob-base-0.3.0.tgz#dbb164f6221b1c0b1ccf82aea328b497df0ea3c4" @@ -4835,16 +4886,6 @@ level-ws@0.1.0: readable-stream "^2.2.8" xtend "^4.0.0" -leveldown@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/leveldown/-/leveldown-4.0.1.tgz#7bc3df93c9fa574feb39ce45a0c4073aa948cfef" - dependencies: - abstract-leveldown "~5.0.0" - bindings "~1.3.0" - fast-future "~1.0.2" - nan "~2.10.0" - prebuild-install "^4.0.0" - levelup@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/levelup/-/levelup-3.0.1.tgz#07794639fd0af185089130aaea09d03023637b8d" @@ -5506,10 +5547,6 @@ mimic-fn@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.1.0.tgz#e667783d92e89dbd342818b5230b9d62a672ad18" -mimic-response@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.0.tgz#df3d3652a73fded6b9b0b24146e6fd052353458e" - minimalistic-assert@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/minimalistic-assert/-/minimalistic-assert-1.0.0.tgz#702be2dda6b37f4836bcb3f5db56641b64a1d3d3" @@ -5698,7 +5735,7 @@ nan@^2.0.7, nan@^2.2.1, nan@^2.3.0, nan@^2.3.3: version "2.8.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a" -nan@^2.9.2, nan@~2.10.0: +nan@^2.9.2: version "2.10.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f" @@ -5739,12 +5776,6 @@ negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" -node-abi@^2.2.0: - version "2.4.1" - resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.4.1.tgz#7628c4d4ec4e9cd3764ceb3652f36b2e7f8d4923" - dependencies: - semver "^5.4.1" - node-fetch@^1.0.1: version "1.7.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" @@ -5818,10 +5849,6 @@ noms@0.0.0: inherits "^2.0.1" readable-stream "~1.0.31" -noop-logger@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2" - nopt@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.1.tgz#d0d4685afd5415193c8c7505602d0d17cd64474d" @@ -5861,7 +5888,7 @@ npm-run-path@^2.0.0: dependencies: path-key "^2.0.0" -npmlog@^4.0.1, npmlog@^4.0.2, npmlog@^4.1.2: +npmlog@^4.0.2, npmlog@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" dependencies: @@ -5982,7 +6009,7 @@ options@>=0.0.5: version "0.0.6" resolved "https://registry.yarnpkg.com/options/-/options-0.0.6.tgz#ec22d312806bb53e731773e7cdaefcf1c643128f" -os-homedir@^1.0.0, os-homedir@^1.0.1: +os-homedir@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" @@ -6235,26 +6262,6 @@ posix-character-classes@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" -prebuild-install@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-4.0.0.tgz#206ce8106ce5efa4b6cf062fc8a0a7d93c17f3a8" - dependencies: - detect-libc "^1.0.3" - expand-template "^1.0.2" - github-from-package "0.0.0" - minimist "^1.2.0" - mkdirp "^0.5.1" - node-abi "^2.2.0" - noop-logger "^0.1.1" - npmlog "^4.0.1" - os-homedir "^1.0.1" - pump "^2.0.1" - rc "^1.1.6" - simple-get "^2.7.0" - tar-fs "^1.13.0" - tunnel-agent "^0.6.0" - which-pm-runs "^1.0.0" - prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" @@ -6391,20 +6398,6 @@ pull-ws@^3.3.1: safe-buffer "^5.1.1" ws "^1.1.0" -pump@^1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/pump/-/pump-1.0.3.tgz#5dfe8311c33bbf6fc18261f9f34702c47c08a954" - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - -pump@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/pump/-/pump-2.0.1.tgz#12399add6e4cf7526d973cbc8b5ce2e2908b3909" - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - pump@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" @@ -7033,18 +7026,6 @@ signed-varint@^2.0.1: dependencies: varint "~5.0.0" -simple-concat@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.0.tgz#7344cbb8b6e26fb27d66b2fc86f9f6d5997521c6" - -simple-get@^2.7.0: - version "2.8.1" - resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-2.8.1.tgz#0e22e91d4575d87620620bc91308d57a77f44b5d" - dependencies: - decompress-response "^3.3.0" - once "^1.3.1" - simple-concat "^1.0.0" - slash@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55" @@ -7374,15 +7355,6 @@ table@4.0.2: slice-ansi "1.0.0" string-width "^2.1.1" -tar-fs@^1.13.0: - version "1.16.0" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-1.16.0.tgz#e877a25acbcc51d8c790da1c57c9cf439817b896" - dependencies: - chownr "^1.0.1" - mkdirp "^0.5.1" - pump "^1.0.0" - tar-stream "^1.1.2" - tar-pack@^3.4.0: version "3.4.1" resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.1.tgz#e1dbc03a9b9d3ba07e896ad027317eb679a10a1f" @@ -7396,15 +7368,6 @@ tar-pack@^3.4.0: tar "^2.2.1" uid-number "^0.0.6" -tar-stream@^1.1.2: - version "1.5.5" - resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.5.5.tgz#5cad84779f45c83b1f2508d96b09d88c7218af55" - dependencies: - bl "^1.0.0" - end-of-stream "^1.0.0" - readable-stream "^2.0.0" - xtend "^4.0.0" - tar@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1" @@ -7931,10 +7894,6 @@ which-module@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" -which-pm-runs@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb" - which@^1.2.12, which@^1.2.9, which@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" From 16bcdb1b905d0edecfef2c57c914750935109e5e Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Tue, 3 Jul 2018 23:08:24 +0200 Subject: [PATCH 4/9] Test interface for Node --experimental-worker --- packages/client-db/package.json | 4 +- packages/client-db/src/Hash.ts | 25 +++++ packages/client-db/src/Memory.spec.js | 91 +++++++++++++++++++ packages/client-db/src/Memory.ts | 57 ++++++++++++ packages/client-db/src/syncify/commands.ts | 13 +++ packages/client-db/src/syncify/test-main.ts | 34 +++++++ packages/client-db/src/syncify/test-worker.js | 46 ++++++++++ packages/client-db/src/syncify/test.spec.js | 30 ++++++ packages/client-db/src/syncify/types.d.ts | 8 ++ .../client-db/src/syncify/worker_threads.d.ts | 28 ++++++ packages/client-db/src/types.d.ts | 13 +++ 11 files changed, 347 insertions(+), 2 deletions(-) create mode 100644 packages/client-db/src/Hash.ts create mode 100644 packages/client-db/src/Memory.spec.js create mode 100644 packages/client-db/src/Memory.ts create mode 100644 packages/client-db/src/syncify/commands.ts create mode 100644 packages/client-db/src/syncify/test-main.ts create mode 100644 packages/client-db/src/syncify/test-worker.js create mode 100644 packages/client-db/src/syncify/test.spec.js create mode 100644 packages/client-db/src/syncify/types.d.ts create mode 100644 packages/client-db/src/syncify/worker_threads.d.ts diff --git a/packages/client-db/package.json b/packages/client-db/package.json index a5b620f9..d515f9ca 100644 --- a/packages/client-db/package.json +++ b/packages/client-db/package.json @@ -35,8 +35,8 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", - "@polkadot/util": "^0.26.10", - "@polkadot/util-triedb": "^0.26.10", + "@polkadot/trie-db": "^0.26.13", + "@polkadot/util": "^0.26.13", "@types/mkdirp": "^0.5.2", "mkdirp": "^0.5.1" }, diff --git a/packages/client-db/src/Hash.ts b/packages/client-db/src/Hash.ts new file mode 100644 index 00000000..bfa75673 --- /dev/null +++ b/packages/client-db/src/Hash.ts @@ -0,0 +1,25 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { BaseDb } from './types'; + +type StorageHash = { + [index: string]: Uint8Array +}; + +export default class HashDb implements BaseDb { + private storage: StorageHash = {}; + + del (key: Uint8Array): void { + delete this.storage[key.toString()]; + } + + get (key: Uint8Array): Uint8Array | null { + return this.storage[key.toString()] || null; + } + + put (key: Uint8Array, value: Uint8Array): void { + this.storage[key.toString()] = value; + } +} diff --git a/packages/client-db/src/Memory.spec.js b/packages/client-db/src/Memory.spec.js new file mode 100644 index 00000000..aba6e8dd --- /dev/null +++ b/packages/client-db/src/Memory.spec.js @@ -0,0 +1,91 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import toU8a from '@polkadot/util/u8a/toU8a'; + +import MemoryDb from './Memory'; + +const EMPTY_ROOT = toU8a('0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421'); +const HELLO_ROOT = toU8a('0x0a915659b88f80bfa200570bd2767a6ab8cb0a4e44fd240cc5ed7a27728c4531'); +const FOO_ROOT = toU8a('0xed4e32371288ee83be74f78fb492f01261b7c3dc9c422581bb705c4376492dc6'); + +describe('MemoryDb', () => { + let memory = new MemoryDb(); + + it('starts with the default root', () => { + expect( + memory.trieRoot() + ).toEqual(EMPTY_ROOT); + }); + + it('has the correct root after a single insertion', () => { + memory.put(toU8a('hello'), toU8a('world')); + + expect( + memory.trieRoot() + ).toEqual( + HELLO_ROOT + ); + }); + + it('has the correct root after a deletion', () => { + memory.del(toU8a('hello')); + + expect( + memory.trieRoot() + ).toEqual( + EMPTY_ROOT + ); + }); + + it('has the correct root after a re-insertion', () => { + memory.put(toU8a('hello'), toU8a('world')); + + expect( + memory.trieRoot() + ).toEqual( + HELLO_ROOT + ); + }); + + it('has the correct root after a second insertion', () => { + memory.put(toU8a('foo'), toU8a('bar')); + + expect( + memory.trieRoot() + ).toEqual( + FOO_ROOT + ); + }); + + it('has the correct root after a re-deletion', () => { + memory.del(toU8a('hello')); + + expect( + memory.trieRoot() + ).toEqual( + toU8a('0x99650c730bbb99f6f58ce8b09bca2a8d90b36ac662e71bf81ec401ed23d199fb') + ); + }); + + it('has the correct root after a re-re-insertion', () => { + memory.put(toU8a('hello'), toU8a('world')); + + expect( + memory.trieRoot() + ).toEqual( + FOO_ROOT + ); + }); + + it('has the correct root after an update', () => { + memory.put(toU8a('hello'), toU8a('mars')); + + expect( + memory.trieRoot() + ).toEqual( + toU8a('0x116267e482ab94eb8824af425ea7f20ee84d10a8fa735f26204d80000e2a907e') + ); + }); +}); diff --git a/packages/client-db/src/Memory.ts b/packages/client-db/src/Memory.ts new file mode 100644 index 00000000..e31d1cd1 --- /dev/null +++ b/packages/client-db/src/Memory.ts @@ -0,0 +1,57 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { TrieDb } from './types'; + +import memdown from 'memdown'; +import Trie from '@polkadot/trie-db/index'; +import syncify from '@polkadot/util/syncify'; + +export default class MemoryDb implements TrieDb { + trie: Trie; + + constructor (root?: Uint8Array) { + this.trie = new Trie(memdown(), root); + } + + checkpoint (): void { + return syncify( + this.trie.checkpoint() + ); + } + + commit (): void { + syncify( + this.trie.commit() + ); + } + + del (key: Uint8Array): void { + syncify( + this.trie.del(key) + ); + } + + get (key: Uint8Array): Uint8Array | null { + return syncify( + this.trie.get(key) + ); + } + + revert (): void { + return syncify( + this.trie.revert() + ); + } + + put (key: Uint8Array, value: Uint8Array): void { + return syncify( + this.trie.put(key, value) + ); + } + + trieRoot (): Uint8Array { + return this.trie.root; + } +} diff --git a/packages/client-db/src/syncify/commands.ts b/packages/client-db/src/syncify/commands.ts new file mode 100644 index 00000000..4d0f7fcb --- /dev/null +++ b/packages/client-db/src/syncify/commands.ts @@ -0,0 +1,13 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +const START = 0x00; +const END = 0x0f; +const ERROR = 0xff; + +export default { + END, + ERROR, + START +}; diff --git a/packages/client-db/src/syncify/test-main.ts b/packages/client-db/src/syncify/test-main.ts new file mode 100644 index 00000000..62119f44 --- /dev/null +++ b/packages/client-db/src/syncify/test-main.ts @@ -0,0 +1,34 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import path from 'path'; +import { Worker } from 'worker_threads'; +import commands from './commands'; + +export default function main () { + const child = new Worker(path.join(__dirname, './test-worker.js')); + + const sendMessage = (type: string): number => { + const state = new Int32Array(new SharedArrayBuffer(8)); + + state[0] = commands.START; + child.postMessage({ + state, + type + }); + + Atomics.wait(state, 0, commands.START); + + return state[0]; + }; + + return { + exit: (): void => + child.terminate(), + callTimeout: (): number => + sendMessage('timeout'), + callError: (): number => + sendMessage('error') + }; +} diff --git a/packages/client-db/src/syncify/test-worker.js b/packages/client-db/src/syncify/test-worker.js new file mode 100644 index 00000000..6a1f2abd --- /dev/null +++ b/packages/client-db/src/syncify/test-worker.js @@ -0,0 +1,46 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +// import { Message } from './types'; + +// import { parentPort } from 'worker_threads'; +const { parentPort } = require('worker_threads'); + +// import commands from './commands'; +// Dangit, also doesn't reolve .ts here... erk :( +// const commands = require('./commands').default; +const commands = { + START: 0x00, + END: 0x0f, + ERROR: 0xff +}; + +// function setState (state: Int32Array, command: number): void { +function setState (state, command) { + state[0] = command; + Atomics.wake(state, 0, +Infinity); +} + +// function handleTimeout (state: Int32Array): void { +function handleTimeout (state) { + setTimeout(() => { + setState(state, commands.END); + }, 1000); +} + +// function error (state: Int32Array): void { +function error (state) { + setState(state, commands.ERROR); +} + +// parentPort.on('message', ({ state, type }: Message) => { +parentPort.on('message', ({ state, type }) => { + switch (type) { + case 'timeout': + return handleTimeout(state); + + default: + return error(state); + } +}); diff --git a/packages/client-db/src/syncify/test.spec.js b/packages/client-db/src/syncify/test.spec.js new file mode 100644 index 00000000..0049170c --- /dev/null +++ b/packages/client-db/src/syncify/test.spec.js @@ -0,0 +1,30 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import commands from './commands'; +import tester from './test-main'; + +describe('syncify (test-{main,worker}', () => { + let test; + + beforeEach(() => { + test = tester(); + }); + + afterEach(() => { + test.exit(); + }); + + it('works for callTimeout (setTimeout)', () => { + expect( + test.callTimeout() + ).toEqual(commands.END); + }); + + it('works for error (immediate)', () => { + expect( + test.callError() + ).toEqual(commands.ERROR); + }); +}); diff --git a/packages/client-db/src/syncify/types.d.ts b/packages/client-db/src/syncify/types.d.ts new file mode 100644 index 00000000..da164e5c --- /dev/null +++ b/packages/client-db/src/syncify/types.d.ts @@ -0,0 +1,8 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +export type Message = { + state: Int32Array, + type: string +} diff --git a/packages/client-db/src/syncify/worker_threads.d.ts b/packages/client-db/src/syncify/worker_threads.d.ts new file mode 100644 index 00000000..7c51416a --- /dev/null +++ b/packages/client-db/src/syncify/worker_threads.d.ts @@ -0,0 +1,28 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +/// + +declare namespace WorkerThreads { + interface ParentPort { + on (type: 'message', db: (message: any) => any): void + } + + class Worker { + constructor (path: string); + + postMessage (message: any): void; + terminate (): void; + } +} + +declare module 'worker_threads' { + const Worker: typeof WorkerThreads.Worker; + const parentPort: WorkerThreads.ParentPort; + + export { + Worker, + parentPort + }; +} diff --git a/packages/client-db/src/types.d.ts b/packages/client-db/src/types.d.ts index 181395d0..4c94bf56 100644 --- a/packages/client-db/src/types.d.ts +++ b/packages/client-db/src/types.d.ts @@ -10,3 +10,16 @@ export type DbConfig = { path: string, type: DbConfig$Type }; + +export interface BaseDb { + del: (key: Uint8Array) => void, + get: (key: Uint8Array) => Uint8Array | null, + put: (key: Uint8Array, value: Uint8Array) => void +} + +export interface TrieDb extends BaseDb { + checkpoint: () => void, + commit: () => void, + revert: () => void, + trieRoot: () => Uint8Array +} From c8f31880212280f25cb32ed18049294a1fa42a03 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Tue, 3 Jul 2018 23:10:54 +0200 Subject: [PATCH 5/9] Node ^10.5 (workers) --- .travis.yml | 2 +- package.json | 2 +- packages/client-db/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3fca8602..bd41b7f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: node_js node_js: - - "10" + - "10.5" cache: yarn: true directories: diff --git a/package.json b/package.json index 017ad3d3..e0aabbc7 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "version": "0.13.3", "private": true, "engines": { - "node": "^10.1.0", + "node": "^10.5.0", "yarn": "^1.3.2" }, "workspaces": [ diff --git a/packages/client-db/package.json b/packages/client-db/package.json index d515f9ca..a80cb9f4 100644 --- a/packages/client-db/package.json +++ b/packages/client-db/package.json @@ -4,7 +4,7 @@ "description": "Database interface layer for network client", "main": "index.js", "engines": { - "node": ">=8.0" + "node": ">=10.5" }, "publishConfig": { "access": "public", From adf854bfec720b0a38df1bc994b54d9b76551431 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Wed, 4 Jul 2018 15:41:33 +0200 Subject: [PATCH 6/9] ... and it actually syncs using trie-db --- package.json | 8 +- packages/client-chains/package.json | 14 +- .../src/chains/chain-dev.spec.js | 16 +- packages/client-chains/src/genesis/index.ts | 4 + packages/client-chains/src/genesis/state.ts | 4 +- packages/client-chains/src/index.spec.js | 20 ++- packages/client-chains/src/state.ts | 4 +- packages/client-db-chain/package.json | 10 +- packages/client-db/package.json | 4 +- packages/client-db/src/Memory.ts | 55 +------ packages/client-db/src/Overlay.ts | 78 ++++++++++ .../src/{syncify => Sync}/commands.ts | 6 + .../{Memory.spec.js => Sync/index.spec.js} | 10 +- packages/client-db/src/Sync/index.ts | 143 ++++++++++++++++++ .../src/{syncify => Sync}/types.d.ts | 7 +- packages/client-db/src/Sync/worker.js | 141 +++++++++++++++++ .../src/{syncify => Sync}/worker_threads.d.ts | 12 +- packages/client-db/src/syncify/test-main.ts | 34 ----- packages/client-db/src/syncify/test-worker.js | 46 ------ packages/client-db/src/syncify/test.spec.js | 30 ---- packages/client-db/src/types.d.ts | 3 +- packages/client-p2p-messages/package.json | 6 +- packages/client-p2p/package.json | 4 +- packages/client-rpc/package.json | 2 +- packages/client-runtime/package.json | 4 +- packages/client-telemetry/package.json | 2 +- packages/client-wasm/package.json | 4 +- packages/client-wasm/src/create/exports.ts | 16 +- packages/client-wasm/src/create/index.ts | 14 +- .../src/executor/executeBlock.spec.js | 19 +-- .../src/executor/generateBlock.spec.js | 19 +-- .../src/executor/initialiseBlock.spec.js | 20 +-- packages/client/package.json | 6 +- yarn.lock | 116 +++++++------- 34 files changed, 559 insertions(+), 322 deletions(-) create mode 100644 packages/client-db/src/Overlay.ts rename packages/client-db/src/{syncify => Sync}/commands.ts (78%) rename packages/client-db/src/{Memory.spec.js => Sync/index.spec.js} (93%) create mode 100644 packages/client-db/src/Sync/index.ts rename packages/client-db/src/{syncify => Sync}/types.d.ts (56%) create mode 100644 packages/client-db/src/Sync/worker.js rename packages/client-db/src/{syncify => Sync}/worker_threads.d.ts (72%) delete mode 100644 packages/client-db/src/syncify/test-main.ts delete mode 100644 packages/client-db/src/syncify/test-worker.js delete mode 100644 packages/client-db/src/syncify/test.spec.js diff --git a/package.json b/package.json index e0aabbc7..7d665bff 100644 --- a/package.json +++ b/package.json @@ -13,10 +13,10 @@ "check": "tslint --project . && tsc --noEmit", "clean": "polkadot-dev-clean-build", "postinstall": "polkadot-dev-yarn-only", - "test": "jest --coverage", - "start": "node packages/client/scripts/polkadot.js --db-path ./tmp/start-1", - "start2": "node packages/client/scripts/polkadot.js --db-path ./tmp/start-2 --p2p-nodes /ip4/127.0.0.1/tcp/39933/ipfs/QmfUiXCYtrKotHgDbP4Kc74NUi2LxckEiAdkK1SMSosLaz --p2p-port 39934 --rpc-port 9934", - "start-rust": "node packages/client/scripts/polkadot.js --db-path ./tmp/start-rust --p2p-nodes /ip4/127.0.0.1/tcp/30333/ipfs/QmX1nafodGEN1m2ueX26UvqLVXhvR2A1g8HdFBVD7GHgjJ --p2p-port 39935 --rpc-port 9935" + "test": "NODE_OPTIONS=--experimental-worker jest --coverage", + "start": "NODE_OPTIONS=--experimental-worker node packages/client/scripts/polkadot.js --db-path ./tmp/start-1", + "start2": "NODE_OPTIONS=--experimental-worker node packages/client/scripts/polkadot.js --db-path ./tmp/start-2 --p2p-nodes /ip4/127.0.0.1/tcp/39933/ipfs/QmfUiXCYtrKotHgDbP4Kc74NUi2LxckEiAdkK1SMSosLaz --p2p-port 39934 --rpc-port 9934", + "start-rust": "NODE_OPTIONS=--experimental-worker node packages/client/scripts/polkadot.js --db-path ./tmp/start-rust --p2p-nodes /ip4/127.0.0.1/tcp/30333/ipfs/QmRZB66LjCH9MSMLsTZjkQGBo7sGmqi6PfRyV5dDL9wPyr --p2p-port 39935 --rpc-port 9935" }, "devDependencies": { "@polkadot/dev": "^0.20.9", diff --git a/packages/client-chains/package.json b/packages/client-chains/package.json index 1fe8e33e..7bcd0860 100644 --- a/packages/client-chains/package.json +++ b/packages/client-chains/package.json @@ -37,16 +37,16 @@ "@polkadot/client-db-chain": "^0.13.3", "@polkadot/client-runtime": "^0.13.3", "@polkadot/client-wasm": "^0.13.3", - "@polkadot/storage": "^0.26.13", - "@polkadot/util": "^0.26.13", - "@polkadot/util-crypto": "^0.26.13", - "@polkadot/util-keyring": "^0.26.13" + "@polkadot/storage": "^0.26.14", + "@polkadot/util": "^0.26.14", + "@polkadot/util-crypto": "^0.26.14", + "@polkadot/util-keyring": "^0.26.14" }, "devDependencies": { "@polkadot/client": "^0.13.3", "@polkadot/client-db": "^0.13.3", - "@polkadot/extrinsics": "^0.26.13", - "@polkadot/primitives": "^0.26.13", - "@polkadot/storage": "^0.26.13" + "@polkadot/extrinsics": "^0.26.14", + "@polkadot/primitives": "^0.26.14", + "@polkadot/storage": "^0.26.14" } } diff --git a/packages/client-chains/src/chains/chain-dev.spec.js b/packages/client-chains/src/chains/chain-dev.spec.js index c712e579..7586e777 100644 --- a/packages/client-chains/src/chains/chain-dev.spec.js +++ b/packages/client-chains/src/chains/chain-dev.spec.js @@ -2,20 +2,16 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. +import toU8a from '@polkadot/util/u8a/toU8a'; import HashDb from '@polkadot/client-db/Hash'; import MemoryDb from '@polkadot/client-db/Memory'; import init from '../index'; describe('genesis', () => { - let genesis; - - beforeEach(() => { - const stateDb = new MemoryDb(); - const blockDb = new HashDb(); - - genesis = init({ chain: 'dev' }, stateDb, blockDb).genesis; - }); + const blockDb = new HashDb(); + const stateDb = new MemoryDb(); + const genesis = init({ chain: 'dev' }, stateDb, blockDb).genesis; it('creates a correct genesis block (stateRoot)', () => { expect( @@ -46,4 +42,8 @@ describe('genesis', () => { ]) ); }); + + it('terminates', () => { + return stateDb.terminate(); + }); }); diff --git a/packages/client-chains/src/genesis/index.ts b/packages/client-chains/src/genesis/index.ts index 9794c514..36ed572c 100644 --- a/packages/client-chains/src/genesis/index.ts +++ b/packages/client-chains/src/genesis/index.ts @@ -16,5 +16,9 @@ export default function initGenesis (self: ChainState, initialState: ChainGenesi self.blockDb.bestNumber.set(0); self.blockDb.block.set(genesis.block, genesis.headerHash); + // self.stateDb.db.checkpoint(); + // self.stateDb.system.blockHashAt.set(genesis.headerHash, 0); + // self.stateDb.db.commit(); + return genesis; } diff --git a/packages/client-chains/src/genesis/state.ts b/packages/client-chains/src/genesis/state.ts index 684e7d86..83bf0f98 100644 --- a/packages/client-chains/src/genesis/state.ts +++ b/packages/client-chains/src/genesis/state.ts @@ -7,11 +7,11 @@ import { ChainState, ChainGenesisState } from '../types'; import hexToU8a from '@polkadot/util/hex/toU8a'; export default function genesisState ({ stateDb: { db } }: ChainState, initial: ChainGenesisState): void { - // db.checkpoint(); + db.checkpoint(); Object.keys(initial).forEach((key) => db.put(hexToU8a(key), hexToU8a(initial[key])) ); - // db.commit(); + db.commit(); } diff --git a/packages/client-chains/src/index.spec.js b/packages/client-chains/src/index.spec.js index 001631ea..9a820b7e 100644 --- a/packages/client-chains/src/index.spec.js +++ b/packages/client-chains/src/index.spec.js @@ -8,17 +8,11 @@ import MemoryDb from '@polkadot/client-db/Memory'; import createChain from './index'; describe('client-chains', () => { - let config; - let blockDb; - let stateDb; - - beforeEach(() => { - config = { - chain: 'dev' - }; - blockDb = new HashDb(); - stateDb = new MemoryDb(); - }); + const config = { + chain: 'dev' + }; + const blockDb = new HashDb(); + const stateDb = new MemoryDb(); it('instantiates a known chain', () => { expect( @@ -33,4 +27,8 @@ describe('client-chains', () => { () => createChain(config, stateDb, blockDb) ).toThrow(/Unable to find builtin chain/); }); + + it('terminates', () => { + return stateDb.terminate(); + }); }); diff --git a/packages/client-chains/src/state.ts b/packages/client-chains/src/state.ts index 09e51726..fc450880 100644 --- a/packages/client-chains/src/state.ts +++ b/packages/client-chains/src/state.ts @@ -12,10 +12,10 @@ import createRuntime from '@polkadot/client-runtime/index'; import logger from '@polkadot/util/logger'; export default function state (config: Config, baseStateDb: TrieDb, baseBlockDb: BaseDb): ChainState { - const l = logger(`chain-${config.chain}`); + const l = logger(`chain/${config.chain}`); const runtime = createRuntime(baseStateDb); const blockDb = createBlockDb(baseBlockDb); - const stateDb = createStateDb(runtime.environment.db); + const stateDb = createStateDb(baseStateDb); return { blockDb, diff --git a/packages/client-db-chain/package.json b/packages/client-db-chain/package.json index c6e848ff..1c3a2808 100644 --- a/packages/client-db-chain/package.json +++ b/packages/client-db-chain/package.json @@ -35,12 +35,12 @@ "dependencies": { "@babel/runtime": "^7.0.0-beta.51", "@polkadot/client-db": "^0.13.3", - "@polkadot/storage": "^0.26.13", - "@polkadot/trie-hash": "^0.26.13", - "@polkadot/util": "^0.26.13", - "@polkadot/util-crypto": "^0.26.13" + "@polkadot/storage": "^0.26.14", + "@polkadot/trie-hash": "^0.26.14", + "@polkadot/util": "^0.26.14", + "@polkadot/util-crypto": "^0.26.14" }, "devDependencies": { - "@polkadot/storage": "^0.26.13" + "@polkadot/storage": "^0.26.14" } } diff --git a/packages/client-db/package.json b/packages/client-db/package.json index a80cb9f4..1024fdb9 100644 --- a/packages/client-db/package.json +++ b/packages/client-db/package.json @@ -35,8 +35,8 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", - "@polkadot/trie-db": "^0.26.13", - "@polkadot/util": "^0.26.13", + "@polkadot/trie-db": "^0.26.14", + "@polkadot/util": "^0.26.14", "@types/mkdirp": "^0.5.2", "mkdirp": "^0.5.1" }, diff --git a/packages/client-db/src/Memory.ts b/packages/client-db/src/Memory.ts index e31d1cd1..563682c2 100644 --- a/packages/client-db/src/Memory.ts +++ b/packages/client-db/src/Memory.ts @@ -2,56 +2,11 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. -import { TrieDb } from './types'; +import OverlayDb from './Overlay'; +import SyncDb from './Sync'; -import memdown from 'memdown'; -import Trie from '@polkadot/trie-db/index'; -import syncify from '@polkadot/util/syncify'; - -export default class MemoryDb implements TrieDb { - trie: Trie; - - constructor (root?: Uint8Array) { - this.trie = new Trie(memdown(), root); - } - - checkpoint (): void { - return syncify( - this.trie.checkpoint() - ); - } - - commit (): void { - syncify( - this.trie.commit() - ); - } - - del (key: Uint8Array): void { - syncify( - this.trie.del(key) - ); - } - - get (key: Uint8Array): Uint8Array | null { - return syncify( - this.trie.get(key) - ); - } - - revert (): void { - return syncify( - this.trie.revert() - ); - } - - put (key: Uint8Array, value: Uint8Array): void { - return syncify( - this.trie.put(key, value) - ); - } - - trieRoot (): Uint8Array { - return this.trie.root; +export default class MemoryDb extends OverlayDb { + constructor () { + super(new SyncDb()); } } diff --git a/packages/client-db/src/Overlay.ts b/packages/client-db/src/Overlay.ts new file mode 100644 index 00000000..0ff6a458 --- /dev/null +++ b/packages/client-db/src/Overlay.ts @@ -0,0 +1,78 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { TrieDb } from './types'; + +import isUndefined from '@polkadot/util/is/undefined'; +import logger from '@polkadot/util/logger'; +import u8aToHex from '@polkadot/util/u8a/toHex'; + +type Cache = { + [index: string]: Uint8Array | null +}; + +const l = logger('db/overlay'); + +export default class OverlayDb implements TrieDb { + private cache: Cache; + private wrapped: TrieDb; + + constructor (wrapped: TrieDb) { + this.cache = {}; + this.wrapped = wrapped; + } + + checkpoint () { + l.debug(() => ['checkpoint at', u8aToHex(this.trieRoot())]); + + this.wrapped.checkpoint(); + } + + commit () { + this.wrapped.commit(); + + l.debug(() => ['committed at', u8aToHex(this.trieRoot())]); + } + + revert () { + this.cache = {}; + this.wrapped.revert(); + + l.debug(() => ['reverted to', u8aToHex(this.trieRoot())]); + } + + del (key: Uint8Array): void { + this.cache[key.toString()] = null; + + return this.wrapped.del(key); + } + + get (key: Uint8Array): Uint8Array | null { + const keyStr = key.toString(); + + if (!isUndefined(this.cache[keyStr])) { + return this.cache[keyStr]; + } + + const result = this.wrapped.get(key); + + this.cache[keyStr] = result; + + return result; + } + + put (key: Uint8Array, value: Uint8Array): void { + this.cache[key.toString()] = value.slice(); + + return this.wrapped.put(key, value); + } + + trieRoot (): Uint8Array { + return this.wrapped.trieRoot(); + } + + terminate () { + return this.wrapped.terminate(); + } +} diff --git a/packages/client-db/src/syncify/commands.ts b/packages/client-db/src/Sync/commands.ts similarity index 78% rename from packages/client-db/src/syncify/commands.ts rename to packages/client-db/src/Sync/commands.ts index 4d0f7fcb..5918e009 100644 --- a/packages/client-db/src/syncify/commands.ts +++ b/packages/client-db/src/Sync/commands.ts @@ -3,11 +3,17 @@ // of the ISC license. See the LICENSE file for details. const START = 0x00; +const SIZE = 0x01; +const FILL = 0x02; +const READ = 0x03; const END = 0x0f; const ERROR = 0xff; export default { END, ERROR, + FILL, + READ, + SIZE, START }; diff --git a/packages/client-db/src/Memory.spec.js b/packages/client-db/src/Sync/index.spec.js similarity index 93% rename from packages/client-db/src/Memory.spec.js rename to packages/client-db/src/Sync/index.spec.js index aba6e8dd..d6a918d8 100644 --- a/packages/client-db/src/Memory.spec.js +++ b/packages/client-db/src/Sync/index.spec.js @@ -4,14 +4,14 @@ import toU8a from '@polkadot/util/u8a/toU8a'; -import MemoryDb from './Memory'; +import SyncDb from './index'; const EMPTY_ROOT = toU8a('0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421'); const HELLO_ROOT = toU8a('0x0a915659b88f80bfa200570bd2767a6ab8cb0a4e44fd240cc5ed7a27728c4531'); const FOO_ROOT = toU8a('0xed4e32371288ee83be74f78fb492f01261b7c3dc9c422581bb705c4376492dc6'); -describe('MemoryDb', () => { - let memory = new MemoryDb(); +describe('SyncDb', () => { + let memory = new SyncDb(); it('starts with the default root', () => { expect( @@ -88,4 +88,8 @@ describe('MemoryDb', () => { toU8a('0x116267e482ab94eb8824af425ea7f20ee84d10a8fa735f26204d80000e2a907e') ); }); + + it('terminates', () => { + return memory.terminate(); + }); }); diff --git a/packages/client-db/src/Sync/index.ts b/packages/client-db/src/Sync/index.ts new file mode 100644 index 00000000..c512554f --- /dev/null +++ b/packages/client-db/src/Sync/index.ts @@ -0,0 +1,143 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +import { TrieDb } from '../types'; + +import path from 'path'; +import { Worker } from 'worker_threads'; +import logger from '@polkadot/util/logger'; +import promisify from '@polkadot/util/promisify'; +import u8aToHex from '@polkadot/util/u8a/toHex'; + +import commands from './commands'; + +const l = logger('db/main'); + +const U32_MAX = 4294967295; +const returnable = ['get', 'root']; + +export default class SyncDb implements TrieDb { + private child: WorkerThreads.Worker; + + constructor () { + // NOTE With --experimental-modules (ESNext targets) this can become + // this.child = new Worker(new URL('./worker.js', import.meta.url).pathname); + this.child = new Worker(path.join(__dirname, './worker.js')); + } + + private _notifyFill (state: Int32Array): void { + state[0] = commands.FILL; + + Atomics.wake(state, 0, +Infinity); + Atomics.wait(state, 0, commands.FILL); + } + + private _sendMessage (type: string, key?: Uint8Array, value?: Uint8Array): Uint8Array | null { + const state = new Int32Array(new SharedArrayBuffer(8)); + let view: DataView | null = null; + let buffer: Uint8Array | null = null; + + const start = () => { + state[0] = commands.START; + this.child.postMessage({ + buffer, + key, + state, + value, + type + }); + + // console.error(type, 'wait', state[0]); + + Atomics.wait(state, 0, commands.START); + + // console.error(type, 'wake', state[0]); + }; + + if (!returnable.includes(type)) { + start(); + return null; + } + + const shared = new SharedArrayBuffer(4096); + + buffer = new Uint8Array(shared); + view = new DataView(shared); + + let result: Uint8Array | null = null; + let size = 0; + let offset = 0; + + start(); + + while (true) { + switch (state[0]) { + case commands.END: + // console.error(type, 'returning', result ? u8aToHex(result) : 'null'); + return result; + + case commands.ERROR: + return null; + + case commands.SIZE: + size = view.getUint32(0); + + if (size === U32_MAX) { + // console.error(type, 'returning not found'); + return null; + } + + result = new Uint8Array(size); + + this._notifyFill(state); + break; + + case commands.READ: + const available = Math.min((buffer as Uint8Array).length, size - offset); + + (result as Uint8Array).set(buffer.subarray(0, available), offset); + offset += available; + + this._notifyFill(state); + break; + + default: + l.error('Unknown worker state', state[0]); + return null; + } + } + } + + checkpoint () { + this._sendMessage('checkpoint'); + } + + commit () { + this._sendMessage('commit'); + } + + revert () { + this._sendMessage('revert'); + } + + del (key: Uint8Array): void { + this._sendMessage('del', key); + } + + get (key: Uint8Array): Uint8Array | null { + return this._sendMessage('get', key); + } + + put (key: Uint8Array, value: Uint8Array): void { + this._sendMessage('put', key, value); + } + + trieRoot (): Uint8Array { + return this._sendMessage('root') as Uint8Array; + } + + async terminate () { + return promisify(this.child, this.child.terminate); + } +} diff --git a/packages/client-db/src/syncify/types.d.ts b/packages/client-db/src/Sync/types.d.ts similarity index 56% rename from packages/client-db/src/syncify/types.d.ts rename to packages/client-db/src/Sync/types.d.ts index da164e5c..670cfa6e 100644 --- a/packages/client-db/src/syncify/types.d.ts +++ b/packages/client-db/src/Sync/types.d.ts @@ -2,7 +2,12 @@ // This software may be modified and distributed under the terms // of the ISC license. See the LICENSE file for details. +export type MessageType = 'checkpoint' | 'commit' | 'del' | 'get' | 'put' | 'revert' | 'root'; + export type Message = { + buffer: Uint8Array | null, + key: Uint8Array, state: Int32Array, - type: string + type: MessageType, + value: Uint8Array } diff --git a/packages/client-db/src/Sync/worker.js b/packages/client-db/src/Sync/worker.js new file mode 100644 index 00000000..55b8f4fb --- /dev/null +++ b/packages/client-db/src/Sync/worker.js @@ -0,0 +1,141 @@ +// Copyright 2017-2018 @polkadot/client-db authors & contributors +// This software may be modified and distributed under the terms +// of the ISC license. See the LICENSE file for details. + +// import { Message } from './types'; + +// import { parentPort } from 'worker_threads'; +const { parentPort } = require('worker_threads'); + +// import Trie from '@polkadot/trie-db'; +const Trie = require('@polkadot/trie-db').default; + +// import commands from './commands'; +// Dangit, also doesn't reolve .ts here... erk :( +// const commands = require('./commands').default; +const commands = { + START: 0x00, + SIZE: 0x01, + FILL: 0x02, + READ: 0x03, + END: 0x0f, + ERROR: 0xff +}; + +const U32_MAX = 4294967295; + +const trie = new Trie(); + +// function notify (state: Int32Array, command: number): void { +function notify (state, command) { + state[0] = command; + Atomics.wake(state, 0, +Infinity); +} + +function wait (state, command) { + Atomics.wait(state, 0, command); +} + +async function notifyResult (state, fn) { + try { + await fn(); + notify(state, commands.END); + } catch (error) { + notify(state, commands.ERROR); + } +} + +function returnValue (state, buffer, value) { + const view = new DataView(buffer.buffer); + + if (!value) { + view.setUint32(0, U32_MAX); + notify(state, commands.SIZE); + return; + } + + const size = value.length; + let offset = 0; + + view.setUint32(0, size); + notify(state, commands.SIZE); + wait(state, commands.SIZE); + + while (offset !== size) { + const available = Math.min(buffer.length, size - offset); + + buffer.set(value.subarray(offset, offset + available), 0); + offset += available; + + notify(state, commands.READ); + wait(state, commands.READ); + } + + notify(state, commands.END); +} + +function checkpoint (state) { + notifyResult(state, () => + trie.checkpoint() + ); +} + +function commit (state) { + notifyResult(state, () => + trie.commit() + ); +} + +function del (state, buffer, key) { + notifyResult(state, () => + trie.del(key) + ); +} + +async function get (state, buffer, key) { + let result; + + try { + result = await trie.get(key); + } catch (error) { + notify(state, commands.ERROR); + return; + } + + returnValue(state, buffer, result); +} + +async function put (state, buffer, key, value) { + notifyResult(state, () => + trie.put(key, value) + ); +} + +async function revert (state) { + notifyResult(state, () => + trie.revert() + ); +} + +function root (state, buffer) { + returnValue(state, buffer, trie.root); +} + +const functions = [ + checkpoint, commit, del, get, put, revert, root +].reduce((fns, fn) => { + fns[fn.name] = fn; + + return fns; +}, {}); + +// parentPort.on('message', ({ state, type }: Message) => { +parentPort.on('message', ({ buffer, key, state, type, value }) => { + const fn = functions[type]; + + if (fn) { + fn(state, buffer, key, value); + } else { + notify(state, commands.ERROR); + } +}); diff --git a/packages/client-db/src/syncify/worker_threads.d.ts b/packages/client-db/src/Sync/worker_threads.d.ts similarity index 72% rename from packages/client-db/src/syncify/worker_threads.d.ts rename to packages/client-db/src/Sync/worker_threads.d.ts index 7c51416a..0bb7513e 100644 --- a/packages/client-db/src/syncify/worker_threads.d.ts +++ b/packages/client-db/src/Sync/worker_threads.d.ts @@ -9,8 +9,18 @@ declare namespace WorkerThreads { on (type: 'message', db: (message: any) => any): void } + type WorkerOptions = { + eval?: boolean, + stderr?: boolean, + stdin?: boolean, + stdout?: boolean, + workerData?: any + } + class Worker { - constructor (path: string); + readonly threadId: number; + + constructor (path: string, options?: WorkerOptions); postMessage (message: any): void; terminate (): void; diff --git a/packages/client-db/src/syncify/test-main.ts b/packages/client-db/src/syncify/test-main.ts deleted file mode 100644 index 62119f44..00000000 --- a/packages/client-db/src/syncify/test-main.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-db authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import path from 'path'; -import { Worker } from 'worker_threads'; -import commands from './commands'; - -export default function main () { - const child = new Worker(path.join(__dirname, './test-worker.js')); - - const sendMessage = (type: string): number => { - const state = new Int32Array(new SharedArrayBuffer(8)); - - state[0] = commands.START; - child.postMessage({ - state, - type - }); - - Atomics.wait(state, 0, commands.START); - - return state[0]; - }; - - return { - exit: (): void => - child.terminate(), - callTimeout: (): number => - sendMessage('timeout'), - callError: (): number => - sendMessage('error') - }; -} diff --git a/packages/client-db/src/syncify/test-worker.js b/packages/client-db/src/syncify/test-worker.js deleted file mode 100644 index 6a1f2abd..00000000 --- a/packages/client-db/src/syncify/test-worker.js +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-db authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -// import { Message } from './types'; - -// import { parentPort } from 'worker_threads'; -const { parentPort } = require('worker_threads'); - -// import commands from './commands'; -// Dangit, also doesn't reolve .ts here... erk :( -// const commands = require('./commands').default; -const commands = { - START: 0x00, - END: 0x0f, - ERROR: 0xff -}; - -// function setState (state: Int32Array, command: number): void { -function setState (state, command) { - state[0] = command; - Atomics.wake(state, 0, +Infinity); -} - -// function handleTimeout (state: Int32Array): void { -function handleTimeout (state) { - setTimeout(() => { - setState(state, commands.END); - }, 1000); -} - -// function error (state: Int32Array): void { -function error (state) { - setState(state, commands.ERROR); -} - -// parentPort.on('message', ({ state, type }: Message) => { -parentPort.on('message', ({ state, type }) => { - switch (type) { - case 'timeout': - return handleTimeout(state); - - default: - return error(state); - } -}); diff --git a/packages/client-db/src/syncify/test.spec.js b/packages/client-db/src/syncify/test.spec.js deleted file mode 100644 index 0049170c..00000000 --- a/packages/client-db/src/syncify/test.spec.js +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017-2018 @polkadot/client-db authors & contributors -// This software may be modified and distributed under the terms -// of the ISC license. See the LICENSE file for details. - -import commands from './commands'; -import tester from './test-main'; - -describe('syncify (test-{main,worker}', () => { - let test; - - beforeEach(() => { - test = tester(); - }); - - afterEach(() => { - test.exit(); - }); - - it('works for callTimeout (setTimeout)', () => { - expect( - test.callTimeout() - ).toEqual(commands.END); - }); - - it('works for error (immediate)', () => { - expect( - test.callError() - ).toEqual(commands.ERROR); - }); -}); diff --git a/packages/client-db/src/types.d.ts b/packages/client-db/src/types.d.ts index 4c94bf56..b62dc093 100644 --- a/packages/client-db/src/types.d.ts +++ b/packages/client-db/src/types.d.ts @@ -21,5 +21,6 @@ export interface TrieDb extends BaseDb { checkpoint: () => void, commit: () => void, revert: () => void, - trieRoot: () => Uint8Array + trieRoot: () => Uint8Array, + terminate: () => Promise } diff --git a/packages/client-p2p-messages/package.json b/packages/client-p2p-messages/package.json index 62adbc33..9de19268 100644 --- a/packages/client-p2p-messages/package.json +++ b/packages/client-p2p-messages/package.json @@ -35,8 +35,8 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", - "@polkadot/jsonrpc": "^0.26.13", - "@polkadot/primitives": "^0.26.13", - "@polkadot/util": "^0.26.13" + "@polkadot/jsonrpc": "^0.26.14", + "@polkadot/primitives": "^0.26.14", + "@polkadot/util": "^0.26.14" } } diff --git a/packages/client-p2p/package.json b/packages/client-p2p/package.json index 54919ff5..c767ea9a 100644 --- a/packages/client-p2p/package.json +++ b/packages/client-p2p/package.json @@ -36,8 +36,8 @@ "dependencies": { "@babel/runtime": "^7.0.0-beta.51", "@polkadot/client-p2p-messages": "^0.13.3", - "@polkadot/primitives": "^0.26.13", - "@polkadot/util": "^0.26.13", + "@polkadot/primitives": "^0.26.14", + "@polkadot/util": "^0.26.14", "eventemitter3": "^2.0.3", "libp2p": "^0.22.0", "libp2p-kad-dht": "^0.10.0", diff --git a/packages/client-rpc/package.json b/packages/client-rpc/package.json index fe4ed352..1a138d20 100644 --- a/packages/client-rpc/package.json +++ b/packages/client-rpc/package.json @@ -39,7 +39,7 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", - "@polkadot/util": "^0.26.13", + "@polkadot/util": "^0.26.14", "@types/co-body": "^0.0.3", "@types/koa": "^2.0.46", "@types/koa-route": "^3.2.4", diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 3563ef45..1c347a5f 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -36,7 +36,7 @@ "dependencies": { "@babel/runtime": "^7.0.0-beta.51", "@polkadot/client-db": "^0.13.3", - "@polkadot/util": "^0.26.13", - "@polkadot/util-crypto": "^0.26.13" + "@polkadot/util": "^0.26.14", + "@polkadot/util-crypto": "^0.26.14" } } diff --git a/packages/client-telemetry/package.json b/packages/client-telemetry/package.json index ac2f2ec8..0b283664 100644 --- a/packages/client-telemetry/package.json +++ b/packages/client-telemetry/package.json @@ -35,7 +35,7 @@ }, "dependencies": { "@babel/runtime": "^7.0.0-beta.51", - "@polkadot/util": "^0.26.13", + "@polkadot/util": "^0.26.14", "websocket": "^1.0.26" } } diff --git a/packages/client-wasm/package.json b/packages/client-wasm/package.json index fc6c1531..72c3c811 100644 --- a/packages/client-wasm/package.json +++ b/packages/client-wasm/package.json @@ -36,8 +36,8 @@ "dependencies": { "@babel/runtime": "^7.0.0-beta.51", "@polkadot/client-runtime": "^0.13.3", - "@polkadot/primitives": "^0.26.13", - "@polkadot/util": "^0.26.13", + "@polkadot/primitives": "^0.26.14", + "@polkadot/util": "^0.26.14", "@polkadot/wasm-bin": "paritytech/polkadot-wasm-bin#5ab4fc6083501b26e5fd979b93c35e2f9010b324", "@types/webassembly-js-api": "^0.0.1", "yargs": "^11.0.0" diff --git a/packages/client-wasm/src/create/exports.ts b/packages/client-wasm/src/create/exports.ts index 2d26a650..318793d9 100644 --- a/packages/client-wasm/src/create/exports.ts +++ b/packages/client-wasm/src/create/exports.ts @@ -23,14 +23,14 @@ export default function createExports (bytecode: Uint8Array, imports?: WasmExtra const codeHash = xxhash(bytecode.subarray(0, 2048), 0); // NOTE compilation is quite resource intensive, here we bypass the actual Uint8Array -> Module compilation when we already have this module bytecode in our cache - if (!cache[codeHash] || forceCreate) { - const table = new WebAssembly.Table(DEFAULT_TABLE); - - cache[codeHash] = new WebAssembly.Instance( - new WebAssembly.Module(bytecode), - createImports(memory, table, imports) - ); - } + // if (!cache[codeHash] || forceCreate) { + const table = new WebAssembly.Table(DEFAULT_TABLE); + + cache[codeHash] = new WebAssembly.Instance( + new WebAssembly.Module(bytecode), + createImports(memory, table, imports) + ); + // } return cache[codeHash].exports; } diff --git a/packages/client-wasm/src/create/index.ts b/packages/client-wasm/src/create/index.ts index 58127d14..69867d3a 100644 --- a/packages/client-wasm/src/create/index.ts +++ b/packages/client-wasm/src/create/index.ts @@ -26,7 +26,7 @@ function instrument (name: string, elapsed: Array, fn: () => T): T { return result; } -let prevChain: WasmInstanceExports; +// let prevChain: WasmInstanceExports; let offset: number = 0; export default function wasm ({ config: { wasm: { heapSize = defaults.HEAP_SIZE_KB } }, l }: Options, runtime: RuntimeInterface, chainCode: Uint8Array, chainProxy: Uint8Array): WasmInstanceExports { @@ -37,15 +37,15 @@ export default function wasm ({ config: { wasm: { heapSize = defaults.HEAP_SIZE_ const chain = instrument('chain', elapsed, (): WasmInstanceExports => createExports(chainCode, { env }) ); - const isNewCode = chain !== prevChain; + // const isNewCode = chain !== prevChain; - if (isNewCode) { - offset = chain.memory.grow(Math.ceil(heapSize / 64)); - prevChain = chain; - } + // if (isNewCode) { + offset = chain.memory.grow(Math.ceil(heapSize / 64)); + // prevChain = chain; + // } const instance = instrument('proxy', elapsed, (): WasmInstanceExports => - createExports(chainProxy, { proxy: chain }, createMemory(0, 0), isNewCode) + createExports(chainProxy, { proxy: chain }, createMemory(0, 0)) // , isNewCode) ); runtime.environment.heap.setWasmMemory(chain.memory, offset * 64 * 1024); diff --git a/packages/client-wasm/src/executor/executeBlock.spec.js b/packages/client-wasm/src/executor/executeBlock.spec.js index 1e2b29a9..5b452ee5 100644 --- a/packages/client-wasm/src/executor/executeBlock.spec.js +++ b/packages/client-wasm/src/executor/executeBlock.spec.js @@ -11,20 +11,21 @@ describe('executeBlock', () => { const TEST = new Uint8Array([ 13, 85, 34, 171, 245, 12, 16, 73, 197, 106, 231, 219, 139, 87, 173, 252, 126, 2, 55, 165, 8, 237, 221, 136, 95, 19, 249, 60, 247, 193, 8, 201, 1, 0, 0, 0, 0, 0, 0, 0, 53, 191, 210, 221, 53, 61, 234, 160, 144, 39, 96, 152, 79, 190, 91, 192, 213, 233, 171, 197, 218, 227, 106, 28, 9, 97, 23, 209, 147, 202, 199, 90, 82, 250, 106, 206, 108, 205, 179, 210, 104, 223, 123, 158, 106, 186, 241, 242, 204, 147, 149, 97, 153, 160, 218, 130, 117, 134, 64, 169, 59, 137, 16, 146, 0, 0, 0, 0, 2, 0, 0, 0, 111, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 239, 129, 56, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]); - let chain; + const config = { + chain: 'dev', + wasm: {} + }; - beforeEach(() => { - const config = { - chain: 'dev', - wasm: {} - }; - - chain = init(config, new MemoryDb(), new HashDb()); - }); + const memory = new MemoryDb(); + const chain = init(config, memory, new HashDb()); it('executes an actual block', () => { expect( chain.executor.executeBlock(TEST) ).toEqual(true); }); + + it('terminates', () => { + return memory.terminate(); + }); }); diff --git a/packages/client-wasm/src/executor/generateBlock.spec.js b/packages/client-wasm/src/executor/generateBlock.spec.js index 5abb8922..346d1e41 100644 --- a/packages/client-wasm/src/executor/generateBlock.spec.js +++ b/packages/client-wasm/src/executor/generateBlock.spec.js @@ -32,16 +32,13 @@ const PARACHAIN = new Uint8Array([ ]); describe('generateBlock', () => { - let chain; + const config = { + chain: 'dev', + wasm: {} + }; - beforeEach(() => { - const config = { - chain: 'dev', - wasm: {} - }; - - chain = init(config, new MemoryDb(), new HashDb()); - }); + const stateDb = new MemoryDb(); + const chain = init(config, stateDb, new HashDb()); it('generates a basic block (empty)', () => { expect( @@ -94,4 +91,8 @@ describe('generateBlock', () => { ) ).not.toBeNull(); }); + + it('terminates', () => { + return stateDb.terminate(); + }); }); diff --git a/packages/client-wasm/src/executor/initialiseBlock.spec.js b/packages/client-wasm/src/executor/initialiseBlock.spec.js index 0b4156cf..288c6fb4 100644 --- a/packages/client-wasm/src/executor/initialiseBlock.spec.js +++ b/packages/client-wasm/src/executor/initialiseBlock.spec.js @@ -10,16 +10,12 @@ import MemoryDb from '@polkadot/client-db/Memory'; import init from '@polkadot/client-chains'; describe('initialiseBlock', () => { - let chain; - - beforeEach(() => { - const config = { - chain: 'dev', - wasm: {} - }; - - chain = init(config, new MemoryDb(), new HashDb()); - }); + const config = { + chain: 'dev', + wasm: {} + }; + const stateDb = new MemoryDb(); + const chain = init(config, stateDb, new HashDb()); it('initialises a block', () => { expect( @@ -33,4 +29,8 @@ describe('initialiseBlock', () => { ) ).toEqual(true); }); + + it('terminates', () => { + return stateDb.terminate(); + }); }); diff --git a/packages/client/package.json b/packages/client/package.json index 7ee59673..2908ce07 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -44,9 +44,9 @@ "@polkadot/client-rpc": "^0.13.3", "@polkadot/client-telemetry": "^0.13.3", "@polkadot/client-wasm": "^0.13.3", - "@polkadot/jsonrpc": "^0.26.13", - "@polkadot/primitives": "^0.26.13", - "@polkadot/util": "^0.26.13", + "@polkadot/jsonrpc": "^0.26.14", + "@polkadot/primitives": "^0.26.14", + "@polkadot/util": "^0.26.14", "@types/package-json": "^4.0.1", "@types/semver-compare": "^1.0.0", "@types/yargs": "^11.0.0", diff --git a/yarn.lock b/yarn.lock index c080d1d6..2f46bfca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -759,20 +759,20 @@ "@polkadot/primitives" "^0.26.10" "@polkadot/util" "^0.26.10" -"@polkadot/extrinsics@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/extrinsics/-/extrinsics-0.26.13.tgz#b6358536c0125a98e3b4659ba9771d13021dba73" +"@polkadot/extrinsics@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/extrinsics/-/extrinsics-0.26.14.tgz#a86dc06c89ba14869351eb28bd235c4a646ec1fb" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/params" "^0.26.13" - "@polkadot/primitives" "^0.26.13" - "@polkadot/util" "^0.26.13" + "@polkadot/params" "^0.26.14" + "@polkadot/primitives" "^0.26.14" + "@polkadot/util" "^0.26.14" -"@polkadot/jsonrpc@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/jsonrpc/-/jsonrpc-0.26.13.tgz#53709aee56834818422e8c0d1a869dc03528cce8" +"@polkadot/jsonrpc@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/jsonrpc/-/jsonrpc-0.26.14.tgz#0002475ca2340b0d1def9e0246b3a8728d3bce4f" dependencies: - "@polkadot/params" "^0.26.13" + "@polkadot/params" "^0.26.14" babel-runtime "^6.26.0" "@polkadot/params@^0.26.10": @@ -784,14 +784,14 @@ "@polkadot/primitives" "^0.26.10" "@polkadot/util" "^0.26.10" -"@polkadot/params@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/params/-/params-0.26.13.tgz#ae0fef6446c5ed2f7700d218060603e6a795b689" +"@polkadot/params@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/params/-/params-0.26.14.tgz#b0be6b54601602fdcfbf7c264b7ae369fad75185" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/extrinsics" "^0.26.13" - "@polkadot/primitives" "^0.26.13" - "@polkadot/util" "^0.26.13" + "@polkadot/extrinsics" "^0.26.14" + "@polkadot/primitives" "^0.26.14" + "@polkadot/util" "^0.26.14" "@polkadot/primitives@^0.26.10": version "0.26.10" @@ -801,13 +801,13 @@ "@polkadot/trie-hash" "^0.26.10" "@polkadot/util" "^0.26.10" -"@polkadot/primitives@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/primitives/-/primitives-0.26.13.tgz#592d1a92865b7dd0fe57ffa12755f068c33e16df" +"@polkadot/primitives@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/primitives/-/primitives-0.26.14.tgz#7f1959ad76154c995a550b4f5361bfde93139137" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/trie-hash" "^0.26.13" - "@polkadot/util" "^0.26.13" + "@polkadot/trie-hash" "^0.26.14" + "@polkadot/util" "^0.26.14" "@polkadot/storage@^0.26.10": version "0.26.10" @@ -820,26 +820,26 @@ "@polkadot/util-crypto" "^0.26.10" "@polkadot/util-keyring" "^0.26.10" -"@polkadot/storage@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/storage/-/storage-0.26.13.tgz#774be216cfc15ef345fc9dc35b78a376e32a5849" +"@polkadot/storage@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/storage/-/storage-0.26.14.tgz#7db4ef7f27393782ddaea04df1e48b64bc63a27b" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/params" "^0.26.13" - "@polkadot/primitives" "^0.26.13" - "@polkadot/util" "^0.26.13" - "@polkadot/util-crypto" "^0.26.13" - "@polkadot/util-keyring" "^0.26.13" + "@polkadot/params" "^0.26.14" + "@polkadot/primitives" "^0.26.14" + "@polkadot/util" "^0.26.14" + "@polkadot/util-crypto" "^0.26.14" + "@polkadot/util-keyring" "^0.26.14" -"@polkadot/trie-db@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/trie-db/-/trie-db-0.26.13.tgz#b3d6d3fc4cfd58db3ae9aea0c3df68ec414bb68d" +"@polkadot/trie-db@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/trie-db/-/trie-db-0.26.14.tgz#826944ceb652eb38de3e098ebd566fd48c2c335d" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/trie-hash" "^0.26.13" - "@polkadot/util" "^0.26.13" - "@polkadot/util-crypto" "^0.26.13" - "@polkadot/util-rlp" "^0.26.13" + "@polkadot/trie-hash" "^0.26.14" + "@polkadot/util" "^0.26.14" + "@polkadot/util-crypto" "^0.26.14" + "@polkadot/util-rlp" "^0.26.14" "@types/async" "^2.0.49" "@types/semaphore" "^1.1.0" async "^1.4.2" @@ -859,14 +859,14 @@ "@polkadot/util-crypto" "^0.26.10" "@polkadot/util-rlp" "^0.26.10" -"@polkadot/trie-hash@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/trie-hash/-/trie-hash-0.26.13.tgz#f47cfa4c9ede895342244e85076279a874e99451" +"@polkadot/trie-hash@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/trie-hash/-/trie-hash-0.26.14.tgz#cf4b001a4b89e366b37fb4ccebe7cc20fd653214" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/util" "^0.26.13" - "@polkadot/util-crypto" "^0.26.13" - "@polkadot/util-rlp" "^0.26.13" + "@polkadot/util" "^0.26.14" + "@polkadot/util-crypto" "^0.26.14" + "@polkadot/util-rlp" "^0.26.14" "@polkadot/ts@^0.1.8": version "0.1.8" @@ -883,12 +883,12 @@ tweetnacl "^1.0.0" xxhashjs "^0.2.2" -"@polkadot/util-crypto@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/util-crypto/-/util-crypto-0.26.13.tgz#b882c79fce24d50586687ef72a30dfa916650b23" +"@polkadot/util-crypto@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/util-crypto/-/util-crypto-0.26.14.tgz#fb57b4a5acbaf0605d68b01031467c7416dae641" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/util" "^0.26.13" + "@polkadot/util" "^0.26.14" blakejs "^1.1.0" js-sha3 "^0.7.0" tweetnacl "^1.0.0" @@ -904,13 +904,13 @@ "@types/bs58" "^3.0.30" bs58 "^4.0.1" -"@polkadot/util-keyring@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/util-keyring/-/util-keyring-0.26.13.tgz#b15e36eda74f5faa0bf923a071a18d20eba35c34" +"@polkadot/util-keyring@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/util-keyring/-/util-keyring-0.26.14.tgz#25f5b62e204a63edabf6069479e4c8b4eaafe2a9" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/util" "^0.26.13" - "@polkadot/util-crypto" "^0.26.13" + "@polkadot/util" "^0.26.14" + "@polkadot/util-crypto" "^0.26.14" "@types/bs58" "^3.0.30" bs58 "^4.0.1" @@ -921,12 +921,12 @@ "@babel/runtime" "^7.0.0-beta.51" "@polkadot/util" "^0.26.10" -"@polkadot/util-rlp@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/util-rlp/-/util-rlp-0.26.13.tgz#0f346ba5b1b41e2232d06523a4b4a0addeed4026" +"@polkadot/util-rlp@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/util-rlp/-/util-rlp-0.26.14.tgz#ca528e5afbf1b8e538d98e0eadbb13348f514c48" dependencies: "@babel/runtime" "^7.0.0-beta.51" - "@polkadot/util" "^0.26.13" + "@polkadot/util" "^0.26.14" "@polkadot/util@^0.26.10": version "0.26.10" @@ -942,9 +942,9 @@ deasync "^0.1.13" ip-regex "^2.1.0" -"@polkadot/util@^0.26.13": - version "0.26.13" - resolved "https://registry.yarnpkg.com/@polkadot/util/-/util-0.26.13.tgz#bca07792761f231b427f8bce0aaeccac8882e707" +"@polkadot/util@^0.26.14": + version "0.26.14" + resolved "https://registry.yarnpkg.com/@polkadot/util/-/util-0.26.14.tgz#2f12c75daddb5ed6e22e0ebe6044d1d420440b22" dependencies: "@babel/runtime" "^7.0.0-beta.51" "@types/bn.js" "^4.11.1" From 5566f05ceb92828fa8d84f50793c14ecf231c764 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Wed, 4 Jul 2018 15:54:42 +0200 Subject: [PATCH 7/9] Re-enable Wasm caching --- packages/client-wasm/src/create/exports.ts | 16 ++++++++-------- packages/client-wasm/src/create/index.ts | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/client-wasm/src/create/exports.ts b/packages/client-wasm/src/create/exports.ts index 318793d9..2d26a650 100644 --- a/packages/client-wasm/src/create/exports.ts +++ b/packages/client-wasm/src/create/exports.ts @@ -23,14 +23,14 @@ export default function createExports (bytecode: Uint8Array, imports?: WasmExtra const codeHash = xxhash(bytecode.subarray(0, 2048), 0); // NOTE compilation is quite resource intensive, here we bypass the actual Uint8Array -> Module compilation when we already have this module bytecode in our cache - // if (!cache[codeHash] || forceCreate) { - const table = new WebAssembly.Table(DEFAULT_TABLE); - - cache[codeHash] = new WebAssembly.Instance( - new WebAssembly.Module(bytecode), - createImports(memory, table, imports) - ); - // } + if (!cache[codeHash] || forceCreate) { + const table = new WebAssembly.Table(DEFAULT_TABLE); + + cache[codeHash] = new WebAssembly.Instance( + new WebAssembly.Module(bytecode), + createImports(memory, table, imports) + ); + } return cache[codeHash].exports; } diff --git a/packages/client-wasm/src/create/index.ts b/packages/client-wasm/src/create/index.ts index 69867d3a..58127d14 100644 --- a/packages/client-wasm/src/create/index.ts +++ b/packages/client-wasm/src/create/index.ts @@ -26,7 +26,7 @@ function instrument (name: string, elapsed: Array, fn: () => T): T { return result; } -// let prevChain: WasmInstanceExports; +let prevChain: WasmInstanceExports; let offset: number = 0; export default function wasm ({ config: { wasm: { heapSize = defaults.HEAP_SIZE_KB } }, l }: Options, runtime: RuntimeInterface, chainCode: Uint8Array, chainProxy: Uint8Array): WasmInstanceExports { @@ -37,15 +37,15 @@ export default function wasm ({ config: { wasm: { heapSize = defaults.HEAP_SIZE_ const chain = instrument('chain', elapsed, (): WasmInstanceExports => createExports(chainCode, { env }) ); - // const isNewCode = chain !== prevChain; + const isNewCode = chain !== prevChain; - // if (isNewCode) { - offset = chain.memory.grow(Math.ceil(heapSize / 64)); - // prevChain = chain; - // } + if (isNewCode) { + offset = chain.memory.grow(Math.ceil(heapSize / 64)); + prevChain = chain; + } const instance = instrument('proxy', elapsed, (): WasmInstanceExports => - createExports(chainProxy, { proxy: chain }, createMemory(0, 0)) // , isNewCode) + createExports(chainProxy, { proxy: chain }, createMemory(0, 0), isNewCode) ); runtime.environment.heap.setWasmMemory(chain.memory, offset * 64 * 1024); From ed0b80ae7062e8e67d10381942ac7ebeb925178d Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Wed, 4 Jul 2018 16:10:55 +0200 Subject: [PATCH 8/9] Cleanups --- packages/client-db/src/Sync/index.ts | 10 ++-------- packages/client-db/src/Sync/worker.js | 22 ++++++++++++++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/client-db/src/Sync/index.ts b/packages/client-db/src/Sync/index.ts index c512554f..6c6279ad 100644 --- a/packages/client-db/src/Sync/index.ts +++ b/packages/client-db/src/Sync/index.ts @@ -8,7 +8,7 @@ import path from 'path'; import { Worker } from 'worker_threads'; import logger from '@polkadot/util/logger'; import promisify from '@polkadot/util/promisify'; -import u8aToHex from '@polkadot/util/u8a/toHex'; +// import u8aToHex from '@polkadot/util/u8a/toHex'; import commands from './commands'; @@ -48,11 +48,7 @@ export default class SyncDb implements TrieDb { type }); - // console.error(type, 'wait', state[0]); - Atomics.wait(state, 0, commands.START); - - // console.error(type, 'wake', state[0]); }; if (!returnable.includes(type)) { @@ -74,7 +70,6 @@ export default class SyncDb implements TrieDb { while (true) { switch (state[0]) { case commands.END: - // console.error(type, 'returning', result ? u8aToHex(result) : 'null'); return result; case commands.ERROR: @@ -84,7 +79,6 @@ export default class SyncDb implements TrieDb { size = view.getUint32(0); if (size === U32_MAX) { - // console.error(type, 'returning not found'); return null; } @@ -94,7 +88,7 @@ export default class SyncDb implements TrieDb { break; case commands.READ: - const available = Math.min((buffer as Uint8Array).length, size - offset); + const available = Math.min(buffer.length, size - offset); (result as Uint8Array).set(buffer.subarray(0, available), offset); offset += available; diff --git a/packages/client-db/src/Sync/worker.js b/packages/client-db/src/Sync/worker.js index 55b8f4fb..a3b39962 100644 --- a/packages/client-db/src/Sync/worker.js +++ b/packages/client-db/src/Sync/worker.js @@ -32,11 +32,13 @@ function notify (state, command) { Atomics.wake(state, 0, +Infinity); } +// function wait (state: Int32Array, command: number): void { function wait (state, command) { Atomics.wait(state, 0, command); } -async function notifyResult (state, fn) { +// function notifyDone (state: Int32Array, fn: () => Promise): Promise { +async function notifyDone (state, fn) { try { await fn(); notify(state, commands.END); @@ -45,6 +47,7 @@ async function notifyResult (state, fn) { } } +// function returnValue (state: Int32Array, buffer: Uint8Array, value: Uint8Array | null): void function returnValue (state, buffer, value) { const view = new DataView(buffer.buffer); @@ -74,24 +77,28 @@ function returnValue (state, buffer, value) { notify(state, commands.END); } +// function checkpoint (state: Int32Array): void function checkpoint (state) { - notifyResult(state, () => + notifyDone(state, () => trie.checkpoint() ); } +// function commit (state: Int32Array): void function commit (state) { - notifyResult(state, () => + notifyDone(state, () => trie.commit() ); } +// function del (state: Int32Array, buffer: Uint8Array, key: Uint8Array): void function del (state, buffer, key) { - notifyResult(state, () => + notifyDone(state, () => trie.del(key) ); } +// function get (state: Int32Array, buffer: Uint8Array, key: Uint8Array): void async function get (state, buffer, key) { let result; @@ -105,18 +112,21 @@ async function get (state, buffer, key) { returnValue(state, buffer, result); } +// function put (state: Int32Array, buffer: Uint8Array, key: Uint8Array, value: Uint8Array): void async function put (state, buffer, key, value) { - notifyResult(state, () => + notifyDone(state, () => trie.put(key, value) ); } +// function revert (state: Int32Array): void async function revert (state) { - notifyResult(state, () => + notifyDone(state, () => trie.revert() ); } +// function root (state: Int32Array, buffer: Uint8Array): void function root (state, buffer) { returnValue(state, buffer, trie.root); } From 27e137df2bcf3ac6fc6d71e09ccd0119b7bac958 Mon Sep 17 00:00:00 2001 From: Jaco Greeff Date: Wed, 4 Jul 2018 17:54:19 +0200 Subject: [PATCH 9/9] State looping comments --- packages/client-db/src/Sync/index.ts | 30 ++++++++++++++++++++------- packages/client-db/src/Sync/worker.js | 27 +++++++++++++++--------- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/packages/client-db/src/Sync/index.ts b/packages/client-db/src/Sync/index.ts index 6c6279ad..0bb948cc 100644 --- a/packages/client-db/src/Sync/index.ts +++ b/packages/client-db/src/Sync/index.ts @@ -26,13 +26,8 @@ export default class SyncDb implements TrieDb { this.child = new Worker(path.join(__dirname, './worker.js')); } - private _notifyFill (state: Int32Array): void { - state[0] = commands.FILL; - - Atomics.wake(state, 0, +Infinity); - Atomics.wait(state, 0, commands.FILL); - } - + // Sends a message to the worker, optionally (in the case of get/root) returning the + // actual result. private _sendMessage (type: string, key?: Uint8Array, value?: Uint8Array): Uint8Array | null { const state = new Int32Array(new SharedArrayBuffer(8)); let view: DataView | null = null; @@ -51,11 +46,15 @@ export default class SyncDb implements TrieDb { Atomics.wait(state, 0, commands.START); }; + // Ok, this is not something that returns a value, just send the message and + // return when we the call has been done if (!returnable.includes(type)) { start(); return null; } + // The shared data buffer that will be used by the worker to send info back + // (wrapped in Uint8Array for binary data or in DataView for Int32 data) const shared = new SharedArrayBuffer(4096); buffer = new Uint8Array(shared); @@ -67,14 +66,20 @@ export default class SyncDb implements TrieDb { start(); + // Here we loop through the states and either read data to fill the buffer + // or return when it is time to do so while (true) { switch (state[0]) { + // we have reached the end, result what we have case commands.END: return result; + // Error, so just return a null for the caller to handle case commands.ERROR: return null; + // Ahah, we need to read the size (first result) to detemine how + // big of a result buffer we need. case commands.SIZE: size = view.getUint32(0); @@ -87,6 +92,8 @@ export default class SyncDb implements TrieDb { this._notifyFill(state); break; + // Get the available data from the buffer and write it into our result + // array. case commands.READ: const available = Math.min(buffer.length, size - offset); @@ -96,6 +103,7 @@ export default class SyncDb implements TrieDb { this._notifyFill(state); break; + // This _should_ never happen... but... default: l.error('Unknown worker state', state[0]); return null; @@ -103,6 +111,14 @@ export default class SyncDb implements TrieDb { } } + // Notifies the worker that it should continue filling the result buffer + private _notifyFill (state: Int32Array): void { + state[0] = commands.FILL; + + Atomics.wake(state, 0, +Infinity); + Atomics.wait(state, 0, commands.FILL); + } + checkpoint () { this._sendMessage('checkpoint'); } diff --git a/packages/client-db/src/Sync/worker.js b/packages/client-db/src/Sync/worker.js index a3b39962..5a84c023 100644 --- a/packages/client-db/src/Sync/worker.js +++ b/packages/client-db/src/Sync/worker.js @@ -26,17 +26,21 @@ const U32_MAX = 4294967295; const trie = new Trie(); -// function notify (state: Int32Array, command: number): void { -function notify (state, command) { +// Sets the state and wakes any Atomics waiting on the current state to change. +// If we are not done, assume that we will have to do something afterwards, so +// enter a wait period. +// function notify (state: Int32Array, command: number, doWait: boolean = false): void { +function notify (state, command, doWait = false) { state[0] = command; Atomics.wake(state, 0, +Infinity); -} -// function wait (state: Int32Array, command: number): void { -function wait (state, command) { - Atomics.wait(state, 0, command); + if (doWait) { + Atomics.wait(state, 0, command); + } } +// Waits on a function (returning a promise) to complete. Notify either on end or +// error, discarding the actual result. // function notifyDone (state: Int32Array, fn: () => Promise): Promise { async function notifyDone (state, fn) { try { @@ -47,6 +51,11 @@ async function notifyDone (state, fn) { } } +// Send the value (caller got it from a function return) back to the parent. Since the result +// may be quite large, we do this is phases - +// - send the actual size of the result first (or U32_MAX if null) +// - loop through the actual result, sending buffer.length bytes per iteration +// - since most results are smaller, the 4K buffer should capture a lot, but :code is >250K // function returnValue (state: Int32Array, buffer: Uint8Array, value: Uint8Array | null): void function returnValue (state, buffer, value) { const view = new DataView(buffer.buffer); @@ -61,8 +70,7 @@ function returnValue (state, buffer, value) { let offset = 0; view.setUint32(0, size); - notify(state, commands.SIZE); - wait(state, commands.SIZE); + notify(state, commands.SIZE, true); while (offset !== size) { const available = Math.min(buffer.length, size - offset); @@ -70,8 +78,7 @@ function returnValue (state, buffer, value) { buffer.set(value.subarray(offset, offset + available), 0); offset += available; - notify(state, commands.READ); - wait(state, commands.READ); + notify(state, commands.READ, true); } notify(state, commands.END);