Skip to content

Commit

Permalink
Merge 70d5a04 into 625ecae
Browse files Browse the repository at this point in the history
  • Loading branch information
@jacogr
jacogr committed Sep 10, 2018
2 parents 625ecae + 70d5a04 commit b9d0d7c
Show file tree
Hide file tree
Showing 20 changed files with 1,111 additions and 1,219 deletions.
5 changes: 3 additions & 2 deletions package.json
View file
Expand Up @@ -10,6 +10,7 @@
],
"resolutions": {
"babel-core": "^7.0.0-bridge.0",
"multistream-select": "^0.14.3",
"typescript": "^3.0.1"
},
"scripts": {
Expand All @@ -23,8 +24,8 @@
"start-rust": "NODE_OPTIONS=--experimental-worker node packages/client/scripts/polkadot.js --db-path ./tmp/start-rust --p2p-nodes /ip4/127.0.0.1/tcp/30333/ipfs/QmXBQmmjNsBLobf5Xx8Kqz7rvUrGgiBLTYm2EmQvcZyrKR --p2p-port 39935 --rpc-port 9935"
},
"devDependencies": {
"@polkadot/dev": "^0.20.18",
"@polkadot/ts": "^0.1.24",
"@polkadot/dev": "^0.20.19",
"@polkadot/ts": "^0.1.27",
"pm2": "^3.0.4"
}
}
16 changes: 8 additions & 8 deletions packages/client-chains/package.json
View file
Expand Up @@ -28,19 +28,19 @@
},
"homepage": "https://github.com/polkadot-js/client/client/tree/master/packages/client-chains#readme",
"dependencies": {
"@babel/runtime": "^7.0.0-rc.1",
"@babel/runtime": "^7.0.0",
"@polkadot/client-db": "^0.15.4",
"@polkadot/client-runtime": "^0.15.4",
"@polkadot/client-wasm": "^0.15.4",
"@polkadot/storage": "^0.28.25",
"@polkadot/util": "^0.29.21",
"@polkadot/util-crypto": "^0.29.21",
"@polkadot/util-keyring": "^0.29.21"
"@polkadot/storage": "^0.28.29",
"@polkadot/util": "^0.29.33",
"@polkadot/util-crypto": "^0.29.33",
"@polkadot/util-keyring": "^0.29.33"
},
"devDependencies": {
"@polkadot/client": "^0.15.4",
"@polkadot/extrinsics": "^0.28.25",
"@polkadot/primitives": "^0.28.25",
"@polkadot/storage": "^0.28.25"
"@polkadot/extrinsics": "^0.28.29",
"@polkadot/primitives": "^0.28.29",
"@polkadot/storage": "^0.28.29"
}
}
2 changes: 2 additions & 0 deletions packages/client-chains/src/index.ts
View file
Expand Up @@ -57,6 +57,8 @@ export default class Chain implements ChainInterface {
const bestNumber = dbs.blocks.bestNumber.get();

l.log(`${this.chain.name}, #${bestNumber.toString()}, ${u8aToHex(bestHash, 48)}`);

dbs.snapshotState();
}

private initGenesis (): ChainGenesis {
Expand Down
4 changes: 3 additions & 1 deletion packages/client-chains/src/types.d.ts
View file
Expand Up @@ -36,7 +36,9 @@ export interface ChainLoader {
readonly genesisRoot: Uint8Array;
}

export interface ChainInterface extends ChainDbs {
export interface ChainInterface {
readonly blocks: BlockDb,
readonly state: StateDb,
readonly chain: ChainJson,
readonly executor: ExecutorInterface,
readonly genesis: ChainGenesis
Expand Down
16 changes: 8 additions & 8 deletions packages/client-db/package.json
View file
Expand Up @@ -28,17 +28,17 @@
},
"homepage": "https://github.com/polkadot-js/client/tree/master/packages/client-db#readme",
"dependencies": {
"@babel/runtime": "^7.0.0-rc.1",
"@polkadot/db": "^0.29.21",
"@polkadot/storage": "^0.28.25",
"@polkadot/trie-db": "^0.29.21",
"@polkadot/trie-hash": "^0.29.21",
"@polkadot/util": "^0.29.21",
"@polkadot/util-crypto": "^0.29.21",
"@babel/runtime": "^7.0.0",
"@polkadot/db": "^0.29.33",
"@polkadot/storage": "^0.28.29",
"@polkadot/trie-db": "^0.29.33",
"@polkadot/trie-hash": "^0.29.33",
"@polkadot/util": "^0.29.33",
"@polkadot/util-crypto": "^0.29.33",
"@types/mkdirp": "^0.5.2",
"mkdirp": "^0.5.1"
},
"devDependencies": {
"@polkadot/storage": "^0.28.25"
"@polkadot/storage": "^0.28.29"
}
}
68 changes: 51 additions & 17 deletions packages/client-db/src/index.ts
View file
Expand Up @@ -2,10 +2,10 @@
// This software may be modified and distributed under the terms
// of the ISC license. See the LICENSE file for details.

import { BaseDb } from '@polkadot/db/types';
import { BaseDb, ProgressCb, ProgressValue, TxDb } from '@polkadot/db/types';
import { Config } from '@polkadot/client/types';
import { ChainLoader } from '@polkadot/client-chains/types';
import { BlockDb, StateDb, ChainDbs } from './types';
import { BlockDb, StateDb, ChainDbs, DbConfig } from './types';

import path from 'path';
import DiskDb from '@polkadot/db/Disk';
Expand All @@ -25,27 +25,31 @@ const l = logger('db');
export default class Dbs implements ChainDbs {
readonly blocks: BlockDb;
readonly state: StateDb;
private stateBackup: StateDb | null;
private basePath: string;

constructor ({ db }: Config, chain: ChainLoader) {
const isDisk = db.type === 'disk';
const dbPath = isDisk
this.basePath = db.type === 'disk'
? path.join(db.path, 'chains', chain.id, u8aToHex(chain.genesisRoot))
: '';

// NOTE blocks compress very well
this.blocks = createBlockDb(
isDisk
// NOTE blocks compress very well
? new DiskDb(dbPath, 'block.db', { isCompressed: true })
: new MemoryDb()
this.createBackingDb(db, 'block.db', true)
);
// NOTE state RLP does not compress well here
this.state = createStateDb(
new TrieDb(
isDisk
// NOTE state RLP does not compress well here
? new DiskDb(dbPath, 'state.db', { isCompressed: false })
: new MemoryDb()
this.createBackingDb(db, 'state.db', false)
)
);
this.stateBackup = db.snapshot
? createStateDb(
new TrieDb(
this.createBackingDb(db, 'state.db.snapshot', false)
)
)
: null;

if (db.compact) {
this.maintain('block', this.blocks.db);
Expand All @@ -56,24 +60,54 @@ export default class Dbs implements ChainDbs {
this.state.db.open();
}

private createBackingDb ({ type }: DbConfig, name: string, isCompressed: boolean): TxDb {
return type === 'disk'
? new DiskDb(this.basePath, name, { isCompressed })
: new MemoryDb();
}

private maintain (name: string, db: BaseDb): void {
l.log(`compacting ${name} database`);

let spin = 0;
db.maintain(this.createProgress());
}

snapshotState (): void {
if (!this.stateBackup) {
return;
}

this.stateBackup.db.open();
this.state.db.snapshot(this.stateBackup.db, this.createProgress());

this.state.db.close();
this.state.db.rename(this.basePath, `state.db.backup-${Date.now()}`);

this.stateBackup.db.close();
this.stateBackup.db.rename(this.basePath, 'state.db');
this.stateBackup.db.open();

this.state.db = this.stateBackup.db;
}

private createProgress (): ProgressCb {
let lastUpdate = 0;
let spin = 0;

db.maintain((progress) => {
return (progress: ProgressValue): void => {
const now = Date.now();

if ((now - lastUpdate) > 200) {
const percent = ` ${progress.percent.toFixed(2)}`.slice(-6);
const keys = (progress.keys / 1000).toFixed(2);
const keys = progress.keys > 9999
? `${(progress.keys / 1000).toFixed(2)}k`
: progress.keys;

process.stdout.write(`${PREPEND}${SPINNER[spin % SPINNER.length]} ${percent}%, ${keys}k keys\r`);
process.stdout.write(`${PREPEND}${SPINNER[spin % SPINNER.length]} ${percent}%, ${keys} keys\r`);

lastUpdate = now;
spin++;
}
});
};
}
}
5 changes: 4 additions & 1 deletion packages/client-db/src/types.d.ts
View file
Expand Up @@ -15,6 +15,7 @@ export type DbConfig = {
compact: boolean,
isTrieDb: boolean,
path: string,
snapshot: boolean,
type: DbConfig$Type
};

Expand Down Expand Up @@ -125,5 +126,7 @@ export type StateDb = {

export interface ChainDbs {
readonly blocks: BlockDb,
readonly state: StateDb
readonly state: StateDb,

snapshotState (): void;
}
8 changes: 4 additions & 4 deletions packages/client-p2p-messages/package.json
View file
Expand Up @@ -29,9 +29,9 @@
},
"homepage": "https://github.com/polkadot-js/client/tree/master/packages/client-p2p-messages#readme",
"dependencies": {
"@babel/runtime": "^7.0.0-rc.1",
"@polkadot/jsonrpc": "^0.28.25",
"@polkadot/primitives": "^0.28.25",
"@polkadot/util": "^0.29.21"
"@babel/runtime": "^7.0.0",
"@polkadot/jsonrpc": "^0.28.29",
"@polkadot/primitives": "^0.28.29",
"@polkadot/util": "^0.29.33"
}
}
10 changes: 5 additions & 5 deletions packages/client-p2p/package.json
View file
Expand Up @@ -29,22 +29,22 @@
},
"homepage": "https://github.com/polkadot-js/client/client/tree/master/packages/client-p2p#readme",
"dependencies": {
"@babel/runtime": "^7.0.0-rc.1",
"@babel/runtime": "^7.0.0",
"@polkadot/client-p2p-messages": "^0.15.4",
"@polkadot/primitives": "^0.28.25",
"@polkadot/util": "^0.29.21",
"@polkadot/primitives": "^0.28.29",
"@polkadot/util": "^0.29.33",
"eventemitter3": "^3.1.0",
"libp2p": "^0.23.1",
"libp2p-bootstrap": "^0.9.3",
"libp2p-kad-dht": "^0.10.1",
"libp2p-kad-dht": "^0.10.2",
"libp2p-mdns": "^0.12.0",
"libp2p-mplex": "^0.8.0",
"libp2p-secio": "^0.10.0",
"libp2p-spdy": "^0.12.1",
"libp2p-tcp": "^0.12.1",
"libp2p-websockets": "^0.12.0",
"pull-pushable": "^2.2.0",
"pull-stream": "^3.6.8",
"pull-stream": "^3.6.9",
"pump": "^3.0.0",
"varint": "^5.0.0"
},
Expand Down
8 changes: 7 additions & 1 deletion packages/client-p2p/src/index.ts
View file
Expand Up @@ -86,6 +86,7 @@ export default class P2p extends EventEmitter implements P2pInterface {
l.log(`Started on address=${this.config.p2p.address}, port=${this.config.p2p.port}`);
this.emit('started');

this.node._dht.randomWalk.start();
this._requestAny();

return true;
Expand All @@ -106,6 +107,7 @@ export default class P2p extends EventEmitter implements P2pInterface {
delete this.node;
delete this.peers;

node._dht.randomWalk.stop();
await promisify(node, node.stop);

l.log('Server stopped');
Expand Down Expand Up @@ -296,7 +298,11 @@ export default class P2p extends EventEmitter implements P2pInterface {

Object.values(this.dialQueue).forEach(
async (item: QueuedPeer): Promise<void> => {
if (item.isDialled || !this.peers || item.nextDial < now) {
if (!this.peers) {
return;
} else if (item.nextDial > now && !item.peer.isActive()) {
item.isDialled = false;
} else if (item.isDialled || item.nextDial < now) {
return;
}

Expand Down

0 comments on commit b9d0d7c

Please sign in to comment.