diff --git a/migrate/README.md b/migrate/README.md new file mode 100644 index 000000000..0cba0e6f1 --- /dev/null +++ b/migrate/README.md @@ -0,0 +1,15 @@ +Bcash Migrations +================ + +There are no migrations necessary for bcash. Current database versions are as +follows: + - ChainDB - `v5` + - WalletDB - `v7` + - Mempool - `v0` + - Indexer - `v0` + +*Note: Lastest version of bcoin does not have separate Indexer and its ChainDB +is at `v4`(See [Refactor Indexers][bcoin-indexers]), but WalletDB is +compatible.* + +[bcoin-indexers]: https://github.com/bcoin-org/bcoin/pull/424 diff --git a/migrate/chaindb2to3.js b/migrate/chaindb2to3.js deleted file mode 100644 index a07a7e4c6..000000000 --- a/migrate/chaindb2to3.js +++ /dev/null @@ -1,701 +0,0 @@ -'use strict'; - -if (process.argv.indexOf('-h') !== -1 - || process.argv.indexOf('--help') !== -1 - || process.argv.length < 3) { - console.error('Bcoin database migration (chaindb v2->v3).'); - console.error(''); - console.error('Usage:'); - console.error(' $ node migrate/chaindb2to3.js [database-path] [--prune]'); - console.error(''); - console.error('Note: use --prune to convert your database to a pruned DB'); - console.error('in the process. This results in a faster migration, but'); - console.error('a pruning of the chain.'); - process.exit(1); - throw new Error('Exit failed.'); -} - -const assert = require('assert'); -const bdb = require('bdb'); -const hash256 = require('bcrypto/lib/hash256'); -const BN = require('bcrypto/lib/bn.js'); -const bio = require('bufio'); -const LRU = require('blru'); -const {BufferMap} = require('buffer-map'); -const util = require('../lib/utils/util'); -const OldCoins = require('./coins/coins'); -const OldUndoCoins = require('./coins/undocoins'); -const CoinEntry = require('../lib/coins/coinentry'); -const UndoCoins = require('../lib/coins/undocoins'); -const Block = require('../lib/primitives/block'); -const consensus = require('../lib/protocol/consensus'); - -const shouldPrune = process.argv.indexOf('--prune') !== -1; - -let hasIndex = false; -let hasPruned = false; -let hasSPV = false; - -const db = bdb.create({ - location: process.argv[2], - compression: true, - cacheSize: 32 << 20, - createIfMissing: false -}); - -// \0\0migrate -const JOURNAL_KEY = Buffer.from('00006d696772617465', 'hex'); -const MIGRATION_ID = 0; -const STATE_VERSION = -1; -const STATE_UNDO = 0; -const STATE_CLEANUP = 1; -const STATE_COINS = 2; -const STATE_ENTRY = 3; -const STATE_FINAL = 4; -const STATE_DONE = 5; - -const metaCache = new BufferMap(); -const lruCache = new LRU(200000, null, BufferMap); - -function writeJournal(batch, state, hash) { - const data = Buffer.allocUnsafe(34); - - if (!hash) - hash = consensus.ZERO_HASH; - - data[0] = MIGRATION_ID; - data[1] = state; - hash.copy(data, 2); - - batch.put(JOURNAL_KEY, data); -} - -async function readJournal() { - const data = await db.get(JOURNAL_KEY); - - if (!data) - return [STATE_VERSION, consensus.ZERO_HASH]; - - if (data.length !== 34) - throw new Error('Bad migration length.'); - - if (data[0] !== MIGRATION_ID) - throw new Error('Bad migration id.'); - - const state = data.readUInt8(1, true); - const hash = data.slice(2, 34); - - console.log('Reading journal.'); - console.log('Recovering from state %d.', state); - - return [state, hash]; -} - -async function updateVersion() { - const batch = db.batch(); - - console.log('Checking version.'); - - const raw = await db.get('V'); - - if (!raw) - throw new Error('No DB version found!'); - - const version = raw.readUInt32LE(0, true); - - if (version !== 2) - throw Error(`DB is version ${version}.`); - - // Set to uint32_max temporarily. - // This is to prevent bcoin from - // trying to access this chain. - const data = Buffer.allocUnsafe(4); - data.writeUInt32LE(-1 >>> 0, 0, true); - batch.put('V', data); - - writeJournal(batch, STATE_UNDO); - - console.log('Updating version.'); - - await batch.write(); - - return [STATE_UNDO, consensus.ZERO_HASH]; -} - -async function reserializeUndo(hash) { - let tip = await getTip(); - - const height = tip.height; - - if (!hash.equals(consensus.ZERO_HASH)) - tip = await getEntry(hash); - - console.log('Reserializing undo coins from tip %s.', - util.revHex(tip.hash)); - - let batch = db.batch(); - let pruning = false; - let total = 0; - let totalCoins = 0; - - while (tip.height !== 0 && !hasSPV) { - if (shouldPrune) { - if (tip.height < height - 288) { - console.log('Pruning block %s (%d).', - util.revHex(tip.hash), tip.height); - - batch.del(pair('u', tip.hash)); - batch.del(pair('b', tip.hash)); - - if (!pruning) { - console.log( - 'Reserialized %d undo records (%d coins).', - total, totalCoins); - writeJournal(batch, STATE_UNDO, tip.prevBlock); - await batch.write(); - metaCache.clear(); - batch = db.batch(); - pruning = true; - } - - tip = await getEntry(tip.prevBlock); - assert(tip); - continue; - } - } - - const undoData = await db.get(pair('u', tip.hash)); - const blockData = await db.get(pair('b', tip.hash)); - - if (!undoData) { - tip = await getEntry(tip.prevBlock); - assert(tip); - continue; - } - - if (!blockData) { - if (!hasPruned) - throw new Error(`Block not found: ${tip.hash}.`); - break; - } - - const block = Block.fromRaw(blockData); - const old = OldUndoCoins.fromRaw(undoData); - const undo = new UndoCoins(); - - console.log( - 'Reserializing coins for block %s (%d).', - util.revHex(tip.hash), tip.height); - - for (let i = block.txs.length - 1; i >= 1; i--) { - const tx = block.txs[i]; - for (let j = tx.inputs.length - 1; j >= 0; j--) { - const {prevout} = tx.inputs[j]; - const coin = old.items.pop(); - const output = coin.toOutput(); - - assert(coin); - - const [version, height, write] = await getMeta(coin, prevout); - - const item = new CoinEntry(); - item.version = version; - item.height = height; - item.coinbase = coin.coinbase; - item.output.script = output.script; - item.output.value = output.value; - item.spent = true; - item.raw = null; - - // Store an index of heights and versions for later. - const meta = [version, height]; - - if (write) { - const data = Buffer.allocUnsafe(8); - data.writeUInt32LE(version, 0, true); - data.writeUInt32LE(height, 4, true); - batch.put(pair(0x01, prevout.hash), data); - metaCache.set(prevout.hash, meta); - } - - if (!lruCache.has(prevout.hash)) - lruCache.set(prevout.hash, meta); - - undo.items.push(item); - } - } - - // We need to reverse everything. - undo.items.reverse(); - - totalCoins += undo.items.length; - - batch.put(pair('u', tip.hash), undo.toRaw()); - - if (++total % 100 === 0) { - console.log( - 'Reserialized %d undo records (%d coins).', - total, totalCoins); - writeJournal(batch, STATE_UNDO, tip.prevBlock); - await batch.write(); - metaCache.clear(); - batch = db.batch(); - } - - tip = await getEntry(tip.prevBlock); - } - - writeJournal(batch, STATE_CLEANUP); - await batch.write(); - - metaCache.clear(); - lruCache.reset(); - - console.log( - 'Reserialized %d undo records (%d coins).', - total, totalCoins); - - return [STATE_CLEANUP, consensus.ZERO_HASH]; -} - -async function cleanupIndex() { - if (hasSPV) - return [STATE_COINS, consensus.ZERO_HASH]; - - const iter = db.iterator({ - gte: pair(0x01, consensus.ZERO_HASH), - lte: pair(0x01, Buffer.alloc(32, 0xff)), - keys: true - }); - - console.log('Removing txid->height undo index.'); - - let batch = db.batch(); - let total = 0; - - while (await iter.next()) { - const {key} = iter; - - batch.del(key); - - if (++total % 10000 === 0) { - console.log('Cleaned up %d undo records.', total); - writeJournal(batch, STATE_CLEANUP); - await batch.write(); - batch = db.batch(); - } - } - - writeJournal(batch, STATE_COINS); - await batch.write(); - - console.log('Cleaned up %d undo records.', total); - - return [STATE_COINS, consensus.ZERO_HASH]; -} - -async function reserializeCoins(hash) { - if (hasSPV) - return [STATE_ENTRY, consensus.ZERO_HASH]; - - const iter = db.iterator({ - gte: pair('c', hash), - lte: pair('c', Buffer.alloc(32, 0xff)), - keys: true, - values: true - }); - - let start = true; - - if (!hash.equals(consensus.ZERO_HASH)) { - const item = await iter.next(); - if (!item) - start = false; - } - - console.log('Reserializing coins from %s.', util.revHex(hash)); - - let batch = db.batch(); - let total = 0; - - while (start) { - const item = await iter.next(); - - if (!item) - break; - - if (item.key.length !== 33) - continue; - - const hash = item.key.slice(1, 33); - const old = OldCoins.fromRaw(item.value, hash); - - let update = false; - - for (let i = 0; i < old.outputs.length; i++) { - const coin = old.getCoin(i); - - if (!coin) - continue; - - const item = new CoinEntry(); - item.version = coin.version; - item.height = coin.height; - item.coinbase = coin.coinbase; - item.output.script = coin.script; - item.output.value = coin.value; - item.spent = false; - item.raw = null; - - batch.put(bpair('c', hash, i), item.toRaw()); - - if (++total % 10000 === 0) - update = true; - } - - batch.del(item.key); - - if (update) { - console.log('Reserialized %d coins.', total); - writeJournal(batch, STATE_COINS, hash); - await batch.write(); - batch = db.batch(); - } - } - - writeJournal(batch, STATE_ENTRY); - await batch.write(); - - console.log('Reserialized %d coins.', total); - - return [STATE_ENTRY, consensus.ZERO_HASH]; -} - -async function reserializeEntries(hash) { - const iter = db.iterator({ - gte: pair('e', hash), - lte: pair('e', Buffer.alloc(32, 0xff)), - values: true - }); - - let start = true; - - if (!hash.equals(consensus.ZERO_HASH)) { - const item = await iter.next(); - if (!item) - start = false; - else - assert(item.key.equals(pair('e', hash))); - } - - console.log('Reserializing entries from %s.', util.revHex(hash)); - - const tip = await getTipHash(); - - let total = 0; - let batch = db.batch(); - - while (start) { - const item = await iter.next(); - - if (!item) - break; - - const entry = entryFromRaw(item.value); - const main = await isMainChain(entry, tip); - - batch.put(item.key, entryToRaw(entry, main)); - - if (++total % 100000 === 0) { - console.log('Reserialized %d entries.', total); - writeJournal(batch, STATE_ENTRY, entry.hash); - await batch.write(); - batch = db.batch(); - } - } - - writeJournal(batch, STATE_FINAL); - await batch.write(); - - console.log('Reserialized %d entries.', total); - - return [STATE_FINAL, consensus.ZERO_HASH]; -} - -async function finalize() { - const batch = db.batch(); - const data = Buffer.allocUnsafe(4); - - data.writeUInt32LE(3, 0, true); - - batch.del(JOURNAL_KEY); - batch.put('V', data); - - // This has bugged me for a while. - batch.del(pair('n', consensus.ZERO_HASH)); - - if (shouldPrune) { - const data = await db.get('O'); - - assert(data); - - let flags = data.readUInt32LE(4, true); - flags |= 1 << 2; - - data.writeUInt32LE(flags, 4, true); - - batch.put('O', data); - } - - console.log('Finalizing database.'); - - await batch.write(); - - console.log('Compacting database...'); - - await db.compactRange(); - - return [STATE_DONE, consensus.ZERO_HASH]; -} - -async function getMeta(coin, prevout) { - // Case 1: Undo coin is the last spend. - if (coin.height !== -1) { - assert(coin.version !== -1, 'Database corruption.'); - return [coin.version, coin.height, hasIndex ? false : true]; - } - - // Case 2: The item is still in the LRU cache. - const lruItem = lruCache.get(prevout.hash); - - if (lruItem) { - const [version, height] = lruItem; - return [version, height, false]; - } - - // Case 3: The database has a tx-index. We - // can just hit that instead of reindexing. - if (hasIndex) { - const txRaw = await db.get(pair('t', prevout.hash)); - assert(txRaw, 'Database corruption.'); - assert(txRaw[txRaw.length - 45] === 1); - const version = txRaw.readUInt32LE(0, true); - const height = txRaw.readUInt32LE(txRaw.length - 12, true); - return [version, height, false]; - } - - // Case 4: We have previously cached - // this coin's metadata, but it's not - // written yet. - const metaItem = metaCache.get(prevout.hash); - - if (metaItem) { - const [version, height] = metaItem; - return [version, height, false]; - } - - // Case 5: We have previously cached - // this coin's metadata, and it is - // written. - const metaRaw = await db.get(pair(0x01, prevout.hash)); - - if (metaRaw) { - const version = metaRaw.readUInt32LE(0, true); - const height = metaRaw.readUInt32LE(4, true); - return [version, height, false]; - } - - // Case 6: The coin's metadata is - // still in the top-level UTXO set. - const coinsRaw = await db.get(pair('c', prevout.hash)); - - // Case 7: We're pruned and are - // under the keepBlocks threshold. - // We don't have access to this - // data. Luckily, it appears that - // all historical transactions - // under height 182 are version 1, - // which means height is not - // necessary to determine CSV - // anyway. Just store the height - // as `1`. - if (!coinsRaw) { - assert(hasPruned, 'Database corruption.'); - return [1, 1, false]; - } - - const br = bio.read(coinsRaw); - const version = br.readVarint(); - const height = br.readU32(); - - return [version, height, true]; -} - -async function getTip() { - const tip = await getTipHash(); - return await getEntry(tip); -} - -async function getTipHash() { - const state = await db.get('R'); - assert(state); - return state.slice(0, 32); -} - -async function getEntry(hash) { - const data = await db.get(pair('e', hash)); - assert(data); - return entryFromRaw(data); -} - -async function isPruned() { - const data = await db.get('O'); - assert(data); - return (data.readUInt32LE(4) & 4) !== 0; -} - -async function isSPV() { - const data = await db.get('O'); - assert(data); - return (data.readUInt32LE(4) & 1) !== 0; -} - -async function isIndexed() { - const data = await db.get('O'); - assert(data); - return (data.readUInt32LE(4) & 8) !== 0; -} - -async function isMainChain(entry, tip) { - if (entry.hash.equals(tip)) - return true; - - if (await db.get(pair('n', entry.hash))) - return true; - - return false; -} - -function entryFromRaw(data) { - const br = bio.read(data, true); - const hash = hash256.digest(br.readBytes(80)); - - br.seek(-80); - - const entry = {}; - entry.hash = hash.toString(); - entry.version = br.readU32(); - entry.prevBlock = br.readHash(); - entry.merkleRoot = br.readHash(); - entry.time = br.readU32(); - entry.bits = br.readU32(); - entry.nonce = br.readU32(); - entry.height = br.readU32(); - entry.chainwork = new BN(br.readBytes(32), 'le'); - - return entry; -} - -function entryToRaw(entry, main) { - const bw = bio.write(116 + 1); - - bw.writeU32(entry.version); - bw.writeHash(entry.prevBlock); - bw.writeHash(entry.merkleRoot); - bw.writeU32(entry.time); - bw.writeU32(entry.bits); - bw.writeU32(entry.nonce); - bw.writeU32(entry.height); - bw.writeBytes(entry.chainwork.toArrayLike(Buffer, 'le', 32)); - bw.writeU8(main ? 1 : 0); - - return bw.render(); -} - -function write(data, hash, off) { - assert(Buffer.isBuffer(hash)); - return hash.copy(data, off); -} - -function pair(prefix, hash) { - const key = Buffer.allocUnsafe(33); - if (typeof prefix === 'string') - prefix = prefix.charCodeAt(0); - key[0] = prefix; - write(key, hash, 1); - return key; -} - -function bpair(prefix, hash, index) { - const key = Buffer.allocUnsafe(37); - if (typeof prefix === 'string') - prefix = prefix.charCodeAt(0); - key[0] = prefix; - write(key, hash, 1); - key.writeUInt32BE(index, 33, true); - return key; -} - -// Make eslint happy. -reserializeEntries; - -(async () => { - await db.open(); - - console.log('Opened %s.', process.argv[2]); - - if (await isSPV()) - hasSPV = true; - - if (await isPruned()) - hasPruned = true; - - if (await isIndexed()) - hasIndex = true; - - if (shouldPrune && hasPruned) - throw new Error('Database is already pruned.'); - - if (shouldPrune && hasSPV) - throw new Error('Database cannot be pruned due to SPV.'); - - console.log('Starting migration in 3 seconds...'); - console.log('If you crash you can start over.'); - - await new Promise(r => setTimeout(r, 3000)); - - let [state, hash] = await readJournal(); - - if (state === STATE_VERSION) - [state, hash] = await updateVersion(); - - if (state === STATE_UNDO) - [state, hash] = await reserializeUndo(hash); - - if (state === STATE_CLEANUP) - [state, hash] = await cleanupIndex(); - - if (state === STATE_COINS) - [state, hash] = await reserializeCoins(hash); - - // if (state === STATE_ENTRY) - // [state, hash] = await reserializeEntries(hash); - - if (state === STATE_ENTRY) - [state, hash] = [STATE_FINAL, consensus.ZERO_HASH]; - - if (state === STATE_FINAL) - [state, hash] = await finalize(); - - assert(state === STATE_DONE); - - console.log('Closing %s.', process.argv[2]); - - await db.close(); - - console.log('Migration complete.'); - process.exit(0); -})().catch((err) => { - console.error(err.stack); - process.exit(1); -}); diff --git a/migrate/chaindb3to4.js b/migrate/chaindb3to4.js deleted file mode 100644 index 497940a63..000000000 --- a/migrate/chaindb3to4.js +++ /dev/null @@ -1,159 +0,0 @@ -'use strict'; - -const assert = require('assert'); -const bdb = require('bdb'); -const layout = require('../lib/blockchain/layout'); - -// changes: -// db version record -// deployment table v->D -// C/T key format - -assert(process.argv.length > 2, 'Please pass in a database path.'); - -let parent = null; - -const db = bdb.create({ - location: process.argv[2], - memory: false, - compression: true, - cacheSize: 32 << 20, - createIfMissing: false -}); - -async function updateVersion() { - console.log('Checking version.'); - - const data = await db.get(layout.V.encode()); - assert(data, 'No version.'); - - const ver = data.readUInt32LE(0, true); - - if (ver !== 3) - throw Error(`DB is version ${ver}.`); - - console.log('Updating version to %d.', ver + 1); - - const buf = Buffer.allocUnsafe(5 + 4); - buf.write('chain', 0, 'ascii'); - buf.writeUInt32LE(4, 5, true); - - parent.put(layout.V.encode(), buf); -} - -async function migrateKeys(id, from, to) { - console.log('Migrating keys for %s.', String.fromCharCode(id)); - - const iter = db.iterator({ - gt: Buffer.from([id]), - lt: Buffer.from([id + 1]), - keys: true - }); - - let batch = db.batch(); - let total = 0; - let items = 0; - - await iter.each(async (key) => { - batch.put(to.encode(...from(key)), null); - batch.del(key); - - total += (key.length + 80) * 2; - items += 1; - - if (total >= (128 << 20)) { - await batch.write(); - batch = db.batch(); - total = 0; - } - }); - - console.log('Migrated %d keys for %s.', items, String.fromCharCode(id)); - - return batch.write(); -} - -async function updateKeys() { - console.log('Updating keys...'); - - const v = Buffer.from('v', 'ascii'); - - const table = await db.get(v); - assert(table); - - parent.put(layout.D.encode(), table); - parent.del(v); - - const raw = await db.get(layout.O.encode()); - assert(raw); - - const flags = raw.readUInt32LE(8, true); - - if (!(flags & 16)) { - console.log('Updated keys.'); - return; - } - - console.log('Updating address index keys...'); - - await migrateKeys(0x54, parseT, layout.T); // T - await migrateKeys(0xab, parseT, layout.T); // W + T - await migrateKeys(0x43, parseC, layout.C); // C - await migrateKeys(0x9a, parseC, layout.C); // W + C - - console.log('Updated keys.'); -} - -function parseT(key) { - assert(Buffer.isBuffer(key)); - - if (key.length === 65) - return [key.slice(1, 33), key.slice(33, 65)]; - - assert(key.length === 53); - return [key.slice(1, 21), key.slice(21, 53)]; -} - -function parseC(key) { - assert(Buffer.isBuffer(key)); - - let addr, hash, index; - - if (key.length === 69) { - addr = key.slice(1, 33); - hash = key.slice(33, 65); - index = key.readUInt32BE(65, 0); - } else if (key.length === 57) { - addr = key.slice(1, 21); - hash = key.slice(21, 53); - index = key.readUInt32BE(53, 0); - } else { - assert(false); - } - - return [addr, hash, index]; -} - -/* - * Execute - */ - -(async () => { - await db.open(); - - console.log('Opened %s.', process.argv[2]); - - parent = db.batch(); - - await updateVersion(); - await updateKeys(); - - await parent.write(); - await db.close(); -})().then(() => { - console.log('Migration complete.'); - process.exit(0); -}).catch((err) => { - console.error(err.stack); - process.exit(1); -}); diff --git a/migrate/chaindb4to5.js b/migrate/chaindb4to5.js deleted file mode 100644 index dba1c4f00..000000000 --- a/migrate/chaindb4to5.js +++ /dev/null @@ -1,79 +0,0 @@ -'use strict'; - -const assert = require('assert'); -const bdb = require('bdb'); -const layout = require('../lib/blockchain/layout'); - -// changes: -// removes tx, addr indexes i.e layout.t, layout.T, layout.C - -assert(process.argv.length > 2, 'Please pass in a database path.'); - -const db = bdb.create({ - location: process.argv[2], - memory: false, - compression: true, - cacheSize: 32 << 20, - createIfMissing: false -}); - -async function removeKey(name, key) { - const iter = db.iterator({ - gte: key.min(), - lte: key.max(), - reverse: true, - keys: true - }); - - let batch = db.batch(); - let total = 0; - - while (await iter.next()) { - const {key} = iter; - batch.del(key); - - if (++total % 10000 === 0) { - console.log('Cleaned up %d %s index records.', total, name); - await batch.write(); - batch = db.batch(); - } - } - await batch.write(); - - console.log('Cleaned up %d %s index records.', total, name); -} - -/* - * Execute - */ - -(async () => { - await db.open(); - - console.log('Opened %s.', process.argv[2]); - console.log('Checking version.'); - await db.verify(layout.V.encode(), 'chain', 4); - - const t = bdb.key('t', ['hhash256']); - const T = bdb.key('T', ['hhash', 'hhash256']); - const C = bdb.key('C', ['hhash', 'hhash256', 'uint32']); - - await removeKey('hash -> tx', t); - await removeKey('addr -> tx', T); - await removeKey('addr -> coin', C); - - console.log('Compacting database...'); - await db.compactRange(); - - console.log('Updating version to %d.', 5); - await db.del(layout.V.encode()); - await db.verify(layout.V.encode(), 'chain', 5); - - await db.close(); -})().then(() => { - console.log('Migration complete.'); - process.exit(0); -}).catch((err) => { - console.error(err.stack); - process.exit(1); -}); diff --git a/migrate/coins/coins.js b/migrate/coins/coins.js deleted file mode 100644 index 401992697..000000000 --- a/migrate/coins/coins.js +++ /dev/null @@ -1,756 +0,0 @@ -/*! - * coins.js - coins object for bcoin - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -/* eslint-disable */ - -'use strict'; - -const assert = require('assert'); -const util = require('../../lib/utils/util'); -const Coin = require('../../lib/primitives/coin'); -const Output = require('../../lib/primitives/output'); -const BufferReader = require('../../lib/utils/reader'); -const StaticWriter = require('../../lib/utils/staticwriter'); -const encoding = require('../../lib/utils/encoding'); -const compressor = require('./compress'); -const compress = compressor.compress; -const decompress = compressor.decompress; - -/** - * Represents the outputs for a single transaction. - * @alias module:coins.Coins - * @constructor - * @param {Object?} options - Options object. - * @property {Hash} hash - Transaction hash. - * @property {Number} version - Transaction version. - * @property {Number} height - Transaction height (-1 if unconfirmed). - * @property {Boolean} coinbase - Whether the containing - * transaction is a coinbase. - * @property {CoinEntry[]} outputs - Coins. - */ - -function Coins(options) { - if (!(this instanceof Coins)) - return new Coins(options); - - this.version = 1; - this.hash = encoding.ZERO_HASH; - this.height = -1; - this.coinbase = true; - this.outputs = []; - - if (options) - this.fromOptions(options); -} - -/** - * Inject properties from options object. - * @private - * @param {Object} options - */ - -Coins.prototype.fromOptions = function fromOptions(options) { - if (options.version != null) { - assert((options.version >>> 0) === options.version); - this.version = options.version; - } - - if (options.hash) { - assert(Buffer.isBuffer(options.hash)); - this.hash = options.hash; - } - - if (options.height != null) { - assert(Number.isSafeInteger(options.height)); - this.height = options.height; - } - - if (options.coinbase != null) { - assert(typeof options.coinbase === 'boolean'); - this.coinbase = options.coinbase; - } - - if (options.outputs) { - assert(Array.isArray(options.outputs)); - this.outputs = options.outputs; - this.cleanup(); - } - - return this; -}; - -/** - * Instantiate coins from options object. - * @param {Object} options - * @returns {Coins} - */ - -Coins.fromOptions = function fromOptions(options) { - return new Coins().fromOptions(options); -}; - -/** - * Add a single entry to the collection. - * @param {Number} index - * @param {CoinEntry} entry - */ - -Coins.prototype.add = function add(index, entry) { - assert(index >= 0); - - while (this.outputs.length <= index) - this.outputs.push(null); - - assert(!this.outputs[index]); - - this.outputs[index] = entry; -}; - -/** - * Add a single output to the collection. - * @param {Number} index - * @param {Output} output - */ - -Coins.prototype.addOutput = function addOutput(index, output) { - assert(!output.script.isUnspendable()); - this.add(index, CoinEntry.fromOutput(output)); -}; - -/** - * Add a single coin to the collection. - * @param {Coin} coin - */ - -Coins.prototype.addCoin = function addCoin(coin) { - assert(!coin.script.isUnspendable()); - this.add(coin.index, CoinEntry.fromCoin(coin)); -}; - -/** - * Test whether the collection has a coin. - * @param {Number} index - * @returns {Boolean} - */ - -Coins.prototype.has = function has(index) { - if (index >= this.outputs.length) - return false; - - return this.outputs[index] != null; -}; - -/** - * Test whether the collection - * has an unspent coin. - * @param {Number} index - * @returns {Boolean} - */ - -Coins.prototype.isUnspent = function isUnspent(index) { - if (index >= this.outputs.length) - return false; - - const output = this.outputs[index]; - - if (!output || output.spent) - return false; - - return true; -}; - -/** - * Get a coin entry. - * @param {Number} index - * @returns {CoinEntry} - */ - -Coins.prototype.get = function get(index) { - if (index >= this.outputs.length) - return; - - return this.outputs[index]; -}; - -/** - * Get an output. - * @param {Number} index - * @returns {Output} - */ - -Coins.prototype.getOutput = function getOutput(index) { - const entry = this.get(index); - - if (!entry) - return; - - return entry.toOutput(); -}; - -/** - * Get a coin. - * @param {Number} index - * @returns {Coin} - */ - -Coins.prototype.getCoin = function getCoin(index) { - const entry = this.get(index); - - if (!entry) - return; - - return entry.toCoin(this, index); -}; - -/** - * Spend a coin entry and return it. - * @param {Number} index - * @returns {CoinEntry} - */ - -Coins.prototype.spend = function spend(index) { - const entry = this.get(index); - - if (!entry || entry.spent) - return; - - entry.spent = true; - - return entry; -}; - -/** - * Remove a coin entry and return it. - * @param {Number} index - * @returns {CoinEntry} - */ - -Coins.prototype.remove = function remove(index) { - const entry = this.get(index); - - if (!entry) - return false; - - this.outputs[index] = null; - this.cleanup(); - - return entry; -}; - -/** - * Calculate unspent length of coins. - * @returns {Number} - */ - -Coins.prototype.length = function length() { - let len = this.outputs.length; - - while (len > 0 && !this.isUnspent(len - 1)) - len--; - - return len; -}; - -/** - * Cleanup spent outputs (remove pruned). - */ - -Coins.prototype.cleanup = function cleanup() { - let len = this.outputs.length; - - while (len > 0 && !this.outputs[len - 1]) - len--; - - this.outputs.length = len; -}; - -/** - * Test whether the coins are fully spent. - * @returns {Boolean} - */ - -Coins.prototype.isEmpty = function isEmpty() { - return this.length() === 0; -}; - -/* - * Coins serialization: - * version: varint - * height: uint32 - * header-code: varint - * bit 1: coinbase - * bit 2: first output unspent - * bit 3: second output unspent - * bit 4-32: spent-field size - * spent-field: bitfield (0=spent, 1=unspent) - * outputs (repeated): - * value: varint - * compressed-script: - * prefix: 0x00 = 20 byte pubkey hash - * 0x01 = 20 byte script hash - * 0x02-0x05 = 32 byte ec-key x-value - * 0x06-0x09 = reserved - * >=0x10 = varint-size + 10 | raw script - * data: script data, dictated by the prefix - * - * The compression below sacrifices some cpu in exchange - * for reduced size, but in some cases the use of varints - * actually increases speed (varint versions and values - * for example). We do as much compression as possible - * without sacrificing too much cpu. Value compression - * is intentionally excluded for now as it seems to be - * too much of a perf hit. Maybe when v8 optimizes - * non-smi arithmetic better we can enable it. - */ - -/** - * Calculate header code. - * @param {Number} len - * @param {Number} size - * @returns {Number} - */ - -Coins.prototype.header = function header(len, size) { - const first = this.isUnspent(0); - const second = this.isUnspent(1); - let offset = 0; - - // Throw if we're fully spent. - assert(len !== 0, 'Cannot serialize fully-spent coins.'); - - // First and second bits - // have a double meaning. - if (!first && !second) { - assert(size !== 0); - offset = 1; - } - - // Calculate header code. - let code = 8 * (size - offset); - - if (this.coinbase) - code += 1; - - if (first) - code += 2; - - if (second) - code += 4; - - return code; -}; - -/** - * Serialize the coins object. - * @returns {Buffer} - */ - -Coins.prototype.toRaw = function toRaw() { - const len = this.length(); - const size = Math.floor((len + 5) / 8); - const code = this.header(len, size); - const total = this.getSize(len, size, code); - const bw = new StaticWriter(total); - - // Write headers. - bw.writeVarint(this.version); - bw.writeU32(this.height); - bw.writeVarint(code); - - // Write the spent field. - for (let i = 0; i < size; i++) { - let ch = 0; - for (let j = 0; j < 8 && 2 + i * 8 + j < len; j++) { - if (this.isUnspent(2 + i * 8 + j)) - ch |= 1 << j; - } - bw.writeU8(ch); - } - - // Write the compressed outputs. - for (let i = 0; i < len; i++) { - const output = this.outputs[i]; - - if (!output || output.spent) - continue; - - output.toWriter(bw); - } - - return bw.render(); -}; - -/** - * Calculate coins size. - * @param {Number} code - * @param {Number} size - * @param {Number} len - * @returns {Number} - */ - -Coins.prototype.getSize = function getSize(len, size, code) { - let total = 0; - - total += encoding.sizeVarint(this.version); - total += 4; - total += encoding.sizeVarint(code); - total += size; - - // Write the compressed outputs. - for (let i = 0; i < len; i++) { - const output = this.outputs[i]; - - if (!output || output.spent) - continue; - - total += output.getSize(); - } - - return total; -}; - -/** - * Inject data from serialized coins. - * @private - * @param {Buffer} data - * @param {Hash} hash - * @returns {Coins} - */ - -Coins.prototype.fromRaw = function fromRaw(data, hash) { - const br = new BufferReader(data); - let first = null; - let second = null; - - // Inject hash (passed by caller). - this.hash = hash; - - // Read headers. - this.version = br.readVarint(); - this.height = br.readU32(); - const code = br.readVarint(); - this.coinbase = (code & 1) !== 0; - - // Recalculate size. - let size = code / 8 | 0; - - if ((code & 6) === 0) - size += 1; - - // Setup spent field. - let offset = br.offset; - br.seek(size); - - // Read first two outputs. - if ((code & 2) !== 0) - first = CoinEntry.fromReader(br); - - if ((code & 4) !== 0) - second = CoinEntry.fromReader(br); - - this.outputs.push(first); - this.outputs.push(second); - - // Read outputs. - for (let i = 0; i < size; i++) { - const ch = br.data[offset++]; - for (let j = 0; j < 8; j++) { - if ((ch & (1 << j)) === 0) { - this.outputs.push(null); - continue; - } - this.outputs.push(CoinEntry.fromReader(br)); - } - } - - this.cleanup(); - - return this; -}; - -/** - * Parse a single serialized coin. - * @param {Buffer} data - * @param {Hash} hash - * @param {Number} index - * @returns {Coin} - */ - -Coins.parseCoin = function parseCoin(data, hash, index) { - const br = new BufferReader(data); - const coin = new Coin(); - - // Inject outpoint (passed by caller). - coin.hash = hash; - coin.index = index; - - // Read headers. - coin.version = br.readVarint(); - coin.height = br.readU32(); - const code = br.readVarint(); - coin.coinbase = (code & 1) !== 0; - - // Recalculate size. - let size = code / 8 | 0; - - if ((code & 6) === 0) - size += 1; - - if (index >= 2 + size * 8) - return; - - // Setup spent field. - let offset = br.offset; - br.seek(size); - - // Read first two outputs. - for (let i = 0; i < 2; i++) { - if ((code & (2 << i)) !== 0) { - if (index === 0) { - decompress.coin(coin, br); - return coin; - } - decompress.skip(br); - } else { - if (index === 0) - return; - } - index -= 1; - } - - // Read outputs. - for (let i = 0; i < size; i++) { - const ch = br.data[offset++]; - for (let j = 0; j < 8; j++) { - if ((ch & (1 << j)) !== 0) { - if (index === 0) { - decompress.coin(coin, br); - return coin; - } - decompress.skip(br); - } else { - if (index === 0) - return; - } - index -= 1; - } - } -}; - -/** - * Instantiate coins from a buffer. - * @param {Buffer} data - * @param {Hash} hash - Transaction hash. - * @returns {Coins} - */ - -Coins.fromRaw = function fromRaw(data, hash) { - return new Coins().fromRaw(data, hash); -}; - -/** - * Inject properties from tx. - * @private - * @param {TX} tx - * @param {Number} height - */ - -Coins.prototype.fromTX = function fromTX(tx, height) { - assert(typeof height === 'number'); - - this.version = tx.version; - this.hash = tx.hash(); - this.height = height; - this.coinbase = tx.isCoinbase(); - - for (const output of tx.outputs) { - if (output.script.isUnspendable()) { - this.outputs.push(null); - continue; - } - this.outputs.push(CoinEntry.fromOutput(output)); - } - - this.cleanup(); - - return this; -}; - -/** - * Instantiate a coins object from a transaction. - * @param {TX} tx - * @param {Number} height - * @returns {Coins} - */ - -Coins.fromTX = function fromTX(tx, height) { - return new Coins().fromTX(tx, height); -}; - -/** - * A coin entry is an object which defers - * parsing of a coin. Say there is a transaction - * with 100 outputs. When a block comes in, - * there may only be _one_ input in that entire - * block which redeems an output from that - * transaction. When parsing the Coins, there - * is no sense to get _all_ of them into their - * abstract form. A coin entry is just a - * pointer to that coin in the Coins buffer, as - * well as a size. Parsing and decompression - * is done only if that coin is being redeemed. - * @alias module:coins.CoinEntry - * @constructor - * @property {Number} offset - * @property {Number} size - * @property {Buffer} raw - * @property {Output|null} output - * @property {Boolean} spent - */ - -function CoinEntry() { - this.offset = 0; - this.size = 0; - this.raw = null; - this.output = null; - this.spent = false; -} - -/** - * Instantiate a reader at the correct offset. - * @private - * @returns {BufferReader} - */ - -CoinEntry.prototype.reader = function reader() { - assert(this.raw); - - const br = new BufferReader(this.raw); - br.offset = this.offset; - - return br; -}; - -/** - * Parse the deferred data and return a coin. - * @param {Coins} coins - * @param {Number} index - * @returns {Coin} - */ - -CoinEntry.prototype.toCoin = function toCoin(coins, index) { - const coin = new Coin(); - const output = this.toOutput(); - - // Load in all necessary properties - // from the parent Coins object. - coin.version = coins.version; - coin.coinbase = coins.coinbase; - coin.height = coins.height; - coin.hash = coins.hash; - coin.index = index; - coin.script = output.script; - coin.value = output.value; - - return coin; -}; - -/** - * Parse the deferred data and return an output. - * @returns {Output} - */ - -CoinEntry.prototype.toOutput = function toOutput() { - if (!this.output) { - this.output = new Output(); - decompress.output(this.output, this.reader()); - } - return this.output; -}; - -/** - * Calculate coin entry size. - * @returns {Number} - */ - -CoinEntry.prototype.getSize = function getSize() { - if (!this.raw) - return compress.size(this.output); - - return this.size; -}; - -/** - * Slice off the part of the buffer - * relevant to this particular coin. - */ - -CoinEntry.prototype.toWriter = function toWriter(bw) { - if (!this.raw) { - assert(this.output); - compress.output(this.output, bw); - return bw; - } - - // If we read this coin from the db and - // didn't use it, it's still in its - // compressed form. Just write it back - // as a buffer for speed. - bw.copy(this.raw, this.offset, this.offset + this.size); - - return bw; -}; - -/** - * Instantiate coin entry from reader. - * @param {BufferReader} br - * @returns {CoinEntry} - */ - -CoinEntry.fromReader = function fromReader(br) { - const entry = new CoinEntry(); - entry.offset = br.offset; - entry.size = decompress.skip(br); - entry.raw = br.data; - return entry; -}; - -/** - * Instantiate coin entry from output. - * @param {Output} output - * @returns {CoinEntry} - */ - -CoinEntry.fromOutput = function fromOutput(output) { - const entry = new CoinEntry(); - entry.output = output; - return entry; -}; - -/** - * Instantiate coin entry from coin. - * @param {Coin} coin - * @returns {CoinEntry} - */ - -CoinEntry.fromCoin = function fromCoin(coin) { - const entry = new CoinEntry(); - const output = new Output(); - output.value = coin.value; - output.script = coin.script; - entry.output = output; - return entry; -}; - -/* - * Expose - */ - -exports = Coins; -exports.Coins = Coins; -exports.CoinEntry = CoinEntry; - -module.exports = exports; diff --git a/migrate/coins/coinview.js b/migrate/coins/coinview.js deleted file mode 100644 index 8d27a08aa..000000000 --- a/migrate/coins/coinview.js +++ /dev/null @@ -1,477 +0,0 @@ -/*! - * coinview.js - coin viewpoint object for bcoin - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -/* eslint-disable */ - -'use strict'; - -const assert = require('assert'); -const {BufferMap} = require('buffer-map'); -const Coins = require('./coins'); -const UndoCoins = require('./undocoins'); -const CoinEntry = Coins.CoinEntry; - -/** - * Represents a coin viewpoint: - * a snapshot of {@link Coins} objects. - * @alias module:coins.CoinView - * @constructor - * @property {Object} map - * @property {UndoCoins} undo - */ - -function CoinView() { - if (!(this instanceof CoinView)) - return new CoinView(); - - this.map = new BufferMap(); - this.undo = new UndoCoins(); -} - -/** - * Get coins. - * @param {Hash} hash - * @returns {Coins} coins - */ - -CoinView.prototype.get = function get(hash) { - return this.map.get(hash); -}; - -/** - * Test whether the view has an entry. - * @param {Hash} hash - * @returns {Boolean} - */ - -CoinView.prototype.has = function has(hash) { - return this.map.has(hash); -}; - -/** - * Add coins to the collection. - * @param {Coins} coins - */ - -CoinView.prototype.add = function add(coins) { - this.map.set(coins.hash, coins); - return coins; -}; - -/** - * Remove coins from the collection. - * @param {Coins} coins - * @returns {Boolean} - */ - -CoinView.prototype.remove = function remove(hash) { - if (!this.map.has(hash)) - return false; - - this.map.delete(hash); - - return true; -}; - -/** - * Add a tx to the collection. - * @param {TX} tx - * @param {Number} height - */ - -CoinView.prototype.addTX = function addTX(tx, height) { - const coins = Coins.fromTX(tx, height); - return this.add(coins); -}; - -/** - * Remove a tx from the collection. - * @param {TX} tx - * @param {Number} height - */ - -CoinView.prototype.removeTX = function removeTX(tx, height) { - const coins = Coins.fromTX(tx, height); - coins.outputs.length = 0; - return this.add(coins); -}; - -/** - * Add a coin to the collection. - * @param {Coin} coin - */ - -CoinView.prototype.addCoin = function addCoin(coin) { - let coins = this.get(coin.hash); - - if (!coins) { - coins = new Coins(); - coins.hash = coin.hash; - coins.height = coin.height; - coins.coinbase = coin.coinbase; - this.add(coins); - } - - if (coin.script.isUnspendable()) - return; - - if (!coins.has(coin.index)) - coins.addCoin(coin); -}; - -/** - * Add an output to the collection. - * @param {Hash} hash - * @param {Number} index - * @param {Output} output - */ - -CoinView.prototype.addOutput = function addOutput(hash, index, output) { - let coins = this.get(hash); - - if (!coins) { - coins = new Coins(); - coins.hash = hash; - coins.height = -1; - coins.coinbase = false; - this.add(coins); - } - - if (output.script.isUnspendable()) - return; - - if (!coins.has(index)) - coins.addOutput(index, output); -}; - -/** - * Spend an output. - * @param {Hash} hash - * @param {Number} index - * @returns {Boolean} - */ - -CoinView.prototype.spendOutput = function spendOutput(hash, index) { - const coins = this.get(hash); - - if (!coins) - return false; - - return this.spendFrom(coins, index); -}; - -/** - * Remove an output. - * @param {Hash} hash - * @param {Number} index - * @returns {Boolean} - */ - -CoinView.prototype.removeOutput = function removeOutput(hash, index) { - const coins = this.get(hash); - - if (!coins) - return false; - - return coins.remove(index); -}; - -/** - * Spend a coin from coins object. - * @param {Coins} coins - * @param {Number} index - * @returns {Boolean} - */ - -CoinView.prototype.spendFrom = function spendFrom(coins, index) { - const entry = coins.spend(index); - - if (!entry) - return false; - - this.undo.push(entry); - - if (coins.isEmpty()) { - const undo = this.undo.top(); - undo.height = coins.height; - undo.coinbase = coins.coinbase; - undo.version = coins.version; - assert(undo.height !== -1); - } - - return true; -}; - -/** - * Get a single coin by input. - * @param {Input} input - * @returns {Coin} - */ - -CoinView.prototype.getCoin = function getCoin(input) { - const coins = this.get(input.prevout.hash); - - if (!coins) - return; - - return coins.getCoin(input.prevout.index); -}; - -/** - * Get a single output by input. - * @param {Input} input - * @returns {Output} - */ - -CoinView.prototype.getOutput = function getOutput(input) { - const coins = this.get(input.prevout.hash); - - if (!coins) - return; - - return coins.getOutput(input.prevout.index); -}; - -/** - * Get a single entry by input. - * @param {Input} input - * @returns {CoinEntry} - */ - -CoinView.prototype.getEntry = function getEntry(input) { - const coins = this.get(input.prevout.hash); - - if (!coins) - return; - - return coins.get(input.prevout.index); -}; - -/** - * Test whether the view has an entry by input. - * @param {Input} input - * @returns {Boolean} - */ - -CoinView.prototype.hasEntry = function hasEntry(input) { - const coins = this.get(input.prevout.hash); - - if (!coins) - return false; - - return coins.has(input.prevout.index); -}; - -/** - * Get coins height by input. - * @param {Input} input - * @returns {Number} - */ - -CoinView.prototype.getHeight = function getHeight(input) { - const coins = this.get(input.prevout.hash); - - if (!coins) - return -1; - - return coins.height; -}; - -/** - * Get coins coinbase flag by input. - * @param {Input} input - * @returns {Boolean} - */ - -CoinView.prototype.isCoinbase = function isCoinbase(input) { - const coins = this.get(input.prevout.hash); - - if (!coins) - return false; - - return coins.coinbase; -}; - -/** - * Retrieve coins from database. - * @method - * @param {ChainDB} db - * @param {TX} tx - * @returns {Promise} - Returns {@link Coins}. - */ - -CoinView.prototype.readCoins = async function readCoins(db, hash) { - let coins = this.map.get(hash); - - if (!coins) { - coins = await db.getCoins(hash); - - if (!coins) - return; - - this.map.set(hash, coins); - } - - return coins; -}; - -/** - * Read all input coins into unspent map. - * @method - * @param {ChainDB} db - * @param {TX} tx - * @returns {Promise} - Returns {Boolean}. - */ - -CoinView.prototype.ensureInputs = async function ensureInputs(db, tx) { - let found = true; - - for (const input of tx.inputs) { - if (!await this.readCoins(db, input.prevout.hash)) - found = false; - } - - return found; -}; - -/** - * Spend coins for transaction. - * @method - * @param {ChainDB} db - * @param {TX} tx - * @returns {Promise} - Returns {Boolean}. - */ - -CoinView.prototype.spendInputs = async function spendInputs(db, tx) { - for (const input of tx.inputs) { - const prevout = input.prevout; - const coins = await this.readCoins(db, prevout.hash); - - if (!coins) - return false; - - if (!this.spendFrom(coins, prevout.index)) - return false; - } - - return true; -}; - -/** - * Convert collection to an array. - * @returns {Coins[]} - */ - -CoinView.prototype.toArray = function toArray() { - const out = []; - - for (const coins of this.map.values()) - out.push(coins); - - return out; -}; - -/** - * Calculate serialization size. - * @returns {Number} - */ - -CoinView.prototype.getSize = function getSize(tx) { - let size = 0; - - size += tx.inputs.length; - - for (const input of tx.inputs) { - const entry = this.getEntry(input); - - if (!entry) - continue; - - size += entry.getSize(); - } - - return size; -}; - -/** - * Write coin data to buffer writer - * as it pertains to a transaction. - * @param {BufferWriter} bw - * @param {TX} tx - */ - -CoinView.prototype.toWriter = function toWriter(bw, tx) { - for (const input of tx.inputs) { - const prevout = input.prevout; - const coins = this.get(prevout.hash); - - if (!coins) { - bw.writeU8(0); - continue; - } - - const entry = coins.get(prevout.index); - - if (!entry) { - bw.writeU8(0); - continue; - } - - bw.writeU8(1); - entry.toWriter(bw); - } - - return bw; -}; - -/** - * Read serialized view data from a buffer - * reader as it pertains to a transaction. - * @private - * @param {BufferReader} br - * @param {TX} tx - */ - -CoinView.prototype.fromReader = function fromReader(br, tx) { - for (const input of tx.inputs) { - const prevout = input.prevout; - - if (br.readU8() === 0) - continue; - - let coins = this.get(prevout.hash); - - if (!coins) { - coins = new Coins(); - coins.hash = prevout.hash; - coins.coinbase = false; - this.add(coins); - } - - const entry = CoinEntry.fromReader(br); - coins.add(prevout.index, entry); - } - - return this; -}; - -/** - * Read serialized view data from a buffer - * reader as it pertains to a transaction. - * @param {BufferReader} br - * @param {TX} tx - * @returns {CoinView} - */ - -CoinView.fromReader = function fromReader(br, tx) { - return new CoinView().fromReader(br, tx); -}; - -/* - * Expose - */ - -module.exports = CoinView; diff --git a/migrate/coins/compress.js b/migrate/coins/compress.js deleted file mode 100644 index 311a5386b..000000000 --- a/migrate/coins/compress.js +++ /dev/null @@ -1,413 +0,0 @@ -/*! - * compress.js - coin compressor for bcoin - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -/* eslint-disable */ - -'use strict'; - -/** - * @module coins/compress - * @ignore - */ - -const assert = require('assert'); -const secp256k1 = require('../../lib/crypto/secp256k1'); -const encoding = require('../../lib/utils/encoding'); -const consensus = require('../../lib/protocol/consensus'); - -/* - * Constants - */ - -const COMPRESS_TYPES = 10; // Space for 4 extra. -const EMPTY_BUFFER = Buffer.alloc(0); - -/** - * Compress a script, write directly to the buffer. - * @param {Script} script - * @param {BufferWriter} bw - */ - -function compressScript(script, bw) { - // Attempt to compress the output scripts. - // We can _only_ ever compress them if - // they are serialized as minimaldata, as - // we need to recreate them when we read - // them. - - // P2PKH -> 0 | key-hash - // Saves 5 bytes. - if (script.isPubkeyhash(true)) { - const data = script.code[2].data; - bw.writeU8(0); - bw.writeBytes(data); - return bw; - } - - // P2SH -> 1 | script-hash - // Saves 3 bytes. - if (script.isScripthash()) { - const data = script.code[1].data; - bw.writeU8(1); - bw.writeBytes(data); - return bw; - } - - // P2PK -> 2-5 | compressed-key - // Only works if the key is valid. - // Saves up to 35 bytes. - if (script.isPubkey(true)) { - let data = script.code[0].data; - if (publicKeyVerify(data)) { - data = compressKey(data); - bw.writeBytes(data); - return bw; - } - } - - // Raw -> varlen + 10 | script - bw.writeVarint(script.raw.length + COMPRESS_TYPES); - bw.writeBytes(script.raw); - - return bw; -} - -/** - * Decompress a script from buffer reader. - * @param {Script} script - * @param {BufferReader} br - */ - -function decompressScript(script, br) { - let size, data; - - // Decompress the script. - switch (br.readU8()) { - case 0: - data = br.readBytes(20, true); - script.fromPubkeyhash(data); - break; - case 1: - data = br.readBytes(20, true); - script.fromScripthash(data); - break; - case 2: - case 3: - case 4: - case 5: - br.offset -= 1; - data = br.readBytes(33, true); - // Decompress the key. If this fails, - // we have database corruption! - data = decompressKey(data); - script.fromPubkey(data); - break; - default: - br.offset -= 1; - size = br.readVarint() - COMPRESS_TYPES; - if (size > consensus.MAX_SCRIPT_SIZE) { - // This violates consensus rules. - // We don't need to read it. - script.fromNulldata(EMPTY_BUFFER); - br.seek(size); - } else { - data = br.readBytes(size); - script.fromRaw(data); - } - break; - } - - return script; -} - -/** - * Calculate script size. - * @returns {Number} - */ - -function sizeScript(script) { - if (script.isPubkeyhash(true)) - return 21; - - if (script.isScripthash()) - return 21; - - if (script.isPubkey(true)) { - const data = script.code[0].data; - if (publicKeyVerify(data)) - return 33; - } - - let size = 0; - size += encoding.sizeVarint(script.raw.length + COMPRESS_TYPES); - size += script.raw.length; - - return size; -} - -/** - * Compress an output. - * @param {Output} output - * @param {BufferWriter} bw - */ - -function compressOutput(output, bw) { - bw.writeVarint(output.value); - compressScript(output.script, bw); - return bw; -} - -/** - * Decompress a script from buffer reader. - * @param {Output} output - * @param {BufferReader} br - */ - -function decompressOutput(output, br) { - output.value = br.readVarint(); - decompressScript(output.script, br); - return output; -} - -/** - * Calculate output size. - * @returns {Number} - */ - -function sizeOutput(output) { - let size = 0; - size += encoding.sizeVarint(output.value); - size += sizeScript(output.script); - return size; -} - -/** - * Compress an output. - * @param {Coin} coin - * @param {BufferWriter} bw - */ - -function compressCoin(coin, bw) { - bw.writeVarint(coin.value); - compressScript(coin.script, bw); - return bw; -} - -/** - * Decompress a script from buffer reader. - * @param {Coin} coin - * @param {BufferReader} br - */ - -function decompressCoin(coin, br) { - coin.value = br.readVarint(); - decompressScript(coin.script, br); - return coin; -} - -/** - * Skip past a compressed output. - * @param {BufferWriter} bw - * @returns {Number} - */ - -function skipOutput(br) { - const start = br.offset; - - // Skip past the value. - br.readVarint(); - - // Skip past the compressed scripts. - switch (br.readU8()) { - case 0: - case 1: - br.seek(20); - break; - case 2: - case 3: - case 4: - case 5: - br.seek(32); - break; - default: - br.offset -= 1; - br.seek(br.readVarint() - COMPRESS_TYPES); - break; - } - - return br.offset - start; -} - -/** - * Compress value using an exponent. Takes advantage of - * the fact that many bitcoin values are divisible by 10. - * @see https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go - * @param {Amount} value - * @returns {Number} - */ - -function compressValue(value) { - if (value === 0) - return 0; - - let exp = 0; - while (value % 10 === 0 && exp < 9) { - value /= 10; - exp++; - } - - if (exp < 9) { - const last = value % 10; - value = (value - last) / 10; - return 1 + 10 * (9 * value + last - 1) + exp; - } - - return 10 + 10 * (value - 1); -} - -/** - * Decompress value. - * @param {Number} value - Compressed value. - * @returns {Amount} value - */ - -function decompressValue(value) { - if (value === 0) - return 0; - - value--; - - let exp = value % 10; - value = (value - exp) / 10; - - let n; - if (exp < 9) { - const last = value % 9; - value = (value - last) / 9; - n = value * 10 + last + 1; - } else { - n = value + 1; - } - - while (exp > 0) { - n *= 10; - exp--; - } - - return n; -} - -/** - * Verify a public key (no hybrid keys allowed). - * @param {Buffer} key - * @returns {Boolean} - */ - -function publicKeyVerify(key) { - if (key.length === 0) - return false; - - switch (key[0]) { - case 0x02: - case 0x03: - return key.length === 33; - case 0x04: - if (key.length !== 65) - return false; - - return secp256k1.publicKeyVerify(key); - default: - return false; - } -} - -/** - * Compress a public key to coins compression format. - * @param {Buffer} key - * @returns {Buffer} - */ - -function compressKey(key) { - let out; - - switch (key[0]) { - case 0x02: - case 0x03: - // Key is already compressed. - out = key; - break; - case 0x04: - // Compress the key normally. - out = secp256k1.publicKeyConvert(key, true); - // Store the oddness. - // Pseudo-hybrid format. - out[0] = 0x04 | (key[64] & 0x01); - break; - default: - throw new Error('Bad point format.'); - } - - assert(out.length === 33); - - return out; -} - -/** - * Decompress a public key from the coins compression format. - * @param {Buffer} key - * @returns {Buffer} - */ - -function decompressKey(key) { - const format = key[0]; - - assert(key.length === 33); - - switch (format) { - case 0x02: - case 0x03: - return key; - case 0x04: - key[0] = 0x02; - break; - case 0x05: - key[0] = 0x03; - break; - default: - throw new Error('Bad point format.'); - } - - // Decompress the key. - const out = secp256k1.publicKeyConvert(key, false); - - // Reset the first byte so as not to - // mutate the original buffer. - key[0] = format; - - return out; -} - -/* - * Expose - */ - -exports.compress = { - output: compressOutput, - coin: compressCoin, - size: sizeOutput, - script: compressScript, - value: compressValue, - key: compressKey -}; - -exports.decompress = { - output: decompressOutput, - coin: decompressCoin, - skip: skipOutput, - script: decompressScript, - value: decompressValue, - key: decompressKey -}; diff --git a/migrate/coins/index.js b/migrate/coins/index.js deleted file mode 100644 index 514c09db5..000000000 --- a/migrate/coins/index.js +++ /dev/null @@ -1,16 +0,0 @@ -/*! - * coins/index.js - utxo management for bcoin - * Copyright (c) 2016-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -/** - * @module coins - */ - -exports.Coins = require('../../lib/coins/coins'); -exports.CoinView = require('../../lib/coins/coinview'); -exports.compress = require('../../lib/coins/compress'); -exports.UndoCoins = require('../../lib/coins/undocoins'); diff --git a/migrate/coins/undocoins.js b/migrate/coins/undocoins.js deleted file mode 100644 index 0e9acedde..000000000 --- a/migrate/coins/undocoins.js +++ /dev/null @@ -1,338 +0,0 @@ -/*! - * undocoins.js - undocoins object for bcoin - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -/* eslint-disable */ - -'use strict'; - -const assert = require('assert'); -const BufferReader = require('../../lib/utils/reader'); -const StaticWriter = require('../../lib/utils/staticwriter'); -const encoding = require('../../lib/utils/encoding'); -const Output = require('../../lib/primitives/output'); -const Coins = require('./coins'); -const compressor = require('./compress'); -const compress = compressor.compress; -const decompress = compressor.decompress; - -/** - * UndoCoins - * Coins need to be resurrected from somewhere - * during a reorg. The undo coins store all - * spent coins in a single record per block - * (in a compressed format). - * @alias module:coins.UndoCoins - * @constructor - * @property {UndoCoin[]} items - */ - -function UndoCoins() { - if (!(this instanceof UndoCoins)) - return new UndoCoins(); - - this.items = []; -} - -/** - * Push coin entry onto undo coin array. - * @param {CoinEntry} - */ - -UndoCoins.prototype.push = function push(entry) { - const undo = new UndoCoin(); - undo.entry = entry; - this.items.push(undo); -}; - -/** - * Calculate undo coins size. - * @returns {Number} - */ - -UndoCoins.prototype.getSize = function getSize() { - let size = 0; - - size += 4; - - for (const coin of this.items) - size += coin.getSize(); - - return size; -}; - -/** - * Serialize all undo coins. - * @returns {Buffer} - */ - -UndoCoins.prototype.toRaw = function toRaw() { - const size = this.getSize(); - const bw = new StaticWriter(size); - - bw.writeU32(this.items.length); - - for (const coin of this.items) - coin.toWriter(bw); - - return bw.render(); -}; - -/** - * Inject properties from serialized data. - * @private - * @param {Buffer} data - * @returns {UndoCoins} - */ - -UndoCoins.prototype.fromRaw = function fromRaw(data) { - const br = new BufferReader(data); - const count = br.readU32(); - - for (let i = 0; i < count; i++) - this.items.push(UndoCoin.fromReader(br)); - - return this; -}; - -/** - * Instantiate undo coins from serialized data. - * @param {Buffer} data - * @returns {UndoCoins} - */ - -UndoCoins.fromRaw = function fromRaw(data) { - return new UndoCoins().fromRaw(data); -}; - -/** - * Test whether the undo coins have any members. - * @returns {Boolean} - */ - -UndoCoins.prototype.isEmpty = function isEmpty() { - return this.items.length === 0; -}; - -/** - * Render the undo coins. - * @returns {Buffer} - */ - -UndoCoins.prototype.commit = function commit() { - const raw = this.toRaw(); - this.items.length = 0; - return raw; -}; - -/** - * Retrieve the last undo coin. - * @returns {UndoCoin} - */ - -UndoCoins.prototype.top = function top() { - return this.items[this.items.length - 1]; -}; - -/** - * Re-apply undo coins to a view, effectively unspending them. - * @param {CoinView} view - * @param {Outpoint} outpoint - */ - -UndoCoins.prototype.apply = function apply(view, outpoint) { - const undo = this.items.pop(); - const hash = outpoint.hash; - const index = outpoint.index; - let coins; - - assert(undo); - - if (undo.height !== -1) { - coins = new Coins(); - - assert(!view.map.has(hash)); - view.map.set(hash, coins); - - coins.hash = hash; - coins.coinbase = undo.coinbase; - coins.height = undo.height; - coins.version = undo.version; - } else { - coins = view.map.get(hash); - assert(coins); - } - - coins.addOutput(index, undo.toOutput()); - - assert(coins.has(index)); -}; - -/** - * UndoCoin - * @alias module:coins.UndoCoin - * @constructor - * @property {CoinEntry|null} entry - * @property {Output|null} output - * @property {Number} version - * @property {Number} height - * @property {Boolean} coinbase - */ - -function UndoCoin() { - this.entry = null; - this.output = null; - this.version = -1; - this.height = -1; - this.coinbase = false; -} - -/** - * Convert undo coin to an output. - * @returns {Output} - */ - -UndoCoin.prototype.toOutput = function toOutput() { - if (!this.output) { - assert(this.entry); - return this.entry.toOutput(); - } - return this.output; -}; - -/** - * Calculate undo coin size. - * @returns {Number} - */ - -UndoCoin.prototype.getSize = function getSize() { - let height = this.height; - let size = 0; - - if (height === -1) - height = 0; - - size += encoding.sizeVarint(height * 2 + (this.coinbase ? 1 : 0)); - - if (this.height !== -1) - size += encoding.sizeVarint(this.version); - - if (this.entry) { - // Cached from spend. - size += this.entry.getSize(); - } else { - size += compress.size(this.output); - } - - return size; -}; - -/** - * Write the undo coin to a buffer writer. - * @param {BufferWriter} bw - */ - -UndoCoin.prototype.toWriter = function toWriter(bw) { - let height = this.height; - - assert(height !== 0); - - if (height === -1) - height = 0; - - bw.writeVarint(height * 2 + (this.coinbase ? 1 : 0)); - - if (this.height !== -1) { - assert(this.version !== -1); - bw.writeVarint(this.version); - } - - if (this.entry) { - // Cached from spend. - this.entry.toWriter(bw); - } else { - compress.output(this.output, bw); - } - - return bw; -}; - -/** - * Serialize the undo coin. - * @returns {Buffer} - */ - -UndoCoin.prototype.toRaw = function toRaw() { - const size = this.getSize(); - return this.toWriter(new StaticWriter(size)).render(); -}; - -/** - * Inject properties from buffer reader. - * @private - * @param {BufferReader} br - * @returns {UndoCoin} - */ - -UndoCoin.prototype.fromReader = function fromReader(br) { - const code = br.readVarint(); - - this.output = new Output(); - - this.height = code / 2 | 0; - - if (this.height === 0) - this.height = -1; - - this.coinbase = (code & 1) !== 0; - - if (this.height !== -1) - this.version = br.readVarint(); - - decompress.output(this.output, br); - - return this; -}; - -/** - * Inject properties from serialized data. - * @private - * @param {Buffer} data - * @returns {UndoCoin} - */ - -UndoCoin.prototype.fromRaw = function fromRaw(data) { - return this.fromReader(new BufferReader(data)); -}; - -/** - * Instantiate undo coin from serialized data. - * @param {Buffer} data - * @returns {UndoCoin} - */ - -UndoCoin.fromReader = function fromReader(br) { - return new UndoCoin().fromReader(br); -}; - -/** - * Instantiate undo coin from serialized data. - * @param {Buffer} data - * @returns {UndoCoin} - */ - -UndoCoin.fromRaw = function fromRaw(data) { - return new UndoCoin().fromRaw(data); -}; - -/* - * Expose - */ - -exports = UndoCoins; -exports.UndoCoins = UndoCoins; -exports.UndoCoin = UndoCoin; - -module.exports = exports; diff --git a/migrate/index.js b/migrate/index.js new file mode 100644 index 000000000..e69de29bb diff --git a/migrate/latest b/migrate/latest index 519479b85..e8df934fa 100755 --- a/migrate/latest +++ b/migrate/latest @@ -1,49 +1,6 @@ #!/usr/bin/env node -const assert = require('assert'); -const fs = require('fs'); -const cp = require('child_process'); -const res = require('path').resolve; -const {argv} = process; +'use strict'; -if (argv.length < 3) { - console.error('Usage: $ ./migrate/latest [bcoin-prefix]'); - console.error('Example: $ ./migrate/latest ~/.bcoin'); - process.exit(1); - return; -} - -function mv(from, to) { - try { - fs.renameSync(from, to); - } catch (e) { - console.error(e.message); - } -} - -function exec(file, ...args) { - try { - const result = cp.spawnSync(file, args, { - stdio: 'inherit', - env: process.env, - maxBuffer: -1 >>> 0, - windowsHide: true - }); - if (result.error) - console.error(result.error.message); - } catch (e) { - console.error(e.message); - } -} - -const node = argv[0]; -const prefix = argv[2]; - -mv(res(prefix, 'chain.ldb'), res(prefix, 'chain')); -mv(res(prefix, 'spvchain.ldb'), res(prefix, 'spvchain')); -mv(res(prefix, 'mempool.ldb'), res(prefix, 'mempool')); -mv(res(prefix, 'walletdb.ldb'), res(prefix, 'wallet')); - -exec(node, res(__dirname, 'chaindb3to4.js'), res(prefix, 'chain')); -exec(node, res(__dirname, 'chaindb3to4.js'), res(prefix, 'spvchain')); -exec(node, res(__dirname, 'walletdb6to7.js'), res(prefix, 'wallet')); +console.error('No migrations available.'); +process.exit(1); diff --git a/migrate/walletdb5to6.js b/migrate/walletdb5to6.js deleted file mode 100644 index ac584fb52..000000000 --- a/migrate/walletdb5to6.js +++ /dev/null @@ -1,264 +0,0 @@ -'use strict'; - -const assert = require('assert'); -const bdb = require('bdb'); -const bio = require('bufio'); - -assert(process.argv.length > 2, 'Please pass in a database path.'); - -let batch; - -const db = bdb.create({ - location: process.argv[2], - compression: true, - cacheSize: 32 << 20, - createIfMissing: false -}); - -async function updateVersion() { - const bak = `${process.env.HOME}/wallet-bak-${Date.now()}`; - - console.log('Checking version.'); - - const raw = await db.get('V'); - assert(raw, 'No version.'); - - const version = raw.readUInt32LE(0, true); - - if (version !== 5) - throw Error(`DB is version ${version}.`); - - console.log('Backing up DB to: %s.', bak); - - await db.backup(bak); - - const data = Buffer.allocUnsafe(4); - data.writeUInt32LE(6, 0, true); - batch.put('V', data); -} - -async function wipeTXDB() { - let total = 0; - - const keys = await db.keys(); - - for (let i = 0; i < keys.length; i++) { - const key = keys[i]; - switch (key[0]) { - case 0x62: // b - case 0x63: // c - case 0x65: // e - case 0x74: // t - case 0x6f: // o - case 0x68: // h - batch.del(key); - total += 1; - break; - } - } - - batch.del(Buffer.from([0x52])); // R - - console.log('Wiped %d txdb records.', total); -} - -async function patchAccounts() { - const items = await db.range({ - gt: Buffer.from([0x61]), // a - lt: Buffer.from([0x62]) - }); - - for (let i = 0; i < items.length; i++) { - const item = items[i]; - const wid = item.key.readUInt32BE(1, true); - const index = item.key.readUInt32BE(5, true); - const account = accountFromRaw(item.value); - console.log('a[%d][%d] -> lookahead=%d', wid, index, account.lookahead); - batch.put(item.key, accountToRaw(account)); - console.log('n[%d][%d] -> %s', wid, index, account.name); - batch.put(n(wid, index), Buffer.from(account.name, 'ascii')); - } -} - -async function indexPaths() { - const items = await db.range({ - gt: Buffer.from([0x50]), // P - lt: Buffer.from([0x51]) - }); - - for (let i = 0; i < items.length; i++) { - const item = items[i]; - const wid = item.key.readUInt32BE(1, true); - const hash = item.key.slice(5); - const index = item.value.readUInt32LE(0, true); - console.log('r[%d][%d][%s] -> NUL', wid, index, hash); - batch.put(r(wid, index, hash), Buffer.from([0])); - } -} - -async function patchPathMaps() { - const items = await db.range({ - gt: Buffer.from([0x70]), // p - lt: Buffer.from([0x71]) - }); - - for (let i = 0; i < items.length; i++) { - const item = items[i]; - const hash = item.key.slice(1); - const wids = parseWallets(item.value); - console.log('p[%s] -> u32(%d)', hash, wids.length); - batch.put(item.key, serializeWallets(wids)); - } -} - -function parseWallets(data) { - const p = bio.read(data); - const wids = []; - - while (p.left()) - wids.push(p.readU32()); - - return wids; -} - -function serializeWallets(wids) { - const p = bio.write(); - - p.writeU32(wids.length); - - for (let i = 0; i < wids.length; i++) { - const wid = wids[i]; - p.writeU32(wid); - } - - return p.render(); -} - -function accountToRaw(account) { - const p = bio.write(); - - p.writeVarString(account.name, 'ascii'); - p.writeU8(account.initialized ? 1 : 0); - p.writeU8(account.witness ? 1 : 0); - p.writeU8(account.type); - p.writeU8(account.m); - p.writeU8(account.n); - p.writeU32(account.accountIndex); - p.writeU32(account.receiveDepth); - p.writeU32(account.changeDepth); - p.writeU32(account.nestedDepth); - p.writeU8(account.lookahead); - p.writeBytes(account.accountKey); - p.writeU8(account.keys.length); - - for (let i = 0; i < account.keys.length; i++) { - const key = account.keys[i]; - p.writeBytes(key); - } - - return p.render(); -}; - -function accountFromRaw(data) { - const account = {}; - const p = bio.read(data); - - account.name = p.readVarString('ascii'); - account.initialized = p.readU8() === 1; - account.witness = p.readU8() === 1; - account.type = p.readU8(); - account.m = p.readU8(); - account.n = p.readU8(); - account.accountIndex = p.readU32(); - account.receiveDepth = p.readU32(); - account.changeDepth = p.readU32(); - account.nestedDepth = p.readU32(); - account.lookahead = 10; - account.accountKey = p.readBytes(82); - account.keys = []; - - const count = p.readU8(); - - for (let i = 0; i < count; i++) { - const key = p.readBytes(82); - account.keys.push(key); - } - - return account; -} - -function n(wid, index) { - const key = Buffer.allocUnsafe(9); - key[0] = 0x6e; - key.writeUInt32BE(wid, 1, true); - key.writeUInt32BE(index, 5, true); - return key; -} - -function r(wid, index, hash) { - const key = Buffer.allocUnsafe(1 + 4 + 4 + (hash.length / 2)); - key[0] = 0x72; - key.writeUInt32BE(wid, 1, true); - key.writeUInt32BE(index, 5, true); - hash.copy(key, 9); - return key; -} - -async function updateLookahead() { - const WalletDB = require('../lib/wallet/walletdb'); - - const db = new WalletDB({ - network: process.argv[3], - db: 'leveldb', - location: process.argv[2], - witness: false, - useCheckpoints: false, - maxFiles: 64, - resolution: false, - verify: false - }); - - await db.open(); - - for (let i = 1; i < db.depth; i++) { - const wallet = await db.get(i); - assert(wallet); - console.log('Updating wallet lookahead: %s', wallet.id); - for (let j = 0; j < wallet.accountDepth; j++) - await wallet.setLookahead(j, 20); - } - - await db.close(); -} - -updateLookahead; - -async function unstate() { - await db.open(); - batch = db.batch(); - await wipeTXDB(); - await batch.write(); - await db.close(); -} - -(async () => { - await db.open(); - batch = db.batch(); - console.log('Opened %s.', process.argv[2]); - await updateVersion(); - await wipeTXDB(); - await patchAccounts(); - await indexPaths(); - await patchPathMaps(); - await batch.write(); - await db.close(); - - // Do not use: - // await updateLookahead(); - await unstate(); -})().then(() => { - console.log('Migration complete.'); - console.log('Rescan is required...'); - console.log('Start bcoin with `--start-height=[wallet-creation-height]`.'); - process.exit(0); -}); diff --git a/migrate/walletdb6to7.js b/migrate/walletdb6to7.js deleted file mode 100644 index 1a7e45773..000000000 --- a/migrate/walletdb6to7.js +++ /dev/null @@ -1,930 +0,0 @@ -'use strict'; - -const assert = require('assert'); -const bdb = require('bdb'); -const bio = require('bufio'); -const layouts = require('../lib/wallet/layout'); -const TX = require('../lib/primitives/tx'); -const Coin = require('../lib/primitives/coin'); -const layout = layouts.wdb; -const tlayout = layouts.txdb; - -// changes: -// db version record -// headers - all headers -// block map - just a map -// input map - only on unconfirmed -// marked byte - no longer a soft fork -// coin `own` flag - no longer a soft fork -// tx map - for unconfirmed -// balances - index account balances -// wallet - serialization -// account - serialization -// path - serialization -// depth - counter record -// hash/ascii - variable length key prefixes - -let parent = null; - -assert(process.argv.length > 2, 'Please pass in a database path.'); - -const db = bdb.create({ - location: process.argv[2], - memory: false, - compression: true, - cacheSize: 32 << 20, - createIfMissing: false -}); - -async function updateVersion() { - const bak = `${process.env.HOME}/wallet-bak-${Date.now()}`; - - console.log('Checking version.'); - - const data = await db.get(layout.V.encode()); - assert(data, 'No version.'); - - const ver = data.readUInt32LE(0, true); - - if (ver !== 6) - throw Error(`DB is version ${ver}.`); - - console.log('Backing up DB to: %s.', bak); - console.log('Updating version to %d.', ver + 1); - - await db.backup(bak); - - const buf = Buffer.allocUnsafe(6 + 4); - buf.write('wallet', 0, 'ascii'); - buf.writeUInt32LE(7, 6, true); - - parent.put(layout.V.encode(), buf); -} - -async function migrateKeys(id, from, to) { - console.log('Migrating keys for %s.', String.fromCharCode(id)); - - const iter = db.iterator({ - gt: Buffer.from([id]), - lt: Buffer.from([id + 1]), - keys: true, - values: true - }); - - let batch = db.batch(); - let total = 0; - let items = 0; - - await iter.each(async (key, value) => { - batch.put(to.encode(...from(key)), value); - batch.del(key); - - total += (key.length + 80) * 2; - total += value.length + 80; - items += 1; - - if (total >= (128 << 20)) { - await batch.write(); - batch = db.batch(); - total = 0; - } - }); - - console.log('Migrated %d keys for %s.', items, String.fromCharCode(id)); - - return batch.write(); -} - -async function updateKeys() { - console.log('Updating keys...'); - - await migrateKeys(0x70, parsep, layout.p); // p - await migrateKeys(0x50, parseP, layout.P); // P - await migrateKeys(0x72, parser, layout.r); // r - await migrateKeys(0x6c, parsel, layout.l); // l - await migrateKeys(0x69, parsei, layout.i); // i - - console.log('Updated keys.'); -} - -async function updateState() { - const raw = await db.get(layout.R.encode()); - - if (!raw) - return; - - console.log('Updating state...'); - - if (raw.length === 40) { - const bw = bio.write(41); - bw.writeBytes(raw); - bw.writeU8(1); - parent.put(layout.R.encode(), bw.render()); - console.log('State updated.'); - } - - const depth = await getDepth(); - - const buf = Buffer.allocUnsafe(4); - buf.writeUInt32LE(depth, 0, true); - - parent.put(layout.D.encode(), buf); -} - -async function updateBlockMap() { - const iter = db.iterator({ - gte: layout.b.min(), - lte: layout.b.max(), - keys: true, - values: true - }); - - console.log('Updating block map...'); - - let total = 0; - - await iter.each((key, value) => { - const [height] = layout.b.decode(key); - const block = BlockMapRecord.fromRaw(height, value); - const map = new Set(); - - for (const tx of block.txs.values()) { - for (const wid of tx.wids) - map.add(wid); - } - - const bw = bio.write(sizeMap(map)); - serializeMap(bw, map); - - parent.put(key, bw.render()); - - total += 1; - }); - - console.log('Updated %d block maps.', total); -} - -async function updateTXDB() { - const wids = await db.keys({ - gte: layout.w.min(), - lte: layout.w.max(), - keys: true, - parse: key => layout.w.decode(key)[0] - }); - - console.log('Updating wallets...'); - - let total = 0; - - for (const wid of wids) { - const bucket = db.bucket(layout.t.encode(wid)); - const batch = bucket.wrap(parent); - - await updateInputs(wid, bucket, batch); - await updateCoins(wid, bucket, batch); - await updateTX(wid, bucket, batch); - await updateWalletBalance(wid, bucket, batch); - await updateAccountBalances(wid, bucket, batch); - await updateWallet(wid); - - total += 1; - } - - console.log('Updated %d wallets.', total); -} - -async function updateInputs(wid, bucket, batch) { - const iter = bucket.iterator({ - gte: tlayout.h.min(), - lte: tlayout.h.max(), - keys: true - }); - - console.log('Updating inputs for %d...', wid); - - let total = 0; - - await iter.each(async (key, value) => { - const [, hash] = tlayout.h.decode(key); - const data = await bucket.get(tlayout.t.encode(hash)); - assert(data); - const tx = TX.fromRaw(data); - - for (const {prevout} of tx.inputs) { - const {hash, index} = prevout; - batch.del(tlayout.s.encode(hash, index)); - total += 1; - } - }); - - console.log('Updated %d inputs for %d.', total, wid); -} - -async function updateCoins(wid, bucket, batch) { - const iter = bucket.iterator({ - gte: tlayout.c.min(), - lte: tlayout.c.max(), - keys: true, - values: true - }); - - console.log('Updating coins for %d...', wid); - - let total = 0; - - await iter.each((key, value) => { - const br = bio.read(value, true); - - Coin.fromReader(br); - br.readU8(); - - if (br.left() === 0) { - const bw = bio.write(value.length + 1); - bw.writeBytes(value); - bw.writeU8(0); - batch.put(key, bw.render()); - total += 1; - } - }); - - console.log('Updated %d coins for %d.', total, wid); -} - -async function updateTX(wid, bucket, batch) { - const iter = bucket.iterator({ - gte: tlayout.p.min(), - lte: tlayout.p.max(), - keys: true - }); - - console.log('Adding TX maps for %d...', wid); - - let total = 0; - - await iter.each(async (key, value) => { - const [hash] = tlayout.p.decode(key); - const raw = await db.get(layout.T.encode(hash)); - - let map = null; - - if (!raw) { - map = new Set(); - } else { - const br = bio.read(raw, true); - map = parseMap(br); - } - - map.add(wid); - - const bw = bio.write(sizeMap(map)); - serializeMap(bw, map); - batch.put(layout.T.encode(hash), bw.render()); - - total += 1; - }); - - console.log('Added %d TX maps for %d.', total, wid); -} - -async function updateWalletBalance(wid, bucket, batch) { - const bal = newBalance(); - - const keys = await bucket.keys({ - gte: tlayout.t.min(), - lte: tlayout.t.max(), - keys: true - }); - - bal.tx = keys.length; - - const iter = bucket.iterator({ - gte: tlayout.c.min(), - lte: tlayout.c.max(), - keys: true, - values: true - }); - - console.log('Updating wallet balance for %d...', wid); - - await iter.each((key, value) => { - const br = bio.read(value, true); - const coin = Coin.fromReader(br); - const spent = br.readU8() === 1; - - bal.coin += 1; - - if (coin.height !== -1) - bal.confirmed += coin.value; - - if (!spent) - bal.unconfirmed += coin.value; - }); - - batch.put(tlayout.R.encode(), serializeBalance(bal)); - - console.log('Updated wallet balance for %d.', wid); -} - -async function updateAccountBalances(wid, bucket, batch) { - const raw = await db.get(layout.w.encode(wid)); - assert(raw); - - const br = bio.read(raw, true); - - br.readU32(); - br.readU32(); - br.readVarString('ascii'); - br.readU8(); - br.readU8(); - - const depth = br.readU32(); - - console.log('Updating account balances for %d...', wid); - - for (let acct = 0; acct < depth; acct++) - await updateAccountBalance(wid, acct, bucket, batch); - - console.log('Updated %d account balances for %d.', depth, wid); -} - -async function updateAccountBalance(wid, acct, bucket, batch) { - const bal = newBalance(); - - const keys = await bucket.keys({ - gte: tlayout.T.min(acct), - lte: tlayout.T.max(acct), - keys: true - }); - - bal.tx = keys.length; - - const iter = bucket.iterator({ - gte: tlayout.C.min(acct), - lte: tlayout.C.max(acct), - keys: true - }); - - console.log('Updating account balance for %d/%d...', wid, acct); - - await iter.each(async (key, value) => { - const [, hash, index] = tlayout.C.decode(key); - const raw = await bucket.get(tlayout.c.encode(hash, index)); - assert(raw); - const br = bio.read(raw, true); - const coin = Coin.fromReader(br); - const spent = br.readU8() === 1; - - bal.coin += 1; - - if (coin.height !== -1) - bal.confirmed += coin.value; - - if (!spent) - bal.unconfirmed += coin.value; - }); - - batch.put(tlayout.r.encode(acct), serializeBalance(bal)); - - console.log('Updated account balance for %d/%d.', wid, acct); -} - -async function updateWallet(wid) { - const raw = await db.get(layout.w.encode(wid)); - assert(raw); - - console.log('Updating wallet: %d.', wid); - - const br = bio.read(raw, true); - - br.readU32(); // Skip network. - br.readU32(); // Skip wid. - const id = br.readVarString('ascii'); - br.readU8(); // Skip initialized. - const watchOnly = br.readU8() === 1; - const accountDepth = br.readU32(); - const token = br.readBytes(32); - const tokenDepth = br.readU32(); - - // We want to get the key - // _out of_ varint serialization. - let key = br.readVarBytes(); - - const kr = bio.read(key, true); - - // Unencrypted? - if (kr.readU8() === 0) { - const bw = bio.write(); - bw.writeU8(0); - - // Skip useless varint. - kr.readVarint(); - - // Skip HD key params. - kr.seek(13); - - // Read/write chain code. - bw.writeBytes(kr.readBytes(32)); - - // Skip zero byte. - assert(kr.readU8() === 0); - - // Read/write private key. - bw.writeBytes(kr.readBytes(32)); - - // Skip checksum. - kr.seek(4); - - // Include mnemonic. - if (kr.readU8() === 1) { - bw.writeU8(1); - const bits = kr.readU16(); - assert(bits % 32 === 0); - const lang = kr.readU8(); - const entropy = kr.readBytes(bits / 8); - - bw.writeU16(bits); - bw.writeU8(lang); - bw.writeBytes(entropy); - } else { - bw.writeU8(0); - } - - key = bw.render(); - } - - let flags = 0; - - if (watchOnly) - flags |= 1; - - // Concatenate wallet with key. - const bw = bio.write(); - bw.writeU8(flags); - bw.writeU32(accountDepth); - bw.writeBytes(token); - bw.writeU32(tokenDepth); - bw.writeBytes(key); - - parent.put(layout.w.encode(wid), bw.render()); - parent.put(layout.W.encode(wid), fromString(id)); - - console.log('Updating accounts for %d...', wid); - - for (let acct = 0; acct < accountDepth; acct++) - await updateAccount(wid, acct); - - console.log('Updated %d accounts for %d.', accountDepth, wid); - - console.log('Updated wallet: %d.', wid); -} - -async function updateAccount(wid, acct) { - const raw = await db.get(layout.a.encode(wid, acct)); - assert(raw); - - console.log('Updating account: %d/%d...', wid, acct); - - const br = bio.read(raw, true); - - const name = br.readVarString('ascii'); - const initialized = br.readU8() === 1; - const witness = br.readU8() === 1; - const type = br.readU8(); - const m = br.readU8(); - const n = br.readU8(); - br.readU32(); // accountIndex - const receiveDepth = br.readU32(); - const changeDepth = br.readU32(); - const nestedDepth = br.readU32(); - const lookahead = br.readU8(); - const accountKey = { - network: br.readU32BE(), - depth: br.readU8(), - parentFingerPrint: br.readU32BE(), - childIndex: br.readU32BE(), - chainCode: br.readBytes(32), - publicKey: br.readBytes(33), - checksum: br.readU32() - }; - - const count = br.readU8(); - const keys = []; - - for (let i = 0; i < count; i++) { - const key = { - network: br.readU32BE(), - depth: br.readU8(), - parentFingerPrint: br.readU32BE(), - childIndex: br.readU32BE(), - chainCode: br.readBytes(32), - publicKey: br.readBytes(33), - checksum: br.readU32() - }; - keys.push(key); - } - - const bw = bio.write(); - - let flags = 0; - - if (initialized) - flags |= 1; - - if (witness) - flags |= 2; - - bw.writeU8(flags); - bw.writeU8(type); - bw.writeU8(m); - bw.writeU8(n); - bw.writeU32(receiveDepth); - bw.writeU32(changeDepth); - bw.writeU32(nestedDepth); - bw.writeU8(lookahead); - - bw.writeU8(accountKey.depth); - bw.writeU32BE(accountKey.parentFingerPrint); - bw.writeU32BE(accountKey.childIndex); - bw.writeBytes(accountKey.chainCode); - bw.writeBytes(accountKey.publicKey); - - bw.writeU8(keys.length); - - for (const key of keys) { - bw.writeU8(key.depth); - bw.writeU32BE(key.parentFingerPrint); - bw.writeU32BE(key.childIndex); - bw.writeBytes(key.chainCode); - bw.writeBytes(key.publicKey); - } - - parent.put(layout.a.encode(wid, acct), bw.render()); - parent.put(layout.n.encode(wid, acct), fromString(name)); - - console.log('Updated account: %d/%d.', wid, acct); -} - -async function updatePaths() { - const iter = db.iterator({ - gte: layout.P.min(), - lte: layout.P.max(), - keys: true, - values: true - }); - - console.log('Updating paths....'); - - let total = 0; - - await iter.each((key, value) => { - const br = bio.read(value, true); - - const account = br.readU32(); - const keyType = br.readU8(); - - let branch = -1; - let index = -1; - let encrypted = false; - let data = null; - - switch (keyType) { - case 0: - branch = br.readU32(); - index = br.readU32(); - break; - case 1: - encrypted = br.readU8() === 1; - data = br.readVarBytes(); - break; - case 2: - break; - default: - assert(false); - break; - } - - let version = br.readI8(); - - let type = br.readU8(); - - if (type === 129 || type === 130) - type = 4; - - type -= 2; - - const bw = bio.write(); - - bw.writeU32(account); - bw.writeU8(keyType); - - if (version === -1) - version = 0x1f; - - const flags = (version << 3) | type; - - bw.writeU8(flags); - - switch (keyType) { - case 0: - assert(!data); - assert(index !== -1); - bw.writeU32(branch); - bw.writeU32(index); - break; - case 1: - assert(data); - assert(index === -1); - bw.writeU8(encrypted ? 1 : 0); - bw.writeVarBytes(data); - break; - case 2: - assert(!data); - assert(index === -1); - break; - default: - assert(false); - break; - } - - parent.put(key, bw.render()); - - total += 1; - }); - - console.log('Updated %d paths.', total); -} - -async function getDepth() { - const iter = db.iterator({ - gte: layout.w.min(), - lte: layout.w.max(), - reverse: true, - limit: 1 - }); - - if (!await iter.next()) - return 1; - - const {key} = iter; - - await iter.end(); - - const [depth] = layout.w.decode(key); - - return depth + 1; -} - -/* - * Old Records - */ - -class BlockMapRecord { - constructor(height) { - this.height = height != null ? height : -1; - this.txs = new Map(); - } - - fromRaw(data) { - const br = bio.read(data); - const count = br.readU32(); - - for (let i = 0; i < count; i++) { - const hash = br.readHash(); - const tx = TXMapRecord.fromReader(hash, br); - this.txs.set(tx.hash, tx); - } - - return this; - } - - static fromRaw(height, data) { - return new BlockMapRecord(height).fromRaw(data); - } - - getSize() { - let size = 0; - - size += 4; - - for (const tx of this.txs.values()) { - size += 32; - size += tx.getSize(); - } - - return size; - } - - toRaw() { - const size = this.getSize(); - const bw = bio.write(size); - - bw.writeU32(this.txs.size); - - for (const [hash, tx] of this.txs) { - bw.writeHash(hash); - tx.toWriter(bw); - } - - return bw.render(); - } - - add(hash, wid) { - let tx = this.txs.get(hash); - - if (!tx) { - tx = new TXMapRecord(hash); - this.txs.set(hash, tx); - } - - return tx.add(wid); - } - - remove(hash, wid) { - const tx = this.txs.get(hash); - - if (!tx) - return false; - - if (!tx.remove(wid)) - return false; - - if (tx.wids.size === 0) - this.txs.delete(tx.hash); - - return true; - } - - toArray() { - const txs = []; - - for (const tx of this.txs.values()) - txs.push(tx); - - return txs; - } -} - -class TXMapRecord { - constructor(hash, wids) { - this.hash = hash || null; - this.wids = wids || new Set(); - } - - add(wid) { - if (this.wids.has(wid)) - return false; - - this.wids.add(wid); - return true; - } - - remove(wid) { - return this.wids.delete(wid); - } - - toWriter(bw) { - return serializeMap(bw, this.wids); - } - - getSize() { - return sizeMap(this.wids); - } - - toRaw() { - const size = this.getSize(); - return this.toWriter(bio.write(size)).render(); - } - - fromReader(br) { - this.wids = parseMap(br); - return this; - } - - fromRaw(data) { - return this.fromReader(bio.read(data)); - } - - static fromReader(hash, br) { - return new TXMapRecord(hash).fromReader(br); - } - - static fromRaw(hash, data) { - return new TXMapRecord(hash).fromRaw(data); - } -} - -function parseMap(br) { - const count = br.readU32(); - const wids = new Set(); - - for (let i = 0; i < count; i++) - wids.add(br.readU32()); - - return wids; -} - -function sizeMap(wids) { - return 4 + wids.size * 4; -} - -function serializeMap(bw, wids) { - bw.writeU32(wids.size); - - for (const wid of wids) - bw.writeU32(wid); - - return bw; -} - -/* - * Helpers - */ - -function newBalance() { - return { - tx: 0, - coin: 0, - unconfirmed: 0, - confirmed: 0 - }; -} - -function serializeBalance(bal) { - const bw = bio.write(32); - - bw.writeU64(bal.tx); - bw.writeU64(bal.coin); - bw.writeU64(bal.unconfirmed); - bw.writeU64(bal.confirmed); - - return bw.render(); -} - -function parsep(key) { // p[hash] - assert(Buffer.isBuffer(key)); - assert(key.length >= 21); - return [key.slice(1)]; -} - -function parseP(key) { // P[wid][hash] - assert(Buffer.isBuffer(key)); - assert(key.length >= 25); - return [key.readUInt32BE(1, true), key.slice(5)]; -} - -function parser(key) { // r[wid][index][hash] - assert(Buffer.isBuffer(key)); - assert(key.length >= 29); - return [ - key.readUInt32BE(1, true), - key.readUInt32BE(5, true), - key.slice(9) - ]; -} - -function parsel(key) { // l[id] - assert(Buffer.isBuffer(key)); - assert(key.length >= 1); - return [key.toString('ascii', 1)]; -} - -function parsei(key) { // i[wid][name] - assert(Buffer.isBuffer(key)); - assert(key.length >= 5); - return [key.readUInt32BE(1, true), key.toString('ascii', 5)]; -} - -function fromString(str) { - const buf = Buffer.alloc(1 + str.length); - buf[0] = str.length; - buf.write(str, 1, str.length, 'ascii'); - return buf; -} - -/* - * Execute - */ - -(async () => { - await db.open(); - - console.log('Opened %s.', process.argv[2]); - - parent = db.batch(); - - await updateVersion(); - await updateKeys(); - await updateState(); - await updateBlockMap(); - await updateTXDB(); - await updatePaths(); - - await parent.write(); - await db.close(); -})().then(() => { - console.log('Migration complete.'); - process.exit(0); -}).catch((err) => { - console.error(err.stack); - process.exit(1); -});