diff --git a/CHANGELOG.md b/CHANGELOG.md index 846103dab7..ac3ed77e0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## unreleased +### Wallet changes + +- Fixes a bug that caused rescans to fail if a name being "watched" was ever +`TRANSFER`ed. A `deepclean` plus `rescan` may be required to fix affected wallets. + ### Wallet API changes - Adds new wallet HTTP endpoint `/wallet/:id/auction` based on `POST /wallet/:id/bid`. diff --git a/lib/primitives/address.js b/lib/primitives/address.js index d73ce3114e..d4c72bb214 100644 --- a/lib/primitives/address.js +++ b/lib/primitives/address.js @@ -542,8 +542,6 @@ class Address extends bio.Struct { throw new Error('Object is not an address.'); if (Buffer.isBuffer(data)) { - if (data.length !== 20 && data.length !== 32) - throw new Error('Object is not an address.'); return data; } diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index efcc514644..1f86c80c56 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -1030,10 +1030,20 @@ class TXDB { // Handle names. if (block) { - const updated = await this.connectNames(b, tx, view, height); + const {updated, index} = await this.connectNames(b, tx, view, height); + + if (updated && !state.updated()) { + if (index) { + // TRANSFER, FINALIZE, and REVOKE transactions must be indexed + // so we can undo them in event of reorg or rescan. + await this.addBlockMap(b, height); + await this.addBlock(b, tx.hash(), block); + } - if (updated && !state.updated()) + // Always save namestate transitions, + // even if they don't affect wallet balance await b.write(); + } } // If this didn't update any coins, @@ -1458,7 +1468,7 @@ class TXDB { for (let i = hashes.length - 1; i >= 0; i--) { const hash = hashes[i]; - await this.unconfirm(hash); + await this.unconfirm(hash, height); } return hashes.length; @@ -1468,14 +1478,24 @@ class TXDB { * Unconfirm a transaction without a batch. * @private * @param {Hash} hash + * @param {Number} height * @returns {Promise} */ - async unconfirm(hash) { + async unconfirm(hash, height) { const wtx = await this.getTX(hash); - if (!wtx) - return null; + if (!wtx) { + this.logger.warning( + 'Reverting namestate without transaction: %x', + hash + ); + const b = this.bucket.batch(); + await this.applyNameUndo(b, hash); + await this.removeBlockMap(b, height); + await this.removeBlock(b, hash, height); + return b.write(); + } if (wtx.height === -1) return null; @@ -1848,6 +1868,9 @@ class TXDB { * @param {Number} i * @param {Path} path * @param {Number} height + * @returns {Object} out + * @returns {Boolean} out.updated + * @returns {Boolean} out.index */ async connectNames(b, tx, view, height) { @@ -1856,8 +1879,19 @@ class TXDB { assert(height !== -1); + // If namestate has been updated we need to write to DB let updated = false; + // If namestate has been updated with a critical property + // we need to not only save the new namestate but also + // index the transaction (whether or not it affects our balance) + // so that we can rollback the change during reorg or rescan. + // Critical properties are anything that are asserted by this + // function the _next_ time we see this transaction: + // TRANSFER / FINALIZE: ns.transfer + // REVOKE: ns.revoked + let index = false; + for (let i = 0; i < tx.outputs.length; i++) { const output = tx.outputs[i]; const {covenant} = output; @@ -2076,6 +2110,7 @@ class TXDB { ns.setTransfer(height); updated = true; + index = true; break; } @@ -2113,6 +2148,7 @@ class TXDB { ns.setRenewals(ns.renewals + 1); updated = true; + index = true; break; } @@ -2127,6 +2163,7 @@ class TXDB { ns.setData(null); updated = true; + index = true; break; } @@ -2151,7 +2188,7 @@ class TXDB { b.put(layout.U.encode(hash), undo.encode()); } - return updated; + return {updated, index}; } /** @@ -2187,6 +2224,19 @@ class TXDB { } } + return this.applyNameUndo(b, hash); + } + + /** + * Apply namestate undo data by hash without transaction. + * Should only be called directly to undo namestate transitions + * that do not affect wallet balance like a TRANSFER for a name + * that is in the nameMap but does not involve wallet addresses. + * @param {Object} b + * @param {Hash} hash + */ + + async applyNameUndo(b, hash) { const raw = await this.bucket.get(layout.U.encode(hash)); if (!raw) diff --git a/test/wallet-rescan-test.js b/test/wallet-rescan-test.js new file mode 100644 index 0000000000..6cf5840f00 --- /dev/null +++ b/test/wallet-rescan-test.js @@ -0,0 +1,516 @@ +/* eslint-env mocha */ +/* eslint prefer-arrow-callback: "off" */ +/* eslint no-implicit-coercion: "off" */ + +'use strict'; + +const assert = require('bsert'); +const FullNode = require('../lib/node/fullnode'); +const MemWallet = require('./util/memwallet'); +const Network = require('../lib/protocol/network'); +const Address = require('../lib/primitives/address'); +const rules = require('../lib/covenants/rules'); +const {Resource} = require('../lib/dns/resource'); +const {forValue} = require('./util/common'); + +const network = Network.get('regtest'); + +const { + treeInterval, + biddingPeriod, + revealPeriod, + transferLockup +} = network.names; + +describe('Wallet rescan with namestate transitions', function() { + describe('Only sends OPEN', function() { + // Bob runs a full node with wallet plugin + const node = new FullNode({ + network: network.type, + memory: true, + plugins: [require('../lib/wallet/plugin')] + }); + node.on('error', (err) => { + assert(false, err); + }); + + const {wdb} = node.require('walletdb'); + let bob, bobAddr; + + // Alice is some other wallet on the network + const alice = new MemWallet({ network }); + const aliceAddr = alice.getAddress(); + + // Connect MemWallet to chain as minimally as possible + node.chain.on('connect', (entry, block) => { + alice.addBlock(entry, block.txs); + }); + alice.getNameStatus = async (nameHash) => { + assert(Buffer.isBuffer(nameHash)); + const height = node.chain.height + 1; + const state = await node.chain.getNextState(); + const hardened = state.hasHardening(); + return node.chain.db.getNameStatus(nameHash, height, hardened); + }; + + const NAME = rules.grindName(4, 4, network); + + // Hash of the FINALIZE transaction + let aliceFinalizeHash; + + async function mineBlocks(n, addr) { + addr = addr ? addr : new Address().toString('regtest'); + const blocks = []; + for (let i = 0; i < n; i++) { + const block = await node.miner.mineBlock(null, addr); + await node.chain.add(block); + blocks.push(block); + } + + return blocks; + } + + before(async () => { + await node.open(); + bob = await wdb.create(); + bobAddr = await bob.receiveAddress(); + }); + + after(async () => { + await node.close(); + }); + + it('should fund wallets', async () => { + const blocks = 10; + await mineBlocks(blocks, aliceAddr); + await mineBlocks(blocks, bobAddr); + + const bobBal = await bob.getBalance(); + assert.strictEqual(bobBal.confirmed, blocks * 2000 * 1e6); + assert.strictEqual(alice.balance, blocks * 2000 * 1e6); + }); + + it('should run auction', async () => { + // Poor Bob, all he does is send an OPEN but his wallet will + // watch all the other activity including TRANSFERS for this name + await bob.sendOpen(NAME, true); + const openBlocks = await mineBlocks(1); + // Coinbase plus open + assert.strictEqual(openBlocks[0].txs.length, 2); + + // Advance to bidding phase + await mineBlocks(treeInterval); + await forValue(alice, 'height', node.chain.height); + + // Alice sends only bid + const aliceBid = await alice.createBid(NAME, 20000, 20000); + await node.mempool.addTX(aliceBid.toTX()); + const bidBlocks = await mineBlocks(1); + assert.strictEqual(bidBlocks[0].txs.length, 2); + + // Advance to reveal phase + await mineBlocks(biddingPeriod); + const aliceReveal = await alice.createReveal(NAME); + await node.mempool.addTX(aliceReveal.toTX()); + const revealBlocks = await mineBlocks(1); + assert.strictEqual(revealBlocks[0].txs.length, 2); + + // Close auction + await mineBlocks(revealPeriod); + + // Alice registers + const aliceRegister = await alice.createRegister( + NAME, + Resource.fromJSON({records:[]}) + ); + await node.mempool.addTX(aliceRegister.toTX()); + + const registerBlocks = await mineBlocks(1); + assert.strictEqual(registerBlocks[0].txs.length, 2); + }); + + it('should get namestate', async () => { + const ns = await bob.getNameStateByName(NAME); + // Bob has the namestate + assert(ns); + + // Bob is not the name owner + const {hash, index} = ns.owner; + const coin = await bob.getCoin(hash, index); + assert.strictEqual(coin, null); + + // Name is not in mid-TRANSFER + assert.strictEqual(ns.transfer, 0); + }); + + it('should process TRANSFER', async () => { + // Alice transfers the name to her own address + const aliceTransfer = await alice.createTransfer(NAME, aliceAddr); + await node.mempool.addTX(aliceTransfer.toTX()); + const transferBlocks = await mineBlocks(1); + assert.strictEqual(transferBlocks[0].txs.length, 2); + + // Bob detects the TRANSFER even though it doesn't involve him at all + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.transfer, node.chain.height); + + // Bob's wallet has not indexed the TRANSFER + const bobTransfer = await bob.getTX(aliceTransfer.hash()); + assert.strictEqual(bobTransfer, null); + }); + + it('should fully rescan', async () => { + // Complete chain rescan + await wdb.rescan(0); + await forValue(wdb, 'height', node.chain.height); + + // No change + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.transfer, node.chain.height); + }); + + it('should process FINALIZE', async () => { + await mineBlocks(transferLockup); + + // Alice finalizes the name + const aliceFinalize = await alice.createFinalize(NAME); + await node.mempool.addTX(aliceFinalize.toTX()); + const finalizeBlocks = await mineBlocks(1); + assert.strictEqual(finalizeBlocks[0].txs.length, 2); + + aliceFinalizeHash = aliceFinalize.hash(); + + // Bob detects the FINALIZE even though it doesn't involve him at all + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.bufferEqual(ns.owner.hash, aliceFinalizeHash); + + // Bob's wallet has not indexed the FINALIZE + const bobFinalize = await bob.getTX(aliceFinalize.hash()); + assert.strictEqual(bobFinalize, null); + }); + + it('should fully rescan', async () => { + // Complete chain rescan + await wdb.rescan(0); + await forValue(wdb, 'height', node.chain.height); + + // No change + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.bufferEqual(ns.owner.hash, aliceFinalizeHash); + }); + + it('should process TRANSFER (again)', async () => { + // Alice transfers the name to her own address + const aliceTransfer = await alice.createTransfer(NAME, aliceAddr); + await node.mempool.addTX(aliceTransfer.toTX()); + const transferBlocks = await mineBlocks(1); + assert.strictEqual(transferBlocks[0].txs.length, 2); + + // Bob detects the TRANSFER even though it doesn't involve him at all + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.transfer, node.chain.height); + + // Bob's wallet has not indexed the TRANSFER + const bobTransfer = await bob.getTX(aliceTransfer.hash()); + assert.strictEqual(bobTransfer, null); + }); + + it('should process REVOKE', async () => { + // Alice revokes the name + const aliceRevoke = await alice.createRevoke(NAME); + await node.mempool.addTX(aliceRevoke.toTX()); + const revokeBlocks = await mineBlocks(1); + assert.strictEqual(revokeBlocks[0].txs.length, 2); + + // Bob detects the REVOKE even though it doesn't involve him at all + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.revoked, node.chain.height); + + // Bob's wallet has not indexed the REVOKE + const bobTransfer = await bob.getTX(aliceRevoke.hash()); + assert.strictEqual(bobTransfer, null); + }); + + it('should fully rescan', async () => { + // Complete chain rescan + await wdb.rescan(0); + await forValue(wdb, 'height', node.chain.height); + + // No change + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.revoked, node.chain.height); + }); + }); + + describe('Bids, loses, shallow rescan', function() { + // Bob runs a full node with wallet plugin + const node = new FullNode({ + network: network.type, + memory: true, + plugins: [require('../lib/wallet/plugin')] + }); + node.on('error', (err) => { + assert(false, err); + }); + + const {wdb} = node.require('walletdb'); + let bob, bobAddr; + + // Alice is some other wallet on the network + const alice = new MemWallet({ network }); + const aliceAddr = alice.getAddress(); + + // Connect MemWallet to chain as minimally as possible + node.chain.on('connect', (entry, block) => { + alice.addBlock(entry, block.txs); + }); + alice.getNameStatus = async (nameHash) => { + assert(Buffer.isBuffer(nameHash)); + const height = node.chain.height + 1; + const state = await node.chain.getNextState(); + const hardened = state.hasHardening(); + return node.chain.db.getNameStatus(nameHash, height, hardened); + }; + + const NAME = rules.grindName(4, 4, network); + + // Block that confirmed the bids + let bidBlockHash; + // Hash of the FINALIZE transaction + let aliceFinalizeHash; + + async function mineBlocks(n, addr) { + addr = addr ? addr : new Address().toString('regtest'); + const blocks = []; + for (let i = 0; i < n; i++) { + const block = await node.miner.mineBlock(null, addr); + await node.chain.add(block); + blocks.push(block); + } + + return blocks; + } + + before(async () => { + await node.open(); + bob = await wdb.create(); + bobAddr = await bob.receiveAddress(); + }); + + after(async () => { + await node.close(); + }); + + it('should fund wallets', async () => { + const blocks = 10; + await mineBlocks(blocks, aliceAddr); + await mineBlocks(blocks, bobAddr); + + const bobBal = await bob.getBalance(); + assert.strictEqual(bobBal.confirmed, blocks * 2000 * 1e6); + assert.strictEqual(alice.balance, blocks * 2000 * 1e6); + }); + + it('should run auction', async () => { + // Alice opens + const aliceOpen = await alice.createOpen(NAME); + await node.mempool.addTX(aliceOpen.toTX()); + const openBlocks = await mineBlocks(1); + // Coinbase plus open + assert.strictEqual(openBlocks[0].txs.length, 2); + + // Advance to bidding phase + await mineBlocks(treeInterval); + await forValue(alice, 'height', node.chain.height); + + // Poor Bob, all he does is send one (losing) bid but his wallet will + // watch all the other activity including TRANSFERS for this name + await bob.sendBid(NAME, 10000, 10000); + + // Alice sends winning bid + const aliceBid = await alice.createBid(NAME, 20000, 20000); + await node.mempool.addTX(aliceBid.toTX()); + const bidBlocks = await mineBlocks(1); + assert.strictEqual(bidBlocks[0].txs.length, 3); + + bidBlockHash = bidBlocks[0].hash(); + + // Advance to reveal phase + await mineBlocks(biddingPeriod); + await bob.sendReveal(NAME); + const aliceReveal = await alice.createReveal(NAME); + await node.mempool.addTX(aliceReveal.toTX()); + const revealBlocks = await mineBlocks(1); + assert.strictEqual(revealBlocks[0].txs.length, 3); + + // Close auction + await mineBlocks(revealPeriod); + + // Alice registers + const aliceRegister = await alice.createRegister( + NAME, + Resource.fromJSON({records:[]}) + ); + await node.mempool.addTX(aliceRegister.toTX()); + + const registerBlocks = await mineBlocks(1); + assert.strictEqual(registerBlocks[0].txs.length, 2); + }); + + it('should get namestate', async () => { + const ns = await bob.getNameStateByName(NAME); + // Bob has the namestate + assert(ns); + + // Bob is not the name owner + const {hash, index} = ns.owner; + const coin = await bob.getCoin(hash, index); + assert.strictEqual(coin, null); + + // Name is not in mid-TRANSFER + assert.strictEqual(ns.transfer, 0); + }); + + it('should process TRANSFER', async () => { + // Alice transfers the name to her own address + const aliceTransfer = await alice.createTransfer(NAME, aliceAddr); + await node.mempool.addTX(aliceTransfer.toTX()); + const transferBlocks = await mineBlocks(1); + assert.strictEqual(transferBlocks[0].txs.length, 2); + + // Bob detects the TRANSFER even though it doesn't involve him at all + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.transfer, node.chain.height); + + // Bob's wallet has indexed the TRANSFER + const bobTransfer = await bob.getTX(aliceTransfer.hash()); + assert.strictEqual(bobTransfer, null); + }); + + it('should fully rescan', async () => { + await wdb.rescan(0); + await forValue(wdb, 'height', node.chain.height); + + // No change + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.transfer, node.chain.height); + }); + + it('should rescan since, but not including, the BIDs', async () => { + const bidBlock = await node.chain.getEntry(bidBlockHash); + await wdb.rescan(bidBlock.height); + await forValue(wdb, 'height', node.chain.height); + + // No change + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.transfer, node.chain.height); + }); + + it('should process FINALIZE', async () => { + await mineBlocks(transferLockup); + + // Alice finalizes the name + const aliceFinalize = await alice.createFinalize(NAME); + await node.mempool.addTX(aliceFinalize.toTX()); + const finalizeBlocks = await mineBlocks(1); + assert.strictEqual(finalizeBlocks[0].txs.length, 2); + + aliceFinalizeHash = aliceFinalize.hash(); + + // Bob detects the FINALIZE even though it doesn't involve him at all + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.bufferEqual(ns.owner.hash, aliceFinalizeHash); + + // Bob's wallet has not indexed the FINALIZE + const bobFinalize = await bob.getTX(aliceFinalize.hash()); + assert.strictEqual(bobFinalize, null); + }); + + it('should fully rescan', async () => { + await wdb.rescan(0); + await forValue(wdb, 'height', node.chain.height); + + // No change + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.bufferEqual(ns.owner.hash, aliceFinalizeHash); + }); + + it('should rescan since, but not including, the BIDs', async () => { + const bidBlock = await node.chain.getEntry(bidBlockHash); + await wdb.rescan(bidBlock.height); + await forValue(wdb, 'height', node.chain.height); + + // No change + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.bufferEqual(ns.owner.hash, aliceFinalizeHash); + }); + + it('should process TRANSFER (again)', async () => { + // Alice transfers the name to her own address + const aliceTransfer = await alice.createTransfer(NAME, aliceAddr); + await node.mempool.addTX(aliceTransfer.toTX()); + const transferBlocks = await mineBlocks(1); + assert.strictEqual(transferBlocks[0].txs.length, 2); + + // Bob detects the TRANSFER even though it doesn't involve him at all + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.transfer, node.chain.height); + + // Bob's wallet has not indexed the TRANSFER + const bobTransfer = await bob.getTX(aliceTransfer.hash()); + assert.strictEqual(bobTransfer, null); + }); + + it('should process REVOKE', async () => { + // Alice revokes the name + const aliceRevoke = await alice.createRevoke(NAME); + await node.mempool.addTX(aliceRevoke.toTX()); + const revokeBlocks = await mineBlocks(1); + assert.strictEqual(revokeBlocks[0].txs.length, 2); + + // Bob detects the REVOKE even though it doesn't involve him at all + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.revoked, node.chain.height); + + // Bob's wallet has not indexed the REVOKE + const bobTransfer = await bob.getTX(aliceRevoke.hash()); + assert.strictEqual(bobTransfer, null); + }); + + it('should fully rescan', async () => { + await wdb.rescan(0); + await forValue(wdb, 'height', node.chain.height); + + // No change + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.revoked, node.chain.height); + }); + + it('should rescan since, but not including, the BIDs', async () => { + const bidBlock = await node.chain.getEntry(bidBlockHash); + await wdb.rescan(bidBlock.height); + await forValue(wdb, 'height', node.chain.height); + + // No change + const ns = await bob.getNameStateByName(NAME); + assert(ns); + assert.strictEqual(ns.revoked, node.chain.height); + }); + }); +});