wallet: separate open and connect. Add wallet-migrate-no-rescan option.

CHANGELOG.md

@@ -3,8 +3,18 @@
 ## Unreleased
 
 ### Wallet Changes
+#### Configuration
+  Wallet now has option `wallet-migrate-no-rescan`/`migrate-no-rescan` if you
+want to disable rescan when migration recommends it. It may result in the
+incorrect txdb state, but can be useful if you know the issue does not affect
+your wallet or is not critical.
+
 #### Wallet API
-- WalletDB Now emits events for: `open`, `close`, `connect`, `disconnect`.
+- WalletDB
+  - `open()` no longer calls `connect` and needs separate call `connect`.
+  - `open()` no longer calls scan, instead only rollbacks and waits for
+    sync to do the rescan.
+  - emits events for: `open`, `close`, `connect`, `disconnect`, `sync done`.
 - HTTP Changes:
   - All transaction creating endpoints now accept `hardFee` for specifying the
     exact fee.

lib/wallet/node.js

@@ -52,6 +52,7 @@ class WalletNode extends Node {
       wipeNoReally: this.config.bool('wipe-no-really'),
       spv: this.config.bool('spv'),
       walletMigrate: this.config.uint('migrate'),
+      migrateNoRescan: this.config.bool('migrate-no-rescan', false),
       icannlockup: this.config.bool('icannlockup', false)
     });
 
@@ -107,6 +108,7 @@ class WalletNode extends Node {
     await this.openPlugins();
 
     await this.http.open();
+    await this.wdb.connect();
     await this.handleOpen();
 
     this.logger.info('Wallet node is loaded.');
@@ -128,6 +130,7 @@ class WalletNode extends Node {
 
     this.rpc.wallet = null;
 
+    await this.wdb.disconnect();
     await this.wdb.close();
     await this.handleClose();
   }

lib/wallet/plugin.js

@@ -57,6 +57,7 @@ class Plugin extends EventEmitter {
       wipeNoReally: this.config.bool('wipe-no-really'),
       spv: node.spv,
       walletMigrate: this.config.uint('migrate'),
+      migrateNoRescan: this.config.bool('migrate-no-rescan', false),
       icannlockup: this.config.bool('icannlockup', false)
     });
 
@@ -90,11 +91,13 @@ class Plugin extends EventEmitter {
     await this.wdb.open();
     this.rpc.wallet = this.wdb.primary;
     await this.http.open();
+    await this.wdb.connect();
   }
 
   async close() {
     await this.http.close();
     this.rpc.wallet = null;
+    await this.wdb.disconnect();
     await this.wdb.close();
   }
 }

lib/wallet/walletdb.js

@@ -64,12 +64,18 @@ class WalletDB extends EventEmitter {
     this.name = 'wallet';
     this.version = 2;
 
-    this.primary = null;
+    // chain state.
+    this.hasStateCache = false;
     this.state = new ChainState();
-    this.confirming = false;
     this.height = 0;
+
+    // wallets
+    this.primary = null;
     this.wallets = new Map();
     this.depth = 0;
+
+    // guards
+    this.confirming = false;
     this.rescanning = false;
     this.filterSent = false;
 
@@ -203,7 +209,7 @@ class WalletDB extends EventEmitter {
       await this.wipe();
 
     await this.watch();
-    await this.connect();
+    await this.loadState();
 
     this.logger.info(
       'WalletDB loaded (depth=%d, height=%d, start=%d).',
@@ -224,8 +230,13 @@ class WalletDB extends EventEmitter {
     this.primary = wallet;
 
     if (migrationResult.rescan) {
-      this.logger.info('Rescanning...');
-      await this.scan(0);
+      if (!this.options.migrateNoRescan) {
+        this.logger.info('Migration rollback...');
+        await this.rollback(0);
+      } else {
+        this.logger.warning(
+          'Migration rescan skipped, state may be incorrect.');
+      }
     }
 
     this.logger.info('WalletDB opened.');
@@ -275,7 +286,8 @@ class WalletDB extends EventEmitter {
    */
 
   async close() {
-    await this.disconnect();
+    if (this.client.opened)
+      await this.disconnect();
 
     for (const wallet of this.wallets.values()) {
       await wallet.destroy();
@@ -375,7 +387,7 @@ class WalletDB extends EventEmitter {
     const unlock = await this.txLock.lock();
     try {
       this.logger.info('Resyncing from server...');
-      await this.syncState();
+      await this.syncInitState();
       await this.syncFilter();
       await this.syncChain();
       await this.resend();
@@ -385,18 +397,31 @@ class WalletDB extends EventEmitter {
   }
 
   /**
-   * Initialize and write initial sync state.
+   * Recover state from the cache.
    * @returns {Promise}
    */
 
-  async syncState() {
+  async loadState() {
     const cache = await this.getState();
 
-    if (cache) {
-      this.state = cache;
-      this.height = cache.height;
-      return undefined;
-    }
+    if (!cache)
+      return;
+
+    this.logger.info('Initialized chain state from the database.');
+    this.hasStateCache = true;
+    this.state = cache;
+    this.height = cache.height;
+  }
+
+  /**
+   * Initialize and write initial sync state.
+   * @returns {Promise}
+   */
+
+  async syncInitState() {
+    // We have recovered from the cache.
+    if (this.hasStateCache)
+      return;
 
     this.logger.info('Initializing database state from server.');
 
@@ -427,7 +452,7 @@ class WalletDB extends EventEmitter {
     this.state = state;
     this.height = state.height;
 
-    return undefined;
+    return;
   }
 
   /**
@@ -2167,6 +2192,7 @@ class WalletDB extends EventEmitter {
     await iter.each(async (key, value) => {
       const [height] = layout.b.decode(key);
       const block = MapRecord.decode(value);
+      this.logger.info('Reverting block: %d', height);
 
       for (const wid of block.wids) {
         const wallet = await this.get(wid);
@@ -2476,6 +2502,7 @@ class WalletOptions {
     this.spv = false;
     this.wipeNoReally = false;
     this.walletMigrate = -1;
+    this.migrateNoRescan = false;
     this.icannlockup = false;
 
     if (options)
@@ -2564,6 +2591,11 @@ class WalletOptions {
       this.icannlockup = options.icannlockup;
     }
 
+    if (options.migrateNoRescan != null) {
+      assert(typeof options.migrateNoRescan === 'boolean');
+      this.migrateNoRescan = options.migrateNoRescan;
+    }
+
     return this;
   }
 

test/mempool-reorg-test.js

@@ -8,6 +8,7 @@ const plugin = require('../lib/wallet/plugin');
 const Coin = require('../lib/primitives/coin');
 const Address = require('../lib/primitives/address');
 const MTX = require('../lib/primitives/mtx');
+const {forEvent} = require('./util/common');
 
 const network = Network.get('regtest');
 const {
@@ -25,8 +26,13 @@ describe('Mempool Covenant Reorg', function () {
   let wallet, name;
 
   before(async () => {
+    const wdb = node.require('walletdb').wdb;
+    const syncDone = forEvent(wdb, 'sync done');
     await node.open();
+
     wallet = node.get('walletdb').wdb.primary;
+
+    await syncDone;
   });
 
   after(async () => {

test/wallet-auction-test.js

@@ -64,6 +64,7 @@ describe('Wallet Auction', function() {
     await chain.open();
     await miner.open();
     await wdb.open();
+    await wdb.connect();
 
     // Set up wallet
     winner = await wdb.create();
@@ -81,6 +82,7 @@ describe('Wallet Auction', function() {
   });
 
   after(async () => {
+    await wdb.disconnect();
     await wdb.close();
     await miner.close();
     await chain.close();

test/wallet-events-test.js

@@ -7,11 +7,13 @@ const Wallet = require('../lib/wallet/wallet');
 const WalletKey = require('../lib/wallet/walletkey');
 const ChainEntry = require('../lib/blockchain/chainentry');
 const MemWallet = require('./util/memwallet');
+const {forEvent} = require('./util/common');
 
 describe('WalletDB Events', function () {
   const node = new FullNode({
     memory: true,
     network: 'regtest',
+    noDNS: true,
     plugins: [require('../lib/wallet/plugin')]
   });
 
@@ -35,15 +37,31 @@ describe('WalletDB Events', function () {
     await node.close();
   });
 
-  it('should emit `tx` events', async () => {
-    const waiter = new Promise((resolve) => {
-      wdb.once('tx', (w, tx) => resolve([w, tx]));
-    });
+  it('should emit `open`/`close` and `connect`/`disconnect` events', async () => {
+    const closeEvent = forEvent(wdb, 'close');
+    const disconnectEvent = forEvent(wdb, 'disconnect');
+    await node.close();
+    await disconnectEvent;
+    await closeEvent;
 
+    const openEvent = forEvent(wdb, 'open');
+    const connectEvent = forEvent(wdb, 'connect');
+    const syncDone = forEvent(wdb, 'sync done');
+    await node.open();
+    await openEvent;
+    await connectEvent;
+    await syncDone;
+
+    wallet = await wdb.get('primary');
+  });
+
+  it('should emit `tx` events', async () => {
+    const waiter = forEvent(wdb, 'tx');
     const walletReceive = await wallet.receiveAddress();
     await mineBlocks(1, walletReceive);
 
-    const [w, tx] = await waiter;
+    const events = await waiter;
+    const [w, tx] = events[0].values;
 
     assert(w);
     assert(w instanceof Wallet);
@@ -55,14 +73,13 @@ describe('WalletDB Events', function () {
   });
 
   it('should emit `address` events', async () => {
-    const waiter = new Promise((resolve) => {
-      wdb.once('address', (w, walletKey) => resolve([w, walletKey]));
-    });
+    const waiter = forEvent(wdb, 'address');
 
     const walletReceive = await wallet.receiveAddress();
     await mineBlocks(1, walletReceive);
 
-    const [w, walletKey] = await waiter;
+    const events = await waiter;
+    const [w, walletKey] = events[0].values;
 
     assert(w);
     assert(w instanceof Wallet);
@@ -75,10 +92,7 @@ describe('WalletDB Events', function () {
 
   describe('should emit `block connect` events', () => {
     it('with a block that includes a wallet tx', async () => {
-      const waiter = new Promise((resolve) => {
-        wdb.once('block connect', (entry, txs) => resolve([entry, txs]));
-      });
-
+      const waiter = forEvent(wdb, 'block connect');
       const walletReceive = await wallet.receiveAddress();
 
       await wallet.send({
@@ -89,7 +103,8 @@ describe('WalletDB Events', function () {
 
       await mineBlocks(1);
 
-      const [entry, txs] = await waiter;
+      const events = await waiter;
+      const [entry, txs] = events[0].values;
 
       assert(entry);
       assert(entry instanceof ChainEntry);
@@ -127,15 +142,14 @@ describe('WalletDB Events', function () {
         otherTx = otherMtx.toTX();
       }
 
-      const waiter = new Promise((resolve) => {
-        wdb.once('block connect', (entry, txs) => resolve([entry, txs]));
-      });
+      const waiter = forEvent(wdb, 'block connect');
 
       await node.sendTX(otherTx);
 
       await mineBlocks(1);
 
-      const [entry, txs] = await waiter;
+      const events = await waiter;
+      const [entry, txs] = events[0].values;
 
       assert(entry);
       assert(entry instanceof ChainEntry);
@@ -158,33 +172,30 @@ describe('WalletDB Events', function () {
       await mineBlocks(1);
 
       // Disconnect it
-      const waiter = new Promise((resolve) => {
-        wdb.once('block disconnect', entry => resolve(entry));
-      });
+      const waiter = forEvent(wdb, 'block disconnect');
 
       const entryToDisconnect = node.chain.tip;
       await node.chain.disconnect(entryToDisconnect);
 
-      const entry = await waiter;
+      const events = await waiter;
+      const entry = events[0].values[0];
 
       assert(entry);
       assert(entry instanceof ChainEntry);
       assert(entry.hash = entryToDisconnect.hash);
     });
 
     it('with a block that does not include a wallet tx', async () => {
-        const waiter = new Promise((resolve) => {
-          wdb.once('block disconnect', entry => resolve(entry));
-        });
-
-        const entryToDisconnect = node.chain.tip;
-        await node.chain.disconnect(entryToDisconnect);
+      const waiter = forEvent(wdb, 'block disconnect');
+      const entryToDisconnect = node.chain.tip;
+      await node.chain.disconnect(entryToDisconnect);
 
-        const entry = await waiter;
+      const events = await waiter;
+      const entry = events[0].values[0];
 
-        assert(entry);
-        assert(entry instanceof ChainEntry);
-        assert(entry.hash = entryToDisconnect.hash);
+      assert(entry);
+      assert(entry instanceof ChainEntry);
+      assert(entry.hash = entryToDisconnect.hash);
     });
   });
 });