wallet: Use interactive scan on initial sync and rescan.

Check issue #872

CHANGELOG.md

@@ -40,6 +40,9 @@ process and allows parallel rescans.
     - expects ws hook for `block rescan interactive abort` param `message`.
 
 ### Wallet Changes
+- Add migration that recalculates txdb balances to fix any inconsistencies.
+- Wallet will now use `interactive scan` for initial sync(on open) and rescan.
+
 #### Configuration
 - Wallet now has option `wallet-migrate-no-rescan`/`migrate-no-rescan` if you
   want to disable rescan when migration recommends it. It may result in the
@@ -51,20 +54,21 @@ process and allows parallel rescans.
 
 #### Wallet API
 
-- Add migration that recalculates txdb balances to fix any inconsistencies.
 - WalletNode now emits `open` and `close` events.
 - WalletDB Now emits events for: `open`, `close`, `connect`, `disconnect`.
 - WalletDB
   - `open()` no longer calls `connect` and needs separate call `connect`.
   - `open()` no longer calls scan, instead only rollbacks and waits for
     sync to do the rescan.
   - emits events for: `open`, `close`, `connect`, `disconnect`, `sync done`.
-- HTTP Changes:
+
+#### Wallet HTTP
   - All transaction creating endpoints now accept `hardFee` for specifying the
     exact fee.
   - All transaction sending endpoints now fundlock/queue tx creation. (no more
     conflicting transactions)
 
+
 ## v6.0.0
 
 ### Node and Wallet HTTP API

lib/wallet/client.js

@@ -17,6 +17,7 @@ const parsers = {
   'block connect': (entry, txs) => parseBlock(entry, txs),
   'block disconnect': entry => [parseEntry(entry)],
   'block rescan': (entry, txs) => parseBlock(entry, txs),
+  'block rescan interactive': (entry, txs) => parseBlock(entry, txs),
   'chain reset': entry => [parseEntry(entry)],
   'tx': tx => [TX.decode(tx)]
 };
@@ -75,10 +76,27 @@ class WalletClient extends NodeClient {
     return super.setFilter(filter.encode());
   }
 
+  /**
+   * Rescan for any missed transactions.
+   * @param {Number|Hash} start - Start block.
+   * @returns {Promise}
+   */
+
   async rescan(start) {
     return super.rescan(start);
   }
 
+  /**
+   * Rescan interactive for any missed transactions.
+   * @param {Number|Hash} start - Start block.
+   * @param {Boolean} [fullLock=false]
+   * @returns {Promise}
+   */
+
+  async rescanInteractive(start, fullLock) {
+    return super.rescanInteractive(start, null, fullLock);
+  }
+
   async getNameStatus(nameHash) {
     const json = await super.getNameStatus(nameHash);
     return NameState.fromJSON(json);

lib/wallet/nodeclient.js

@@ -262,6 +262,34 @@ class NodeClient extends AsyncEmitter {
     });
   }
 
+  /**
+   * Rescan interactive for any missed transactions.
+   * @param {Number|Hash} start - Start block.
+   * @param {Boolean} [fullLock=false]
+   * @returns {Promise}
+   */
+
+  async rescanInteractive(start, fullLock = true) {
+    if (this.node.spv)
+      return this.node.chain.reset(start);
+
+    const iter = async (entry, txs) => {
+      return await this.handleCall('block rescan interactive', entry, txs);
+    };
+
+    try {
+      return await this.node.scanInteractive(
+        start,
+        this.filter,
+        iter,
+        fullLock
+      );
+    } catch (e) {
+      await this.handleCall('block rescan interactive abort', e.message);
+      throw e;
+    }
+  }
+
   /**
    * Get name state.
    * @param {Buffer} nameHash

lib/wallet/nullclient.js

@@ -165,15 +165,24 @@ class NullClient extends EventEmitter {
   /**
    * Rescan for any missed transactions.
    * @param {Number|Hash} start - Start block.
-   * @param {Bloom} filter
-   * @param {Function} iter - Iterator.
    * @returns {Promise}
    */
 
   async rescan(start) {
     ;
   }
 
+  /**
+   * Rescan interactive for any missed transactions.
+   * @param {Number|Hash} start - Start block.
+   * @param {Boolean} [fullLock=false]
+   * @returns {Promise}
+   */
+
+  async rescanInteractive(start, fullLock) {
+    ;
+  }
+
   /**
    * Get opening bid height.
    * @param {Buffer} nameHash

lib/wallet/txdb.js

@@ -14,14 +14,13 @@ const Amount = require('../ui/amount');
 const CoinView = require('../coins/coinview');
 const Coin = require('../primitives/coin');
 const Outpoint = require('../primitives/outpoint');
-const records = require('./records');
 const layout = require('./layout').txdb;
 const consensus = require('../protocol/consensus');
 const policy = require('../protocol/policy');
 const rules = require('../covenants/rules');
 const NameState = require('../covenants/namestate');
 const NameUndo = require('../covenants/undo');
-const {TXRecord} = records;
+const {TXRecord} = require('./records');
 const {types} = rules;
 
 /*
@@ -1484,7 +1483,7 @@ class TXDB {
   /**
    * Revert a block.
    * @param {Number} height
-   * @returns {Promise<Number>} - blocks length
+   * @returns {Promise<Number>} - number of txs removed.
    */
 
   async revert(height) {

lib/wallet/wallet.js

@@ -4785,7 +4785,7 @@ class Wallet extends EventEmitter {
   /**
    * Revert a block.
    * @param {Number} height
-   * @returns {Promise}
+   * @returns {Promise<Number>} - number of txs removed.
    */
 
   async revert(height) {

lib/wallet/walletdb.js

@@ -29,8 +29,10 @@ const WalletMigrator = require('./migrations');
 const layout = layouts.wdb;
 const tlayout = layouts.txdb;
 const {states} = require('../covenants/namestate');
+const {scanActions} = require('../blockchain/common');
 
 /** @typedef {import('../primitives/tx')} TX */
+/** @typedef {import('../blockchain/common').ScanAction} ScanAction */
 
 const {
   ChainState,
@@ -189,6 +191,21 @@ class WalletDB extends EventEmitter {
       }
     });
 
+    this.client.hook('block rescan interactive', async (entry, txs) => {
+      try {
+        return await this.rescanBlockInteractive(entry, txs);
+      } catch (e) {
+        this.emit('error', e);
+        return {
+          type: scanActions.ABORT
+        };
+      }
+    });
+
+    this.client.hook('block rescan interactive abort', async (message) => {
+      this.emit('error', new Error(message));
+    });
+
     this.client.bind('tx', async (tx) => {
       try {
         await this.addTX(tx);
@@ -522,7 +539,7 @@ class WalletDB extends EventEmitter {
     }
 
     // syncNode sets the rescanning to true.
-    return this.scan(height);
+    return this.scanInteractive(height);
   }
 
   /**
@@ -534,11 +551,17 @@ class WalletDB extends EventEmitter {
    */
 
   async scan(height) {
+    assert(this.rescanning, 'WDB: Rescanning guard not set.');
+
     if (height == null)
       height = this.state.startHeight;
 
     assert((height >>> 0) === height, 'WDB: Must pass in a height.');
 
+    this.logger.info(
+      'Rolling back %d blocks.',
+      this.height - height + 1);
+
     await this.rollback(height);
 
     this.logger.info(
@@ -550,6 +573,38 @@ class WalletDB extends EventEmitter {
     return this.client.rescan(tip.hash);
   }
 
+  /**
+   * Interactive scan blockchain from a given height.
+   * Expect this.rescanning to be set to true.
+   * @private
+   * @param {Number} [height=this.state.startHeight]
+   * @param {Boolean} [fullLock=true]
+   * @returns {Promise}
+   */
+
+  async scanInteractive(height, fullLock = true) {
+    assert(this.rescanning, 'WDB: Rescanning guard not set.');
+
+    if (height == null)
+      height = this.state.startHeight;
+
+    assert((height >>> 0) === height, 'WDB: Must pass in a height.');
+
+    this.logger.info(
+      'Rolling back %d blocks.',
+      this.height - height + 1);
+
+    await this.rollback(height);
+
+    this.logger.info(
+      'WalletDB is scanning %d blocks.',
+      this.state.height - height + 1);
+
+    const tip = await this.getTip();
+
+    return this.client.rescanInteractive(tip.hash, fullLock);
+  }
+
   /**
    * Deep Clean:
    * Keep all keys, account data, wallet maps (name and path).
@@ -660,7 +715,7 @@ class WalletDB extends EventEmitter {
     this.rescanning = true;
 
     try {
-      return await this.scan(height);
+      return await this.scanInteractive(height);
     } finally {
       this.rescanning = false;
     }
@@ -2132,7 +2187,7 @@ class WalletDB extends EventEmitter {
    */
 
   async addOutpointMap(b, hash, index, wid) {
-    await this.addOutpoint(hash, index);
+    this.addOutpoint(hash, index);
     return this.addMap(b, layout.o.encode(hash, index), wid);
   }
 
@@ -2431,7 +2486,7 @@ class WalletDB extends EventEmitter {
    * Unconfirm a block's transactions
    * and write the new best hash (SPV version).
    * @param {ChainEntry} entry
-   * @returns {Promise}
+   * @returns {Promise<Number>} - number of txs removed.
    */
 
   async removeBlock(entry) {
@@ -2447,7 +2502,7 @@ class WalletDB extends EventEmitter {
    * Unconfirm a block's transactions.
    * @private
    * @param {ChainEntry} entry
-   * @returns {Promise}
+   * @returns {Promise<Number>} - number of txs removed.
    */
 
   async _removeBlock(entry) {
@@ -2524,6 +2579,41 @@ class WalletDB extends EventEmitter {
     }
   }
 
+  /**
+   * Rescan a block interactively.
+   * @param {ChainEntry} entry
+   * @param {TX[]} txs
+   * @returns {Promise<ScanAction>} - interactive action
+   */
+
+  async rescanBlockInteractive(entry, txs) {
+    if (!this.rescanning)
+      throw new Error(`WDB: Unsolicited rescan block: ${entry.height}.`);
+
+    if (entry.height > this.state.height + 1)
+      throw new Error(`WDB: Rescan block too high: ${entry.height}.`);
+
+    const blockAdded = await this._addBlock(entry, txs);
+
+    if (!blockAdded)
+      throw new Error('WDB: Block not added.');
+
+    if (blockAdded.filterUpdated) {
+      // We remove block, because adding the same block twice, will ignore
+      // already indexed transactions. This handles the case where single
+      // transaction has undiscovered outputs.
+      await this._removeBlock(entry);
+
+      return {
+        type: scanActions.REPEAT
+      };
+    }
+
+    return {
+      type: scanActions.NEXT
+    };
+  }
+
   /**
    * Add a transaction to the database, map addresses
    * to wallet IDs, potentially store orphans, resolve