fix(node): detecting dupe transactions and coins with mismatched heights

Improving verification scripts to detect duplicate transactions and coins with differing heights
matiu · Feb 27, 2019 · 3891140 · 3891140
1 parent 499dc72
commit 3891140
Show file tree

Hide file tree

Showing 2 changed files with 82 additions and 28 deletions.
diff --git a/packages/bitcore-node/test/verification/db-repair.ts b/packages/bitcore-node/test/verification/db-repair.ts
@@ -7,6 +7,7 @@ import { P2pWorker } from '../../src/services/p2p';
 import { Config } from '../../src/services/config';
 import { BlockStorage } from '../../src/models/block';
 import { validateDataForBlock } from './db-verify';
+import { TransactionStorage } from '../../src/models/transaction';
 
 (async () => {
   const { CHAIN, NETWORK, FILE, DRYRUN } = process.env;
@@ -23,38 +24,70 @@ import { validateDataForBlock } from './db-verify';
 
   const handleRepair = async data => {
     switch (data.type) {
-      case 'DUPE_COIN':
-        const coin = data.payload.coin;
-        const dupeCoins = await CoinStorage.collection
-          .find({ chain, network, mintTxid: coin.mintTxid, mintIndex: coin.mintIndex })
-          .sort({ _id: -1 })
-          .toArray();
+      case 'DUPE_TRANSACTION':
+        {
+          const tx = data.payload.tx;
+          const dupeTxs = await TransactionStorage.collection
+            .find({ chain: tx.chain, network: tx.network, txid: tx.txid })
+            .sort({ blockHeight: -1 })
+            .toArray();
 
-        if (dupeCoins.length < 2) {
-          console.log('No action required.', dupeCoins.length, 'coin');
-          return;
+          if (dupeTxs.length < 2) {
+            console.log('No action required.', dupeTxs.length, 'transaction');
+            return;
+          }
+
+          let toKeep = dupeTxs[0];
+          const wouldBeDeleted = dupeTxs.filter(c => c._id != toKeep._id);
+
+          if (DRYRUN) {
+            console.log('WOULD DELETE');
+            console.log(wouldBeDeleted);
+          } else {
+            console.log('Deleting', wouldBeDeleted.length, 'transactions');
+            await TransactionStorage.collection.deleteMany({
+              chain,
+              network,
+              _id: { $in: wouldBeDeleted.map(c => c._id) }
+            });
+          }
         }
+        break;
+      case 'DUPE_COIN':
+        {
+          const coin = data.payload.coin;
+          const dupeCoins = await CoinStorage.collection
+            .find({ chain, network, mintTxid: coin.mintTxid, mintIndex: coin.mintIndex })
+            .sort({ _id: -1 })
+            .toArray();
 
-        let toKeep = dupeCoins[0];
-        const spentCoin = dupeCoins.find(c => c.spentHeight > toKeep.spentHeight);
-        toKeep = spentCoin || toKeep;
-        const wouldBeDeleted = dupeCoins.filter(c => c._id != toKeep._id);
+          if (dupeCoins.length < 2) {
+            console.log('No action required.', dupeCoins.length, 'coin');
+            return;
+          }
 
-        if (DRYRUN) {
-          console.log('WOULD DELETE');
-          console.log(wouldBeDeleted);
-        } else {
-          const { mintIndex, mintTxid } = toKeep;
-          console.log('Deleting', wouldBeDeleted.length, 'coins');
-          await CoinStorage.collection.deleteMany({
-            chain,
-            network,
-            mintTxid,
-            mintIndex,
-            _id: { $in: wouldBeDeleted.map(c => c._id) }
-          });
+          let toKeep = dupeCoins[0];
+          const spentCoin = dupeCoins.find(c => c.spentHeight > toKeep.spentHeight);
+          toKeep = spentCoin || toKeep;
+          const wouldBeDeleted = dupeCoins.filter(c => c._id != toKeep._id);
+
+          if (DRYRUN) {
+            console.log('WOULD DELETE');
+            console.log(wouldBeDeleted);
+          } else {
+            const { mintIndex, mintTxid } = toKeep;
+            console.log('Deleting', wouldBeDeleted.length, 'coins');
+            await CoinStorage.collection.deleteMany({
+              chain,
+              network,
+              mintTxid,
+              mintIndex,
+              _id: { $in: wouldBeDeleted.map(c => c._id) }
+            });
+          }
         }
         break;
+      case 'COIN_HEIGHT_MISMATCH':
       case 'MISSING_BLOCK':
       case 'MISSING_TX':
       case 'MISSING_COIN_FOR_TXID':

diff --git a/packages/bitcore-node/test/verification/db-verify.ts b/packages/bitcore-node/test/verification/db-verify.ts
@@ -4,6 +4,7 @@ import { BlockStorage } from '../../src/models/block';
 import { CoinStorage, ICoin } from '../../src/models/coin';
 import { TransactionStorage, ITransaction } from '../../src/models/transaction';
 import { Storage } from '../../src/services/storage';
+import * as _ from 'lodash';
 
 const { CHAIN, NETWORK, HEIGHT } = process.env;
 const resumeHeight = Number(HEIGHT) || 1;
@@ -33,10 +34,20 @@ export async function validateDataForBlock(blockNum: number, log = false) {
         console.log(JSON.stringify(error));
       }
     }
-    seenTxs[tx.txid] = tx;
+    if (seenTxs[tx.txid]) {
+      success = false;
+      const error = { model: 'transaction', err: true, type: 'DUPE_TRANSACTION', payload: { tx, blockNum } };
+      errors.push(error);
+      if (log) {
+        console.log(JSON.stringify(error));
+      }
+    } else {
+      seenTxs[tx.txid] = tx;
+    }
   }
 
   const blockTxids = blockTxs.map(t => t.txid);
+
   const coinsForTx = await CoinStorage.collection.find({ chain, network, mintTxid: { $in: blockTxids } }).toArray();
   for (let coin of coinsForTx) {
     if (seenTxCoins[coin.mintTxid] && seenTxCoins[coin.mintTxid][coin.mintIndex]) {
@@ -52,6 +63,16 @@ export async function validateDataForBlock(blockNum: number, log = false) {
     }
   }
 
+  const mintHeights = _.uniq(coinsForTx.map(c => c.mintHeight));
+  if (mintHeights.length > 1) {
+    success = false;
+    const error = { model: 'coin', err: true, type: 'COIN_HEIGHT_MISMATCH', payload: { blockNum } };
+    errors.push(error);
+    if (log) {
+      console.log(JSON.stringify(error));
+    }
+  }
+
   for (let txid of Object.keys(seenTxs)) {
     const coins = seenTxCoins[txid];
     if (!coins) {
@@ -152,7 +173,7 @@ if (require.main === module) {
     const tip = await BlockStorage.getLocalTip({ chain, network });
 
     if (tip) {
-      for (let i = resumeHeight; i < tip.height; i++) {
+      for (let i = resumeHeight; i <= tip.height; i++) {
         const { success } = await validateDataForBlock(i, true);
         console.log({ block: i, success });
       }