Merge PR #515 from 'pinheadmz/expiry1'

CHANGELOG.md

@@ -12,6 +12,10 @@ the miner will add transactions up to the minimum weight that would normally be
 ignored for being "free" (paying a fee below policy limit). The default value is
 raised from `0` to `5000` (a 1-in, 2-out BID transaction has a weight of about `889`).
 
+- Transactions that have sat unconfirmed in the mempool for 3 days will be evicted.
+This is the default `MEMPOOL_EXPIRY_TIME` value set in `policy.js` but can be
+configured (in seconds) with the `FullNode` option `--mempool-expiry-time`.
+
 ### Wallet API changes
 
 - Adds new wallet HTTP endpoint `/deepclean` that requires a parameter

lib/mempool/mempool.js

@@ -583,14 +583,29 @@ class Mempool extends EventEmitter {
 
   limitSize(added) {
     const maxSize = this.options.maxSize;
+    const expiryTime = this.options.expiryTime;
+    const now = util.now();
+
+    // Clear out expired transactions first
+    for (const entry of this.map.values()) {
+      // If this tx has ancestors in the mempool, we will find and
+      // evict the oldest and its entire chain of descendants
+      if (this.hasDepends(entry.tx))
+        continue;
+
+      if (now >= entry.time + expiryTime) {
+        this.logger.debug(
+          'Removing package %x from mempool (too old).',
+          entry.hash());
+
+        this.evictEntry(entry);
+      }
+    }
 
     if (this.size <= maxSize)
       return false;
 
     const threshold = maxSize - (maxSize / 10);
-    const expiryTime = this.options.expiryTime;
-
-    const now = util.now();
     const queue = new Heap(cmpRate);
 
     let start = util.bench();
@@ -599,16 +614,7 @@ class Mempool extends EventEmitter {
       if (this.hasDepends(entry.tx))
         continue;
 
-      if (now < entry.time + expiryTime) {
-        queue.insert(entry);
-        continue;
-      }
-
-      this.logger.debug(
-        'Removing package %x from mempool (too old).',
-        entry.hash());
-
-      this.evictEntry(entry);
+      queue.insert(entry);
     }
 
     if (this.size <= threshold) {

lib/node/fullnode.js

@@ -77,7 +77,8 @@ class FullNode extends Node {
       limitFreeRelay: this.config.uint('limit-free-relay'),
       requireStandard: this.config.bool('require-standard'),
       rejectAbsurdFees: this.config.bool('reject-absurd-fees'),
-      indexAddress: this.config.bool('index-address')
+      indexAddress: this.config.bool('index-address'),
+      expiryTime: this.config.uint('mempool-expiry-time')
     });
 
     // Pool needs access to the chain and mempool.

test/mempool-test.js

@@ -1099,4 +1099,207 @@ describe('Mempool', function() {
       assert(chain.tip.time < data.expiration);
     });
   });
+
+  describe('Mempool eviction', function () {
+    // Computed in advance with MempoolEntry.memUsage()
+    const txMemUsage = 1728;
+    // Should allow 9 transactions in mempool.
+    // The 10th transaction will push the mempool size over 100% of the limit.
+    // Mempool will then remove two transactions to get under 90% limit.
+    const maxSize = txMemUsage * 10 - 1;
+    // 1 hour
+    const expiryTime = 60 * 60;
+
+    const workers = new WorkerPool({
+      enabled: true,
+      size: 2
+    });
+
+    const chain = new Chain({
+      memory: true,
+      workers,
+      network: 'regtest'
+    });
+
+    const mempool = new Mempool({
+      chain,
+      workers,
+      memory: true,
+      maxSize,
+      expiryTime
+    });
+
+    before(async () => {
+      await mempool.open();
+      await chain.open();
+      await workers.open();
+    });
+
+    after(async () => {
+      await workers.close();
+      await chain.close();
+      await mempool.close();
+    });
+
+    // Number of coins available in
+    // chaincoins (100k satoshi per coin).
+    const N = 100;
+    const chaincoins = new MemWallet();
+    const wallet = new MemWallet();
+
+    async function getMockBlock(chain, txs = [], cb = true) {
+      if (cb) {
+        const raddr = KeyRing.generate().getAddress();
+        const mtx = new MTX();
+        mtx.addInput(new Input());
+        mtx.addOutput(raddr, 0);
+        mtx.locktime = chain.height + 1;
+
+        txs = [mtx.toTX(), ...txs];
+      }
+
+      const view = new CoinView();
+      for (const tx of txs) {
+        view.addTX(tx, -1);
+      }
+
+      const now = Math.floor(Date.now() / 1000);
+      const time = chain.tip.time <= now ? chain.tip.time + 1 : now;
+
+      const block = new Block();
+      block.txs = txs;
+      block.prevBlock = chain.tip.hash;
+      block.time = time;
+      block.bits = await chain.getTarget(block.time, chain.tip);
+
+      // Ensure mockblocks are unique (required for reorg testing)
+      block.merkleRoot = block.createMerkleRoot();
+      block.witnessRoot = block.createWitnessRoot();
+      block.treeRoot = chain.db.treeRoot();
+
+      return [block, view];
+    }
+
+    it('should create coins in chain', async () => {
+      const mtx = new MTX();
+      mtx.locktime = chain.height + 1;
+      mtx.addInput(new Input());
+
+      for (let i = 0; i < N; i++) {
+        const addr = chaincoins.createReceive().getAddress();
+        mtx.addOutput(addr, 100000);
+      }
+
+      const cb = mtx.toTX();
+      const [block] = await getMockBlock(chain, [cb], false);
+      const entry = await chain.add(block, VERIFY_BODY);
+
+      await mempool._addBlock(entry, block.txs);
+
+      // Add 100 blocks so we don't get
+      // premature spend of coinbase.
+      for (let i = 0; i < 100; i++) {
+        const [block] = await getMockBlock(chain);
+        const entry = await chain.add(block, VERIFY_BODY);
+
+        await mempool._addBlock(entry, block.txs);
+      }
+
+      chaincoins.addTX(cb);
+    });
+
+    it('should limit mempool size', async () => {
+      let expectedSize = 0;
+
+      for (let i = 0; i < N; i++) {
+        // Spend a different coin each time to avoid exceeding max ancestors.
+        const coin = chaincoins.getCoins()[i];
+        const addr = wallet.createReceive().getAddress();
+
+        const mtx = new MTX();
+        mtx.addCoin(coin);
+        // Increment fee with each TX so oldest TX gets evicted first.
+        // Otherwise the new TX might be the one that gets evicted,
+        // resulting in a "mempool full" error instead.
+        mtx.addOutput(addr, 90000 - (10 * i));
+        chaincoins.sign(mtx);
+        const tx = mtx.toTX();
+
+        expectedSize += txMemUsage;
+
+        if (expectedSize < maxSize) {
+          await mempool.addTX(tx);
+        } else {
+          assert(i >= 9);
+          let evicted = false;
+          mempool.once('remove entry', () => {
+            evicted = true;
+            // We've exceeded the max size by 1 TX
+            // Mempool will remove 2 TXs to get below 90% limit.
+            expectedSize -= txMemUsage * 2;
+          });
+          await mempool.addTX(tx);
+          assert(evicted);
+        }
+      }
+    });
+
+    it('should evict old transactions', async () => {
+      // Clear mempool. Note that TXs in last test were not
+      // added to the wallet: we can re-spend those coins.
+      await mempool.reset();
+
+      let now = 0;
+      const original = util.now;
+      try {
+        util.now = () => {
+          return now;
+        };
+
+        // After we cross the expiry threshold, one TX at a time
+        // will start to expire, starting with the oldest.
+        const sent = [];
+        let evicted = 0;
+        mempool.on('remove entry', (entry) => {
+          const expected = sent.shift();
+          assert.bufferEqual(entry.tx.hash(), expected);
+          evicted++;
+        });
+
+        for (let i = 0; i < N; i++) {
+          // Spend a different coin each time to avoid exceeding max ancestors.
+          const coin = chaincoins.getCoins()[i];
+          const addr = wallet.createReceive().getAddress();
+
+          const mtx = new MTX();
+          mtx.addCoin(coin);
+          mtx.addOutput(addr, 90000);
+          chaincoins.sign(mtx);
+          const tx = mtx.toTX();
+
+          sent.push(tx.hash());
+
+          // mempool size is not a factor
+          assert(mempool.size  + (txMemUsage * 2) < maxSize);
+
+          await mempool.addTX(tx);
+
+          // Time travel forward ten minutes
+          now += 60 * 10;
+
+          // The first 6 TXs are added without incident.
+          // After that, a virtual hour will have passed, and
+          // each new TX will trigger the eviction of one old TX.
+          if (i < 6) {
+            assert(mempool.map.size === i + 1);
+          } else {
+            assert(mempool.map.size === 6);
+            assert.strictEqual(evicted, (i + 1) - 6);
+          }
+        }
+      } finally {
+        util.now = original;
+      }
+    });
+  });
 });