Merge pull request #407 from clark800/rangeset

Add support for efficient range checking to RangeSet

src/api/ledger/transaction.js

@@ -11,12 +11,7 @@ function hasCompleteLedgerRange(remote, options) {
   const minLedgerVersion = options.minLedgerVersion || MIN_LEDGER_VERSION;
   const maxLedgerVersion = options.maxLedgerVersion
     || remote.getLedgerSequence();
-  for (let i = minLedgerVersion; i <= maxLedgerVersion; i++) {
-    if (!remote.getServer().hasLedger(i)) { // TODO: optimize this
-      return false;
-    }
-  }
-  return true;
+  return remote.getServer().hasLedgerRange(minLedgerVersion, maxLedgerVersion);
 }
 
 function attachTransactionDate(remote, tx, callback) {

src/core/index.js

@@ -18,13 +18,13 @@ exports.utils = require('./utils');
 exports.Server = require('./server').Server;
 exports.Ledger = require('./ledger').Ledger;
 exports.TransactionQueue = require('./transactionqueue').TransactionQueue;
-exports.RangeSet = require('./rangeset').RangeSet;
 exports.convertBase = require('./baseconverter');
 
 exports._test = {
   Log: require('./log'),
   PathFind: require('./pathfind').PathFind,
-  TransactionManager: require('./transactionmanager').TransactionManager
+  TransactionManager: require('./transactionmanager').TransactionManager,
+  RangeSet: require('./rangeset').RangeSet
 };
 
 // Important: We do not guarantee any specific version of SJCL or for any

src/core/rangeset.js

@@ -1,67 +1,61 @@
-var assert = require('assert');
-var lodash = require('lodash');
-
-function RangeSet() {
-  this._ranges = [ ];
-};
-
-/**
- * Add a ledger range
- *
- * @param {Number|String} range string (n-n2,n3-n4)
- */
-
-RangeSet.prototype.add = function(range) {
-  assert(typeof range !== 'number' || !isNaN(range), 'Ledger range malformed');
-
-  range = String(range).split(',');
-
-  if (range.length > 1) {
-    return range.forEach(this.add, this);
-  }
-
-  range = range[0].split('-').map(Number);
-
-  var lRange = {
-    start: range[0],
-    end: range[range.length === 1 ? 0 : 1]
-  };
-
-  // Comparisons on NaN should be falsy
-  assert(lRange.start <= lRange.end, 'Ledger range malformed');
-
-  var insertionPoint = lodash.sortedIndex(this._ranges, lRange, function(r) {
-    return r.start;
+/* @flow */
+'use strict';
+const _ = require('lodash');
+const assert = require('assert');
+const ranges = Symbol();
+
+function mergeIntervals(intervals: Array<[number, number]>) {
+  const stack = [[-Infinity, -Infinity]];
+  _.forEach(_.sortBy(intervals, x => x[0]), interval => {
+    const lastInterval = stack.pop();
+    if (interval[0] <= lastInterval[1] + 1) {
+      stack.push([lastInterval[0], Math.max(interval[1], lastInterval[1])]);
+    } else {
+      stack.push(lastInterval);
+      stack.push(interval);
+    }
   });
+  return stack.slice(1);
+}
 
-  this._ranges.splice(insertionPoint, 0, lRange);
-};
+class RangeSet {
+  constructor() {
+    this.reset();
+  }
 
+  reset() {
+    this[ranges] = [];
+  }
 
-/*
- * Check presence of ledger in range
- *
- * @param {Number|String} ledger
- * @return Boolean
- */
+  serialize() {
+    return this[ranges].map(range =>
+      range[0].toString() + '-' + range[1].toString()).join(',');
+  }
 
-RangeSet.prototype.has =
-RangeSet.prototype.contains = function(ledger) {
-  assert(ledger != null && !isNaN(ledger), 'Ledger must be a number');
+  addRange(start: number, end: number) {
+    assert(start <= end, 'invalid range');
+    this[ranges] = mergeIntervals(this[ranges].concat([[start, end]]));
+  }
 
-  ledger = Number(ledger);
+  addValue(value: number) {
+    this.addRange(value, value);
+  }
 
-  return this._ranges.some(function(r) {
-    return ledger >= r.start && ledger <= r.end;
-  });
-};
+  parseAndAddRanges(rangesString: string) {
+    const rangeStrings = rangesString.split(',');
+    _.forEach(rangeStrings, rangeString => {
+      const range = rangeString.split('-').map(Number);
+      this.addRange(range[0], range[1]);
+    });
+  }
 
-/**
- * Reset ledger ranges
- */
+  containsRange(start: number, end: number) {
+    return _.some(this[ranges], range => range[0] <= start && range[1] >= end);
+  }
 
-RangeSet.prototype.reset = function() {
-  this._ranges = [ ];
-};
+  containsValue(value: number) {
+    return this.containsRange(value, value);
+  }
+}
 
 exports.RangeSet = RangeSet;

src/core/server.js

@@ -597,7 +597,7 @@ Server.prototype._handleMessage = function(message) {
 Server.prototype._handleLedgerClosed = function(message) {
   this._lastLedgerIndex = message.ledger_index;
   this._lastLedgerClose = Date.now();
-  this._ledgerRanges.add(message.ledger_index);
+  this._ledgerRanges.addValue(message.ledger_index);
   this._ledgerMap.set(message.ledger_hash, message.ledger_index);
   this.emit('ledger_closed', message);
 };
@@ -713,7 +713,7 @@ Server.prototype._handleResponseSubscribe = function(message) {
 
   if (message.validated_ledgers) {
     // Add validated ledgers to ledger range set
-    this._ledgerRanges.add(message.validated_ledgers);
+    this._ledgerRanges.parseAndAddRanges(message.validated_ledgers);
   }
 
   if (~Server.onlineStates.indexOf(message.server_status)) {
@@ -913,12 +913,16 @@ Server.prototype.hasLedger = function(ledger) {
   if (typeof ledger === 'string' && /^[A-F0-9]{64}$/.test(ledger)) {
     result = this._ledgerMap.has(ledger);
   } else if (ledger !== null && !isNaN(ledger)) {
-    result = this._ledgerRanges.has(ledger);
+    result = this._ledgerRanges.containsValue(ledger);
   }
 
   return result;
 };
 
+Server.prototype.hasLedgerRange = function(startLedger, endLedger) {
+  return this._ledgerRanges.containsRange(startLedger, endLedger);
+};
+
 /**
  * Get ledger index of last seen validated ledger
  *

test/rangeset-test.js

@@ -1,78 +1,55 @@
-var assert = require('assert');
-var RangeSet = require('ripple-lib').RangeSet;
+'use strict';
+const assert = require('assert');
+const RangeSet = require('ripple-lib')._test.RangeSet;
 
 describe('RangeSet', function() {
-  it('add()', function() {
-    var r = new RangeSet();
+  it('addRange()/addValue()', function() {
+    const r = new RangeSet();
 
-    r.add('4-5');
-    r.add('7-10');
-    r.add('1-2');
-    r.add('3');
+    r.addRange(4, 5);
+    r.addRange(7, 10);
+    r.addRange(1, 2);
+    r.addValue(3);
 
-    assert.deepEqual(r._ranges, [
-      { start: 1, end: 2 },
-      { start: 3, end: 3 },
-      { start: 4, end: 5 },
-      { start: 7, end: 10 }
-    ]);
+    assert.deepEqual(r.serialize(), '1-5,7-10');
   });
 
-  it('add() -- malformed range', function() {
-    var r = new RangeSet();
-
-    assert.throws(function() {
-      r.add(null);
-    });
-    assert.throws(function() {
-      r.add(void(0));
-    });
-    assert.throws(function() {
-      r.add('a');
-    });
+  it('addValue()/addRange() -- malformed', function() {
+    const r = new RangeSet();
     assert.throws(function() {
-      r.add('2-1');
+      r.addRange(2, 1);
     });
   });
 
-  it('contains()', function() {
-    var r = new RangeSet();
-
-    r.add('32570-11005146');
-    r.add('11005147');
-
-    assert.strictEqual(r.contains(1), false);
-    assert.strictEqual(r.contains(32569), false);
-    assert.strictEqual(r.contains(32570), true);
-    assert.strictEqual(r.contains('32570'), true);
-    assert.strictEqual(r.contains(50000), true);
-    assert.strictEqual(r.contains(11005146), true);
-    assert.strictEqual(r.contains(11005147), true);
-    assert.strictEqual(r.contains(11005148), false);
-    assert.strictEqual(r.contains(12000000), false);
+  it('parseAndAddRanges()', function() {
+    const r = new RangeSet();
+    r.parseAndAddRanges('4-5,7-10,1-2,3-3');
+    assert.deepEqual(r.serialize(), '1-5,7-10');
   });
 
-  it('contains() -- invalid ledger', function() {
-    var r = new RangeSet();
+  it('containsValue()', function() {
+    const r = new RangeSet();
 
-    assert.throws(function() {
-      r.contains(null);
-    });
-    assert.throws(function() {
-      r.contains(void(0));
-    });
-    assert.throws(function() {
-      r.contains('a');
-    });
+    r.addRange(32570, 11005146);
+    r.addValue(11005147);
+
+    assert.strictEqual(r.containsValue(1), false);
+    assert.strictEqual(r.containsValue(32569), false);
+    assert.strictEqual(r.containsValue(32570), true);
+    assert.strictEqual(r.containsValue(50000), true);
+    assert.strictEqual(r.containsValue(11005146), true);
+    assert.strictEqual(r.containsValue(11005147), true);
+    assert.strictEqual(r.containsValue(11005148), false);
+    assert.strictEqual(r.containsValue(12000000), false);
   });
 
   it('reset()', function() {
-    var r = new RangeSet();
+    const r = new RangeSet();
 
-    r.add('4-5');
-    r.add('7-10');
+    r.addRange(4, 5);
+    r.addRange(7, 10);
     r.reset();
 
-    assert.deepEqual(r._ranges, [ ]);
+    assert.deepEqual(r.serialize(), '');
   });
 });

test/request-test.js

@@ -257,8 +257,8 @@ describe('Request', function() {
       });
     };
 
-    servers[0]._ledgerRanges.add('5-6');
-    servers[1]._ledgerRanges.add('1-4');
+    servers[0]._ledgerRanges.addRange(5, 6);
+    servers[1]._ledgerRanges.addRange(1, 4);
 
     const remote = new Remote();
     remote._connected = true;