Merge pull request #319 from karlfloersch/feat/range_bucket_iterator

Add range bucket iterator

packages/core/src/app/db/iterator.ts

@@ -5,9 +5,18 @@
 
 /* External Imports */
 import { AbstractIterator } from 'abstract-leveldown'
+import BigNum = require('bn.js')
 
 /* Internal Imports */
-import { Iterator, IteratorOptions, K, V, KV, DB } from '../../types'
+import {
+  Iterator,
+  IteratorOptions,
+  K,
+  V,
+  KV,
+  DB,
+  RangeEntry,
+} from '../../types'
 
 const defaultIteratorOptions: IteratorOptions = {
   reverse: false,
@@ -217,3 +226,33 @@ export class BaseIterator implements Iterator {
     return value ? value.slice(this.prefix.length) : value
   }
 }
+
+/**
+ * A special purpose iterator which includes a nextRange() function that returns RangeEntrys instead of simple KVs.
+ * This is used by the RangeBucket class.
+ */
+export class BaseRangeIterator extends BaseIterator {
+  /**
+   * Constructs a RangeIterator with a particular `resultToRange()` function that will transform
+   * the it.next() result into a RangeEntry.
+   */
+  constructor(
+    db: DB,
+    options: IteratorOptions = {},
+    readonly resultToRange: (result: KV) => RangeEntry
+  ) {
+    super(db, options)
+  }
+
+  /**
+   * Advances the iterator to the next key and converts its result into a RangeEntry.
+   * @returns the RangeEntry at the next key.
+   */
+  public async nextRange(): Promise<RangeEntry> {
+    const res: KV = await this.next()
+    if (typeof res.key === 'undefined') {
+      return
+    }
+    return this.resultToRange(res)
+  }
+}

packages/core/src/app/db/range-bucket.ts

@@ -15,12 +15,14 @@ import {
   Iterator,
   K,
   V,
+  KV,
   RangeBucket,
   RangeEntry,
+  RangeIterator,
   Endianness,
   PUT_BATCH_TYPE,
 } from '../../types'
-import { bufferUtils, intersects, BaseDB } from '../../app'
+import { bufferUtils, intersects, BaseDB, BaseRangeIterator } from '../../app'
 
 /* Logging */
 import debug from 'debug'
@@ -46,12 +48,14 @@ export class BaseRangeBucket implements RangeBucket {
   ) {}
 
   /**
-   * Adds this RangeBucket's prefix to the target Buffer
-   * @param target A Buffer which will have the prefix prepended to it.
-   * @returns resulting Buffer `prefix+target`.
+   * Concatenates some value to this bucket's prefix.
+   * @param value Value to concatenate.
+   * @returns the value concatenated to the prefix.
    */
-  private addPrefix(target: Buffer): Buffer {
-    return Buffer.concat([this.prefix, target])
+  private addPrefix(value: Buffer): Buffer {
+    return value !== undefined
+      ? Buffer.concat([this.prefix, value])
+      : this.prefix
   }
 
   /**
@@ -134,7 +138,7 @@ export class BaseRangeBucket implements RangeBucket {
    * @param result The resulting value which has been extracted from our DB.
    * @returns a range object with {start, end, value}
    */
-  private resultToRange(result): RangeEntry {
+  private resultToRange(result: KV): RangeEntry {
     // Helper function which gets the start and end position from a DB seek result
     return {
       start: new BigNum(this.getStartFromValue(result.value)),
@@ -288,6 +292,23 @@ export class BaseRangeBucket implements RangeBucket {
     return ranges
   }
 
+  /**
+   * Creates an iterator with some options.
+   * @param options Parameters for the iterator.
+   * @returns the iterator instance.
+   */
+  public iterator(options: IteratorOptions = {}): RangeIterator {
+    const rangeIterator = new BaseRangeIterator(
+      this.db,
+      {
+        ...options,
+        prefix: this.addPrefix(options.prefix),
+      },
+      (res: KV) => this.resultToRange(res)
+    )
+    return rangeIterator
+  }
+
   /**
    * Creates a prefixed bucket underneath
    * this bucket.

packages/core/src/types/db/db.interface.ts

@@ -6,7 +6,7 @@ import {
 } from 'abstract-leveldown'
 
 /* Internal Imports */
-import { RangeBucket } from '../../types'
+import { RangeBucket, RangeEntry } from '../../types'
 
 export type K = NonNullable<Buffer>
 export type V = NonNullable<Buffer>
@@ -182,3 +182,7 @@ export interface Iterator {
    */
   end(): Promise<void>
 }
+
+export interface RangeIterator extends Iterator {
+  nextRange(): Promise<RangeEntry>
+}

packages/core/src/types/db/range-db.interface.ts

@@ -4,7 +4,13 @@ import BigNum = require('bn.js')
 import level from 'level'
 
 /* Internal Imports */
-import { KeyValueStore, V, Bucket } from './db.interface'
+import {
+  KeyValueStore,
+  V,
+  Bucket,
+  RangeIterator,
+  IteratorOptions,
+} from './db.interface'
 
 /**
  * Represents a range of values. Note start & end are big numbers!
@@ -57,6 +63,13 @@ export interface RangeStore {
    * @returns all of the ranges which have been deleted.
    */
   del(start: BigNum, end: BigNum): Promise<RangeEntry[]>
+
+  /**
+   * Creates an iterator with some options.
+   * @param options Parameters for the iterator.
+   * @returns the iterator instance.
+   */
+  iterator(options?: IteratorOptions): RangeIterator
 }
 
 export interface RangeBucket extends RangeStore {

packages/core/test/app/db/range-bucket.spec.ts

@@ -45,6 +45,22 @@ const testPutResults = async (
   db: RangeBucket,
   putContents: any[],
   expectedResults: any[]
+): Promise<void> => {
+  // First put the ranges
+  putRanges(db, putContents)
+  // Now check that they were added correctly
+  const res = await db.get(
+    new BigNum('0', 'hex'),
+    new BigNum('100000000000', 'hex')
+  )
+  for (let i = 0; i < res.length; i++) {
+    compareResult(res[i], expectedResults[i])
+  }
+}
+
+const putRanges = async (
+  db: RangeBucket,
+  putContents: any[]
 ): Promise<void> => {
   for (const putContent of putContents) {
     await db.put(
@@ -53,14 +69,11 @@ const testPutResults = async (
       Buffer.from(putContent.value)
     )
   }
-  const res = await db.get(
-    new BigNum('0', 'hex'),
-    new BigNum('100000000000', 'hex')
-  )
-  for (let i = 0; i < res.length; i++) {
-    const strResult = new StringRangeEntry(res[i])
-    strResult.stringRangeEntry.should.deep.equal(expectedResults[i])
-  }
+}
+
+const compareResult = (res: any, expectedResult: any): void => {
+  const strResult = new StringRangeEntry(res)
+  strResult.stringRangeEntry.should.deep.equal(expectedResult)
 }
 
 describe('RangeDB', () => {
@@ -203,7 +216,7 @@ describe('RangeDB', () => {
   })
 
   it('splits `put(0, 100, x), put(50, 150, y)` into (0, 50, x), (50, 150, y)', async () => {
-    testPutResults(
+    await testPutResults(
       rangeDB,
       [
         { start: '0', end: '100', value: 'x1' },
@@ -217,7 +230,7 @@ describe('RangeDB', () => {
   })
 
   it('splits `put(50, 150, x), put(0, 100, y)` into (0, 50, x), (50, 150, y)', async () => {
-    testPutResults(
+    await testPutResults(
       rangeDB,
       [
         { start: '50', end: '150', value: 'x2' },
@@ -231,7 +244,7 @@ describe('RangeDB', () => {
   })
 
   it('splits `put(0, 100, x), put(0, 100, y)` into (0, 100, y)', async () => {
-    testPutResults(
+    await testPutResults(
       rangeDB,
       [
         { start: '0', end: '100', value: 'x3' },
@@ -242,7 +255,7 @@ describe('RangeDB', () => {
   })
 
   it('splits `put(0, 100, x), put(100, 200, y), put(50, 150, z)` into (0, 50, x), (50, 150, z), (150, 200, y)', async () => {
-    testPutResults(
+    await testPutResults(
       rangeDB,
       [
         { start: '0', end: '100', value: 'x4' },
@@ -256,4 +269,32 @@ describe('RangeDB', () => {
       ]
     )
   })
+
+  describe('iterator()', () => {
+    it('allows nextRange() to be called by the iterator returning a RangeEntry instead of a KV', async () => {
+      const testRanges = {
+        inputs: [
+          { start: '0', end: '100', value: 'x' },
+          { start: '100', end: '200', value: 'y' },
+          { start: '200', end: '225', value: 'z' },
+        ],
+        expectedResults: [
+          { start: '0', end: '100', value: 'x' },
+          { start: '100', end: '200', value: 'y' },
+          { start: '200', end: '225', value: 'z' },
+        ],
+      }
+
+      // Put our ranges
+      await putRanges(rangeDB, testRanges.inputs)
+      // Use a range iterator to get values we expect
+      const it = rangeDB.iterator()
+      const range0 = await it.nextRange()
+      const range1 = await it.nextRange()
+      const range2 = await it.nextRange()
+      compareResult(range0, testRanges.expectedResults[0])
+      compareResult(range1, testRanges.expectedResults[1])
+      compareResult(range2, testRanges.expectedResults[2])
+    })
+  })
 })