HAMT: rehashing when hash is exhausted

src/hamt/bucket.js

@@ -7,9 +7,10 @@ const defaultOptions = {
 }
 
 module.exports = class Bucket {
-  constructor (options, parent, posAtParent) {
+  constructor (options, depth, parent, posAtParent) {
     this._options = Object.assign({}, defaultOptions, options)
     this._popCount = 0
+    this._depth = depth || 0
     this._parent = parent
     this._posAtParent = posAtParent
 
@@ -99,19 +100,13 @@ module.exports = class Bucket {
   }
 
   _findPlace (key, callback) {
-    this._options.hash(key, (err, hashValue) => {
+    const hashValue = this._options.hash(key)
+    hashValue.take(this._options.bits, (err, index) => {
       if (err) {
         callback(err)
         return // early
       }
 
-      if (hashValue.availableBits() < this._options.bits) {
-        // TODO: handle this better by calculating another hash
-        callback(new Error('no more hash bits'))
-        return // early
-      }
-      const index = hashValue.take(this._options.bits)
-
       // and use it as an index into the child
       const child = this._children[index]
       if (child instanceof Bucket) {
@@ -138,7 +133,7 @@ module.exports = class Bucket {
       if (child && child.key !== key) {
         // conflict
 
-        const bucket = new Bucket(this._options, place.bucket, place.pos)
+        const bucket = new Bucket(this._options, this._depth + 1, place.bucket, place.pos)
         place.bucket._putObjectAt(place.pos, bucket)
 
         // put the previous value
@@ -185,7 +180,7 @@ module.exports = class Bucket {
     if (this._parent && this._popCount <= 1) {
       if (this._popCount === 1) {
         // remove myself from parent, replacing me with my only child
-        const onlyChild = this._children.find(identity)
+        const onlyChild = this._children.find(exists)
         if (!(onlyChild instanceof Bucket)) {
           const hash = onlyChild.hash
           hash.untake(this._options.bits)
@@ -206,8 +201,8 @@ module.exports = class Bucket {
   }
 }
 
-function identity (o) {
-  return o
+function exists (o) {
+  return Boolean(o)
 }
 
 function mapNode (node, index) {

src/hamt/consumable-buffer.js

@@ -79,4 +79,4 @@ function byteBitsToInt (byte, start, length) {
 
 function maskFor (start, length) {
   return START_MASKS[start] & STOP_MASKS[Math.min(length + start - 1, 7)]
-}
+}

src/hamt/consumable-hash.js

@@ -5,7 +5,7 @@ const ConsumableBuffer = require('./consumable-buffer')
 
 module.exports = function wrapHash (hashFn) {
   return function hashing (value) {
-    if (value instanceof ConsumableBuffer) {
+    if (value instanceof InfiniteHash) {
       // already a hash. return it
       return value
     } else {
@@ -16,6 +16,9 @@ module.exports = function wrapHash (hashFn) {
 
 class InfiniteHash {
   constructor (value, hashFn) {
+    if ((typeof value) !== 'string' && !Buffer.isBuffer(value)) {
+      throw new Error('can only hash strings or buffers')
+    }
     this._value = value
     this._hashFn = hashFn
     this._depth = -1
@@ -24,7 +27,7 @@ class InfiniteHash {
     this._buffers = []
   }
 
-  take(bits, callback) {
+  take (bits, callback) {
     let pendingBits = bits
     whilst(
       () => this._availableBits < pendingBits,
@@ -59,7 +62,7 @@ class InfiniteHash {
               return // early
             }
 
-            callback(null, result)
+            process.nextTick(() => callback(null, result))
           }
         )
       }
@@ -74,7 +77,7 @@ class InfiniteHash {
       hash.untake(availableForUntake)
       pendingBits -= availableForUntake
       this._availableBits += availableForUntake
-      if (this._currentBufferIndex > 0 && hash.totalBits() == hash.availableBits()) {
+      if (this._currentBufferIndex > 0 && hash.totalBits() === hash.availableBits()) {
         this._depth--
         this._currentBufferIndex--
       }
@@ -96,4 +99,4 @@ class InfiniteHash {
       callback()
     })
   }
-}
+}

test/browser.js

@@ -60,5 +60,6 @@ describe('IPFS data importing tests on the Browser', function () {
   require('./test-import-export')(repo)
   require('./test-hash-parity-with-go-ipfs')(repo)
   require('./test-consumable-buffer')
+  require('./test-consumable-hash')
   require('./test-hamt')
 })

test/node.js

@@ -45,5 +45,6 @@ describe('IPFS UnixFS Engine', () => {
   require('./test-import-export')(repo)
   require('./test-hash-parity-with-go-ipfs')(repo)
   require('./test-consumable-buffer')
+  require('./test-consumable-hash')
   require('./test-hamt')
 })

test/test-consumable-hash.js

@@ -43,7 +43,7 @@ describe('consumable hash', () => {
           values.push(result)
           expect(result).to.be.below(1024)
           expect(result).to.be.above(0)
-          iter --
+          iter--
           callback()
         })
       },
@@ -64,7 +64,7 @@ describe('consumable hash', () => {
           expect(err).to.not.exist
           values.push(result)
           expect(result).to.be.eql(values.shift())
-          iter --
+          iter--
           callback()
         })
       },
@@ -73,10 +73,9 @@ describe('consumable hash', () => {
   })
 })
 
-
-function hashFn(value, callback) {
+function hashFn (value, callback) {
   callback(null, crypto
     .createHash('sha256')
     .update(value)
     .digest())
-}
+}

test/test-hamt.js

@@ -4,6 +4,7 @@
 const crypto = require('crypto')
 const expect = require('chai').expect
 const each = require('async/each')
+const eachSeries = require('async/eachSeries')
 
 const HAMT = require('../src/hamt')
 
@@ -149,4 +150,36 @@ describe('HAMT', () => {
       })
     })
   })
+
+  describe('exhausting hash', () => {
+    let bucket
+
+    before(() => {
+      bucket = HAMT({
+        hashFn: smallHashFn,
+        bits: 2
+      })
+    })
+
+    it('iterates', (callback) => {
+      const size = 300
+      const keys = Array(size)
+      for (let i = 0; i < size; i++) {
+        keys[i] = i.toString()
+      }
+
+      eachSeries(keys, (key, callback) => bucket.put(key, key, callback), (err) => {
+        expect(err).to.not.exist
+        callback()
+      })
+    })
+
+    function smallHashFn (value, callback) {
+      callback(null, crypto
+        .createHash('sha256')
+        .update(value)
+        .digest()
+        .slice(0, 2)) // just return the 2 first bytes of the hash
+    }
+  })
 })