From 4915f6c301785e756f92b111156a662a1f60ecdf Mon Sep 17 00:00:00 2001 From: Pedro Teixeira Date: Fri, 20 Jan 2017 18:43:08 +0000 Subject: [PATCH] HAMT: rehashing when hash is exhausted --- src/hamt/bucket.js | 21 ++++++++------------- src/hamt/consumable-buffer.js | 2 +- src/hamt/consumable-hash.js | 13 ++++++++----- test/browser.js | 1 + test/node.js | 1 + test/test-consumable-hash.js | 9 ++++----- test/test-hamt.js | 33 +++++++++++++++++++++++++++++++++ 7 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/hamt/bucket.js b/src/hamt/bucket.js index 63c26366..1b3ce4d5 100644 --- a/src/hamt/bucket.js +++ b/src/hamt/bucket.js @@ -7,9 +7,10 @@ const defaultOptions = { } module.exports = class Bucket { - constructor (options, parent, posAtParent) { + constructor (options, depth, parent, posAtParent) { this._options = Object.assign({}, defaultOptions, options) this._popCount = 0 + this._depth = depth || 0 this._parent = parent this._posAtParent = posAtParent @@ -99,19 +100,13 @@ module.exports = class Bucket { } _findPlace (key, callback) { - this._options.hash(key, (err, hashValue) => { + const hashValue = this._options.hash(key) + hashValue.take(this._options.bits, (err, index) => { if (err) { callback(err) return // early } - if (hashValue.availableBits() < this._options.bits) { - // TODO: handle this better by calculating another hash - callback(new Error('no more hash bits')) - return // early - } - const index = hashValue.take(this._options.bits) - // and use it as an index into the child const child = this._children[index] if (child instanceof Bucket) { @@ -138,7 +133,7 @@ module.exports = class Bucket { if (child && child.key !== key) { // conflict - const bucket = new Bucket(this._options, place.bucket, place.pos) + const bucket = new Bucket(this._options, this._depth + 1, place.bucket, place.pos) place.bucket._putObjectAt(place.pos, bucket) // put the previous value @@ -185,7 +180,7 @@ module.exports = class Bucket { if (this._parent && this._popCount <= 1) { if (this._popCount === 1) { // remove myself from parent, replacing me with my only child - const onlyChild = this._children.find(identity) + const onlyChild = this._children.find(exists) if (!(onlyChild instanceof Bucket)) { const hash = onlyChild.hash hash.untake(this._options.bits) @@ -206,8 +201,8 @@ module.exports = class Bucket { } } -function identity (o) { - return o +function exists (o) { + return Boolean(o) } function mapNode (node, index) { diff --git a/src/hamt/consumable-buffer.js b/src/hamt/consumable-buffer.js index e0990259..69f6a140 100644 --- a/src/hamt/consumable-buffer.js +++ b/src/hamt/consumable-buffer.js @@ -79,4 +79,4 @@ function byteBitsToInt (byte, start, length) { function maskFor (start, length) { return START_MASKS[start] & STOP_MASKS[Math.min(length + start - 1, 7)] -} \ No newline at end of file +} diff --git a/src/hamt/consumable-hash.js b/src/hamt/consumable-hash.js index 45a41d4c..3f1c2fdc 100644 --- a/src/hamt/consumable-hash.js +++ b/src/hamt/consumable-hash.js @@ -5,7 +5,7 @@ const ConsumableBuffer = require('./consumable-buffer') module.exports = function wrapHash (hashFn) { return function hashing (value) { - if (value instanceof ConsumableBuffer) { + if (value instanceof InfiniteHash) { // already a hash. return it return value } else { @@ -16,6 +16,9 @@ module.exports = function wrapHash (hashFn) { class InfiniteHash { constructor (value, hashFn) { + if ((typeof value) !== 'string' && !Buffer.isBuffer(value)) { + throw new Error('can only hash strings or buffers') + } this._value = value this._hashFn = hashFn this._depth = -1 @@ -24,7 +27,7 @@ class InfiniteHash { this._buffers = [] } - take(bits, callback) { + take (bits, callback) { let pendingBits = bits whilst( () => this._availableBits < pendingBits, @@ -59,7 +62,7 @@ class InfiniteHash { return // early } - callback(null, result) + process.nextTick(() => callback(null, result)) } ) } @@ -74,7 +77,7 @@ class InfiniteHash { hash.untake(availableForUntake) pendingBits -= availableForUntake this._availableBits += availableForUntake - if (this._currentBufferIndex > 0 && hash.totalBits() == hash.availableBits()) { + if (this._currentBufferIndex > 0 && hash.totalBits() === hash.availableBits()) { this._depth-- this._currentBufferIndex-- } @@ -96,4 +99,4 @@ class InfiniteHash { callback() }) } -} \ No newline at end of file +} diff --git a/test/browser.js b/test/browser.js index 1c314bda..be2cf997 100644 --- a/test/browser.js +++ b/test/browser.js @@ -60,5 +60,6 @@ describe('IPFS data importing tests on the Browser', function () { require('./test-import-export')(repo) require('./test-hash-parity-with-go-ipfs')(repo) require('./test-consumable-buffer') + require('./test-consumable-hash') require('./test-hamt') }) diff --git a/test/node.js b/test/node.js index 5fd36fdb..191f2691 100644 --- a/test/node.js +++ b/test/node.js @@ -45,5 +45,6 @@ describe('IPFS UnixFS Engine', () => { require('./test-import-export')(repo) require('./test-hash-parity-with-go-ipfs')(repo) require('./test-consumable-buffer') + require('./test-consumable-hash') require('./test-hamt') }) diff --git a/test/test-consumable-hash.js b/test/test-consumable-hash.js index 6cecefd5..0807fed0 100644 --- a/test/test-consumable-hash.js +++ b/test/test-consumable-hash.js @@ -43,7 +43,7 @@ describe('consumable hash', () => { values.push(result) expect(result).to.be.below(1024) expect(result).to.be.above(0) - iter -- + iter-- callback() }) }, @@ -64,7 +64,7 @@ describe('consumable hash', () => { expect(err).to.not.exist values.push(result) expect(result).to.be.eql(values.shift()) - iter -- + iter-- callback() }) }, @@ -73,10 +73,9 @@ describe('consumable hash', () => { }) }) - -function hashFn(value, callback) { +function hashFn (value, callback) { callback(null, crypto .createHash('sha256') .update(value) .digest()) -} \ No newline at end of file +} diff --git a/test/test-hamt.js b/test/test-hamt.js index 9015e29a..bf1c36a7 100644 --- a/test/test-hamt.js +++ b/test/test-hamt.js @@ -4,6 +4,7 @@ const crypto = require('crypto') const expect = require('chai').expect const each = require('async/each') +const eachSeries = require('async/eachSeries') const HAMT = require('../src/hamt') @@ -149,4 +150,36 @@ describe('HAMT', () => { }) }) }) + + describe('exhausting hash', () => { + let bucket + + before(() => { + bucket = HAMT({ + hashFn: smallHashFn, + bits: 2 + }) + }) + + it('iterates', (callback) => { + const size = 300 + const keys = Array(size) + for (let i = 0; i < size; i++) { + keys[i] = i.toString() + } + + eachSeries(keys, (key, callback) => bucket.put(key, key, callback), (err) => { + expect(err).to.not.exist + callback() + }) + }) + + function smallHashFn (value, callback) { + callback(null, crypto + .createHash('sha256') + .update(value) + .digest() + .slice(0, 2)) // just return the 2 first bytes of the hash + } + }) })