diff --git a/_checksum.js b/_checksum.js new file mode 100644 index 00000000..1bbeb0d7 --- /dev/null +++ b/_checksum.js @@ -0,0 +1,21 @@ +var crypto = require('crypto') + +module.exports = function (checksum, binary) { + if (typeof checksum == 'function') return checksum + var algorithm = checksum || 'sha1' + if (algorithm == 'none') { + return null + } + if (binary) { + return function (buffer, start, end) { + var hash = crypto.createHash(algorithm) + hash.update(buffer.slice(start, end)) + return new Buffer(hash.digest('hex'), 'hex') + } + } + return function (buffer, start, end) { + var hash = crypto.createHash(algorithm) + hash.update(buffer.slice(start, end)) + return hash.digest('hex') + } +} diff --git a/benchmark/checksum.js b/benchmark/checksum.js new file mode 100644 index 00000000..dcea647f --- /dev/null +++ b/benchmark/checksum.js @@ -0,0 +1,96 @@ +var ok = require('assert').ok +var checksum = require('../_checksum') +var murmur3 = require('./murmur3') +var fnv = require('./fnv') +var djb = require('./djb') +var Benchmark = require('benchmark').Benchmark +var crypto = require('crypto') + +var suite = new Benchmark.Suite('frame') + +var buffer = crypto.randomBytes(1024) + +function djbTest () { + djb(buffer, 0, buffer.length) +} + +function fnvTest () { + fnv(buffer, 0, buffer.length) +} + +function murmur3Test () { + murmur3(buffer, 0, buffer.length) +} + +var sha1 = checksum('sha1', true) +function sha1Test () { + sha1(buffer, 0, buffer.length) +} + +var md5 = checksum('md5', true) +function md5Test () { + md5(buffer, 0, buffer.length) +} + +var sha1hex = checksum('sha1') +function sha1hexTest () { + sha1hex(buffer, 0, buffer.length) +} + +var md5hex = checksum('md5') +function md5hexTest () { + md5hex(buffer, 0, buffer.length) +} + +djbTest() +fnvTest() +murmur3Test() +sha1Test() +md5Test() + +for (var i = 0; i < 1; i++) { + suite.add({ + name: 'djbTest ' + i, + fn: djbTest + }) + + suite.add({ + name: 'fnvTest ' + i, + fn: fnvTest + }) + + suite.add({ + name: 'murmur3Test ' + i, + fn: murmur3Test + }) + + suite.add({ + name: 'sha1 ' + i, + fn: sha1Test + }) + + suite.add({ + name: 'md5 ' + i, + fn: md5Test + }) + + suite.add({ + name: 'sha1hex ' + i, + fn: sha1hexTest + }) + + suite.add({ + name: 'md5hex ' + i, + fn: md5hexTest + }) +} + +suite.on('cycle', function(event) { + console.log(String(event.target)); +}) + +suite.on('complete', function() { + console.log('Fastest is ' + this.filter('fastest').pluck('name')); +}) + +suite.run() diff --git a/benchmark/djb.js b/benchmark/djb.js new file mode 100644 index 00000000..45058086 --- /dev/null +++ b/benchmark/djb.js @@ -0,0 +1,8 @@ +function djb (block, start, end) { + var seed = 0 + for (var i = start; i < end; i++) { + seed = (seed * 33 + block[i]) >>> 0 + } + return seed +} +module.exports = djb diff --git a/benchmark/fnv.js b/benchmark/fnv.js new file mode 100644 index 00000000..3123e827 --- /dev/null +++ b/benchmark/fnv.js @@ -0,0 +1,12 @@ +function fnv (block, start, end) { + var hash = (0 ^ 2166136261) >>> 0 + for (var i = start; i < end; i++) { + hash = (hash ^ block[i]) >>> 0 + hash += (hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24) + hash = hash >>> 0 + } + var buffer = new Buffer(4) + buffer.writeUInt32LE(hash, 0) + return buffer +} +module.exports = fnv diff --git a/benchmark/frame.js b/benchmark/frame.js new file mode 100644 index 00000000..dadd2e40 --- /dev/null +++ b/benchmark/frame.js @@ -0,0 +1,57 @@ +var ok = require('assert').ok +var UTF8 = require('../frame/utf8') +var Binary = require('../frame/binary') +var Benchmark = require('benchmark') +var Queue = require('../queue') +var json = require('../json') + +var suite = new Benchmark.Suite('frame') + +var utf8 = new UTF8('none') +var binary = new Binary('none') + +function createTest (framer) { + return function () { + var queue = new Queue + for (var i = 0; i < 512; i++) { + framer.serialize(json.serializer, queue, [ 1, 2, 3 ], { a: 1 }) + } + queue.finish() + var buffer = queue.buffers.shift(), offset = 0, count = 0 + for (;;) { + var entry = framer.deserialize(json.deserialize, buffer, offset) + if (entry == null) { + break + } + offset += entry.length + } + } +} + +var utf8test = createTest(utf8) +var binaryTest = createTest(binary) + +utf8test() +binaryTest() + +for (var i = 0; i < 1; i++) { + suite.add({ + name: 'utf8 ' + i, + fn: utf8test + }) + + suite.add({ + name: 'binary ' + i, + fn: binaryTest + }) +} + +suite.on('cycle', function(event) { + console.log(String(event.target)); +}) + +suite.on('complete', function() { + console.log('Fastest is ' + this.filter('fastest').pluck('name')); +}) + +suite.run() diff --git a/benchmark/murmur3.js b/benchmark/murmur3.js new file mode 100644 index 00000000..2e454ac3 --- /dev/null +++ b/benchmark/murmur3.js @@ -0,0 +1,91 @@ +var util = require('util') + +var c1 = 0xcc9e2d51 +var c2 = 0x1b873593 + +function multiply (a, b) { + var aHigh = (a >> 16) & 0xffff + var aLow = a & 0xffff + var bHigh = (b >> 16) & 0xffff + var bLow = b & 0xffff + var high = ((aHigh * bLow) + (aLow * bHigh)) & 0xffff + return (high << 16) + (aLow * bLow) +} + +// We don't use `>>> 0`. We let the values negate. The only use of addition in +// Murmur uses the result of a multiplication, which will be converted to +// unsigned integer by our 16-bit at a time multiplication. + +function fmix32 (hash) { + hash ^= hash >>> 16 + hash = multiply(hash, 0x85ebca6b) + hash ^= hash >>> 13 + hash = multiply(hash, 0xc2b2ae35) + hash ^= hash >>> 16 + return hash +} + +// With this, unused, function we always make sure we have an unsigned integer +// value, but it's not absolutely necessary. We're only interested in the +// integer value when we perform addition or write the value to our buffer. We +// do not do this within Murmur's mix function. I'm leaving it in place for a +// benchmark where I can gauge the cost of `>>> 0`. + +function fmix32_pure (hash) { + hash = (hash ^ (hash >>> 16)) >>> 0 + hash = multiply(hash, 0x85ebca6b) + hash = (hash ^ (hash >>> 13)) >>> 0 + hash = multiply(hash, 0xc2b2ae35) + hash = (hash ^ (hash >>> 16)) >>> 0 + return hash +} + +function rotl32 (number, bits) { + return ((number << bits) | (number >>> 32 - bits)) >>> 0 +} + +function murmur (buffer, start, end) { + var hash = 0 + var length = end - start + + var count = length / 4 + var remainder = length % 4 + + for (var i = 0; i < count; i++) { + var k1 = buffer[i * 4 + start] + + (buffer[i * 4 + 1 + start] << 8) + + (buffer[i * 4 + 2 + start] << 16) + + (buffer[i * 4 + 3 + start] << 24) + + k1 = multiply(k1, c1) + k1 = rotl32(k1, 15) + k1 = multiply(k1, c2) + + hash ^= k1 + hash = rotl32(hash, 13) + hash = multiply(hash, 5) + 0xe6546b64 + } + length += count * 4 + + var k1 = 0 + + switch (remainder) { + case 3: k1 ^= buffer[i + 2 + start] << 16 + case 2: k1 ^= buffer[i + 1 + start] << 8 + case 1: k1 ^= buffer[i + 0 + start] + k1 = multiply(k1, c1) + k1 = rotl32(k1, 15) + k1 = multiply(k1, c2) + hash ^= k1 + } + + hash ^= length + remainder + + hash = fmix32(hash) >>> 0 + + var buffer = new Buffer(4) + buffer.writeUInt32LE(hash, 0, true) + return buffer +} + +module.exports = murmur diff --git a/frame/binary.js b/frame/binary.js new file mode 100644 index 00000000..8d391dfa --- /dev/null +++ b/frame/binary.js @@ -0,0 +1,74 @@ +var createChecksum = require('../_checksum') + +function Binary (checksum) { + checksum = this.checksum = createChecksum(checksum, true) + this.checksumLength = checksum ? checksum(new Buffer(0), 0, 0).length : 0 +} + +Binary.prototype.serialize = function (serializer, queue, header, body) { + var bodyLength = 0 + if (body) { + var body = serializer.serialize(body) + var bodyLength = serializer.sizeOf(body) + } + var length = 8 + this.checksumLength + ((header.length + 1) * 4) + bodyLength + var buffer = queue.slice(length) + var offset = -4 + var payloadStart + buffer.writeUInt32BE(buffer.length, offset += 4, true) + buffer.writeUInt32BE(0xaaaaaaaa, offset += 4, true) + payloadStart = (offset += this.checksumLength) + 4 + buffer.writeUInt32BE(header.length, offset += 4, true) + for (var i = 0, I = header.length; i < I; i++) { + buffer.writeInt32BE(header[i], offset += 4, true) + } + if (body) { + serializer.write(body, buffer, offset += 4, buffer.length) + } + var checksum = this.checksum + if (checksum) { + var digest = checksum(buffer, payloadStart, buffer.length) + digest.copy(buffer, 8, 0, digest.length) + } + return length +} + +Binary.prototype.deserialize = function (deserialize, buffer, offset) { + var start = offset + var remaining = buffer.length - offset + if (remaining < 4) { + return null + } + var length = buffer.readUInt32BE(offset, true) + var end = offset + length + if (remaining < length) { + return null + } + offset += 8 + var checksum = this.checksum + if (checksum != null) { + var digest = checksum(buffer, offset + this.checksumLength, end) + for (var i = 0, I = digest.length; i < I; i++) { + if (buffer[offset++] != digest[i]) { + throw new Error('invalid checksum') + } + } + } + var headerCount = buffer.readUInt32BE(offset, true) + var header = [] + for (var i = 0; i < headerCount; i++) { + header.push(buffer.readInt32BE(offset += 4, true)) + } + offset += 4 + if (offset < end) { + var body = deserialize(buffer, offset, end) + } + return { + length: end - start, + heft: body == null ? null : end - offset, + header: header, + body: body || null + } +} + +module.exports = Binary diff --git a/frame/utf8.js b/frame/utf8.js new file mode 100644 index 00000000..87a50e93 --- /dev/null +++ b/frame/utf8.js @@ -0,0 +1,111 @@ +var ok = require('assert').ok +var createChecksum = require('../_checksum') + +function UTF8 (checksum) { + var checksum = this.checksum = createChecksum(checksum) + this.dummyDigest = checksum ? checksum(new Buffer(0), 0, 0) : '0' +} + +UTF8.prototype.serialize = function (serializer, queue, header, body) { + var checksum = this.checksum, digest = this.dummyDigest, + entry, buffer, json, line, length + + ok(header.every(function (n) { + return typeof n == 'number' + }), 'header values must be numbers') + + entry = header.slice() + json = JSON.stringify(entry) + + length = 0 + + var separator = '' + if (body != null) { + body = serializer.serialize(body) + separator = ' ' + var bodyLength = serializer.sizeOf(body) + length += bodyLength + var temporary = new Buffer(bodyLength) + serializer.write(body, temporary, 0, temporary.length) + body = temporary + } + + line = this.dummyDigest + ' ' + json + separator + + length += Buffer.byteLength(line, 'utf8') + 1 + + var entire = length + String(length).length + 1 + if (entire < length + String(entire).length + 1) { + length = length + String(entire).length + 1 + } else { + length = entire + } + + buffer = queue.slice(length) + + buffer.write(String(length) + ' ' + line) + if (body != null) { + body.copy(buffer, buffer.length - 1 - body.length) + } + buffer[length - 1] = 0x0A + + if (checksum) { + var digest = checksum(buffer, String(length).length + digest.length + 2, length - 1) + buffer.write(digest, String(length).length + 1) + } + + return length +} + +UTF8.prototype.deserialize = function (deserialize, buffer, offset) { + for (var i = offset, I = buffer.length; i < I; i++) { + if (buffer[i] == 0x20) break + } + if (buffer[i] != 0x20) { + return null + } + var size = parseInt(buffer.toString('utf8', offset, i)) + if (buffer.length - offset < size) { + return null + } + for (var count = 2, i = offset, I = buffer.length; i < I && count; i++) { + if (buffer[i] == 0x20) count-- + } + if (count) { + throw new Error('invalid record') + } + var checksumStart = i + for (count = 1; i < I && count; i++) { + if (buffer[i] == 0x20 || buffer[i] == 0x0a) count-- + } + if (count) { + throw new Error('invalid record') + } + var fields = buffer.toString('utf8', 0, i - 1).split(' ') + var checksum = this.checksum + if (checksum) { + var digest = checksum(buffer, checksumStart, offset + size - 1) + ok(fields[1] == '-' || digest == fields[1], 'corrupt line: invalid checksum') + } + var body, length + if (buffer[i - 1] == 0x20) { + body = buffer.slice(i, offset + size - 1) + length = body.length + } + if (buffer[i - 1] == 0x20) { + i += body.length + 1 + body = deserialize(body, 0, body.length) + } + var entry = { + heft: length || null, + length: i - offset, + header: JSON.parse(fields[2]), + body: body || null + } + ok(entry.header.every(function (n) { + return typeof n == 'number' + }), 'header values must be numbers') + return entry +} + +module.exports = UTF8 diff --git a/json.js b/json.js new file mode 100644 index 00000000..1b964122 --- /dev/null +++ b/json.js @@ -0,0 +1,15 @@ +exports.serializer = { + serialize: function (body) { + return JSON.stringify(body) + }, + sizeOf: function (body) { + return Buffer.byteLength(body, 'utf8') + }, + write: function (body, buffer, offset, length) { + buffer.write(body, offset, length, 'utf8') + } +} + +exports.deserialize = function (buffer, start, end) { + return JSON.parse(buffer.toString('utf8', start, end, 'utf8')) +} diff --git a/t/frame/binary.t.js b/t/frame/binary.t.js new file mode 100755 index 00000000..d0668db4 --- /dev/null +++ b/t/frame/binary.t.js @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +require('proof')(4, prove) + +function prove (assert) { + var json = require('../../json') + var Queue = require('../../queue') + var Framer = require('../../frame/binary') + var framer = new Framer('sha1') + var queue = new Queue + var length = framer.serialize(json.serializer, queue, [ 1, 2, 3 ], { a: 1 }) + assert(length, 51, 'bodied length') + queue.finish() + var entry = framer.deserialize(json.deserialize, queue.buffers.shift(), 0) + assert(entry, { length: 51, heft: 7, header: [ 1, 2, 3 ], body: { a: 1 } }, 'bodied') + var queue = new Queue + var length = framer.serialize(json.serializer, queue, [ 1, 2, 3 ]) + assert(length, 44, 'unbodied length') + queue.finish() + var entry = framer.deserialize(json.deserialize, queue.buffers.shift(), 0) + assert(entry, { length: 44, heft: null, header: [ 1, 2, 3 ], body: null }, 'bodied') +} diff --git a/t/frame/utf8.t.js b/t/frame/utf8.t.js new file mode 100755 index 00000000..fc5d4682 --- /dev/null +++ b/t/frame/utf8.t.js @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +require('proof')(4, prove) + +function prove (assert) { + var json = require('../../json') + var Queue = require('../../queue') + var Framer = require('../../frame/utf8') + var framer = new Framer('sha1') + var queue = new Queue + var length = framer.serialize(json.serializer, queue, [ 1, 2, 3 ], { a: 1 }) + assert(length, 60, 'bodied length') + queue.finish() + var entry = framer.deserialize(json.deserialize, queue.buffers.shift(), 0) + assert(entry, { length: 60, heft: 7, header: [ 1, 2, 3 ], body: { a: 1 } }, 'bodied') + var queue = new Queue + var length = framer.serialize(json.serializer, queue, [ 1, 2, 3 ]) + assert(length, 52, 'unbodied length') + queue.finish() + var entry = framer.deserialize(json.deserialize, queue.buffers.shift(), 0) + assert(entry, { length: 52, heft: null, header: [ 1, 2, 3 ], body: null }, 'bodied') +}