Create binary frame format.

Also created different checksum formats. SHA1 from `crypto` is still
faster than non-cryptograhic checksums written in JavaScript.

See #474.
Closes #471.
See #470.

_checksum.js

@@ -0,0 +1,21 @@
+var crypto = require('crypto')
+
+module.exports = function (checksum, binary) {
+    if (typeof checksum == 'function') return checksum
+    var algorithm = checksum || 'sha1'
+    if (algorithm == 'none') {
+        return null
+    }
+    if (binary) {
+        return function (buffer, start, end) {
+            var hash = crypto.createHash(algorithm)
+            hash.update(buffer.slice(start, end))
+            return new Buffer(hash.digest('hex'), 'hex')
+        }
+    }
+    return function (buffer, start, end) {
+        var hash = crypto.createHash(algorithm)
+        hash.update(buffer.slice(start, end))
+        return hash.digest('hex')
+    }
+}

benchmark/checksum.js

@@ -0,0 +1,96 @@
+var ok = require('assert').ok
+var checksum = require('../_checksum')
+var murmur3 = require('./murmur3')
+var fnv = require('./fnv')
+var djb = require('./djb')
+var Benchmark = require('benchmark').Benchmark
+var crypto = require('crypto')
+
+var suite = new Benchmark.Suite('frame')
+
+var buffer = crypto.randomBytes(1024)
+
+function djbTest () {
+    djb(buffer, 0, buffer.length)
+}
+
+function fnvTest () {
+    fnv(buffer, 0, buffer.length)
+}
+
+function murmur3Test () {
+    murmur3(buffer, 0, buffer.length)
+}
+
+var sha1 = checksum('sha1', true)
+function sha1Test () {
+    sha1(buffer, 0, buffer.length)
+}
+
+var md5 = checksum('md5', true)
+function md5Test () {
+    md5(buffer, 0, buffer.length)
+}
+
+var sha1hex = checksum('sha1')
+function sha1hexTest () {
+    sha1hex(buffer, 0, buffer.length)
+}
+
+var md5hex = checksum('md5')
+function md5hexTest () {
+    md5hex(buffer, 0, buffer.length)
+}
+
+djbTest()
+fnvTest()
+murmur3Test()
+sha1Test()
+md5Test()
+
+for (var i = 0; i < 1; i++)  {
+    suite.add({
+        name: 'djbTest ' + i,
+        fn: djbTest
+    })
+
+    suite.add({
+        name: 'fnvTest ' + i,
+        fn: fnvTest
+    })
+
+    suite.add({
+        name: 'murmur3Test ' + i,
+        fn: murmur3Test
+    })
+
+    suite.add({
+        name: 'sha1 ' + i,
+        fn: sha1Test
+    })
+
+    suite.add({
+        name: 'md5 ' + i,
+        fn: md5Test
+    })
+
+    suite.add({
+        name: 'sha1hex ' + i,
+        fn: sha1hexTest
+    })
+
+    suite.add({
+        name: 'md5hex ' + i,
+        fn: md5hexTest
+    })
+}
+
+suite.on('cycle', function(event) {
+    console.log(String(event.target));
+})
+
+suite.on('complete', function() {
+    console.log('Fastest is ' + this.filter('fastest').pluck('name'));
+})
+
+suite.run()

benchmark/djb.js

@@ -0,0 +1,8 @@
+function djb (block, start, end) {
+    var seed = 0
+    for (var i = start; i < end; i++) {
+        seed = (seed * 33 + block[i]) >>> 0
+    }
+    return seed
+}
+module.exports = djb

benchmark/fnv.js

@@ -0,0 +1,12 @@
+function fnv (block, start, end) {
+    var hash = (0 ^ 2166136261) >>> 0
+    for (var i = start; i < end; i++) {
+        hash = (hash ^ block[i]) >>> 0
+        hash += (hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24)
+        hash = hash >>> 0
+    }
+    var buffer = new Buffer(4)
+    buffer.writeUInt32LE(hash, 0)
+    return buffer
+}
+module.exports = fnv

benchmark/frame.js

@@ -0,0 +1,57 @@
+var ok = require('assert').ok
+var UTF8 = require('../frame/utf8')
+var Binary = require('../frame/binary')
+var Benchmark = require('benchmark')
+var Queue = require('../queue')
+var json = require('../json')
+
+var suite = new Benchmark.Suite('frame')
+
+var utf8 = new UTF8('none')
+var binary = new Binary('none')
+
+function createTest (framer) {
+    return function () {
+        var queue = new Queue
+        for (var i = 0; i < 512; i++) {
+            framer.serialize(json.serializer, queue, [ 1, 2, 3 ], { a: 1 })
+        }
+        queue.finish()
+        var buffer = queue.buffers.shift(), offset = 0, count = 0
+        for (;;) {
+            var entry = framer.deserialize(json.deserialize, buffer, offset)
+            if (entry == null) {
+                break
+            }
+            offset += entry.length
+        }
+    }
+}
+
+var utf8test = createTest(utf8)
+var binaryTest = createTest(binary)
+
+utf8test()
+binaryTest()
+
+for (var i = 0; i < 1; i++)  {
+    suite.add({
+        name: 'utf8 ' + i,
+        fn: utf8test
+    })
+
+    suite.add({
+        name: 'binary ' + i,
+        fn: binaryTest
+    })
+}
+
+suite.on('cycle', function(event) {
+    console.log(String(event.target));
+})
+
+suite.on('complete', function() {
+    console.log('Fastest is ' + this.filter('fastest').pluck('name'));
+})
+
+suite.run()

benchmark/murmur3.js

@@ -0,0 +1,91 @@
+var util = require('util')
+
+var c1 = 0xcc9e2d51
+var c2 = 0x1b873593
+
+function multiply (a, b) {
+    var aHigh = (a >> 16) & 0xffff
+    var aLow = a & 0xffff
+    var bHigh = (b >> 16) & 0xffff
+    var bLow = b & 0xffff
+    var high = ((aHigh * bLow) + (aLow * bHigh)) & 0xffff
+    return (high << 16) + (aLow * bLow)
+}
+
+// We don't use `>>> 0`. We let the values negate. The only use of addition in
+// Murmur uses the result of a multiplication, which will be converted to
+// unsigned integer by our 16-bit at a time multiplication.
+
+function fmix32 (hash) {
+    hash ^= hash >>> 16
+    hash = multiply(hash, 0x85ebca6b)
+    hash ^= hash >>> 13
+    hash = multiply(hash, 0xc2b2ae35)
+    hash ^= hash >>> 16
+    return hash
+}
+
+// With this, unused, function we always make sure we have an unsigned integer
+// value, but it's not absolutely necessary. We're only interested in the
+// integer value when we perform addition or write the value to our buffer. We
+// do not do this within Murmur's mix function. I'm leaving it in place for a
+// benchmark where I can gauge the cost of `>>> 0`.
+
+function fmix32_pure (hash) {
+    hash = (hash ^ (hash >>> 16)) >>> 0
+    hash = multiply(hash, 0x85ebca6b)
+    hash = (hash ^ (hash >>> 13)) >>> 0
+    hash = multiply(hash, 0xc2b2ae35)
+    hash = (hash ^ (hash >>> 16)) >>> 0
+    return hash
+}
+
+function rotl32 (number, bits) {
+    return ((number << bits) | (number >>> 32 - bits)) >>> 0
+}
+
+function murmur (buffer, start, end) {
+    var hash = 0
+    var length = end - start
+
+    var count = length / 4
+    var remainder = length % 4
+
+    for (var i = 0; i < count; i++) {
+        var k1 =  buffer[i * 4 + start] +
+                 (buffer[i * 4 + 1 + start] <<  8) +
+                 (buffer[i * 4 + 2 + start] << 16) +
+                 (buffer[i * 4 + 3 + start] << 24)
+
+        k1 = multiply(k1, c1)
+        k1 = rotl32(k1, 15)
+        k1 = multiply(k1, c2)
+
+        hash ^= k1
+        hash = rotl32(hash, 13)
+        hash = multiply(hash, 5) + 0xe6546b64
+    }
+    length += count * 4
+
+    var k1 = 0
+
+    switch (remainder) {
+    case 3: k1 ^= buffer[i + 2 + start] << 16
+    case 2: k1 ^= buffer[i + 1 + start] << 8
+    case 1: k1 ^= buffer[i + 0 + start]
+        k1 = multiply(k1, c1)
+        k1 = rotl32(k1, 15)
+        k1 = multiply(k1, c2)
+        hash ^= k1
+    }
+
+    hash ^= length + remainder
+
+    hash = fmix32(hash) >>> 0
+
+    var buffer = new Buffer(4)
+    buffer.writeUInt32LE(hash, 0, true)
+    return buffer
+}
+
+module.exports = murmur

frame/binary.js

@@ -0,0 +1,74 @@
+var createChecksum = require('../_checksum')
+
+function Binary (checksum) {
+    checksum = this.checksum = createChecksum(checksum, true)
+    this.checksumLength = checksum ? checksum(new Buffer(0), 0, 0).length : 0
+}
+
+Binary.prototype.serialize = function (serializer, queue, header, body) {
+    var bodyLength = 0
+    if (body) {
+        var body = serializer.serialize(body)
+        var bodyLength = serializer.sizeOf(body)
+    }
+    var length = 8 + this.checksumLength + ((header.length + 1) * 4) + bodyLength
+    var buffer = queue.slice(length)
+    var offset = -4
+    var payloadStart
+    buffer.writeUInt32BE(buffer.length, offset += 4, true)
+    buffer.writeUInt32BE(0xaaaaaaaa, offset += 4, true)
+    payloadStart = (offset += this.checksumLength) + 4
+    buffer.writeUInt32BE(header.length, offset += 4, true)
+    for (var i = 0, I = header.length; i < I; i++) {
+        buffer.writeInt32BE(header[i], offset += 4, true)
+    }
+    if (body) {
+        serializer.write(body, buffer, offset += 4, buffer.length)
+    }
+    var checksum = this.checksum
+    if (checksum) {
+        var digest = checksum(buffer, payloadStart, buffer.length)
+        digest.copy(buffer, 8, 0, digest.length)
+    }
+    return length
+}
+
+Binary.prototype.deserialize = function (deserialize, buffer, offset) {
+    var start = offset
+    var remaining = buffer.length - offset
+    if (remaining < 4) {
+        return null
+    }
+    var length = buffer.readUInt32BE(offset, true)
+    var end = offset + length
+    if (remaining < length) {
+        return null
+    }
+    offset += 8
+    var checksum = this.checksum
+    if (checksum != null) {
+        var digest = checksum(buffer, offset + this.checksumLength, end)
+        for (var i = 0, I = digest.length; i < I; i++) {
+            if (buffer[offset++] != digest[i]) {
+                throw new Error('invalid checksum')
+            }
+        }
+    }
+    var headerCount = buffer.readUInt32BE(offset, true)
+    var header = []
+    for (var i = 0; i < headerCount; i++) {
+        header.push(buffer.readInt32BE(offset += 4, true))
+    }
+    offset += 4
+    if (offset < end) {
+        var body = deserialize(buffer, offset, end)
+    }
+    return {
+        length: end - start,
+        heft: body == null ? null : end - offset,
+        header: header,
+        body: body || null
+    }
+}
+
+module.exports = Binary