Merge pull request #206 from mapbox/append-friendly

Append-friendlier serial tiles serializer

lib/stream-deserialize.js

@@ -55,6 +55,7 @@ Deserialize.prototype.deserialize = function(serializedObj) {
 
     var obj = deserialize(serializedObj);
 
+    if (obj === null) return;
     if (obj instanceof Info) this.emit('info', obj);
     if (obj instanceof Tile) this.emit('tile', obj);
     this.push(obj);

lib/stream-serialize.js

@@ -1,7 +1,5 @@
 var stream = require('stream');
 var util = require('util');
-var Tile = require('./stream-util').Tile;
-var Info = require('./stream-util').Info;
 var serialize = require('./stream-util').serialize;
 var serialHeader = require('./stream-util').serialHeader;
 
@@ -19,13 +17,19 @@ function Serialize() {
 Serialize.prototype._transform = function(chunk, enc, callback) {
     var data = this._buffer.pop();
     if (data) this.push(data + '\n');
-    if (chunk instanceof Tile || chunk instanceof Info)
-        this._buffer.push(serialize(chunk));
+
+    var serialized;
+    try {
+        serialized = serialize(chunk);
+    } catch(err) {
+        return callback(err);
+    }
+    this._buffer.push(serialized);
     callback();
 };
 
 Serialize.prototype._flush = function(callback) {
     var data = this._buffer.pop();
-    if (data) this.push(data);
+    if (data) this.push(data + '\n');
     callback();
 };

lib/stream-util.js

@@ -4,6 +4,7 @@ var util = require('util');
 module.exports = {};
 module.exports.Tile = Tile;
 module.exports.Info = Info;
+module.exports.SerializationError = SerializationError;
 module.exports.DeserializationError = DeserializationError;
 module.exports.Stats = Stats;
 module.exports.addChildren = addChildren;
@@ -21,6 +22,12 @@ module.exports.putTileRetry = putTileRetry;
 module.exports.retryBackoff = 1000;
 module.exports.slowTime = 10e3;
 
+function SerializationError(msg) {
+    this.message = msg;
+    this.name = 'SerializationError';
+}
+util.inherits(SerializationError, Error);
+
 function DeserializationError(msg) {
     this.message = msg;
     this.name = 'DeserializationError';
@@ -43,13 +50,15 @@ function Info(info) {
 function serialize(obj) {
     if (obj instanceof Tile) return serializeTile(obj);
     if (obj instanceof Info) return serializeInfo(obj);
-    return '';
+
+    throw new SerializationError('Invalid data');
 }
 
 function deserialize(data, property) {
     if (property) return getSerializedProperty(data, property);
     if (data.indexOf('{"z":') === 0) return deserializeTile(data);
     if (data.indexOf('{') === 0) return deserializeInfo(data);
+    if (data === '') return null;
 
     throw new DeserializationError('Invalid data');
 }

test/copy.test.js

@@ -81,7 +81,7 @@ test('copy streams', function(t) {
         t.ifError(err, 'no errors');
         t.equal(stderr, '', 'no stderr');
         t.ok(stdout.indexOf('JSONBREAKFASTTIME\n') === 0);
-        t.equal(stdout.length, 647001);
+        t.equal(stdout.length, 647002);
         t.end();
     });
 });

test/stream-serialize.test.js

@@ -3,6 +3,8 @@ var MBTiles = require('@mapbox/mbtiles');
 var tilelive = require('..');
 var fs = require('fs');
 var path = require('path');
+var Tile = require('../lib/stream-util').Tile;
+var Info = require('../lib/stream-util').Info;
 var deserialize = require('../lib/stream-util').deserialize;
 var serialHeader = require('../lib/stream-util').serialHeader;
 
@@ -27,15 +29,13 @@ test('serialize: list', function(t) {
             out += data;
         })
         .on('finish', function() {
-            var data = out.split('\n').slice(1);
-            t.equal(data.length, 77, 'correct number of tiles');
-
-            function roundtrip() {
-                data.forEach(function(tile) {
-                    var obj = deserialize(tile);
-                });
-            }
-            t.doesNotThrow(roundtrip, 'serialized data can be deserialized');
+            var counter = out.split('\n').slice(1).reduce(function(memo, tile) {
+                var obj = deserialize(tile);
+                if (obj instanceof Tile) memo.tiles++;
+                if (obj instanceof Info) memo.info++;
+                return memo;
+            }, { tiles: 0, info: 0 });
+            t.deepEqual(counter, { tiles: 77, info: 0 }, 'deserialized accurately');
             t.end();
         });
 });
@@ -50,15 +50,13 @@ test('serialize: scanline', function(t) {
             out += data;
         })
         .on('finish', function() {
-            var data = out.split('\n').slice(1);
-            t.equal(data.length, 286, 'correct number of tiles');
-
-            function roundtrip() {
-                data.forEach(function(tile) {
-                    var obj = deserialize(tile);
-                });
-            }
-            t.doesNotThrow(roundtrip, 'serialized data can be deserialized');
+            var counter = out.split('\n').slice(1).reduce(function(memo, tile) {
+                var obj = deserialize(tile);
+                if (obj instanceof Tile) memo.tiles++;
+                if (obj instanceof Info) memo.info++;
+                return memo;
+            }, { tiles: 0, info: 0 });
+            t.deepEqual(counter, { tiles: 285, info: 1 }, 'deserialized accurately');
             t.end();
         });
 });
@@ -74,15 +72,13 @@ test('serialize: pyramid', function(t) {
             out += data;
         })
         .on('finish', function() {
-            var data = out.split('\n').slice(1);
-            t.equal(data.length, 286, 'correct number of tiles');
-
-            function roundtrip() {
-                data.forEach(function(tile) {
-                    var obj = deserialize(tile);
-                });
-            }
-            t.doesNotThrow(roundtrip, 'serialized data can be deserialized');
+            var counter = out.split('\n').slice(1).reduce(function(memo, tile) {
+                var obj = deserialize(tile);
+                if (obj instanceof Tile) memo.tiles++;
+                if (obj instanceof Info) memo.info++;
+                return memo;
+            }, { tiles: 0, info: 0 });
+            t.deepEqual(counter, { tiles: 285, info: 1 }, 'deserialized accurately');
             t.end();
         });
 });
@@ -91,7 +87,13 @@ test('serialize: garbage', function(t) {
     t.plan(2);
     fs.createReadStream(path.join(__dirname,'fixtures','filescheme.flat'))
         .pipe(tilelive.serialize())
-        .on('error', function(err) { t.ifError(err, 'no error should be thrown'); })
-        .on('data', function(d) { t.ok(d.toString() === serialHeader + '\n', 'no data should be received'); })
-        .on('end', function() { t.ok(true, 'no data was serialized'); });
+        .on('error', function(err) {
+            t.equal(err && err.toString(), 'SerializationError: Invalid data', 'no error should be thrown');
+        })
+        .on('data', function(d) {
+            t.equal(d.toString(), serialHeader + '\n', 'only data passed is serialization header');
+        })
+        .on('end', function() {
+            t.fail('stream interrupted by error');
+        });
 });

test/stream-util.test.js

@@ -319,6 +319,17 @@ test('Info: deserialize', function(t) {
     }
 });
 
+test('serialize/deserialize corner cases', function(t) {
+    t.deepEqual(util.deserialize(''), null, 'deserialize interprets empty strings as null');
+    t.throws(function() {
+        util.serialize('boogie woogie');
+    }, /SerializationError: Invalid data/, 'serialize throws on invalid data');
+    t.throws(function() {
+        util.serialize('');
+    }, /SerializationError: Invalid data/, 'serialize throws on invalid data');
+    t.end();
+});
+
 test('Limit bounds', function(t) {
 
     // these inputs should simply be equal to themselves, since they don't contain