Merge pull request #147 from mapbox/hat

switch to hat + hat.rack for batch

index.js

@@ -5,7 +5,6 @@ var queue = require('queue-async');
 var Dyno = require('dyno');
 var AWS = require('aws-sdk');
 var extent = require('geojson-extent');
-var cuid = require('cuid');
 var tilebelt = require('tilebelt');
 var geobuf = require('geobuf');
 var stream = require('stream');

lib/batch.js

@@ -3,6 +3,7 @@ var queue = require('queue-async');
 var geobuf = require('geobuf');
 var _ = require('lodash');
 var Dyno = require('dyno');
+var hat = require('hat');
 
 module.exports = function(config) {
     if (!config.bucket) throw new Error('No bucket set');
@@ -34,8 +35,9 @@ module.exports = function(config) {
         var encoded;
         var q = queue(150);
 
+        var rack = hat.rack();
         for (var i = 0; i < collection.features.length; i++) {
-            try { encoded = utils.toDatabaseRecord(collection.features[i], dataset); }
+            try { encoded = utils.toDatabaseRecord(collection.features[i], dataset, rack); }
             catch (err) { return callback(err); }
 
             records.push(encoded[0]);

lib/utils.js

@@ -3,7 +3,7 @@ var geobuf = require('geobuf');
 var url = require('url');
 var geojsonNormalize = require('geojson-normalize');
 var _ = require('lodash');
-var cuid = require('cuid');
+var hat = require('hat');
 var Metadata = require('./metadata');
 var tilebelt = require('tilebelt');
 
@@ -58,11 +58,12 @@ var Utils = module.exports = function(config) {
      * Converts a single GeoJSON feature into backend format
      * @param {object} feature - a GeoJSON feature
      * @param {string} dataset - the name of the dataset the feature belongs to
+     * @param {string} [rack] - a hat rack to use for random id generation
      * @returns {object[]} the first element is a DynamoDB record suitable for inserting via `dyno.putItem`, the second are parameters suitable for uploading via `s3.putObject`.
      */
-    utils.toDatabaseRecord = function(feature, dataset) {
+    utils.toDatabaseRecord = function(feature, dataset, rack) {
         if (feature.id === 0) feature.id = '0';
-        var f = feature.id ? _.clone(feature) : _.extend({}, feature, { id: cuid() });
+        var f = feature.id ? _.clone(feature) : _.extend({}, feature, { id: (rack || hat)() });
         var primary = f.id;
 
         if (!f.geometry || !f.geometry.coordinates)

package.json

@@ -34,6 +34,7 @@
     "geobuf": "0.2.4",
     "geojson-extent": "^0.1.0",
     "geojson-normalize": "0.0.0",
+    "hat": "0.0.3",
     "lodash": "~2.4.1",
     "minimist": "0.0.9",
     "queue-async": "~1.0.7",

test/cli.test.js

@@ -119,6 +119,8 @@ test('[cli] list', function(assert) {
         exec(params.join(' '), function(err, stdout, stderr) {
             assert.ifError(err, 'success');
             var found = JSON.parse(stdout.trim());
+            found.features = _(found.features).sortBy('id').value();
+            putResults.features = _(putResults.features).sortBy('id').value();
             assert.deepEqual(found, putResults, 'got expected FeatureCollection');
             assert.end();
         });

test/indexing.js

@@ -454,10 +454,13 @@ test('list first page with maxFeatures', function(t) {
     cardboard.batch.put(features, 'default', function page(err, putResult) {
         t.equal(err, null);
         t.pass('collection inserted');
+
+        var indexed = _.indexBy(putResult.features, 'id');
+
         cardboard.list('default', {maxFeatures: 1}, function(err, data) {
             t.equal(err, null, 'no error');
             t.deepEqual(data.features.length, 1, 'first page has one feature');
-            t.deepEqual(data.features[0].id, putResult.features[0].id, 'id as expected');
+            t.deepEqual(data.features[0], indexed[data.features[0].id], 'obj as expected');
             t.end();
         });
     });

test/utils.test.js

@@ -82,6 +82,30 @@ test('[utils] toDatabaseRecord - no ID', function(assert) {
     assert.end();
 });
 
+test('[utils] toDatabaseRecord - no ID + rack', function(assert) {
+    var noId = {
+        type: 'Feature',
+        properties: {
+            hasNo: 'id'
+        },
+        geometry: {
+            type: 'Point',
+            coordinates: [0, 0]
+        }
+    };
+
+    var rack = function() {
+        return 'big-mac';
+    };
+
+    var encoded = utils.toDatabaseRecord(noId, 'dataset', rack);
+    var item = encoded[0];
+
+    assert.equal(item.id, 'id!big-mac', 'an id was assigned');
+
+    assert.end();
+});
+
 test('[utils] toDatabaseRecord - with ID', function(assert) {
     var hasId = {
         id: 'bacon-lettuce-tomato',