Skip to content

Commit

Permalink
Merge f1acf3c into cacd387
Browse files Browse the repository at this point in the history
  • Loading branch information
Ryan Clark committed Jun 12, 2015
2 parents cacd387 + f1acf3c commit d978ccc
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 135 deletions.
64 changes: 14 additions & 50 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@ var url = require('url');
var tilebelt = require('tilebelt');
var geobuf = require('geobuf');

var MAX_GEOMETRY_SIZE = 1024 * 10; //10KB
var LARGE_INDEX_DISTANCE = 50; //bbox more then 100 miles corner to corner.
var MAX_GEOMETRY_SIZE = 1024 * 10; // 10KB

module.exports = function Cardboard(config) {
config = config || {};
Expand Down Expand Up @@ -86,8 +85,20 @@ module.exports = function Cardboard(config) {
config.dyno.createTable(table, callback);
};

function listIds(dataset, callback) {
var query = { dataset: { EQ: dataset }, id: {BEGINS_WITH: 'id!'} };
var opts = { attributes: ['id'], pages: 0 };

config.dyno.query(query, opts, function(err, items) {
if (err) return callback(err);
callback(err, items.map(function(_) {
return _.id.split('!')[1];
}));
});
}

cardboard.delDataset = function(dataset, callback) {
cardboard.listIds(dataset, function(err, res) {
listIds(dataset, function(err, res) {
var keys = res.map(function(id) {
return { dataset: dataset, id: 'id!' + id };
});
Expand Down Expand Up @@ -122,18 +133,6 @@ module.exports = function Cardboard(config) {
});
};

cardboard.listIds = function(dataset, callback) {
var query = { dataset: { EQ: dataset }, id: {BEGINS_WITH: 'id!'} };
var opts = { attributes: ['id'], pages: 0 };

config.dyno.query(query, opts, function(err, items) {
if (err) return callback(err);
callback(err, items.map(function(_) {
return _.id.split('!')[1];
}));
});
};

cardboard.listDatasets = function(callback) {
var opts = { attributes: ['dataset'], pages:0 };

Expand Down Expand Up @@ -281,40 +280,5 @@ module.exports = function Cardboard(config) {
});
};

cardboard.dump = function(cb) {
return config.dyno.scan(cb);
};

cardboard.export = function(_) {
return config.dyno.scan()
.pipe(through({ objectMode: true }, function(data, enc, cb) {
var output = this.push.bind(this);

if (data.id.indexOf('id!') === 0) {
return utils.resolveFeatures([data], function(err, features) {
output(features.features[0]);
cb();
});
}

cb();
}))
.pipe(geojsonStream.stringify());
};

return cardboard;
};

function indexLevel(feature) {
var bbox = extent(feature);
var sw = point(bbox[0], bbox[1]);
var ne = point(bbox[2], bbox[3]);
var dist = distance(sw, ne, 'miles');
return dist >= LARGE_INDEX_DISTANCE ? 0 : 1;
}

function truncateNum(num, digits) {
digits = digits || 6;
var exp = Math.pow(10, digits);
return Math.round(exp * num) / exp;
}
129 changes: 44 additions & 85 deletions test/indexing.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,31 +38,14 @@ test('teardown', s.teardown);

test('setup', s.setup);

test('dump', function(t) {
var cardboard = Cardboard(config);
cardboard.dump(function(err, items) {
t.equal(err, null);
t.deepEqual(items, [], 'no results with a new database');
t.end();
});
});

test('teardown', s.teardown);

test('setup', s.setup);

test('insert & dump', function(t) {
test('insert', function(t) {
var cardboard = Cardboard(config);
var dataset = 'default';

cardboard.put(fixtures.nullIsland, dataset, function(err, res) {
t.equal(err, null);
t.pass('inserted');
cardboard.dump(function(err, data) {
t.equal(err, null);
t.equal(data.length, 1, 'creates data');
t.end();
});
t.end();
});
});

Expand Down Expand Up @@ -358,24 +341,6 @@ test('teardown', s.teardown);

test('setup', s.setup);

test('listIds', function(t) {
var cardboard = Cardboard(config);

cardboard.put(fixtures.nullIsland, 'default', function(err, primary) {
t.equal(err, null);
t.pass('inserted');

cardboard.listIds('default', function(err, data) {
t.deepEqual(data, [primary.id]);
t.end();
});
});
});

test('teardown', s.teardown);

test('setup', s.setup);

test('list first page with maxFeatures', function(t) {
var cardboard = Cardboard(config);
var features = featureCollection([_.clone(fixtures.haiti), _.clone(fixtures.haiti), _.clone(fixtures.haiti)]);
Expand Down Expand Up @@ -482,54 +447,6 @@ test('teardown', s.teardown);

test('setup', s.setup);

test('export', function(t) {
var cardboard = new Cardboard(config);
var first = geojsonFixtures.featurecollection.idaho.features.slice(0, 10);
var second = geojsonFixtures.featurecollection.idaho.features.slice(10, 20);

var q = queue();
first.forEach(function(f) {
q.defer(cardboard.put, f, 'first');
});

second.forEach(function(f) {
q.defer(cardboard.put, f, 'second');
});

q.defer(cardboard.calculateDatasetInfo, 'first');
q.defer(cardboard.calculateDatasetInfo, 'second');
q.awaitAll(function(err, ids) {
t.ifError(err, 'inserted and calc metadata');

first = first.map(function(f, i) {
return _.defaults({ id: ids[0] }, f);
});

second = second.map(function(f, i) {
return _.defaults({ id: ids[i + 10] }, f);
});

var expected = { type: 'FeatureCollection', features: _.union(first, second) };
var found = '';

cardboard.export()
.on('data', function(chunk) { found = found + chunk; })
.on('error', function(err) {
t.notOk(err, 'stream error');
t.end();
})
.on('end', function() {
found = JSON.parse(found);
t.equal(found.features.length, expected.features.length, 'expected number of features');
t.end();
});
});
});

test('teardown', s.teardown);

test('setup', s.setup);

test('insert & query', function(t) {
var queries = [
{
Expand Down Expand Up @@ -1375,3 +1292,45 @@ test('getDatasetInfo', function(t) {
});

test('teardown', s.teardown);

test('setup', s.setup);

test('calculateDatasetInfo', function(t) {
var cardboard = new Cardboard(config);
var features = geojsonFixtures.featurecollection.idaho.features.slice(0, 50);
var expectedBounds = geojsonExtent({ type: 'FeatureCollection', features: features });

cardboard.batch.put(featureCollection(features), dataset, function(err, putFeatures) {
t.ifError(err, 'inserted');

features = features.map(function(f, i) {
var feature = _.defaults({ id: putFeatures.features[i].id }, f);
return feature;
});

var expectedSize = features.reduce(function(memo, feature) {
memo = memo + Buffer.byteLength(JSON.stringify(feature));
return memo;
}, 0);

var expected = {
dataset: dataset,
id: metadata.recordId,
size: expectedSize,
count: features.length,
west: expectedBounds[0],
south: expectedBounds[1],
east: expectedBounds[2],
north: expectedBounds[3]
};

cardboard.calculateDatasetInfo(dataset, function(err, info) {
t.ifError(err, 'calculated');
t.ok(info.updated, 'has updated date');
t.deepEqual(_.omit(info, 'updated'), expected, 'returned expected info');
t.end();
});
});
});

test('teardown', s.teardown);

0 comments on commit d978ccc

Please sign in to comment.