Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Changed CLI to use Dyno.serialize #88

Merged
merged 2 commits into from
Jun 5, 2015
Merged

Changed CLI to use Dyno.serialize #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9208ce8
Switched CLI to use wire-format via serialize
Jun 5, 2015
693e5fb
Changed tests to base64
Jun 5, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,15 +73,15 @@ dyno local table my-table

##### Scan a table:

Outputs line delimited JSON for every item in the table.
Outputs line delimited JSON in wire-format for every item in the table.

```
dyno local scan my-table

{"id":"0.9410678697749972","collection":"somethign:0","attr":"moredata 64"}
{"id":"0.9417226337827742","collection":"somethign:0","attr":"moredata 24"}
{"id":"0.9447696127463132","collection":"somethign:0","attr":"moredata 48"}
{"id":"0.9472108569461852","collection":"somethign:0","attr":"moredata 84"}
{"id":{"N":"0.9410678697749972"},"collection":{"S":"somethign:0"},"attr":{"S":"moredata 64"}}
{"id":{"N":"0.9417226337827742"},"collection":{"S":"somethign:0"},"attr":{"S":"moredata 24"}}
{"id":{"N":"0.9447696127463132"},"collection":{"S":"somethign:0"},"attr":{"S":"moredata 48"}}
{"id":{"N":"0.9472108569461852"},"collection":{"S":"somethign:0"},"attr":{"S":"moredata 84"}}
....

```
Expand All @@ -94,10 +94,10 @@ Outputs the table schema then does a scan (like above)
dyno local export my-table

{"AttributeDefinitions":[{"AttributeName":"collection","AttributeType":"S"},...]}
{"id":"0.9410678697749972","collection":"somethign:0","attr":"moredata 64"}
{"id":"0.9417226337827742","collection":"somethign:0","attr":"moredata 24"}
{"id":"0.9447696127463132","collection":"somethign:0","attr":"moredata 48"}
{"id":"0.9472108569461852","collection":"somethign:0","attr":"moredata 84"}
{"id":{"N":"0.9410678697749972"},"collection":{"S":"somethign:0"},"attr":{"S":"moredata 64"}}
{"id":{"N":"0.9417226337827742"},"collection":{"S":"somethign:0"},"attr":{"S":"moredata 24"}}
{"id":{"N":"0.9447696127463132"},"collection":{"S":"somethign:0"},"attr":{"S":"moredata 48"}}
{"id":{"N":"0.9472108569461852"},"collection":{"S":"somethign:0"},"attr":{"S":"moredata 84"}}
....

```
Expand Down
15 changes: 2 additions & 13 deletions bin/cli.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,7 @@ function Stringifier() {
stringifier._readableState.objectMode = false;

stringifier._transform = function(record, enc, callback) {
var str = JSON.stringify(record, function(key, value) {
var val = this[key];
if (Buffer.isBuffer(val)) return 'base64:' + val.toString('base64');
return value;
});
var str = Dyno.serialize(record);

this.push(str + '\n');
setImmediate(callback);
Expand All @@ -91,14 +87,7 @@ function Parser() {
parser._transform = function(record, enc, callback) {
if (!record) return;

record = JSON.parse(record);

var val;
for (var key in record) {
val = record[key];
if (typeof val === 'string' && val.indexOf('base64:') === 0)
record[key] = new Buffer(val.split('base64:').pop(), 'base64');
}
record = Dyno.deserialize(record);

this.push(record);
setImmediate(callback);
Expand Down
21 changes: 11 additions & 10 deletions test/test.cli.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ var fixtures = require('./fixtures');
var setup = require('./setup')();
var test = setup.test;
var dyno = setup.dyno;
var Dyno = require('..');

// use --verbose to print cli output to terminal during the test run
var verbose = process.argv.indexOf('--verbose') > -1;
Expand Down Expand Up @@ -150,7 +151,7 @@ test('cli: export table', function(assert) {
assert.deepEqual(table, cleanedTable, 'printed cleaned table definition to stdout');

var expectedRecords = records.map(function(record) {
return JSON.stringify(record);
return Dyno.serialize(record);
});

var intersection = _.intersection(expectedRecords, results);
Expand All @@ -177,7 +178,7 @@ test('cli: scan table', function(assert) {
var results = stdout.trim().split('\n');

var expectedRecords = records.map(function(record) {
return JSON.stringify(record);
return Dyno.serialize(record);
});

var intersection = _.intersection(expectedRecords, results);
Expand All @@ -192,7 +193,7 @@ test('cli: setup', setup.setup());
test('cli: import table', function(assert) {
var records = fixtures.randomItems(10);
var serialized = _.union([cleanedTable], records).map(function(line) {
return JSON.stringify(line);
return Dyno.serialize(line);
}).join('\n');

var proc = runCli(['import', 'local/' + setup.tableName], function(err, stdout, stderr) {
Expand All @@ -204,11 +205,11 @@ test('cli: import table', function(assert) {
}

var expectedRecords = records.map(function(record) {
return JSON.stringify(record);
return Dyno.serialize(record);
});

items = items.map(function(record) {
return JSON.stringify(record);
return Dyno.serialize(record);
});

var intersection = _.intersection(expectedRecords, items);
Expand All @@ -229,7 +230,7 @@ test('cli: setup table', setup.setupTable);
test('cli: import data', function(assert) {
var records = fixtures.randomItems(10);
var serialized = records.map(function(line) {
return JSON.stringify(line);
return Dyno.serialize(line);
}).join('\n');

var proc = runCli(['put', 'local/' + setup.tableName], function(err, stdout, stderr) {
Expand All @@ -241,11 +242,11 @@ test('cli: import data', function(assert) {
}

var expectedRecords = records.map(function(record) {
return JSON.stringify(record);
return Dyno.serialize(record);
});

items = items.map(function(record) {
return JSON.stringify(record);
return Dyno.serialize(record);
});

var intersection = _.intersection(expectedRecords, items);
Expand Down Expand Up @@ -279,7 +280,7 @@ test('cli: export complicated record', function(assert) {

runCli(['scan', 'local/' + setup.tableName], function(err, stdout, stderr) {
assert.ifError(err, 'cli success');
assert.equal(stdout.trim(), '{"id":"id:1","range":1,"buffer":"base64:aGVsbG8gd29ybGQh","array":[0,1,2],"newline":"0\\n1"}', 'printed record to stdout');
assert.equal(stdout.trim(), '{"id":{"S":"id:1"},"range":{"N":"1"},"buffer":{"B":"aGVsbG8gd29ybGQh"},"array":{"L":[{"N":"0"},{"N":"1"},{"N":"2"}]},"newline":{"S":"0\\n1"}}', 'printed record to stdout');
assert.end();
});
});
Expand Down Expand Up @@ -312,7 +313,7 @@ test('cli: import complicated record', function(assert) {
});
});

proc.stdin.write('{"id":"id:1","range":1,"buffer":"base64:aGVsbG8gd29ybGQh","array":[0,1,2],"newline":"0\\n1"}');
proc.stdin.write('{"id":{"S":"id:1"},"range":{"N":"1"},"buffer":{"B":"aGVsbG8gd29ybGQh"},"array":{"L":[{"N":"0"},{"N":"1"},{"N":"2"}]},"newline":{"S":"0\\n1"}}');
proc.stdin.end();
});
test('cli: deleteTable', setup.deleteTable);
Expand Down