diff --git a/bin/import.js b/bin/import.js index 92948c8b..5ef97cef 100644 --- a/bin/import.js +++ b/bin/import.js @@ -24,13 +24,13 @@ function importCmd(dat, opts, cb) { if (!opts.quiet) console.error('No import file specified, using STDIN as input') input = process.stdin } else if (filename) { - if (!(opts.json || opts.csv)) { + if (!(opts.json || opts.csv || opts.tsv)) { var ending = path.extname(filename) if (ending === '.json') { opts.json = true; } else if (ending === '.tsv') { - opts.csv = true; - opts.separator = ' '; // use tab separator + opts.tsv = true; + opts.separator = '\t'; // use tab separator } else if (ending === '.csv') { opts.csv = true; } diff --git a/lib/commands.js b/lib/commands.js index 6d765674..6a5ebfb4 100644 --- a/lib/commands.js +++ b/lib/commands.js @@ -739,13 +739,16 @@ dat.createReadStream = function(opts) { if (!opts) opts = {} var pipeline = [this.storage.createReadStream(opts), decoder(this)] - if(!opts.format) { if(opts.csv) opts.format = 'csv' + if(opts.tsv) { + opts.format = 'csv' + opts.separator = '\t' + } if(opts.json) opts.format = 'json' if(opts.ndjson) opts.format = 'ndjson' } - + if(opts.format && opts.format !== 'objectMode') pipeline.push(formatData(opts)) diff --git a/lib/write-stream.js b/lib/write-stream.js index 7c06a865..a48cf635 100644 --- a/lib/write-stream.js +++ b/lib/write-stream.js @@ -40,6 +40,7 @@ module.exports = function writeStream(dat, opts) { function parseStream() { if (opts.csv || opts.f === 'csv') return parseCSV(opts.separator) + if (opts.tsv || opts.f === 'tsv') return parseCSV('\t') if (opts.json || opts.f === 'json') return parseJSON(opts.jsonpath) if (opts.protobuf || opts.f === 'protobuf') return parseProtobuf() if (opts.objects || opts.f === 'objects') return parseObjects() diff --git a/test/tests/cli.js b/test/tests/cli.js index 7d0754f5..00438e42 100644 --- a/test/tests/cli.js +++ b/test/tests/cli.js @@ -200,6 +200,43 @@ module.exports.importCSV = function(test, common) { }) } +module.exports.importCSVstdin = function(test, common) { + test('CLI dat import csv from stdin', function(t) { + common.destroyTmpDats(function() { + mkdirp(common.dat1tmp, function(err) { + t.notOk(err, 'no err') + initDat({cwd: common.dat1tmp, timeout: timeout, rpc: common.rpc}, function(cleanup) { + var cmd = datCmd + ' import --csv --quiet --results' + var dat = child.exec(cmd, {timeout: timeout, cwd: common.dat1tmp}, done) + dat.stdin.write('a,b,c\n1,2,3\n4,5,6\n7,8,9') + dat.stdin.end() + + function done(err, stdo, stde) { + if (process.env.DEBUG) { + process.stdout.write(stdo.toString()) + process.stdout.write(stde.toString()) + } + + t.notOk(err, 'no err') + t.equals(stde.toString(), '', 'empty stderr') + var lines = stdo.toString().split('\n') + var rows = [] + lines.map(function(l) { + if (l !== '') rows.push(JSON.parse(l)) + }) + t.equal(rows.length, 3) + rows.map(function(r) { t.ok(r.key, 'row has key') }) + common.destroyTmpDats(function() { + cleanup() + t.end() + }) + } + }) + }) + }) + }) +} + module.exports.importTSV = function(test, common) { test('CLI dat import tsv', function(t) { common.destroyTmpDats(function() { @@ -210,13 +247,50 @@ module.exports.importTSV = function(test, common) { fs.writeFileSync(testTsv, 'a\tb\tc\n1\t2\t3\n4\t5\t6\n7\t8\t9') var cmd = datCmd + ' import "' + testTsv + '" --quiet --results' child.exec(cmd, {timeout: timeout, cwd: common.dat1tmp}, done) - + function done(err, stdo, stde) { if (process.env.DEBUG) { process.stdout.write(stdo.toString()) process.stdout.write(stde.toString()) } - + + t.notOk(err, 'no err') + t.equals(stde.toString(), '', 'empty stderr') + var lines = stdo.toString().split('\n') + var rows = [] + lines.map(function(l) { + if (l !== '') rows.push(JSON.parse(l)) + }) + t.equal(rows.length, 3) + rows.map(function(r) { t.ok(r.key, 'row has key') }) + common.destroyTmpDats(function() { + cleanup() + t.end() + }) + } + }) + }) + }) + }) +} + +module.exports.importTSVstdin = function(test, common) { + test('CLI dat import tsv from stdin', function(t) { + common.destroyTmpDats(function() { + mkdirp(common.dat1tmp, function(err) { + t.notOk(err, 'no err') + initDat({cwd: common.dat1tmp, timeout: timeout, rpc: common.rpc}, function(cleanup) { + var cmd = datCmd + ' import --tsv --quiet --results' + var dat = child.exec(cmd, {timeout: timeout, cwd: common.dat1tmp}, done) + dat.stdin.write('a\tb\tc\n1\t2\t3\n4\t5\t6\n7\t8\t9') + dat.stdin.end() + + function done(err, stdo, stde) { + if (process.env.DEBUG) { + process.stdout.write(stdo.toString()) + process.stdout.write(stde.toString()) + } + t.notOk(err, 'no err') t.equals(stde.toString(), '', 'empty stderr') var lines = stdo.toString().split('\n') @@ -639,7 +713,9 @@ module.exports.all = function (test, common) { module.exports.listenEmptyDir(test, common) module.exports.listenPort(test, common) module.exports.importCSV(test, common) + module.exports.importCSVstdin(test, common) module.exports.importTSV(test, common) + module.exports.importTSVstdin(test, common) module.exports.blobs(test, common) module.exports.rows(test, common) module.exports.badCommand(test, common)