detect .tsv file extension

add --tsv command-line option
Tab-separated tables (.tsv) are really common in unix-based informatics pipelines. Adding a --tsv cli option makes it much simpler to pipe in and out tab-separated data.
Tab-separated tables (.tsv) are really common in unix-based informatics pipelines. Adding a --tsv cli option makes it much simpler to pipe in and out tab-separated data.
datproject
Unified Split

Showing with 87 additions and 7 deletions.

+3 −3 bin/import.js

+5 −2 lib/commands.js

+1 −0 lib/write-stream.js

+78 −2 test/tests/cli.js
diff --git a/bin/import.js b/bin/import.js
@@ -24,13 +24,13 @@ function importCmd(dat, opts, cb) {
    if (!opts.quiet) console.error('No import file specified, using STDIN as input')
    input = process.stdin
  } else if (filename) {
-    if (!(opts.json || opts.csv)) {
+    if (!(opts.json || opts.csv || opts.tsv)) {
      var ending = path.extname(filename)
      if (ending === '.json') {
          opts.json = true;
      } else if (ending === '.tsv') {
-          opts.csv = true;
-          opts.separator = '	';  // use tab separator
+          opts.tsv = true;
+          opts.separator = '\t';  // use tab separator
      } else if (ending === '.csv') {
          opts.csv = true;
      }
diff --git a/lib/commands.js b/lib/commands.js
@@ -739,13 +739,16 @@ dat.createReadStream = function(opts) {
  if (!opts) opts = {}

  var pipeline = [this.storage.createReadStream(opts), decoder(this)]
-  
  if(!opts.format) {
    if(opts.csv) opts.format = 'csv'
+    if(opts.tsv) {
+      opts.format = 'csv'
+      opts.separator = '\t'
+    }             
    if(opts.json) opts.format = 'json'
    if(opts.ndjson) opts.format = 'ndjson'
  }
-  
+
  if(opts.format && opts.format !== 'objectMode')
    pipeline.push(formatData(opts)) 

diff --git a/lib/write-stream.js b/lib/write-stream.js
@@ -40,6 +40,7 @@ module.exports = function writeStream(dat, opts) {
  
  function parseStream() {
    if (opts.csv || opts.f === 'csv') return parseCSV(opts.separator)
+    if (opts.tsv || opts.f === 'tsv') return parseCSV('\t')
    if (opts.json || opts.f === 'json') return parseJSON(opts.jsonpath)
    if (opts.protobuf || opts.f === 'protobuf') return parseProtobuf()
    if (opts.objects || opts.f === 'objects') return parseObjects()
diff --git a/test/tests/cli.js b/test/tests/cli.js
@@ -200,6 +200,43 @@ module.exports.importCSV = function(test, common) {
  })
 }

+module.exports.importCSVstdin = function(test, common) {
+  test('CLI dat import csv from stdin', function(t) {
+    common.destroyTmpDats(function() {
+      mkdirp(common.dat1tmp, function(err) {
+        t.notOk(err, 'no err')
+        initDat({cwd: common.dat1tmp, timeout: timeout, rpc: common.rpc}, function(cleanup) {
+          var cmd = datCmd + ' import --csv --quiet --results'
+          var dat = child.exec(cmd, {timeout: timeout, cwd: common.dat1tmp}, done)
+          dat.stdin.write('a,b,c\n1,2,3\n4,5,6\n7,8,9')
+          dat.stdin.end()
+
+          function done(err, stdo, stde) {
+            if (process.env.DEBUG) {
+              process.stdout.write(stdo.toString())
+              process.stdout.write(stde.toString())
+            }
+
+            t.notOk(err, 'no err')
+            t.equals(stde.toString(), '', 'empty stderr')
+            var lines = stdo.toString().split('\n')
+            var rows = []
+            lines.map(function(l) {
+              if (l !== '') rows.push(JSON.parse(l))
+            })
+            t.equal(rows.length, 3)
+            rows.map(function(r) { t.ok(r.key, 'row has key') })
+            common.destroyTmpDats(function() {
+              cleanup()
+              t.end()
+            })
+          }
+        })
+      })
+    })
+  })
+}
+
 module.exports.importTSV = function(test, common) {
  test('CLI dat import tsv', function(t) {
    common.destroyTmpDats(function() {
@@ -210,13 +247,50 @@ module.exports.importTSV = function(test, common) {
          fs.writeFileSync(testTsv, 'a\tb\tc\n1\t2\t3\n4\t5\t6\n7\t8\t9')
          var cmd = datCmd + ' import "' + testTsv + '" --quiet --results'
          child.exec(cmd, {timeout: timeout, cwd: common.dat1tmp}, done)
-          
+
          function done(err, stdo, stde) {
            if (process.env.DEBUG) {
              process.stdout.write(stdo.toString())
              process.stdout.write(stde.toString())
            }
-            
+
+            t.notOk(err, 'no err')
+            t.equals(stde.toString(), '', 'empty stderr')
+            var lines = stdo.toString().split('\n')
+            var rows = []
+            lines.map(function(l) {
+              if (l !== '') rows.push(JSON.parse(l))
+            })
+            t.equal(rows.length, 3)
+            rows.map(function(r) { t.ok(r.key, 'row has key') })
+            common.destroyTmpDats(function() {
+              cleanup()
+              t.end()
+            })
+          }
+        })
+      })
+    })
+  })
+}
+
+module.exports.importTSVstdin = function(test, common) {
+  test('CLI dat import tsv from stdin', function(t) {
+    common.destroyTmpDats(function() {
+      mkdirp(common.dat1tmp, function(err) {
+        t.notOk(err, 'no err')
+        initDat({cwd: common.dat1tmp, timeout: timeout, rpc: common.rpc}, function(cleanup) {
+          var cmd = datCmd + ' import --tsv --quiet --results'
+          var dat = child.exec(cmd, {timeout: timeout, cwd: common.dat1tmp}, done)
+          dat.stdin.write('a\tb\tc\n1\t2\t3\n4\t5\t6\n7\t8\t9')
+          dat.stdin.end()
+
+          function done(err, stdo, stde) {
+            if (process.env.DEBUG) {
+              process.stdout.write(stdo.toString())
+              process.stdout.write(stde.toString())
+            }
+
            t.notOk(err, 'no err')
            t.equals(stde.toString(), '', 'empty stderr')
            var lines = stdo.toString().split('\n')
@@ -639,7 +713,9 @@ module.exports.all = function (test, common) {
  module.exports.listenEmptyDir(test, common)
  module.exports.listenPort(test, common)
  module.exports.importCSV(test, common)
+  module.exports.importCSVstdin(test, common)
  module.exports.importTSV(test, common)
+  module.exports.importTSVstdin(test, common)
  module.exports.blobs(test, common)
  module.exports.rows(test, common)
  module.exports.badCommand(test, common)