separate file loading from reading and thus support dbf files through…

… readData

src/directReaders/readDbf.js

@@ -1,5 +1,5 @@
-var shapefile = require('shapefile')
 import identity from '../utils/identity'
+import readData from '../readers/readData'
 
 /**
  * Asynchronously read a dbf file. Returns an empty array if file is empty.
@@ -32,13 +32,5 @@ export default function readDbf (filePath, opts_, cb) {
   } else {
     parserOptions = typeof opts_ === 'function' ? {map: opts_} : opts_
   }
-  var values = []
-  shapefile.openDbf(filePath)
-    .then(source => source.read()
-      .then(function log (result) {
-        if (result.done) return cb(null, values)
-        values.push(parserOptions.map(result.value)) // TODO, figure out i
-        return source.read().then(log)
-      }))
-    .catch(error => cb(error.stack))
+  readData(filePath, parserOptions, cb)
 }

src/helpers/discernLoader.js

@@ -0,0 +1,13 @@
+import discernFormat from './discernFormat'
+import loaders from '../loaders/index'
+
+export default function discernLoader (filePath, opts_ = {}) {
+  var which = opts_.sync === true ? 'sync' : 'async'
+  var format = discernFormat(filePath)
+  var loader = loaders[which][format]
+  // If we don't have a loader for this format, read in as a normal file
+  if (typeof loader === 'undefined') {
+    loader = loaders[which]['txt']
+  }
+  return loader
+}

src/loaders/dbf.js

@@ -0,0 +1,13 @@
+var shapefile = require('shapefile')
+
+export default function dbf (filePath, parser, parserOptions, cb) {
+  var values = []
+  shapefile.openDbf(filePath)
+    .then(source => source.read()
+      .then(function log (result) {
+        if (result.done) return cb(null, values)
+        values.push(parserOptions.map(result.value)) // TODO, figure out i
+        return source.read().then(log)
+      }))
+    .catch(error => cb(error.stack))
+}

src/loaders/file.js

@@ -0,0 +1,30 @@
+import fs from 'fs'
+import discernFormat from '../helpers/discernFormat'
+import {formatsIndex} from '../config/equivalentFormats'
+
+export default function file (filePath, parser, parserOptions, cb) {
+  fs.readFile(filePath, 'utf8', function (err, data) {
+    var fileFormat = discernFormat(filePath)
+    if ((fileFormat === 'json' || formatsIndex.json.indexOf(fileFormat) > -1) && data === '') {
+      data = '[]'
+    }
+    if (err) {
+      cb(err)
+      return false
+    }
+    var parsed
+    try {
+      if (typeof parser === 'function') {
+        parsed = parser(data, parserOptions)
+      } else if (typeof parser === 'object' && typeof parser.parse === 'function') {
+        parsed = parser.parse(data, parserOptions)
+      } else {
+        parsed = 'Your specified parser is not properly formatted. It must either be a function or have a `parse` method.'
+      }
+    } catch (err) {
+      cb(err)
+      return
+    }
+    cb(null, parsed)
+  })
+}

src/loaders/fileSync.js

@@ -0,0 +1,25 @@
+import fs from 'fs'
+import discernFormat from '../helpers/discernFormat'
+import {formatsIndex} from '../config/equivalentFormats'
+
+export default function file (filePath, parser, parserOptions, cb) {
+  var data = fs.readFileSync(filePath, 'utf8')
+  var fileFormat = discernFormat(filePath)
+  if ((fileFormat === 'json' || formatsIndex.json.indexOf(fileFormat) > -1) && data === '') {
+    data = '[]'
+  }
+
+  var parsed
+  if (typeof parser === 'function') {
+    parsed = parser(data, parserOptions)
+  } else if (typeof parser === 'object' && typeof parser.parse === 'function') {
+    parsed = parser.parse(data, parserOptions)
+  } else {
+    return new Error('Your specified parser is not properly formatted. It must either be a function or have a `parse` method.')
+  }
+
+  // if (opts_ && opts_.flatten) {
+  //   parsed = _.map(parsed, flatten)
+  // }
+  return parsed
+}

src/loaders/index.js

@@ -0,0 +1,36 @@
+import file from './file'
+import fileSync from './fileSync'
+import dbf from './dbf'
+import {formatsList} from '../config/equivalentFormats'
+
+let loaders = {
+  async: {
+    aml: file,
+    csv: file,
+    psv: file,
+    tsv: file,
+    txt: file,
+    json: file,
+    yaml: file,
+    dbf
+  },
+  sync: {
+    aml: fileSync,
+    csv: fileSync,
+    psv: fileSync,
+    tsv: fileSync,
+    txt: fileSync,
+    json: fileSync,
+    yaml: fileSync
+  }
+}
+
+formatsList.forEach(format => {
+  format.equivalents.forEach(equivalent => {
+    Object.keys(loaders).forEach(key => {
+      loaders[key][equivalent] = loaders[key][format.name]
+    })
+  })
+})
+
+export default loaders

src/readers/readData.js

@@ -1,9 +1,7 @@
-import fs from 'fs'
+import _ from 'underscore'
 import getParser from '../helpers/getParser'
+import discernLoader from '../helpers/discernLoader'
 import discernParser from '../helpers/discernParser'
-import discernFormat from '../helpers/discernFormat'
-import {formatsIndex} from '../config/equivalentFormats'
-import _ from 'underscore'
 
 /**
  * Asynchronously read data given a path ending in the file format.
@@ -17,10 +15,9 @@ import _ from 'underscore'
  * * `.yaml` or `.yml` Yaml file
  * * `.aml` ArchieML
  * * `.txt` Text file (a string)
+ * * `.dbf` Database format used for shapefiles
  * * other All others are read as a text file
  *
- * *Note: Does not currently support `.dbf` files.
- *
  * @function readData
  * @param {String} filePath Input file path
  * @param {Function|Object} [parserOptions] Optional map function or an object specifying the optional options below.
@@ -109,28 +106,6 @@ export default function readData (filePath, opts_, cb_) {
   } else {
     parser = discernParser(filePath)
   }
-  fs.readFile(filePath, 'utf8', function (err, data) {
-    var fileFormat = discernFormat(filePath)
-    if ((fileFormat === 'json' || formatsIndex.json.indexOf(fileFormat) > -1) && data === '') {
-      data = '[]'
-    }
-    if (err) {
-      cb(err)
-      return false
-    }
-    var parsed
-    try {
-      if (typeof parser === 'function') {
-        parsed = parser(data, parserOptions)
-      } else if (typeof parser === 'object' && typeof parser.parse === 'function') {
-        parsed = parser.parse(data, parserOptions)
-      } else {
-        parsed = 'Your specified parser is not properly formatted. It must either be a function or have a `parse` method.'
-      }
-    } catch (err) {
-      cb(err)
-      return
-    }
-    cb(null, parsed)
-  })
+  var loader = discernLoader(filePath)
+  loader(filePath, parser, parserOptions, cb)
 }

src/readers/readDataSync.js

@@ -1,12 +1,19 @@
-import fs from 'fs'
 import getParser from '../helpers/getParser'
 import discernParser from '../helpers/discernParser'
-import discernFormat from '../helpers/discernFormat'
-import {formatsIndex} from '../config/equivalentFormats'
+import discernLoader from '../helpers/discernLoader'
 import _ from 'underscore'
 
 /**
- * Syncronous version of {@link readData}. Read data given a path ending in the file format.
+ * Syncronous version of {@link readData}. Read data given a path ending in the file format. This function detects the same formats as the asynchronous {@link readData} except for `.dbf` files, which it cannot read.
+ *
+ * * `.json` Array of objects or object
+ * * `.csv` Comma-separated
+ * * `.tsv` Tab-separated
+ * * `.psv` Pipe-separated
+ * * `.yaml` or `.yml` Yaml file
+ * * `.aml` ArchieML
+ * * `.txt` Text file (a string)
+ * * other All others are read as a text file
  *
  * @function readDataSync
  * @param {String} filePath Input file path
@@ -89,23 +96,6 @@ export default function readDataSync (filePath, opts_) {
   } else {
     parser = discernParser(filePath)
   }
-  var data = fs.readFileSync(filePath, 'utf8')
-  var fileFormat = discernFormat(filePath)
-  if ((fileFormat === 'json' || formatsIndex.json.indexOf(fileFormat) > -1) && data === '') {
-    data = '[]'
-  }
-
-  var parsed
-  if (typeof parser === 'function') {
-    parsed = parser(data, parserOptions)
-  } else if (typeof parser === 'object' && typeof parser.parse === 'function') {
-    parsed = parser.parse(data, parserOptions)
-  } else {
-    return new Error('Your specified parser is not properly formatted. It must either be a function or have a `parse` method.')
-  }
-
-  // if (opts_ && opts_.flatten) {
-  //   parsed = _.map(parsed, flatten)
-  // }
-  return parsed
+  var loader = discernLoader(filePath, {sync: true})
+  return loader(filePath, parser, parserOptions)
 }

test/test.js

@@ -948,29 +948,23 @@ describe('readers', function () {
           done()
         })
       })
-    })
 
-    describe('json', function () {
       it('should match expected geojson', function (done) {
         io.readData(testDataPath('geojson/basic.geojson'), function (err, json) {
           assert.equal(err, null)
           assertBasicValid(json)
           done()
         })
       })
-    })
 
-    describe('json', function () {
       it('should match expected topojson', function (done) {
         io.readData(testDataPath('topojson/basic.topojson'), function (err, json) {
           assert.equal(err, null)
           assertBasicValid(json)
           done()
         })
       })
-    })
 
-    describe('json with map', function () {
       it('should match expected json', function (done) {
         io.readData(testDataPath('json/basic.json'), {
           map: function (row, i) {
@@ -1374,6 +1368,55 @@ describe('readers', function () {
       })
     })
 
+    describe('dbf', function () {
+      describe('empty', function () {
+        it('should be empty array', function (done) {
+          io.readData(testDataPath('dbf/empty.dbf'), function (err, json) {
+            assert.equal(err.split('\n')[0], 'TypeError: Cannot read property \'buffer\' of null')
+            done()
+          })
+        })
+      })
+
+      describe('basic', function () {
+        it('should match expected json', function (done) {
+          io.readDbf(testDataPath('dbf/basic.dbf'), function (err, json) {
+            assert.equal(err, null)
+            assert(_.isEqual(JSON.stringify(json), '[{"foo":"orange","bar":0},{"foo":"blue","bar":1},{"foo":"green","bar":2}]'))
+            done()
+          })
+        })
+      })
+
+      describe('basic map', function () {
+        it('should match expected json', function (done) {
+          io.readData(testDataPath('dbf/basic.dbf'), {
+            map: function (row, i) {
+              row.bar = row.bar * 2
+              return row
+            }
+          }, function (err, json) {
+            assert.equal(err, null)
+            assert(_.isEqual(JSON.stringify(json), '[{"foo":"orange","bar":0},{"foo":"blue","bar":2},{"foo":"green","bar":4}]'))
+            done()
+          })
+        })
+      })
+
+      describe('basic map shorthand', function () {
+        it('should match expected json', function (done) {
+          io.readData(testDataPath('dbf/basic.dbf'), function (row, i) {
+            row.bar = row.bar * 2
+            return row
+          }, function (err, json) {
+            assert.equal(err, null)
+            assert(_.isEqual(JSON.stringify(json), '[{"foo":"orange","bar":0},{"foo":"blue","bar":2},{"foo":"green","bar":4}]'))
+            done()
+          })
+        })
+      })
+    })
+
     describe('custom delimiter string: `_`', function () {
       it('should match expected json', function (done) {
         io.readData(testDataPath('other/basic.usv'), {parser: '_'}, function (err, json) {