WIP strip bom

src/loaders/file.js

@@ -2,6 +2,7 @@
 import fs from 'fs'
 import discernFormat from '../helpers/discernFormat'
 import {formatsIndex} from '../config/equivalentFormats'
+import stripBom from '../utils/stripBom.js'
 
 export default function file (filePath, parser, parserOptions, cb) {
   fs.readFile(filePath, 'utf8', function (err, data) {
@@ -26,6 +27,6 @@ export default function file (filePath, parser, parserOptions, cb) {
       cb(err)
       return
     }
-    cb(null, parsed)
+    cb(null, stripBom(parsed))
   })
 }

src/loaders/fileSync.js

@@ -2,6 +2,7 @@
 import fs from 'fs'
 import discernFormat from '../helpers/discernFormat'
 import {formatsIndex} from '../config/equivalentFormats'
+import stripBom from '../utils/stripBom.js'
 
 export default function file (filePath, parser, parserOptions, cb) {
   var data = fs.readFileSync(filePath, 'utf8')
@@ -19,8 +20,5 @@ export default function file (filePath, parser, parserOptions, cb) {
     return new Error('Your specified parser is not properly formatted. It must either be a function or have a `parse` method.')
   }
 
-  // if (opts_ && opts_.flatten) {
-  //   parsed = _.map(parsed, flatten)
-  // }
-  return parsed
+  return stripBom(parsed)
 }

src/utils/identity.js

@@ -1 +1 @@
-export default (d => d)
+export default d => d

src/utils/stripBom.js

@@ -0,0 +1,11 @@
+// from https://github.com/sindresorhus/strip-bom/blob/d5696fdc9eeb6cc8d97e390cf1de7558f74debd5/index.js#L3
+
+export default function stripBom (string) {
+  // Catches EFBBBF (UTF-8 BOM) because the buffer-to-string
+  // conversion translates it to FEFF (UTF-16 BOM)
+  if (string.charCodeAt(0) === 0xFEFF) {
+    return string.slice(1)
+  }
+
+  return string
+};

test/data/bom/bom.csv

@@ -0,0 +1,3 @@
+name,occupation,height
+jim,land surveyor,70
+francis,conductor,63

test/test.js

@@ -62,6 +62,44 @@ function assertBasicValidObject (obj, strings, row) {
   }
 }
 
+describe('csv with bom characters', function () {
+  describe('readDataSync()', function () {
+    it('should be read proper keys', function () {
+      var file = io.readDataSync(testDataPath('bom/bom.csv'))
+      // file.forEach(d => {
+      //   Object.keys(d).forEach(k => {
+      //     d[k.trim()]= d[k]
+      //   })
+      // })
+      assert.equal(file[0].name, 'jim')
+      assert.equal(file[0].occupation, 'land surveyor')
+      assert.equal(file[0].height, 70)
+      assert.equal(file[1].name, 'francis')
+      assert.equal(file[1].occupation, 'conductor')
+      assert.equal(file[1].height, 63)
+    })
+  })
+
+  describe('readData()', function () {
+    it('should be read proper keys', function (done) {
+      io.readData(testDataPath('bom/bom.csv'), (err, file) => {
+        // file.forEach(d => {
+        //   Object.keys(d).forEach(k => {
+        //     d[k.trim()]= d[k]
+        //   })
+        // })
+        assert.equal(file[0].name, 'jim')
+        assert.equal(file[0].occupation, 'land surveyor')
+        assert.equal(file[0].height, 70)
+        assert.equal(file[1].name, 'francis')
+        assert.equal(file[1].occupation, 'conductor')
+        assert.equal(file[1].height, 63)
+        done()
+      })
+    })
+  })
+
+})
 describe('discernFormat()', function () {
   describe('no extension', function () {
     it('should be false', function () {