Skip to content

Commit

Permalink
allow for native json parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@mhkeller
mhkeller committed Oct 30, 2016
1 parent f30f855 commit 570e6a9
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 38 deletions.
6 changes: 5 additions & 1 deletion lib/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ var path = require('path')
var queue = require('d3-queue').queue
var _ = require('underscore')
var chalk = require('chalk')
var jsonParser = require('parse-json') // This library has better error reporting of malformed json.
var parseJson = require('parse-json') // This library has better error reporting of malformed json.
var yamlParser = require('js-yaml')
var mkdirp = require('mkdirp')
var archieml = require('archieml')
Expand Down Expand Up @@ -79,6 +79,7 @@ var parsers = {
parserOptions = parserOptions || {}
// Do a naive test whether this is a string or an object
var mapFn = parserOptions.map ? str.trim().charAt(0) === '[' ? _.map : _.mapObject : _.identity
var jsonParser = parserOptions.nativeParser === true ? JSON.parse : parseJson
return mapFn(jsonParser(str, parserOptions.reviver, parserOptions.filename), parserOptions.map)
},
csv: function (str, parserOptions) {
Expand Down Expand Up @@ -543,6 +544,7 @@ var readers = {}
* @param {Object|Function} [options] Optional. Set this as a function as a shorthand for `map`.
* @param {String|Function|Object} [options.parser] optional This can be a string that is the file's delimiter or a function that returns the json. See `parsers` in library source for examples. For convenience, this can also take a dsv object such as `dsv.dsv('_')` or any object that has a `parse` method.
* @param {Function} [options.map] Transformation function. Takes `(fileString, options)` where `options` is the hash you pass in minus the `parser` key. See {@link shorthandReaders} for specifics.
* @param {Boolean} [options.nativeParser Used in {@link shorthandReaders.readJson} for now. Otherwise ignored.
* @param {Function} [options.comments] Used in {@link shorthandReaders.readAml}. Otherwise ignored.
* @param {Function} [options.reviver] Used in {@link shorthandReaders.readJson}. Otherwise ignored.
* @param {Function} [options.filename] Used in {@link shorthandReaders.readJson}. Otherwise ignored.
Expand Down Expand Up @@ -890,6 +892,7 @@ shorthandReaders.readCsvSync = function (path, opts_) {
* @param {String} fileName the name of the file
* @param {Function|String|Object} [parserOptions] Can be a map function, a filename string that's display as an error message or an object specifying both and other options.
* @param {Function} [parserOptions.map] Optional map function, called once for each row (header row skipped). If your file is an array (tests if first non-whitespace character is a `[`), has signature `(row, i)`, delegates to `_.map`. If file is an object has signature `(value, key)`, delegates to `_.mapObject`. See example below.
* @param {Boolean} [options.nativeParser Use native JSON parser instead of parse-json library for better performance but not as good error messaging. This can be nice to turn on for produciton use.
* @param {String} [parserOptions.filename] Filename displayed in the error message.
* @param {String} [parserOptions.reviver] Prescribes how the value originally produced by parsing is mapped, before being returned. See JSON.parse docs for more: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse#Using_the_reviver_parameter
* @param {Function} callback Function invoked after data is read, takes error (if any) and the data read
Expand Down Expand Up @@ -965,6 +968,7 @@ shorthandReaders.readJson = function (path, opts_, cb) {
* @param {String} fileName the name of the file
* @param {Function|String|Object} [parserOptions] Can be a map function, a filename string that's display as an error message or an object specifying both and other options.
* @param {Function} [parserOptions.map] Optional map function, called once for each row (header row skipped). If your file is an array (tests if first non-whitespace character is a `[`), has signature `(row, i)`, delegates to `_.map`. If file is an object has signature `(value, key)`, delegates to `_.mapObject`. See example below.
* @param {Boolean} [options.nativeParser Use native JSON parser instead of parse-json library for better performance but not as good error messaging. This can be nice to turn on for produciton use.
* @param {String} [parserOptions.filename] Filename displayed in the error message.
* @param {String} [parserOptions.reviver] Prescribes how the value originally produced by parsing is mapped, before being returned. See JSON.parse docs for more: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse#Using_the_reviver_parameter
* @returns {Array} the contents of the file as JSON
Expand Down
140 changes: 103 additions & 37 deletions test/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1615,60 +1615,112 @@ describe('readers', function () {

describe('shorthandReaders', function () {
describe('readJsonSync()', function () {
describe('empty', function () {
it('should be empty', function () {
assert.lengthOf(io.readJsonSync(testDataPath('json/empty.json')), 0)
describe('parseJson parser', function () {
describe('empty', function () {
it('should be empty', function () {
assert.lengthOf(io.readJsonSync(testDataPath('json/empty.json')), 0)
})
})
})

describe('basic', function () {
it('should match expected json', function () {
var json = io.readJsonSync(testDataPath('json/basic.json'))
assertBasicValid(json)
describe('basic', function () {
it('should match expected json', function () {
var json = io.readJsonSync(testDataPath('json/basic.json'))
assertBasicValid(json)
})
})
})

describe('basic', function () {
it('should match expected geojson', function () {
var json = io.readJsonSync(testDataPath('geojson/basic.geojson'))
assertBasicValid(json)
describe('basic', function () {
it('should match expected geojson', function () {
var json = io.readJsonSync(testDataPath('geojson/basic.geojson'))
assertBasicValid(json)
})
})
})

describe('basic', function () {
it('should match expected topojson', function () {
var json = io.readJsonSync(testDataPath('topojson/basic.topojson'))
assertBasicValid(json)
describe('basic', function () {
it('should match expected topojson', function () {
var json = io.readJsonSync(testDataPath('topojson/basic.topojson'))
assertBasicValid(json)
})
})
})

describe('basic map', function () {
it('should use map', function () {
var json = io.readJsonSync(testDataPath('json/basic.json'), {
map: function (row, i) {
describe('basic map', function () {
it('should use map', function () {
var json = io.readJsonSync(testDataPath('json/basic.json'), {
map: function (row, i) {
row.height = row.height * 2
return row
}
})
assert(_.isEqual(JSON.stringify(json), '[{"name":"jim","occupation":"land surveyor","height":140},{"name":"francis","occupation":"conductor","height":126}]'))
})
})

describe('basic map shorthand', function () {
it('should use map shorthand', function () {
var json = io.readJsonSync(testDataPath('json/basic.json'), function (row, i) {
row.height = row.height * 2
return row
}
})
assert(_.isEqual(JSON.stringify(json), '[{"name":"jim","occupation":"land surveyor","height":140},{"name":"francis","occupation":"conductor","height":126}]'))
})
assert(_.isEqual(JSON.stringify(json), '[{"name":"jim","occupation":"land surveyor","height":140},{"name":"francis","occupation":"conductor","height":126}]'))
})
})

describe('basic map shorthand', function () {
it('should use map shorthand', function () {
var json = io.readJsonSync(testDataPath('json/basic.json'), function (row, i) {
row.height = row.height * 2
return row
describe('invalid', function () {
it('should raise an error', function () {
assert.throws(function () {
io.readJsonSync(testDataPath('json/invalid.json'))
}, Error)
})
assert(_.isEqual(JSON.stringify(json), '[{"name":"jim","occupation":"land surveyor","height":140},{"name":"francis","occupation":"conductor","height":126}]'))
})
})

describe('invalid', function () {
it('should raise an error', function () {
assert.throws(function () {
io.readJsonSync(testDataPath('json/invalid.json'))
}, Error)
describe('native parser', function () {
describe('empty', function () {
it('should be empty', function () {
assert.lengthOf(io.readJsonSync(testDataPath('json/empty.json'), {nativeParser: true}), 0)
})
})

describe('basic', function () {
it('should match expected json', function () {
var json = io.readJsonSync(testDataPath('json/basic.json'), {nativeParser: true})
assertBasicValid(json)
})
})

describe('basic', function () {
it('should match expected geojson', function () {
var json = io.readJsonSync(testDataPath('geojson/basic.geojson'), {nativeParser: true})
assertBasicValid(json)
})
})

describe('basic', function () {
it('should match expected topojson', function () {
var json = io.readJsonSync(testDataPath('topojson/basic.topojson'), {nativeParser: true})
assertBasicValid(json)
})
})

describe('basic map', function () {
it('should use map', function () {
var json = io.readJsonSync(testDataPath('json/basic.json'), {
map: function (row, i) {
row.height = row.height * 2
return row
},
nativeParser: true
})
assert(_.isEqual(JSON.stringify(json), '[{"name":"jim","occupation":"land surveyor","height":140},{"name":"francis","occupation":"conductor","height":126}]'))
})
})

describe('invalid', function () {
it('should raise an error', function () {
assert.throws(function () {
io.readJsonSync(testDataPath('json/invalid.json'), {nativeParser: true})
}, Error)
})
})
})
})
Expand Down Expand Up @@ -2446,6 +2498,20 @@ describe('writers', function () {
})
})

describe('json', function () {
it('should write as json without making directory', function (done) {
var filePath = ['test', 'test-out-data.json']
io.writeData(filePath.join(path.sep), testData, function (err) {
assert.equal(err, null)
readAssertBasicValid(filePath.join(path.sep))
rimraf(filePath.join(path.sep), {glob: false}, function (err) {
assert.equal(err, null)
done()
})
})
})
})

describe('geojson', function () {
it('should write as geojson', function (done) {
var filePath = ['test', 'tmp-write-data-geojson', 'data.geojson']
Expand Down

0 comments on commit 570e6a9

Please sign in to comment.