From 4c7ece211793d77ed686f97e98a23b63095ae65b Mon Sep 17 00:00:00 2001 From: Nick Babcock Date: Thu, 5 Feb 2015 13:51:13 -0500 Subject: [PATCH] Use jison and not hand rolled parsing --- .gitignore | 1 + cli.js | 14 +- gulpfile.js | 17 +- index.js | 4 +- lib/header.js | 39 ---- lib/jomini.jison | 86 ++++++++ lib/parse.js | 10 - lib/parser.js | 495 ----------------------------------------- lib/setProp.js | 20 ++ package.json | 9 +- test/header.js | 51 ----- test/parse.js | 296 +++++++++++++++++++++++-- test/toDate.js | 4 + test/toJson.js | 559 ----------------------------------------------- 14 files changed, 417 insertions(+), 1188 deletions(-) delete mode 100644 lib/header.js create mode 100644 lib/jomini.jison delete mode 100644 lib/parse.js delete mode 100644 lib/parser.js create mode 100644 lib/setProp.js delete mode 100644 test/header.js delete mode 100644 test/toJson.js diff --git a/.gitignore b/.gitignore index 25fbf5a..b3d6dfe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ coverage/ +lib/jomini.js diff --git a/cli.js b/cli.js index 17369c2..766326b 100644 --- a/cli.js +++ b/cli.js @@ -1,8 +1,10 @@ var jomini = require('./'); var JS = require('json3'); -var header = new jomini.Header({header: 'EU4txt'}); -var parser = new jomini.Parser(); -process.stdin.pipe(header).pipe(parser); -parser.on('finish', function() { - process.stdout.write(JS.stringify(parser.obj), 'utf8', function() {}); -}); \ No newline at end of file +var concat = require('concat-stream'); +var concatStream = concat(function(buf) { + var str = buf.toString('utf8'); + var obj = jomini.parse(str); + process.stdout.write(JS.stringify(obj)); +}); + +process.stdin.pipe(concatStream) diff --git a/gulpfile.js b/gulpfile.js index 04ed796..30be73e 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -3,9 +3,10 @@ var jscs = require('gulp-jscs'); var mocha = require('gulp-mocha'); var jshint = require('gulp-jshint'); var istanbul = require('gulp-istanbul'); +var jison = require('gulp-jison'); -gulp.task('test', function(cb) { - gulp.src('lib/*.js') +gulp.task('test', ['jison'], function(cb) { + gulp.src(['lib/*.js', '!lib/jomini.js']) .pipe(istanbul()) .pipe(istanbul.hookRequire()) .on('finish', function() { @@ -20,15 +21,21 @@ gulp.task('test', function(cb) { }); }); +gulp.task('jison', function() { + return gulp.src('./src/*.jison') + .pipe(jison({ moduleType: 'commonjs' })) + .pipe(gulp.dest('./src/')); +}); + gulp.task('lint', function() { - return gulp.src(['lib/**/*.js', 'test/**/*.js']) + return gulp.src(['lib/**/*.js', 'test/**/*.js', '!lib/jomini.js']) .pipe(jshint()) .pipe(jshint.reporter('jshint-stylish')) .pipe(jshint.reporter('fail')); }); -gulp.task('main', ['test', 'lint'], function() { - return gulp.src(['lib/*js']).pipe(jscs({ +gulp.task('main', ['jison', 'test', 'lint'], function() { + return gulp.src(['lib/*js', '!lib/jomini.js']).pipe(jscs({ preset: 'google' })); }); diff --git a/index.js b/index.js index 0563f5b..fdd9d8f 100644 --- a/index.js +++ b/index.js @@ -1,7 +1,5 @@ exports = module.exports; -exports.Parser = require('./lib/parser'); -exports.Header = require('./lib/header'); -exports.parse = require('./lib/parse'); +exports.parse = require('./lib/jomini').parser.parse; exports.toArray = require('./lib/toArray'); exports.toDate = require('./lib/toDate'); exports.toBool = require('./lib/toBool'); diff --git a/lib/header.js b/lib/header.js deleted file mode 100644 index 44e8420..0000000 --- a/lib/header.js +++ /dev/null @@ -1,39 +0,0 @@ -var util = require('util'); -var Transform = require('stream').Transform; - -util.inherits(Header, Transform); - -function Header(options) { - if (!(this instanceof Header)) { - return new Header(options); - } - - Transform.call(this, options); - this.header = options.header; - this.first = true; -} - -Header.prototype._transform = function(data, encoding, cb) { - // If this is the first time seeing data, check to see if the buffer starts - // with our header. If it is not the first time, that means we saw the right - // header so just pass through the data - if (this.first === true) { - var header = data.toString('utf8', 0, this.header.length); - var payload = data.slice(this.header.length); - this.first = false; - - // Error out if unexpected header - if (header !== this.header) { - cb(new Error('Expected ' + this.header + ' but received ' + header), - payload); - return; - } - - // Pass along everything in the first chunk after the header - cb(null, payload); - } else { - cb(null, data); - } -}; - -module.exports = Header; diff --git a/lib/jomini.jison b/lib/jomini.jison new file mode 100644 index 0000000..1f98634 --- /dev/null +++ b/lib/jomini.jison @@ -0,0 +1,86 @@ +%{ +var toDate = require('./toDate'); +var setProp = require('./setProp'); +%} + +/* lexical grammar */ +%lex +%% + +\s+ /* skip whitespace */ +"yes"\b return 'BOOL' +"no"\b return 'BOOL' +[0-9]+"."[0-9]+"."[0-9]+\b return 'DATE' +'"'[0-9]+"."[0-9]+"."[0-9]+'"' yytext = yytext.substr(1,yyleng-2); return 'QDATE' +"-"?[0-9]+("."[0-9]+)?\b return 'NUMBER' +"{" return '{' +"}" return '}' +"=" return '=' +\"[^\"]*\" yytext = yytext.substr(1,yyleng-2); return 'QIDENTIFIER' +[a-zA-Z0-9_]+ return 'IDENTIFIER' +"#"[^\r\n]*((\r\n)|<>) /* skip comments */ +. return 'INVALID' +<> return 'EOF' + + +/lex + +%start expressions + +%% /* language grammar */ + +expressions + : PMemberList EOF + { return obj; } + | IDENTIFIER PMemberList EOF + { return obj; } + ; + +PMemberList + : PMember + {if(key) {nest.push(obj); obj = {}; setProp(obj, key, value);}} + | PMemberList PMember + {if (key) {setProp(obj, key, value);}} + ; + +PMember + : IDENTIFIER '=' PValue + {key = $1; value = $3;} + | DATE '=' PValue + {key = $1; value = $3;} + | IDENTIFIER '=' '{' '}' + {key = $1; value = {};} + | '{' '}' + {key = undefined;} + ; + +PList + : PValue + {nest.push(obj); obj = [$1];} + | PList PValue + {obj.push($2);} + ; + +PValue + : NUMBER + {$$ = +yytext;} + | BOOL + {$$ = yytext === 'yes';} + | QIDENTIFIER + {$$ = yytext;} + | IDENTIFIER + {$$ = yytext;} + | DATE + {$$ = toDate(yytext);} + | QDATE + {$$ = toDate(yytext);} + | '{' PMemberList '}' + {$$ = obj; obj = nest.pop();} + | '{' PList '}' + {$$ = obj; obj = nest.pop();} + ; + +%% + +nest = []; +obj = {}; diff --git a/lib/parse.js b/lib/parse.js deleted file mode 100644 index 3c2abe8..0000000 --- a/lib/parse.js +++ /dev/null @@ -1,10 +0,0 @@ -var Parser = require('./parser'); - -function parse(text, cb) { - var p = new Parser(); - p.end(text, 'utf8', function() { - cb(null, p.obj); - }); -} - -module.exports = parse; diff --git a/lib/parser.js b/lib/parser.js deleted file mode 100644 index aa8cbc8..0000000 --- a/lib/parser.js +++ /dev/null @@ -1,495 +0,0 @@ -var util = require('util'); -var Writable = require('stream').Writable; -var toDate = require('./toDate'); -var toBool = require('./toBool'); - -util.inherits(Parser, Writable); - -function Parser(options) { - if (!(this instanceof Parser)) { - return new Parser(options); - } - - Writable.call(this, options); - this.on('finish', function() { - this.isEnding = true; - this._parse(function() { }); - }.bind(this)); - - // The current object being population - this.obj = {}; - - // Data structure to monitor the object as we go deeper into the hierarchy. - // Once the object has been read completely, it is popped off - this.nest = []; - - // Current byte in the stream - this.current = ''; - - // Buffer used to aggregate characters across chunks - this.tok = new Buffer(256); -} - -var eq = '='.charCodeAt(0); -var rcurl = '{'.charCodeAt(0); -var lcurl = '}'.charCodeAt(0); -var hash = '#'.charCodeAt(0); -var comma = ','.charCodeAt(0); -var semicolon = ';'.charCodeAt(0); -var quote = '"'.charCodeAt(0); -var tab = '\t'.charCodeAt(0); -var space = ' '.charCodeAt(0); -var newline = '\n'.charCodeAt(0); -var carriage = '\r'.charCodeAt(0); - -// Returns whether the given byte is untyped. Untyped means it is not a -// delimiter for these types of files. Examples of untyped are alphanumeric -// characters and whitespace -Parser._untyped = function(c) { - return !(c === eq || c === rcurl || c === lcurl || c === hash || - c === comma || c === semicolon); -}; - -// Advances the stream through all whitespace and comments -Parser.prototype._trimmer = function() { - var retry = false; - do { - while (this._read() && Parser._isspace(this.current)) { - } - - retry = false; - if (Parser._untyped(this.current) && !Parser._isspace(this.current)) { - this._unpeek(); - } else if (this.current === hash) { - while (this._read() && this.current !== carriage) { - } - retry = true; - } - } while (retry); -}; - -// Returns whether the given byte is a white space character -Parser._isspace = function(c) { - return c === space || c === tab || c === newline || c === carriage; -}; - -Parser.prototype._read = function() { - if (this.eoc === true) { - return false; - } - - if (this.readFirst) { - this.current = this.prevBuf[this.bufPos++]; - if (this.bufPos === this.prevBuf.length) { - this.readFirst = false; - this.bufPos = 0; - } - - return true; - } - - if (this.bufPos < this.buf.length) { - this.current = this.buf[this.bufPos++]; - return true; - } - - this.eoc = true; - return false; -}; - -// Moves the stream backwards by one byte. If a previous buffer exists and the -// buffer position is zero, start backing up at the end of the previous buffer. -Parser.prototype._unpeek = function() { - this.eoc = false; - if (this.bufPos === 0 && this.prevBuf) { - this.bufPos = this.prevBuf.length - 1; - this.readFirst = true; - } else { - this.bufPos--; - } -}; - -// Returns the numerical value of the string if it is a number else undefined -Parser._number = function(str) { - var result = +str; - if (!isNaN(result) && str !== '') { - return result; - } -}; - -// Advances the streams returns the next identifier. The result will be -// undefined if the function didn't get enough room to extract the identifier -Parser.prototype._sliceIdentifier = function() { - if (this.eoc === true) { - return undefined; - } - - do { - this._trimmer(); - - // The while check is important so we skip empty objects with no - // identifier (I think it is a bug with Paradox, but we'll be - // accomodating) - } while ((this.current === lcurl || this.current === rcurl) && !this.eoc); - - var pos = 0; - - // Check for a quote. If we are looking at a quote, then the identifier - // stretches through all until the end quote is found. This means that the - // resulting value can contain whitespace! If we don't see a quote then we - // continue until the chunk ends or a delimiter is found (such as an equals - // or whitespace) - if (this.current === quote) { - this._read(); - while (this._read() && this.current !== quote) { - this.tok[pos++] = this.current; - } - } else { - while (this._read() && Parser._untyped(this.current) && - !Parser._isspace(this.current)) { - this.tok[pos++] = this.current; - } - - // We read too far if it is not the end of the chunk, so back up - if (this.eoc === false) { - this._unpeek(); - } - } - - if (this.eoc && !this.isEnding) { - return undefined; - } - - var result = this.tok.toString('utf8', 0, pos); - return result; -}; - -// Reads through the stream and attempts to detect a list. If a list is -// detected, the object that is being parsed changes to a list and the function -// returns true. -Parser.prototype._list = function() { - while (this._read() && - (Parser._isspace(this.current) || this.current === eq)) { - } - - if (this.current === rcurl) { - this.nest.push(this.obj); - this.obj = []; - this.realBufPos = this.bufPos; - return true; - } else { - this._unpeek(); - return undefined; - } -}; - -// Reads through the stream and attemps to detect a list. If a list is -// detected, the object that is being parsed changes to a list and the function -// returns true. If the function knows that the stream doesn't contain an -// object, it returns false. If there isn't enough data to determine, it -// returns undefined. -Parser.prototype._obj = function() { - while (this._read() && - (Parser._isspace(this.current) || this.current === eq)) { - } - - // If we hit the end of the chunk, well we don't know if we are looking at an - // object. Else if we aren't looking at a right curly then we aren't looking - // at an object - if (this.eoc === true) { - return undefined; - } else if (this.current !== rcurl) { - this._unpeek(); - return false; - } - - var pos = this.bufPos; - var foundUntyped = false; - - // Attempt to advance the stream to the next delimiter, we are looking for an - // equal - while (this._read() && - (Parser._untyped(this.current) || Parser._isspace(this.current))) { - foundUntyped |= Parser._untyped(this.current) && - !Parser._isspace(this.current); - } - - // We possibly read into the next chunk, so make a note of that as we reset - // to an earlier position so that we can re-read the identifier - this.readFirst = pos > this.bufPos; - this.bufPos = pos - 1; - - if (this.eoc === true && (this.current !== eq && this.current !== lcurl) && - Parser._untyped(this.current)) { - return undefined; - } else { - this.eoc = false; - } - - // We hit '=', so we know we are parsing an object! And that is cool and - // all that we are in an object, but make sure we rewind ourselves to the - // start of the first property. - if (this.current === eq || (this.current === lcurl && !foundUntyped)) { - this.nest.push(this.obj); - this.obj = {}; - this.bufPos++; - if (this.current === lcurl) { - this.emptyObject = true; - } - return true; - } - - return false; -}; - -// Convert the string value to the most restrictive type. Return the new value -// in its restrictive type -Parser.prototype._identify = function(value) { - var val = toBool(value); - if (val !== undefined) { - return val; - } - - val = toDate(value); - if (val) { - return val; - } - - val = Parser._number(value); - if (val !== undefined) { - return val; - } - - return value; -}; - -// Convert the array to an array of the least common demoninator types. An -// array of strings and ints will be converted to an array of strings. -Parser._lcd = function(arr) { - var i = 0; - - // If the array is an array of objects. Let's jettison. - if (arr.length > 0 && typeof arr[0] === 'object') { - return; - } - - // Is this an array of bools? - bools = arr.map(function(val) { return toBool(val); }); - if (bools.every(function(val) { return val !== undefined; })) { - for (; i < arr.length; i++) { - arr[i] = toBool(arr[i]); - } - return; - } - - // Is this an array of dates? - dates = arr.map(function(val) { return toDate(val); }); - if (dates.every(function(val) { return val !== undefined; })) { - for (; i < arr.length; i++) { - arr[i] = toDate(arr[i]); - } - return; - } - - // Is this an array of ints? - var nums = arr.map(function(val) { return Parser._number(val); }); - if (nums.every(function(val) { return val !== undefined; })) { - for (; i < arr.length; i++) { - arr[i] = Parser._number(arr[i]); - } - } -}; - -Parser.prototype._parseList = function() { - // This could be a list of objects so we first check if we are lookgin at an - // object. We may not have enough buffer space to properly evaluate - var isObj = this._obj(); - if (isObj === undefined) { - this.bufPos = this.realBufPos; - this.eoc = false; - return true; - } else if (isObj === true) { - this.nest[this.nest.length - 1].push(this.obj); - return this._parseObj(); - } - - var value = this._sliceIdentifier(); - - // The end of the list, the list was empty, or we ran out of buffer - if (value === undefined) { - if (this.current === lcurl) { - Parser._lcd(this.obj); - this.obj = this.nest.pop(); - return false; - } - return true; - } - - this.obj.push(value); - - while (this._read() && Parser._isspace(this.current)) { - } - - if (this.eoc === true && !this.isEnding) { - return true; - } - - // We probably read too far so backup by one if the current character is - // something we probably want to be looking at. - if (Parser._untyped(this.current)) { - this._unpeek(); - } - - // Convert the list to the least common denominator type and pop it from the - // list as we are done parsing the list - if (this.current === lcurl) { - Parser._lcd(this.obj); - this.obj = this.nest.pop(); - this.realBufPos = this.bufPos; - return false; - } -}; - -Parser._setProp = function(obj, identifier, value) { - if (obj.hasOwnProperty(identifier)) { - // Since the object has the key, we need to check if the value is an array - // or is single valued. If the property is already an array, push the new - // value to the end. Else the property is still single valued, then create - // a list with the two elements - if (util.isArray(obj[identifier])) { - obj[identifier].push(value); - } else { - obj[identifier] = [obj[identifier], value]; - } - } else { - // New property so we just shove it into the object - obj[identifier] = value; - } -}; - -Parser.prototype._parseObj = function() { - this._trimmer(); - - while (this.current === rcurl && !this.eoc) { - this._trimmer(); - if (this.current === lcurl) { - return false; - } - } - - if (this.current === lcurl && !this.eoc && this.nest.length > 0) { - this.obj = this.nest.pop(); - this.realBufPos = this.bufPos; - return false; - } - - var identifier = this._sliceIdentifier(); - var isObj = this._obj(); - if (isObj === undefined) { - this.bufPos = this.realBufPos; - this.eoc = false; - return true; - } else if (isObj === true) { - Parser._setProp(this.nest[this.nest.length - 1], identifier, this.obj); - if (this.emptyObject) { - this.realBufPos = this.bufPos; - this.emptyObject = false; - return false; - } - return this._parseObj(); - } - - var isList = this._list(); - if (isList) { - this.nest[this.nest.length - 1][identifier] = this.obj; - return this._parseList(); - } - - var value = this._identify(this._sliceIdentifier()); - if (identifier === undefined || value === undefined) { - this.readFirst = this.realBufPos >= this.bufPos; - this.bufPos = this.realBufPos; - this.eoc = false; - return true; - } - - this.realBufPos = this.bufPos; - Parser._setProp(this.obj, identifier, value); - this._trimmer(); - - // Another chance to chunk empty objects - while (this.current === rcurl && !this.eoc) { - this._trimmer(); - if (this.current === lcurl) { - this._read(); - } - } - - if (this.current === lcurl) { - this.obj = this.nest.pop(); - this.realBufPos = this.bufPos; - } - - return false; -}; - -Parser.prototype._parse = function(cb) { - while (this.eoc === false) { - // If the object we are adding to is an array. We keep on processing - // elements and adding it to the end of the array. Else if we are - // dealing with an object, continue processing key value pairs - var cutoffed = util.isArray(this.obj) ? this._parseList() : - this._parseObj(); - - // While parsing, we may have run out of buffer room for parsing. If so we - // invoke the call back and wait for more data. - if (cutoffed) { - cb(); - - // Because this could be the last chunk before the "finish" event - if (this.buf.prevBuf) { - this.readFirst = true; - } - return; - } else if (this.current === lcurl) { - // As long as there are left curlies lined up in the buffer, pop them off - // and finalize them - var redo = true; - while (redo) { - redo = false; - while (this._read() && Parser._isspace(this.current)) { - } - - if (this.current === lcurl && this.nest.length > 0) { - redo = true; - if (util.isArray(this.obj)) { - Parser._lcd(this.obj); - } - this.obj = this.nest.pop(); - } else if (this.current !== lcurl) { - this._unpeek(); - } - } - } - } - cb(); -}; - -Parser.prototype._write = function(chunk, enc, cb) { - // If there is something in the buffer we squirrel it away as we may need to - // reference the data in it - if (this.buf !== undefined) { - this.prevBuf = this.buf; - this.readFirst = true; - this.bufPos = Math.min(this.bufPos, this.buf.length - 1); - } else { - this.bufPos = 0; - this.realBufPos = 0; - } - - this.eoc = false; - this.buf = chunk; - this._parse(cb); -}; - -module.exports = Parser; diff --git a/lib/setProp.js b/lib/setProp.js new file mode 100644 index 0000000..bab8a56 --- /dev/null +++ b/lib/setProp.js @@ -0,0 +1,20 @@ +var util = require('util'); + +function setProp(obj, identifier, value) { + if (obj.hasOwnProperty(identifier)) { + // Since the object has the key, we need to check if the value is an array + // or is single valued. If the property is already an array, push the new + // value to the end. Else the property is still single valued, then create + // a list with the two elements + if (util.isArray(obj[identifier])) { + obj[identifier].push(value); + } else { + obj[identifier] = [obj[identifier], value]; + } + } else { + // New property so we just shove it into the object + obj[identifier] = value; + } +} + +module.exports = setProp; diff --git a/package.json b/package.json index 1cdb5ec..84de399 100644 --- a/package.json +++ b/package.json @@ -15,18 +15,17 @@ "license": "MIT", "devDependencies": { "chai": "^1.10.0", + "concat-stream": "^1.4.7", "gulp": "^3.8.10", "gulp-istanbul": "^0.5.0", + "gulp-jison": "^1.0.0", "gulp-jscs": "^1.4.0", "gulp-jshint": "^1.9.0", "gulp-mocha": "^2.0.0", "jshint-stylish": "^1.0.0", - "lodash": "^2.4.1", - "through2": "^0.6.3", + "json3": "^3.3.2", "mocha": "^2.1.0", - "zuul": "^1.17.0", - "bluebird": "^2.9.4", - "json3": "^3.3.2" + "zuul": "^1.17.0" }, "homepage": "https://github.com/nickbabcock/jomini", "keywords": [ diff --git a/test/header.js b/test/header.js deleted file mode 100644 index 91506ba..0000000 --- a/test/header.js +++ /dev/null @@ -1,51 +0,0 @@ -var Header = require('../').Header; -var stream = require('stream'); -var expect = require('chai').expect; - -describe('Header', function() { - it('detect expected header', function(done) { - var s = new stream.Readable(); - s.push('EU4txtblah'); - s.push(null); - var head = new Header({header: 'EU4txt'}); - s.pipe(head); - head.on('data', function(data) { - expect(data.toString()).to.equal('blah'); - done(); - }); - }); - - it('error on unexpected header', function(done) { - var s = new stream.Readable(); - s.push('EU4binblah'); - s.push(null); - var head = new Header({header: 'EU4txt'}); - s.pipe(head); - head.on('error', function(err) { - expect(err.message).to.equal('Expected EU4txt but received EU4bin'); - done(); - }); - }); - - it('should write through subsequent data', function(done) { - var head = new Header({header: 'EU4txt'}); - head.write('EU4txt\nblah', 'utf8', function() { - head.write('blue', 'utf8', function() { - var actual = head.read().toString(); - expect(actual).to.equal('\nblahblue'); - done(); - }); - }); - }); - - it('should compensate for the missing new', function(done) { - var head = Header({header: 'EU4txt'}); - head.write('EU4txt\nblah', 'utf8', function() { - head.write('blue', 'utf8', function() { - var actual = head.read().toString(); - expect(actual).to.equal('\nblahblue'); - done(); - }); - }); - }); -}); diff --git a/test/parse.js b/test/parse.js index e2566cf..03e4e9e 100644 --- a/test/parse.js +++ b/test/parse.js @@ -1,20 +1,286 @@ -var parse = require('../').parse; +var parse = require('../lib/jomini').parse; var expect = require('chai').expect; -var blue = require('bluebird'); describe('parse', function() { - it('should handle the simple parse case', function(done) { - parse('foo=bar', function(err, actual) { - expect(actual).to.deep.equal({'foo': 'bar'}); - done(); - }); - }); - - it('should be able to be promisified', function(done) { - var parseAsync = blue.promisify(parse); - parseAsync('foo=bar').then(function(data) { - expect(data).to.deep.equal({'foo': 'bar'}); - done(); - }); + it('should handle the simple parse case', function() { + expect(parse('foo=bar')).to.deep.equal({foo: 'bar'}); + }); + + it('should handle the simple header case', function() { + expect(parse('EU4txt\nfoo=bar')).to.deep.equal({foo: 'bar'}); + }); + + it('should handle empty quoted strings', function() { + expect(parse('foo=""')).to.deep.equal({foo: ''}); + }); + + it('should handle whitespace', function() { + expect(parse('\tfoo = bar ')).to.deep.equal({'foo':'bar'}); + }); + + it('should handle the simple quoted case', function() { + expect(parse('foo="bar"')).to.deep.equal({'foo':'bar'}); + }); + + it('should handle string list accumulation', function() { + expect(parse('foo=bar\nfoo=qux')).to.deep.equal({'foo':['bar', 'qux']}); + }); + + it('should handle string list accumulation long', function() { + expect(parse('foo=bar\nfoo=qux\nfoo=baz')).to.deep. + equal({'foo':['bar', 'qux', 'baz']}); + }); + + it('should handle quoted string list accumulation', function() { + expect(parse('foo="bar"\nfoo="qux"')).to.deep. + equal({'foo':['bar', 'qux']}); + }); + + it('should handle boolen', function() { + expect(parse('foo=yes')).to.deep.equal({'foo': true}); + }); + + it('should handle boolen list', function() { + expect(parse('foo={yes no}')).to.deep.equal({'foo': [true, false]}); + }); + + it('should handle whole numbers', function() { + expect(parse('foo=1')).to.deep.equal({'foo': 1}); + }); + + it('should handle zero', function() { + expect(parse('foo=0')).to.deep.equal({'foo': 0}); + }); + + it('should handle negative whole numbers', function() { + expect(parse('foo=-1')).to.deep.equal({'foo': -1}); + }); + + it('should handle decimal number', function() { + expect(parse('foo=1.23')).to.deep.equal({'foo': 1.23}); + }); + + it('should handle negative decimal number', function() { + expect(parse('foo=-1.23')).to.deep.equal({'foo': -1.23}); + }); + + it('should handle number list accumulation', function() { + expect(parse('foo=1\nfoo=-1.23')).to.deep.equal({'foo':[1, -1.23]}); + }); + + it('should handle dates', function() { + expect(parse('date=1821.1.1')).to.deep. + equal({'date': new Date(Date.UTC(1821, 0, 1))}); + }); + +/* it('should deceptive dates', function() { + expect(parse('date=1821.a.1')).to.deep.equal({date': '1821.a.1'}); + });*/ + + it('should handle quoted dates', function() { + expect(parse('date="1821.1.1"')).to.deep. + equal({'date': new Date(Date.UTC(1821, 0, 1))}); + }); + + it('should handle accumulated dates', function() { + expect(parse('date="1821.1.1"\ndate=1821.2.1')).to.deep.equal( + {'date':[new Date(Date.UTC(1821, 0, 1)), new Date(Date.UTC(1821, 1, 1))]} + ); + }); + + it('should handle consecutive strings', function() { + expect(parse('foo = { bar baz }')).to.deep.equal({'foo': ['bar', 'baz']}); + }); + + it('should handle consecutive strings no space', function() { + expect(parse('foo={bar baz}')).to.deep.equal({'foo': ['bar', 'baz']}); + }); + + it('should handle consecutive quoted strings', function() { + expect(parse('foo = { "bar" "baz" }')).to.deep. + equal({'foo': ['bar', 'baz']}); + }); + + it('should handle empty object', function() { + expect(parse('foo = {}')).to.deep.equal({'foo': {}}); + }); + + it('should handle space empty object', function() { + expect(parse('foo = { }')).to.deep.equal({'foo': {}}); + }); + + it('should handle the object after empty object', function() { + var obj = { + foo: {}, + catholic: { + defender: 'me' + } + }; + + expect(parse('foo={} catholic={defender="me"}')).to.deep.equal(obj); + }); + + it('should handle the object after empty object nested', function() { + var obj = { + religion: { + foo: {}, + catholic: { + defender: 'me' + } + } + }; + + expect(parse('religion={foo={} catholic={defender="me"}}')).to.deep. + equal(obj); + }); + + it('should ignore empty objects with no identifier at end', function() { + expect(parse('foo={bar=val {}} { } me=you')).to.deep. + equal({foo: {bar: 'val'}, me: 'you'}); + }); + + it('should understand a list of objects', function() { + var str = 'attachments={ { id=258579 type=4713 } ' + + ' { id=258722 type=4713 } }'; + var obj = { + attachments: [{ + id: 258579, + type: 4713 + }, { + id: 258722, + type: 4713 + }] + }; + + expect(parse(str)).to.deep.equal(obj); + }); + + it('should parse minimal spacing for objects', function() { + var str = 'nation={ship={name="ship1"} ship={name="ship2"}}'; + var obj = { + nation: { + ship: [{name: 'ship1'}, {name: 'ship2'}] + } + }; + + expect(parse(str)).to.deep.equal(obj); + }); + + it('should understand a simple EU4 header', function() { + var str = 'date=1640.7.1\r\nplayer="FRA"\r\nsavegame_version=' + + '\r\n{\r\n\tfirst=1\r\n\tsecond=9\r\n\tthird=2\r\n\tforth=0\r\n}'; + var obj = { + date: new Date(Date.UTC(1640, 6, 1)), + player: 'FRA', + savegame_version: { + first: 1, + second: 9, + third: 2, + forth: 0 + } + }; + expect(parse(str)).to.deep.equal(obj); + }); + + it('should understand EU4 gameplay settings', function() { + var str = 'gameplaysettings=\r\n{\r\n\tsetgameplayoptions=' + + '\r\n\t{\r\n\t\t1 1 2 0 1 0 0 0 1 1 1 1 \r\n\t}\r\n}'; + var obj = { + gameplaysettings: { + setgameplayoptions: [1, 1, 2, 0, 1, 0, 0, 0, 1, 1, 1, 1] + } + }; + expect(parse(str)).to.deep.equal(obj); + }); + + it('should parse multiple objects accumulated', function() { + var str = 'army=\r\n{\r\n\tname="1st army"\r\n\tunit={\r\n\t\t' + + 'name="1st unit"\r\n\t}\r\n}\r\narmy=\r\n{\r\n\tname="2nd army"' + + '\r\n\tunit={\r\n\t\tname="1st unit"\r\n\t}\r\n\tunit={\r\n\t\t' + + 'name="2nd unit"\r\n\t}\r\n}'; + + var obj = { + army: [{ + name: '1st army', + unit: { + name: '1st unit' + } + }, { + name: '2nd army', + unit: [{ + name: '1st unit' + }, { + name: '2nd unit' + }] + }] + }; + + expect(parse(str)).to.deep.equal(obj); + }); + + it('should handle back to backs', function() { + var str1 = 'POR={type=0 max_demand=2.049 t_in=49.697 t_from=\r\n' + + '{ C00=5.421 C18=44.276 } }'; + var str2 = 'SPA= { type=0 val=3.037 max_pow=1.447 max_demand=2.099 ' + + 'province_power=1.447 t_in=44.642 t_from= { C01=1.794 C17=42.848 } }'; + + var expected = { + POR: { + type: 0, + max_demand: 2.049, + t_in: 49.697, + t_from: {'C00': 5.421, 'C18': 44.276} + }, + SPA: { + type: 0, + val: 3.037, + max_pow: 1.447, + max_demand: 2.099, + province_power: 1.447, + t_in: 44.642, + t_from: {'C01': 1.794, 'C17': 42.848} + } + }; + + expect(parse(str1 + str2)).to.deep.equal(expected); + }); + + it('should handle dates as identifiers', function() { + expect(parse('1480.1.1=yes')).to.deep.equal({'1480.1.1': true}); + }); + + it('should handle consecutive numbers', function() { + expect(parse('foo = { 1 -1.23 }')).to.deep.equal({'foo': [1, -1.23]}); + }); + + it('should handle consecutive dates', function() { + expect(parse('foo = { 1821.1.1 1821.2.1 }')).to.deep.equal({'foo': + [new Date(Date.UTC(1821, 0, 1)), new Date(Date.UTC(1821, 1, 1))]}); + }); + + it('should understand comments mean skip line', function() { + expect(parse('# boo\r\n# baa\r\nfoo=a\r\n# bee')).to.deep. + equal({'foo': 'a'}); + }); + + it('should understand simple objects', function() { + expect(parse('foo={bar=val}')).to.deep.equal({'foo': {'bar': 'val'}}); + }); + + it('should understand nested list objects', function() { + expect(parse('foo={bar={val}}')).to.deep.equal({'foo': {'bar': ['val']}}); + }); + + it('should understand objects with start spaces', function() { + expect(parse('foo= { bar=val}')).to.deep.equal({'foo': {'bar': 'val'}}); + }); + + it('should understand objects with end spaces', function() { + expect(parse('foo={bar=val }')).to.deep.equal({'foo': {'bar': 'val'}}); + }); + + it('should ignore empty objects with no identifier', function() { + expect(parse('foo={bar=val} {} { } me=you')).to.deep. + equal({foo: {bar: 'val'}, me: 'you'}); }); }); diff --git a/test/toDate.js b/test/toDate.js index 77235a6..3e5a962 100644 --- a/test/toDate.js +++ b/test/toDate.js @@ -20,4 +20,8 @@ describe('toDate', function() { it('should return undefined for deceptive date', function() { expect(toDate('1800.1.a')).to.equal(undefined); }); + + it('should return the input for a false value', function() { + expect(toDate(undefined)).to.equal(undefined); + }); }); diff --git a/test/toJson.js b/test/toJson.js deleted file mode 100644 index 09204e0..0000000 --- a/test/toJson.js +++ /dev/null @@ -1,559 +0,0 @@ -var Parser = require('../').Parser; -var stream = require('stream'); -var expect = require('chai').expect; - -function conversion(text, cb) { - var s = new stream.Readable(); - s.push(text); - s.push(null); - var p = Parser(); - var res = s.pipe(p); - p.on('finish', function() { - cb(p.obj); - }); -} - -function parse(input, expected, cb) { - conversion(input, function(actual) { - expect(actual).to.deep.equal(expected); - cb(); - }); -} - -describe('toJson', function() { - it('should handle the simple case', function(done) { - parse('foo=bar', {'foo':'bar'}, done); - }); - - it('should handle empty quoted strings', function(done) { - parse('foo=""', {'foo': ''}, done); - }); - - it('should handle whitespace', function(done) { - parse('\tfoo = bar ', {'foo':'bar'}, done); - }); - - it('should handle the simple quoted case', function(done) { - parse('foo="bar"', {'foo':'bar'}, done); - }); - - it('should handle string list accumulation', function(done) { - parse('foo=bar\nfoo=qux', {'foo':['bar', 'qux']}, done); - }); - - it('should handle string list accumulation long', function(done) { - parse('foo=bar\nfoo=qux\nfoo=baz', {'foo':['bar', 'qux', 'baz']}, done); - }); - - it('should handle quoted string list accumulation', function(done) { - parse('foo="bar"\nfoo="qux"', {'foo':['bar', 'qux']}, done); - }); - - it('should handle boolen', function(done) { - parse('foo=yes', {'foo': true}, done); - }); - - it('should handle boolen list', function(done) { - parse('foo={yes no}', {'foo': [true, false]}, done); - }); - - it('should handle whole numbers', function(done) { - parse('foo=1', {'foo': 1}, done); - }); - - it('should handle zero', function(done) { - parse('foo=0', {'foo': 0}, done); - }); - - it('should handle negative whole numbers', function(done) { - parse('foo=-1', {'foo': -1}, done); - }); - - it('should handle decimal number', function(done) { - parse('foo=1.23', {'foo': 1.23}, done); - }); - - it('should handle negative decimal number', function(done) { - parse('foo=-1.23', {'foo': -1.23}, done); - }); - - it('should handle number list accumulation', function(done) { - parse('foo=1\nfoo=-1.23', {'foo':[1, -1.23]}, done); - }); - - it('should handle dates', function(done) { - parse('date=1821.1.1', {'date': new Date(Date.UTC(1821, 0, 1))}, done); - }); - - it('should deceptive dates', function(done) { - parse('date=1821.a.1', {'date': '1821.a.1'}, done); - }); - - it('should handle quoted dates', function(done) { - parse('date="1821.1.1"', {'date': new Date(Date.UTC(1821, 0, 1))}, done); - }); - - it('should handle accumulated dates', function(done) { - parse('date="1821.1.1"\ndate=1821.2.1', - {'date':[new Date(Date.UTC(1821, 0, 1)), new Date(Date.UTC(1821, 1, 1))]}, - done); - }); - - it('should handle consecutive strings', function(done) { - parse('foo = { bar baz }', {'foo': ['bar', 'baz']}, done); - }); - - it('should handle consecutive strings no space', function(done) { - parse('foo={bar baz}', {'foo': ['bar', 'baz']}, done); - }); - - it('should handle consecutive quoted strings', function(done) { - parse('foo = { "bar" "baz" }', {'foo': ['bar', 'baz']}, done); - }); - - it('should handle empty object', function(done) { - parse('foo = {}', {'foo': {}}, done); - }); - - it('should handle space empty object', function(done) { - parse('foo = { }', {'foo': {}}, done); - }); - - it('should handle the object after empty object', function(done) { - var obj = { - foo: {}, - catholic: { - defender: 'me' - } - }; - - parse('foo={} catholic={defender="me"}', obj, done); - }); - - it('should handle the object after empty object nested', function(done) { - var obj = { - religion: { - foo: {}, - catholic: { - defender: 'me' - } - } - }; - - parse('religion={foo={} catholic={defender="me"}}', obj, done); - }); - - it('should handle consecutive numbers', function(done) { - parse('foo = { 1 -1.23 }', {'foo': [1, -1.23]}, done); - }); - - it('should handle consecutive dates', function(done) { - parse('foo = { 1821.1.1 1821.2.1 }', {'foo': - [new Date(Date.UTC(1821, 0, 1)), new Date(Date.UTC(1821, 1, 1))]}, done); - }); - - it('should make least common demoninator list', function(done) { - parse('foo = { 1 a 1821.1.1 }', {'foo': ['1', 'a', '1821.1.1']}, done); - }); - - it('should understand comments mean skip line', function(done) { - parse('# boo\r\n# baa\r\nfoo=a\r\n# bee', {'foo': 'a'}, done); - }); - - it('should understand simple objects', function(done) { - parse('foo={bar=val}', {'foo': {'bar': 'val'}}, done); - }); - - it('should understand nested list objects', function(done) { - parse('foo={bar={val}}', {'foo': {'bar': ['val']}}, done); - }); - - it('should understand objects with start spaces', function(done) { - parse('foo= { bar=val}', {'foo': {'bar': 'val'}}, done); - }); - - it('should understand objects with end spaces', function(done) { - parse('foo={bar=val }', {'foo': {'bar': 'val'}}, done); - }); - - it('should ignore empty objects with no identifier', function(done) { - parse('foo={bar=val} {} { } me=you', {foo: {bar: 'val'}, me: 'you'}, done); - }); - - it('should ignore empty objects with no identifier at end', function(done) { - parse('foo={bar=val {}} { } me=you', {foo: {bar: 'val'}, me: 'you'}, done); - }); - - it('should understand a list of objects', function(done) { - var str = 'attachments={ { id=258579 type=4713 } ' + - ' { id=258722 type=4713 } }'; - var obj = { - attachments: [{ - id: 258579, - type: 4713 - }, { - id: 258722, - type: 4713 - }] - }; - - parse(str, obj, done); - }); - - it('should parse minimal spacing for objects', function(done) { - var str = 'nation{ship={name="ship1"} ship={name="ship2"}}'; - var obj = { - nation: { - ship: [{name: 'ship1'}, {name: 'ship2'}] - } - }; - - parse(str, obj, done); - }); - - it('should understand a simple EU4 header', function(done) { - var str = 'date=1640.7.1\r\nplayer="FRA"\r\nsavegame_version=' + - '\r\n{\r\n\tfirst=1\r\n\tsecond=9\r\n\tthird=2\r\n\tforth=0\r\n}'; - var obj = { - date: new Date(Date.UTC(1640, 6, 1)), - player: 'FRA', - savegame_version: { - first: 1, - second: 9, - third: 2, - forth: 0 - } - }; - parse(str, obj, done); - }); - - it('should understand EU4 gameplay settings', function(done) { - var str = 'gameplaysettings=\r\n{\r\n\tsetgameplayoptions=' + - '\r\n\t{\r\n\t\t1 1 2 0 1 0 0 0 1 1 1 1 \r\n\t}\r\n}'; - var obj = { - gameplaysettings: { - setgameplayoptions: [1, 1, 2, 0, 1, 0, 0, 0, 1, 1, 1, 1] - } - }; - parse(str, obj, done); - }); - - it('should handle a constructor without new', function(done) { - var p = Parser(); - p.write('foo=bar\n', 'utf8', function() { - p.end(); - }); - - p.on('finish', function() { - expect(p.obj).to.deep.equal({'foo': 'bar'}); - done(); - }); - }); - - it('should handle string list accumulation chunky norm', function(done) { - var p = new Parser(); - p.write('foo=bar\n', 'utf8', function() { - p.write('foo=qux', 'utf8', function() { - p.end(); - }); - }); - - p.on('finish', function() { - expect(p.obj).to.deep.equal({'foo': ['bar', 'qux']}); - done(); - }); - }); - - it('should handle string list accumulate chunky break', function(done) { - var p = new Parser(); - p.write('foo=bar\nf', 'utf8', function() { - p.write('oo=qux', 'utf8', function() { - p.end(); - }); - }); - - p.on('finish', function() { - expect(p.obj).to.deep.equal({'foo': ['bar', 'qux']}); - done(); - }); - }); - - it('should handle string list accumulate chunky space break', function(done) { - var p = new Parser(); - p.write('foo = bar \n f', 'utf8', function() { - p.write('oo = qux', 'utf8', function() { - p.end(); - }); - }); - - p.on('finish', function() { - expect(p.obj).to.deep.equal({'foo': ['bar', 'qux']}); - done(); - }); - }); - - it('should handle string list accumulate chunky equal', function(done) { - var p = new Parser(); - p.write('foo=bar\nfoo=', 'utf8', function() { - p.write('qux', 'utf8', function() { - p.end(); - }); - }); - - p.on('finish', function() { - expect(p.obj).to.deep.equal({'foo': ['bar', 'qux']}); - done(); - }); - }); - - it('should handle string list accumulate chunky value', function(done) { - var p = new Parser(); - p.write('foo=bar\nfoo=qu', 'utf8', function() { - p.write('x', 'utf8', function() { - p.end(); - }); - }); - - p.on('finish', function() { - expect(p.obj).to.deep.equal({'foo': ['bar', 'qux']}); - done(); - }); - }); - - it('should handle a chunky list', function(done) { - var p = new Parser(); - p.write('foo= {1 1', 'utf8', function() { - p.write('1 2}', 'utf8', function() { - p.end(); - }); - }); - - p.on('finish', function() { - expect(p.obj).to.deep.equal({'foo': [1, 11, 2]}); - done(); - }); - }); - - it('should handle back to backs', function(done) { - var p = new Parser(); - var str1 = 'POR={type=0 max_demand=2.049 t_in=49.697 t_from=\r\n' + - '{ C00=5.421 C18=44.276 } }'; - var str2 = 'SPA= { type=0 val=3.037 max_pow=1.447 max_demand=2.099 ' + - 'province_power=1.447 t_in=44.642 t_from= { C01=1.794 C17=42.848 } }'; - - p.write(str1, 'utf8', function() { - p.write(str2, 'utf8', function() { - p.end(); - }); - }); - - var expected = { - POR: { - type: 0, - max_demand: 2.049, - t_in: 49.697, - t_from: {'C00': 5.421, 'C18': 44.276} - }, - SPA: { - type: 0, - val: 3.037, - max_pow: 1.447, - max_demand: 2.099, - province_power: 1.447, - t_in: 44.642, - t_from: {'C01': 1.794, 'C17': 42.848} - } - }; - - p.on('finish', function() { - expect(p.obj).to.deep.equal(expected); - done(); - }); - }); - - it('should handle empty objects on buffer edge', function(done) { - var p = new Parser(); - var str1 = 'foo={bar=val} {'; - var str2 = '} me=you'; - - p.write(str1, 'utf8', function() { - p.write(str2, 'utf8', function() { - p.end(); - }); - }); - - var expected = { - foo: {bar: 'val'}, - me: 'you' - }; - - p.on('finish', function() { - expect(p.obj).to.deep.equal(expected); - done(); - }); - }); - - it('should handle empty objects on buffer edge2', function(done) { - var p = new Parser(); - var str1 = 'foo={bar=val {'; - var str2 = '}} me=you'; - - p.write(str1, 'utf8', function() { - p.write(str2, 'utf8', function() { - p.end(); - }); - }); - - var expected = { - foo: {bar: 'val'}, - me: 'you' - }; - - p.on('finish', function() { - expect(p.obj).to.deep.equal(expected); - done(); - }); - }); - - it('should understand a list of objects chunky', function(done) { - var p = new Parser(); - var str1 = 'attachments={ { i'; - var str2 = 'd=258579 type=4713 } { id=258722 type=4713 } }'; - - p.write(str1, 'utf8', function() { - p.write(str2, 'utf8', function() { - p.end(); - }); - }); - - var obj = { - attachments: [{ - id: 258579, - type: 4713 - }, { - id: 258722, - type: 4713 - }] - }; - - p.on('finish', function() { - expect(p.obj).to.deep.equal(obj); - done(); - }); - }); - - it('should understand new object chunky', function(done) { - var p = new Parser(); - var str1 = 'id={active=no} reb'; - var str2 = 'el_faction={id=1}'; - - p.write(str1, 'utf8', function() { - p.write(str2, 'utf8', function() { - p.end(); - }); - }); - - var obj = { - id: { - active: false - }, - rebel_faction: { - id: 1 - } - }; - - p.on('finish', function() { - expect(p.obj).to.deep.equal(obj); - done(); - }); - }); - - it('should handle a chunky start of an object', function(done) { - var p = new Parser(); - - var str1 = 'nation{ship={name="ship1"} ship='; - var str2 = '{name="ship2"}}'; - p.write(str1, 'utf8', function() { - p.write(str2, 'utf8', function() { - p.end(); - }); - }); - - var obj = { - nation: { - ship: [{name: 'ship1'}, {name: 'ship2'}] - } - }; - - p.on('finish', function() { - expect(p.obj).to.deep.equal(obj); - done(); - }); - }); - - it('should understand new list chunky', function(done) { - var p = new Parser(); - var str1 = 'id={1 2 3} reb'; - var str2 = 'el_faction={id=5}'; - - p.write(str1, 'utf8', function() { - p.write(str2, 'utf8', function() { - p.end(); - }); - }); - - var obj = { - id: [1, 2, 3], - rebel_faction: { - id: 5 - } - }; - - p.on('finish', function() { - expect(p.obj).to.deep.equal(obj); - done(); - }); - }); - - it('should parse a chunky object on space', function(done) { - var p = new Parser(); - var str1 = 'foo={\r\nbar={qux=bax '; - var str2 = '}}'; - var obj = {foo: {bar: {qux: 'bax'}}}; - p.write(str1, 'utf8', function() { - p.write(str2, 'utf8', function() { - p.end(); - }); - }); - - p.on('finish', function() { - expect(p.obj).to.deep.equal(obj); - done(); - }); - }); - - it('should parse multiple objects accumulated', function(done) { - var str = 'army=\r\n{\r\n\tname="1st army"\r\n\tunit={\r\n\t\t' + - 'name="1st unit"\r\n\t}\r\n}\r\narmy=\r\n{\r\n\tname="2nd army"' + - '\r\n\tunit={\r\n\t\tname="1st unit"\r\n\t}\r\n\tunit={\r\n\t\t' + - 'name="2nd unit"\r\n\t}\r\n}'; - - var obj = { - army: [{ - name: '1st army', - unit: { - name: '1st unit' - } - }, { - name: '2nd army', - unit: [{ - name: '1st unit' - }, { - name: '2nd unit' - }] - }] - }; - - parse(str, obj, done); - }); -});