From 01536483be6c7dfcedd3588c1b96eeb92a56d0b3 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Fri, 6 Apr 2018 19:13:36 +0700 Subject: [PATCH 1/5] Add "sourcemap" support --- src/defaults.coffee | 3 +++ src/parser.coffee | 34 ++++++++++++++++++++++++++++++++++ test/parser.test.coffee | 29 +++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/src/defaults.coffee b/src/defaults.coffee index a9bd214b..012d22c9 100644 --- a/src/defaults.coffee +++ b/src/defaults.coffee @@ -70,4 +70,7 @@ exports.defaults = { chunkSize: 10000 emptyTag: '' cdata: false + sourcemap: false + sourcemapkey: '$source' + sourcemapEnumerable: false } diff --git a/src/parser.coffee b/src/parser.coffee index 8a375197..54c4df96 100644 --- a/src/parser.coffee +++ b/src/parser.coffee @@ -61,6 +61,35 @@ class exports.Parser extends events.EventEmitter obj[key] = [obj[key]] if not (obj[key] instanceof Array) obj[key].push newValue + storeSourcemapStart: (obj, value) => + if @options.sourcemap + Object.defineProperty obj, @options.sourcemapkey, + value: + start: + line: value.line + column: value.column + position: value.position + enumerable: true + configurable: true + + storeSourcemapEnd: (obj, value) => + if @options.sourcemap + currentValue = obj[@options.sourcemapkey] + Object.defineProperty obj, @options.sourcemapkey, + value: + start: currentValue.start + end: + line: value.line + column: value.column + position: value.position + + setNonEnumerableSourcemap: (obj) => + if obj instanceof Object + if @options.sourcemapkey of obj + Object.defineProperty obj, @options.sourcemapkey, enumerable: false + for own key of obj + @setNonEnumerableSourcemap obj[key] + reset: => # remove all previous listeners for events, to prevent event listener # accumulation @@ -104,6 +133,8 @@ class exports.Parser extends events.EventEmitter @saxParser.onopentag = (node) => obj = {} obj[charkey] = "" + @storeSourcemapStart obj, @saxParser + unless @options.ignoreAttrs for own key of node.attributes if attrkey not of obj and not @options.mergeAttrs @@ -125,6 +156,7 @@ class exports.Parser extends events.EventEmitter obj = stack.pop() nodeName = obj["#name"] delete obj["#name"] if not @options.explicitChildren or not @options.preserveChildrenOrder + @storeSourcemapEnd obj, @saxParser if obj.cdata == true cdata = obj.cdata @@ -199,6 +231,8 @@ class exports.Parser extends events.EventEmitter obj[nodeName] = old @resultObject = obj + if @options.sourcemap and not @options.sourcemapEnumerable + @setNonEnumerableSourcemap @resultObject # parsing has ended, mark that so we won't throw exceptions from # here anymore @saxParser.ended = true diff --git a/test/parser.test.coffee b/test/parser.test.coffee index 6d531d56..46b9b7de 100644 --- a/test/parser.test.coffee +++ b/test/parser.test.coffee @@ -574,3 +574,32 @@ module.exports = console.log 'Result object: ' + util.inspect r, false, 10 equ r.hasOwnProperty('SAMP'), true equ r.SAMP.hasOwnProperty('TAGN'), true) + + 'test source map with defaults': (test) -> + xml = "\n hello\n \n" + xml2js.parseString xml, {sourcemap: true}, (err, parsed) -> + # sourcemap doesn't appear, but can be accessed anyway + equ JSON.stringify(parsed).indexOf("$source") < 0, true + equ parsed.a.$source.start.line, 0 + equ parsed.a.$source.end.line, 3 + equ parsed.a.b[0].$source.start.line, 1 + equ parsed.a.b[0].$source.end.line, 1 + equ parsed.a.c[0].$source.start.line, 2 + equ parsed.a.c[0].$source.end.line, 2 + equ parsed.a.c[0].d[0].$source.start.line, 2 + test.finish() + + 'test source map with sourcemapEnumerable': (test) -> + xml = "\n hello\n \n" + xml2js.parseString xml, {sourcemap: true, sourcemapEnumerable: true}, (err, parsed) -> + # sourcemap is public + equ JSON.stringify(parsed).indexOf("$source") > 0, true + equ parsed.a.$source.start.line, 0 + equ parsed.a.$source.end.line, 3 + equ parsed.a.b[0].$source.start.line, 1 + equ parsed.a.b[0].$source.end.line, 1 + equ parsed.a.c[0].$source.start.line, 2 + equ parsed.a.c[0].$source.end.line, 2 + equ parsed.a.c[0].d[0].$source.start.line, 2 + test.finish() + From 052aa4c6d6ebdba4a1035c15089f02abfb8c41c3 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Fri, 6 Apr 2018 19:14:15 +0700 Subject: [PATCH 2/5] Update built javascript --- lib/defaults.js | 5 ++++- lib/parser.js | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/lib/defaults.js b/lib/defaults.js index 0a21da0a..6234ce0e 100644 --- a/lib/defaults.js +++ b/lib/defaults.js @@ -65,7 +65,10 @@ headless: false, chunkSize: 10000, emptyTag: '', - cdata: false + cdata: false, + sourcemap: false, + sourcemapkey: '$source', + sourcemapEnumerable: false } }; diff --git a/lib/parser.js b/lib/parser.js index 9e8261eb..24466d30 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -37,6 +37,9 @@ function Parser(opts) { this.parseString = bind(this.parseString, this); this.reset = bind(this.reset, this); + this.setNonEnumerableSourcemap = bind(this.setNonEnumerableSourcemap, this); + this.storeSourcemapEnd = bind(this.storeSourcemapEnd, this); + this.storeSourcemapStart = bind(this.storeSourcemapStart, this); this.assignOrPush = bind(this.assignOrPush, this); this.processAsync = bind(this.processAsync, this); var key, ref, value; @@ -105,6 +108,56 @@ } }; + Parser.prototype.storeSourcemapStart = function(obj, value) { + if (this.options.sourcemap) { + return Object.defineProperty(obj, this.options.sourcemapkey, { + value: { + start: { + line: value.line, + column: value.column, + position: value.position + } + }, + enumerable: true, + configurable: true + }); + } + }; + + Parser.prototype.storeSourcemapEnd = function(obj, value) { + var currentValue; + if (this.options.sourcemap) { + currentValue = obj[this.options.sourcemapkey]; + return Object.defineProperty(obj, this.options.sourcemapkey, { + value: { + start: currentValue.start, + end: { + line: value.line, + column: value.column, + position: value.position + } + } + }); + } + }; + + Parser.prototype.setNonEnumerableSourcemap = function(obj) { + var key, results; + if (obj instanceof Object) { + if (this.options.sourcemapkey in obj) { + Object.defineProperty(obj, this.options.sourcemapkey, { + enumerable: false + }); + } + results = []; + for (key in obj) { + if (!hasProp.call(obj, key)) continue; + results.push(this.setNonEnumerableSourcemap(obj[key])); + } + return results; + } + }; + Parser.prototype.reset = function() { var attrkey, charkey, ontext, stack; this.removeAllListeners(); @@ -142,6 +195,7 @@ var key, newValue, obj, processedKey, ref; obj = {}; obj[charkey] = ""; + _this.storeSourcemapStart(obj, _this.saxParser); if (!_this.options.ignoreAttrs) { ref = node.attributes; for (key in ref) { @@ -176,6 +230,7 @@ if (!_this.options.explicitChildren || !_this.options.preserveChildrenOrder) { delete obj["#name"]; } + _this.storeSourcemapEnd(obj, _this.saxParser); if (obj.cdata === true) { cdata = obj.cdata; delete obj.cdata; @@ -257,6 +312,9 @@ obj[nodeName] = old; } _this.resultObject = obj; + if (_this.options.sourcemap && !_this.options.sourcemapEnumerable) { + _this.setNonEnumerableSourcemap(_this.resultObject); + } _this.saxParser.ended = true; return _this.emit("end", _this.resultObject); } From c8a9ed8c8ea148695d85cc742fa7f01a454440d3 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Fri, 6 Apr 2018 20:00:04 +0700 Subject: [PATCH 3/5] Update documentation related to sourcemap --- README.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 54 insertions(+) diff --git a/README.md b/README.md index be7b83f4..b94f422e 100644 --- a/README.md +++ b/README.md @@ -325,6 +325,59 @@ value})``. Possible options are: } ``` Added in 0.4.6 + * `sourcemap` (default: `false`): Allows exporting node positions in the XML + source document. By default it adds to all nodes a non-enumerable property + called `$source` (doesn't show up during enumeration of the properties on the objects). + The name of the property and whether or not you want it enumerable can be + configured with the options below. + * `sourcemapkey` (default: `$source`): Change the key to use when exporting sourcemaps. + * `sourcemapEnumerable` (default: `false`): Make the sourcemap enumerable (visible). + Note that converting back the JSON to XML will include `$source` attributes + if you make them enumerable. Here's a small example: + ```javascript + xml = "\n hello\n"; + xml2js.parseString(xml, {sourcemap: true, sourcemapEnumerable: true}, (err, parsed) => { + console.log(JSON.stringify(parsed)); + }); + ``` + This will output: + ```json + { + "a": { + "$source": { + "start": { + "line": 0, + "column": 3, + "position": 3 + }, + "end": { + "line": 2, + "column": 4, + "position": 23 + } + }, + "b": [ + { + "_": "hello", + "$source": { + "start": { + "line": 1, + "column": 5, + "position": 9 + }, + "end": { + "line": 1, + "column": 14, + "position": 18 + } + } + } + ] + } + } + ``` + If `sourcemapEnumerable` was not set, `$source` wouldn't appear in the generated + JSON but would be accessible the same way. Options for the `Builder` class ------------------------------- diff --git a/package.json b/package.json index 590d4cee..51746f53 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "David Wood (http://codesleuth.co.uk/)", "Nicolas Maquet (https://github.com/nmaquet)", "Lovell Fuller (http://lovell.info/)", + "Jean-Christophe Hoelt (http://github.com/j3k0)", "d3adc0d3 (https://github.com/d3adc0d3)" ], "main": "./lib/xml2js", From 21206086c4cbc0f2d296aacb6cc0036dc3055723 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Sat, 7 Apr 2018 16:39:06 +0700 Subject: [PATCH 4/5] Use $source to restore the original XML nodes order Quite a refactoring the Builder class. It is now a 2 steps process: 1. Preprocessing Convert nodes to a unified format. In this format, children are extracted as an array, then sorted by their position in the original XML document (if made available using the `sourcemap` option). 2. XML building The code here has been simplified as special cases are handled at processing stage. It build the XML. --- lib/builder.js | 116 +++++++++++++++++++++++++++------------ src/builder.coffee | 85 ++++++++++++++++++---------- test/builder.test.coffee | 19 +++++++ 3 files changed, 156 insertions(+), 64 deletions(-) diff --git a/lib/builder.js b/lib/builder.js index 58f36384..37a76ae6 100644 --- a/lib/builder.js +++ b/lib/builder.js @@ -1,13 +1,16 @@ // Generated by CoffeeScript 1.12.7 (function() { "use strict"; - var builder, defaults, escapeCDATA, requiresCDATA, wrapCDATA, + var CHILDREN_KEY, builder, defaults, escapeCDATA, requiresCDATA, wrapCDATA, + bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; }, hasProp = {}.hasOwnProperty; builder = require('xmlbuilder'); defaults = require('./defaults').defaults; + CHILDREN_KEY = '$$children_da914993d9904559be754444a4685d08$$'; + requiresCDATA = function(entry) { return typeof entry === "string" && (entry.indexOf('&') >= 0 || entry.indexOf('>') >= 0 || entry.indexOf('<') >= 0); }; @@ -22,6 +25,7 @@ exports.Builder = (function() { function Builder(opts) { + this.preprocess = bind(this.preprocess, this); var key, ref, value; this.options = {}; ref = defaults["0.2"]; @@ -38,7 +42,7 @@ } Builder.prototype.buildObject = function(rootObj) { - var attrkey, charkey, render, rootElement, rootName; + var attrkey, charkey, entry, i, key, len, obj, render, rootArray, rootElement, rootName; attrkey = this.options.attrkey; charkey = this.options.charkey; if ((Object.keys(rootObj).length === 1) && (this.options.rootName === defaults['0.2'].rootName)) { @@ -49,22 +53,13 @@ } render = (function(_this) { return function(element, obj) { - var attr, child, entry, index, key, value; + var attr, child, childObj, i, key, len, value; if (typeof obj !== 'object') { if (_this.options.cdata && requiresCDATA(obj)) { element.raw(wrapCDATA(obj)); } else { element.txt(obj); } - } else if (Array.isArray(obj)) { - for (index in obj) { - if (!hasProp.call(obj, index)) continue; - child = obj[index]; - for (key in child) { - entry = child[key]; - element = render(element.ele(key), entry).up(); - } - } } else { for (key in obj) { if (!hasProp.call(obj, key)) continue; @@ -82,31 +77,13 @@ } else { element = element.txt(child); } - } else if (Array.isArray(child)) { - for (index in child) { - if (!hasProp.call(child, index)) continue; - entry = child[index]; - if (typeof entry === 'string') { - if (_this.options.cdata && requiresCDATA(entry)) { - element = element.ele(key).raw(wrapCDATA(entry)).up(); - } else { - element = element.ele(key, entry).up(); - } - } else { - element = render(element.ele(key), entry).up(); - } + } else if (key === CHILDREN_KEY) { + for (i = 0, len = child.length; i < len; i++) { + childObj = child[i]; + element = render(element.ele(childObj.name), childObj.node).up(); } - } else if (typeof child === "object") { - element = render(element.ele(key), child).up(); } else { - if (typeof child === 'string' && _this.options.cdata && requiresCDATA(child)) { - element = element.ele(key).raw(wrapCDATA(child)).up(); - } else { - if (child == null) { - child = ''; - } - element = element.ele(key, child.toString()).up(); - } + throw new Error('Invalid object given to xml2js builder'); } } } @@ -117,9 +94,78 @@ headless: this.options.headless, allowSurrogateChars: this.options.allowSurrogateChars }); + if (Array.isArray(rootObj)) { + rootArray = rootObj; + rootObj = {}; + for (i = 0, len = rootArray.length; i < len; i++) { + obj = rootArray[i]; + for (key in obj) { + entry = obj[key]; + if (rootObj[key]) { + rootObj[key].push(entry); + } else { + rootObj[key] = [entry]; + } + } + } + } + rootObj = this.preprocess(rootObj); return render(rootElement, rootObj).end(this.options.renderOpts); }; + Builder.prototype.preprocess = function(obj) { + var child, children, i, key, len, ref, ref1, ref2, ref3, ret, sourcePosition, value; + if (typeof obj !== 'object') { + return obj || ''; + } + if (Array.isArray(obj)) { + return obj.map(this.preprocess); + } + ret = {}; + children = []; + sourcePosition = 0; + for (key in obj) { + if (!hasProp.call(obj, key)) continue; + value = obj[key]; + if (key === this.options.attrkey || key === this.options.charkey) { + ret[key] = value; + } else { + if (Array.isArray(value)) { + for (i = 0, len = value.length; i < len; i++) { + child = value[i]; + sourcePosition = (child != null ? (ref = child.$source) != null ? (ref1 = ref.start) != null ? ref1.position : void 0 : void 0 : void 0) || (sourcePosition + 1); + children.push({ + name: key, + sourcePosition: sourcePosition, + node: this.preprocess(child) + }); + } + } else if (typeof value === 'object') { + sourcePosition = (value != null ? (ref2 = value.$source) != null ? (ref3 = ref2.start) != null ? ref3.position : void 0 : void 0 : void 0) || (sourcePosition + 1); + children.push({ + name: key, + sourcePosition: sourcePosition, + node: this.preprocess(value) + }); + } else { + sourcePosition = sourcePosition + 1; + children.push({ + name: key, + sourcePosition: sourcePosition, + node: value || '' + }); + } + } + } + if (children.length > 0) { + children.sort(function(a, b) { + return a.sourcePosition - b.sourcePosition; + }); + ret[CHILDREN_KEY] = children; + } + return ret; + }; + return Builder; })(); diff --git a/src/builder.coffee b/src/builder.coffee index 5653fde0..8e044267 100644 --- a/src/builder.coffee +++ b/src/builder.coffee @@ -3,6 +3,8 @@ builder = require 'xmlbuilder' defaults = require('./defaults').defaults +CHILDREN_KEY = '$$children_da914993d9904559be754444a4685d08$$' + requiresCDATA = (entry) -> return typeof entry is "string" && (entry.indexOf('&') >= 0 || entry.indexOf('>') >= 0 || entry.indexOf('<') >= 0) @@ -48,11 +50,6 @@ class exports.Builder element.raw wrapCDATA obj else element.txt obj - else if Array.isArray obj - # fix issue #119 - for own index, child of obj - for key, entry of child - element = render(element.ele(key), entry).up() else for own key, child of obj # Case #1 Attribute @@ -61,37 +58,18 @@ class exports.Builder # Inserts tag attributes for attr, value of child element = element.att(attr, value) - # Case #2 Char data (CDATA, etc.) else if key is charkey if @options.cdata && requiresCDATA child element = element.raw wrapCDATA child else element = element.txt child - - # Case #3 Array data - else if Array.isArray child - for own index, entry of child - if typeof entry is 'string' - if @options.cdata && requiresCDATA entry - element = element.ele(key).raw(wrapCDATA entry).up() - else - element = element.ele(key, entry).up() - else - element = render(element.ele(key), entry).up() - - # Case #4 Objects - else if typeof child is "object" - element = render(element.ele(key), child).up() - - # Case #5 String and remaining types + # Case #3 Preprocessed node + else if key is CHILDREN_KEY + for childObj in child + element = render(element.ele(childObj.name), childObj.node).up() else - if typeof child is 'string' && @options.cdata && requiresCDATA child - element = element.ele(key).raw(wrapCDATA child).up() - else - if not child? - child = '' - element = element.ele(key, child.toString()).up() + throw new Error 'Invalid object given to xml2js builder' element @@ -99,4 +77,53 @@ class exports.Builder headless: @options.headless allowSurrogateChars: @options.allowSurrogateChars) + # fix issue #119 + if Array.isArray rootObj + rootArray = rootObj + rootObj = {} + for obj in rootArray + for key, entry of obj + if rootObj[key] + rootObj[key].push entry + else + rootObj[key] = [entry] + + rootObj = @preprocess rootObj render(rootElement, rootObj).end(@options.renderOpts) + + preprocess: (obj) => + if typeof obj != 'object' + return obj || '' + if Array.isArray obj + return obj.map @preprocess + ret = {} + children = [] + sourcePosition = 0 + for own key, value of obj + if key is @options.attrkey or key is @options.charkey + ret[key] = value + else + if Array.isArray value + for child in value + sourcePosition = (child?.$source?.start?.position) || (sourcePosition + 1) + children.push + name: key + sourcePosition: sourcePosition + node: @preprocess child + else if typeof value == 'object' + sourcePosition = (value?.$source?.start?.position) || (sourcePosition + 1) + children.push + name: key + sourcePosition: sourcePosition + node: @preprocess value + else + sourcePosition = sourcePosition + 1 + children.push + name: key + sourcePosition: sourcePosition + node: value || '' + + if children.length > 0 + children.sort (a, b) -> a.sourcePosition - b.sourcePosition + ret[CHILDREN_KEY] = children + ret diff --git a/test/builder.test.coffee b/test/builder.test.coffee index e9c12be3..3df6a2de 100644 --- a/test/builder.test.coffee +++ b/test/builder.test.coffee @@ -17,6 +17,25 @@ diffeq = (expected, actual) -> throw patch unless patch is diffless module.exports = + 'test restores children order when $source is available': (test) -> + expected = '123' + obj = {"xml":{ + "A":[{"_":"1"}, {"_": "3"}], + "B":[{"_": "2"}] + }} + definePosition = (node, value) -> + Object.defineProperty(node, '$source', { + value: {start: position: value} + enuerable: false + }) + definePosition obj.xml.A[0], 64 + definePosition obj.xml.A[1], 79 + definePosition obj.xml.B[0], 71 + builder = new xml2js.Builder renderOpts: pretty: false + actual = builder.buildObject obj + diffeq expected, actual + test.finish() + 'test building basic XML structure': (test) -> expected = '' obj = {"xml":{"Label":[""],"MsgId":["5850440872586764820"]}} From 96d459c05d22d0041a58b1d60e5d5530e9a23142 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Hoelt Date: Sat, 7 Apr 2018 20:21:43 +0700 Subject: [PATCH 5/5] Make sure sourcemap isn't exported in XML --- lib/builder.js | 2 ++ src/builder.coffee | 2 ++ test/builder.test.coffee | 55 ++++++++++++++++++++++++++-------------- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/lib/builder.js b/lib/builder.js index 37a76ae6..47961795 100644 --- a/lib/builder.js +++ b/lib/builder.js @@ -129,6 +129,8 @@ value = obj[key]; if (key === this.options.attrkey || key === this.options.charkey) { ret[key] = value; + } else if (key === this.options.sourcemapkey) { + continue; } else { if (Array.isArray(value)) { for (i = 0, len = value.length; i < len; i++) { diff --git a/src/builder.coffee b/src/builder.coffee index 8e044267..dd67453c 100644 --- a/src/builder.coffee +++ b/src/builder.coffee @@ -102,6 +102,8 @@ class exports.Builder for own key, value of obj if key is @options.attrkey or key is @options.charkey ret[key] = value + else if key is @options.sourcemapkey + continue # do not export metadata to the final XML else if Array.isArray value for child in value diff --git a/test/builder.test.coffee b/test/builder.test.coffee index 3df6a2de..0e019d49 100644 --- a/test/builder.test.coffee +++ b/test/builder.test.coffee @@ -17,25 +17,6 @@ diffeq = (expected, actual) -> throw patch unless patch is diffless module.exports = - 'test restores children order when $source is available': (test) -> - expected = '123' - obj = {"xml":{ - "A":[{"_":"1"}, {"_": "3"}], - "B":[{"_": "2"}] - }} - definePosition = (node, value) -> - Object.defineProperty(node, '$source', { - value: {start: position: value} - enuerable: false - }) - definePosition obj.xml.A[0], 64 - definePosition obj.xml.A[1], 79 - definePosition obj.xml.B[0], 71 - builder = new xml2js.Builder renderOpts: pretty: false - actual = builder.buildObject obj - diffeq expected, actual - test.finish() - 'test building basic XML structure': (test) -> expected = '' obj = {"xml":{"Label":[""],"MsgId":["5850440872586764820"]}} @@ -300,3 +281,39 @@ module.exports = actual = builder.buildObject obj diffeq expected, actual test.finish() + + 'test restores children order when $source is available': (test) -> + expected = '123' + obj = {"xml":{ + "A":[{"_":"1"}, {"_": "3"}], + "B":[{"_": "2","$":{"attr":"x"}}] + }} + definePosition = (node, value) -> + Object.defineProperty(node, '$source', { + value: {start: position: value} + enuerable: false + }) + definePosition obj.xml.A[0], 64 + definePosition obj.xml.A[1], 79 + definePosition obj.xml.B[0], 71 + builder = new xml2js.Builder renderOpts: pretty: false + actual = builder.buildObject obj + diffeq expected, actual + test.finish() + + 'test uses enumerable sourcemap but dont export to XML': (test) -> + expected = '123' + obj = {"xml":{ + "A":[{"_":"1"}, {"_": "3"}], + "B":[{"_": "2","$":{"attr":"x"}}] + }} + definePosition = (node, value) -> + node['$source'] = {start: position: value} + definePosition obj.xml.A[0], 64 + definePosition obj.xml.A[1], 79 + definePosition obj.xml.B[0], 71 + builder = new xml2js.Builder renderOpts: pretty: false + actual = builder.buildObject obj + diffeq expected, actual + test.finish() +