diff --git a/moo.js b/moo.js index 87dcd1e..8632761 100644 --- a/moo.js +++ b/moo.js @@ -352,15 +352,15 @@ var match = this._eat(re) var i = this._getGroup(match) - var group, value + var group, text if (i === -1) { group = this.error // consume rest of buffer - value = buffer.slice(index) + text = buffer.slice(index) } else { - value = match[0] // i+1 + text = match[0] group = this.groups[i] } @@ -369,25 +369,26 @@ if (group.lineBreaks) { var matchNL = /\n/g var nl = 1 - if (value === '\n') { + if (text === '\n') { lineBreaks = 1 } else { - while (matchNL.exec(value)) { lineBreaks++; nl = matchNL.lastIndex } + while (matchNL.exec(text)) { lineBreaks++; nl = matchNL.lastIndex } } } - var size = value.length var token = { - type: (group.getType && group.getType(value)) || group.tokenType, - value: (group.value && group.value(value)) || value, + type: (group.getType && group.getType(text)) || group.tokenType, + value: (group.value && group.value(text)) || text, + text: text, toString: tokenToString, offset: index, - size: size, lineBreaks: lineBreaks, line: this.line, col: this.col, } + // nb. adding more props to token object will make V8 sad! + var size = text.length this.index += size this.line += lineBreaks if (lineBreaks !== 0) { diff --git a/test/test.js b/test/test.js index b85e5e2..a116f23 100644 --- a/test/test.js +++ b/test/test.js @@ -196,7 +196,7 @@ describe('value transforms', () => { })).toThrow("has capture groups") }) - test('transform & report correct size', () => { + test('transform & keep original', () => { let lexer = moo.compile({ fubar: {match: /fubar/, value: x => x.slice(2)}, string: {match: /".*?"/, value: x => x.slice(1, -1)}, @@ -206,10 +206,10 @@ describe('value transforms', () => { }) lexer.reset('fubar "yes" quxx moomoomoomoo') let tokens = lexAll(lexer).filter(t => t.type !== 'space') - expect(tokens.shift()).toMatchObject({ type: 'fubar', value: 'bar', size: 5 }) - expect(tokens.shift()).toMatchObject({ type: 'string', value: 'yes', size: 5 }) - expect(tokens.shift()).toMatchObject({ value: 'quxx', size: 4 }) - expect(tokens.shift()).toMatchObject({ value: 'moomoo', size: 12 }) + expect(tokens.shift()).toMatchObject({ type: 'fubar', text: 'fubar', value: 'bar' }) + expect(tokens.shift()).toMatchObject({ type: 'string', text: '"yes"', value: 'yes' }) + expect(tokens.shift()).toMatchObject({ value: 'quxx' }) + expect(tokens.shift()).toMatchObject({ value: 'moomoo' }) }) }) @@ -650,6 +650,7 @@ describe('errors', () => { }) lexer.reset('foo\nbar') expect(lexer.next()).toMatchObject({type: 'error', value: 'foo\nbar', lineBreaks: 1}) + expect(lexer.save()).toMatchObject({line: 2}) expect(lexer.next()).toBe(undefined) // consumes rest of input })