From fbddcd5ff07b0f72b48af29dc931b69f10f4d0dd Mon Sep 17 00:00:00 2001 From: Dylan Scott Date: Fri, 8 Feb 2019 15:11:01 -0800 Subject: [PATCH] lexer more accurately tracks token line and column information --- nunjucks/src/lexer.js | 2 +- tests/compiler.js | 2 +- tests/lexer.js | 295 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 297 insertions(+), 2 deletions(-) diff --git a/nunjucks/src/lexer.js b/nunjucks/src/lexer.js index 2e33adb4..ef992274 100644 --- a/nunjucks/src/lexer.js +++ b/nunjucks/src/lexer.js @@ -375,7 +375,7 @@ class Tokenizer { _extractString(str) { if (this._matches(str)) { - this.index += str.length; + this.forwardN(str.length); return str; } return null; diff --git a/tests/compiler.js b/tests/compiler.js index a282ecca..8bdebe66 100644 --- a/tests/compiler.js +++ b/tests/compiler.js @@ -816,7 +816,7 @@ tmpl.render({}, function(err, res) { expect(res).to.be(undefined); expect(err.toString()).to.be([ - 'Template render error: (parse-error.njk) [Line 1, Column 24]', + 'Template render error: (parse-error.njk) [Line 1, Column 26]', ' unexpected token: ,', ].join('\n')); done(); diff --git a/tests/lexer.js b/tests/lexer.js index 0c406e54..515c46e0 100644 --- a/tests/lexer.js +++ b/tests/lexer.js @@ -32,6 +32,17 @@ if (lib.isArray(type)) { expect(tok.type).to.be(type[0]); expect(tok.value).to.be(type[1]); + } else if (lib.isObject(type)) { + expect(tok.type).to.be(type.type); + if (type.value != null) { + expect(tok.value).to.be(type.value); + } + if (type.lineno != null) { + expect(tok.lineno).to.be(type.lineno); + } + if (type.colno != null) { + expect(tok.colno).to.be(type.colno); + } } else { expect(tok.type).to.be(type); } @@ -436,5 +447,289 @@ lexer.TOKEN_SYMBOL, lexer.TOKEN_VARIABLE_END); }); + + it('should keep track of token positions', function() { + hasTokens(lexer.lex('{{ 3 != 4 == 5 <= 6 >= 7 < 8 > 9 }}'), + { + type: lexer.TOKEN_VARIABLE_START, + lineno: 0, + colno: 0, + }, + { + type: lexer.TOKEN_INT, + value: '3', + lineno: 0, + colno: 3, + }, + { + type: lexer.TOKEN_OPERATOR, + value: '!=', + lineno: 0, + colno: 5, + }, + { + type: lexer.TOKEN_INT, + value: '4', + lineno: 0, + colno: 8, + }, + { + type: lexer.TOKEN_OPERATOR, + value: '==', + lineno: 0, + colno: 10, + }, + { + type: lexer.TOKEN_INT, + value: '5', + lineno: 0, + colno: 13, + }, + { + type: lexer.TOKEN_OPERATOR, + value: '<=', + lineno: 0, + colno: 15, + }, + { + type: lexer.TOKEN_INT, + value: '6', + lineno: 0, + colno: 18, + }, + { + type: lexer.TOKEN_OPERATOR, + lineno: 0, + colno: 20, + value: '>=', + }, + { + type: lexer.TOKEN_INT, + lineno: 0, + colno: 23, + value: '7', + }, + { + type: lexer.TOKEN_OPERATOR, + value: '<', + lineno: 0, + colno: 25, + }, + { + type: lexer.TOKEN_INT, + value: '8', + lineno: 0, + colno: 27, + }, + { + type: lexer.TOKEN_OPERATOR, + value: '>', + lineno: 0, + colno: 29, + }, + { + type: lexer.TOKEN_INT, + value: '9', + lineno: 0, + colno: 31, + }, + { + type: lexer.TOKEN_VARIABLE_END, + lineno: 0, + colno: 33, + }); + + hasTokens(lexer.lex('{% if something %}{{ value }}{% else %}{{ otherValue }}{% endif %}'), + { + type: lexer.TOKEN_BLOCK_START, + lineno: 0, + colno: 0, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'if', + lineno: 0, + colno: 3, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'something', + lineno: 0, + colno: 6, + }, + { + type: lexer.TOKEN_BLOCK_END, + lineno: 0, + colno: 16, + }, + { + type: lexer.TOKEN_VARIABLE_START, + lineno: 0, + colno: 18, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'value', + lineno: 0, + colno: 21, + }, + { + type: lexer.TOKEN_VARIABLE_END, + lineno: 0, + colno: 27, + }, + { + type: lexer.TOKEN_BLOCK_START, + lineno: 0, + colno: 29, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'else', + lineno: 0, + colno: 32, + }, + { + type: lexer.TOKEN_BLOCK_END, + lineno: 0, + colno: 37, + }, + { + type: lexer.TOKEN_VARIABLE_START, + lineno: 0, + colno: 39, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'otherValue', + lineno: 0, + colno: 42, + }, + { + type: lexer.TOKEN_VARIABLE_END, + lineno: 0, + colno: 53, + }, + { + type: lexer.TOKEN_BLOCK_START, + lineno: 0, + colno: 55, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'endif', + lineno: 0, + colno: 58, + }, + { + type: lexer.TOKEN_BLOCK_END, + lineno: 0, + colno: 64, + }); + + hasTokens(lexer.lex('{% if something %}\n{{ value }}\n{% else %}\n{{ otherValue }}\n{% endif %}'), + { + type: lexer.TOKEN_BLOCK_START, + lineno: 0, + colno: 0, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'if', + lineno: 0, + colno: 3, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'something', + lineno: 0, + colno: 6, + }, + { + type: lexer.TOKEN_BLOCK_END, + lineno: 0, + colno: 16, + }, + { + type: lexer.TOKEN_DATA, + value: '\n', + }, + { + type: lexer.TOKEN_VARIABLE_START, + lineno: 1, + colno: 0, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'value', + lineno: 1, + colno: 3, + }, + { + type: lexer.TOKEN_VARIABLE_END, + lineno: 1, + colno: 9, + }, + { + type: lexer.TOKEN_DATA, + value: '\n', + }, + { + type: lexer.TOKEN_BLOCK_START, + lineno: 2, + colno: 0, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'else', + lineno: 2, + colno: 3, + }, + { + type: lexer.TOKEN_BLOCK_END, + lineno: 2, + colno: 8, + }, + { + type: lexer.TOKEN_DATA, + value: '\n', + }, + { + type: lexer.TOKEN_VARIABLE_START, + lineno: 3, + colno: 0, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'otherValue', + lineno: 3, + colno: 3, + }, + { + type: lexer.TOKEN_VARIABLE_END, + lineno: 3, + colno: 14, + }, + { + type: lexer.TOKEN_DATA, + value: '\n', + }, + { + type: lexer.TOKEN_BLOCK_START, + lineno: 4, + colno: 0, + }, + { + type: lexer.TOKEN_SYMBOL, + value: 'endif', + lineno: 4, + colno: 3, + }, + { + type: lexer.TOKEN_BLOCK_END, + lineno: 4, + colno: 9, + }); + }); }); }());