Permalink
Browse files

[[FIX]] Allow lexer to communicate completion

Currently, JSHint's lexer continuously emits new "end-of-file" token
once all input has been consumed. This behavior makes the internal
`advance` function dangerous to use within iteration contexts. Such
usages need to explicitly check for the "end-of-file" token in order to
avoid entering non-terminating states.

Some of JSHint's parsing logic (such as that for template literal
parsing) takes this detail into account. Other areas (i.e. parsing class
bodies and function parameter lists) does not. While it would be
possible to extend each use with explicit safety checks, the possibility
for error in future feature additions would remain.

Update the lexer to consistently emit `null` after producing the
"end-of-file" token, alleviating the requirement of explicit safety
checks in `advance`-calling code. Update existing unit tests with the
new parsing error information (which is more accurate in many cases).
  • Loading branch information...
jugglinmike committed Apr 23, 2015
1 parent e22b21a commit a093f784ee5298973c6233e9113842cbe8d16c74
Showing with 86 additions and 28 deletions.
  1. +12 −2 src/jshint.js
  2. +10 −1 src/lex.js
  3. +2 −3 tests/unit/core.js
  4. +2 −4 tests/unit/options.js
  5. +60 −18 tests/unit/parser.js
@@ -730,6 +730,12 @@ var JSHINT = (function() {
}
j += 1;
}

// Peeking past the end of the program should produce the "(end)" token.
if (!t && state.tokens.next.id === "(end)") {
return state.tokens.next;
}

return t;
}

@@ -3000,7 +3006,7 @@ var JSHINT = (function() {
var depth = this.depth;

if (!noSubst) {
while (!end() && state.tokens.next.id !== "(end)") {
while (!end()) {
if (!state.tokens.next.template || state.tokens.next.depth > depth) {
expression(0); // should probably have different rbp?
} else {
@@ -5374,7 +5380,11 @@ var JSHINT = (function() {

statements();
}
advance((state.tokens.next && state.tokens.next.value !== ".") ? "(end)" : undefined);

if (state.tokens.next.id !== "(end)") {
quit("E041", state.tokens.curr.line);
}

funct["(blockscope)"].unstack();

var markDefined = function(name, context) {
@@ -1668,7 +1668,16 @@ Lexer.prototype = {

for (;;) {
if (!this.input.length) {
return create(this.nextLine() ? "(endline)" : "(end)", "");
if (this.nextLine()) {
return create("(endline)", "");
}

if (this.exhausted) {
return null;
}

this.exhausted = true;
return create("(end)", "");
}

token = this.next(checks);
@@ -363,9 +363,8 @@ exports.insideEval = function (test) {
// The "TestRun" class (and these errors) probably needs some
// facility for checking the expected scope of the error
.addError(1, "Unexpected early end of program.")
.addError(1, "Expected an identifier and instead saw '(end)'.")
.addError(1, "Expected ')' and instead saw ''.")
.addError(1, "Missing semicolon.")
.addError(1, "Unrecoverable syntax error. (100% scanned).")
.addError(1, "Unrecoverable syntax error. (100% scanned).")

.test(src, { es3: true, evil: false });

@@ -1068,10 +1068,9 @@ exports.immed = function (test) {
.addError(1, "Expected an identifier and instead saw ')'.")
.addError(1, "Expected an assignment or function call and instead saw an expression.")
.addError(1, "Unmatched '{'.")
.addError(1, "Unmatched '('.")
.addError(1, "Wrapping non-IIFE function literals in parens is unnecessary.")
.addError(1, "Expected an assignment or function call and instead saw an expression.")
.addError(1, "Missing semicolon.")
.addError(1, "Unrecoverable syntax error. (100% scanned).")
.test("(function () { if (true) { }());", { es3: true, immed: true });

test.done();
@@ -1359,8 +1358,7 @@ exports.quotesAndTemplateLiterals = function (test) {
TestRun(test)
.addError(2, "Unexpected '`'.")
.addError(2, "Unexpected early end of program.")
.addError(2, "Expected an identifier and instead saw '(end)'.")
.addError(2, "Missing semicolon.")
.addError(2, "Unrecoverable syntax error. (100% scanned).")
.test(src);

// With esnext
@@ -30,7 +30,6 @@ exports.other = function (test) {
TestRun(test)
.addError(1, "Unexpected '\\'.")
.addError(2, "Unexpected early end of program.")
.addError(2, "Expected an identifier and instead saw '(end)'.")
.addError(2, "Unrecoverable syntax error. (100% scanned).")
.test(code, {es3: true});

@@ -339,8 +338,7 @@ exports.numbers = function (test) {
.addError(16, "Missing semicolon.")
.addError(17, "Unexpected '1'.")
.addError(17, "Unexpected early end of program.")
.addError(17, "Expected an identifier and instead saw '(end)'.")
.addError(17, "Missing semicolon.")
.addError(17, "Unrecoverable syntax error. (100% scanned).")
.test(code, {es3: true});

// Octals are prohibited in strict mode.
@@ -693,7 +691,7 @@ exports.badJSON = function (test) {

var run3 = TestRun(test)
.addError(1, "Expected '}' and instead saw 'k2'.")
.addError(1, "Expected '(end)' and instead saw ':'.");
.addError(1, "Unrecoverable syntax error. (100% scanned).");

run3.test(objMissingComma, {multistr: true, es3: true});
run3.test(objMissingComma, {multistr: true}); // es5
@@ -706,7 +704,7 @@ exports.badJSON = function (test) {

var run4 = TestRun(test)
.addError(1, "Expected ']' and instead saw 'v2'.")
.addError(1, "Expected '(end)' and instead saw ']'.");
.addError(1, "Unrecoverable syntax error. (100% scanned).");

run4.test(arrayMissingComma, {multistr: true, es3: true});
run4.test(arrayMissingComma, {multistr: true}); // es5
@@ -722,7 +720,7 @@ exports.badJSON = function (test) {
.addError(1, "Expected ':' and instead saw 'k2'.")
.addError(1, "Expected a JSON value.")
.addError(1, "Expected '}' and instead saw ':'.")
.addError(1, "Expected '(end)' and instead saw 'v2'.");
.addError(1, "Unrecoverable syntax error. (100% scanned).");

run5.test(objDoubleComma, {multistr: true, es3: true});
run5.test(objDoubleComma, {multistr: true}); // es5
@@ -747,7 +745,8 @@ exports.badJSON = function (test) {
];

var run7 = TestRun(test)
.addError(1, "Expected '}' and instead saw ''.");
.addError(1, "Expected '}' and instead saw ''.")
.addError(1, "Unrecoverable syntax error. (100% scanned).");

run7.test(objUnclosed, {multistr: true, es3: true});
run7.test(objUnclosed, {multistr: true}); // es5
@@ -759,7 +758,8 @@ exports.badJSON = function (test) {
];

var run8 = TestRun(test)
.addError(1, "Expected ']' and instead saw ''.");
.addError(1, "Expected ']' and instead saw ''.")
.addError(1, "Unrecoverable syntax error. (100% scanned).");

run8.test(arrayUnclosed, {multistr: true, es3: true});
run8.test(arrayUnclosed, {multistr: true}); // es5
@@ -772,9 +772,7 @@ exports.badJSON = function (test) {

var run9 = TestRun(test)
.addError(1, "Missing '}' to match '{' from line 1.")
.addError(1, "Expected ':' and instead saw ''.")
.addError(1, "Expected a JSON value.")
.addError(1, "Expected '}' and instead saw ''.");
.addError(1, "Unrecoverable syntax error. (100% scanned).");

run9.test(objUnclosed2, {multistr: true, es3: true});
run9.test(objUnclosed2, {multistr: true}); // es5
@@ -788,7 +786,8 @@ exports.badJSON = function (test) {
var run10 = TestRun(test)
.addError(1, "Missing ']' to match '[' from line 1.")
.addError(1, "Expected a JSON value.")
.addError(1, "Expected ']' and instead saw ''.");
.addError(1, "Expected ']' and instead saw ''.")
.addError(1, "Unrecoverable syntax error. (100% scanned).");

run10.test(arrayUnclosed2, {multistr: true, es3: true});
run10.test(arrayUnclosed2, {multistr: true}); // es5
@@ -826,7 +825,7 @@ exports.badJSON = function (test) {
var run13 = TestRun(test)
.addError(1, "Expected a JSON value.")
.addError(1, "Expected '}' and instead saw '/$^/'.")
.addError(1, "Expected '(end)' and instead saw '}'.");
.addError(1, "Unrecoverable syntax error. (100% scanned).");

run13.test(objBadValue, {multistr: true, es3: true});
run13.test(objBadValue, {multistr: true}); // es5
@@ -892,8 +891,8 @@ exports.blocks = function (test) {
var src = fs.readFileSync(__dirname + "/fixtures/blocks.js", "utf8");

var run = TestRun(test)
.addError(29, "Unmatched \'{\'.")
.addError(31, "Unmatched \'{\'.");
.addError(31, "Unmatched \'{\'.")
.addError(32, "Unrecoverable syntax error. (100% scanned).");
run.test(src, {es3: true});
run.test(src, {}); // es5
run.test(src, {esnext: true});
@@ -1003,7 +1002,7 @@ exports.badIdentifiers = function (test) {
var run = TestRun(test)
.addError(1, "Unexpected '\\'.")
.addError(1, "Expected an identifier and instead saw ''.")
.addError(1, "Missing semicolon.");
.addError(1, "Unrecoverable syntax error. (100% scanned).");
run.test(badUnicode, {es3: true});
run.test(badUnicode, {}); // es5
run.test(badUnicode, {esnext: true});
@@ -1016,7 +1015,7 @@ exports.badIdentifiers = function (test) {
var run = TestRun(test)
.addError(1, "Unexpected '\\'.")
.addError(1, "Expected an identifier and instead saw ''.")
.addError(1, "Missing semicolon.");
.addError(1, "Unrecoverable syntax error. (100% scanned).");
run.test(invalidUnicodeIdent, {es3: true});
run.test(invalidUnicodeIdent, {}); // es5
run.test(invalidUnicodeIdent, {esnext: true});
@@ -1042,9 +1041,9 @@ exports["regression for GH-910"] = function (test) {
.addError(1, "Expected an identifier and instead saw ')'.")
.addError(1, "Expected an operator and instead saw '('.")
.addError(1, "Unmatched '{'.")
.addError(1, "Unmatched '('.")
.addError(1, "Expected an assignment or function call and instead saw an expression.")
.addError(1, "Missing semicolon.")
.addError(1, "Unrecoverable syntax error. (100% scanned).")
.test(src, { es3: true, nonew: true });
test.done();
};
@@ -5430,6 +5429,29 @@ exports.classElementEmpty = function (test) {
test.done();
};

exports.invalidClasses = function (test) {
// Regression test for GH-2324
TestRun(test)
.addError(1, "Class properties must be methods. Expected '(' but instead saw ''.")
.addError(1, "Unrecoverable syntax error. (100% scanned).")
.test("class a { b", { esnext: true });

// Regression test for GH-2339
TestRun(test)
.addError(2, "Class properties must be methods. Expected '(' but instead saw ':'.")
.addError(3, "Expected '(' and instead saw '}'.")
.addError(4, "Expected an identifier and instead saw '}'.")
.addError(4, "Unrecoverable syntax error. (100% scanned).")
.test([
"class Test {",
" constructor: {",
" }",
"}"
], { esnext: true });

test.done();
};

exports["test for GH-1018"] = function (test) {
var code = [
"if (a = 42) {}",
@@ -6520,3 +6542,23 @@ exports.getAsIdentifierProp = function (test) {

test.done();
};

exports.invalidParams = function (test) {
TestRun(test)
.addError(1, "Expected an identifier and instead saw '!'.")
.addError(1, "Unrecoverable syntax error. (100% scanned).")
.test("(function(!", { esnext: true });

test.done();
};

// Regression test for gh-2362
exports.functionKeyword = function (test) {
TestRun(test)
.addError(1, "Missing name in function declaration.")
.addError(1, "Expected '(' and instead saw ''.")
.addError(1, "Unrecoverable syntax error. (100% scanned).")
.test("function");

test.done();
};

0 comments on commit a093f78

Please sign in to comment.