From e011c9e0ff9fb6b5ed343ef48ea5e75b61a96a0b Mon Sep 17 00:00:00 2001 From: Kasper Lund Date: Thu, 6 Feb 2014 12:20:42 +0100 Subject: [PATCH] chore(lexer): Introduce different classes of tokens. Use the new token classes to restrict the grammar to only allow identifiers or keywords as filter and member names. For object literal keys we also allow strings. --- bin/parser_generator_for_spec.dart | 9 + lib/core/parser/dynamic_parser_impl.dart | 195 +++++++++++--------- lib/core/parser/lexer.dart | 60 ++---- lib/core/parser/tokens.dart | 65 +++++++ test/core/parser/lexer_spec.dart | 223 +++++++++++++---------- test/core/parser/parser_spec.dart | 25 ++- 6 files changed, 353 insertions(+), 224 deletions(-) create mode 100644 lib/core/parser/tokens.dart diff --git a/bin/parser_generator_for_spec.dart b/bin/parser_generator_for_spec.dart index 8820674e2..e0dbe26e8 100644 --- a/bin/parser_generator_for_spec.dart +++ b/bin/parser_generator_for_spec.dart @@ -434,5 +434,14 @@ main(arguments) { "o.void()", "o.while()", "o.with()", + + '"Foo"|(', + '"Foo"|1234', + '"Foo"|"uppercase"', + 'x.(', + 'x. 1234', + 'x."foo"', + '{(:0}', + '{1234:0}', ]); } diff --git a/lib/core/parser/dynamic_parser_impl.dart b/lib/core/parser/dynamic_parser_impl.dart index 0b893ddcf..d7edfc4d2 100644 --- a/lib/core/parser/dynamic_parser_impl.dart +++ b/lib/core/parser/dynamic_parser_impl.dart @@ -3,9 +3,9 @@ library angular.core.parser.dynamic_parser_impl; import 'package:angular/core/parser/parser.dart' show ParserBackend; import 'package:angular/core/parser/lexer.dart'; import 'package:angular/core/parser/syntax.dart'; +import 'package:angular/core/parser/characters.dart'; class DynamicParserImpl { - static Token EOF = new Token(-1, null); final ParserBackend backend; final String input; final List tokens; @@ -15,26 +15,28 @@ class DynamicParserImpl { : this.input = input, tokens = lexer.call(input); Token get peek { - return (index < tokens.length) ? tokens[index] : EOF; + return (index < tokens.length) ? tokens[index] : Token.EOF; } parseChain() { bool isChain = false; - while (optional(';')) { + while (optionalCharacter($SEMICOLON)) { isChain = true; } List expressions = []; while (index < tokens.length) { - if (peek.text == ')' || peek.text == '}' || peek.text == ']') { - error('Unconsumed token ${peek.text}'); + if (peek.isCharacter($RPAREN) || + peek.isCharacter($RBRACE) || + peek.isCharacter($RBRACKET)) { + error('Unconsumed token $peek'); } var expr = parseFilter(); expressions.add(expr); - while (optional(';')) { + while (optionalCharacter($SEMICOLON)) { isChain = true; } if (isChain && expr is Filter) { - error('cannot have a filter in a chain'); + error('Cannot have a filter in a chain'); } } return (expressions.length == 1) @@ -44,11 +46,10 @@ class DynamicParserImpl { parseFilter() { var result = parseExpression(); - while (optional('|')) { - String name = peek.text; // TODO(kasperl): Restrict to identifier? - advance(); + while (optionalOperator('|')) { + String name = expectIdentifierOrKeyword(); List arguments = []; - while (optional(':')) { + while (optionalCharacter($COLON)) { // TODO(kasperl): Is this really supposed to be expressions? arguments.add(parseExpression()); } @@ -60,13 +61,13 @@ class DynamicParserImpl { parseExpression() { int start = peek.index; var result = parseConditional(); - while (peek.text == '=') { + while (peek.isOperator('=')) { if (!backend.isAssignable(result)) { int end = (index < tokens.length) ? peek.index : input.length; String expression = input.substring(start, end); error('Expression $expression is not assignable'); } - expect('='); + expectOperator('='); result = backend.newAssign(result, parseConditional()); } return result; @@ -75,9 +76,9 @@ class DynamicParserImpl { parseConditional() { int start = peek.index; var result = parseLogicalOr(); - if (optional('?')) { + if (optionalOperator('?')) { var yes = parseExpression(); - if (!optional(':')) { + if (!optionalCharacter($COLON)) { int end = (index < tokens.length) ? peek.index : input.length; String expression = input.substring(start, end); error('Conditional expression $expression requires all 3 expressions'); @@ -91,7 +92,7 @@ class DynamicParserImpl { parseLogicalOr() { // '||' var result = parseLogicalAnd(); - while (optional('||')) { + while (optionalOperator('||')) { result = backend.newBinaryLogicalOr(result, parseLogicalAnd()); } return result; @@ -100,7 +101,7 @@ class DynamicParserImpl { parseLogicalAnd() { // '&&' var result = parseEquality(); - while (optional('&&')) { + while (optionalOperator('&&')) { result = backend.newBinaryLogicalAnd(result, parseEquality()); } return result; @@ -110,9 +111,9 @@ class DynamicParserImpl { // '==','!=' var result = parseRelational(); while (true) { - if (optional('==')) { + if (optionalOperator('==')) { result = backend.newBinaryEqual(result, parseRelational()); - } else if (optional('!=')) { + } else if (optionalOperator('!=')) { result = backend.newBinaryNotEqual(result, parseRelational()); } else { return result; @@ -124,13 +125,13 @@ class DynamicParserImpl { // '<', '>', '<=', '>=' var result = parseAdditive(); while (true) { - if (optional('<')) { + if (optionalOperator('<')) { result = backend.newBinaryLessThan(result, parseAdditive()); - } else if (optional('>')) { + } else if (optionalOperator('>')) { result = backend.newBinaryGreaterThan(result, parseAdditive()); - } else if (optional('<=')) { + } else if (optionalOperator('<=')) { result = backend.newBinaryLessThanEqual(result, parseAdditive()); - } else if (optional('>=')) { + } else if (optionalOperator('>=')) { result = backend.newBinaryGreaterThanEqual(result, parseAdditive()); } else { return result; @@ -142,9 +143,9 @@ class DynamicParserImpl { // '+', '-' var result = parseMultiplicative(); while (true) { - if (optional('+')) { + if (optionalOperator('+')) { result = backend.newBinaryPlus(result, parseMultiplicative()); - } else if (optional('-')) { + } else if (optionalOperator('-')) { result = backend.newBinaryMinus(result, parseMultiplicative()); } else { return result; @@ -156,13 +157,13 @@ class DynamicParserImpl { // '*', '%', '/', '~/' var result = parsePrefix(); while (true) { - if (optional('*')) { + if (optionalOperator('*')) { result = backend.newBinaryMultiply(result, parsePrefix()); - } else if (optional('%')) { + } else if (optionalOperator('%')) { result = backend.newBinaryModulo(result, parsePrefix()); - } else if (optional('/')) { + } else if (optionalOperator('/')) { result = backend.newBinaryDivide(result, parsePrefix()); - } else if (optional('~/')) { + } else if (optionalOperator('~/')) { result = backend.newBinaryTruncatingDivide(result, parsePrefix()); } else { return result; @@ -171,12 +172,12 @@ class DynamicParserImpl { } parsePrefix() { - if (optional('+')) { + if (optionalOperator('+')) { // TODO(kasperl): This is different than the original parser. return backend.newPrefixPlus(parsePrefix()); - } else if (optional('-')) { + } else if (optionalOperator('-')) { return backend.newPrefixMinus(parsePrefix()); - } else if (optional('!')) { + } else if (optionalOperator('!')) { return backend.newPrefixNot(parsePrefix()); } else { return parseAccessOrCallMember(); @@ -186,24 +187,22 @@ class DynamicParserImpl { parseAccessOrCallMember() { var result = parsePrimary(); while (true) { - if (optional('.')) { - // TODO(kasperl): Check that this is an identifier. Are keywords okay? - String name = peek.text; - advance(); - if (optional('(')) { - List arguments = parseExpressionList(')'); - expect(')'); + if (optionalCharacter($PERIOD)) { + String name = expectIdentifierOrKeyword(); + if (optionalCharacter($LPAREN)) { + List arguments = parseExpressionList($RPAREN); + expectCharacter($RPAREN); result = backend.newCallMember(result, name, arguments); } else { result = backend.newAccessMember(result, name); } - } else if (optional('[')) { + } else if (optionalCharacter($LBRACKET)) { var key = parseExpression(); - expect(']'); + expectCharacter($RBRACKET); result = backend.newAccessKeyed(result, key); - } else if (optional('(')) { - List arguments = parseExpressionList(')'); - expect(')'); + } else if (optionalCharacter($LPAREN)) { + List arguments = parseExpressionList($RPAREN); + expectCharacter($RPAREN); result = backend.newCallFunction(result, arguments); } else { return result; @@ -212,77 +211,78 @@ class DynamicParserImpl { } parsePrimary() { - if (optional('(')) { + if (optionalCharacter($LPAREN)) { var result = parseExpression(); - expect(')'); + expectCharacter($RPAREN); return result; - } else if (optional('null') || optional('undefined')) { + } else if (peek.isKeywordNull || peek.isKeywordUndefined) { + advance(); return backend.newLiteralNull(); - } else if (optional('true')) { + } else if (peek.isKeywordTrue) { + advance(); return backend.newLiteralBoolean(true); - } else if (optional('false')) { + } else if (peek.isKeywordFalse) { + advance(); return backend.newLiteralBoolean(false); - } else if (optional('[')) { - List elements = parseExpressionList(']'); - expect(']'); + } else if (optionalCharacter($LBRACKET)) { + List elements = parseExpressionList($RBRACKET); + expectCharacter($RBRACKET); return backend.newLiteralArray(elements); - } else if (peek.text == '{') { + } else if (peek.isCharacter($LBRACE)) { return parseObject(); - } else if (peek.key != null) { + } else if (peek.isIdentifier) { return parseAccessOrCallScope(); - } else if (peek.value != null) { - var value = peek.value; + } else if (peek.isNumber) { + num value = peek.toNumber(); + advance(); + return backend.newLiteralNumber(value); + } else if (peek.isString) { + String value = peek.toString(); advance(); - return (value is num) - ? backend.newLiteralNumber(value) - : backend.newLiteralString(value); + return backend.newLiteralString(value); } else if (index >= tokens.length) { throw 'Unexpected end of expression: $input'; } else { - error('Unexpected token ${peek.text}'); + error('Unexpected token $peek'); } } parseAccessOrCallScope() { - String name = peek.key; - advance(); - if (!optional('(')) return backend.newAccessScope(name); - List arguments = parseExpressionList(')'); - expect(')'); + String name = expectIdentifierOrKeyword(); + if (!optionalCharacter($LPAREN)) return backend.newAccessScope(name); + List arguments = parseExpressionList($RPAREN); + expectCharacter($RPAREN); return backend.newCallScope(name, arguments); } parseObject() { List keys = []; List values = []; - expect('{'); - if (peek.text != '}') { + expectCharacter($LBRACE); + if (!optionalCharacter($RBRACE)) { do { - // TODO(kasperl): Stricter checking. Only allow identifiers - // and strings as keys. Maybe also keywords? - var value = peek.value; - keys.add(value is String ? value : peek.text); - advance(); - expect(':'); + String key = expectIdentifierOrKeywordOrString(); + keys.add(key); + expectCharacter($COLON); values.add(parseExpression()); - } while (optional(',')); + } while (optionalCharacter($COMMA)); + expectCharacter($RBRACE); } - expect('}'); return backend.newLiteralObject(keys, values); } - List parseExpressionList(String terminator) { + List parseExpressionList(int terminator) { List result = []; - if (peek.text != terminator) { + if (!peek.isCharacter(terminator)) { do { result.add(parseExpression()); - } while (optional(',')); + } while (optionalCharacter($COMMA)); } return result; } - bool optional(text) { - if (peek.text == text) { + bool optionalCharacter(int code) { + if (peek.isCharacter(code)) { advance(); return true; } else { @@ -290,12 +290,41 @@ class DynamicParserImpl { } } - void expect(text) { - if (peek.text == text) { + bool optionalOperator(String operator) { + if (peek.isOperator(operator)) { advance(); + return true; } else { - error('Missing expected $text'); + return false; + } + } + + void expectCharacter(int code) { + if (optionalCharacter(code)) return; + error('Missing expected ${new String.fromCharCode(code)}'); + } + + void expectOperator(String operator) { + if (optionalOperator(operator)) return; + error('Missing expected operator $operator'); + } + + String expectIdentifierOrKeyword() { + if (!peek.isIdentifier && !peek.isKeyword) { + error('Unexpected token $peek, expected identifier or keyword'); } + String result = peek.toString(); + advance(); + return result; + } + + String expectIdentifierOrKeywordOrString() { + if (!peek.isIdentifier && !peek.isKeyword && !peek.isString) { + error('Unexpected token $peek, expected identifier, keyword, or string'); + } + String result = peek.toString(); + advance(); + return result; } void advance() { diff --git a/lib/core/parser/lexer.dart b/lib/core/parser/lexer.dart index 3361c8f53..5ad803489 100644 --- a/lib/core/parser/lexer.dart +++ b/lib/core/parser/lexer.dart @@ -3,30 +3,7 @@ library angular.core.parser.lexer; import 'package:angular/core/module.dart' show NgInjectableService; import 'package:angular/core/parser/characters.dart'; -class Token { - final int index; - final String text; - - var value; - // Tokens should have one of these set. - String opKey; - String key; - - Token(this.index, this.text); - - withOp(op) { - this.opKey = op; - } - - withGetterSetter(key) { - this.key = key; - } - - withValue(value) { this.value = value; } - - toString() => "Token($text)"; -} - +part 'tokens.dart'; @NgInjectableService() class Lexer { @@ -72,7 +49,7 @@ class Scanner { switch (peek) { case $PERIOD: advance(); - return isDigit(peek) ? scanNumber(start) : new Token(start, '.'); + return isDigit(peek) ? scanNumber(start) : new CharacterToken(start, $PERIOD); case $LPAREN: case $RPAREN: case $LBRACE: @@ -82,7 +59,7 @@ class Scanner { case $COMMA: case $COLON: case $SEMICOLON: - return scanCharacter(start, new String.fromCharCode(peek)); + return scanCharacter(start, peek); case $SQ: case $DQ: return scanString(); @@ -115,17 +92,17 @@ class Scanner { return null; } - Token scanCharacter(int start, String string) { - assert(peek == string.codeUnitAt(0)); + Token scanCharacter(int start, int code) { + assert(peek == code); advance(); - return new Token(start, string); + return new CharacterToken(start, code); } Token scanOperator(int start, String string) { assert(peek == string.codeUnitAt(0)); assert(OPERATORS.contains(string)); advance(); - return new Token(start, string)..withOp(string); + return new OperatorToken(start, string); } Token scanComplexOperator(int start, int code, String one, String two) { @@ -137,7 +114,7 @@ class Scanner { string += two; } assert(OPERATORS.contains(string)); - return new Token(start, string)..withOp(string); + return new OperatorToken(start, string); } Token scanIdentifier() { @@ -146,15 +123,7 @@ class Scanner { advance(); while (isIdentifierPart(peek)) advance(); String string = input.substring(start, index); - Token result = new Token(start, string); - // TODO(kasperl): Deal with null, undefined, true, and false in - // a cleaner and faster way. - if (OPERATORS.contains(string)) { - result.withOp(string); - } else { - result.withGetterSetter(string); - } - return result; + return new IdentifierToken(start, string, KEYWORDS.contains(string)); } Token scanNumber(int start) { @@ -178,7 +147,7 @@ class Scanner { } String string = input.substring(start, index); num value = simple ? int.parse(string) : double.parse(string); - return new Token(start, string)..withValue(value); + return new NumberToken(start, value); } Token scanString() { @@ -226,7 +195,7 @@ class Scanner { buffer.write(last); unescaped = buffer.toString(); } - return new Token(start, string)..withValue(unescaped); + return new StringToken(start, string, unescaped); } void advance() { @@ -242,11 +211,14 @@ class Scanner { } } -Set OPERATORS = new Set.from([ - 'undefined', +Set KEYWORDS = new Set.from([ 'null', + 'undefined', 'true', 'false', +]); + +Set OPERATORS = new Set.from([ '+', '-', '*', diff --git a/lib/core/parser/tokens.dart b/lib/core/parser/tokens.dart new file mode 100644 index 000000000..b3b5bf18f --- /dev/null +++ b/lib/core/parser/tokens.dart @@ -0,0 +1,65 @@ +part of angular.core.parser.lexer; + +class Token { + static const Token EOF = const Token._(-1); + final int index; + const Token._(this.index); + + bool get isIdentifier => false; + bool get isString => false; + bool get isNumber => false; + + bool isCharacter(int code) => false; + bool isOperator(String operator) => false; + + bool get isKeyword => false; + bool get isKeywordNull => false; + bool get isKeywordUndefined => false; + bool get isKeywordTrue => false; + bool get isKeywordFalse => false; + + num toNumber() => null; +} + +class CharacterToken extends Token { + final int _code; + CharacterToken(int index, this._code) : super._(index); + bool isCharacter(int code) => _code == code; + String toString() => new String.fromCharCode(_code); +} + +class IdentifierToken extends Token { + final String _text; + final bool _isKeyword; + IdentifierToken(int index, this._text, this._isKeyword) : super._(index); + bool get isIdentifier => !_isKeyword; + bool get isKeyword => _isKeyword; + bool get isKeywordNull => _isKeyword && _text == "null"; + bool get isKeywordUndefined => _isKeyword && _text == "undefined"; + bool get isKeywordTrue => _isKeyword && _text == "true"; + bool get isKeywordFalse => _isKeyword && _text == "false"; + String toString() => _text; +} + +class OperatorToken extends Token { + final String _text; + OperatorToken(int index, this._text) : super._(index); + bool isOperator(String operator) => _text == operator; + String toString() => _text; +} + +class NumberToken extends Token { + final num _value; + NumberToken(int index, this._value) : super._(index); + bool get isNumber => true; + num toNumber() => _value; + String toString() => "$_value"; +} + +class StringToken extends Token { + final String input; + final String _value; + StringToken(int index, this.input, this._value) : super._(index); + bool get isString => true; + String toString() => _value; +} diff --git a/test/core/parser/lexer_spec.dart b/test/core/parser/lexer_spec.dart index 7844268e2..219c6f9fb 100644 --- a/test/core/parser/lexer_spec.dart +++ b/test/core/parser/lexer_spec.dart @@ -4,12 +4,50 @@ import '../../_specs.dart'; class LexerExpect extends Expect { LexerExpect(actual) : super(actual); - toBeToken(int index, String text) { + + toBeToken(int index) { expect(actual is Token).toEqual(true); expect(actual.index).toEqual(index); - expect(actual.text).toEqual(text); + } + + toBeCharacterToken(int index, String character) { + toBeToken(index); + expect(character.length).toEqual(1); + expect(actual.isCharacter(character.codeUnitAt(0))).toEqual(true); + } + + toBeIdentifierToken(int index, String text) { + toBeToken(index); + expect(actual.isIdentifier).toEqual(true); + expect(actual.toString()).toEqual(text); + } + + toBeKeywordUndefinedToken(int index) { + toBeToken(index); + expect(actual.isKeywordUndefined).toEqual(true); + } + + toBeOperatorToken(int index, String operator) { + toBeToken(index); + expect(actual.isOperator(operator)).toEqual(true); + } + + toBeStringToken(int index, String input, String value) { + toBeToken(index); + expect(actual.isString).toEqual(true); + StringToken token = actual; + expect(token.input).toEqual(input); + expect(token.toString()).toEqual(value); + } + + toBeNumberToken(int index, num value) { + toBeToken(index); + expect(actual.isNumber).toEqual(true); + NumberToken token = actual; + expect(token.toNumber()).toEqual(value); } } + expect(actual) => new LexerExpect(actual); main() { @@ -21,204 +59,199 @@ main() { // New test case it('should tokenize a simple identifier', () { - var tokens = lex("j"); + List tokens = lex("j"); expect(tokens.length).toEqual(1); - expect(tokens[0]).toBeToken(0, 'j'); + expect(tokens[0]).toBeIdentifierToken(0, 'j'); }); // New test case it('should tokenize a dotted identifier', () { - var tokens = lex("j.k"); + List tokens = lex("j.k"); expect(tokens.length).toEqual(3); - expect(tokens[0]).toBeToken(0, 'j'); - expect(tokens[1]).toBeToken(1, '.'); - expect(tokens[2]).toBeToken(2, 'k'); + expect(tokens[0]).toBeIdentifierToken(0, 'j'); + expect(tokens[1]).toBeCharacterToken(1, '.'); + expect(tokens[2]).toBeIdentifierToken(2, 'k'); }); it('should tokenize an operator', () { - var tokens = lex('j-k'); + List tokens = lex('j-k'); expect(tokens.length).toEqual(3); - expect(tokens[1]).toBeToken(1, '-'); + expect(tokens[1]).toBeOperatorToken(1, '-'); }); it('should tokenize an indexed operator', () { - var tokens = lex('j[k]'); + List tokens = lex('j[k]'); expect(tokens.length).toEqual(4); - expect(tokens[1]).toBeToken(1, '['); + expect(tokens[1]).toBeCharacterToken(1, '['); }); it('should tokenize numbers', () { - var tokens = lex('88'); + List tokens = lex('88'); expect(tokens.length).toEqual(1); - expect(tokens[0]).toBeToken(0, '88'); + expect(tokens[0]).toBeNumberToken(0, 88); }); it('should tokenize numbers within index ops', () { - expect(lex('a[22]')[2]).toBeToken(2, '22'); + expect(lex('a[22]')[2]).toBeNumberToken(2, 22); }); it('should tokenize simple quoted strings', () { - expect(lex('"a"')[0]).toBeToken(0, '"a"'); + expect(lex('"a"')[0]).toBeStringToken(0, '"a"', 'a'); }); it('should tokenize quoted strings with escaped quotes', () { - expect(lex('"a\\""')[0]).toBeToken(0, '"a\\""'); + expect(lex('"a\\""')[0]).toBeStringToken(0, '"a\\""', 'a"'); }); it('should tokenize a string', () { - var tokens = lex("j-a.bc[22]+1.3|f:'a\\\'c':\"d\\\"e\""); + List tokens = lex("j-a.bc[22]+1.3|f:'a\\\'c':\"d\\\"e\""); var i = 0; - expect(tokens[i]).toBeToken(0, 'j'); + expect(tokens[i]).toBeIdentifierToken(0, 'j'); i++; - expect(tokens[i]).toBeToken(1, '-'); + expect(tokens[i]).toBeOperatorToken(1, '-'); i++; - expect(tokens[i]).toBeToken(2, 'a'); + expect(tokens[i]).toBeIdentifierToken(2, 'a'); i++; - expect(tokens[i]).toBeToken(3, '.'); + expect(tokens[i]).toBeCharacterToken(3, '.'); i++; - expect(tokens[i]).toBeToken(4, 'bc'); + expect(tokens[i]).toBeIdentifierToken(4, 'bc'); i++; - expect(tokens[i]).toBeToken(6, '['); + expect(tokens[i]).toBeCharacterToken(6, '['); i++; - expect(tokens[i]).toBeToken(7, '22'); + expect(tokens[i]).toBeNumberToken(7, 22); i++; - expect(tokens[i]).toBeToken(9, ']'); + expect(tokens[i]).toBeCharacterToken(9, ']'); i++; - expect(tokens[i]).toBeToken(10, '+'); + expect(tokens[i]).toBeOperatorToken(10, '+'); i++; - expect(tokens[i]).toBeToken(11, '1.3'); + expect(tokens[i]).toBeNumberToken(11, 1.3); i++; - expect(tokens[i]).toBeToken(14, '|'); + expect(tokens[i]).toBeOperatorToken(14, '|'); i++; - expect(tokens[i]).toBeToken(15, 'f'); + expect(tokens[i]).toBeIdentifierToken(15, 'f'); i++; - expect(tokens[i]).toBeToken(16, ':'); + expect(tokens[i]).toBeCharacterToken(16, ':'); i++; - expect(tokens[i]).toBeToken(17, '\'a\\\'c\''); + expect(tokens[i]).toBeStringToken(17, "'a\\'c'", "a'c"); i++; - expect(tokens[i]).toBeToken(23, ':'); + expect(tokens[i]).toBeCharacterToken(23, ':'); i++; - expect(tokens[i]).toBeToken(24, '"d\\"e"'); + expect(tokens[i]).toBeStringToken(24, '"d\\"e"', 'd"e'); }); it('should tokenize undefined', () { - var tokens = lex("undefined"); + List tokens = lex("undefined"); var i = 0; - expect(tokens[i]).toBeToken(0, 'undefined'); - expect(tokens[i].value).toEqual(null); + expect(tokens[i]).toBeKeywordUndefinedToken(0); }); it('should ignore whitespace', () { - var tokens = lex("a \t \n \r b"); - expect(tokens[0].text).toEqual('a'); - expect(tokens[1].text).toEqual('b'); + List tokens = lex("a \t \n \r b"); + expect(tokens[0]).toBeIdentifierToken(0, 'a'); + expect(tokens[1]).toBeIdentifierToken(8, 'b'); }); it('should tokenize quoted string', () { var str = "['\\'', \"\\\"\"]"; - var tokens = lex(str); - - expect(tokens[1].index).toEqual(1); - expect(tokens[1].value).toEqual("'"); - - expect(tokens[3].index).toEqual(7); - expect(tokens[3].value).toEqual('"'); + List tokens = lex(str); + expect(tokens[1]).toBeStringToken(1, "'\\''", "'"); + expect(tokens[3]).toBeStringToken(7, '"\\""', '"'); }); it('should tokenize escaped quoted string', () { var str = '"\\"\\n\\f\\r\\t\\v\\u00A0"'; - var tokens = lex(str); - - expect(tokens[0].value).toEqual('"\n\f\r\t\v\u00A0'); + List tokens = lex(str); + expect(tokens.length).toEqual(1); + expect(tokens[0].toString()).toEqual('"\n\f\r\t\v\u00A0'); }); it('should tokenize unicode', () { - var tokens = lex('"\\u00A0"'); + List tokens = lex('"\\u00A0"'); expect(tokens.length).toEqual(1); - expect(tokens[0].value).toEqual('\u00a0'); + expect(tokens[0].toString()).toEqual('\u00a0'); }); it('should tokenize relation', () { - var tokens = lex("! == != < > <= >="); - expect(tokens[0].text).toEqual('!'); - expect(tokens[1].text).toEqual('=='); - expect(tokens[2].text).toEqual('!='); - expect(tokens[3].text).toEqual('<'); - expect(tokens[4].text).toEqual('>'); - expect(tokens[5].text).toEqual('<='); - expect(tokens[6].text).toEqual('>='); + List tokens = lex("! == != < > <= >="); + expect(tokens[0]).toBeOperatorToken(0, '!'); + expect(tokens[1]).toBeOperatorToken(2, '=='); + expect(tokens[2]).toBeOperatorToken(5, '!='); + expect(tokens[3]).toBeOperatorToken(8, '<'); + expect(tokens[4]).toBeOperatorToken(10, '>'); + expect(tokens[5]).toBeOperatorToken(12, '<='); + expect(tokens[6]).toBeOperatorToken(15, '>='); }); it('should tokenize statements', () { - var tokens = lex("a;b;"); - expect(tokens[0].text).toEqual('a'); - expect(tokens[1].text).toEqual(';'); - expect(tokens[2].text).toEqual('b'); - expect(tokens[3].text).toEqual(';'); + List tokens = lex("a;b;"); + expect(tokens[0]).toBeIdentifierToken(0, 'a'); + expect(tokens[1]).toBeCharacterToken(1, ';'); + expect(tokens[2]).toBeIdentifierToken(2, 'b'); + expect(tokens[3]).toBeCharacterToken(3, ';'); }); it('should tokenize function invocation', () { - var tokens = lex("a()"); - expect(tokens[0]).toBeToken(0, 'a'); - expect(tokens[1]).toBeToken(1, '('); - expect(tokens[2]).toBeToken(2, ')'); + List tokens = lex("a()"); + expect(tokens[0]).toBeIdentifierToken(0, 'a'); + expect(tokens[1]).toBeCharacterToken(1, '('); + expect(tokens[2]).toBeCharacterToken(2, ')'); }); it('should tokenize simple method invocations', () { - var tokens = lex("a.method()"); - expect(tokens[2]).toBeToken(2, 'method'); + List tokens = lex("a.method()"); + expect(tokens[2]).toBeIdentifierToken(2, 'method'); }); it('should tokenize method invocation', () { - var tokens = lex("a.b.c (d) - e.f()"); - expect(tokens[0]).toBeToken(0, 'a'); - expect(tokens[1]).toBeToken(1, '.'); - expect(tokens[2]).toBeToken(2, 'b'); - expect(tokens[3]).toBeToken(3, '.'); - expect(tokens[4]).toBeToken(4, 'c'); - expect(tokens[5]).toBeToken(6, '('); - expect(tokens[6]).toBeToken(7, 'd'); - expect(tokens[7]).toBeToken(8, ')'); - expect(tokens[8]).toBeToken(10, '-'); - expect(tokens[9]).toBeToken(12, 'e'); - expect(tokens[10]).toBeToken(13, '.'); - expect(tokens[11]).toBeToken(14, 'f'); - expect(tokens[12]).toBeToken(15, '('); - expect(tokens[13]).toBeToken(16, ')'); + List tokens = lex("a.b.c (d) - e.f()"); + expect(tokens[0]).toBeIdentifierToken(0, 'a'); + expect(tokens[1]).toBeCharacterToken(1, '.'); + expect(tokens[2]).toBeIdentifierToken(2, 'b'); + expect(tokens[3]).toBeCharacterToken(3, '.'); + expect(tokens[4]).toBeIdentifierToken(4, 'c'); + expect(tokens[5]).toBeCharacterToken(6, '('); + expect(tokens[6]).toBeIdentifierToken(7, 'd'); + expect(tokens[7]).toBeCharacterToken(8, ')'); + expect(tokens[8]).toBeOperatorToken(10, '-'); + expect(tokens[9]).toBeIdentifierToken(12, 'e'); + expect(tokens[10]).toBeCharacterToken(13, '.'); + expect(tokens[11]).toBeIdentifierToken(14, 'f'); + expect(tokens[12]).toBeCharacterToken(15, '('); + expect(tokens[13]).toBeCharacterToken(16, ')'); }); it('should tokenize number', () { - var tokens = lex("0.5"); - expect(tokens[0].value).toEqual(0.5); + List tokens = lex("0.5"); + expect(tokens[0]).toBeNumberToken(0, 0.5); }); // NOTE(deboer): NOT A LEXER TEST // it('should tokenize negative number', () { - // var tokens = lex("-0.5"); - // expect(tokens[0].value).toEqual(-0.5); + // List tokens = lex("-0.5"); + // expect(tokens[0]).toBeNumberToken(0, -0.5); // }); it('should tokenize number with exponent', () { - var tokens = lex("0.5E-10"); + List tokens = lex("0.5E-10"); expect(tokens.length).toEqual(1); - expect(tokens[0].value).toEqual(0.5E-10); + expect(tokens[0]).toBeNumberToken(0, 0.5E-10); tokens = lex("0.5E+10"); - expect(tokens[0].value).toEqual(0.5E+10); + expect(tokens[0]).toBeNumberToken(0, 0.5E+10); }); it('should throws exception for invalid exponent', () { @@ -232,8 +265,8 @@ main() { }); it('should tokenize number starting with a dot', () { - var tokens = lex(".5"); - expect(tokens[0].value).toEqual(0.5); + List tokens = lex(".5"); + expect(tokens[0]).toBeNumberToken(0, 0.5); }); it('should throw error on invalid unicode', () { diff --git a/test/core/parser/parser_spec.dart b/test/core/parser/parser_spec.dart index 1441a93e5..6e1003a8a 100644 --- a/test/core/parser/parser_spec.dart +++ b/test/core/parser/parser_spec.dart @@ -324,6 +324,7 @@ main() { }).toThrow("notAProperty"); }); + it('should fail on private field access', () { expect(parser('publicField').eval(new WithPrivateField())).toEqual(4); // On Dartium, this fails with "NoSuchMethod: no instance getter" @@ -333,6 +334,26 @@ main() { parser('_privateField').eval(new WithPrivateField()); }).toThrow(); }); + + + it('should only allow identifier or keyword as filter names', () { + expect(() => parser('"Foo"|(')).toThrow('identifier or keyword'); + expect(() => parser('"Foo"|1234')).toThrow('identifier or keyword'); + expect(() => parser('"Foo"|"uppercase"')).toThrow('identifier or keyword'); + }); + + + it('should only allow identifier or keyword as member names', () { + expect(() => parser('x.(')).toThrow('identifier or keyword'); + expect(() => parser('x. 1234')).toThrow('identifier or keyword'); + expect(() => parser('x."foo"')).toThrow('identifier or keyword'); + }); + + + it('should only allow identifier, string, or keyword as object literal key', () { + expect(() => parser('{(:0}')).toThrow('expected identifier, keyword, or string'); + expect(() => parser('{1234:0}')).toThrow('expected identifier, keyword, or string'); + }); }); describe('setters', () { @@ -997,10 +1018,10 @@ main() { it('should not allow filters in a chain', () { expect(() { parser("1;'World'|hello"); - }).toThrow('cannot have a filter in a chain the end of the expression [1;\'World\'|hello]'); + }).toThrow('Cannot have a filter in a chain the end of the expression [1;\'World\'|hello]'); expect(() { parser("'World'|hello;1"); - }).toThrow('cannot have a filter in a chain at column 15 in [\'World\'|hello;1]'); + }).toThrow('Cannot have a filter in a chain at column 15 in [\'World\'|hello;1]'); }); }); });