diff --git a/bin/parser_generator_for_spec.dart b/bin/parser_generator_for_spec.dart index 8820674e2..e0dbe26e8 100644 --- a/bin/parser_generator_for_spec.dart +++ b/bin/parser_generator_for_spec.dart @@ -434,5 +434,14 @@ main(arguments) { "o.void()", "o.while()", "o.with()", + + '"Foo"|(', + '"Foo"|1234', + '"Foo"|"uppercase"', + 'x.(', + 'x. 1234', + 'x."foo"', + '{(:0}', + '{1234:0}', ]); } diff --git a/lib/core/parser/dynamic_parser_impl.dart b/lib/core/parser/dynamic_parser_impl.dart index 0b893ddcf..d7edfc4d2 100644 --- a/lib/core/parser/dynamic_parser_impl.dart +++ b/lib/core/parser/dynamic_parser_impl.dart @@ -3,9 +3,9 @@ library angular.core.parser.dynamic_parser_impl; import 'package:angular/core/parser/parser.dart' show ParserBackend; import 'package:angular/core/parser/lexer.dart'; import 'package:angular/core/parser/syntax.dart'; +import 'package:angular/core/parser/characters.dart'; class DynamicParserImpl { - static Token EOF = new Token(-1, null); final ParserBackend backend; final String input; final List tokens; @@ -15,26 +15,28 @@ class DynamicParserImpl { : this.input = input, tokens = lexer.call(input); Token get peek { - return (index < tokens.length) ? tokens[index] : EOF; + return (index < tokens.length) ? tokens[index] : Token.EOF; } parseChain() { bool isChain = false; - while (optional(';')) { + while (optionalCharacter($SEMICOLON)) { isChain = true; } List expressions = []; while (index < tokens.length) { - if (peek.text == ')' || peek.text == '}' || peek.text == ']') { - error('Unconsumed token ${peek.text}'); + if (peek.isCharacter($RPAREN) || + peek.isCharacter($RBRACE) || + peek.isCharacter($RBRACKET)) { + error('Unconsumed token $peek'); } var expr = parseFilter(); expressions.add(expr); - while (optional(';')) { + while (optionalCharacter($SEMICOLON)) { isChain = true; } if (isChain && expr is Filter) { - error('cannot have a filter in a chain'); + error('Cannot have a filter in a chain'); } } return (expressions.length == 1) @@ -44,11 +46,10 @@ class DynamicParserImpl { parseFilter() { var result = parseExpression(); - while (optional('|')) { - String name = peek.text; // TODO(kasperl): Restrict to identifier? - advance(); + while (optionalOperator('|')) { + String name = expectIdentifierOrKeyword(); List arguments = []; - while (optional(':')) { + while (optionalCharacter($COLON)) { // TODO(kasperl): Is this really supposed to be expressions? arguments.add(parseExpression()); } @@ -60,13 +61,13 @@ class DynamicParserImpl { parseExpression() { int start = peek.index; var result = parseConditional(); - while (peek.text == '=') { + while (peek.isOperator('=')) { if (!backend.isAssignable(result)) { int end = (index < tokens.length) ? peek.index : input.length; String expression = input.substring(start, end); error('Expression $expression is not assignable'); } - expect('='); + expectOperator('='); result = backend.newAssign(result, parseConditional()); } return result; @@ -75,9 +76,9 @@ class DynamicParserImpl { parseConditional() { int start = peek.index; var result = parseLogicalOr(); - if (optional('?')) { + if (optionalOperator('?')) { var yes = parseExpression(); - if (!optional(':')) { + if (!optionalCharacter($COLON)) { int end = (index < tokens.length) ? peek.index : input.length; String expression = input.substring(start, end); error('Conditional expression $expression requires all 3 expressions'); @@ -91,7 +92,7 @@ class DynamicParserImpl { parseLogicalOr() { // '||' var result = parseLogicalAnd(); - while (optional('||')) { + while (optionalOperator('||')) { result = backend.newBinaryLogicalOr(result, parseLogicalAnd()); } return result; @@ -100,7 +101,7 @@ class DynamicParserImpl { parseLogicalAnd() { // '&&' var result = parseEquality(); - while (optional('&&')) { + while (optionalOperator('&&')) { result = backend.newBinaryLogicalAnd(result, parseEquality()); } return result; @@ -110,9 +111,9 @@ class DynamicParserImpl { // '==','!=' var result = parseRelational(); while (true) { - if (optional('==')) { + if (optionalOperator('==')) { result = backend.newBinaryEqual(result, parseRelational()); - } else if (optional('!=')) { + } else if (optionalOperator('!=')) { result = backend.newBinaryNotEqual(result, parseRelational()); } else { return result; @@ -124,13 +125,13 @@ class DynamicParserImpl { // '<', '>', '<=', '>=' var result = parseAdditive(); while (true) { - if (optional('<')) { + if (optionalOperator('<')) { result = backend.newBinaryLessThan(result, parseAdditive()); - } else if (optional('>')) { + } else if (optionalOperator('>')) { result = backend.newBinaryGreaterThan(result, parseAdditive()); - } else if (optional('<=')) { + } else if (optionalOperator('<=')) { result = backend.newBinaryLessThanEqual(result, parseAdditive()); - } else if (optional('>=')) { + } else if (optionalOperator('>=')) { result = backend.newBinaryGreaterThanEqual(result, parseAdditive()); } else { return result; @@ -142,9 +143,9 @@ class DynamicParserImpl { // '+', '-' var result = parseMultiplicative(); while (true) { - if (optional('+')) { + if (optionalOperator('+')) { result = backend.newBinaryPlus(result, parseMultiplicative()); - } else if (optional('-')) { + } else if (optionalOperator('-')) { result = backend.newBinaryMinus(result, parseMultiplicative()); } else { return result; @@ -156,13 +157,13 @@ class DynamicParserImpl { // '*', '%', '/', '~/' var result = parsePrefix(); while (true) { - if (optional('*')) { + if (optionalOperator('*')) { result = backend.newBinaryMultiply(result, parsePrefix()); - } else if (optional('%')) { + } else if (optionalOperator('%')) { result = backend.newBinaryModulo(result, parsePrefix()); - } else if (optional('/')) { + } else if (optionalOperator('/')) { result = backend.newBinaryDivide(result, parsePrefix()); - } else if (optional('~/')) { + } else if (optionalOperator('~/')) { result = backend.newBinaryTruncatingDivide(result, parsePrefix()); } else { return result; @@ -171,12 +172,12 @@ class DynamicParserImpl { } parsePrefix() { - if (optional('+')) { + if (optionalOperator('+')) { // TODO(kasperl): This is different than the original parser. return backend.newPrefixPlus(parsePrefix()); - } else if (optional('-')) { + } else if (optionalOperator('-')) { return backend.newPrefixMinus(parsePrefix()); - } else if (optional('!')) { + } else if (optionalOperator('!')) { return backend.newPrefixNot(parsePrefix()); } else { return parseAccessOrCallMember(); @@ -186,24 +187,22 @@ class DynamicParserImpl { parseAccessOrCallMember() { var result = parsePrimary(); while (true) { - if (optional('.')) { - // TODO(kasperl): Check that this is an identifier. Are keywords okay? - String name = peek.text; - advance(); - if (optional('(')) { - List arguments = parseExpressionList(')'); - expect(')'); + if (optionalCharacter($PERIOD)) { + String name = expectIdentifierOrKeyword(); + if (optionalCharacter($LPAREN)) { + List arguments = parseExpressionList($RPAREN); + expectCharacter($RPAREN); result = backend.newCallMember(result, name, arguments); } else { result = backend.newAccessMember(result, name); } - } else if (optional('[')) { + } else if (optionalCharacter($LBRACKET)) { var key = parseExpression(); - expect(']'); + expectCharacter($RBRACKET); result = backend.newAccessKeyed(result, key); - } else if (optional('(')) { - List arguments = parseExpressionList(')'); - expect(')'); + } else if (optionalCharacter($LPAREN)) { + List arguments = parseExpressionList($RPAREN); + expectCharacter($RPAREN); result = backend.newCallFunction(result, arguments); } else { return result; @@ -212,77 +211,78 @@ class DynamicParserImpl { } parsePrimary() { - if (optional('(')) { + if (optionalCharacter($LPAREN)) { var result = parseExpression(); - expect(')'); + expectCharacter($RPAREN); return result; - } else if (optional('null') || optional('undefined')) { + } else if (peek.isKeywordNull || peek.isKeywordUndefined) { + advance(); return backend.newLiteralNull(); - } else if (optional('true')) { + } else if (peek.isKeywordTrue) { + advance(); return backend.newLiteralBoolean(true); - } else if (optional('false')) { + } else if (peek.isKeywordFalse) { + advance(); return backend.newLiteralBoolean(false); - } else if (optional('[')) { - List elements = parseExpressionList(']'); - expect(']'); + } else if (optionalCharacter($LBRACKET)) { + List elements = parseExpressionList($RBRACKET); + expectCharacter($RBRACKET); return backend.newLiteralArray(elements); - } else if (peek.text == '{') { + } else if (peek.isCharacter($LBRACE)) { return parseObject(); - } else if (peek.key != null) { + } else if (peek.isIdentifier) { return parseAccessOrCallScope(); - } else if (peek.value != null) { - var value = peek.value; + } else if (peek.isNumber) { + num value = peek.toNumber(); + advance(); + return backend.newLiteralNumber(value); + } else if (peek.isString) { + String value = peek.toString(); advance(); - return (value is num) - ? backend.newLiteralNumber(value) - : backend.newLiteralString(value); + return backend.newLiteralString(value); } else if (index >= tokens.length) { throw 'Unexpected end of expression: $input'; } else { - error('Unexpected token ${peek.text}'); + error('Unexpected token $peek'); } } parseAccessOrCallScope() { - String name = peek.key; - advance(); - if (!optional('(')) return backend.newAccessScope(name); - List arguments = parseExpressionList(')'); - expect(')'); + String name = expectIdentifierOrKeyword(); + if (!optionalCharacter($LPAREN)) return backend.newAccessScope(name); + List arguments = parseExpressionList($RPAREN); + expectCharacter($RPAREN); return backend.newCallScope(name, arguments); } parseObject() { List keys = []; List values = []; - expect('{'); - if (peek.text != '}') { + expectCharacter($LBRACE); + if (!optionalCharacter($RBRACE)) { do { - // TODO(kasperl): Stricter checking. Only allow identifiers - // and strings as keys. Maybe also keywords? - var value = peek.value; - keys.add(value is String ? value : peek.text); - advance(); - expect(':'); + String key = expectIdentifierOrKeywordOrString(); + keys.add(key); + expectCharacter($COLON); values.add(parseExpression()); - } while (optional(',')); + } while (optionalCharacter($COMMA)); + expectCharacter($RBRACE); } - expect('}'); return backend.newLiteralObject(keys, values); } - List parseExpressionList(String terminator) { + List parseExpressionList(int terminator) { List result = []; - if (peek.text != terminator) { + if (!peek.isCharacter(terminator)) { do { result.add(parseExpression()); - } while (optional(',')); + } while (optionalCharacter($COMMA)); } return result; } - bool optional(text) { - if (peek.text == text) { + bool optionalCharacter(int code) { + if (peek.isCharacter(code)) { advance(); return true; } else { @@ -290,12 +290,41 @@ class DynamicParserImpl { } } - void expect(text) { - if (peek.text == text) { + bool optionalOperator(String operator) { + if (peek.isOperator(operator)) { advance(); + return true; } else { - error('Missing expected $text'); + return false; + } + } + + void expectCharacter(int code) { + if (optionalCharacter(code)) return; + error('Missing expected ${new String.fromCharCode(code)}'); + } + + void expectOperator(String operator) { + if (optionalOperator(operator)) return; + error('Missing expected operator $operator'); + } + + String expectIdentifierOrKeyword() { + if (!peek.isIdentifier && !peek.isKeyword) { + error('Unexpected token $peek, expected identifier or keyword'); } + String result = peek.toString(); + advance(); + return result; + } + + String expectIdentifierOrKeywordOrString() { + if (!peek.isIdentifier && !peek.isKeyword && !peek.isString) { + error('Unexpected token $peek, expected identifier, keyword, or string'); + } + String result = peek.toString(); + advance(); + return result; } void advance() { diff --git a/lib/core/parser/lexer.dart b/lib/core/parser/lexer.dart index 3361c8f53..5ad803489 100644 --- a/lib/core/parser/lexer.dart +++ b/lib/core/parser/lexer.dart @@ -3,30 +3,7 @@ library angular.core.parser.lexer; import 'package:angular/core/module.dart' show NgInjectableService; import 'package:angular/core/parser/characters.dart'; -class Token { - final int index; - final String text; - - var value; - // Tokens should have one of these set. - String opKey; - String key; - - Token(this.index, this.text); - - withOp(op) { - this.opKey = op; - } - - withGetterSetter(key) { - this.key = key; - } - - withValue(value) { this.value = value; } - - toString() => "Token($text)"; -} - +part 'tokens.dart'; @NgInjectableService() class Lexer { @@ -72,7 +49,7 @@ class Scanner { switch (peek) { case $PERIOD: advance(); - return isDigit(peek) ? scanNumber(start) : new Token(start, '.'); + return isDigit(peek) ? scanNumber(start) : new CharacterToken(start, $PERIOD); case $LPAREN: case $RPAREN: case $LBRACE: @@ -82,7 +59,7 @@ class Scanner { case $COMMA: case $COLON: case $SEMICOLON: - return scanCharacter(start, new String.fromCharCode(peek)); + return scanCharacter(start, peek); case $SQ: case $DQ: return scanString(); @@ -115,17 +92,17 @@ class Scanner { return null; } - Token scanCharacter(int start, String string) { - assert(peek == string.codeUnitAt(0)); + Token scanCharacter(int start, int code) { + assert(peek == code); advance(); - return new Token(start, string); + return new CharacterToken(start, code); } Token scanOperator(int start, String string) { assert(peek == string.codeUnitAt(0)); assert(OPERATORS.contains(string)); advance(); - return new Token(start, string)..withOp(string); + return new OperatorToken(start, string); } Token scanComplexOperator(int start, int code, String one, String two) { @@ -137,7 +114,7 @@ class Scanner { string += two; } assert(OPERATORS.contains(string)); - return new Token(start, string)..withOp(string); + return new OperatorToken(start, string); } Token scanIdentifier() { @@ -146,15 +123,7 @@ class Scanner { advance(); while (isIdentifierPart(peek)) advance(); String string = input.substring(start, index); - Token result = new Token(start, string); - // TODO(kasperl): Deal with null, undefined, true, and false in - // a cleaner and faster way. - if (OPERATORS.contains(string)) { - result.withOp(string); - } else { - result.withGetterSetter(string); - } - return result; + return new IdentifierToken(start, string, KEYWORDS.contains(string)); } Token scanNumber(int start) { @@ -178,7 +147,7 @@ class Scanner { } String string = input.substring(start, index); num value = simple ? int.parse(string) : double.parse(string); - return new Token(start, string)..withValue(value); + return new NumberToken(start, value); } Token scanString() { @@ -226,7 +195,7 @@ class Scanner { buffer.write(last); unescaped = buffer.toString(); } - return new Token(start, string)..withValue(unescaped); + return new StringToken(start, string, unescaped); } void advance() { @@ -242,11 +211,14 @@ class Scanner { } } -Set OPERATORS = new Set.from([ - 'undefined', +Set KEYWORDS = new Set.from([ 'null', + 'undefined', 'true', 'false', +]); + +Set OPERATORS = new Set.from([ '+', '-', '*', diff --git a/lib/core/parser/tokens.dart b/lib/core/parser/tokens.dart new file mode 100644 index 000000000..b3b5bf18f --- /dev/null +++ b/lib/core/parser/tokens.dart @@ -0,0 +1,65 @@ +part of angular.core.parser.lexer; + +class Token { + static const Token EOF = const Token._(-1); + final int index; + const Token._(this.index); + + bool get isIdentifier => false; + bool get isString => false; + bool get isNumber => false; + + bool isCharacter(int code) => false; + bool isOperator(String operator) => false; + + bool get isKeyword => false; + bool get isKeywordNull => false; + bool get isKeywordUndefined => false; + bool get isKeywordTrue => false; + bool get isKeywordFalse => false; + + num toNumber() => null; +} + +class CharacterToken extends Token { + final int _code; + CharacterToken(int index, this._code) : super._(index); + bool isCharacter(int code) => _code == code; + String toString() => new String.fromCharCode(_code); +} + +class IdentifierToken extends Token { + final String _text; + final bool _isKeyword; + IdentifierToken(int index, this._text, this._isKeyword) : super._(index); + bool get isIdentifier => !_isKeyword; + bool get isKeyword => _isKeyword; + bool get isKeywordNull => _isKeyword && _text == "null"; + bool get isKeywordUndefined => _isKeyword && _text == "undefined"; + bool get isKeywordTrue => _isKeyword && _text == "true"; + bool get isKeywordFalse => _isKeyword && _text == "false"; + String toString() => _text; +} + +class OperatorToken extends Token { + final String _text; + OperatorToken(int index, this._text) : super._(index); + bool isOperator(String operator) => _text == operator; + String toString() => _text; +} + +class NumberToken extends Token { + final num _value; + NumberToken(int index, this._value) : super._(index); + bool get isNumber => true; + num toNumber() => _value; + String toString() => "$_value"; +} + +class StringToken extends Token { + final String input; + final String _value; + StringToken(int index, this.input, this._value) : super._(index); + bool get isString => true; + String toString() => _value; +} diff --git a/test/core/parser/lexer_spec.dart b/test/core/parser/lexer_spec.dart index 7844268e2..219c6f9fb 100644 --- a/test/core/parser/lexer_spec.dart +++ b/test/core/parser/lexer_spec.dart @@ -4,12 +4,50 @@ import '../../_specs.dart'; class LexerExpect extends Expect { LexerExpect(actual) : super(actual); - toBeToken(int index, String text) { + + toBeToken(int index) { expect(actual is Token).toEqual(true); expect(actual.index).toEqual(index); - expect(actual.text).toEqual(text); + } + + toBeCharacterToken(int index, String character) { + toBeToken(index); + expect(character.length).toEqual(1); + expect(actual.isCharacter(character.codeUnitAt(0))).toEqual(true); + } + + toBeIdentifierToken(int index, String text) { + toBeToken(index); + expect(actual.isIdentifier).toEqual(true); + expect(actual.toString()).toEqual(text); + } + + toBeKeywordUndefinedToken(int index) { + toBeToken(index); + expect(actual.isKeywordUndefined).toEqual(true); + } + + toBeOperatorToken(int index, String operator) { + toBeToken(index); + expect(actual.isOperator(operator)).toEqual(true); + } + + toBeStringToken(int index, String input, String value) { + toBeToken(index); + expect(actual.isString).toEqual(true); + StringToken token = actual; + expect(token.input).toEqual(input); + expect(token.toString()).toEqual(value); + } + + toBeNumberToken(int index, num value) { + toBeToken(index); + expect(actual.isNumber).toEqual(true); + NumberToken token = actual; + expect(token.toNumber()).toEqual(value); } } + expect(actual) => new LexerExpect(actual); main() { @@ -21,204 +59,199 @@ main() { // New test case it('should tokenize a simple identifier', () { - var tokens = lex("j"); + List tokens = lex("j"); expect(tokens.length).toEqual(1); - expect(tokens[0]).toBeToken(0, 'j'); + expect(tokens[0]).toBeIdentifierToken(0, 'j'); }); // New test case it('should tokenize a dotted identifier', () { - var tokens = lex("j.k"); + List tokens = lex("j.k"); expect(tokens.length).toEqual(3); - expect(tokens[0]).toBeToken(0, 'j'); - expect(tokens[1]).toBeToken(1, '.'); - expect(tokens[2]).toBeToken(2, 'k'); + expect(tokens[0]).toBeIdentifierToken(0, 'j'); + expect(tokens[1]).toBeCharacterToken(1, '.'); + expect(tokens[2]).toBeIdentifierToken(2, 'k'); }); it('should tokenize an operator', () { - var tokens = lex('j-k'); + List tokens = lex('j-k'); expect(tokens.length).toEqual(3); - expect(tokens[1]).toBeToken(1, '-'); + expect(tokens[1]).toBeOperatorToken(1, '-'); }); it('should tokenize an indexed operator', () { - var tokens = lex('j[k]'); + List tokens = lex('j[k]'); expect(tokens.length).toEqual(4); - expect(tokens[1]).toBeToken(1, '['); + expect(tokens[1]).toBeCharacterToken(1, '['); }); it('should tokenize numbers', () { - var tokens = lex('88'); + List tokens = lex('88'); expect(tokens.length).toEqual(1); - expect(tokens[0]).toBeToken(0, '88'); + expect(tokens[0]).toBeNumberToken(0, 88); }); it('should tokenize numbers within index ops', () { - expect(lex('a[22]')[2]).toBeToken(2, '22'); + expect(lex('a[22]')[2]).toBeNumberToken(2, 22); }); it('should tokenize simple quoted strings', () { - expect(lex('"a"')[0]).toBeToken(0, '"a"'); + expect(lex('"a"')[0]).toBeStringToken(0, '"a"', 'a'); }); it('should tokenize quoted strings with escaped quotes', () { - expect(lex('"a\\""')[0]).toBeToken(0, '"a\\""'); + expect(lex('"a\\""')[0]).toBeStringToken(0, '"a\\""', 'a"'); }); it('should tokenize a string', () { - var tokens = lex("j-a.bc[22]+1.3|f:'a\\\'c':\"d\\\"e\""); + List tokens = lex("j-a.bc[22]+1.3|f:'a\\\'c':\"d\\\"e\""); var i = 0; - expect(tokens[i]).toBeToken(0, 'j'); + expect(tokens[i]).toBeIdentifierToken(0, 'j'); i++; - expect(tokens[i]).toBeToken(1, '-'); + expect(tokens[i]).toBeOperatorToken(1, '-'); i++; - expect(tokens[i]).toBeToken(2, 'a'); + expect(tokens[i]).toBeIdentifierToken(2, 'a'); i++; - expect(tokens[i]).toBeToken(3, '.'); + expect(tokens[i]).toBeCharacterToken(3, '.'); i++; - expect(tokens[i]).toBeToken(4, 'bc'); + expect(tokens[i]).toBeIdentifierToken(4, 'bc'); i++; - expect(tokens[i]).toBeToken(6, '['); + expect(tokens[i]).toBeCharacterToken(6, '['); i++; - expect(tokens[i]).toBeToken(7, '22'); + expect(tokens[i]).toBeNumberToken(7, 22); i++; - expect(tokens[i]).toBeToken(9, ']'); + expect(tokens[i]).toBeCharacterToken(9, ']'); i++; - expect(tokens[i]).toBeToken(10, '+'); + expect(tokens[i]).toBeOperatorToken(10, '+'); i++; - expect(tokens[i]).toBeToken(11, '1.3'); + expect(tokens[i]).toBeNumberToken(11, 1.3); i++; - expect(tokens[i]).toBeToken(14, '|'); + expect(tokens[i]).toBeOperatorToken(14, '|'); i++; - expect(tokens[i]).toBeToken(15, 'f'); + expect(tokens[i]).toBeIdentifierToken(15, 'f'); i++; - expect(tokens[i]).toBeToken(16, ':'); + expect(tokens[i]).toBeCharacterToken(16, ':'); i++; - expect(tokens[i]).toBeToken(17, '\'a\\\'c\''); + expect(tokens[i]).toBeStringToken(17, "'a\\'c'", "a'c"); i++; - expect(tokens[i]).toBeToken(23, ':'); + expect(tokens[i]).toBeCharacterToken(23, ':'); i++; - expect(tokens[i]).toBeToken(24, '"d\\"e"'); + expect(tokens[i]).toBeStringToken(24, '"d\\"e"', 'd"e'); }); it('should tokenize undefined', () { - var tokens = lex("undefined"); + List tokens = lex("undefined"); var i = 0; - expect(tokens[i]).toBeToken(0, 'undefined'); - expect(tokens[i].value).toEqual(null); + expect(tokens[i]).toBeKeywordUndefinedToken(0); }); it('should ignore whitespace', () { - var tokens = lex("a \t \n \r b"); - expect(tokens[0].text).toEqual('a'); - expect(tokens[1].text).toEqual('b'); + List tokens = lex("a \t \n \r b"); + expect(tokens[0]).toBeIdentifierToken(0, 'a'); + expect(tokens[1]).toBeIdentifierToken(8, 'b'); }); it('should tokenize quoted string', () { var str = "['\\'', \"\\\"\"]"; - var tokens = lex(str); - - expect(tokens[1].index).toEqual(1); - expect(tokens[1].value).toEqual("'"); - - expect(tokens[3].index).toEqual(7); - expect(tokens[3].value).toEqual('"'); + List tokens = lex(str); + expect(tokens[1]).toBeStringToken(1, "'\\''", "'"); + expect(tokens[3]).toBeStringToken(7, '"\\""', '"'); }); it('should tokenize escaped quoted string', () { var str = '"\\"\\n\\f\\r\\t\\v\\u00A0"'; - var tokens = lex(str); - - expect(tokens[0].value).toEqual('"\n\f\r\t\v\u00A0'); + List tokens = lex(str); + expect(tokens.length).toEqual(1); + expect(tokens[0].toString()).toEqual('"\n\f\r\t\v\u00A0'); }); it('should tokenize unicode', () { - var tokens = lex('"\\u00A0"'); + List tokens = lex('"\\u00A0"'); expect(tokens.length).toEqual(1); - expect(tokens[0].value).toEqual('\u00a0'); + expect(tokens[0].toString()).toEqual('\u00a0'); }); it('should tokenize relation', () { - var tokens = lex("! == != < > <= >="); - expect(tokens[0].text).toEqual('!'); - expect(tokens[1].text).toEqual('=='); - expect(tokens[2].text).toEqual('!='); - expect(tokens[3].text).toEqual('<'); - expect(tokens[4].text).toEqual('>'); - expect(tokens[5].text).toEqual('<='); - expect(tokens[6].text).toEqual('>='); + List tokens = lex("! == != < > <= >="); + expect(tokens[0]).toBeOperatorToken(0, '!'); + expect(tokens[1]).toBeOperatorToken(2, '=='); + expect(tokens[2]).toBeOperatorToken(5, '!='); + expect(tokens[3]).toBeOperatorToken(8, '<'); + expect(tokens[4]).toBeOperatorToken(10, '>'); + expect(tokens[5]).toBeOperatorToken(12, '<='); + expect(tokens[6]).toBeOperatorToken(15, '>='); }); it('should tokenize statements', () { - var tokens = lex("a;b;"); - expect(tokens[0].text).toEqual('a'); - expect(tokens[1].text).toEqual(';'); - expect(tokens[2].text).toEqual('b'); - expect(tokens[3].text).toEqual(';'); + List tokens = lex("a;b;"); + expect(tokens[0]).toBeIdentifierToken(0, 'a'); + expect(tokens[1]).toBeCharacterToken(1, ';'); + expect(tokens[2]).toBeIdentifierToken(2, 'b'); + expect(tokens[3]).toBeCharacterToken(3, ';'); }); it('should tokenize function invocation', () { - var tokens = lex("a()"); - expect(tokens[0]).toBeToken(0, 'a'); - expect(tokens[1]).toBeToken(1, '('); - expect(tokens[2]).toBeToken(2, ')'); + List tokens = lex("a()"); + expect(tokens[0]).toBeIdentifierToken(0, 'a'); + expect(tokens[1]).toBeCharacterToken(1, '('); + expect(tokens[2]).toBeCharacterToken(2, ')'); }); it('should tokenize simple method invocations', () { - var tokens = lex("a.method()"); - expect(tokens[2]).toBeToken(2, 'method'); + List tokens = lex("a.method()"); + expect(tokens[2]).toBeIdentifierToken(2, 'method'); }); it('should tokenize method invocation', () { - var tokens = lex("a.b.c (d) - e.f()"); - expect(tokens[0]).toBeToken(0, 'a'); - expect(tokens[1]).toBeToken(1, '.'); - expect(tokens[2]).toBeToken(2, 'b'); - expect(tokens[3]).toBeToken(3, '.'); - expect(tokens[4]).toBeToken(4, 'c'); - expect(tokens[5]).toBeToken(6, '('); - expect(tokens[6]).toBeToken(7, 'd'); - expect(tokens[7]).toBeToken(8, ')'); - expect(tokens[8]).toBeToken(10, '-'); - expect(tokens[9]).toBeToken(12, 'e'); - expect(tokens[10]).toBeToken(13, '.'); - expect(tokens[11]).toBeToken(14, 'f'); - expect(tokens[12]).toBeToken(15, '('); - expect(tokens[13]).toBeToken(16, ')'); + List tokens = lex("a.b.c (d) - e.f()"); + expect(tokens[0]).toBeIdentifierToken(0, 'a'); + expect(tokens[1]).toBeCharacterToken(1, '.'); + expect(tokens[2]).toBeIdentifierToken(2, 'b'); + expect(tokens[3]).toBeCharacterToken(3, '.'); + expect(tokens[4]).toBeIdentifierToken(4, 'c'); + expect(tokens[5]).toBeCharacterToken(6, '('); + expect(tokens[6]).toBeIdentifierToken(7, 'd'); + expect(tokens[7]).toBeCharacterToken(8, ')'); + expect(tokens[8]).toBeOperatorToken(10, '-'); + expect(tokens[9]).toBeIdentifierToken(12, 'e'); + expect(tokens[10]).toBeCharacterToken(13, '.'); + expect(tokens[11]).toBeIdentifierToken(14, 'f'); + expect(tokens[12]).toBeCharacterToken(15, '('); + expect(tokens[13]).toBeCharacterToken(16, ')'); }); it('should tokenize number', () { - var tokens = lex("0.5"); - expect(tokens[0].value).toEqual(0.5); + List tokens = lex("0.5"); + expect(tokens[0]).toBeNumberToken(0, 0.5); }); // NOTE(deboer): NOT A LEXER TEST // it('should tokenize negative number', () { - // var tokens = lex("-0.5"); - // expect(tokens[0].value).toEqual(-0.5); + // List tokens = lex("-0.5"); + // expect(tokens[0]).toBeNumberToken(0, -0.5); // }); it('should tokenize number with exponent', () { - var tokens = lex("0.5E-10"); + List tokens = lex("0.5E-10"); expect(tokens.length).toEqual(1); - expect(tokens[0].value).toEqual(0.5E-10); + expect(tokens[0]).toBeNumberToken(0, 0.5E-10); tokens = lex("0.5E+10"); - expect(tokens[0].value).toEqual(0.5E+10); + expect(tokens[0]).toBeNumberToken(0, 0.5E+10); }); it('should throws exception for invalid exponent', () { @@ -232,8 +265,8 @@ main() { }); it('should tokenize number starting with a dot', () { - var tokens = lex(".5"); - expect(tokens[0].value).toEqual(0.5); + List tokens = lex(".5"); + expect(tokens[0]).toBeNumberToken(0, 0.5); }); it('should throw error on invalid unicode', () { diff --git a/test/core/parser/parser_spec.dart b/test/core/parser/parser_spec.dart index 1441a93e5..6e1003a8a 100644 --- a/test/core/parser/parser_spec.dart +++ b/test/core/parser/parser_spec.dart @@ -324,6 +324,7 @@ main() { }).toThrow("notAProperty"); }); + it('should fail on private field access', () { expect(parser('publicField').eval(new WithPrivateField())).toEqual(4); // On Dartium, this fails with "NoSuchMethod: no instance getter" @@ -333,6 +334,26 @@ main() { parser('_privateField').eval(new WithPrivateField()); }).toThrow(); }); + + + it('should only allow identifier or keyword as filter names', () { + expect(() => parser('"Foo"|(')).toThrow('identifier or keyword'); + expect(() => parser('"Foo"|1234')).toThrow('identifier or keyword'); + expect(() => parser('"Foo"|"uppercase"')).toThrow('identifier or keyword'); + }); + + + it('should only allow identifier or keyword as member names', () { + expect(() => parser('x.(')).toThrow('identifier or keyword'); + expect(() => parser('x. 1234')).toThrow('identifier or keyword'); + expect(() => parser('x."foo"')).toThrow('identifier or keyword'); + }); + + + it('should only allow identifier, string, or keyword as object literal key', () { + expect(() => parser('{(:0}')).toThrow('expected identifier, keyword, or string'); + expect(() => parser('{1234:0}')).toThrow('expected identifier, keyword, or string'); + }); }); describe('setters', () { @@ -997,10 +1018,10 @@ main() { it('should not allow filters in a chain', () { expect(() { parser("1;'World'|hello"); - }).toThrow('cannot have a filter in a chain the end of the expression [1;\'World\'|hello]'); + }).toThrow('Cannot have a filter in a chain the end of the expression [1;\'World\'|hello]'); expect(() { parser("'World'|hello;1"); - }).toThrow('cannot have a filter in a chain at column 15 in [\'World\'|hello;1]'); + }).toThrow('Cannot have a filter in a chain at column 15 in [\'World\'|hello;1]'); }); }); });