Skip to content
This repository was archived by the owner on Feb 22, 2018. It is now read-only.

chore(lexer): Introduce different classes of tokens. #711

Closed
wants to merge 1 commit into from
Closed

chore(lexer): Introduce different classes of tokens. #711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions bin/parser_generator_for_spec.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,5 +434,14 @@ main(arguments) {
"o.void()",
"o.while()",
"o.with()",

'"Foo"|(',
'"Foo"|1234',
'"Foo"|"uppercase"',
'x.(',
'x. 1234',
'x."foo"',
'{(:0}',
'{1234:0}',
]);
}
195 changes: 112 additions & 83 deletions lib/core/parser/dynamic_parser_impl.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ library angular.core.parser.dynamic_parser_impl;
import 'package:angular/core/parser/parser.dart' show ParserBackend;
import 'package:angular/core/parser/lexer.dart';
import 'package:angular/core/parser/syntax.dart';
import 'package:angular/core/parser/characters.dart';

class DynamicParserImpl {
static Token EOF = new Token(-1, null);
final ParserBackend backend;
final String input;
final List<Token> tokens;
Expand All @@ -15,26 +15,28 @@ class DynamicParserImpl {
: this.input = input, tokens = lexer.call(input);

Token get peek {
return (index < tokens.length) ? tokens[index] : EOF;
return (index < tokens.length) ? tokens[index] : Token.EOF;
}

parseChain() {
bool isChain = false;
while (optional(';')) {
while (optionalCharacter($SEMICOLON)) {
isChain = true;
}
List expressions = [];
while (index < tokens.length) {
if (peek.text == ')' || peek.text == '}' || peek.text == ']') {
error('Unconsumed token ${peek.text}');
if (peek.isCharacter($RPAREN) ||
peek.isCharacter($RBRACE) ||
peek.isCharacter($RBRACKET)) {
error('Unconsumed token $peek');
}
var expr = parseFilter();
expressions.add(expr);
while (optional(';')) {
while (optionalCharacter($SEMICOLON)) {
isChain = true;
}
if (isChain && expr is Filter) {
error('cannot have a filter in a chain');
error('Cannot have a filter in a chain');
}
}
return (expressions.length == 1)
Expand All @@ -44,11 +46,10 @@ class DynamicParserImpl {

parseFilter() {
var result = parseExpression();
while (optional('|')) {
String name = peek.text; // TODO(kasperl): Restrict to identifier?
advance();
while (optionalOperator('|')) {
String name = expectIdentifierOrKeyword();
List arguments = [];
while (optional(':')) {
while (optionalCharacter($COLON)) {
// TODO(kasperl): Is this really supposed to be expressions?
arguments.add(parseExpression());
}
Expand All @@ -60,13 +61,13 @@ class DynamicParserImpl {
parseExpression() {
int start = peek.index;
var result = parseConditional();
while (peek.text == '=') {
while (peek.isOperator('=')) {
if (!backend.isAssignable(result)) {
int end = (index < tokens.length) ? peek.index : input.length;
String expression = input.substring(start, end);
error('Expression $expression is not assignable');
}
expect('=');
expectOperator('=');
result = backend.newAssign(result, parseConditional());
}
return result;
Expand All @@ -75,9 +76,9 @@ class DynamicParserImpl {
parseConditional() {
int start = peek.index;
var result = parseLogicalOr();
if (optional('?')) {
if (optionalOperator('?')) {
var yes = parseExpression();
if (!optional(':')) {
if (!optionalCharacter($COLON)) {
int end = (index < tokens.length) ? peek.index : input.length;
String expression = input.substring(start, end);
error('Conditional expression $expression requires all 3 expressions');
Expand All @@ -91,7 +92,7 @@ class DynamicParserImpl {
parseLogicalOr() {
// '||'
var result = parseLogicalAnd();
while (optional('||')) {
while (optionalOperator('||')) {
result = backend.newBinaryLogicalOr(result, parseLogicalAnd());
}
return result;
Expand All @@ -100,7 +101,7 @@ class DynamicParserImpl {
parseLogicalAnd() {
// '&&'
var result = parseEquality();
while (optional('&&')) {
while (optionalOperator('&&')) {
result = backend.newBinaryLogicalAnd(result, parseEquality());
}
return result;
Expand All @@ -110,9 +111,9 @@ class DynamicParserImpl {
// '==','!='
var result = parseRelational();
while (true) {
if (optional('==')) {
if (optionalOperator('==')) {
result = backend.newBinaryEqual(result, parseRelational());
} else if (optional('!=')) {
} else if (optionalOperator('!=')) {
result = backend.newBinaryNotEqual(result, parseRelational());
} else {
return result;
Expand All @@ -124,13 +125,13 @@ class DynamicParserImpl {
// '<', '>', '<=', '>='
var result = parseAdditive();
while (true) {
if (optional('<')) {
if (optionalOperator('<')) {
result = backend.newBinaryLessThan(result, parseAdditive());
} else if (optional('>')) {
} else if (optionalOperator('>')) {
result = backend.newBinaryGreaterThan(result, parseAdditive());
} else if (optional('<=')) {
} else if (optionalOperator('<=')) {
result = backend.newBinaryLessThanEqual(result, parseAdditive());
} else if (optional('>=')) {
} else if (optionalOperator('>=')) {
result = backend.newBinaryGreaterThanEqual(result, parseAdditive());
} else {
return result;
Expand All @@ -142,9 +143,9 @@ class DynamicParserImpl {
// '+', '-'
var result = parseMultiplicative();
while (true) {
if (optional('+')) {
if (optionalOperator('+')) {
result = backend.newBinaryPlus(result, parseMultiplicative());
} else if (optional('-')) {
} else if (optionalOperator('-')) {
result = backend.newBinaryMinus(result, parseMultiplicative());
} else {
return result;
Expand All @@ -156,13 +157,13 @@ class DynamicParserImpl {
// '*', '%', '/', '~/'
var result = parsePrefix();
while (true) {
if (optional('*')) {
if (optionalOperator('*')) {
result = backend.newBinaryMultiply(result, parsePrefix());
} else if (optional('%')) {
} else if (optionalOperator('%')) {
result = backend.newBinaryModulo(result, parsePrefix());
} else if (optional('/')) {
} else if (optionalOperator('/')) {
result = backend.newBinaryDivide(result, parsePrefix());
} else if (optional('~/')) {
} else if (optionalOperator('~/')) {
result = backend.newBinaryTruncatingDivide(result, parsePrefix());
} else {
return result;
Expand All @@ -171,12 +172,12 @@ class DynamicParserImpl {
}

parsePrefix() {
if (optional('+')) {
if (optionalOperator('+')) {
// TODO(kasperl): This is different than the original parser.
return backend.newPrefixPlus(parsePrefix());
} else if (optional('-')) {
} else if (optionalOperator('-')) {
return backend.newPrefixMinus(parsePrefix());
} else if (optional('!')) {
} else if (optionalOperator('!')) {
return backend.newPrefixNot(parsePrefix());
} else {
return parseAccessOrCallMember();
Expand All @@ -186,24 +187,22 @@ class DynamicParserImpl {
parseAccessOrCallMember() {
var result = parsePrimary();
while (true) {
if (optional('.')) {
// TODO(kasperl): Check that this is an identifier. Are keywords okay?
String name = peek.text;
advance();
if (optional('(')) {
List arguments = parseExpressionList(')');
expect(')');
if (optionalCharacter($PERIOD)) {
String name = expectIdentifierOrKeyword();
if (optionalCharacter($LPAREN)) {
List arguments = parseExpressionList($RPAREN);
expectCharacter($RPAREN);
result = backend.newCallMember(result, name, arguments);
} else {
result = backend.newAccessMember(result, name);
}
} else if (optional('[')) {
} else if (optionalCharacter($LBRACKET)) {
var key = parseExpression();
expect(']');
expectCharacter($RBRACKET);
result = backend.newAccessKeyed(result, key);
} else if (optional('(')) {
List arguments = parseExpressionList(')');
expect(')');
} else if (optionalCharacter($LPAREN)) {
List arguments = parseExpressionList($RPAREN);
expectCharacter($RPAREN);
result = backend.newCallFunction(result, arguments);
} else {
return result;
Expand All @@ -212,90 +211,120 @@ class DynamicParserImpl {
}

parsePrimary() {
if (optional('(')) {
if (optionalCharacter($LPAREN)) {
var result = parseExpression();
expect(')');
expectCharacter($RPAREN);
return result;
} else if (optional('null') || optional('undefined')) {
} else if (peek.isKeywordNull || peek.isKeywordUndefined) {
advance();
return backend.newLiteralNull();
} else if (optional('true')) {
} else if (peek.isKeywordTrue) {
advance();
return backend.newLiteralBoolean(true);
} else if (optional('false')) {
} else if (peek.isKeywordFalse) {
advance();
return backend.newLiteralBoolean(false);
} else if (optional('[')) {
List elements = parseExpressionList(']');
expect(']');
} else if (optionalCharacter($LBRACKET)) {
List elements = parseExpressionList($RBRACKET);
expectCharacter($RBRACKET);
return backend.newLiteralArray(elements);
} else if (peek.text == '{') {
} else if (peek.isCharacter($LBRACE)) {
return parseObject();
} else if (peek.key != null) {
} else if (peek.isIdentifier) {
return parseAccessOrCallScope();
} else if (peek.value != null) {
var value = peek.value;
} else if (peek.isNumber) {
num value = peek.toNumber();
advance();
return backend.newLiteralNumber(value);
} else if (peek.isString) {
String value = peek.toString();
advance();
return (value is num)
? backend.newLiteralNumber(value)
: backend.newLiteralString(value);
return backend.newLiteralString(value);
} else if (index >= tokens.length) {
throw 'Unexpected end of expression: $input';
} else {
error('Unexpected token ${peek.text}');
error('Unexpected token $peek');
}
}

parseAccessOrCallScope() {
String name = peek.key;
advance();
if (!optional('(')) return backend.newAccessScope(name);
List arguments = parseExpressionList(')');
expect(')');
String name = expectIdentifierOrKeyword();
if (!optionalCharacter($LPAREN)) return backend.newAccessScope(name);
List arguments = parseExpressionList($RPAREN);
expectCharacter($RPAREN);
return backend.newCallScope(name, arguments);
}

parseObject() {
List<String> keys = [];
List values = [];
expect('{');
if (peek.text != '}') {
expectCharacter($LBRACE);
if (!optionalCharacter($RBRACE)) {
do {
// TODO(kasperl): Stricter checking. Only allow identifiers
// and strings as keys. Maybe also keywords?
var value = peek.value;
keys.add(value is String ? value : peek.text);
advance();
expect(':');
String key = expectIdentifierOrKeywordOrString();
keys.add(key);
expectCharacter($COLON);
values.add(parseExpression());
} while (optional(','));
} while (optionalCharacter($COMMA));
expectCharacter($RBRACE);
}
expect('}');
return backend.newLiteralObject(keys, values);
}

List parseExpressionList(String terminator) {
List parseExpressionList(int terminator) {
List result = [];
if (peek.text != terminator) {
if (!peek.isCharacter(terminator)) {
do {
result.add(parseExpression());
} while (optional(','));
} while (optionalCharacter($COMMA));
}
return result;
}

bool optional(text) {
if (peek.text == text) {
bool optionalCharacter(int code) {
if (peek.isCharacter(code)) {
advance();
return true;
} else {
return false;
}
}

void expect(text) {
if (peek.text == text) {
bool optionalOperator(String operator) {
if (peek.isOperator(operator)) {
advance();
return true;
} else {
error('Missing expected $text');
return false;
}
}

void expectCharacter(int code) {
if (optionalCharacter(code)) return;
error('Missing expected ${new String.fromCharCode(code)}');
}

void expectOperator(String operator) {
if (optionalOperator(operator)) return;
error('Missing expected operator $operator');
}

String expectIdentifierOrKeyword() {
if (!peek.isIdentifier && !peek.isKeyword) {
error('Unexpected token $peek, expected identifier or keyword');
}
String result = peek.toString();
advance();
return result;
}

String expectIdentifierOrKeywordOrString() {
if (!peek.isIdentifier && !peek.isKeyword && !peek.isString) {
error('Unexpected token $peek, expected identifier, keyword, or string');
}
String result = peek.toString();
advance();
return result;
}

void advance() {
Expand Down
Loading