From 4908e8287fd6b1514140fb20a305f1b122084004 Mon Sep 17 00:00:00 2001 From: Aliaksandr Zhuhrou Date: Wed, 20 Jun 2012 00:39:18 +0300 Subject: [PATCH] implement parser. --- CompilerLib.dart | 2 +- Lib.dart | 2 +- RuntimeLib.dart | 2 +- lib/parser.dart | 315 ++++++++++++++++++++++++++++++++++++----- test/scanner_test.dart | 2 +- 5 files changed, 286 insertions(+), 37 deletions(-) diff --git a/CompilerLib.dart b/CompilerLib.dart index 6df0ad6..15e0342 100644 --- a/CompilerLib.dart +++ b/CompilerLib.dart @@ -2,7 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -#library("hypcomm:edt:cmd"); +#library("desntech:edt:cmd"); #import("Lib.dart"); void main() { diff --git a/Lib.dart b/Lib.dart index 23ef459..f17dfbf 100644 --- a/Lib.dart +++ b/Lib.dart @@ -2,7 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -#library("hypcomm:edt"); +#library("desntech:edt"); #import("dart:io"); #source("lib/args.dart"); #source("lib/args_utils.dart"); diff --git a/RuntimeLib.dart b/RuntimeLib.dart index 99ab544..5727fa6 100644 --- a/RuntimeLib.dart +++ b/RuntimeLib.dart @@ -4,5 +4,5 @@ //An runtime library contains support classes that needed for execution //the template at runtime -#library("hypcomm:edt:runtime"); +#library("desntech:edt:runtime"); #source("runtime/template.dart"); \ No newline at end of file diff --git a/lib/parser.dart b/lib/parser.dart index 544223e..cf240e1 100644 --- a/lib/parser.dart +++ b/lib/parser.dart @@ -4,12 +4,11 @@ /** Parses a edt template */ class Parser { + /** The newline char code */ static final int NEW_LINE_CODE = 47; - /** numbers of symbols we need to consume in order to determine next action */ - static final int LOOK_AHEAD_SYMBOLS = 3; /** the current parser state */ - int _state = ParserStates.ANALYZING; + int _state = ParserStates.UNKNOWN; Parser() {} @@ -17,8 +16,234 @@ class Parser { List parse(String src) { var parsed = []; var lines = _toLines(src); - - } + var scanner = new Scanner(lines); + return _processTokenStream(scanner.tokenize()); + } + + /** Process a token stream */ + List _processTokenStream(List tokens) { + Iterator tokenIterator = tokens.iterator(); + Queue stack = new Queue(); + while (tokenIterator.hasNext()) { + Token token = tokenIterator.next(); + switch (_state) { + case ParserStates.UNKNOWN: + _processTokenInUnknownState(token, stack); + break; + case ParserStates.TEMPLATE: + _processTokenInTemplateState(token, stack); + break; + case ParserStates.INCLUDE: + _processTokenInIncludeState(token, stack); + break; + case ParserStates.CODE: + _processTokenInCodeState(token, stack); + break; + case ParserStates.ESCAPED_EXPRESSION: + _processTokenInEscapedExpressionState(token, stack); + break; + case ParserStates.UNESCAPED_EXPRESSION: + _processTokenInUnescapedExpressionState(token, stack); + break; + default: + throw new Exception("Illegal state: $_state"); + } + } + return _toList(stack); + } + + void _processTokenInUnknownState(Token token, Queue stack) { + if (token is TextToken) { + _state = ParserStates.TEXT; + stack.add(new TextFragment(token.content, token.line)); + } + else if (token is OpenIncludeToken) { + _state = ParserStates.INCLUDE; + stack.add(new IncludeFragment(token.line)); + } + else if (token is OpenCodeToken) { + _state = ParserStates.CODE; + stack.add(new CodeFragment(token.line)); + } + else if (token is OpenExpressionToken) { + _state = ParserStates.ESCAPED_EXPRESSION; + stack.add(new EscapedOutputFragment(token.line)); + } + else if (token is OpenUnescapedExpressionToken) { + _state = ParserStates.UNESCAPED_EXPRESSION; + stack.add(new UnescapedOutputFragment(token.line)); + } + else if (token is CloseToken) { + throw const ParseException(token.line, token); + } + else { + throw new IllegalArgumentException(token); + } + } + + void _processTokenInTemplateState(Token token, Queue stack) { + if (token is TextToken) { + throw new ParseException(token.line, token); + } + else if (token is OpenIncludeToken) { + _state = ParserStates.INCLUDE; + stack.add(new IncludeFragment(token.line)); + } + else if (token is OpenCodeToken) { + _state = ParserStates.CODE; + stack.add(new CodeFragment(token.line)); + } + else if (token is OpenExpressionToken) { + _state = ParserStates.ESCAPED_EXPRESSION; + stack.add(new EscapedOutputFragment(token.line)); + } + else if (token is OpenUnescapedExpressionToken) { + _state = ParserStates.UNESCAPED_EXPRESSION; + stack.add(new UnescapedOutputFragment(token.line)); + } + else if (token is CloseToken) { + throw const ParseException(token.line, token); + } + else { + throw new IllegalArgumentException(token); + } + } + + void _processTokenInIncludeState(Token token, Queue stack) { + if (token is TextToken) { + IncludeFragment includeToken = stack.last(); + if (includeToken.include != null) { + throw const ParseException(token.line, token); + } + includeToken.include = token.content; + } + else if (token is OpenIncludeToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenCodeToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenExpressionToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenUnescapedExpressionToken) { + throw const ParseException(token.line, token); + } + else if (token is CloseToken) { + IncludeFragment includeToken = stack.last(); + if (includeToken.include == null) { + throw const ParseException(token.line, token); + } + _state = ParserStates.UNKNOWN; + } + else { + throw new IllegalArgumentException(token); + } + } + + void _processTokenInCodeState(Token token, Queue stack) { + if (token is TextToken) { + CodeFragment codeFragment = stack.last(); + if (codeFragment.code != null) { + throw const ParseException(token.line, token); + } + codeFragment.code = token.content; + } + else if (token is OpenIncludeToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenCodeToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenExpressionToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenUnescapedExpressionToken) { + throw const ParseException(token.line, token); + } + else if (token is CloseToken) { + CodeFragment codeFragment = stack.last(); + if (codeFragment.code == null) { + throw const ParseException(token.line, token); + } + _state = ParserStates.UNKNOWN; + } + else { + throw new IllegalArgumentException(token); + } + } + + void _processTokenInEscapedExpressionState(Token token, Queue stack) { + if (token is TextToken) { + EscapedOutputFragment expressionFragment = stack.last(); + if (expressionFragment.expression != null) { + throw const ParseException(token.line, token); + } + expressionFragment.expression = token.content; + } + else if (token is OpenIncludeToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenCodeToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenExpressionToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenUnescapedExpressionToken) { + throw const ParseException(token.line, token); + } + else if (token is CloseToken) { + EscapedOutputFragment expressionFragment = stack.last(); + if (expressionFragment == null) { + throw const ParseException(token.line, token); + } + _state = ParserStates.UNKNOWN; + } + else { + throw new IllegalArgumentException(token); + } + } + + void _processTokenInUnescapedExpressionState(Token token, Queue stack) { + if (token is TextToken) { + UnescapedOutputFragment expressionFragment = stack.last(); + if (expressionFragment.expression != null) { + throw const ParseException(token.line, token); + } + expressionFragment.expression = token.content; + } + else if (token is OpenIncludeToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenCodeToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenExpressionToken) { + throw const ParseException(token.line, token); + } + else if (token is OpenUnescapedExpressionToken) { + throw const ParseException(token.line, token); + } + else if (token is CloseToken) { + UnescapedOutputFragment expressionFragment = stack.last(); + if (expressionFragment == null) { + throw const ParseException(token.line, token); + } + _state = ParserStates.UNKNOWN; + } + else { + throw new IllegalArgumentException(token); + } + } + + List _toList(Queue queue) { + var result = []; + for (var item in queue) { + result.add(item); + } + return result; + } /** Transforms to lines */ @@ -43,12 +268,10 @@ class Parser { * a AST, but in our case we have just a plain list of parsed elements. */ abstract class Fragment { - /** A fragment content */ - String _content; /** A line number of this fragment */ int _line; - Fragment(this._content, this._line); + Fragment(this._line); abstract String toCode(); @@ -57,7 +280,13 @@ abstract class Fragment { /** An include */ class IncludeFragment extends Fragment { - IncludeFragment(String fragment, int line): super(fragment, line); + String _include; + + IncludeFragment(int line): super(line); + + String get include() => _include; + + set include(String value) => _include = value; String toCode() { return null; @@ -66,9 +295,13 @@ class IncludeFragment extends Fragment { } /** An plain fragment of template */ -class TemplateFragment extends Fragment { +class TextFragment extends Fragment { + + String _text; - TemplateFragment(String fragment, int line): super(fragment, line); + TextFragment(this._text, int line): super(line); + + String get text() => _text; String toCode() { return null; @@ -79,7 +312,14 @@ class TemplateFragment extends Fragment { /** A fragment of code */ class CodeFragment extends Fragment { - CodeFragment(String fragment, int line): super(fragment, line); + String _code; + + CodeFragment(int line): super(line); + + String get code() => _code; + + set code(String value) => _code = value; + String toCode() { return null; @@ -90,7 +330,13 @@ class CodeFragment extends Fragment { /** An expression which value is html escaped before it appended to an output */ class EscapedOutputFragment extends Fragment { - EscapedOutputFragment(String fragment, int line): super(fragment, line); + String _expression; + + EscapedOutputFragment(int line): super(line); + + String get expression() => _expression; + + set expression(String value) => _expression = value; String toCode() { return null; @@ -101,7 +347,13 @@ class EscapedOutputFragment extends Fragment { /** An expression which value is directly appended to an output */ class UnescapedOutputFragment extends Fragment { - UnescapedOutputFragment(String fragment, int line): super(fragment, line); + String _expression; + + UnescapedOutputFragment(int line): super(line); + + String get expression() => _expression; + + set expression(String value) => _expression = value; String toCode() { return null; @@ -111,8 +363,10 @@ class UnescapedOutputFragment extends Fragment { /** Any available parser states */ class ParserStates { - /** trying to determine which action should be taken */ - static final int ANALYZING = 0; + /** an unitialized state */ + static final int UNKNOWN = -1; + /** processing a template text */ + static final int TEXT = 0; /** processing a template content */ static final int TEMPLATE = 1; /** processing an include content */ @@ -125,21 +379,16 @@ class ParserStates { static final int UNESCAPED_EXPRESSION = 5; } -/** A set of actions that we can possibly take */ -class Actions { +/** thrown when parse exception arise */ +class ParseException implements Exception { + final int _line; + final Token _unexpectedToken; -} - - - - - - - - - - - - - - + const ParseException(this._line, this._unexpectedToken); + + String get exceptionName() => "ParseException"; + /** A line where parse exception occured */ + int get line() => _line; + /** An unexpected token */ + Token unexpectedToken() => _unexpectedToken; +} \ No newline at end of file diff --git a/test/scanner_test.dart b/test/scanner_test.dart index 1b3fdb3..750a283 100644 --- a/test/scanner_test.dart +++ b/test/scanner_test.dart @@ -2,7 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -#library("hypcomm:scanner-test"); +#library("desntech:scanner-test"); #import("../Lib.dart"); //yes, it sucks. you need define your path in order to run this test #import("../../side-projects/dart/dart-sdk/lib/unittest/unittest.dart");