Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Added curly method support

  • Loading branch information...
commit 9de765c602f88808cc8b9410466b10bac2d10015 1 parent 0072a61
Peter van der Zee authored
Showing with 138 additions and 13 deletions.
  1. +113 −4 Tokenizer.js
  2. +25 −9 ZeParser.js
View
117 Tokenizer.js
@@ -70,6 +70,24 @@ function Tokenizer(inp, options){
};
Tokenizer.prototype = {
+ // token constants... (should use these some day)
+ REGEX: 1,
+ IDENTIFIER: 2,
+ NUMERIC_HEX: 3,
+ NUMERIC_DEC: 4,
+ STRING_SINGLE: 5,
+ STRING_DOUBLE: 6,
+ COMMENT_SINGLE: 7,
+ COMMENT_MULTI: 8,
+ WHITE_SPACE: 9,
+ LINETERMINATOR: 10,
+ PUNCTUATOR: 11,
+ EOF: 12,
+ ASI: 13,
+ ERROR: 14,
+ TAG: 15,
+ CURLY_METHOD: 16,
+
inp:null,
shadowInp:null,
pos:null,
@@ -97,6 +115,8 @@ Tokenizer.prototype = {
Unicode:null,
+ tagLiterals: false, // custom tag literal support. allows <div></div> kind of (sub-expression) tokens
+
// storeCurrentAndFetchNextToken(bool, false, false true) to get just one token
storeCurrentAndFetchNextToken: function(noRegex, returnValue, stack, _dontStore){
var regex = !noRegex; // TOFIX :)
@@ -647,8 +667,8 @@ Tokenizer.prototype = {
}
if (returnValue) {
- // note that ASI's are slipstreamed in here from the parser since the tokenizer cant determine that
- // if this part ever changes, make sure you change that too :)
+ // note that ASI's and errors are slipstreamed in here from the parser since the tokenizer cant determine that
+ // if this part ever changes, make sure you change that too :) (see also this.addTokenToStreamBefore)
returnValue.tokposw = this.wtree.length;
this.wtree.push(returnValue);
if (!returnValue.isWhite) {
@@ -658,7 +678,7 @@ Tokenizer.prototype = {
}
- } while (stack && returnValue && returnValue.isWhite); // WHITE_SPACE LINETERMINATOR COMMENT_SINGLE COMMENT_MULTI
+ } while (returnValue && returnValue.isWhite); // WHITE_SPACE LINETERMINATOR COMMENT_SINGLE COMMENT_MULTI
++this.tokenCountNoWhite;
this.pos = pos;
@@ -666,6 +686,7 @@ Tokenizer.prototype = {
if (matchedNewline) returnValue.newline = true;
return returnValue;
},
+ // used by ASI and error stuff (in parser)
addTokenToStreamBefore: function(token, match){
var wtree = this.wtree;
var btree = this.btree;
@@ -688,6 +709,89 @@ Tokenizer.prototype = {
}
}
},
+ // (unused) replaces the range of tokens in the
+ // white and black streams with the specified token.
+ replaceTokensInStreamWithToken: function(token, wfrom, wto, bfrom, bto){
+ this.wtree.splice(wfrom, wto-wfrom, token);
+ this.btree.splice(bfrom, bto-bfrom, token);
+ },
+ parseCurlyMethodLiteral: function(match){
+ var error = false;
+ var pos = this.pos;
+ // match should be an opening curly with no preceeding newline
+ if (match.hasNewline) {
+ // so this is bad because if we would not demand this, the language could
+ // be amibiguous with a block
+ error = 'CurlyMethodsMayNotFollowNewline';
+ } else {
+ var input = this.inp;
+
+ // remember number of curlies, you'll want that many closers as well
+ var curlies = 1;
+ while (input[pos] == '{') {
+ ++pos;
+ ++curlies;
+ }
+
+ // keep parsing characters until you reach a curly.
+ // backslashes may only escape backslashes or curlies
+ while (!error && pos < input.length && input[pos] != '}') {
+ if (input[pos] == '{') {
+ error = CurlyMethodsCannotContainOpeningCurly;
+ } else if (input[pos] == '\\') {
+ if (input[pos+1] != '{' && input[pos+1] != '}' && input[pos+1] != '\\') {
+ error = 'CurlyMethodsMayOnlyEscapeCurlies';
+ } else {
+ // skip curly or backslash
+ ++pos;
+ }
+ }
+ ++pos;
+ }
+
+ if (!error) {
+ if (pos >= input.length) {
+ error = 'CurlyMethodsUnexpectedEof';
+ } else {
+ var n = curlies;
+ while (n && pos<input.length) {
+ if (input[pos] == '}') {
+ ++pos;
+ --n;
+ } else {
+ break;
+ }
+ }
+// while (n-- && pos<=input.length && input[pos++] == '}') console.log('yes');
+
+ if (pos>input.length) error = 'CurlyMethodsUnexpectedEof';
+ else if (n) error = 'CurlyMethodsWasOpenedWithMoreCurliesThanClosed';
+ }
+ }
+ if (!error) {
+ // transform this match to a CURLY_METHOD instead of the opening curly it was
+ match.name = this.CURLY_METHOD;
+ match.stop = pos;
+ match.value = this.inp.slice(match.start,pos);
+ match.curlies = curlies;
+
+ this.pos = pos;
+ }
+ }
+
+ if (error) {
+ this.addTokenToStreamBefore(
+ {
+ start: match.start,
+ stop: pos,
+ name: this.ERROR,
+ tokenError:true,
+ error: Tokenizer.Error.NumberExponentRequiresDigits
+ },
+ match
+ );
+ }
+ },
oldNumberParser: function(pos, chr, inp, returnValue, start, Tokenizer){
++pos;
// either: 0x 0X 0 .3
@@ -851,5 +955,10 @@ Tokenizer.Error = {
DollarShouldBeEnd: {msg: 'The $ signifies the stop of match but was not found at a stop'},
QuantifierRequiresNumber: {msg:'Quantifier curly requires at least one digit before the comma'},
QuantifierRequiresClosingCurly: {msg:'Quantifier curly requires to be closed'},
- MissingOpeningCurly: {msg:'Encountered closing quantifier curly without seeing an opening curly'}
+ MissingOpeningCurly: {msg:'Encountered closing quantifier curly without seeing an opening curly'},
+ CurlyMethodsMayNotFollowNewline: {msg:'There may not be any newlines between the expression and the curly method'},
+ CurlyMethodsMayOnlyEscapeCurlies: {msg:'You may only escape curlies {} and backslashes in curly methods'},
+ CurlyMethodsCannotContainOpeningCurly: {msg:'There\'s no way an opening curly could be part of a curly method, yet'},
+ CurlyMethodsWasOpenedWithMoreCurliesThanClosed: {msg:'The curly method must be closed with as many curlies as it was started with'},
+ CurlyMethodsUnexpectedEof: {msg:'Encountered EOF while parsing a curly method'},
};
View
34 ZeParser.js
@@ -93,6 +93,8 @@ ZeParser.prototype = {
ast: null,
+ queryMethodExtension: false,
+
parse: function(match){
if (match) match = this.tokenizer.storeCurrentAndFetchNextToken(false, match, this.stack); // meh
else match = this.tokenizer.storeCurrentAndFetchNextToken(false, null, this.stack, true); // initialization step, dont store the match (there isnt any!)
@@ -240,7 +242,7 @@ ZeParser.prototype = {
if (!(/*is left hand side start?*/ match.name <= 6 || match.name == 15/*TAG*/ || this.regexLhsStart.test(match.value))) match = this.failsafe('ExpectedAnotherExpressionRhs', match);
// not allowed to parse assignment
parsedUnaryOperator = true;
- };
+ }
// if we parsed any kind of unary operator, we cannot be parsing a labeled statement
if (parsedUnaryOperator) mayParseLabeledStatementInstead = false;
@@ -548,7 +550,8 @@ ZeParser.prototype = {
stack = oldstack;
this.scope = oldscope;
} //#endif
- } else if (match.name <= 6 || match.name == 15/*TAG*/) { // IDENTIFIER STRING_SINGLE STRING_DOUBLE NUMERIC_HEX NUMERIC_DEC REG_EX or TAG
+ // this is the core (sub-)expression part:
+ } else if (match.name <= 6 || match.name == 15/*TAG*/) { // IDENTIFIER STRING_SINGLE STRING_DOUBLE NUMERIC_HEX NUMERIC_DEC REG_EX or (custom extension) TAG
// save it in case it turns out to be a label.
var possibleLabel = match;
@@ -577,7 +580,9 @@ ZeParser.prototype = {
// only accept assignments after a member expression (identifier or ending with a [] suffix)
acceptAssignment = true;
- } else if (isBreakOrContinueArg) match = this.failsafe('BreakOrContinueArgMustBeJustIdentifier', match);
+ } else if (isBreakOrContinueArg) {
+ match = this.failsafe('BreakOrContinueArgMustBeJustIdentifier', match);
+ }
// the current match is the lead value being queried. tag it that way
if (this.ast) { //#ifdef FULL_AST
@@ -656,8 +661,8 @@ ZeParser.prototype = {
match = this.tokenizer.storeCurrentAndFetchNextToken(false, match, stack);
}
- // search for "value" suffix. property access and call parens.
- while (match.value == '.' || match.value == '[' || match.value == '(') {
+ // search for "value" suffix. property access and call parens. (and query methods: {)
+ while (match.value == '.' || match.value == '[' || match.value == '(' || (this.queryMethodExtension && match.value == '{')) {
if (isBreakOrContinueArg) match = this.failsafe('BreakOrContinueArgMustBeJustIdentifier', match);
if (match.value == '.') {
@@ -719,7 +724,18 @@ ZeParser.prototype = {
} //#endif
match = this.tokenizer.storeCurrentAndFetchNextToken(true, match, stack); // might be div
acceptAssignment = false;
- }
+ } else if (match.value == '{') {
+ if (!this.queryMethodExtension) {
+ console.warn("This should never happen. The `foo{.bar}` syntax is only parsed under a flag that's set to false... wtf?");
+ }
+
+ // this transforms the given match inline. so the match stays the same, it just
+ // has a new name and covers more input after this method finishes ;)
+ this.tokenizer.parseCurlyMethodLiteral(match);
+ match = this.tokenizer.storeCurrentAndFetchNextToken(true, match, stack); // might be div
+ } else {
+ throw "Coding error, should never happen";
+ }
}
// check for postfix operators ++ and --
@@ -810,8 +826,8 @@ ZeParser.prototype = {
}
} while (ternary); // if we just parsed a ternary expression, we need to check _again_ whether the next token is a binary operator.
- // start over. match is the rhs for the lhs we just parsed, but lhs for the next expression
- if (parseAnotherExpression && !(/*is left hand side start?*/ match.name <= 6 || match.name == 15/*TAG*/ || this.regexLhsStart.test(match.value))) {
+ // start over. match is the rhs for the lhs we just parsed, but lhs for the next expression
+ if (parseAnotherExpression && !(/*is left hand side start?*/ match.name <= 6 || match.name == 15/*TAG*/ || this.regexLhsStart.test(match.value))) {
// no idea what to do now. lets just ignore and see where it ends. TOFIX: maybe just break the loop or return?
this.failignore('InvalidRhsExpression', match, stack);
}
@@ -2183,5 +2199,5 @@ ZeParser.Errors = {
CaseMissingExpression: {msg:'Case expects an expression before the colon'},
TryMustHaveCatchOrFinally: {msg:'The try statement must have a catch or finally block'},
UnexpectedInputSwitch: {msg:'Unexpected input while parsing a switch clause...'},
- ForInCanOnlyDeclareOnVar: {msg:'For-in header can only introduce one new variable'}
+ ForInCanOnlyDeclareOnVar: {msg:'For-in header can only introduce one new variable'},
};
Please sign in to comment.
Something went wrong with that request. Please try again.