Skip to content
Fetching contributors…
Cannot retrieve contributors at this time
179 lines (159 sloc) 8.96 KB
/*
* Copyright (C) 2009 Google Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// Generate js file as follows:
//
// re2c -isc WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js \
// | sed 's|^yy\([^:]*\)*\:|case \1:|' \
// | sed 's|[*]cursor[+][+]|this._charAt(cursor++)|' \
// | sed 's|[[*][+][+]cursor|this._charAt(++cursor)|' \
// | sed 's|[*]cursor|this._charAt(cursor)|' \
// | sed 's|yych = \*\([^;]*\)|yych = this._charAt\1|' \
// | sed 's|goto case \([^;]*\)|{ gotoCase = \1; continue; }|' \
// | sed 's|unsigned\ int|var|' \
// | sed 's|var\ yych|case 1: var yych|'
WebInspector.SourceJavaScriptTokenizer = function()
{
WebInspector.SourceTokenizer.call(this);
this._keywords = [
"null", "true", "false", "break", "case", "catch", "const", "default", "finally", "for",
"instanceof", "new", "var", "continue", "function", "return", "void", "delete", "if",
"this", "do", "while", "else", "in", "switch", "throw", "try", "typeof", "debugger",
"class", "enum", "export", "extends", "import", "super", "get", "set", "with"
].keySet();
this._lexConditions = {
DIV: 0,
NODIV: 1,
COMMENT: 2,
DSTRING: 3,
SSTRING: 4,
REGEX: 5
};
this.case_DIV = 1000;
this.case_NODIV = 1001;
this.case_COMMENT = 1002;
this.case_DSTRING = 1003;
this.case_SSTRING = 1004;
this.case_REGEX = 1005;
this.initialCondition = { lexCondition: this._lexConditions.NODIV }
this.condition = this.initialCondition;
}
WebInspector.SourceJavaScriptTokenizer.prototype = {
nextToken: function(cursor)
{
var cursorOnEnter = cursor;
var gotoCase = 1;
while (1) {
switch (gotoCase)
// Following comment is replaced with generated state machine.
/*!re2c
re2c:define:YYCTYPE = "var";
re2c:define:YYCURSOR = cursor;
re2c:define:YYGETCONDITION = "this.getLexCondition";
re2c:define:YYSETCONDITION = "this.setLexCondition";
re2c:condprefix = "case this.case_";
re2c:condenumprefix = "this._lexConditions.";
re2c:yyfill:enable = 0;
re2c:labelprefix = "case ";
re2c:indent:top = 2;
re2c:indent:string = " ";
LineComment = "//" [^\r\n]*;
CommentContent = ([^*\r\n] | ("*"+[^/*]))*;
Comment = "/*" CommentContent "*"+ "/";
CommentStart = "/*" CommentContent [\r\n];
CommentEnd = CommentContent "*"+ "/";
DecimalDigit = [0-9];
NonZeroDigit = [1-9];
OctalDigit = [0-7];
HexDigit = [0-9a-fA-F];
SignedInteger = ("+"|"-")? DecimalDigit+;
ExponentPart = ("e" | "E") SignedInteger;
DecimalIntegerLiteral = "0" | NonZeroDigit DecimalDigit*;
DecimalLiteral = DecimalIntegerLiteral "." DecimalDigit* ExponentPart? | "." DecimalDigit+ ExponentPart? | DecimalIntegerLiteral ExponentPart?;
HexIntegerLiteral = "0" ("x"|"X") HexDigit+;
OctalIntegerLiteral = "0" OctalDigit+;
NumericLiteral = DecimalLiteral | HexIntegerLiteral | OctalIntegerLiteral;
Punctuation = [\!\%\&\(\*\+\,\-\.\:\;\<\=\>\?\[\]\^\{\|\}\~] | "!=" | "!==" | "%=" | "&&" | "&=" | "*=" | "++" | "+=" | "--" | "-=" | "<<" | "<<=" | "<=" | "==" | "===" | ">=" | ">>" | ">>=" | ">>>" | ">>>=" | "^=" | "|=" | "||";
Division = "/" | "/=";
RightParen = ")";
Letter = [a-zA-Z\x80-\xFF];
UnicodeEscapeSequence = "\\u" HexDigit HexDigit HexDigit HexDigit;
IdentifierStart = Letter | "_" | "$" | UnicodeEscapeSequence;
IdentifierPart = IdentifierStart | DecimalDigit;
Identifier = IdentifierStart IdentifierPart *;
DoubleStringContent = ([^\r\n\"\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*;
SingleStringContent = ([^\r\n\'\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*;
StringLiteral = "\"" DoubleStringContent "\"" | "'" SingleStringContent "'";
DoubleStringStart = "\"" DoubleStringContent "\\" [\r\n];
DoubleStringEnd = DoubleStringContent "\"";
SingleStringStart = "'" SingleStringContent "\\" [\r\n];
SingleStringEnd = SingleStringContent "'";
BackslashSequence = "\\" [^\r\n];
RegexSet = "[" ([^\r\n*\\/] | BackslashSequence)* "]";
RegexFirstChar = [^\r\n*\\/\[\]] | BackslashSequence | RegexSet;
RegexChar = [^\r\n\\/\[\]] | BackslashSequence | RegexSet;
RegexContent = RegexChar*;
Regex = "/" RegexFirstChar RegexContent "/" [igm]*;
RegexStart = "/" RegexFirstChar RegexContent "\\";
RegexEnd = RegexContent "/" [igm]*;
<DIV,NODIV> LineComment { this.tokenType = "javascript-comment"; return cursor; }
<DIV,NODIV> Comment { this.tokenType = "javascript-comment"; return cursor; }
<DIV,NODIV> CommentStart => COMMENT { this.tokenType = "javascript-comment"; return cursor; }
<COMMENT> CommentContent => COMMENT { this.tokenType = "javascript-comment"; return cursor; }
<COMMENT> CommentEnd => NODIV { this.tokenType = "javascript-comment"; return cursor; }
<DIV,NODIV> StringLiteral { this.tokenType = "javascript-string"; return cursor; }
<DIV,NODIV> DoubleStringStart => DSTRING { this.tokenType = "javascript-string"; return cursor; }
<DSTRING> DoubleStringContent => DSTRING { this.tokenType = "javascript-string"; return cursor; }
<DSTRING> DoubleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; }
<DIV,NODIV> SingleStringStart => SSTRING { this.tokenType = "javascript-string"; return cursor; }
<SSTRING> SingleStringContent => SSTRING { this.tokenType = "javascript-string"; return cursor; }
<SSTRING> SingleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; }
<NODIV> Regex { this.tokenType = "javascript-regexp"; return cursor; }
<NODIV> RegexStart => REGEX { this.tokenType = "javascript-regexp"; return cursor; }
<REGEX> RegexContent => REGEX { this.tokenType = "javascript-regexp"; return cursor; }
<REGEX> RegexEnd => NODIV { this.tokenType = "javascript-regexp"; return cursor; }
<DIV,NODIV> NumericLiteral => DIV { this.tokenType = "javascript-number"; return cursor; }
<DIV,NODIV> Identifier => DIV
{
var token = this._line.substring(cursorOnEnter, cursor);
if (token in this._keywords)
this.tokenType = "javascript-keyword";
else
this.tokenType = "javascript-ident";
return cursor;
}
<DIV,NODIV> RightParen => DIV { this.tokenType = null; return cursor; }
<DIV,NODIV> Punctuation => NODIV { this.tokenType = null; return cursor; }
<DIV> Division => NODIV { this.tokenType = null; return cursor; }
<*> [^] { this.tokenType = null; return cursor; }
*/
}
}
}
WebInspector.SourceJavaScriptTokenizer.prototype.__proto__ = WebInspector.SourceTokenizer.prototype;
Jump to Line
Something went wrong with that request. Please try again.