-
Notifications
You must be signed in to change notification settings - Fork 84
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- move util functions from scanner.js to utils.js - implement Scanner#setSource() method - use its own buffers for each scanner instance - add constants (const.js) to Scanner as properties - add util functions to Scanner as static methods of Scanner - reuse scanner instance in parser - expose Scanner class - internal renaming and related refactoring
- Loading branch information
Showing
6 changed files
with
395 additions
and
341 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,132 +1,132 @@ | ||
// token types (note: value shouldn't intersect with using char codes) | ||
// token types (note: value shouldn't intersect with used char codes) | ||
var WHITESPACE = 1; | ||
var IDENTIFIER = 2; | ||
var NUMBER = 3; | ||
var STRING = 4; | ||
var COMMENT = 5; | ||
var PUNCTUATION = 6; | ||
var PUNCTUATOR = 6; | ||
|
||
var TAB = 9; | ||
var N = 10; | ||
var F = 12; | ||
var R = 13; | ||
var SPACE = 32; | ||
|
||
var TokenType = { | ||
var TYPE = { | ||
Whitespace: WHITESPACE, | ||
Identifier: IDENTIFIER, | ||
Number: NUMBER, | ||
String: STRING, | ||
Comment: COMMENT, | ||
Punctuation: PUNCTUATION, | ||
Punctuator: PUNCTUATOR, | ||
|
||
ExclamationMark: 33, // ! | ||
QuotationMark: 34, // " | ||
NumberSign: 35, // # | ||
DollarSign: 36, // $ | ||
PercentSign: 37, // % | ||
Ampersand: 38, // & | ||
Apostrophe: 39, // ' | ||
LeftParenthesis: 40, // ( | ||
RightParenthesis: 41, // ) | ||
Asterisk: 42, // * | ||
PlusSign: 43, // + | ||
Comma: 44, // , | ||
HyphenMinus: 45, // - | ||
FullStop: 46, // . | ||
Solidus: 47, // / | ||
Colon: 58, // : | ||
Semicolon: 59, // ; | ||
LessThanSign: 60, // < | ||
EqualsSign: 61, // = | ||
GreaterThanSign: 62, // > | ||
QuestionMark: 63, // ? | ||
CommercialAt: 64, // @ | ||
LeftSquareBracket: 91, // [ | ||
RightSquareBracket: 93, // ] | ||
CircumflexAccent: 94, // ^ | ||
LowLine: 95, // _ | ||
LeftCurlyBracket: 123, // { | ||
VerticalLine: 124, // | | ||
RightCurlyBracket: 125, // } | ||
Tilde: 126 // ~ | ||
ExclamationMark: 33, // ! | ||
QuotationMark: 34, // " | ||
NumberSign: 35, // # | ||
DollarSign: 36, // $ | ||
PercentSign: 37, // % | ||
Ampersand: 38, // & | ||
Apostrophe: 39, // ' | ||
LeftParenthesis: 40, // ( | ||
RightParenthesis: 41, // ) | ||
Asterisk: 42, // * | ||
PlusSign: 43, // + | ||
Comma: 44, // , | ||
HyphenMinus: 45, // - | ||
FullStop: 46, // . | ||
Solidus: 47, // / | ||
Colon: 58, // : | ||
Semicolon: 59, // ; | ||
LessThanSign: 60, // < | ||
EqualsSign: 61, // = | ||
GreaterThanSign: 62, // > | ||
QuestionMark: 63, // ? | ||
CommercialAt: 64, // @ | ||
LeftSquareBracket: 91, // [ | ||
RightSquareBracket: 93, // ] | ||
CircumflexAccent: 94, // ^ | ||
LowLine: 95, // _ | ||
LeftCurlyBracket: 123, // { | ||
VerticalLine: 124, // | | ||
RightCurlyBracket: 125, // } | ||
Tilde: 126 // ~ | ||
}; | ||
|
||
var TokenName = Object.keys(TokenType).reduce(function(result, key) { | ||
result[TokenType[key]] = key; | ||
var NAME = Object.keys(TYPE).reduce(function(result, key) { | ||
result[TYPE[key]] = key; | ||
return result; | ||
}, {}); | ||
|
||
var punctuation = [ | ||
TokenType.ExclamationMark, // '!' | ||
TokenType.QuotationMark, // '"' | ||
TokenType.NumberSign, // '#' | ||
TokenType.DollarSign, // '$' | ||
TokenType.PercentSign, // '%' | ||
TokenType.Ampersand, // '&' | ||
TokenType.Apostrophe, // '\'' | ||
TokenType.LeftParenthesis, // '(' | ||
TokenType.RightParenthesis, // ')' | ||
TokenType.Asterisk, // '*' | ||
TokenType.PlusSign, // '+' | ||
TokenType.Comma, // ',' | ||
TokenType.HyphenMinus, // '-' | ||
TokenType.FullStop, // '.' | ||
TokenType.Solidus, // '/' | ||
TokenType.Colon, // ':' | ||
TokenType.Semicolon, // ';' | ||
TokenType.LessThanSign, // '<' | ||
TokenType.EqualsSign, // '=' | ||
TokenType.GreaterThanSign, // '>' | ||
TokenType.QuestionMark, // '?' | ||
TokenType.CommercialAt, // '@' | ||
TokenType.LeftSquareBracket, // '[' | ||
TokenType.RightSquareBracket, // ']' | ||
TokenType.CircumflexAccent, // '^' | ||
TokenType.LeftCurlyBracket, // '{' | ||
TokenType.VerticalLine, // '|' | ||
TokenType.RightCurlyBracket, // '}' | ||
TokenType.Tilde // '~' | ||
]; | ||
var SYMBOL_CATEGORY_LENGTH = Math.max.apply(null, punctuation) + 1; | ||
var SYMBOL_CATEGORY = new Uint32Array(SYMBOL_CATEGORY_LENGTH); | ||
var IS_PUNCTUATION = new Uint32Array(SYMBOL_CATEGORY_LENGTH); | ||
var SafeUint32Array = typeof Uint32Array !== 'undefined' ? Uint32Array : Array; // fallback on Array when TypedArray is not supported | ||
var SYMBOL_TYPE = new SafeUint32Array(Math.max.apply(null, Object.keys(NAME).map(Number)) + 1); | ||
var PUNCTUATION = new SafeUint32Array(SYMBOL_TYPE.length); | ||
|
||
for (var i = 0; i < SYMBOL_CATEGORY.length; i++) { | ||
SYMBOL_CATEGORY[i] = IDENTIFIER; | ||
for (var i = 0; i < SYMBOL_TYPE.length; i++) { | ||
SYMBOL_TYPE[i] = IDENTIFIER; | ||
} | ||
|
||
// fill categories | ||
punctuation.forEach(function(key) { | ||
SYMBOL_CATEGORY[Number(key)] = PUNCTUATION; | ||
IS_PUNCTUATION[Number(key)] = PUNCTUATION; | ||
}, SYMBOL_CATEGORY); | ||
|
||
IS_PUNCTUATION[TokenType.HyphenMinus] = 0; | ||
// whitespace is punctuator | ||
IS_PUNCTUATION[SPACE] = PUNCTUATION; | ||
IS_PUNCTUATION[TAB] = PUNCTUATION; | ||
IS_PUNCTUATION[N] = PUNCTUATION; | ||
IS_PUNCTUATION[R] = PUNCTUATION; | ||
IS_PUNCTUATION[F] = PUNCTUATION; | ||
[ | ||
TYPE.ExclamationMark, // '!' | ||
TYPE.QuotationMark, // '"' | ||
TYPE.NumberSign, // '#' | ||
TYPE.DollarSign, // '$' | ||
TYPE.PercentSign, // '%' | ||
TYPE.Ampersand, // '&' | ||
TYPE.Apostrophe, // '\'' | ||
TYPE.LeftParenthesis, // '(' | ||
TYPE.RightParenthesis, // ')' | ||
TYPE.Asterisk, // '*' | ||
TYPE.PlusSign, // '+' | ||
TYPE.Comma, // ',' | ||
TYPE.HyphenMinus, // '-' | ||
TYPE.FullStop, // '.' | ||
TYPE.Solidus, // '/' | ||
TYPE.Colon, // ':' | ||
TYPE.Semicolon, // ';' | ||
TYPE.LessThanSign, // '<' | ||
TYPE.EqualsSign, // '=' | ||
TYPE.GreaterThanSign, // '>' | ||
TYPE.QuestionMark, // '?' | ||
TYPE.CommercialAt, // '@' | ||
TYPE.LeftSquareBracket, // '[' | ||
TYPE.RightSquareBracket, // ']' | ||
TYPE.CircumflexAccent, // '^' | ||
TYPE.LeftCurlyBracket, // '{' | ||
TYPE.VerticalLine, // '|' | ||
TYPE.RightCurlyBracket, // '}' | ||
TYPE.Tilde // '~' | ||
].forEach(function(key) { | ||
SYMBOL_TYPE[Number(key)] = PUNCTUATOR; | ||
PUNCTUATION[Number(key)] = PUNCTUATOR; | ||
}, SYMBOL_TYPE); | ||
|
||
for (var i = 48; i <= 57; i++) { | ||
SYMBOL_CATEGORY[i] = NUMBER; | ||
SYMBOL_TYPE[i] = NUMBER; | ||
} | ||
|
||
SYMBOL_CATEGORY[SPACE] = WHITESPACE; | ||
SYMBOL_CATEGORY[TAB] = WHITESPACE; | ||
SYMBOL_CATEGORY[N] = WHITESPACE; | ||
SYMBOL_CATEGORY[R] = WHITESPACE; | ||
SYMBOL_CATEGORY[F] = WHITESPACE; | ||
SYMBOL_TYPE[SPACE] = WHITESPACE; | ||
SYMBOL_TYPE[TAB] = WHITESPACE; | ||
SYMBOL_TYPE[N] = WHITESPACE; | ||
SYMBOL_TYPE[R] = WHITESPACE; | ||
SYMBOL_TYPE[F] = WHITESPACE; | ||
|
||
SYMBOL_TYPE[TYPE.Apostrophe] = STRING; | ||
SYMBOL_TYPE[TYPE.QuotationMark] = STRING; | ||
|
||
SYMBOL_CATEGORY[TokenType.Apostrophe] = STRING; | ||
SYMBOL_CATEGORY[TokenType.QuotationMark] = STRING; | ||
// whitespace is punctuation ... | ||
PUNCTUATION[SPACE] = PUNCTUATOR; | ||
PUNCTUATION[TAB] = PUNCTUATOR; | ||
PUNCTUATION[N] = PUNCTUATOR; | ||
PUNCTUATION[R] = PUNCTUATOR; | ||
PUNCTUATION[F] = PUNCTUATOR; | ||
// ... hyper minus is not | ||
PUNCTUATION[TYPE.HyphenMinus] = 0; | ||
|
||
module.exports = { | ||
TokenType: TokenType, | ||
TokenName: TokenName, | ||
TYPE: TYPE, | ||
NAME: NAME, | ||
|
||
SYMBOL_CATEGORY: SYMBOL_CATEGORY, | ||
IS_PUNCTUATION: IS_PUNCTUATION | ||
SYMBOL_TYPE: SYMBOL_TYPE, | ||
PUNCTUATION: PUNCTUATION | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.