Skip to content

Commit

Permalink
scanner refactoring
Browse files Browse the repository at this point in the history 
- move util functions from scanner.js to utils.js
- implement Scanner#setSource() method
- use its own buffers for each scanner instance
- add constants (const.js) to Scanner as properties
- add util functions to Scanner as static methods of Scanner
- reuse scanner instance in parser
- expose Scanner class
- internal renaming and related refactoring
  • Loading branch information
@lahmatiy
lahmatiy committed Jan 11, 2017
1 parent 532d757 commit d953d69
Show file tree
Hide file tree
Showing 6 changed files with 395 additions and 341 deletions.
2 changes: 2 additions & 0 deletions lib/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ var names = require('./utils/names');

module.exports = {
List: require('./utils/list'),
Scanner: require('./parser/scanner'),

syntax: require('./syntax'),
property: names.property,
keyword: names.keyword,
Expand Down
192 changes: 96 additions & 96 deletions lib/parser/const.js
View file
Original file line number Diff line number Diff line change
@@ -1,132 +1,132 @@
// token types (note: value shouldn't intersect with using char codes)
// token types (note: value shouldn't intersect with used char codes)
var WHITESPACE = 1;
var IDENTIFIER = 2;
var NUMBER = 3;
var STRING = 4;
var COMMENT = 5;
var PUNCTUATION = 6;
var PUNCTUATOR = 6;

var TAB = 9;
var N = 10;
var F = 12;
var R = 13;
var SPACE = 32;

var TokenType = {
var TYPE = {
Whitespace: WHITESPACE,
Identifier: IDENTIFIER,
Number: NUMBER,
String: STRING,
Comment: COMMENT,
Punctuation: PUNCTUATION,
Punctuator: PUNCTUATOR,

ExclamationMark: 33, // !
QuotationMark: 34, // "
NumberSign: 35, // #
DollarSign: 36, // $
PercentSign: 37, // %
Ampersand: 38, // &
Apostrophe: 39, // '
LeftParenthesis: 40, // (
RightParenthesis: 41, // )
Asterisk: 42, // *
PlusSign: 43, // +
Comma: 44, // ,
HyphenMinus: 45, // -
FullStop: 46, // .
Solidus: 47, // /
Colon: 58, // :
Semicolon: 59, // ;
LessThanSign: 60, // <
EqualsSign: 61, // =
GreaterThanSign: 62, // >
QuestionMark: 63, // ?
CommercialAt: 64, // @
LeftSquareBracket: 91, // [
RightSquareBracket: 93, // ]
CircumflexAccent: 94, // ^
LowLine: 95, // _
LeftCurlyBracket: 123, // {
VerticalLine: 124, // |
RightCurlyBracket: 125, // }
Tilde: 126 // ~
ExclamationMark: 33, // !
QuotationMark: 34, // "
NumberSign: 35, // #
DollarSign: 36, // $
PercentSign: 37, // %
Ampersand: 38, // &
Apostrophe: 39, // '
LeftParenthesis: 40, // (
RightParenthesis: 41, // )
Asterisk: 42, // *
PlusSign: 43, // +
Comma: 44, // ,
HyphenMinus: 45, // -
FullStop: 46, // .
Solidus: 47, // /
Colon: 58, // :
Semicolon: 59, // ;
LessThanSign: 60, // <
EqualsSign: 61, // =
GreaterThanSign: 62, // >
QuestionMark: 63, // ?
CommercialAt: 64, // @
LeftSquareBracket: 91, // [
RightSquareBracket: 93, // ]
CircumflexAccent: 94, // ^
LowLine: 95, // _
LeftCurlyBracket: 123, // {
VerticalLine: 124, // |
RightCurlyBracket: 125, // }
Tilde: 126 // ~
};

var TokenName = Object.keys(TokenType).reduce(function(result, key) {
result[TokenType[key]] = key;
var NAME = Object.keys(TYPE).reduce(function(result, key) {
result[TYPE[key]] = key;
return result;
}, {});

var punctuation = [
TokenType.ExclamationMark, // '!'
TokenType.QuotationMark, // '"'
TokenType.NumberSign, // '#'
TokenType.DollarSign, // '$'
TokenType.PercentSign, // '%'
TokenType.Ampersand, // '&'
TokenType.Apostrophe, // '\''
TokenType.LeftParenthesis, // '('
TokenType.RightParenthesis, // ')'
TokenType.Asterisk, // '*'
TokenType.PlusSign, // '+'
TokenType.Comma, // ','
TokenType.HyphenMinus, // '-'
TokenType.FullStop, // '.'
TokenType.Solidus, // '/'
TokenType.Colon, // ':'
TokenType.Semicolon, // ';'
TokenType.LessThanSign, // '<'
TokenType.EqualsSign, // '='
TokenType.GreaterThanSign, // '>'
TokenType.QuestionMark, // '?'
TokenType.CommercialAt, // '@'
TokenType.LeftSquareBracket, // '['
TokenType.RightSquareBracket, // ']'
TokenType.CircumflexAccent, // '^'
TokenType.LeftCurlyBracket, // '{'
TokenType.VerticalLine, // '|'
TokenType.RightCurlyBracket, // '}'
TokenType.Tilde // '~'
];
var SYMBOL_CATEGORY_LENGTH = Math.max.apply(null, punctuation) + 1;
var SYMBOL_CATEGORY = new Uint32Array(SYMBOL_CATEGORY_LENGTH);
var IS_PUNCTUATION = new Uint32Array(SYMBOL_CATEGORY_LENGTH);
var SafeUint32Array = typeof Uint32Array !== 'undefined' ? Uint32Array : Array; // fallback on Array when TypedArray is not supported
var SYMBOL_TYPE = new SafeUint32Array(Math.max.apply(null, Object.keys(NAME).map(Number)) + 1);
var PUNCTUATION = new SafeUint32Array(SYMBOL_TYPE.length);

for (var i = 0; i < SYMBOL_CATEGORY.length; i++) {
SYMBOL_CATEGORY[i] = IDENTIFIER;
for (var i = 0; i < SYMBOL_TYPE.length; i++) {
SYMBOL_TYPE[i] = IDENTIFIER;
}

// fill categories
punctuation.forEach(function(key) {
SYMBOL_CATEGORY[Number(key)] = PUNCTUATION;
IS_PUNCTUATION[Number(key)] = PUNCTUATION;
}, SYMBOL_CATEGORY);

IS_PUNCTUATION[TokenType.HyphenMinus] = 0;
// whitespace is punctuator
IS_PUNCTUATION[SPACE] = PUNCTUATION;
IS_PUNCTUATION[TAB] = PUNCTUATION;
IS_PUNCTUATION[N] = PUNCTUATION;
IS_PUNCTUATION[R] = PUNCTUATION;
IS_PUNCTUATION[F] = PUNCTUATION;
[
TYPE.ExclamationMark, // '!'
TYPE.QuotationMark, // '"'
TYPE.NumberSign, // '#'
TYPE.DollarSign, // '$'
TYPE.PercentSign, // '%'
TYPE.Ampersand, // '&'
TYPE.Apostrophe, // '\''
TYPE.LeftParenthesis, // '('
TYPE.RightParenthesis, // ')'
TYPE.Asterisk, // '*'
TYPE.PlusSign, // '+'
TYPE.Comma, // ','
TYPE.HyphenMinus, // '-'
TYPE.FullStop, // '.'
TYPE.Solidus, // '/'
TYPE.Colon, // ':'
TYPE.Semicolon, // ';'
TYPE.LessThanSign, // '<'
TYPE.EqualsSign, // '='
TYPE.GreaterThanSign, // '>'
TYPE.QuestionMark, // '?'
TYPE.CommercialAt, // '@'
TYPE.LeftSquareBracket, // '['
TYPE.RightSquareBracket, // ']'
TYPE.CircumflexAccent, // '^'
TYPE.LeftCurlyBracket, // '{'
TYPE.VerticalLine, // '|'
TYPE.RightCurlyBracket, // '}'
TYPE.Tilde // '~'
].forEach(function(key) {
SYMBOL_TYPE[Number(key)] = PUNCTUATOR;
PUNCTUATION[Number(key)] = PUNCTUATOR;
}, SYMBOL_TYPE);

for (var i = 48; i <= 57; i++) {
SYMBOL_CATEGORY[i] = NUMBER;
SYMBOL_TYPE[i] = NUMBER;
}

SYMBOL_CATEGORY[SPACE] = WHITESPACE;
SYMBOL_CATEGORY[TAB] = WHITESPACE;
SYMBOL_CATEGORY[N] = WHITESPACE;
SYMBOL_CATEGORY[R] = WHITESPACE;
SYMBOL_CATEGORY[F] = WHITESPACE;
SYMBOL_TYPE[SPACE] = WHITESPACE;
SYMBOL_TYPE[TAB] = WHITESPACE;
SYMBOL_TYPE[N] = WHITESPACE;
SYMBOL_TYPE[R] = WHITESPACE;
SYMBOL_TYPE[F] = WHITESPACE;

SYMBOL_TYPE[TYPE.Apostrophe] = STRING;
SYMBOL_TYPE[TYPE.QuotationMark] = STRING;

SYMBOL_CATEGORY[TokenType.Apostrophe] = STRING;
SYMBOL_CATEGORY[TokenType.QuotationMark] = STRING;
// whitespace is punctuation ...
PUNCTUATION[SPACE] = PUNCTUATOR;
PUNCTUATION[TAB] = PUNCTUATOR;
PUNCTUATION[N] = PUNCTUATOR;
PUNCTUATION[R] = PUNCTUATOR;
PUNCTUATION[F] = PUNCTUATOR;
// ... hyper minus is not
PUNCTUATION[TYPE.HyphenMinus] = 0;

module.exports = {
TokenType: TokenType,
TokenName: TokenName,
TYPE: TYPE,
NAME: NAME,

SYMBOL_CATEGORY: SYMBOL_CATEGORY,
IS_PUNCTUATION: IS_PUNCTUATION
SYMBOL_TYPE: SYMBOL_TYPE,
PUNCTUATION: PUNCTUATION
};
84 changes: 41 additions & 43 deletions lib/parser/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,49 +1,49 @@
'use strict';

var TokenType = require('./const').TokenType;
var Scanner = require('./scanner');
var List = require('../utils/list');
var cmpChar = require('./utils').cmpChar;
var cmpStr = require('./utils').cmpStr;
var endsWith = require('./utils').endsWith;
var isHex = require('./utils').isHex;
var Scanner = require('./scanner');

var scanner = new Scanner();
var cmpChar = Scanner.cmpChar;
var cmpStr = Scanner.cmpStr;
var endsWith = Scanner.endsWith;
var isHex = Scanner.isHex;
var needPositions;
var filename;
var scanner;

var DESCENDANT_COMBINATOR = {};
var SPACE_NODE = { type: 'Space' };

var WHITESPACE = TokenType.Whitespace;
var IDENTIFIER = TokenType.Identifier;
var NUMBER = TokenType.Number;
var STRING = TokenType.String;
var COMMENT = TokenType.Comment;
var EXCLAMATIONMARK = TokenType.ExclamationMark;
var NUMBERSIGN = TokenType.NumberSign;
var DOLLARSIGN = TokenType.DollarSign;
var PERCENTSIGN = TokenType.PercentSign;
var LEFTPARENTHESIS = TokenType.LeftParenthesis;
var RIGHTPARENTHESIS = TokenType.RightParenthesis;
var ASTERISK = TokenType.Asterisk;
var PLUSSIGN = TokenType.PlusSign;
var COMMA = TokenType.Comma;
var HYPHENMINUS = TokenType.HyphenMinus;
var FULLSTOP = TokenType.FullStop;
var SOLIDUS = TokenType.Solidus;
var COLON = TokenType.Colon;
var SEMICOLON = TokenType.Semicolon;
var EQUALSSIGN = TokenType.EqualsSign;
var GREATERTHANSIGN = TokenType.GreaterThanSign;
var QUESTIONMARK = TokenType.QuestionMark;
var COMMERCIALAT = TokenType.CommercialAt;
var LEFTSQUAREBRACKET = TokenType.LeftSquareBracket;
var RIGHTSQUAREBRACKET = TokenType.RightSquareBracket;
var CIRCUMFLEXACCENT = TokenType.CircumflexAccent;
var LEFTCURLYBRACKET = TokenType.LeftCurlyBracket;
var VERTICALLINE = TokenType.VerticalLine;
var RIGHTCURLYBRACKET = TokenType.RightCurlyBracket;
var TILDE = TokenType.Tilde;
var WHITESPACE = Scanner.TYPE.Whitespace;
var IDENTIFIER = Scanner.TYPE.Identifier;
var NUMBER = Scanner.TYPE.Number;
var STRING = Scanner.TYPE.String;
var COMMENT = Scanner.TYPE.Comment;
var EXCLAMATIONMARK = Scanner.TYPE.ExclamationMark;
var NUMBERSIGN = Scanner.TYPE.NumberSign;
var DOLLARSIGN = Scanner.TYPE.DollarSign;
var PERCENTSIGN = Scanner.TYPE.PercentSign;
var LEFTPARENTHESIS = Scanner.TYPE.LeftParenthesis;
var RIGHTPARENTHESIS = Scanner.TYPE.RightParenthesis;
var ASTERISK = Scanner.TYPE.Asterisk;
var PLUSSIGN = Scanner.TYPE.PlusSign;
var COMMA = Scanner.TYPE.Comma;
var HYPHENMINUS = Scanner.TYPE.HyphenMinus;
var FULLSTOP = Scanner.TYPE.FullStop;
var SOLIDUS = Scanner.TYPE.Solidus;
var COLON = Scanner.TYPE.Colon;
var SEMICOLON = Scanner.TYPE.Semicolon;
var EQUALSSIGN = Scanner.TYPE.EqualsSign;
var GREATERTHANSIGN = Scanner.TYPE.GreaterThanSign;
var QUESTIONMARK = Scanner.TYPE.QuestionMark;
var COMMERCIALAT = Scanner.TYPE.CommercialAt;
var LEFTSQUAREBRACKET = Scanner.TYPE.LeftSquareBracket;
var RIGHTSQUAREBRACKET = Scanner.TYPE.RightSquareBracket;
var CIRCUMFLEXACCENT = Scanner.TYPE.CircumflexAccent;
var LEFTCURLYBRACKET = Scanner.TYPE.LeftCurlyBracket;
var VERTICALLINE = Scanner.TYPE.VerticalLine;
var RIGHTCURLYBRACKET = Scanner.TYPE.RightCurlyBracket;
var TILDE = Scanner.TYPE.Tilde;
var N = 110; // 'n'.charCodeAt(0)

var SCOPE_ATRULE_EXPRESSION = {
Expand All @@ -59,7 +59,7 @@ var SCOPE_VALUE = {
var: getVarFunction
};

var initialContext = {
var CONTEXT = {
stylesheet: getStylesheet,
atrule: getAtrule,
atruleExpression: getAtruleExpression,
Expand Down Expand Up @@ -1729,20 +1729,18 @@ function parse(source, options) {
needPositions = Boolean(options.positions);
filename = options.filename || '<unknown>';

if (!initialContext.hasOwnProperty(context)) {
if (!CONTEXT.hasOwnProperty(context)) {
throw new Error('Unknown context `' + context + '`');
}

scanner = new Scanner(source, options.line, options.column);
scanner.setSource(source, options.line, options.column);

if (context === 'value') {
ast = getValue(false, options.property ? String(options.property) : null);
} else {
ast = initialContext[context]();
ast = CONTEXT[context]();
}

scanner = null;

// console.log(JSON.stringify(ast, null, 4));
return ast;
};
Expand Down

0 comments on commit d953d69

Please sign in to comment.