Skip to content
This repository has been archived by the owner on May 19, 2018. It is now read-only.

Refactor tokenizer types file #263

Merged
merged 9 commits into from
Jan 10, 2017
Merged

Refactor tokenizer types file #263

Changes from 4 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e6f9b33
refactor(tokenizer): use class for keywork tokens
xtuc Dec 20, 2016
0adc3f6
refactor(tokenizer): re-use constacts in types
xtuc Dec 20, 2016
9e3aa03
refactor(tokenizer): binop token type
xtuc Dec 20, 2016
b0a2b3c
feat(tokenizer): use beforeAndStartExpr for shortcut
xtuc Jan 7, 2017
004f425
fix(tokenizer): typo in keywordTokenType
xtuc Jan 8, 2017
482cc67
refactor(tokenizer): don't use spread operator
xtuc Jan 8, 2017
a564d40
refactor(tokenizer): constant for isLoop, isAssign, prefix, postfix
xtuc Jan 8, 2017
6b2dfd0
fix(tokenizer): remove constant beforeAndStartExpr
xtuc Jan 8, 2017
c65ae91
style(tokenizer): space after comma
xtuc Jan 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 82 additions & 71 deletions src/tokenizer/types.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@
// to know when parsing a label, in order to allow or disallow
// continue jumps to that label.


const beforeExpr = { beforeExpr: true };
const startsExpr = { startsExpr: true };
const beforeAndStartExpr = { ...beforeExpr, ...startsExpr };

export class TokenType {
constructor(label, conf = {}) {
this.label = label;
Expand All @@ -32,10 +37,19 @@ export class TokenType {
}
}

function binop(name, prec) {
return new TokenType(name, {beforeExpr: true, binop: prec});
class KeyworkTokenType extends TokenType {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typo: KeywordTokenType

constructor(name, options = {}) {
options.keyword = name;

super(name, options);
}
}

export class BinopTokenType extends TokenType {
constructor(name, prec) {
super(name, { ...beforeExpr, binop: prec });
}
}
const beforeExpr = {beforeExpr: true}, startsExpr = {startsExpr: true};

export const types = {
num: new TokenType("num", startsExpr),
Expand All @@ -45,13 +59,13 @@ export const types = {
eof: new TokenType("eof"),

// Punctuation token types.
bracketL: new TokenType("[", {beforeExpr: true, startsExpr: true}),
bracketL: new TokenType("[", beforeAndStartExpr),
bracketR: new TokenType("]"),
braceL: new TokenType("{", {beforeExpr: true, startsExpr: true}),
braceBarL: new TokenType("{|", {beforeExpr: true, startsExpr: true}),
braceL: new TokenType("{", beforeAndStartExpr),
braceBarL: new TokenType("{|", beforeAndStartExpr),
braceR: new TokenType("}"),
braceBarR: new TokenType("|}"),
parenL: new TokenType("(", {beforeExpr: true, startsExpr: true}),
parenL: new TokenType("(", beforeAndStartExpr),
parenR: new TokenType(")"),
comma: new TokenType(",", beforeExpr),
semi: new TokenType(";", beforeExpr),
Expand All @@ -63,7 +77,7 @@ export const types = {
template: new TokenType("template"),
ellipsis: new TokenType("...", beforeExpr),
backQuote: new TokenType("`", startsExpr),
dollarBraceL: new TokenType("${", {beforeExpr: true, startsExpr: true}),
dollarBraceL: new TokenType("${", beforeAndStartExpr),
at: new TokenType("@"),

// Operators. These carry several kinds of properties to help the
Expand All @@ -80,69 +94,66 @@ export const types = {
// binary operators with a very low precedence, that should result
// in AssignmentExpression nodes.

eq: new TokenType("=", {beforeExpr: true, isAssign: true}),
assign: new TokenType("_=", {beforeExpr: true, isAssign: true}),
incDec: new TokenType("++/--", {prefix: true, postfix: true, startsExpr: true}),
prefix: new TokenType("prefix", {beforeExpr: true, prefix: true, startsExpr: true}),
logicalOR: binop("||", 1),
logicalAND: binop("&&", 2),
bitwiseOR: binop("|", 3),
bitwiseXOR: binop("^", 4),
bitwiseAND: binop("&", 5),
equality: binop("==/!=", 6),
relational: binop("</>", 7),
bitShift: binop("<</>>", 8),
plusMin: new TokenType("+/-", {beforeExpr: true, binop: 9, prefix: true, startsExpr: true}),
modulo: binop("%", 10),
star: binop("*", 10),
slash: binop("/", 10),
exponent: new TokenType("**", {beforeExpr: true, binop: 11, rightAssociative: true})
eq: new TokenType("=", { ...beforeExpr, isAssign: true}),
assign: new TokenType("_=", { ...beforeExpr, isAssign: true}),
incDec: new TokenType("++/--", { prefix: true, postfix: true, ...startsExpr }),
prefix: new TokenType("prefix", { ...beforeExpr, prefix: true, ...startsExpr }),
logicalOR: new BinopTokenType("||", 1),
logicalAND: new BinopTokenType("&&", 2),
bitwiseOR: new BinopTokenType("|", 3),
bitwiseXOR: new BinopTokenType("^", 4),
bitwiseAND: new BinopTokenType("&", 5),
equality: new BinopTokenType("==/!=", 6),
relational: new BinopTokenType("</>", 7),
bitShift: new BinopTokenType("<</>>", 8),
plusMin: new TokenType("+/-", { ...beforeExpr, binop: 9, prefix: true, ...startsExpr }),
modulo: new BinopTokenType("%", 10),
star: new BinopTokenType("*", 10),
slash: new BinopTokenType("/", 10),
exponent: new TokenType("**", { ...beforeExpr, binop: 11, rightAssociative: true })
};

// Map keyword names to token types.

export const keywords = {};

// Succinct definitions of keyword token types
function kw(name, options = {}) {
options.keyword = name;
keywords[name] = types["_" + name] = new TokenType(name, options);
}
export const keywords = {
"break": new KeyworkTokenType("break"),
"case": new KeyworkTokenType("case", beforeExpr),
"catch": new KeyworkTokenType("catch"),
"continue": new KeyworkTokenType("continue"),
"debugger": new KeyworkTokenType("debugger"),
"default": new KeyworkTokenType("default", beforeExpr),
"do": new KeyworkTokenType("do", { isLoop: true, ...beforeExpr }),
"else": new KeyworkTokenType("else", beforeExpr),
"finally": new KeyworkTokenType("finally"),
"for": new KeyworkTokenType("for", { isLoop: true }),
"function": new KeyworkTokenType("function", startsExpr),
"if": new KeyworkTokenType("if"),
"return": new KeyworkTokenType("return", beforeExpr),
"switch": new KeyworkTokenType("switch"),
"throw": new KeyworkTokenType("throw", beforeExpr),
"try": new KeyworkTokenType("try"),
"var": new KeyworkTokenType("var"),
"let": new KeyworkTokenType("let"),
"const": new KeyworkTokenType("const"),
"while": new KeyworkTokenType("while", { isLoop: true }),
"with": new KeyworkTokenType("with"),
"new": new KeyworkTokenType("new", beforeAndStartExpr),
"this": new KeyworkTokenType("this", startsExpr),
"super": new KeyworkTokenType("super", startsExpr),
"class": new KeyworkTokenType("class"),
"extends": new KeyworkTokenType("extends", beforeExpr),
"export": new KeyworkTokenType("export"),
"import": new KeyworkTokenType("import"),
"yield": new KeyworkTokenType("yield", beforeAndStartExpr),
"null": new KeyworkTokenType("null", startsExpr),
"true": new KeyworkTokenType("true", startsExpr),
"false": new KeyworkTokenType("false", startsExpr),
"in": new KeyworkTokenType("in", { ...beforeExpr, binop: 7 }),
"instanceof": new KeyworkTokenType("instanceof", { ...beforeExpr, binop: 7 }),
"typeof": new KeyworkTokenType("typeof", { ...beforeExpr, prefix: true, ...startsExpr }),
"void": new KeyworkTokenType("void", { ...beforeExpr, prefix: true, ...startsExpr }),
"delete": new KeyworkTokenType("delete", { ...beforeExpr, prefix: true, ...startsExpr })
};

kw("break");
kw("case", beforeExpr);
kw("catch");
kw("continue");
kw("debugger");
kw("default", beforeExpr);
kw("do", {isLoop: true, beforeExpr: true});
kw("else", beforeExpr);
kw("finally");
kw("for", {isLoop: true});
kw("function", startsExpr);
kw("if");
kw("return", beforeExpr);
kw("switch");
kw("throw", beforeExpr);
kw("try");
kw("var");
kw("let");
kw("const");
kw("while", {isLoop: true});
kw("with");
kw("new", {beforeExpr: true, startsExpr: true});
kw("this", startsExpr);
kw("super", startsExpr);
kw("class");
kw("extends", beforeExpr);
kw("export");
kw("import");
kw("yield", {beforeExpr: true, startsExpr: true});
kw("null", startsExpr);
kw("true", startsExpr);
kw("false", startsExpr);
kw("in", {beforeExpr: true, binop: 7});
kw("instanceof", {beforeExpr: true, binop: 7});
kw("typeof", {beforeExpr: true, prefix: true, startsExpr: true});
kw("void", {beforeExpr: true, prefix: true, startsExpr: true});
kw("delete", {beforeExpr: true, prefix: true, startsExpr: true});
// Map keyword names to token types.
Object.keys(keywords).forEach((name) => {
types["_" + name] = keywords[name];
});