Skip to content

Commit

Permalink
No prototype inheritance for Token Types. (#608)
Browse files Browse the repository at this point in the history 
Fixes #607 
Fixes  #564
  • Loading branch information
Shahar Soel committed Dec 4, 2017
1 parent 8cf81ca commit f8c087d
Show file tree
Hide file tree
Showing 66 changed files with 1,363 additions and 1,317 deletions.
1 change: 1 addition & 0 deletions .idea/chevrotain.iml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

32 changes: 32 additions & 0 deletions .idea/codeStyles/Project.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions .idea/codeStyles/codeStyleConfig.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 14 additions & 14 deletions docs/tutorial/src/step1_lexing/step1_lexing.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,24 @@ let tokenVocabulary = {}
// So use chevrotain.tokenMatcher instead of "instanceof" when matching
const Identifier = createToken({ name: "Identifier", pattern: /[a-zA-Z]\w*/ })

// This is an "abstract" Token type used to categorize all the keywords
const Keyword = createToken({
name: "Keyword",
pattern: Lexer.NA,
// the longer alt property allows resolving the keywords vs identifiers ambiguity
// See: https://github.com/SAP/chevrotain/blob/master/examples/lexer/keywords_vs_identifiers/keywords_vs_identifiers.js
longer_alt: Identifier
})

// By indicating that these Token types parent is "Keyword" they will inherit the "longer_alt"
// property to resolve keywords vs identifiers ambiguity.
// We specify the "longer_alt" property to resolve keywords vs identifiers ambiguity.
// See: https://github.com/SAP/chevrotain/blob/master/examples/lexer/keywords_vs_identifiers/keywords_vs_identifiers.js
const Select = createToken({
name: "Select",
pattern: /SELECT/,
parent: Keyword
longer_alt: Identifier
})

const From = createToken({
name: "From",
pattern: /FROM/,
longer_alt: Identifier
})
const Where = createToken({
name: "Where",
pattern: /WHERE/,
longer_alt: Identifier
})
const From = createToken({ name: "From", pattern: /FROM/, parent: Keyword })
const Where = createToken({ name: "Where", pattern: /WHERE/, parent: Keyword })

const Comma = createToken({ name: "Comma", pattern: /,/ })
const Integer = createToken({ name: "Integer", pattern: /0|[1-9]\d*/ })
Expand Down
3 changes: 2 additions & 1 deletion docs/tutorial/src/step4_error_recovery/step4_error_recovery_spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ describe("Chevrotain Tutorial", () => {
endLine: NaN,
startColumn: NaN,
endColumn: NaN,
isInsertedInRecovery: true
isInsertedInRecovery: true,
type: errRecovery.Colon
}
],
// the value rule appears AFTER the error (missing colon) yet it was still parsed successfully
Expand Down
16 changes: 5 additions & 11 deletions docs/tutorial/step1_lexing.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,17 +71,11 @@ const WhiteSpace = createToken({

```javascript

const Keyword = createToken({
name: "Keyword",
pattern: Lexer.NA,
// the longer alt property allows resolving the keywords vs identifiers ambiguity
// See: https://github.com/SAP/chevrotain/blob/master/examples/lexer/keywords_vs_identifiers/keywords_vs_identifiers.js
longer_alt:tokenVocabulary.Identifier
});

const Select = createToken({name: "Select", pattern: /SELECT/, parent: Keyword});
const From = createToken({name: "From", pattern: /FROM/, parent: Keyword});
const Where = createToken({name: "Where", pattern: /WHERE/, parent: Keyword});
// We specify the "longer_alt" property to resolve keywords vs identifiers ambiguity.
// See: https://github.com/SAP/chevrotain/blob/master/examples/lexer/keywords_vs_identifiers/keywords_vs_identifiers.js
const Select = createToken({name: "Select", pattern: /SELECT/, longer_alt: Identifier});
const From = createToken({name: "From", pattern: /FROM/, longer_alt: Identifier});
const Where = createToken({name: "Where", pattern: /WHERE/, longer_alt: Identifier});

const Comma = createToken({name: "Comma", pattern: /,/});

Expand Down
14 changes: 7 additions & 7 deletions examples/grammars/calculator/calculator_embedded_actions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ var AdditionOperator = createToken({
var Plus = createToken({
name: "Plus",
pattern: /\+/,
parent: AdditionOperator
categories: AdditionOperator
})
var Minus = createToken({
name: "Minus",
pattern: /-/,
parent: AdditionOperator
categories: AdditionOperator
})

var MultiplicationOperator = createToken({
Expand All @@ -42,12 +42,12 @@ var MultiplicationOperator = createToken({
var Multi = createToken({
name: "Multi",
pattern: /\*/,
parent: MultiplicationOperator
categories: MultiplicationOperator
})
var Div = createToken({
name: "Div",
pattern: /\//,
parent: MultiplicationOperator
categories: MultiplicationOperator
})

var LParen = createToken({ name: "LParen", pattern: /\(/ })
Expand Down Expand Up @@ -141,11 +141,11 @@ function Calculator(input) {

$.RULE("atomicExpression", function() {
return $.OR([
// parenthesisExpression has the highest precedence and thus it appears
// categorieshesisExpression has the highest precedence and thus it appears
// in the "lowest" leaf in the expression ParseTree.
{
ALT: function() {
return $.SUBRULE($.parenthesisExpression)
return $.SUBRULE($.categorieshesisExpression)
}
},
{
Expand All @@ -161,7 +161,7 @@ function Calculator(input) {
])
})

$.RULE("parenthesisExpression", function() {
$.RULE("categorieshesisExpression", function() {
var expValue

$.CONSUME(LParen)
Expand Down
22 changes: 11 additions & 11 deletions examples/grammars/calculator/calculator_pure_grammar.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,12 @@ var AdditionOperator = createToken({
var Plus = createToken({
name: "Plus",
pattern: /\+/,
parent: AdditionOperator
categories: AdditionOperator
})
var Minus = createToken({
name: "Minus",
pattern: /-/,
parent: AdditionOperator
categories: AdditionOperator
})

var MultiplicationOperator = createToken({
Expand All @@ -43,12 +43,12 @@ var MultiplicationOperator = createToken({
var Multi = createToken({
name: "Multi",
pattern: /\*/,
parent: MultiplicationOperator
categories: MultiplicationOperator
})
var Div = createToken({
name: "Div",
pattern: /\//,
parent: MultiplicationOperator
categories: MultiplicationOperator
})

var LParen = createToken({ name: "LParen", pattern: /\(/ })
Expand Down Expand Up @@ -118,11 +118,11 @@ function CalculatorPure(input) {

$.RULE("atomicExpression", function() {
return $.OR([
// parenthesisExpression has the highest precedence and thus it appears
// categorieshesisExpression has the highest precedence and thus it appears
// in the "lowest" leaf in the expression ParseTree.
{
ALT: function() {
return $.SUBRULE($.parenthesisExpression)
return $.SUBRULE($.categorieshesisExpression)
}
},
{
Expand All @@ -138,7 +138,7 @@ function CalculatorPure(input) {
])
})

$.RULE("parenthesisExpression", function() {
$.RULE("categorieshesisExpression", function() {
$.CONSUME(LParen)
$.SUBRULE($.expression)
$.CONSUME(RParen)
Expand Down Expand Up @@ -219,18 +219,18 @@ class CalculatorInterpreter extends BaseCstVisitor {
}

atomicExpression(ctx) {
if (ctx.parenthesisExpression.length > 0) {
if (ctx.categorieshesisExpression.length > 0) {
// TODO: allow accepting array for less verbose syntax
return this.visit(ctx.parenthesisExpression[0])
return this.visit(ctx.categorieshesisExpression[0])
} else if (ctx.NumberLiteral.length > 0) {
return parseInt(ctx.NumberLiteral[0].image, 10)
} else if (ctx.powerFunction.length > 0) {
return this.visit(ctx.powerFunction[0])
}
}

parenthesisExpression(ctx) {
// The ctx will also contain the parenthesis tokens, but we don't care about those
categorieshesisExpression(ctx) {
// The ctx will also contain the categorieshesis tokens, but we don't care about those
// in the context of calculating the result.
return this.visit(ctx.expression[0])
}
Expand Down
30 changes: 15 additions & 15 deletions examples/grammars/css/css.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,14 @@
pattern: MAKE_PATTERN(
"url\\((:?{{spaces}}})?({{string1}}|{{string2}})(:?{{spaces}})?\\)"
),
parent: Uri
categories: Uri
})
var UriUrl = createToken({
name: "UriUrl",
pattern: MAKE_PATTERN(
"url\\((:?{{spaces}}})?{{url}}(:?{{spaces}})?\\)"
),
parent: Uri
categories: Uri
})
var Func = createToken({
name: "Func",
Expand Down Expand Up @@ -171,73 +171,73 @@
var Px = createToken({
name: "Px",
pattern: MAKE_PATTERN("{{num}}px", "i"),
parent: Length
categories: Length
})
var Cm = createToken({
name: "Cm",
pattern: MAKE_PATTERN("{{num}}cm", "i"),
parent: Length
categories: Length
})
var Mm = createToken({
name: "Mm",
pattern: MAKE_PATTERN("{{num}}mm", "i"),
parent: Length
categories: Length
})
var In = createToken({
name: "In",
pattern: MAKE_PATTERN("{{num}}in", "i"),
parent: Length
categories: Length
})
var Pt = createToken({
name: "Pt",
pattern: MAKE_PATTERN("{{num}}pt", "i"),
parent: Length
categories: Length
})
var Pc = createToken({
name: "Pc",
pattern: MAKE_PATTERN("{{num}}pc", "i"),
parent: Length
categories: Length
})

var Angle = createToken({ name: "Angle", pattern: Lexer.NA })
var Deg = createToken({
name: "Deg",
pattern: MAKE_PATTERN("{{num}}deg", "i"),
parent: Angle
categories: Angle
})
var Rad = createToken({
name: "Rad",
pattern: MAKE_PATTERN("{{num}}rad", "i"),
parent: Angle
categories: Angle
})
var Grad = createToken({
name: "Grad",
pattern: MAKE_PATTERN("{{num}}grad", "i"),
parent: Angle
categories: Angle
})

var Time = createToken({ name: "Time", pattern: Lexer.NA })
var Ms = createToken({
name: "Ms",
pattern: MAKE_PATTERN("{{num}}ms", "i"),
parent: Time
categories: Time
})
var Sec = createToken({
name: "Sec",
pattern: MAKE_PATTERN("{{num}}sec", "i"),
parent: Time
categories: Time
})

var Freq = createToken({ name: "Freq", pattern: Lexer.NA })
var Hz = createToken({
name: "Hz",
pattern: MAKE_PATTERN("{{num}}hz", "i"),
parent: Freq
categories: Freq
})
var Khz = createToken({
name: "Khz",
pattern: MAKE_PATTERN("{{num}}khz", "i"),
parent: Freq
categories: Freq
})

var Percentage = createToken({
Expand Down
Loading

0 comments on commit f8c087d

Please sign in to comment.