Remove abstraction around the concept of token Identity.

It is no longer required as the multiple types of tokens have long since been removed. This seems to provide about 3% performance boost. Fixes #536
Chevrotain · Sep 29, 2017 · d82fb7a · d82fb7a
1 parent a962ffe
commit d82fb7a
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 115 deletions.
diff --git a/src/parse/grammar/lookahead.ts b/src/parse/grammar/lookahead.ts
@@ -15,8 +15,6 @@ import {
     Predicate,
     IAnyOrAlt,
     TokenMatcher,
-    TokenInstanceIdentityFunc,
-    TokenClassIdentityFunc,
     lookAheadSequence
 } from "../parser_public"
 import { TokenConstructor } from "../../scan/lexer_public"
@@ -54,8 +52,6 @@ export function buildLookaheadFuncForOr(
     k: number,
     hasPredicates: boolean,
     tokenMatcher: TokenMatcher,
-    tokenClassIdentityFunc: TokenClassIdentityFunc,
-    tokenIdentityFunc: TokenInstanceIdentityFunc,
     dynamicTokensEnabled: boolean,
     laFuncBuilder: Function
 ): (orAlts?: IAnyOrAlt<any>[]) => number {
@@ -65,8 +61,6 @@ export function buildLookaheadFuncForOr(
         lookAheadPaths,
         hasPredicates,
         tokenMatcher,
-        tokenClassIdentityFunc,
-        tokenIdentityFunc,
         dynamicTokensEnabled
     )
 }
@@ -88,15 +82,11 @@ export function buildLookaheadFuncForOptionalProd(
     ruleGrammar: gast.Rule,
     k: number,
     tokenMatcher: TokenMatcher,
-    tokenClassIdentityFunc: TokenClassIdentityFunc,
-    tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
     dynamicTokensEnabled: boolean,
     prodType: PROD_TYPE,
     lookaheadBuilder: (
         lookAheadSequence,
         TokenMatcher,
-        TokenClassIdentityFunc,
-        TokenInstanceIdentityFunc,
         boolean
     ) => () => boolean
 ): () => boolean {
@@ -110,8 +100,6 @@ export function buildLookaheadFuncForOptionalProd(
     return lookaheadBuilder(
         lookAheadPaths[0],
         tokenMatcher,
-        tokenClassIdentityFunc,
-        tokenInstanceIdentityFunc,
         dynamicTokensEnabled
     )
 }
@@ -122,8 +110,6 @@ export function buildAlternativesLookAheadFunc(
     alts: lookAheadSequence[],
     hasPredicates: boolean,
     tokenMatcher: TokenMatcher,
-    tokenClassIdentityFunc: TokenClassIdentityFunc,
-    tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
     dynamicTokensEnabled: boolean
 ): (orAlts?: IAnyOrAlt<any>[]) => number {
     let numOfAlts = alts.length
@@ -185,8 +171,8 @@ export function buildAlternativesLookAheadFunc(
             singleTokenAlts,
             (result, currAlt, idx) => {
                 forEach(currAlt, currTokClass => {
-                    if (!has(result, tokenClassIdentityFunc(currTokClass))) {
-                        result[tokenClassIdentityFunc(currTokClass)] = idx
+                    if (!has(result, currTokClass.tokenType)) {
+                        result[currTokClass.tokenType] = idx
                     }
                     forEach(
                         currTokClass.extendingTokenTypes,
@@ -207,7 +193,7 @@ export function buildAlternativesLookAheadFunc(
          */
         return function(): number {
             let nextToken = this.LA(1)
-            return choiceToAlt[tokenInstanceIdentityFunc(nextToken)]
+            return choiceToAlt[nextToken.tokenType]
         }
     } else {
         // optimized lookahead without needing to check the predicates at all.
@@ -246,8 +232,6 @@ export function buildAlternativesLookAheadFunc(
 export function buildSingleAlternativeLookaheadFunction(
     alt: lookAheadSequence,
     tokenMatcher: TokenMatcher,
-    tokenClassIdentityFunc: TokenClassIdentityFunc,
-    tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
     dynamicTokensEnabled: boolean
 ): () => boolean {
     let areAllOneTokenLookahead = every(alt, currPath => {
@@ -266,20 +250,16 @@ export function buildSingleAlternativeLookaheadFunction(
             isEmpty((<any>singleTokensClasses[0]).extendingTokenTypes)
         ) {
             let expectedTokenType = singleTokensClasses[0]
-            let expectedTokenUniqueKey = tokenClassIdentityFunc(
-                <any>expectedTokenType
-            )
+            let expectedTokenUniqueKey = (<any>expectedTokenType).tokenType
+
             return function(): boolean {
-                return (
-                    tokenInstanceIdentityFunc(this.LA(1)) ===
-                    expectedTokenUniqueKey
-                )
+                return this.LA(1).tokenType === expectedTokenUniqueKey
             }
         } else {
             let choiceToAlt = reduce(
                 singleTokensClasses,
                 (result, currTokClass, idx) => {
-                    result[tokenClassIdentityFunc(currTokClass)] = true
+                    result[currTokClass.tokenType] = true
                     forEach(
                         currTokClass.extendingTokenTypes,
                         currExtendingType => {
@@ -292,9 +272,7 @@ export function buildSingleAlternativeLookaheadFunction(
             )
             return function(): boolean {
                 let nextToken = this.LA(1)
-                return (
-                    choiceToAlt[tokenInstanceIdentityFunc(nextToken)] === true
-                )
+                return choiceToAlt[nextToken.tokenType] === true
             }
         }
     } else {
@@ -304,7 +282,7 @@ export function buildSingleAlternativeLookaheadFunction(
                 let currPathLength = currPath.length
                 for (let i = 0; i < currPathLength; i++) {
                     let nextToken = this.LA(i + 1)
-                    if (!tokenMatcher(nextToken, currPath[i])) {
+                    if (tokenMatcher(nextToken, currPath[i]) === false) {
                         // mismatch in current path
                         // try the next pth
                         continue nextPath

diff --git a/src/parse/parser_public.ts b/src/parse/parser_public.ts
@@ -79,7 +79,6 @@ import {
 import {
     augmentTokenClasses,
     isExtendingTokenType,
-    tokenStructuredIdentity,
     tokenStructuredMatcher
 } from "../scan/tokens"
 import { CstNode, ICstVisitor } from "./cst/cst_public"
@@ -140,8 +139,6 @@ export type TokenMatcher = (
     token: IToken,
     tokClass: TokenConstructor
 ) => boolean
-export type TokenInstanceIdentityFunc = (tok: IToken) => string
-export type TokenClassIdentityFunc = (tok: TokenConstructor) => string
 
 export type lookAheadSequence = TokenConstructor[][]
 
@@ -575,8 +572,6 @@ export class Parser {
     // The shortName Index must be coded "after" the first 8bits to enable building unique lookahead keys
     private ruleShortNameIdx = 256
     private tokenMatcher: TokenMatcher
-    private tokenClassIdentityFunc: TokenClassIdentityFunc
-    private tokenInstanceIdentityFunc: TokenInstanceIdentityFunc
     private LAST_EXPLICIT_RULE_STACK: number[] = []
     private selfAnalysisDone = false
 
@@ -704,10 +699,8 @@ export class Parser {
             )
         }
 
+        // TODO: do we really want to keep this on the "this" instance?
         this.tokenMatcher = tokenStructuredMatcher
-        this.tokenClassIdentityFunc = tokenStructuredIdentity
-        // same IdentityFunc used in structured Mode
-        this.tokenInstanceIdentityFunc = tokenStructuredIdentity
 
         // always add EOF to the tokenNames -> constructors map. it is useful to assure all the input has been
         // parsed with a clear error message ("expecting EOF but found ...")
@@ -2899,8 +2892,6 @@ export class Parser {
                 this.maxLookahead,
                 hasPredicates,
                 this.tokenMatcher,
-                this.tokenClassIdentityFunc,
-                this.tokenInstanceIdentityFunc,
                 this.dynamicTokensEnabled,
                 this.lookAheadBuilderForAlternatives
             )
@@ -3015,8 +3006,6 @@ export class Parser {
                 ruleGrammar,
                 maxLookahead,
                 this.tokenMatcher,
-                this.tokenClassIdentityFunc,
-                this.tokenInstanceIdentityFunc,
                 this.dynamicTokensEnabled,
                 prodType,
                 this.lookAheadBuilderForOptional
@@ -3241,15 +3230,11 @@ export class Parser {
     protected lookAheadBuilderForOptional(
         alt: lookAheadSequence,
         tokenMatcher: TokenMatcher,
-        tokenClassIdentityFunc: TokenClassIdentityFunc,
-        tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
         dynamicTokensEnabled: boolean
     ): () => boolean {
         return buildSingleAlternativeLookaheadFunction(
             alt,
             tokenMatcher,
-            tokenClassIdentityFunc,
-            tokenInstanceIdentityFunc,
             dynamicTokensEnabled
         )
     }
@@ -3258,16 +3243,12 @@ export class Parser {
         alts: lookAheadSequence[],
         hasPredicates: boolean,
         tokenMatcher: TokenMatcher,
-        tokenClassIdentityFunc: TokenClassIdentityFunc,
-        tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
         dynamicTokensEnabled: boolean
     ): (orAlts?: IAnyOrAlt<any>[]) => number | undefined {
         return buildAlternativesLookAheadFunc(
             alts,
             hasPredicates,
             tokenMatcher,
-            tokenClassIdentityFunc,
-            tokenInstanceIdentityFunc,
             dynamicTokensEnabled
         )
     }

diff --git a/src/scan/tokens.ts b/src/scan/tokens.ts
@@ -28,16 +28,6 @@ export function tokenStructuredMatcher(tokInstance, tokConstructor) {
     }
 }
 
-export function tokenClassIdentity(tokenConstructor: TokenConstructor): string {
-    return (<any>tokenConstructor).tokenType
-}
-
-export function tokenStructuredIdentity(
-    token: TokenConstructor | IToken
-): string {
-    return (<any>token).tokenType
-}
-
 export function isBaseTokenOrObject(tokClass: TokenConstructor): boolean {
     return isBaseTokenClass(tokClass) || <any>tokClass === Object
 }

diff --git a/test/full_flow/ecma_quirks/ecma_quirks.ts b/test/full_flow/ecma_quirks/ecma_quirks.ts
@@ -5,8 +5,6 @@ import {
     IAnyOrAlt,
     lookAheadSequence,
     Parser,
-    TokenClassIdentityFunc,
-    TokenInstanceIdentityFunc,
     TokenMatcher
 } from "../../../src/parse/parser_public"
 import { exceptions } from "../../../src/parse/exceptions_public"
@@ -184,8 +182,6 @@ class EcmaScriptQuirksParser extends Parser {
     protected lookAheadBuilderForOptional(
         alt: lookAheadSequence,
         tokenMatcher: TokenMatcher,
-        tokenClassIdentityFunc: TokenClassIdentityFunc,
-        tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
         dynamicTokensEnabled: boolean
     ): () => boolean {
         if (!every(alt, currAlt => currAlt.length === 1)) {
@@ -221,8 +217,6 @@ class EcmaScriptQuirksParser extends Parser {
         alts: lookAheadSequence[],
         hasPredicates: boolean,
         tokenMatcher: TokenMatcher,
-        tokenClassIdentityFunc: TokenClassIdentityFunc,
-        tokenInstanceIdentityFunc: TokenInstanceIdentityFunc,
         dynamicTokensEnabled: boolean
     ): (orAlts?: IAnyOrAlt<any>[]) => number | undefined {
         if (