diff --git a/benchmark/parsers/cssDevParser.js b/benchmark/parsers/cssDevParser.js index ad18960e9..a97601bf8 100644 --- a/benchmark/parsers/cssDevParser.js +++ b/benchmark/parsers/cssDevParser.js @@ -230,7 +230,6 @@ function CssParser(input) { // medium [ COMMA S* medium]* this.media_list = this.RULE('media_list', function() { - $.SUBRULE($.medium) $.MANY_SEP(Comma, function() { $.SUBRULE2($.medium) }) @@ -313,6 +312,7 @@ function CssParser(input) { // selector [ ',' S* selector ]* // '{' S* declaration? [ ';' S* declaration? ]* '}' S* this.ruleset = this.RULE('ruleset', function() { + // TODO: try without SEP? $.AT_LEAST_ONE_SEP(Comma, function() { $.SUBRULE($.selector) }) diff --git a/benchmark/parsers/cssOldParser.js b/benchmark/parsers/cssOldParser.js index ea167f4da..6596effa2 100644 --- a/benchmark/parsers/cssOldParser.js +++ b/benchmark/parsers/cssOldParser.js @@ -230,7 +230,6 @@ function CssParser(input) { // medium [ COMMA S* medium]* this.media_list = this.RULE('media_list', function() { - $.SUBRULE($.medium) $.MANY_SEP(Comma, function() { $.SUBRULE2($.medium) }) diff --git a/docs/faq.md b/docs/faq.md index 50302bde5..3eb8fe99d 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -83,6 +83,10 @@ See [related documentation](../examples/parser/minification/README.md) for detai ### How do I Maximize my parser's performance? +#### Major Performance Benefits + +These are highly recommended for each and every parser. + 1. **Do not create a new Parser instance for each new input**. Instead re-use a single instance and reset its state between iterations. For example: @@ -118,8 +122,12 @@ See [related documentation](../examples/parser/minification/README.md) for detai 2. **Choose the optimal Token Type for your use case.** See [Token Types Docs](docs/token_types.md) for more details. - -3. **Avoid creating parsing rules which only parse a single Terminal.** + +#### Minor Performance Benefits + +These are only required if you are trying to squeeze every tiny bit of performance out of your parser. + +1. **Avoid creating parsing rules which only parse a single Terminal.** There is a certain fixed overhead for the invocation of each parsing rule. Normally there is no reason to pay it for a Rule which only consumes a single Terminal. @@ -132,5 +140,47 @@ See [related documentation](../examples/parser/minification/README.md) for detai ``` Instead such a rule's contents should be (manually) in-lined in its call sites. - +2. **Avoid *_SEP DSL methods (MANY_SEP / AT_LEAST_ONE_SEP).** + + The *_SEP DSL methods also collect the separator Tokens parsed. Creating these arrays has a small overhead (several percentage). + Which is a complete waste in most cases where those separators tokens are not needed for any output data structure. + +3. **Use the returned values of iteration DSL methods (MANY/MANY_SEP/AT_LEAST_ONE/AT_LEAST_ONE_SEP).** + + Consider the following grammar rule: + + ```javascript + this.RULE("array", function() { + let myArr = [] + $.CONSUME(LSquare); + values.push($.SUBRULE($.value)); + $.MANY(() => { + $.CONSUME(Comma); + values.push($.SUBRULE2($.value)); + }); + $.CONSUME(RSquare); + }); + ``` + + The values of the array are manually collected inside the "myArr" array. + However another result array is already created by invoking the iteration DSL method "MANY" + This is obviously a waste of cpu cycles... + + A slightly more efficient (but syntactically ugly) alternative would be: + ```javascript + this.RULE("array", function() { + let myArr = [] + $.CONSUME(LSquare); + values.push($.SUBRULE($.value)); + + let iterationResult = + $.MANY(() => { + $.CONSUME(Comma); + return $.SUBRULE2($.value); + }); + + myArr = myArr.concat(iterationResult) + $.CONSUME(RSquare); + }); + ``` diff --git a/examples/language_services/src/examples/json/parser.ts b/examples/language_services/src/examples/json/parser.ts index 1bfab405c..e5295619b 100644 --- a/examples/language_services/src/examples/json/parser.ts +++ b/examples/language_services/src/examples/json/parser.ts @@ -43,7 +43,7 @@ export class JsonParser extends Parser { lCurlyTok = this.CONSUME(LCurly) commas = this.MANY_SEP(Comma, () => { objectItemPTs.push(this.SUBRULE2(this.objectItem)) - }) + }).separators rCurlyTok = this.CONSUME(RCurly) return PT(ObjectPT, CHILDREN(objectItemPTs, @@ -68,7 +68,7 @@ export class JsonParser extends Parser { lSquareTok = this.CONSUME(LSquare) commas = this.MANY_SEP(Comma, () => { valuePTs.push(this.SUBRULE(this.value)) - }) + }).separators rSquareTok = this.CONSUME(RSquare) return PT(ArrayPT, CHILDREN(valuePTs, diff --git a/src/parse/parser_public.ts b/src/parse/parser_public.ts index 005e38820..342bc7a9b 100644 --- a/src/parse/parser_public.ts +++ b/src/parse/parser_public.ts @@ -250,7 +250,13 @@ export interface IParserState { } export type Predicate = () => boolean -export type GrammarAction = () => void +export type GrammarAction = () => OUT + +export type ISeparatedIterationResult = + { + values:OUT[], // The aggregated results of the values returned by each iteration. + separators:ISimpleTokenOrIToken[] // the separator tokens between the iterations + } /** * Convenience used to express an empty alternative in an OR (alternation). @@ -764,8 +770,8 @@ export class Parser { * Convenience method equivalent to OPTION1. * @see OPTION1 */ - protected OPTION(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):boolean { + protected OPTION(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT { return this.OPTION1.call(this, predicateOrAction, action) } @@ -788,42 +794,42 @@ export class Parser { * or the grammar action to optionally invoke once. * @param {Function} [action] - The action to optionally invoke. * - * @returns {boolean} - True iff the OPTION's action has been invoked + * @returns {OUT} */ - protected OPTION1(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):boolean { + protected OPTION1(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT { return this.optionInternal(predicateOrAction, action, 1) } /** * @see OPTION1 */ - protected OPTION2(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):boolean { + protected OPTION2(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT { return this.optionInternal(predicateOrAction, action, 2) } /** * @see OPTION1 */ - protected OPTION3(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):boolean { + protected OPTION3(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT { return this.optionInternal(predicateOrAction, action, 3) } /** * @see OPTION1 */ - protected OPTION4(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):boolean { + protected OPTION4(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT { return this.optionInternal(predicateOrAction, action, 4) } /** * @see OPTION1 */ - protected OPTION5(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):boolean { + protected OPTION5(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT { return this.optionInternal(predicateOrAction, action, 5) } @@ -909,8 +915,8 @@ export class Parser { * Convenience method equivalent to MANY1. * @see MANY1 */ - protected MANY(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):void { + protected MANY(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT[] { return this.MANY1.call(this, predicateOrAction, action) } @@ -936,49 +942,51 @@ export class Parser { * @param {Function} predicateOrAction - The predicate / gate function that implements the constraint on the grammar * or the grammar action to optionally invoke multiple times. * @param {Function} [action] - The action to optionally invoke multiple times. + * + * @returns {OUT[]} */ - protected MANY1(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):void { - this.manyInternal(this.MANY1, 1, predicateOrAction, action) + protected MANY1(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT[] { + return this.manyInternal(1, predicateOrAction, action, []) } /** * @see MANY1 */ - protected MANY2(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):void { - this.manyInternal(this.MANY2, 2, predicateOrAction, action) + protected MANY2(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT[] { + return this.manyInternal(2, predicateOrAction, action, []) } /** * @see MANY1 */ - protected MANY3(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):void { - this.manyInternal(this.MANY3, 3, predicateOrAction, action) + protected MANY3(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT[] { + return this.manyInternal(3, predicateOrAction, action, []) } /** * @see MANY1 */ - protected MANY4(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):void { - this.manyInternal(this.MANY4, 4, predicateOrAction, action) + protected MANY4(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT[] { + return this.manyInternal(4, predicateOrAction, action, []) } /** * @see MANY1 */ - protected MANY5(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction):void { - this.manyInternal(this.MANY5, 5, predicateOrAction, action) + protected MANY5(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction):OUT[] { + return this.manyInternal(5, predicateOrAction, action, []) } /** * Convenience method equivalent to MANY_SEP1. * @see MANY_SEP1 */ - protected MANY_SEP(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] { + protected MANY_SEP(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult { return this.MANY_SEP1.call(this, separator, action) } @@ -1004,47 +1012,47 @@ export class Parser { * @param {TokenConstructor} separator - The Token class which will be used as a separator between repetitions. * @param {Function} [action] - The action to optionally invoke. * - * @return {Token[]} - The consumed separator Tokens. + * @return {ISeparatedIterationResult} */ - protected MANY_SEP1(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] { - return this.manySepFirstInternal(this.MANY_SEP1, 1, separator, action) + protected MANY_SEP1(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult { + return this.manySepFirstInternal(1, separator, action, {values: [], separators: []}) } /** * @see MANY_SEP1 */ - protected MANY_SEP2(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] { - return this.manySepFirstInternal(this.MANY_SEP2, 2, separator, action) + protected MANY_SEP2(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult { + return this.manySepFirstInternal(2, separator, action, {values: [], separators: []}) } /** * @see MANY_SEP1 */ - protected MANY_SEP3(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] { - return this.manySepFirstInternal(this.MANY_SEP3, 3, separator, action) + protected MANY_SEP3(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult { + return this.manySepFirstInternal(3, separator, action, {values: [], separators: []}) } /** * @see MANY_SEP1 */ - protected MANY_SEP4(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] { - return this.manySepFirstInternal(this.MANY_SEP4, 4, separator, action) + protected MANY_SEP4(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult { + return this.manySepFirstInternal(4, separator, action, {values: [], separators: []}) } /** * @see MANY_SEP1 */ - protected MANY_SEP5(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] { - return this.manySepFirstInternal(this.MANY_SEP5, 5, separator, action) + protected MANY_SEP5(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult { + return this.manySepFirstInternal(5, separator, action, {values: [], separators: []}) } /** * Convenience method equivalent to AT_LEAST_ONE1. * @see AT_LEAST_ONE1 */ - protected AT_LEAST_ONE(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction | string, - errMsg?:string):void { + protected AT_LEAST_ONE(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction | string, + errMsg?:string):void { return this.AT_LEAST_ONE1.call(this, predicateOrAction, action, errMsg) } @@ -1059,56 +1067,58 @@ export class Parser { * or the grammar action to invoke at least once. * @param {Function} [action] - The action to optionally invoke. * @param {string} [errMsg] - Short title/classification to what is being matched. + * + * @return {OUT[]} */ - protected AT_LEAST_ONE1(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction | string, - errMsg?:string):void { - this.atLeastOneInternal(this.AT_LEAST_ONE1, 1, predicateOrAction, action, errMsg) + protected AT_LEAST_ONE1(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction | string, + errMsg?:string):OUT[] { + return this.atLeastOneInternal(1, predicateOrAction, action, errMsg, []) } /** * @see AT_LEAST_ONE1 */ - protected AT_LEAST_ONE2(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction | string, - errMsg?:string):void { - this.atLeastOneInternal(this.AT_LEAST_ONE2, 2, predicateOrAction, action, errMsg) + protected AT_LEAST_ONE2(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction | string, + errMsg?:string):OUT[] { + return this.atLeastOneInternal(2, predicateOrAction, action, errMsg, []) } /** * @see AT_LEAST_ONE1 */ - protected AT_LEAST_ONE3(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction | string, - errMsg?:string):void { - this.atLeastOneInternal(this.AT_LEAST_ONE3, 3, predicateOrAction, action, errMsg) + protected AT_LEAST_ONE3(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction | string, + errMsg?:string):OUT[] { + return this.atLeastOneInternal(3, predicateOrAction, action, errMsg, []) } /** * @see AT_LEAST_ONE1 */ - protected AT_LEAST_ONE4(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction | string, - errMsg?:string):void { - this.atLeastOneInternal(this.AT_LEAST_ONE4, 4, predicateOrAction, action, errMsg) + protected AT_LEAST_ONE4(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction | string, + errMsg?:string):OUT[] { + return this.atLeastOneInternal(4, predicateOrAction, action, errMsg, []) } /** * @see AT_LEAST_ONE1 */ - protected AT_LEAST_ONE5(predicateOrAction:Predicate | GrammarAction, - action?:GrammarAction | string, - errMsg?:string):void { - this.atLeastOneInternal(this.AT_LEAST_ONE5, 5, predicateOrAction, action, errMsg) + protected AT_LEAST_ONE5(predicateOrAction:Predicate | GrammarAction, + action?:GrammarAction | string, + errMsg?:string):OUT[] { + return this.atLeastOneInternal(5, predicateOrAction, action, errMsg, []) } /** * Convenience method equivalent to AT_LEAST_ONE_SEP1. * @see AT_LEAST_ONE1 */ - protected AT_LEAST_ONE_SEP(separator:TokenConstructor, - action:GrammarAction | string, - errMsg?:string):ISimpleTokenOrIToken[] { + protected AT_LEAST_ONE_SEP(separator:TokenConstructor, + action:GrammarAction | string, + errMsg?:string):ISeparatedIterationResult { return this.AT_LEAST_ONE_SEP1.call(this, separator, action, errMsg) } @@ -1123,47 +1133,49 @@ export class Parser { * @param {TokenConstructor} separator - The Token class which will be used as a separator between repetitions. * @param {Function} [action] - The action to optionally invoke. * @param {string} [errMsg] - Short title/classification to what is being matched. + * + * @return {ISeparatedIterationResult} */ - protected AT_LEAST_ONE_SEP1(separator:TokenConstructor, - action:GrammarAction | string, - errMsg?:string):ISimpleTokenOrIToken[] { - return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 1, separator, action, errMsg) + protected AT_LEAST_ONE_SEP1(separator:TokenConstructor, + action:GrammarAction | string, + errMsg?:string):ISeparatedIterationResult { + return this.atLeastOneSepFirstInternal(1, separator, action, errMsg, {values: [], separators: []}) } /** * @see AT_LEAST_ONE_SEP1 */ - protected AT_LEAST_ONE_SEP2(separator:TokenConstructor, - action:GrammarAction | string, - errMsg?:string):ISimpleTokenOrIToken[] { - return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 2, separator, action, errMsg) + protected AT_LEAST_ONE_SEP2(separator:TokenConstructor, + action:GrammarAction | string, + errMsg?:string):ISeparatedIterationResult { + return this.atLeastOneSepFirstInternal(2, separator, action, errMsg, {values: [], separators: []}) } /** * @see AT_LEAST_ONE_SEP1 */ - protected AT_LEAST_ONE_SEP3(separator:TokenConstructor, - action:GrammarAction | string, - errMsg?:string):ISimpleTokenOrIToken[] { - return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 3, separator, action, errMsg) + protected AT_LEAST_ONE_SEP3(separator:TokenConstructor, + action:GrammarAction | string, + errMsg?:string):ISeparatedIterationResult { + return this.atLeastOneSepFirstInternal(3, separator, action, errMsg, {values: [], separators: []}) } /** * @see AT_LEAST_ONE_SEP1 */ - protected AT_LEAST_ONE_SEP4(separator:TokenConstructor, - action:GrammarAction | string, - errMsg?:string):ISimpleTokenOrIToken[] { - return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 4, separator, action, errMsg) + protected AT_LEAST_ONE_SEP4(separator:TokenConstructor, + action:GrammarAction | string, + errMsg?:string):ISeparatedIterationResult { + return this.atLeastOneSepFirstInternal(4, separator, action, errMsg, {values: [], separators: []}) } /** * @see AT_LEAST_ONE_SEP1 */ - protected AT_LEAST_ONE_SEP5(separator:TokenConstructor, - action:GrammarAction | string, - errMsg?:string):ISimpleTokenOrIToken[] { - return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 5, separator, action, errMsg) + protected AT_LEAST_ONE_SEP5(separator:TokenConstructor, + action:GrammarAction | string, + errMsg?:string):ISeparatedIterationResult { + return this.atLeastOneSepFirstInternal(5, separator, action, errMsg, {values: [], separators: []}) } /** @@ -1812,27 +1824,26 @@ export class Parser { } // Implementation of parsing DSL - private optionInternal(predicateOrAction:Predicate | GrammarAction, action:GrammarAction, occurrence:number):boolean { + private optionInternal(predicateOrAction:Predicate | GrammarAction, action:GrammarAction, occurrence:number):OUT { let lookAheadFunc = this.getLookaheadFuncForOption(occurrence) if (action === undefined) { action = predicateOrAction } // predicate present else if (!(predicateOrAction as Predicate).call(this)) { - return false + return undefined } if ((lookAheadFunc).call(this)) { - action.call(this) - return true + return action.call(this) } - return false + return undefined } - private atLeastOneInternal(prodFunc:Function, - prodOccurrence:number, - predicate:Predicate | GrammarAction, - action:GrammarAction | string, - userDefinedErrMsg?:string):void { + private atLeastOneInternal(prodOccurrence:number, + predicate:Predicate | GrammarAction, + action:GrammarAction | string, + userDefinedErrMsg:string, + result:OUT[]):OUT[] { let lookAheadFunc = this.getLookaheadFuncForAtLeastOne(prodOccurrence) if (!isFunction(action)) { userDefinedErrMsg = action @@ -1846,11 +1857,10 @@ export class Parser { orgLookAheadFunc.call(this) } } - if ((lookAheadFunc).call(this)) { - (action).call(this) + result.push((action).call(this)) while ((lookAheadFunc).call(this)) { - (action).call(this) + result.push((action).call(this)) } } else { @@ -1862,36 +1872,39 @@ export class Parser { // from the tryInRepetitionRecovery(...) will only happen IFF there really are TWO/THREE/.... items. // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled - this.attemptInRepetitionRecovery(prodFunc, [lookAheadFunc, action, userDefinedErrMsg], + this.attemptInRepetitionRecovery(this.atLeastOneInternal, [prodOccurrence, lookAheadFunc, action, userDefinedErrMsg, result], lookAheadFunc, AT_LEAST_ONE_IDX, prodOccurrence, NextTerminalAfterAtLeastOneWalker) + + return result } - private atLeastOneSepFirstInternal(prodFunc:Function, - prodOccurrence:number, - separator:TokenConstructor, - action:GrammarAction | string, - userDefinedErrMsg?:string):ISimpleTokenOrIToken[] { + private atLeastOneSepFirstInternal(prodOccurrence:number, + separator:TokenConstructor, + action:GrammarAction | string, + userDefinedErrMsg:string, + result:ISeparatedIterationResult):ISeparatedIterationResult { - let separatorsResult = [] let firstIterationLookaheadFunc = this.getLookaheadFuncForAtLeastOneSep(prodOccurrence) + let values = result.values + let separators = result.separators + // 1st iteration if (firstIterationLookaheadFunc.call(this)) { - (action).call(this) + values.push((>action).call(this)) let separatorLookAheadFunc = () => {return this.tokenMatcher(this.LA(1), separator)} // 2nd..nth iterations - while (separatorLookAheadFunc()) { + while (this.tokenMatcher(this.LA(1), separator)) { // note that this CONSUME will never enter recovery because // the separatorLookAheadFunc checks that the separator really does exist. - separatorsResult.push(this.CONSUME(separator)); - (action).call(this) + separators.push(this.CONSUME(separator)) + values.push((>action).call(this)) } // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled this.attemptInRepetitionRecovery(this.repetitionSepSecondInternal, - [prodOccurrence, separator, separatorLookAheadFunc, - action, separatorsResult, NextTerminalAfterAtLeastOneSepWalker], + [prodOccurrence, separator, separatorLookAheadFunc, action, NextTerminalAfterAtLeastOneSepWalker, result], separatorLookAheadFunc, AT_LEAST_ONE_SEP_IDX, prodOccurrence, @@ -1901,13 +1914,13 @@ export class Parser { throw this.raiseEarlyExitException(prodOccurrence, PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR, userDefinedErrMsg) } - return separatorsResult + return result } - private manyInternal(prodFunc:Function, - prodOccurrence:number, - predicate:Predicate | GrammarAction, - action?:GrammarAction):void { + private manyInternal(prodOccurrence:number, + predicate:Predicate | GrammarAction, + action:GrammarAction, + result:OUT[]):OUT[] { let lookaheadFunction = this.getLookaheadFuncForMany(prodOccurrence) if (action === undefined) { @@ -1923,64 +1936,67 @@ export class Parser { } while (lookaheadFunction.call(this)) { - action.call(this) + result.push(action.call(this)) } // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled - this.attemptInRepetitionRecovery(prodFunc, - [lookaheadFunction, action], + this.attemptInRepetitionRecovery(this.manyInternal, + [prodOccurrence, lookaheadFunction, action, result], lookaheadFunction, MANY_IDX, prodOccurrence, NextTerminalAfterManyWalker) + + return result } - private manySepFirstInternal(prodFunc:Function, - prodOccurrence:number, - separator:TokenConstructor, - action:GrammarAction):ISimpleTokenOrIToken[] { + private manySepFirstInternal(prodOccurrence:number, + separator:TokenConstructor, + action:GrammarAction, + result:ISeparatedIterationResult):ISeparatedIterationResult { + let firstIterationLaFunc = this.getLookaheadFuncForManySep(prodOccurrence) - let separatorsResult = [] + let values = result.values + let separators = result.separators - let firstIterationLaFunc = this.getLookaheadFuncForManySep(prodOccurrence) // 1st iteration if (firstIterationLaFunc.call(this)) { - action.call(this) + values.push(action.call(this)) let separatorLookAheadFunc = () => {return this.tokenMatcher(this.LA(1), separator)} // 2nd..nth iterations - while (separatorLookAheadFunc()) { + while (this.tokenMatcher(this.LA(1), separator)) { // note that this CONSUME will never enter recovery because // the separatorLookAheadFunc checks that the separator really does exist. - separatorsResult.push(this.CONSUME(separator)) - action.call(this) + separators.push(this.CONSUME(separator)) + values.push(action.call(this)) } // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled this.attemptInRepetitionRecovery(this.repetitionSepSecondInternal, - [prodOccurrence, separator, separatorLookAheadFunc, action, separatorsResult, NextTerminalAfterManySepWalker], + [prodOccurrence, separator, separatorLookAheadFunc, action, NextTerminalAfterManySepWalker, result], separatorLookAheadFunc, MANY_SEP_IDX, prodOccurrence, NextTerminalAfterManySepWalker) } - return separatorsResult + return result } - private repetitionSepSecondInternal(prodOccurrence:number, - separator:TokenConstructor, - separatorLookAheadFunc:() => boolean, - action:GrammarAction, - separatorsResult:ISimpleTokenOrIToken[], - nextTerminalAfterWalker:typeof AbstractNextTerminalAfterProductionWalker):void { + private repetitionSepSecondInternal(prodOccurrence:number, + separator:TokenConstructor, + separatorLookAheadFunc:() => boolean, + action:GrammarAction, + nextTerminalAfterWalker:typeof AbstractNextTerminalAfterProductionWalker, + result:ISeparatedIterationResult):void { while (separatorLookAheadFunc()) { // note that this CONSUME will never enter recovery because // the separatorLookAheadFunc checks that the separator really does exist. - separatorsResult.push(this.CONSUME(separator)) - action.call(this) + result.separators.push(this.CONSUME(separator)) + result.values.push(action.call(this)) } // we can only arrive to this function after an error @@ -1990,8 +2006,7 @@ export class Parser { // Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled /* istanbul ignore else */ this.attemptInRepetitionRecovery(this.repetitionSepSecondInternal, - [prodOccurrence, separator, separatorLookAheadFunc, - action, separatorsResult, nextTerminalAfterWalker], + [prodOccurrence, separator, separatorLookAheadFunc, action, nextTerminalAfterWalker, result], separatorLookAheadFunc, AT_LEAST_ONE_SEP_IDX, prodOccurrence, diff --git a/test/parse/recognizer_lookahead_spec.ts b/test/parse/recognizer_lookahead_spec.ts index 6d90db817..2f6855726 100644 --- a/test/parse/recognizer_lookahead_spec.ts +++ b/test/parse/recognizer_lookahead_spec.ts @@ -234,27 +234,27 @@ function defineLookaheadSpecs(contextName, extendToken, createToken, tokenMatche separators = separators.concat(this.MANY_SEP1(Comma, () => { this.CONSUME1(OneTok) total += "1" - })) + }).separators) separators = separators.concat(this.MANY_SEP2(Comma, () => { this.CONSUME1(TwoTok) total += "2" - })) + }).separators) separators = separators.concat(this.MANY_SEP3(Comma, () => { this.CONSUME1(ThreeTok) total += "3" - })) + }).separators) separators = separators.concat(this.MANY_SEP4(Comma, () => { this.CONSUME1(FourTok) total += "4" - })) + }).separators) separators = separators.concat(this.MANY_SEP5(Comma, () => { this.CONSUME1(FiveTok) total += "5" - })) + }).separators) return {total: total, separators: separators} } @@ -410,27 +410,27 @@ function defineLookaheadSpecs(contextName, extendToken, createToken, tokenMatche separators = separators.concat(this.AT_LEAST_ONE_SEP1(Comma, () => { this.CONSUME1(OneTok) total += "1" - }, "Ones")) + }, "Ones").separators) separators = separators.concat(this.AT_LEAST_ONE_SEP2(Comma, () => { this.CONSUME1(TwoTok) total += "2" - }, "Twos")) + }, "Twos").separators) separators = separators.concat(this.AT_LEAST_ONE_SEP3(Comma, () => { this.CONSUME1(ThreeTok) total += "3" - }, "Threes")) + }, "Threes").separators) separators = separators.concat(this.AT_LEAST_ONE_SEP4(Comma, () => { this.CONSUME1(FourTok) total += "4" - }, "Fours")) + }, "Fours").separators) separators = separators.concat(this.AT_LEAST_ONE_SEP5(Comma, () => { this.CONSUME1(FiveTok) total += "5" - }, "Fives")) + }, "Fives").separators) return {total: total, separators: separators} } diff --git a/test/parse/recognizer_spec.ts b/test/parse/recognizer_spec.ts index e421cccf6..40bea4998 100644 --- a/test/parse/recognizer_spec.ts +++ b/test/parse/recognizer_spec.ts @@ -31,10 +31,6 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch const ALL_TOKENS = [PlusTok, MinusTok, IntTok, IdentTok, DotTok] augmentTokenClasses(ALL_TOKENS) - // function isQualifiedNamePart():boolean { - // return this.LA(1) instanceof DotTok - // } - describe("The Parsing DSL", () => { it("provides a production SUBRULE1-5 that invokes another rule", () => { @@ -429,6 +425,180 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch }) }) + describe("The Parsing DSL methods are expressions", () => { + + it("OR will return the chosen alternative's grammar action's returned value", () => { + class OrExpressionParser extends Parser { + + constructor(input:Token[] = []) { + super(input, ALL_TOKENS); + (Parser).performSelfAnalysis(this) + } + + public or = this.RULE("or", () => { + return this.OR([ + { + ALT: () => { + this.CONSUME1(MinusTok) + return 666 + } + }, + { + ALT: () => { + this.CONSUME1(PlusTok) + return "bamba" + } + } + ]) + }) + } + + let parser = new OrExpressionParser([]) + + parser.input = [createToken(MinusTok)] + expect(parser.or()).to.equal(666) + + parser.input = [createToken(PlusTok)] + expect(parser.or()).to.equal("bamba") + }) + + it("OPTION will return the grammar action value or undefined if the option was not taken", () => { + class OptionExpressionParser extends Parser { + + constructor(input:Token[] = []) { + super(input, ALL_TOKENS); + (Parser).performSelfAnalysis(this) + } + + public option = this.RULE("option", () => { + return this.OPTION(() => { + this.CONSUME(IdentTok) + return "bamba" + }) + }) + } + + let parser = new OptionExpressionParser([]) + + parser.input = [createToken(IdentTok)] + expect(parser.option()).to.equal("bamba") + + parser.input = [createToken(IntTok)] + expect(parser.option()).to.be.undefined + }) + + it("MANY will return an array of grammar action values", () => { + let num = 0 + class ManyExpressionParser extends Parser { + + constructor(input:Token[] = []) { + super(input, ALL_TOKENS); + (Parser).performSelfAnalysis(this) + } + + public many = this.RULE("many", () => { + return this.MANY(() => { + this.CONSUME(IntTok) + return num++ + }) + }) + } + + let parser = new ManyExpressionParser([]) + + parser.input = [createToken(IntTok), createToken(IntTok), createToken(IntTok)] + expect(parser.many()).to.deep.equal([0, 1, 2]) + + parser.input = [] + expect(parser.many()).to.deep.equal([]) + }) + + it("AT_LEAST_ONE will return an array of grammar action values", () => { + class AtLeastOneExpressionParser extends Parser { + + constructor(input:Token[] = []) { + super(input, ALL_TOKENS); + (Parser).performSelfAnalysis(this) + } + + public atLeastOne = this.RULE("atLeastOne", () => { + let num = 0 + return this.AT_LEAST_ONE(() => { + this.CONSUME(IntTok) + num = num + 3 + return num + }) + }) + } + + let parser = new AtLeastOneExpressionParser([]) + + parser.input = [createToken(IntTok), createToken(IntTok), createToken(IntTok), createToken(IntTok)] + expect(parser.atLeastOne()).to.deep.equal([3, 6, 9, 12]) + + parser.input = [createToken(IntTok)] + expect(parser.atLeastOne()).to.deep.equal([3]) + }) + + it("MANY_SEP will return an array of grammar action values and an array of Separators", () => { + + class ManySepExpressionParser extends Parser { + + constructor(input:Token[] = []) { + super(input, ALL_TOKENS); + (Parser).performSelfAnalysis(this) + } + + public manySep = this.RULE("manySep", () => { + let num = 0 + return this.MANY_SEP(PlusTok, () => { + this.CONSUME(IntTok) + return num++ + }) + }) + } + + let parser = new ManySepExpressionParser([]) + + let separator1 = createToken(PlusTok) + let separator2 = createToken(PlusTok) + parser.input = [createToken(IntTok), separator1, createToken(IntTok), separator2, createToken(IntTok)] + expect(parser.manySep()).to.deep.equal({values: [0, 1, 2], separators: [separator1, separator2]}) + + parser.input = [] + expect(parser.manySep()).to.deep.equal({values: [], separators: []}) + }) + + it("AT_LEAST_ONE_SEP will return an array of grammar action values and an array of Separators", () => { + class AtLeastOneSepExpressionParser extends Parser { + + constructor(input:Token[] = []) { + super(input, ALL_TOKENS); + (Parser).performSelfAnalysis(this) + } + + public atLeastOneSep = this.RULE("atLeastOneSep", () => { + let num = 0 + return this.AT_LEAST_ONE_SEP(PlusTok, () => { + this.CONSUME(IntTok) + num = num + 3 + return num + }) + }) + } + + let parser = new AtLeastOneSepExpressionParser([]) + + let separator1 = createToken(PlusTok) + let separator2 = createToken(PlusTok) + parser.input = [createToken(IntTok), separator1, createToken(IntTok), separator2, createToken(IntTok)] + expect(parser.atLeastOneSep()).to.deep.equal({values: [3, 6, 9], separators: [separator1, separator2]}) + + parser.input = [createToken(IntTok)] + expect(parser.atLeastOneSep()).to.deep.equal({values: [3], separators: []}) + }) + }) + describe("The BaseRecognizer", () => { it("can be initialized without supplying an input vector", () => { @@ -456,7 +626,7 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch // see: http://en.wikipedia.org/wiki/Tony_Hoare#Apologies_and_retractions }) - it("invoking an OPTION will return true/false depending if it succeeded or not", () => { + it("invoking an OPTION will return the inner grammar action's value or undefined", () => { class OptionsReturnValueParser extends Parser { @@ -468,12 +638,14 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch public trueOptionRule = this.RULE("trueOptionRule", () => { return this.OPTION(() => true, () => { this.CONSUME(IntTok) + return true }) }) public falseOptionRule = this.RULE("falseOptionRule", () => { return this.OPTION(() => false, () => { this.CONSUME(IntTok) + return false }) }) } @@ -482,7 +654,7 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch expect(successfulOption).to.equal(true) let failedOption = new OptionsReturnValueParser().falseOptionRule() - expect(failedOption).to.equal(false) + expect(failedOption).to.equal(undefined) }) it("will return false if a RecognitionException is thrown during backtracking and rethrow any other kind of Exception", () => {