diff --git a/benchmark/parsers/cssDevParser.js b/benchmark/parsers/cssDevParser.js
index ad18960e9..a97601bf8 100644
--- a/benchmark/parsers/cssDevParser.js
+++ b/benchmark/parsers/cssDevParser.js
@@ -230,7 +230,6 @@ function CssParser(input) {
// medium [ COMMA S* medium]*
this.media_list = this.RULE('media_list', function() {
- $.SUBRULE($.medium)
$.MANY_SEP(Comma, function() {
$.SUBRULE2($.medium)
})
@@ -313,6 +312,7 @@ function CssParser(input) {
// selector [ ',' S* selector ]*
// '{' S* declaration? [ ';' S* declaration? ]* '}' S*
this.ruleset = this.RULE('ruleset', function() {
+ // TODO: try without SEP?
$.AT_LEAST_ONE_SEP(Comma, function() {
$.SUBRULE($.selector)
})
diff --git a/benchmark/parsers/cssOldParser.js b/benchmark/parsers/cssOldParser.js
index ea167f4da..6596effa2 100644
--- a/benchmark/parsers/cssOldParser.js
+++ b/benchmark/parsers/cssOldParser.js
@@ -230,7 +230,6 @@ function CssParser(input) {
// medium [ COMMA S* medium]*
this.media_list = this.RULE('media_list', function() {
- $.SUBRULE($.medium)
$.MANY_SEP(Comma, function() {
$.SUBRULE2($.medium)
})
diff --git a/docs/faq.md b/docs/faq.md
index 50302bde5..3eb8fe99d 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -83,6 +83,10 @@ See [related documentation](../examples/parser/minification/README.md) for detai
### How do I Maximize my parser's performance?
+#### Major Performance Benefits
+
+These are highly recommended for each and every parser.
+
1. **Do not create a new Parser instance for each new input**.
Instead re-use a single instance and reset its state between iterations. For example:
@@ -118,8 +122,12 @@ See [related documentation](../examples/parser/minification/README.md) for detai
2. **Choose the optimal Token Type for your use case.**
See [Token Types Docs](docs/token_types.md) for more details.
-
-3. **Avoid creating parsing rules which only parse a single Terminal.**
+
+#### Minor Performance Benefits
+
+These are only required if you are trying to squeeze every tiny bit of performance out of your parser.
+
+1. **Avoid creating parsing rules which only parse a single Terminal.**
There is a certain fixed overhead for the invocation of each parsing rule.
Normally there is no reason to pay it for a Rule which only consumes a single Terminal.
@@ -132,5 +140,47 @@ See [related documentation](../examples/parser/minification/README.md) for detai
```
Instead such a rule's contents should be (manually) in-lined in its call sites.
-
+2. **Avoid *_SEP DSL methods (MANY_SEP / AT_LEAST_ONE_SEP).**
+
+ The *_SEP DSL methods also collect the separator Tokens parsed. Creating these arrays has a small overhead (several percentage).
+ Which is a complete waste in most cases where those separators tokens are not needed for any output data structure.
+
+3. **Use the returned values of iteration DSL methods (MANY/MANY_SEP/AT_LEAST_ONE/AT_LEAST_ONE_SEP).**
+
+ Consider the following grammar rule:
+
+ ```javascript
+ this.RULE("array", function() {
+ let myArr = []
+ $.CONSUME(LSquare);
+ values.push($.SUBRULE($.value));
+ $.MANY(() => {
+ $.CONSUME(Comma);
+ values.push($.SUBRULE2($.value));
+ });
+ $.CONSUME(RSquare);
+ });
+ ```
+
+ The values of the array are manually collected inside the "myArr" array.
+ However another result array is already created by invoking the iteration DSL method "MANY"
+ This is obviously a waste of cpu cycles...
+
+ A slightly more efficient (but syntactically ugly) alternative would be:
+ ```javascript
+ this.RULE("array", function() {
+ let myArr = []
+ $.CONSUME(LSquare);
+ values.push($.SUBRULE($.value));
+
+ let iterationResult =
+ $.MANY(() => {
+ $.CONSUME(Comma);
+ return $.SUBRULE2($.value);
+ });
+
+ myArr = myArr.concat(iterationResult)
+ $.CONSUME(RSquare);
+ });
+ ```
diff --git a/examples/language_services/src/examples/json/parser.ts b/examples/language_services/src/examples/json/parser.ts
index 1bfab405c..e5295619b 100644
--- a/examples/language_services/src/examples/json/parser.ts
+++ b/examples/language_services/src/examples/json/parser.ts
@@ -43,7 +43,7 @@ export class JsonParser extends Parser {
lCurlyTok = this.CONSUME(LCurly)
commas = this.MANY_SEP(Comma, () => {
objectItemPTs.push(this.SUBRULE2(this.objectItem))
- })
+ }).separators
rCurlyTok = this.CONSUME(RCurly)
return PT(ObjectPT, CHILDREN(objectItemPTs,
@@ -68,7 +68,7 @@ export class JsonParser extends Parser {
lSquareTok = this.CONSUME(LSquare)
commas = this.MANY_SEP(Comma, () => {
valuePTs.push(this.SUBRULE(this.value))
- })
+ }).separators
rSquareTok = this.CONSUME(RSquare)
return PT(ArrayPT, CHILDREN(valuePTs,
diff --git a/src/parse/parser_public.ts b/src/parse/parser_public.ts
index 005e38820..342bc7a9b 100644
--- a/src/parse/parser_public.ts
+++ b/src/parse/parser_public.ts
@@ -250,7 +250,13 @@ export interface IParserState {
}
export type Predicate = () => boolean
-export type GrammarAction = () => void
+export type GrammarAction = () => OUT
+
+export type ISeparatedIterationResult =
+ {
+ values:OUT[], // The aggregated results of the values returned by each iteration.
+ separators:ISimpleTokenOrIToken[] // the separator tokens between the iterations
+ }
/**
* Convenience used to express an empty alternative in an OR (alternation).
@@ -764,8 +770,8 @@ export class Parser {
* Convenience method equivalent to OPTION1.
* @see OPTION1
*/
- protected OPTION(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):boolean {
+ protected OPTION(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT {
return this.OPTION1.call(this, predicateOrAction, action)
}
@@ -788,42 +794,42 @@ export class Parser {
* or the grammar action to optionally invoke once.
* @param {Function} [action] - The action to optionally invoke.
*
- * @returns {boolean} - True iff the OPTION's action has been invoked
+ * @returns {OUT}
*/
- protected OPTION1(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):boolean {
+ protected OPTION1(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT {
return this.optionInternal(predicateOrAction, action, 1)
}
/**
* @see OPTION1
*/
- protected OPTION2(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):boolean {
+ protected OPTION2(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT {
return this.optionInternal(predicateOrAction, action, 2)
}
/**
* @see OPTION1
*/
- protected OPTION3(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):boolean {
+ protected OPTION3(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT {
return this.optionInternal(predicateOrAction, action, 3)
}
/**
* @see OPTION1
*/
- protected OPTION4(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):boolean {
+ protected OPTION4(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT {
return this.optionInternal(predicateOrAction, action, 4)
}
/**
* @see OPTION1
*/
- protected OPTION5(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):boolean {
+ protected OPTION5(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT {
return this.optionInternal(predicateOrAction, action, 5)
}
@@ -909,8 +915,8 @@ export class Parser {
* Convenience method equivalent to MANY1.
* @see MANY1
*/
- protected MANY(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):void {
+ protected MANY(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT[] {
return this.MANY1.call(this, predicateOrAction, action)
}
@@ -936,49 +942,51 @@ export class Parser {
* @param {Function} predicateOrAction - The predicate / gate function that implements the constraint on the grammar
* or the grammar action to optionally invoke multiple times.
* @param {Function} [action] - The action to optionally invoke multiple times.
+ *
+ * @returns {OUT[]}
*/
- protected MANY1(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):void {
- this.manyInternal(this.MANY1, 1, predicateOrAction, action)
+ protected MANY1(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT[] {
+ return this.manyInternal(1, predicateOrAction, action, [])
}
/**
* @see MANY1
*/
- protected MANY2(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):void {
- this.manyInternal(this.MANY2, 2, predicateOrAction, action)
+ protected MANY2(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT[] {
+ return this.manyInternal(2, predicateOrAction, action, [])
}
/**
* @see MANY1
*/
- protected MANY3(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):void {
- this.manyInternal(this.MANY3, 3, predicateOrAction, action)
+ protected MANY3(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT[] {
+ return this.manyInternal(3, predicateOrAction, action, [])
}
/**
* @see MANY1
*/
- protected MANY4(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):void {
- this.manyInternal(this.MANY4, 4, predicateOrAction, action)
+ protected MANY4(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT[] {
+ return this.manyInternal(4, predicateOrAction, action, [])
}
/**
* @see MANY1
*/
- protected MANY5(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction):void {
- this.manyInternal(this.MANY5, 5, predicateOrAction, action)
+ protected MANY5(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction):OUT[] {
+ return this.manyInternal(5, predicateOrAction, action, [])
}
/**
* Convenience method equivalent to MANY_SEP1.
* @see MANY_SEP1
*/
- protected MANY_SEP(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] {
+ protected MANY_SEP(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult {
return this.MANY_SEP1.call(this, separator, action)
}
@@ -1004,47 +1012,47 @@ export class Parser {
* @param {TokenConstructor} separator - The Token class which will be used as a separator between repetitions.
* @param {Function} [action] - The action to optionally invoke.
*
- * @return {Token[]} - The consumed separator Tokens.
+ * @return {ISeparatedIterationResult}
*/
- protected MANY_SEP1(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] {
- return this.manySepFirstInternal(this.MANY_SEP1, 1, separator, action)
+ protected MANY_SEP1(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult {
+ return this.manySepFirstInternal(1, separator, action, {values: [], separators: []})
}
/**
* @see MANY_SEP1
*/
- protected MANY_SEP2(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] {
- return this.manySepFirstInternal(this.MANY_SEP2, 2, separator, action)
+ protected MANY_SEP2(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult {
+ return this.manySepFirstInternal(2, separator, action, {values: [], separators: []})
}
/**
* @see MANY_SEP1
*/
- protected MANY_SEP3(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] {
- return this.manySepFirstInternal(this.MANY_SEP3, 3, separator, action)
+ protected MANY_SEP3(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult {
+ return this.manySepFirstInternal(3, separator, action, {values: [], separators: []})
}
/**
* @see MANY_SEP1
*/
- protected MANY_SEP4(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] {
- return this.manySepFirstInternal(this.MANY_SEP4, 4, separator, action)
+ protected MANY_SEP4(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult {
+ return this.manySepFirstInternal(4, separator, action, {values: [], separators: []})
}
/**
* @see MANY_SEP1
*/
- protected MANY_SEP5(separator:TokenConstructor, action:GrammarAction):ISimpleTokenOrIToken[] {
- return this.manySepFirstInternal(this.MANY_SEP5, 5, separator, action)
+ protected MANY_SEP5(separator:TokenConstructor, action:GrammarAction):ISeparatedIterationResult {
+ return this.manySepFirstInternal(5, separator, action, {values: [], separators: []})
}
/**
* Convenience method equivalent to AT_LEAST_ONE1.
* @see AT_LEAST_ONE1
*/
- protected AT_LEAST_ONE(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction | string,
- errMsg?:string):void {
+ protected AT_LEAST_ONE(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction | string,
+ errMsg?:string):void {
return this.AT_LEAST_ONE1.call(this, predicateOrAction, action, errMsg)
}
@@ -1059,56 +1067,58 @@ export class Parser {
* or the grammar action to invoke at least once.
* @param {Function} [action] - The action to optionally invoke.
* @param {string} [errMsg] - Short title/classification to what is being matched.
+ *
+ * @return {OUT[]}
*/
- protected AT_LEAST_ONE1(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction | string,
- errMsg?:string):void {
- this.atLeastOneInternal(this.AT_LEAST_ONE1, 1, predicateOrAction, action, errMsg)
+ protected AT_LEAST_ONE1(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction | string,
+ errMsg?:string):OUT[] {
+ return this.atLeastOneInternal(1, predicateOrAction, action, errMsg, [])
}
/**
* @see AT_LEAST_ONE1
*/
- protected AT_LEAST_ONE2(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction | string,
- errMsg?:string):void {
- this.atLeastOneInternal(this.AT_LEAST_ONE2, 2, predicateOrAction, action, errMsg)
+ protected AT_LEAST_ONE2(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction | string,
+ errMsg?:string):OUT[] {
+ return this.atLeastOneInternal(2, predicateOrAction, action, errMsg, [])
}
/**
* @see AT_LEAST_ONE1
*/
- protected AT_LEAST_ONE3(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction | string,
- errMsg?:string):void {
- this.atLeastOneInternal(this.AT_LEAST_ONE3, 3, predicateOrAction, action, errMsg)
+ protected AT_LEAST_ONE3(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction | string,
+ errMsg?:string):OUT[] {
+ return this.atLeastOneInternal(3, predicateOrAction, action, errMsg, [])
}
/**
* @see AT_LEAST_ONE1
*/
- protected AT_LEAST_ONE4(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction | string,
- errMsg?:string):void {
- this.atLeastOneInternal(this.AT_LEAST_ONE4, 4, predicateOrAction, action, errMsg)
+ protected AT_LEAST_ONE4(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction | string,
+ errMsg?:string):OUT[] {
+ return this.atLeastOneInternal(4, predicateOrAction, action, errMsg, [])
}
/**
* @see AT_LEAST_ONE1
*/
- protected AT_LEAST_ONE5(predicateOrAction:Predicate | GrammarAction,
- action?:GrammarAction | string,
- errMsg?:string):void {
- this.atLeastOneInternal(this.AT_LEAST_ONE5, 5, predicateOrAction, action, errMsg)
+ protected AT_LEAST_ONE5(predicateOrAction:Predicate | GrammarAction,
+ action?:GrammarAction | string,
+ errMsg?:string):OUT[] {
+ return this.atLeastOneInternal(5, predicateOrAction, action, errMsg, [])
}
/**
* Convenience method equivalent to AT_LEAST_ONE_SEP1.
* @see AT_LEAST_ONE1
*/
- protected AT_LEAST_ONE_SEP(separator:TokenConstructor,
- action:GrammarAction | string,
- errMsg?:string):ISimpleTokenOrIToken[] {
+ protected AT_LEAST_ONE_SEP(separator:TokenConstructor,
+ action:GrammarAction | string,
+ errMsg?:string):ISeparatedIterationResult {
return this.AT_LEAST_ONE_SEP1.call(this, separator, action, errMsg)
}
@@ -1123,47 +1133,49 @@ export class Parser {
* @param {TokenConstructor} separator - The Token class which will be used as a separator between repetitions.
* @param {Function} [action] - The action to optionally invoke.
* @param {string} [errMsg] - Short title/classification to what is being matched.
+ *
+ * @return {ISeparatedIterationResult}
*/
- protected AT_LEAST_ONE_SEP1(separator:TokenConstructor,
- action:GrammarAction | string,
- errMsg?:string):ISimpleTokenOrIToken[] {
- return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 1, separator, action, errMsg)
+ protected AT_LEAST_ONE_SEP1(separator:TokenConstructor,
+ action:GrammarAction | string,
+ errMsg?:string):ISeparatedIterationResult {
+ return this.atLeastOneSepFirstInternal(1, separator, action, errMsg, {values: [], separators: []})
}
/**
* @see AT_LEAST_ONE_SEP1
*/
- protected AT_LEAST_ONE_SEP2(separator:TokenConstructor,
- action:GrammarAction | string,
- errMsg?:string):ISimpleTokenOrIToken[] {
- return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 2, separator, action, errMsg)
+ protected AT_LEAST_ONE_SEP2(separator:TokenConstructor,
+ action:GrammarAction | string,
+ errMsg?:string):ISeparatedIterationResult {
+ return this.atLeastOneSepFirstInternal(2, separator, action, errMsg, {values: [], separators: []})
}
/**
* @see AT_LEAST_ONE_SEP1
*/
- protected AT_LEAST_ONE_SEP3(separator:TokenConstructor,
- action:GrammarAction | string,
- errMsg?:string):ISimpleTokenOrIToken[] {
- return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 3, separator, action, errMsg)
+ protected AT_LEAST_ONE_SEP3(separator:TokenConstructor,
+ action:GrammarAction | string,
+ errMsg?:string):ISeparatedIterationResult {
+ return this.atLeastOneSepFirstInternal(3, separator, action, errMsg, {values: [], separators: []})
}
/**
* @see AT_LEAST_ONE_SEP1
*/
- protected AT_LEAST_ONE_SEP4(separator:TokenConstructor,
- action:GrammarAction | string,
- errMsg?:string):ISimpleTokenOrIToken[] {
- return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 4, separator, action, errMsg)
+ protected AT_LEAST_ONE_SEP4(separator:TokenConstructor,
+ action:GrammarAction | string,
+ errMsg?:string):ISeparatedIterationResult {
+ return this.atLeastOneSepFirstInternal(4, separator, action, errMsg, {values: [], separators: []})
}
/**
* @see AT_LEAST_ONE_SEP1
*/
- protected AT_LEAST_ONE_SEP5(separator:TokenConstructor,
- action:GrammarAction | string,
- errMsg?:string):ISimpleTokenOrIToken[] {
- return this.atLeastOneSepFirstInternal(this.atLeastOneSepFirstInternal, 5, separator, action, errMsg)
+ protected AT_LEAST_ONE_SEP5(separator:TokenConstructor,
+ action:GrammarAction | string,
+ errMsg?:string):ISeparatedIterationResult {
+ return this.atLeastOneSepFirstInternal(5, separator, action, errMsg, {values: [], separators: []})
}
/**
@@ -1812,27 +1824,26 @@ export class Parser {
}
// Implementation of parsing DSL
- private optionInternal(predicateOrAction:Predicate | GrammarAction, action:GrammarAction, occurrence:number):boolean {
+ private optionInternal(predicateOrAction:Predicate | GrammarAction, action:GrammarAction, occurrence:number):OUT {
let lookAheadFunc = this.getLookaheadFuncForOption(occurrence)
if (action === undefined) {
action = predicateOrAction
} // predicate present
else if (!(predicateOrAction as Predicate).call(this)) {
- return false
+ return undefined
}
if ((lookAheadFunc).call(this)) {
- action.call(this)
- return true
+ return action.call(this)
}
- return false
+ return undefined
}
- private atLeastOneInternal(prodFunc:Function,
- prodOccurrence:number,
- predicate:Predicate | GrammarAction,
- action:GrammarAction | string,
- userDefinedErrMsg?:string):void {
+ private atLeastOneInternal(prodOccurrence:number,
+ predicate:Predicate | GrammarAction,
+ action:GrammarAction | string,
+ userDefinedErrMsg:string,
+ result:OUT[]):OUT[] {
let lookAheadFunc = this.getLookaheadFuncForAtLeastOne(prodOccurrence)
if (!isFunction(action)) {
userDefinedErrMsg = action
@@ -1846,11 +1857,10 @@ export class Parser {
orgLookAheadFunc.call(this)
}
}
-
if ((lookAheadFunc).call(this)) {
- (action).call(this)
+ result.push((action).call(this))
while ((lookAheadFunc).call(this)) {
- (action).call(this)
+ result.push((action).call(this))
}
}
else {
@@ -1862,36 +1872,39 @@ export class Parser {
// from the tryInRepetitionRecovery(...) will only happen IFF there really are TWO/THREE/.... items.
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
- this.attemptInRepetitionRecovery(prodFunc, [lookAheadFunc, action, userDefinedErrMsg],
+ this.attemptInRepetitionRecovery(this.atLeastOneInternal, [prodOccurrence, lookAheadFunc, action, userDefinedErrMsg, result],
lookAheadFunc, AT_LEAST_ONE_IDX, prodOccurrence, NextTerminalAfterAtLeastOneWalker)
+
+ return result
}
- private atLeastOneSepFirstInternal(prodFunc:Function,
- prodOccurrence:number,
- separator:TokenConstructor,
- action:GrammarAction | string,
- userDefinedErrMsg?:string):ISimpleTokenOrIToken[] {
+ private atLeastOneSepFirstInternal(prodOccurrence:number,
+ separator:TokenConstructor,
+ action:GrammarAction | string,
+ userDefinedErrMsg:string,
+ result:ISeparatedIterationResult):ISeparatedIterationResult {
- let separatorsResult = []
let firstIterationLookaheadFunc = this.getLookaheadFuncForAtLeastOneSep(prodOccurrence)
+ let values = result.values
+ let separators = result.separators
+
// 1st iteration
if (firstIterationLookaheadFunc.call(this)) {
- (action).call(this)
+ values.push((>action).call(this))
let separatorLookAheadFunc = () => {return this.tokenMatcher(this.LA(1), separator)}
// 2nd..nth iterations
- while (separatorLookAheadFunc()) {
+ while (this.tokenMatcher(this.LA(1), separator)) {
// note that this CONSUME will never enter recovery because
// the separatorLookAheadFunc checks that the separator really does exist.
- separatorsResult.push(this.CONSUME(separator));
- (action).call(this)
+ separators.push(this.CONSUME(separator))
+ values.push((>action).call(this))
}
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
this.attemptInRepetitionRecovery(this.repetitionSepSecondInternal,
- [prodOccurrence, separator, separatorLookAheadFunc,
- action, separatorsResult, NextTerminalAfterAtLeastOneSepWalker],
+ [prodOccurrence, separator, separatorLookAheadFunc, action, NextTerminalAfterAtLeastOneSepWalker, result],
separatorLookAheadFunc,
AT_LEAST_ONE_SEP_IDX,
prodOccurrence,
@@ -1901,13 +1914,13 @@ export class Parser {
throw this.raiseEarlyExitException(prodOccurrence, PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR, userDefinedErrMsg)
}
- return separatorsResult
+ return result
}
- private manyInternal(prodFunc:Function,
- prodOccurrence:number,
- predicate:Predicate | GrammarAction,
- action?:GrammarAction):void {
+ private manyInternal(prodOccurrence:number,
+ predicate:Predicate | GrammarAction,
+ action:GrammarAction,
+ result:OUT[]):OUT[] {
let lookaheadFunction = this.getLookaheadFuncForMany(prodOccurrence)
if (action === undefined) {
@@ -1923,64 +1936,67 @@ export class Parser {
}
while (lookaheadFunction.call(this)) {
- action.call(this)
+ result.push(action.call(this))
}
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
- this.attemptInRepetitionRecovery(prodFunc,
- [lookaheadFunction, action],
+ this.attemptInRepetitionRecovery(this.manyInternal,
+ [prodOccurrence, lookaheadFunction, action, result],
lookaheadFunction,
MANY_IDX,
prodOccurrence,
NextTerminalAfterManyWalker)
+
+ return result
}
- private manySepFirstInternal(prodFunc:Function,
- prodOccurrence:number,
- separator:TokenConstructor,
- action:GrammarAction):ISimpleTokenOrIToken[] {
+ private manySepFirstInternal(prodOccurrence:number,
+ separator:TokenConstructor,
+ action:GrammarAction,
+ result:ISeparatedIterationResult):ISeparatedIterationResult {
+ let firstIterationLaFunc = this.getLookaheadFuncForManySep(prodOccurrence)
- let separatorsResult = []
+ let values = result.values
+ let separators = result.separators
- let firstIterationLaFunc = this.getLookaheadFuncForManySep(prodOccurrence)
// 1st iteration
if (firstIterationLaFunc.call(this)) {
- action.call(this)
+ values.push(action.call(this))
let separatorLookAheadFunc = () => {return this.tokenMatcher(this.LA(1), separator)}
// 2nd..nth iterations
- while (separatorLookAheadFunc()) {
+ while (this.tokenMatcher(this.LA(1), separator)) {
// note that this CONSUME will never enter recovery because
// the separatorLookAheadFunc checks that the separator really does exist.
- separatorsResult.push(this.CONSUME(separator))
- action.call(this)
+ separators.push(this.CONSUME(separator))
+ values.push(action.call(this))
}
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
this.attemptInRepetitionRecovery(this.repetitionSepSecondInternal,
- [prodOccurrence, separator, separatorLookAheadFunc, action, separatorsResult, NextTerminalAfterManySepWalker],
+ [prodOccurrence, separator, separatorLookAheadFunc, action, NextTerminalAfterManySepWalker, result],
separatorLookAheadFunc,
MANY_SEP_IDX,
prodOccurrence,
NextTerminalAfterManySepWalker)
}
- return separatorsResult
+ return result
}
- private repetitionSepSecondInternal(prodOccurrence:number,
- separator:TokenConstructor,
- separatorLookAheadFunc:() => boolean,
- action:GrammarAction,
- separatorsResult:ISimpleTokenOrIToken[],
- nextTerminalAfterWalker:typeof AbstractNextTerminalAfterProductionWalker):void {
+ private repetitionSepSecondInternal(prodOccurrence:number,
+ separator:TokenConstructor,
+ separatorLookAheadFunc:() => boolean,
+ action:GrammarAction,
+ nextTerminalAfterWalker:typeof AbstractNextTerminalAfterProductionWalker,
+ result:ISeparatedIterationResult):void {
while (separatorLookAheadFunc()) {
// note that this CONSUME will never enter recovery because
// the separatorLookAheadFunc checks that the separator really does exist.
- separatorsResult.push(this.CONSUME(separator))
- action.call(this)
+ result.separators.push(this.CONSUME(separator))
+ result.values.push(action.call(this))
}
// we can only arrive to this function after an error
@@ -1990,8 +2006,7 @@ export class Parser {
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
/* istanbul ignore else */
this.attemptInRepetitionRecovery(this.repetitionSepSecondInternal,
- [prodOccurrence, separator, separatorLookAheadFunc,
- action, separatorsResult, nextTerminalAfterWalker],
+ [prodOccurrence, separator, separatorLookAheadFunc, action, nextTerminalAfterWalker, result],
separatorLookAheadFunc,
AT_LEAST_ONE_SEP_IDX,
prodOccurrence,
diff --git a/test/parse/recognizer_lookahead_spec.ts b/test/parse/recognizer_lookahead_spec.ts
index 6d90db817..2f6855726 100644
--- a/test/parse/recognizer_lookahead_spec.ts
+++ b/test/parse/recognizer_lookahead_spec.ts
@@ -234,27 +234,27 @@ function defineLookaheadSpecs(contextName, extendToken, createToken, tokenMatche
separators = separators.concat(this.MANY_SEP1(Comma, () => {
this.CONSUME1(OneTok)
total += "1"
- }))
+ }).separators)
separators = separators.concat(this.MANY_SEP2(Comma, () => {
this.CONSUME1(TwoTok)
total += "2"
- }))
+ }).separators)
separators = separators.concat(this.MANY_SEP3(Comma, () => {
this.CONSUME1(ThreeTok)
total += "3"
- }))
+ }).separators)
separators = separators.concat(this.MANY_SEP4(Comma, () => {
this.CONSUME1(FourTok)
total += "4"
- }))
+ }).separators)
separators = separators.concat(this.MANY_SEP5(Comma, () => {
this.CONSUME1(FiveTok)
total += "5"
- }))
+ }).separators)
return {total: total, separators: separators}
}
@@ -410,27 +410,27 @@ function defineLookaheadSpecs(contextName, extendToken, createToken, tokenMatche
separators = separators.concat(this.AT_LEAST_ONE_SEP1(Comma, () => {
this.CONSUME1(OneTok)
total += "1"
- }, "Ones"))
+ }, "Ones").separators)
separators = separators.concat(this.AT_LEAST_ONE_SEP2(Comma, () => {
this.CONSUME1(TwoTok)
total += "2"
- }, "Twos"))
+ }, "Twos").separators)
separators = separators.concat(this.AT_LEAST_ONE_SEP3(Comma, () => {
this.CONSUME1(ThreeTok)
total += "3"
- }, "Threes"))
+ }, "Threes").separators)
separators = separators.concat(this.AT_LEAST_ONE_SEP4(Comma, () => {
this.CONSUME1(FourTok)
total += "4"
- }, "Fours"))
+ }, "Fours").separators)
separators = separators.concat(this.AT_LEAST_ONE_SEP5(Comma, () => {
this.CONSUME1(FiveTok)
total += "5"
- }, "Fives"))
+ }, "Fives").separators)
return {total: total, separators: separators}
}
diff --git a/test/parse/recognizer_spec.ts b/test/parse/recognizer_spec.ts
index e421cccf6..40bea4998 100644
--- a/test/parse/recognizer_spec.ts
+++ b/test/parse/recognizer_spec.ts
@@ -31,10 +31,6 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch
const ALL_TOKENS = [PlusTok, MinusTok, IntTok, IdentTok, DotTok]
augmentTokenClasses(ALL_TOKENS)
- // function isQualifiedNamePart():boolean {
- // return this.LA(1) instanceof DotTok
- // }
-
describe("The Parsing DSL", () => {
it("provides a production SUBRULE1-5 that invokes another rule", () => {
@@ -429,6 +425,180 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch
})
})
+ describe("The Parsing DSL methods are expressions", () => {
+
+ it("OR will return the chosen alternative's grammar action's returned value", () => {
+ class OrExpressionParser extends Parser {
+
+ constructor(input:Token[] = []) {
+ super(input, ALL_TOKENS);
+ (Parser).performSelfAnalysis(this)
+ }
+
+ public or = this.RULE("or", () => {
+ return this.OR([
+ {
+ ALT: () => {
+ this.CONSUME1(MinusTok)
+ return 666
+ }
+ },
+ {
+ ALT: () => {
+ this.CONSUME1(PlusTok)
+ return "bamba"
+ }
+ }
+ ])
+ })
+ }
+
+ let parser = new OrExpressionParser([])
+
+ parser.input = [createToken(MinusTok)]
+ expect(parser.or()).to.equal(666)
+
+ parser.input = [createToken(PlusTok)]
+ expect(parser.or()).to.equal("bamba")
+ })
+
+ it("OPTION will return the grammar action value or undefined if the option was not taken", () => {
+ class OptionExpressionParser extends Parser {
+
+ constructor(input:Token[] = []) {
+ super(input, ALL_TOKENS);
+ (Parser).performSelfAnalysis(this)
+ }
+
+ public option = this.RULE("option", () => {
+ return this.OPTION(() => {
+ this.CONSUME(IdentTok)
+ return "bamba"
+ })
+ })
+ }
+
+ let parser = new OptionExpressionParser([])
+
+ parser.input = [createToken(IdentTok)]
+ expect(parser.option()).to.equal("bamba")
+
+ parser.input = [createToken(IntTok)]
+ expect(parser.option()).to.be.undefined
+ })
+
+ it("MANY will return an array of grammar action values", () => {
+ let num = 0
+ class ManyExpressionParser extends Parser {
+
+ constructor(input:Token[] = []) {
+ super(input, ALL_TOKENS);
+ (Parser).performSelfAnalysis(this)
+ }
+
+ public many = this.RULE("many", () => {
+ return this.MANY(() => {
+ this.CONSUME(IntTok)
+ return num++
+ })
+ })
+ }
+
+ let parser = new ManyExpressionParser([])
+
+ parser.input = [createToken(IntTok), createToken(IntTok), createToken(IntTok)]
+ expect(parser.many()).to.deep.equal([0, 1, 2])
+
+ parser.input = []
+ expect(parser.many()).to.deep.equal([])
+ })
+
+ it("AT_LEAST_ONE will return an array of grammar action values", () => {
+ class AtLeastOneExpressionParser extends Parser {
+
+ constructor(input:Token[] = []) {
+ super(input, ALL_TOKENS);
+ (Parser).performSelfAnalysis(this)
+ }
+
+ public atLeastOne = this.RULE("atLeastOne", () => {
+ let num = 0
+ return this.AT_LEAST_ONE(() => {
+ this.CONSUME(IntTok)
+ num = num + 3
+ return num
+ })
+ })
+ }
+
+ let parser = new AtLeastOneExpressionParser([])
+
+ parser.input = [createToken(IntTok), createToken(IntTok), createToken(IntTok), createToken(IntTok)]
+ expect(parser.atLeastOne()).to.deep.equal([3, 6, 9, 12])
+
+ parser.input = [createToken(IntTok)]
+ expect(parser.atLeastOne()).to.deep.equal([3])
+ })
+
+ it("MANY_SEP will return an array of grammar action values and an array of Separators", () => {
+
+ class ManySepExpressionParser extends Parser {
+
+ constructor(input:Token[] = []) {
+ super(input, ALL_TOKENS);
+ (Parser).performSelfAnalysis(this)
+ }
+
+ public manySep = this.RULE("manySep", () => {
+ let num = 0
+ return this.MANY_SEP(PlusTok, () => {
+ this.CONSUME(IntTok)
+ return num++
+ })
+ })
+ }
+
+ let parser = new ManySepExpressionParser([])
+
+ let separator1 = createToken(PlusTok)
+ let separator2 = createToken(PlusTok)
+ parser.input = [createToken(IntTok), separator1, createToken(IntTok), separator2, createToken(IntTok)]
+ expect(parser.manySep()).to.deep.equal({values: [0, 1, 2], separators: [separator1, separator2]})
+
+ parser.input = []
+ expect(parser.manySep()).to.deep.equal({values: [], separators: []})
+ })
+
+ it("AT_LEAST_ONE_SEP will return an array of grammar action values and an array of Separators", () => {
+ class AtLeastOneSepExpressionParser extends Parser {
+
+ constructor(input:Token[] = []) {
+ super(input, ALL_TOKENS);
+ (Parser).performSelfAnalysis(this)
+ }
+
+ public atLeastOneSep = this.RULE("atLeastOneSep", () => {
+ let num = 0
+ return this.AT_LEAST_ONE_SEP(PlusTok, () => {
+ this.CONSUME(IntTok)
+ num = num + 3
+ return num
+ })
+ })
+ }
+
+ let parser = new AtLeastOneSepExpressionParser([])
+
+ let separator1 = createToken(PlusTok)
+ let separator2 = createToken(PlusTok)
+ parser.input = [createToken(IntTok), separator1, createToken(IntTok), separator2, createToken(IntTok)]
+ expect(parser.atLeastOneSep()).to.deep.equal({values: [3, 6, 9], separators: [separator1, separator2]})
+
+ parser.input = [createToken(IntTok)]
+ expect(parser.atLeastOneSep()).to.deep.equal({values: [3], separators: []})
+ })
+ })
+
describe("The BaseRecognizer", () => {
it("can be initialized without supplying an input vector", () => {
@@ -456,7 +626,7 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch
// see: http://en.wikipedia.org/wiki/Tony_Hoare#Apologies_and_retractions
})
- it("invoking an OPTION will return true/false depending if it succeeded or not", () => {
+ it("invoking an OPTION will return the inner grammar action's value or undefined", () => {
class OptionsReturnValueParser extends Parser {
@@ -468,12 +638,14 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch
public trueOptionRule = this.RULE("trueOptionRule", () => {
return this.OPTION(() => true, () => {
this.CONSUME(IntTok)
+ return true
})
})
public falseOptionRule = this.RULE("falseOptionRule", () => {
return this.OPTION(() => false, () => {
this.CONSUME(IntTok)
+ return false
})
})
}
@@ -482,7 +654,7 @@ function defineRecognizerSpecs(contextName, extendToken, createToken, tokenMatch
expect(successfulOption).to.equal(true)
let failedOption = new OptionsReturnValueParser().falseOptionRule()
- expect(failedOption).to.equal(false)
+ expect(failedOption).to.equal(undefined)
})
it("will return false if a RecognitionException is thrown during backtracking and rethrow any other kind of Exception", () => {