From a3e737dacf6edb38f07f888de61cb956aaf68898 Mon Sep 17 00:00:00 2001 From: Carl Morris Date: Thu, 7 Feb 2019 18:02:09 -0600 Subject: [PATCH 1/3] no while checks if scanning a capture --- src/grammar.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/grammar.ts b/src/grammar.ts index 2cb1052e..a5967be2 100644 --- a/src/grammar.ts +++ b/src/grammar.ts @@ -757,12 +757,16 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo const lineLength = lineText.content.length; let STOP = false; + let whileCheckResult: IWhileCheckResult - let whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); - stack = whileCheckResult.stack; - linePos = whileCheckResult.linePos; - isFirstLine = whileCheckResult.isFirstLine; - let anchorPosition = whileCheckResult.anchorPosition; + // check while conditions only if input starting position is begining of the string + if (linePos == 0) { + whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); + stack = whileCheckResult.stack; + linePos = whileCheckResult.linePos; + isFirstLine = whileCheckResult.isFirstLine; + } + let anchorPosition = (linePos == 0) ? whileCheckResult.anchorPosition : -1; while (!STOP) { scanNext(); // potentially modifies linePos && anchorPosition From 489094054d1bfde89a3f5b83cd2ae0a36a68321f Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 12 Jul 2019 11:27:10 +0200 Subject: [PATCH 2/3] Simplify code --- src/grammar.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/grammar.ts b/src/grammar.ts index a5967be2..c76b6c3d 100644 --- a/src/grammar.ts +++ b/src/grammar.ts @@ -757,16 +757,16 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo const lineLength = lineText.content.length; let STOP = false; - let whileCheckResult: IWhileCheckResult + let anchorPosition = -1; // check while conditions only if input starting position is begining of the string - if (linePos == 0) { - whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); + if (linePos === 0) { + let whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); stack = whileCheckResult.stack; linePos = whileCheckResult.linePos; isFirstLine = whileCheckResult.isFirstLine; + anchorPosition = whileCheckResult.anchorPosition; } - let anchorPosition = (linePos == 0) ? whileCheckResult.anchorPosition : -1; while (!STOP) { scanNext(); // potentially modifies linePos && anchorPosition From 2d17859597a71c4694e7def67d0beaa59c07f78a Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 12 Jul 2019 12:57:49 +0200 Subject: [PATCH 3/3] Make sure that even capture rules starting at position 0 are not checked for while conditions --- release/main.js | 20 +++++++++++--------- src/grammar.ts | 14 +++++--------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/release/main.js b/release/main.js index a79ca3b3..68e55367 100644 --- a/release/main.js +++ b/release/main.js @@ -2317,7 +2317,6 @@ function nameMatcher(identifers, scopes) { return false; }); } -; function collectInjections(result, selector, rule, ruleFactoryHelper, grammar) { var matchers = matcher_1.createMatchers(selector, nameMatcher); var ruleId = rule_1.RuleFactory.getCompiledRuleId(rule, ruleFactoryHelper, grammar.repository); @@ -2579,7 +2578,7 @@ var Grammar = /** @class */ (function () { var onigLineText = this.createOnigString(lineText); var lineLength = onigLineText.content.length; var lineTokens = new LineTokens(emitBinaryTokens, lineText, this._tokenTypeMatchers); - var nextState = _tokenizeString(this, onigLineText, isFirstLine, 0, prevState, lineTokens); + var nextState = _tokenizeString(this, onigLineText, isFirstLine, 0, prevState, lineTokens, true); disposeOnigString(onigLineText); return { lineLength: lineLength, @@ -2649,7 +2648,7 @@ function handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, captu var contentNameScopesList = nameScopesList.push(grammar, contentName); var stackClone = stack.push(captureRule.retokenizeCapturedWithRuleId, captureIndex.start, -1, null, nameScopesList, contentNameScopesList); var onigSubStr = grammar.createOnigString(lineTextContent.substring(0, captureIndex.end)); - _tokenizeString(grammar, onigSubStr, (isFirstLine && captureIndex.start === 0), captureIndex.start, stackClone, lineTokens); + _tokenizeString(grammar, onigSubStr, (isFirstLine && captureIndex.start === 0), captureIndex.start, stackClone, lineTokens, false); disposeOnigString(onigSubStr); continue; } @@ -2814,14 +2813,17 @@ function _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, l } return { stack: stack, linePos: linePos, anchorPosition: anchorPosition, isFirstLine: isFirstLine }; } -function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTokens) { +function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTokens, checkWhileConditions) { var lineLength = lineText.content.length; var STOP = false; - var whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); - stack = whileCheckResult.stack; - linePos = whileCheckResult.linePos; - isFirstLine = whileCheckResult.isFirstLine; - var anchorPosition = whileCheckResult.anchorPosition; + var anchorPosition = -1; + if (checkWhileConditions) { + var whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); + stack = whileCheckResult.stack; + linePos = whileCheckResult.linePos; + isFirstLine = whileCheckResult.isFirstLine; + anchorPosition = whileCheckResult.anchorPosition; + } while (!STOP) { scanNext(); // potentially modifies linePos && anchorPosition } diff --git a/src/grammar.ts b/src/grammar.ts index 39ebe116..8d6c4a67 100644 --- a/src/grammar.ts +++ b/src/grammar.ts @@ -137,7 +137,7 @@ function nameMatcher(identifers: string[], scopes: string[]) { } return false; }); -}; +} function collectInjections(result: Injection[], selector: string, rule: IRawRule, ruleFactoryHelper: IRuleFactoryHelper, grammar: IRawGrammar): void { let matchers = createMatchers(selector, nameMatcher); @@ -455,7 +455,7 @@ export class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib { let onigLineText = this.createOnigString(lineText); let lineLength = onigLineText.content.length; let lineTokens = new LineTokens(emitBinaryTokens, lineText, this._tokenTypeMatchers); - let nextState = _tokenizeString(this, onigLineText, isFirstLine, 0, prevState, lineTokens); + let nextState = _tokenizeString(this, onigLineText, isFirstLine, 0, prevState, lineTokens, true); disposeOnigString(onigLineText); @@ -538,10 +538,7 @@ function handleCaptures(grammar: Grammar, lineText: OnigString, isFirstLine: boo let stackClone = stack.push(captureRule.retokenizeCapturedWithRuleId, captureIndex.start, -1, null, nameScopesList, contentNameScopesList); let onigSubStr = grammar.createOnigString(lineTextContent.substring(0, captureIndex.end)); - _tokenizeString(grammar, - onigSubStr, - (isFirstLine && captureIndex.start === 0), captureIndex.start, stackClone, lineTokens - ); + _tokenizeString(grammar, onigSubStr, (isFirstLine && captureIndex.start === 0), captureIndex.start, stackClone, lineTokens, false); disposeOnigString(onigSubStr); continue; } @@ -753,14 +750,13 @@ function _checkWhileConditions(grammar: Grammar, lineText: OnigString, isFirstLi return { stack: stack, linePos: linePos, anchorPosition: anchorPosition, isFirstLine: isFirstLine }; } -function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, lineTokens: LineTokens): StackElement { +function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, lineTokens: LineTokens, checkWhileConditions: boolean): StackElement { const lineLength = lineText.content.length; let STOP = false; let anchorPosition = -1; - // check while conditions only if input starting position is begining of the string - if (linePos === 0) { + if (checkWhileConditions) { let whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); stack = whileCheckResult.stack; linePos = whileCheckResult.linePos;