fix bad rebase

microsoft · May 11, 2022 · f260b7f · f260b7f
1 parent d0528a4
commit f260b7f
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/src/vs/editor/standalone/common/monarch/monarchLexer.ts b/src/vs/editor/standalone/common/monarch/monarchLexer.ts
@@ -10,7 +10,7 @@
 
 import { IDisposable } from 'vs/base/common/lifecycle';
 import * as languages from 'vs/editor/common/languages';
-import { NullState, nullTokenizeEncoded } from 'vs/editor/common/languages/nullTokenize';
+import { NullState, nullTokenizeEncoded, nullTokenize } from 'vs/editor/common/languages/nullTokenize';
 import { TokenTheme } from 'vs/editor/common/languages/supports/tokenization';
 import { ILanguageService } from 'vs/editor/common/languages/language';
 import * as monarchCommon from 'vs/editor/standalone/common/monarch/monarchCommon';
@@ -479,6 +479,9 @@ export class MonarchTokenizer implements languages.ITokenizationSupport {
 	}
 
 	public tokenize(line: string, hasEOL: boolean, lineState: languages.IState): languages.TokenizationResult {
+		if (line.length >= this._maxTokenizationLineLength) {
+			return nullTokenize(this._languageId, lineState);
+		}
 		const tokensCollector = new MonarchClassicTokensCollector();
 		const endLineState = this._tokenize(line, hasEOL, <MonarchLineState>lineState, tokensCollector);
 		return tokensCollector.finalize(endLineState);

diff --git a/src/vs/editor/standalone/test/browser/monarch.test.ts b/src/vs/editor/standalone/test/browser/monarch.test.ts
@@ -299,12 +299,14 @@ suite('Monarch', () => {
 		languageService.dispose();
 	});
 
-	test('microsoft/monaco-editor#3025: Check maxTokenizationLineLength before tokenizing', () => {
+	test('microsoft/monaco-editor#3025: Check maxTokenizationLineLength before tokenizing', async () => {
 		const configurationService = new StandaloneConfigurationService();
 		const languageService = new LanguageService();
 
+		// Set maxTokenizationLineLength to 4 so that "ham" works but "hamham" would fail
+		await configurationService.updateValue('editor.maxTokenizationLineLength', 4);
+
 		const tokenizer = createMonarchTokenizer(languageService, 'test', {
-			maxTokenizationLineLength: 4,
 			tokenizer: {
 				root: [
 					{