Move lineRange to editor/common/core and adds editor.experimental.asyncTokenizationLogging

build/monaco/monaco.d.ts.recipe

@@ -74,7 +74,8 @@ export interface ICommandHandler {
 #includeAll(vs/editor/common/model): IScrollEvent
 #include(vs/editor/common/diff/smartLinesDiffComputer): IChange, ICharChange, ILineChange
 #include(vs/editor/common/diff/documentDiffProvider): IDocumentDiffProvider, IDocumentDiffProviderOptions, IDocumentDiff
-#include(vs/editor/common/diff/linesDiffComputer): LineRangeMapping, LineRange, RangeMapping
+#include(vs/editor/common/core/lineRange): LineRange
+#include(vs/editor/common/diff/linesDiffComputer): LineRangeMapping, RangeMapping
 #include(vs/editor/common/core/dimension): IDimension
 #includeAll(vs/editor/common/editorCommon): IScrollEvent
 #includeAll(vs/editor/common/textModelEvents):

src/vs/editor/browser/services/editorWorkerService.ts

@@ -27,7 +27,8 @@ import { IEditorWorkerHost } from 'vs/editor/common/services/editorWorkerHost';
 import { ILanguageFeaturesService } from 'vs/editor/common/services/languageFeatures';
 import { IChange } from 'vs/editor/common/diff/smartLinesDiffComputer';
 import { IDocumentDiff, IDocumentDiffProviderOptions } from 'vs/editor/common/diff/documentDiffProvider';
-import { LineRangeMapping, LineRange, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 
 /**
  * Stop syncing a model to the worker if it was not needed for 1 min.

src/vs/editor/common/config/editorConfigurationSchema.ts

@@ -101,6 +101,11 @@ const editorConfiguration: IConfigurationNode = {
 			description: nls.localize('editor.experimental.asyncTokenization', "Controls whether the tokenization should happen asynchronously on a web worker."),
 			tags: ['experimental'],
 		},
+		'editor.experimental.asyncTokenizationLogging': {
+			type: 'boolean',
+			default: false,
+			description: nls.localize('editor.experimental.asyncTokenizationLogging', "Controls whether async tokenization should be logged. For debugging only."),
+		},
 		'editor.language.brackets': {
 			type: ['array', 'null'],
 			default: null, // We want to distinguish the empty array from not configured.

src/vs/editor/common/core/lineRange.ts

@@ -0,0 +1,62 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/**
+ * A range of lines (1-based).
+ */
+export class LineRange {
+	/**
+	 * The start line number.
+	 */
+	public readonly startLineNumber: number;
+
+	/**
+	 * The end line number (exclusive).
+	 */
+	public readonly endLineNumberExclusive: number;
+
+	constructor(
+		startLineNumber: number,
+		endLineNumberExclusive: number,
+	) {
+		this.startLineNumber = startLineNumber;
+		this.endLineNumberExclusive = endLineNumberExclusive;
+	}
+
+	/**
+	 * Indicates if this line range is empty.
+	 */
+	get isEmpty(): boolean {
+		return this.startLineNumber === this.endLineNumberExclusive;
+	}
+
+	/**
+	 * Moves this line range by the given offset of line numbers.
+	 */
+	public delta(offset: number): LineRange {
+		return new LineRange(this.startLineNumber + offset, this.endLineNumberExclusive + offset);
+	}
+
+	/**
+	 * The number of lines this line range spans.
+	 */
+	public get length(): number {
+		return this.endLineNumberExclusive - this.startLineNumber;
+	}
+
+	/**
+	 * Creates a line range that combines this and the given line range.
+	 */
+	public join(other: LineRange): LineRange {
+		return new LineRange(
+			Math.min(this.startLineNumber, other.startLineNumber),
+			Math.max(this.endLineNumberExclusive, other.endLineNumberExclusive)
+		);
+	}
+
+	public toString(): string {
+		return `[${this.startLineNumber},${this.endLineNumberExclusive})`;
+	}
+}

src/vs/editor/common/diff/linesDiffComputer.ts

@@ -3,6 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { LineRange } from 'vs/editor/common/core/lineRange';
 import { Range } from 'vs/editor/common/core/range';
 
 export interface ILinesDiffComputer {
@@ -83,61 +84,3 @@ export class RangeMapping {
 		return `{${this.originalRange.toString()}->${this.modifiedRange.toString()}}`;
 	}
 }
-
-/**
- * A range of lines (1-based).
- */
-export class LineRange {
-	/**
-	 * The start line number.
-	 */
-	public readonly startLineNumber: number;
-
-	/**
-	 * The end line number (exclusive).
-	 */
-	public readonly endLineNumberExclusive: number;
-
-	constructor(
-		startLineNumber: number,
-		endLineNumberExclusive: number,
-	) {
-		this.startLineNumber = startLineNumber;
-		this.endLineNumberExclusive = endLineNumberExclusive;
-	}
-
-	/**
-	 * Indicates if this line range is empty.
-	 */
-	get isEmpty(): boolean {
-		return this.startLineNumber === this.endLineNumberExclusive;
-	}
-
-	/**
-	 * Moves this line range by the given offset of line numbers.
-	 */
-	public delta(offset: number): LineRange {
-		return new LineRange(this.startLineNumber + offset, this.endLineNumberExclusive + offset);
-	}
-
-	/**
-	 * The number of lines this line range spans.
-	 */
-	public get length(): number {
-		return this.endLineNumberExclusive - this.startLineNumber;
-	}
-
-	/**
-	 * Creates a line range that combines this and the given line range.
-	 */
-	public join(other: LineRange): LineRange {
-		return new LineRange(
-			Math.min(this.startLineNumber, other.startLineNumber),
-			Math.max(this.endLineNumberExclusive, other.endLineNumberExclusive)
-		);
-	}
-
-	public toString(): string {
-		return `[${this.startLineNumber},${this.endLineNumberExclusive})`;
-	}
-}

src/vs/editor/common/diff/smartLinesDiffComputer.ts

@@ -5,10 +5,11 @@
 
 import { CharCode } from 'vs/base/common/charCode';
 import { IDiffChange, ISequence, LcsDiff, IDiffResult } from 'vs/base/common/diff/diff';
-import { ILinesDiffComputer, ILinesDiff, ILinesDiffComputerOptions, LineRange, RangeMapping, LineRangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { ILinesDiffComputer, ILinesDiff, ILinesDiffComputerOptions, RangeMapping, LineRangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
 import * as strings from 'vs/base/common/strings';
 import { Range } from 'vs/editor/common/core/range';
 import { assertFn, checkAdjacentItems } from 'vs/base/common/assert';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 
 const MINIMUM_MATCHING_CHARACTER_LENGTH = 3;
 

src/vs/editor/common/diff/standardLinesDiffComputer.ts

@@ -5,13 +5,14 @@
 
 import { assertFn, checkAdjacentItems } from 'vs/base/common/assert';
 import { CharCode } from 'vs/base/common/charCode';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { OffsetRange, SequenceDiff, ISequence } from 'vs/editor/common/diff/algorithms/diffAlgorithm';
 import { DynamicProgrammingDiffing } from 'vs/editor/common/diff/algorithms/dynamicProgrammingDiffing';
 import { optimizeSequenceDiffs, smoothenSequenceDiffs } from 'vs/editor/common/diff/algorithms/joinSequenceDiffs';
 import { MyersDiffAlgorithm } from 'vs/editor/common/diff/algorithms/myersDiffAlgorithm';
-import { ILinesDiff, ILinesDiffComputer, ILinesDiffComputerOptions, LineRange, LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { ILinesDiff, ILinesDiffComputer, ILinesDiffComputerOptions, LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
 
 export class StandardLinesDiffComputer implements ILinesDiffComputer {
 	private readonly dynamicProgrammingDiffing = new DynamicProgrammingDiffing();

src/vs/editor/common/tokens/contiguousMultilineTokens.ts

@@ -9,12 +9,12 @@ import { Position } from 'vs/editor/common/core/position';
 import { IRange } from 'vs/editor/common/core/range';
 import { countEOL } from 'vs/editor/common/core/eolCounter';
 import { ContiguousTokensEditing } from 'vs/editor/common/tokens/contiguousTokensEditing';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 
 /**
  * Represents contiguous tokens over a contiguous range of lines.
  */
 export class ContiguousMultilineTokens {
-
 	public static deserialize(buff: Uint8Array, offset: number, result: ContiguousMultilineTokens[]): number {
 		const view32 = new Uint32Array(buff.buffer);
 		const startLineNumber = readUInt32BE(buff, offset); offset += 4;
@@ -64,6 +64,10 @@ export class ContiguousMultilineTokens {
 		this._tokens = tokens;
 	}
 
+	getLineRange(): LineRange {
+		return new LineRange(this._startLineNumber, this._startLineNumber + this._tokens.length);
+	}
+
 	/**
 	 * @see {@link _tokens}
 	 */

src/vs/editor/standalone/browser/standaloneEditor.ts

@@ -34,7 +34,8 @@ import { EditorCommand, ServicesAccessor } from 'vs/editor/browser/editorExtensi
 import { IMenuItem, MenuRegistry, MenuId } from 'vs/platform/actions/common/actions';
 import { ContextKeyExpr } from 'vs/platform/contextkey/common/contextkey';
 import { PLAINTEXT_LANGUAGE_ID } from 'vs/editor/common/languages/modesRegistry';
-import { LineRange, LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 
 /**
  * Create a new editor under `domElement`.

src/vs/monaco.d.ts

@@ -2294,30 +2294,6 @@ declare namespace monaco.editor {
 		 */
 		readonly changes: LineRangeMapping[];
 	}
-
-	/**
-	 * Maps a line range in the original text model to a line range in the modified text model.
-	 */
-	export class LineRangeMapping {
-		/**
-		 * The line range in the original text model.
-		 */
-		readonly originalRange: LineRange;
-		/**
-		 * The line range in the modified text model.
-		 */
-		readonly modifiedRange: LineRange;
-		/**
-		 * If inner changes have not been computed, this is set to undefined.
-		 * Otherwise, it represents the character-level diff in this line range.
-		 * The original range of each range mapping should be contained in the original line range (same for modified).
-		 * Must not be an empty array.
-		 */
-		readonly innerChanges: RangeMapping[] | undefined;
-		constructor(originalRange: LineRange, modifiedRange: LineRange, innerChanges: RangeMapping[] | undefined);
-		toString(): string;
-	}
-
 	/**
 	 * A range of lines (1-based).
 	 */
@@ -2350,6 +2326,29 @@ declare namespace monaco.editor {
 		toString(): string;
 	}
 
+	/**
+	 * Maps a line range in the original text model to a line range in the modified text model.
+	 */
+	export class LineRangeMapping {
+		/**
+		 * The line range in the original text model.
+		 */
+		readonly originalRange: LineRange;
+		/**
+		 * The line range in the modified text model.
+		 */
+		readonly modifiedRange: LineRange;
+		/**
+		 * If inner changes have not been computed, this is set to undefined.
+		 * Otherwise, it represents the character-level diff in this line range.
+		 * The original range of each range mapping should be contained in the original line range (same for modified).
+		 * Must not be an empty array.
+		 */
+		readonly innerChanges: RangeMapping[] | undefined;
+		constructor(originalRange: LineRange, modifiedRange: LineRange, innerChanges: RangeMapping[] | undefined);
+		toString(): string;
+	}
+
 	/**
 	 * Maps a range in the original text model to a range in the modified text model.
 	 */

src/vs/workbench/contrib/mergeEditor/browser/model/diffComputer.ts

@@ -5,13 +5,14 @@
 
 import { assertFn, checkAdjacentItems } from 'vs/base/common/assert';
 import { IReader } from 'vs/base/common/observable';
-import { LineRange as DiffLineRange, RangeMapping as DiffRangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { RangeMapping as DiffRangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
 import { ITextModel } from 'vs/editor/common/model';
 import { IEditorWorkerService } from 'vs/editor/common/services/editorWorker';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { LineRange } from 'vs/workbench/contrib/mergeEditor/browser/model/lineRange';
 import { DetailedLineRangeMapping, RangeMapping } from 'vs/workbench/contrib/mergeEditor/browser/model/mapping';
 import { observableConfigValue } from 'vs/workbench/contrib/mergeEditor/browser/utils';
+import { LineRange as DiffLineRange } from 'vs/editor/common/core/lineRange';
 
 export interface IMergeDiffComputer {
 	computeDiff(textModel1: ITextModel, textModel2: ITextModel, reader: IReader): Promise<IMergeDiffComputerResult>;

src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts

@@ -144,7 +144,7 @@ export class TextMateWorkerHost implements IDisposable {
 			}
 
 			store.add(keepAliveWhenAttached(textModel, () => {
-				const controller = new TextMateWorkerTokenizerController(textModel, workerProxy, this._languageService.languageIdCodec, tokenStore, INITIAL);
+				const controller = new TextMateWorkerTokenizerController(textModel, workerProxy, this._languageService.languageIdCodec, tokenStore, INITIAL, this._configurationService);
 				this._workerTokenizerControllers.set(textModel.uri.toString(), controller);
 
 				return toDisposable(() => {