Use markdown engine to enable/disable smart paste

extensions/markdown-language-features/package.json

@@ -756,6 +756,7 @@
     "@types/picomatch": "^2.3.0",
     "@types/vscode-notebook-renderer": "^1.60.0",
     "@types/vscode-webview": "^1.57.0",
+    "@vscode/markdown-it-katex": "^1.0.1",
     "lodash.throttle": "^4.1.1",
     "vscode-languageserver-types": "^3.17.2",
     "vscode-markdown-languageservice": "^0.3.0-alpha.3"

extensions/markdown-language-features/src/extension.shared.ts

@@ -7,13 +7,13 @@ import * as vscode from 'vscode';
 import { MdLanguageClient } from './client/client';
 import { CommandManager } from './commandManager';
 import { registerMarkdownCommands } from './commands/index';
-import { registerLinkPasteSupport } from './languageFeatures/copyFiles/pasteUrlProvider';
+import { registerPasteUrlSupport } from './languageFeatures/copyFiles/pasteUrlProvider';
 import { registerResourceDropOrPasteSupport } from './languageFeatures/copyFiles/dropOrPasteResource';
 import { registerDiagnosticSupport } from './languageFeatures/diagnostics';
 import { registerFindFileReferenceSupport } from './languageFeatures/fileReferences';
 import { registerUpdateLinksOnRename } from './languageFeatures/linkUpdater';
 import { ILogger } from './logging';
-import { MarkdownItEngine } from './markdownEngine';
+import { IMdParser, MarkdownItEngine } from './markdownEngine';
 import { MarkdownContributionProvider } from './markdownExtensions';
 import { MdDocumentRenderer } from './preview/documentRenderer';
 import { MarkdownPreviewManager } from './preview/previewManager';
@@ -40,7 +40,7 @@ export function activateShared(
 	const previewManager = new MarkdownPreviewManager(contentProvider, logger, contributions, opener);
 	context.subscriptions.push(previewManager);
 
-	context.subscriptions.push(registerMarkdownLanguageFeatures(client, commandManager));
+	context.subscriptions.push(registerMarkdownLanguageFeatures(client, commandManager, engine));
 	context.subscriptions.push(registerMarkdownCommands(commandManager, previewManager, telemetryReporter, cspArbiter, engine));
 
 	context.subscriptions.push(vscode.workspace.onDidChangeConfiguration(() => {
@@ -51,15 +51,15 @@ export function activateShared(
 function registerMarkdownLanguageFeatures(
 	client: MdLanguageClient,
 	commandManager: CommandManager,
+	parser: IMdParser,
 ): vscode.Disposable {
 	const selector: vscode.DocumentSelector = { language: 'markdown', scheme: '*' };
 	return vscode.Disposable.from(
 		// Language features
 		registerDiagnosticSupport(selector, commandManager),
 		registerFindFileReferenceSupport(commandManager, client),
 		registerResourceDropOrPasteSupport(selector),
-		registerLinkPasteSupport(selector),
+		registerPasteUrlSupport(selector, parser),
 		registerUpdateLinksOnRename(client),
 	);
 }
-

extensions/markdown-language-features/src/languageFeatures/copyFiles/pasteUrlProvider.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as vscode from 'vscode';
+import { IMdParser } from '../../markdownEngine';
 import { ITextDocument } from '../../types/textDocument';
 import { Mime } from '../../util/mimes';
 import { createInsertUriListEdit, externalUriSchemes } from './shared';
@@ -31,6 +32,10 @@ class PasteUrlEditProvider implements vscode.DocumentPasteEditProvider {
 
 	public static readonly pasteMimeTypes = [Mime.textPlain];
 
+	constructor(
+		private readonly _parser: IMdParser,
+	) { }
+
 	async provideDocumentPasteEdits(
 		document: vscode.TextDocument,
 		ranges: readonly vscode.Range[],
@@ -63,53 +68,67 @@ class PasteUrlEditProvider implements vscode.DocumentPasteEditProvider {
 		workspaceEdit.set(document.uri, edit.edits);
 		pasteEdit.additionalEdit = workspaceEdit;
 
-		if (!shouldInsertMarkdownLinkByDefault(document, pasteUrlSetting, ranges)) {
+		if (!shouldInsertMarkdownLinkByDefault(this._parser, document, pasteUrlSetting, ranges, token)) {
 			pasteEdit.yieldTo = [{ mimeType: Mime.textPlain }];
 		}
 
 		return pasteEdit;
 	}
 }
 
-export function registerLinkPasteSupport(selector: vscode.DocumentSelector,) {
-	return vscode.languages.registerDocumentPasteEditProvider(selector, new PasteUrlEditProvider(), {
+export function registerPasteUrlSupport(selector: vscode.DocumentSelector, parser: IMdParser) {
+	return vscode.languages.registerDocumentPasteEditProvider(selector, new PasteUrlEditProvider(parser), {
 		id: PasteUrlEditProvider.id,
 		pasteMimeTypes: PasteUrlEditProvider.pasteMimeTypes,
 	});
 }
 
-const smartPasteRegexes = [
+const smartPasteLineRegexes = [
 	{ regex: /(\[[^\[\]]*](?:\([^\(\)]*\)|\[[^\[\]]*]))/g }, // In a Markdown link
-	{ regex: /^```[\s\S]*?```$/gm }, // In a backtick fenced code block
-	{ regex: /^~~~[\s\S]*?~~~$/gm }, // In a tildefenced code block
-	{ regex: /^\$\$[\s\S]*?\$\$$/gm }, // In a fenced math block
+	{ regex: /\$\$[\s\S]*?\$\$/gm }, // In a fenced math block
 	{ regex: /`[^`]*`/g }, // In inline code
 	{ regex: /\$[^$]*\$/g }, // In inline math
+	{ regex: /^[ ]{0,3}\[\w+\]:\s.*$/g }, // Block link definition (needed as tokens are not generated for these)
 ];
 
-export function shouldInsertMarkdownLinkByDefault(document: ITextDocument, pasteUrlSetting: PasteUrlAsMarkdownLink, ranges: readonly vscode.Range[]): boolean {
+export async function shouldInsertMarkdownLinkByDefault(
+	parser: IMdParser,
+	document: ITextDocument,
+	pasteUrlSetting: PasteUrlAsMarkdownLink,
+	ranges: readonly vscode.Range[],
+	token: vscode.CancellationToken,
+): Promise<boolean> {
 	switch (pasteUrlSetting) {
 		case PasteUrlAsMarkdownLink.Always: {
 			return true;
 		}
 		case PasteUrlAsMarkdownLink.Smart: {
-			return ranges.every(range => shouldSmartPasteForSelection(document, range));
+			return checkSmart();
 		}
 		case PasteUrlAsMarkdownLink.SmartWithSelection: {
-			return (
-				// At least one range must not be empty
-				ranges.some(range => document.getText(range).trim().length > 0)
-				// And all ranges must be smart
-				&& ranges.every(range => shouldSmartPasteForSelection(document, range))
-			);
+			// At least one range must not be empty
+			if (!ranges.some(range => document.getText(range).trim().length > 0)) {
+				return false;
+			}
+			// And all ranges must be smart
+			return checkSmart();
 		}
 		default: {
 			return false;
 		}
 	}
+
+	async function checkSmart(): Promise<boolean> {
+		return (await Promise.all(ranges.map(range => shouldSmartPasteForSelection(parser, document, range, token)))).every(x => x);
+	}
 }
 
-function shouldSmartPasteForSelection(document: ITextDocument, selectedRange: vscode.Range): boolean {
+async function shouldSmartPasteForSelection(
+	parser: IMdParser,
+	document: ITextDocument,
+	selectedRange: vscode.Range,
+	token: vscode.CancellationToken,
+): Promise<boolean> {
 	// Disable for multi-line selections
 	if (selectedRange.start.line !== selectedRange.end.line) {
 		return false;
@@ -125,18 +144,25 @@ function shouldSmartPasteForSelection(document: ITextDocument, selectedRange: vs
 		return false;
 	}
 
-	// TODO: use proper parsing instead of regexes
-	for (const regex of smartPasteRegexes) {
-		const matches = [...document.getText().matchAll(regex.regex)];
-		for (const match of matches) {
-			if (match.index !== undefined) {
-				const matchRange = new vscode.Range(
-					document.positionAt(match.index),
-					document.positionAt(match.index + match[0].length)
-				);
-				if (matchRange.intersection(selectedRange)) {
-					return false;
-				}
+	// Check if selection is inside a special block level element using markdown engine
+	const tokens = await parser.tokenize(document);
+	if (token.isCancellationRequested) {
+		return false;
+	}
+	for (const token of tokens) {
+		if (token.map && token.map[0] <= selectedRange.start.line && token.map[1] > selectedRange.start.line) {
+			if (!['paragraph_open', 'inline', 'heading_open', 'ordered_list_open', 'bullet_list_open', 'list_item_open', 'blockquote_open'].includes(token.type)) {
+				return false;
+			}
+		}
+	}
+
+	// Run additional regex checks on the current line to check if we are inside an inline element
+	const line = document.getText(new vscode.Range(selectedRange.start.line, 0, selectedRange.start.line, Number.MAX_SAFE_INTEGER));
+	for (const regex of smartPasteLineRegexes) {
+		for (const match of line.matchAll(regex.regex)) {
+			if (match.index !== undefined && selectedRange.start.character >= match.index && selectedRange.start.character <= match.index + match[0].length) {
+				return false;
 			}
 		}
 	}

extensions/markdown-language-features/src/markdownEngine.ts

@@ -118,6 +118,12 @@ export class MarkdownItEngine implements IMdParser {
 		});
 	}
 
+
+	public async getEngine(resource: vscode.Uri | undefined): Promise<MarkdownIt> {
+		const config = this._getConfig(resource);
+		return this._getEngine(config);
+	}
+
 	private async _getEngine(config: MarkdownItConfig): Promise<MarkdownIt> {
 		if (!this._md) {
 			this._md = (async () => {

extensions/markdown-language-features/src/test/markdownLink.test.ts

@@ -8,6 +8,8 @@ import * as vscode from 'vscode';
 import { InMemoryDocument } from '../client/inMemoryDocument';
 import { PasteUrlAsMarkdownLink, findValidUriInText, shouldInsertMarkdownLinkByDefault } from '../languageFeatures/copyFiles/pasteUrlProvider';
 import { createInsertUriListEdit } from '../languageFeatures/copyFiles/shared';
+import { createNewMarkdownEngine } from './engine';
+import { noopToken } from '../util/cancellation';
 
 function makeTestDoc(contents: string) {
 	return new InMemoryDocument(vscode.Uri.file('test.md'), contents);
@@ -136,88 +138,135 @@ suite('createEditAddingLinksForUriList', () => {
 
 	suite('shouldInsertMarkdownLinkByDefault', () => {
 
-		test('Smart should enabled for selected plain text', () => {
+		test('Smart should be enabled for selected plain text', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('hello world'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 12)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('hello world'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 12)], noopToken),
 				true);
 		});
 
-		test('Smart should enabled for empty selection', () => {
+		test('Smart should be enabled in headers', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('xyz'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 0, 0, 0)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('# title'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 2, 0, 2)], noopToken),
 				true);
 		});
 
-		test('SmartWithSelection should disable for empty selection', () => {
+		test('Smart should be enabled in lists', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('xyz'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 0)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('1. text'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 3, 0, 3)], noopToken),
+				true);
+		});
+
+		test('Smart should be enabled in blockquotes', async () => {
+			assert.strictEqual(
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('> text'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 3, 0, 3)], noopToken),
+				true);
+		});
+
+		test('Smart should be disabled in indented code blocks', async () => {
+			assert.strictEqual(
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('    code'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 4, 0, 4)], noopToken),
 				false);
 		});
 
-		test('Smart should disable for selected link', () => {
+		test('Smart should be disabled in fenced code blocks', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('https://www.microsoft.com'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 25)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('```\r\n\r\n```'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 5, 0, 5)], noopToken),
+				false);
+
+			assert.strictEqual(
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('~~~\r\n\r\n~~~'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 5, 0, 5)], noopToken),
 				false);
 		});
 
-		test('Smart should disable for selected link with trailing whitespace', () => {
+		test('Smart should be disabled in math blocks', async () => {
+			const katex = (await import('@vscode/markdown-it-katex')).default;
+			const engine = createNewMarkdownEngine();
+			(await engine.getEngine(undefined)).use(katex);
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('   https://www.microsoft.com  '), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 30)]),
+				await shouldInsertMarkdownLinkByDefault(engine, makeTestDoc('$$\r\n\r\n$$'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 5, 0, 5)], noopToken),
 				false);
 		});
 
-		test('Should evaluate pasteAsMarkdownLink as true for a link pasted in square brackets', () => {
+		test('Smart should be disabled in link definitions', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('[abc]'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 1, 0, 4)]),
-				true);
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('[ref]: http://example.com'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 4, 0, 6)], noopToken),
+				false);
+
+			assert.strictEqual(
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('[ref]: '), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 7, 0, 7)], noopToken),
+				false);
 		});
 
-		test('Should evaluate pasteAsMarkdownLink as false for selected whitespace and new lines', () => {
+		test('Smart should be disabled in html blocks', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('   \r\n\r\n'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 7)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('<p>\na\n</p>'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(1, 0, 1, 0)], noopToken),
 				false);
 		});
 
-		test('Should evaluate pasteAsMarkdownLink as false for pasting within a backtick code block', () => {
+		test('Smart should be disabled in Markdown links', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('```\r\n\r\n```'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 5, 0, 5)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('[a](bcdef)'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 4, 0, 6)], noopToken),
 				false);
 		});
 
-		test('Should evaluate pasteAsMarkdownLink as false for pasting within a tilde code block', () => {
+		test('Smart should be disabled in Markdown images', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('~~~\r\n\r\n~~~'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 5, 0, 5)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('![a](bcdef)'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 5, 0, 10)], noopToken),
 				false);
 		});
 
-		test('Should evaluate pasteAsMarkdownLink as false for pasting within a math block', () => {
+		test('Smart should be disabled in inline code', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('$$$\r\n\r\n$$$'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 5, 0, 5)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('``'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 1, 0, 1)], noopToken),
 				false);
+
+			assert.strictEqual(
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('``'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 0, 0, 0)], noopToken),
+				false);
+		});
+
+		test('Smart should be disabled in inline math', async () => {
+			assert.strictEqual(
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('$$'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 1, 0, 1)], noopToken),
+				false);
+		});
+
+		test('Smart should be enabled for empty selection', async () => {
+			assert.strictEqual(
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('xyz'), PasteUrlAsMarkdownLink.Smart, [new vscode.Range(0, 0, 0, 0)], noopToken),
+				true);
 		});
 
-		test('Should evaluate pasteAsMarkdownLink as false for pasting within a Markdown link', () => {
+		test('SmartWithSelection should disable for empty selection', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('[a](bcdef)'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 4, 0, 6)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('xyz'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 0)], noopToken),
 				false);
 		});
 
-		test('Should evaluate pasteAsMarkdownLink as false for pasting within a Markdown image link', () => {
+		test('Smart should disable for selected link', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('![a](bcdef)'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 5, 0, 10)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('https://www.microsoft.com'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 25)], noopToken),
 				false);
 		});
 
-		test('Should evaluate pasteAsMarkdownLink as false for pasting within inline code', () => {
+		test('Smart should disable for selected link with trailing whitespace', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('``'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 1, 0, 1)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('   https://www.microsoft.com  '), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 30)], noopToken),
 				false);
 		});
 
-		test('Should evaluate pasteAsMarkdownLink as false for pasting within inline math', () => {
+		test('Should evaluate pasteAsMarkdownLink as true for a link pasted in square brackets', async () => {
+			assert.strictEqual(
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('[abc]'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 1, 0, 4)], noopToken),
+				true);
+		});
+
+		test('Should evaluate pasteAsMarkdownLink as false for selected whitespace and new lines', async () => {
 			assert.strictEqual(
-				shouldInsertMarkdownLinkByDefault(makeTestDoc('$$'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 1, 0, 1)]),
+				await shouldInsertMarkdownLinkByDefault(createNewMarkdownEngine(), makeTestDoc('   \r\n\r\n'), PasteUrlAsMarkdownLink.SmartWithSelection, [new vscode.Range(0, 0, 0, 7)], noopToken),
 				false);
 		});
+
+
 	});
 });

extensions/markdown-language-features/src/typings/markdown-it-katex.d.ts

@@ -0,0 +1,6 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+declare module '@vscode/markdown-it-katex';