fix: Fix word break issue with large documents (#3156)

packages/_server/src/DocumentValidationController.mts

@@ -7,6 +7,7 @@ import type { TextDocumentInfoWithText, TextDocumentRef } from './api.js';
 import type { CSpellUserSettings } from './config/cspellConfig/index.mjs';
 import type { DocumentSettings } from './config/documentSettings.mjs';
 import { defaultCheckLimit } from './constants.mjs';
+import { breakTextAtLimit } from './utils/breakTextAtLimit.mjs';
 
 interface DocValEntry {
     uri: string;
@@ -88,7 +89,8 @@ export async function createDocumentValidator(
 ): Promise<DocumentValidator> {
     const settings = await pSettings;
     const limit = (settings.checkLimit || defaultCheckLimit) * 1024;
-    const content = ('getText' in textDocument ? textDocument.getText() : textDocument.text).slice(0, limit);
+    const fullContent = 'getText' in textDocument ? textDocument.getText() : textDocument.text;
+    const content = breakTextAtLimit(fullContent, limit);
     const { uri, languageId, version } = textDocument;
     const docInfo = { uri, content, languageId, version };
     const doc = createTextDocument(docInfo);

packages/_server/src/utils/breakTextAtLimit.mts

@@ -0,0 +1,16 @@
+export function breakTextAtLimit(text: string, limit: number): string {
+    if (!text[limit]) return text;
+
+    const regIsNotChar = /[^\p{L}._0-9-]/uy;
+
+    regIsNotChar.lastIndex = limit;
+    if (regIsNotChar.test(text)) return text.slice(0, limit);
+
+    let idx = limit - 1;
+    for (; idx > 0; --idx) {
+        if ((text.charCodeAt(idx) & 0xfc00) === 0xdc00) continue;
+        regIsNotChar.lastIndex = idx;
+        if (regIsNotChar.test(text)) return text.slice(0, idx + 1);
+    }
+    return '';
+}

packages/_server/src/utils/breakTextAtLimit.test.mts

@@ -0,0 +1,19 @@
+import { describe, expect, test } from 'vitest';
+
+import { breakTextAtLimit } from './breakTextAtLimit.mjs';
+
+describe('breakTextAtLimit', () => {
+    test.each`
+        line         | limit | expected
+        ${'a'}       | ${1}  | ${'a'}
+        ${'abc'}     | ${1}  | ${''}
+        ${'abc def'} | ${3}  | ${'abc'}
+        ${'abc def'} | ${4}  | ${'abc '}
+        ${'abc def'} | ${5}  | ${'abc '}
+        ${'abc def'} | ${6}  | ${'abc '}
+        ${'abc def'} | ${7}  | ${'abc def'}
+        ${'abc def'} | ${8}  | ${'abc def'}
+    `('breakTextAtLimit $line $limit', ({ line, limit, expected }) => {
+        expect(breakTextAtLimit(line, limit)).toBe(expected);
+    });
+});