feat: Support UTF-16 words lists (#2549)

packages/client/src/settings/CSpellSettings.test.ts

@@ -6,6 +6,18 @@ import * as CSS from './CSpellSettings';
 import { readSettings, writeSettings } from './CSpellSettings';
 import { createDictionaryTargetForFile } from './DictionaryTarget';
 
+import { replaceDocText } from './replaceDocText';
+import { vscodeFs } from './fs';
+
+jest.mock('./replaceDocText');
+
+const mock_replaceDocText = jest.mocked(replaceDocText);
+
+mock_replaceDocText.mockImplementation(async (doc, text) => {
+    await vscodeFs.writeFile(doc.uri, text);
+    return true;
+});
+
 describe('Validate CSpellSettings functions', () => {
     const filenameSampleCSpellFile = getUriToSample('cSpell.json');
 

packages/client/src/settings/DictionaryTarget.test.ts

@@ -7,6 +7,18 @@ import { DictionaryTarget } from './DictionaryTarget';
 import { configTargetToDictionaryTarget } from './DictionaryTargetHelper';
 import { createConfigFile } from './settings';
 
+import { vscodeFs } from './fs';
+import { replaceDocText } from './replaceDocText';
+
+jest.mock('./replaceDocText');
+
+const mock_replaceDocText = jest.mocked(replaceDocText);
+
+mock_replaceDocText.mockImplementation(async (doc, text) => {
+    await vscodeFs.writeFile(doc.uri, text);
+    return true;
+});
+
 describe('Validate DictionaryTarget', () => {
     interface TestUpdating {
         createFn: (tempDir: Uri) => Promise<TargetAndReader>;

packages/client/src/settings/DictionaryTarget.ts

@@ -1,11 +1,12 @@
 import { isErrnoException } from 'common-utils/index.js';
 import { uriToName } from 'common-utils/uriHelper.js';
-import * as fs from 'fs-extra';
-import * as path from 'path';
 import { format } from 'util';
-import { Uri } from 'vscode';
+import { Uri, window, workspace } from 'vscode';
+import { Utils as UriUtils } from 'vscode-uri';
 import { ConfigRepository, createCSpellConfigRepository } from './configRepository';
 import { addWordsFn, removeWordsFn, updaterAddWords, updaterRemoveWords } from './configUpdaters';
+import { vscodeFs as fs } from './fs';
+import { replaceDocText } from './replaceDocText';
 
 const regBlockUpdateDictionaryFormat = /(\.((gz|jsonc?|yaml|yml|c?js)$|trie\b)|^$|[/\\]$)/i;
 
@@ -80,17 +81,28 @@ async function addWordsToCustomDictionary(words: string[], dict: CustomDictDef):
 }
 
 async function updateWordInCustomDictionary(updateFn: (words: string[]) => string[], dict: CustomDictDef): Promise<void> {
-    const fsPath = dict.uri.fsPath;
-    if (regBlockUpdateDictionaryFormat.test(fsPath)) {
+    if (regBlockUpdateDictionaryFormat.test(dict.uri.path)) {
         return Promise.reject(new Error(`Failed to add words to dictionary "${dict.name}", unsupported format: "${dict.uri.fsPath}".`));
     }
     try {
-        const data = await fs.readFile(fsPath, 'utf8').catch(() => '');
+        await ensureFileExists(dict.uri);
+        const doc = await workspace.openTextDocument(dict.uri);
+        const data = doc.getText();
         const lines = updateFn(data.split(/\r?\n/g).filter((a) => !!a));
-        await fs.mkdirp(path.dirname(fsPath));
-        await fs.writeFile(fsPath, lines.join('\n').trim().concat('\n'));
+        const text = lines.join('\n').trim() + '\n';
+        const success = await replaceDocText(doc, text);
+        if (!success) {
+            await window.showInformationMessage(`Unable to add words to dictionary "${dict.name}"`);
+        }
     } catch (e) {
         const errMsg = isErrnoException(e) ? e.message : format(e);
         return Promise.reject(new Error(`Failed to add words to dictionary "${dict.name}", ${errMsg}`));
     }
 }
+
+async function ensureFileExists(uri: Uri): Promise<void> {
+    if (await fs.fileExists(uri)) return;
+
+    await fs.createDirectory(UriUtils.dirname(uri));
+    await fs.writeFile(uri, '');
+}

packages/client/src/settings/replaceDocText.ts

@@ -0,0 +1,11 @@
+import { Range, TextDocument, TextEdit, workspace, WorkspaceEdit } from 'vscode';
+
+export async function replaceDocText(doc: TextDocument, text: string): Promise<boolean> {
+    const wsEdit = new WorkspaceEdit();
+    const range = new Range(doc.positionAt(0), doc.positionAt(doc.getText().length));
+    const teReplaceDoc = TextEdit.replace(range, text);
+    wsEdit.set(doc.uri, [teReplaceDoc]);
+    const success = await workspace.applyEdit(wsEdit);
+    success && (await doc.save());
+    return success;
+}

samples/utf16/README.md

@@ -0,0 +1,8 @@
+# Test UTF16 word lists.
+
+mywordd
+otherword
+
+mywordbe
+
+mywordle

samples/utf16/cspell.config.yaml

@@ -0,0 +1,10 @@
+dictionaryDefinitions:
+  - name: words16le
+    path: ./words16le.dic
+    addWords: true
+  - name: words16be
+    path: ./words/words16be.dic
+    addWords: true
+dictionaries:
+  - words16be
+  - words16le