fix: Make sure dictionary files are refreshed if they have changed

packages/cspell-lib/src/SpellingDictionary/Dictionaries.test.ts

@@ -1,6 +1,8 @@
 import { expect } from 'chai';
 import * as Dictionaries from './Dictionaries';
 import { getDefaultSettings } from '../Settings';
+import * as path from 'path';
+import * as fs from 'fs-extra';
 
 // cspell:ignore café rhône
 
@@ -60,4 +62,46 @@ describe('Validate getDictionary', () => {
         expect(dict.has('rhone', { ignoreCase: true })).to.be.true;
         expect(dict.has('cafe', { ignoreCase: true })).to.be.true;
     });
+
+    test('Refresh Dictionary Cache', async () => {
+        const tempDictPath = path.join(__dirname, '..', '..', 'temp', 'words.txt');
+        await fs.mkdirp(path.dirname(tempDictPath));
+        await fs.writeFile(tempDictPath, "one\ntwo\nthree\n");
+
+        const settings = getDefaultSettings();
+        const defs = (settings.dictionaryDefinitions || []).concat([
+            {
+                name: 'temp',
+                path: tempDictPath
+            }
+        ]);
+        const toLoad = ['node', 'html', 'css', 'temp'];
+        const dicts = await Promise.all(Dictionaries.loadDictionaries(toLoad, defs));
+
+        expect(dicts[3].has('one')).to.be.true;
+        expect(dicts[3].has('four')).to.be.false;
+
+        await Dictionaries.refreshDictionaryCache(0);
+        const dicts2 = await Promise.all(Dictionaries.loadDictionaries(toLoad, defs));
+
+        // Since noting changed, expect them to be the same.
+        expect(dicts.length).to.eq(toLoad.length);
+        expect(dicts2.length).to.be.eq(dicts.length);
+        dicts.forEach((d, i) => expect(dicts2[i]).to.be.equal(d));
+
+        // Update one of the dictionaries to see if it loads.
+        await fs.writeFile(tempDictPath, "one\ntwo\nthree\nfour\n");
+
+        const dicts3 = await Promise.all(Dictionaries.loadDictionaries(toLoad, defs));
+        // Should be using cache and will not contain the new words.
+        expect(dicts3[3].has('one')).to.be.true;
+        expect(dicts3[3].has('four')).to.be.false;
+
+        await Dictionaries.refreshDictionaryCache(0);
+
+        const dicts4 = await Promise.all(Dictionaries.loadDictionaries(toLoad, defs));
+        // Should be using the latest copy of the words.
+        expect(dicts4[3].has('one')).to.be.true;
+        expect(dicts4[3].has('four')).to.be.true;
+    });
 });

packages/cspell-lib/src/SpellingDictionary/Dictionaries.ts

@@ -1,6 +1,6 @@
 import { DictionaryDefinition, DictionaryId, CSpellUserSettings } from '../Settings';
 import { filterDictDefsToLoad } from '../Settings/DictionarySettings';
-import { loadDictionary } from './DictionaryLoader';
+import { loadDictionary, refreshCacheEntries } from './DictionaryLoader';
 import { SpellingDictionary, createSpellingDictionary } from './SpellingDictionary';
 import { createCollectionP } from './SpellingDictionaryCollection';
 import { SpellingDictionaryCollection } from './index';
@@ -14,6 +14,10 @@ export function loadDictionaries(dictIds: DictionaryId[], defs: DictionaryDefini
         .map(def => loadDictionary(def.path!, def));
 }
 
+export function refreshDictionaryCache(maxAge?: number) {
+    return refreshCacheEntries(maxAge);
+}
+
 export function getDictionary(settings: CSpellUserSettings): Promise<SpellingDictionaryCollection> {
     const { words = [], userWords = [], dictionaries = [], dictionaryDefinitions = [], flagWords = [], caseSensitive = false } = settings;
     const spellDictionaries = loadDictionaries(dictionaries, dictionaryDefinitions);

packages/cspell-lib/src/SpellingDictionary/DictionaryLoader.ts

@@ -4,6 +4,10 @@ import * as path from 'path';
 import { ReplaceMap } from '../Settings';
 import { genSequence } from 'gensequence';
 import { readLines } from '../util/fileReader';
+import { Stats } from 'fs-extra';
+import * as fs from 'fs-extra';
+
+const MAX_AGE = 10000;
 
 export interface LoadOptions {
     // Type of file:
@@ -39,18 +43,62 @@ const loaders: Loaders = {
     default: loadSimpleWordList,
 };
 
-const dictionaryCache = new Map<string, Promise<SpellingDictionary>>();
+interface CacheEntry {
+    uri: string;
+    options: LoadOptions;
+    ts: number;
+    state: Promise<Stats>;
+    dictionary: Promise<SpellingDictionary>;
+}
+
+const dictionaryCache = new Map<string, CacheEntry>();
 
 export function loadDictionary(uri: string, options: LoadOptions): Promise<SpellingDictionary> {
-    const loaderType = determineType(uri, options);
-    const key = [uri, loaderType].join('|');
+    const key = calcKey(uri, options);
     if (!dictionaryCache.has(key)) {
-        dictionaryCache.set(key, load(uri, options));
+        dictionaryCache.set(key, loadEntry(uri, options));
     }
+    const entry = dictionaryCache.get(key)!;
+    return entry.dictionary;
+}
 
-    return dictionaryCache.get(key)!;
+function calcKey(uri: string, options: LoadOptions) {
+    const loaderType = determineType(uri, options);
+    return [uri, loaderType].join('|');
 }
 
+export async function refreshCacheEntries(maxAge = MAX_AGE, now = Date.now()) {
+    await Promise.all([...dictionaryCache]
+        .map(([, entry]) => refreshEntry(entry, maxAge, now))
+    );
+}
+
+async function refreshEntry(entry: CacheEntry, maxAge = MAX_AGE, now = Date.now()) {
+    if (now - entry.ts >= maxAge) {
+        // Write to the ts, so the next one will not do it.
+        entry.ts = now;
+        const [state, oldState] = await Promise.all([fs.stat(entry.uri), entry.state]);
+        if (entry.ts === now && (
+            state.mtimeMs !== oldState.mtimeMs ||
+            state.size !== oldState.size
+        )) {
+            dictionaryCache.set(
+                calcKey(entry.uri, entry.options),
+                loadEntry(entry.uri, entry.options)
+            );
+        }
+    }
+}
+
+function loadEntry(uri: string, options: LoadOptions, now = Date.now()): CacheEntry {
+    return {
+        uri,
+        options,
+        ts: now,
+        state: fs.stat(uri),
+        dictionary: load(uri, options),
+    }
+}
 
 function determineType(uri: string, options: LoadOptions): LoaderType {
     const defType = uri.endsWith('.trie.gz') ? 'T' : uri.endsWith('.txt.gz') ? 'S' : 'C';
@@ -90,3 +138,9 @@ async function loadCodeWordList(filename: string, options: LoadOptions) {
 async function loadTrie(filename: string, options: LoadOptions) {
     return createSpellingDictionaryTrie(await loadWordsNoError(filename), path.basename(filename), filename, options);
 }
+
+export const testing = {
+    dictionaryCache,
+    refreshEntry,
+    loadEntry
+};

packages/cspell-lib/src/index.test.ts

@@ -15,4 +15,8 @@ describe('Validate the cspell API', () => {
                 expect(results.map(to => to.text)).to.contain('Jansons');
             });
     });
+
+    test('clearCachedSettings', () => {
+        return cspell.clearCachedFiles();
+    });
 });

packages/cspell-lib/src/index.ts

@@ -11,7 +11,6 @@ export {
 } from './validator';
 export {
     calcOverrideSettings,
-    clearCachedFiles as clearCachedSettings,
     defaultFileName as defaultSettingsFilename,
     mergeSettings,
     readSettings,
@@ -42,3 +41,15 @@ export {
 
 export { getLanguagesForExt } from './LanguageIds';
 export * from './trace';
+
+import { clearCachedFiles } from './Settings';
+import { refreshDictionaryCache } from './SpellingDictionary'
+
+export async function clearCachedSettings() {
+    await Promise.all([
+        clearCachedFiles(),
+        refreshDictionaryCache(),
+    ]);
+}
+
+export { clearCachedSettings as clearCachedFiles };