dev: Prepare to support Hunspell AFF to improve suggestions. (#2332)

streetsidesoftware · Jan 25, 2022 · b9fa206 · b9fa206
1 parent 65c12df
commit b9fa206
Show file tree

Hide file tree

Showing 32 changed files with 1,354 additions and 301 deletions.
diff --git a/cspell.schema.json b/cspell.schema.json
@@ -430,10 +430,19 @@
     "HunspellInformation": {
       "additionalProperties": false,
       "properties": {
+        "aff": {
+          "description": "Selected Hunspell AFF content. The content must be UTF-8\n\nSections:\n- TRY\n- MAP\n- REP\n- KEY\n- ICONV\n- OCONV\n\nExample: ```hunspell # Comment TRY aeistlunkodmrvpgjhäõbüoöfcwzxðqþ` MAP aàâäAÀÂÄ MAP eéèêëEÉÈÊË MAP iîïyIÎÏY MAP oôöOÔÖ MAP (IJ)(Ĳ) ```",
+          "type": "string"
+        },
         "costs": {
           "additionalProperties": false,
           "description": "The costs to apply when using the hunspell settings",
           "properties": {
+            "firstLetterPenalty": {
+              "default": 4,
+              "description": "The extra cost incurred for changing the first letter of a word. This value should be less than `100 - tryCharCost`.",
+              "type": "number"
+            },
             "ioConvertCost": {
               "default": 30,
               "description": "The cost to convert between convert pairs.\n\nThe value should be slightly higher than the mapCost.",
@@ -461,71 +470,11 @@
             }
           },
           "type": "object"
-        },
-        "inConvert": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "items": {
-                "type": "string"
-              },
-              "type": "array"
-            }
-          ]
-        },
-        "keyboard": {
-          "description": "This represents the layout of a common keyboard. Used for adjacency mistakes.\n\nHunspell: ```hunspell KEY qwertyuiop|asdfghjkl|zxcvbnm ``` Use: ```yaml keyboard: qwertyuiop|asdfghjkl|zxcvbnm ``` <!--- cspell:ignore asdfghjkl qwertyuiop zxcvbnm -->",
-          "type": "string"
-        },
-        "mapSets": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "items": {
-                "type": "string"
-              },
-              "type": "array"
-            }
-          ],
-          "description": "This defines characters that are good substitutes for each other.\n- A `()` is need for multi character sequences like letters with multiple accents.\n- Separate unrelated characters with `|` or `\\n`\n\nExample:\n- Hunspell:   ```hunspell   MAP aàâäAÀÂÄ   MAP eéèêëEÉÈÊË   MAP iîïyIÎÏY   MAP oôöOÔÖ   MAP (IJ)(Ĳ)   ```\n- Use:   `yaml`   ```yaml   mapSets: |     aàâäAÀÂÄ|eéèêëEÉÈÊË`     eéèêëEÉÈÊË|iîïyIÎÏY     oôöOÔÖ|(IJ)(Ĳ)   ``` <!--- cspell:ignore aàâä eéèêë iîïy -->"
-        },
-        "outConvert": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "items": {
-                "type": "string"
-              },
-              "type": "array"
-            }
-          ],
-          "description": "Output Conversions"
-        },
-        "replace": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "items": {
-                "type": "string"
-              },
-              "type": "array"
-            }
-          ],
-          "description": "Common Substitutions. Special characters:\n- `^` - matches the beginning of a word.\n- `$` - matches the end of a word.\n\nHunspell: ```hunspell REP c ss REP e ĳ REP é ee REP g ch REP ï ii REP t d\t# gebiest=>gebiesd REP u ĳ ```\n\nUse: ```yaml replace: |   c ss   e ĳ   é ee   g ch   ï ii   t d   u ĳ ```\n\n<!--- cspell:ignore gebiesd gebiest -->"
-        },
-        "tryChars": {
-          "description": "This is generally the alphabet used in the dictionary. It stands for the characters to try using for substitutions when looking for a suggestions.\n\nExample:\n- Hunspell:   ```hunspell   TRY aeistlunkodmrvpgjhäõbüoöfcwzxðqþ`   ```\n- Use: yaml   ```yaml   tryChars: \"aeistlunkodmrvpgjhäõbüoöfcwzxðqþ\"   ``` <!--- cspell:ignore aeistlunkodmrvpgjhäõbüoöfcwzxðqþ  --",
-          "type": "string"
         }
       },
+      "required": [
+        "aff"
+      ],
       "type": "object"
     },
     "LanguageId": {

diff --git a/packages/cspell-lib/api/api.d.ts b/packages/cspell-lib/api/api.d.ts
@@ -1,11 +1,11 @@
 /// <reference types="node" />
-import { Glob, CSpellSettingsWithSourceTrace, DictionaryDefinitionPreferred, DictionaryDefinitionAugmented, DictionaryDefinitionCustom, PnPSettings, ImportFileRef, ReplaceMap, SuggestionCostsDefs, TextOffset, TextDocumentOffset, CSpellUserSettings, LocaleId, CSpellSettings } from '@cspell/cspell-types';
+import { Glob, CSpellSettingsWithSourceTrace, ReplaceMap, SuggestionCostsDefs, DictionaryDefinitionPreferred, DictionaryDefinitionAugmented, DictionaryDefinitionCustom, TextOffset, TextDocumentOffset, PnPSettings, ImportFileRef, CSpellUserSettings, LocaleId, CSpellSettings } from '@cspell/cspell-types';
 export * from '@cspell/cspell-types';
-import { URI } from 'vscode-uri';
-import { CompoundWordsMethod, SuggestionResult, SuggestionCollector } from 'cspell-trie-lib';
+import { CompoundWordsMethod, SuggestionResult, SuggestionCollector, WeightMap } from 'cspell-trie-lib';
 export { CompoundWordsMethod, SuggestionCollector, SuggestionResult } from 'cspell-trie-lib';
 import { Sequence } from 'gensequence';
 export * from 'cspell-io';
+import { URI } from 'vscode-uri';
 
 declare type ExclusionFunction = (fileUri: string) => boolean;
 declare type FileExclusionFunction = (file: string) => boolean;
@@ -46,92 +46,6 @@ declare namespace exclusionHelper_d {
   };
 }
 
-declare const SymbolCSpellSettingsInternal: unique symbol;
-interface CSpellSettingsInternal extends Omit<CSpellSettingsWithSourceTrace, 'dictionaryDefinitions'> {
-    [SymbolCSpellSettingsInternal]: true;
-    dictionaryDefinitions?: DictionaryDefinitionInternal[];
-}
-declare type DictionaryDefinitionCustomUniqueFields = Omit<DictionaryDefinitionCustom, keyof DictionaryDefinitionPreferred>;
-interface DictionaryDefinitionInternal extends DictionaryDefinitionPreferred, Partial<DictionaryDefinitionCustomUniqueFields>, DictionaryDefinitionAugmented {
-}
-
-/**
- * Handles loading of `.pnp.js` and `.pnp.js` files.
- */
-
-declare type LoaderResult = URI | undefined;
-
-declare type CSpellSettingsWST = CSpellSettingsWithSourceTrace;
-declare type CSpellSettingsI = CSpellSettingsInternal;
-declare const currentSettingsFileVersion = "0.2";
-declare const sectionCSpell = "cSpell";
-declare const defaultFileName = "cspell.json";
-declare const ENV_CSPELL_GLOB_ROOT = "CSPELL_GLOB_ROOT";
-declare const defaultConfigFilenames: readonly string[];
-declare function readSettings(filename: string): CSpellSettingsI;
-declare function readSettings(filename: string, defaultValues: CSpellSettingsWST): CSpellSettingsI;
-declare function readSettings(filename: string, relativeTo: string): CSpellSettingsI;
-declare function readSettings(filename: string, relativeTo: string, defaultValues: CSpellSettingsWST): CSpellSettingsI;
-declare function searchForConfig(searchFrom: string | undefined, pnpSettings?: PnPSettings): Promise<CSpellSettingsI | undefined>;
-/**
- * Load a CSpell configuration files.
- * @param file - path or package reference to load.
- * @param pnpSettings - PnP settings
- * @returns normalized CSpellSettings
- */
-declare function loadConfig(file: string, pnpSettings?: PnPSettings): Promise<CSpellSettingsI>;
-declare function loadPnP(pnpSettings: PnPSettings, searchFrom: URI): Promise<LoaderResult>;
-declare function loadPnPSync(pnpSettings: PnPSettings, searchFrom: URI): LoaderResult;
-declare function readRawSettings(filename: string, relativeTo?: string): CSpellSettingsWST;
-/**
- *
- * @param filenames - settings files to read
- * @returns combined configuration
- * @deprecated true
- */
-declare function readSettingsFiles(filenames: string[]): CSpellSettingsI;
-declare function mergeSettings(left: CSpellSettingsWST | CSpellSettingsI, ...settings: (CSpellSettingsWST | CSpellSettingsI)[]): CSpellSettingsI;
-declare function mergeInDocSettings(left: CSpellSettingsWST, right: CSpellSettingsWST): CSpellSettingsWST;
-declare function calcOverrideSettings(settings: CSpellSettingsWST, filename: string): CSpellSettingsI;
-/**
- *
- * @param settings - settings to finalize
- * @returns settings where all globs and file paths have been resolved.
- */
-declare function finalizeSettings(settings: CSpellSettingsWST | CSpellSettingsI): CSpellSettingsI;
-declare function getGlobalSettings(): CSpellSettingsI;
-declare function getCachedFileSize(): number;
-declare function clearCachedSettingsFiles(): void;
-/**
- * @param filename - filename
- * @param globs - globs
- * @returns true if it matches
- * @deprecated true
- * @deprecationMessage No longer actively supported. Use package: `cspell-glob`.
- */
-declare function checkFilenameMatchesGlob(filename: string, globs: Glob | Glob[]): boolean;
-/**
- * Return a list of Setting Sources used to create this Setting.
- * @param settings the settings to search
- */
-declare function getSources(settings: CSpellSettingsWST): CSpellSettingsWST[];
-interface ImportFileRefWithError extends ImportFileRef {
-    error: Error;
-}
-declare function extractImportErrors(settings: CSpellSettingsWST): ImportFileRefWithError[];
-interface ConfigurationDependencies {
-    configFiles: string[];
-    dictionaryFiles: string[];
-}
-declare function extractDependencies(settings: CSpellSettingsWST | CSpellSettingsI): ConfigurationDependencies;
-
-declare function getDefaultSettings(): CSpellSettingsInternal;
-
-declare class ImportError extends Error {
-    readonly cause?: Error;
-    constructor(msg: string, cause?: Error | unknown);
-}
-
 interface ListGlobalImportsResult {
     filename: string;
     name: string | undefined;
@@ -294,6 +208,21 @@ declare class SpellingDictionaryCollection implements SpellingDictionary {
     private _isNoSuggestWord;
 }
 
+declare const SymbolCSpellSettingsInternal: unique symbol;
+interface CSpellSettingsInternal extends Omit<CSpellSettingsWithSourceTrace, 'dictionaryDefinitions'> {
+    [SymbolCSpellSettingsInternal]: true;
+    dictionaryDefinitions?: DictionaryDefinitionInternal[];
+}
+declare type DictionaryDefinitionCustomUniqueFields = Omit<DictionaryDefinitionCustom, keyof DictionaryDefinitionPreferred>;
+interface DictionaryDefinitionInternal extends DictionaryDefinitionPreferred, Partial<DictionaryDefinitionCustomUniqueFields>, DictionaryDefinitionAugmented {
+    /**
+     * Optional weight map used to improve suggestions.
+     */
+    weightMap?: WeightMap;
+    /** The path to the config file that contains this dictionary definition */
+    __source?: string;
+}
+
 declare function refreshDictionaryCache(maxAge?: number): Promise<void>;
 
 interface IterableLike<T> {
@@ -424,6 +353,83 @@ declare namespace text_d {
 declare type LanguageId = string;
 declare function getLanguagesForExt(ext: string): string[];
 
+/**
+ * Handles loading of `.pnp.js` and `.pnp.js` files.
+ */
+
+declare type LoaderResult = URI | undefined;
+
+declare type CSpellSettingsWST = CSpellSettingsWithSourceTrace;
+declare type CSpellSettingsI = CSpellSettingsInternal;
+declare const currentSettingsFileVersion = "0.2";
+declare const sectionCSpell = "cSpell";
+declare const defaultFileName = "cspell.json";
+declare const ENV_CSPELL_GLOB_ROOT = "CSPELL_GLOB_ROOT";
+declare const defaultConfigFilenames: readonly string[];
+declare function readSettings(filename: string): CSpellSettingsI;
+declare function readSettings(filename: string, defaultValues: CSpellSettingsWST): CSpellSettingsI;
+declare function readSettings(filename: string, relativeTo: string): CSpellSettingsI;
+declare function readSettings(filename: string, relativeTo: string, defaultValues: CSpellSettingsWST): CSpellSettingsI;
+declare function searchForConfig(searchFrom: string | undefined, pnpSettings?: PnPSettings): Promise<CSpellSettingsI | undefined>;
+/**
+ * Load a CSpell configuration files.
+ * @param file - path or package reference to load.
+ * @param pnpSettings - PnP settings
+ * @returns normalized CSpellSettings
+ */
+declare function loadConfig(file: string, pnpSettings?: PnPSettings): Promise<CSpellSettingsI>;
+declare function loadPnP(pnpSettings: PnPSettings, searchFrom: URI): Promise<LoaderResult>;
+declare function loadPnPSync(pnpSettings: PnPSettings, searchFrom: URI): LoaderResult;
+declare function readRawSettings(filename: string, relativeTo?: string): CSpellSettingsWST;
+/**
+ *
+ * @param filenames - settings files to read
+ * @returns combined configuration
+ * @deprecated true
+ */
+declare function readSettingsFiles(filenames: string[]): CSpellSettingsI;
+declare function mergeSettings(left: CSpellSettingsWST | CSpellSettingsI, ...settings: (CSpellSettingsWST | CSpellSettingsI)[]): CSpellSettingsI;
+declare function mergeInDocSettings(left: CSpellSettingsWST, right: CSpellSettingsWST): CSpellSettingsWST;
+declare function calcOverrideSettings(settings: CSpellSettingsWST, filename: string): CSpellSettingsI;
+/**
+ *
+ * @param settings - settings to finalize
+ * @returns settings where all globs and file paths have been resolved.
+ */
+declare function finalizeSettings(settings: CSpellSettingsWST | CSpellSettingsI): CSpellSettingsI;
+declare function getGlobalSettings(): CSpellSettingsI;
+declare function getCachedFileSize(): number;
+declare function clearCachedSettingsFiles(): void;
+/**
+ * @param filename - filename
+ * @param globs - globs
+ * @returns true if it matches
+ * @deprecated true
+ * @deprecationMessage No longer actively supported. Use package: `cspell-glob`.
+ */
+declare function checkFilenameMatchesGlob(filename: string, globs: Glob | Glob[]): boolean;
+/**
+ * Return a list of Setting Sources used to create this Setting.
+ * @param settings the settings to search
+ */
+declare function getSources(settings: CSpellSettingsWST): CSpellSettingsWST[];
+interface ImportFileRefWithError extends ImportFileRef {
+    error: Error;
+}
+declare function extractImportErrors(settings: CSpellSettingsWST): ImportFileRefWithError[];
+interface ConfigurationDependencies {
+    configFiles: string[];
+    dictionaryFiles: string[];
+}
+declare function extractDependencies(settings: CSpellSettingsWST | CSpellSettingsI): ConfigurationDependencies;
+
+declare function getDefaultSettings(): CSpellSettingsInternal;
+
+declare class ImportError extends Error {
+    readonly cause?: Error;
+    constructor(msg: string, cause?: Error | unknown);
+}
+
 declare function combineTextAndLanguageSettings(settings: CSpellUserSettings, text: string, languageId: string | string[]): CSpellUserSettings;
 
 interface IncludeExcludeOptions {

diff --git a/...src/Settings/CSpellSettingsInternalDef.ts → ...b/src/Models/CSpellSettingsInternalDef.ts b/...src/Settings/CSpellSettingsInternalDef.ts → ...b/src/Models/CSpellSettingsInternalDef.ts
@@ -4,6 +4,7 @@ import {
     DictionaryDefinitionCustom,
     DictionaryDefinitionPreferred,
 } from '@cspell/cspell-types';
+import { WeightMap } from 'cspell-trie-lib';
 
 export const SymbolCSpellSettingsInternal = Symbol('CSpellSettingsInternal');
 
@@ -17,10 +18,16 @@ type DictionaryDefinitionCustomUniqueFields = Omit<DictionaryDefinitionCustom, k
 export interface DictionaryDefinitionInternal
     extends DictionaryDefinitionPreferred,
         Partial<DictionaryDefinitionCustomUniqueFields>,
-        DictionaryDefinitionAugmented {}
-
-export interface DictionaryDefinitionWithSource extends DictionaryDefinitionInternal {
+        DictionaryDefinitionAugmented {
+    /**
+     * Optional weight map used to improve suggestions.
+     */
+    weightMap?: WeightMap;
     /** The path to the config file that contains this dictionary definition */
+    __source?: string;
+}
+
+export interface DictionaryDefinitionInternalWithSource extends DictionaryDefinitionInternal {
     __source: string;
 }
 

diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts
@@ -2,7 +2,7 @@ import type { CSpellSettingsWithSourceTrace, CSpellUserSettings, ImportFileRef }
 import * as path from 'path';
 import { URI } from 'vscode-uri';
 import { logError, logWarning } from '../util/logger';
-import { createCSpellSettingsInternal as csi } from './CSpellSettingsInternalDef';
+import { createCSpellSettingsInternal as csi } from '../Models/CSpellSettingsInternalDef';
 import {
     calcOverrideSettings,
     checkFilenameMatchesGlob,

diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts
@@ -21,8 +21,8 @@ import {
     createCSpellSettingsInternal as csi,
     CSpellSettingsInternal,
     isCSpellSettingsInternal,
-} from './CSpellSettingsInternalDef';
-import { calcDictionaryDefsToLoad, normalizePathForDictDefs } from './DictionarySettings';
+} from '../Models/CSpellSettingsInternalDef';
+import { calcDictionaryDefsToLoad, mapDictDefsToInternal } from './DictionarySettings';
 import { getRawGlobalSettings } from './GlobalSettings';
 import { ImportError } from './ImportError';
 import { resolvePatterns } from './patterns';
@@ -608,7 +608,7 @@ export function toInternalSettings(settings?: CSpellSettingsI | CSpellSettingsWS
 
     const { dictionaryDefinitions: defs, ...rest } = settings;
 
-    const dictionaryDefinitions = normalizePathForDictDefs(
+    const dictionaryDefinitions = mapDictDefsToInternal(
         defs,
         filenameToDirectory(settings.source?.filename) || resolveCwd()
     );
@@ -814,11 +814,11 @@ function toGlobDef(
 type NormalizeDictionaryDefsParams = Pick<CSpellUserSettings, 'dictionaryDefinitions' | 'languageSettings'>;
 
 function normalizeDictionaryDefs(settings: NormalizeDictionaryDefsParams, pathToSettingsFile: string) {
-    const dictionaryDefinitions = normalizePathForDictDefs(settings.dictionaryDefinitions, pathToSettingsFile);
+    const dictionaryDefinitions = mapDictDefsToInternal(settings.dictionaryDefinitions, pathToSettingsFile);
     const languageSettings = settings.languageSettings?.map((langSetting) =>
         util.clean({
             ...langSetting,
-            dictionaryDefinitions: normalizePathForDictDefs(langSetting.dictionaryDefinitions, pathToSettingsFile),
+            dictionaryDefinitions: mapDictDefsToInternal(langSetting.dictionaryDefinitions, pathToSettingsFile),
         })
     );
 

diff --git a/packages/cspell-lib/src/Settings/DefaultSettings.ts b/packages/cspell-lib/src/Settings/DefaultSettings.ts
@@ -1,6 +1,6 @@
 import type { PredefinedPatterns, RegExpPatternDefinition } from '@cspell/cspell-types';
 import { resolveFile } from '../util/resolveFile';
-import { CSpellSettingsInternal, createCSpellSettingsInternal } from './CSpellSettingsInternalDef';
+import { CSpellSettingsInternal, createCSpellSettingsInternal } from '../Models/CSpellSettingsInternalDef';
 import { readSettings } from './CSpellSettingsServer';
 import { mergeSettings } from './index';
 import * as LanguageSettings from './LanguageSettings';