fix: Make sure cspell-tool-cli keeps accents

In `--split` mode, the compiler would loose free floating accents.
streetsidesoftware · Feb 1, 2022 · 6302b6e · 6302b6e
1 parent bb9baaa
commit 6302b6e
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/packages/cspell-tools/src/compiler/wordListCompiler.test.ts b/packages/cspell-tools/src/compiler/wordListCompiler.test.ts
@@ -115,7 +115,10 @@ describe('Validate the wordListCompiler', () => {
         ${'Namespace DNSLookup'}                    | ${['Namespace', 'DNSLookup']}                 | ${true}    | ${true}
         ${'well-educated'}                          | ${['well-educated']}                          | ${true}    | ${true}
         ${'--abort-on-uncaught-exception'}          | ${['--abort-on-uncaught-exception']}          | ${true}    | ${true}
+        ${'corner cafe\u0301\u0304'}                | ${['corner café\u0304']}                      | ${false}   | ${true}
         ${'corner café'}                            | ${['café', 'corner']}                         | ${true}    | ${true}
+        ${'corner café'.normalize('NFD')}           | ${['café', 'corner']}                         | ${true}    | ${true}
+        ${'corner café\u0304'.normalize('NFD')}     | ${['café\u0304', 'corner']}                   | ${true}    | ${true}
         ${'El Niño'}                                | ${['El', 'Niño']}                             | ${true}    | ${true}
         ${'El Nin\u0303o'}                          | ${['El', 'Niño']}                             | ${true}    | ${true}
         ${'CURLcode'}                               | ${['CURLcode']}                               | ${true}    | ${true}

diff --git a/packages/cspell-tools/src/compiler/wordListCompiler.ts b/packages/cspell-tools/src/compiler/wordListCompiler.ts
@@ -7,8 +7,8 @@ import { writeSeqToFile } from './fileWriter';
 import { uniqueFilter } from 'hunspell-reader/dist/util';
 import { extractInlineSettings, InlineSettings } from './inlineSettings';
 
-const regNonWordOrSpace = /[^\p{L}' ]+/giu;
-const regNonWordOrDigit = /[^\p{L}'\w-]+/giu;
+const regNonWordOrSpace = /[^\p{L}\p{M}' ]+/giu;
+const regNonWordOrDigit = /[^\p{L}\p{M}'\w-]+/giu;
 const regExpSpaceOrDash = /[- ]+/g;
 const regExpRepeatChars = /(.)\1{4,}/i;