fix(internet): userName, email and slugify return only ascii (#1554)

src/modules/helpers/index.ts

@@ -33,8 +33,10 @@ export class HelpersModule {
    */
   slugify(string: string = ''): string {
     return string
-      .replace(/ /g, '-')
-      .replace(/[^\一-龠\ぁ-ゔ\ァ-ヴー\w\.\-]+/g, '');
+      .normalize('NFKD') //for example è decomposes to as e +  ̀
+      .replace(/[\u0300-\u036f]/g, '') // removes combining marks
+      .replace(/ /g, '-') // replaces spaces with hyphens
+      .replace(/[^\w\.\-]+/g, ''); // removes all non-word characters except for dots and hyphens
   }
 
   /**

src/modules/internet/char-mappings.ts

@@ -0,0 +1,285 @@
+// Selectively sourced from https://github.com/sindresorhus/transliterate/blob/08bbfd3a13ac393d945a430ed5ec62f044a08d70/replacements.js (under MIT license)
+const cyrillicMapping: { [key: string]: string } = Object.fromEntries([
+  ['А', 'A'],
+  ['а', 'a'],
+  ['Б', 'B'],
+  ['б', 'b'],
+  ['В', 'V'],
+  ['в', 'v'],
+  ['Г', 'G'],
+  ['г', 'g'],
+  ['Д', 'D'],
+  ['д', 'd'],
+  ['ъе', 'ye'],
+  ['Ъе', 'Ye'],
+  ['ъЕ', 'yE'],
+  ['ЪЕ', 'YE'],
+  ['Е', 'E'],
+  ['е', 'e'],
+  ['Ё', 'Yo'],
+  ['ё', 'yo'],
+  ['Ж', 'Zh'],
+  ['ж', 'zh'],
+  ['З', 'Z'],
+  ['з', 'z'],
+  ['И', 'I'],
+  ['и', 'i'],
+  ['ый', 'iy'],
+  ['Ый', 'Iy'],
+  ['ЫЙ', 'IY'],
+  ['ыЙ', 'iY'],
+  ['Й', 'Y'],
+  ['й', 'y'],
+  ['К', 'K'],
+  ['к', 'k'],
+  ['Л', 'L'],
+  ['л', 'l'],
+  ['М', 'M'],
+  ['м', 'm'],
+  ['Н', 'N'],
+  ['н', 'n'],
+  ['О', 'O'],
+  ['о', 'o'],
+  ['П', 'P'],
+  ['п', 'p'],
+  ['Р', 'R'],
+  ['р', 'r'],
+  ['С', 'S'],
+  ['с', 's'],
+  ['Т', 'T'],
+  ['т', 't'],
+  ['У', 'U'],
+  ['у', 'u'],
+  ['Ф', 'F'],
+  ['ф', 'f'],
+  ['Х', 'Kh'],
+  ['х', 'kh'],
+  ['Ц', 'Ts'],
+  ['ц', 'ts'],
+  ['Ч', 'Ch'],
+  ['ч', 'ch'],
+  ['Ш', 'Sh'],
+  ['ш', 'sh'],
+  ['Щ', 'Sch'],
+  ['щ', 'sch'],
+  ['Ъ', ''],
+  ['ъ', ''],
+  ['Ы', 'Y'],
+  ['ы', 'y'],
+  ['Ь', ''],
+  ['ь', ''],
+  ['Э', 'E'],
+  ['э', 'e'],
+  ['Ю', 'Yu'],
+  ['ю', 'yu'],
+  ['Я', 'Ya'],
+  ['я', 'ya'],
+]);
+const greekMapping: { [key: string]: string } = Object.fromEntries([
+  ['α', 'a'],
+  ['β', 'v'],
+  ['γ', 'g'],
+  ['δ', 'd'],
+  ['ε', 'e'],
+  ['ζ', 'z'],
+  ['η', 'i'],
+  ['θ', 'th'],
+  ['ι', 'i'],
+  ['κ', 'k'],
+  ['λ', 'l'],
+  ['μ', 'm'],
+  ['ν', 'n'],
+  ['ξ', 'ks'],
+  ['ο', 'o'],
+  ['π', 'p'],
+  ['ρ', 'r'],
+  ['σ', 's'],
+  ['τ', 't'],
+  ['υ', 'y'],
+  ['φ', 'f'],
+  ['χ', 'x'],
+  ['ψ', 'ps'],
+  ['ω', 'o'],
+  ['ά', 'a'],
+  ['έ', 'e'],
+  ['ί', 'i'],
+  ['ό', 'o'],
+  ['ύ', 'y'],
+  ['ή', 'i'],
+  ['ώ', 'o'],
+  ['ς', 's'],
+  ['ϊ', 'i'],
+  ['ΰ', 'y'],
+  ['ϋ', 'y'],
+  ['ΐ', 'i'],
+  ['Α', 'A'],
+  ['Β', 'B'],
+  ['Γ', 'G'],
+  ['Δ', 'D'],
+  ['Ε', 'E'],
+  ['Ζ', 'Z'],
+  ['Η', 'I'],
+  ['Θ', 'TH'],
+  ['Ι', 'I'],
+  ['Κ', 'K'],
+  ['Λ', 'L'],
+  ['Μ', 'M'],
+  ['Ν', 'N'],
+  ['Ξ', 'KS'],
+  ['Ο', 'O'],
+  ['Π', 'P'],
+  ['Ρ', 'R'],
+  ['Σ', 'S'],
+  ['Τ', 'T'],
+  ['Υ', 'Y'],
+  ['Φ', 'F'],
+  ['Χ', 'X'],
+  ['Ψ', 'PS'],
+  ['Ω', 'O'],
+  ['Ά', 'A'],
+  ['Έ', 'E'],
+  ['Ί', 'I'],
+  ['Ό', 'O'],
+  ['Ύ', 'Y'],
+  ['Ή', 'I'],
+  ['Ώ', 'O'],
+  ['Ϊ', 'I'],
+  ['Ϋ', 'Y'],
+]);
+const arabicMapping: { [key: string]: string } = Object.fromEntries([
+  ['ء', 'e'],
+  ['آ', 'a'],
+  ['أ', 'a'],
+  ['ؤ', 'w'],
+  ['إ', 'i'],
+  ['ئ', 'y'],
+  ['ا', 'a'],
+  ['ب', 'b'],
+  ['ة', 't'],
+  ['ت', 't'],
+  ['ث', 'th'],
+  ['ج', 'j'],
+  ['ح', 'h'],
+  ['خ', 'kh'],
+  ['د', 'd'],
+  ['ذ', 'dh'],
+  ['ر', 'r'],
+  ['ز', 'z'],
+  ['س', 's'],
+  ['ش', 'sh'],
+  ['ص', 's'],
+  ['ض', 'd'],
+  ['ط', 't'],
+  ['ظ', 'z'],
+  ['ع', 'e'],
+  ['غ', 'gh'],
+  ['ـ', '_'],
+  ['ف', 'f'],
+  ['ق', 'q'],
+  ['ك', 'k'],
+  ['ل', 'l'],
+  ['م', 'm'],
+  ['ن', 'n'],
+  ['ه', 'h'],
+  ['و', 'w'],
+  ['ى', 'a'],
+  ['ي', 'y'],
+  ['َ‎', 'a'],
+  ['ُ', 'u'],
+  ['ِ‎', 'i'],
+]);
+const armenianMapping: { [key: string]: string } = Object.fromEntries([
+  ['ա', 'a'],
+  ['Ա', 'A'],
+  ['բ', 'b'],
+  ['Բ', 'B'],
+  ['գ', 'g'],
+  ['Գ', 'G'],
+  ['դ', 'd'],
+  ['Դ', 'D'],
+  ['ե', 'ye'],
+  ['Ե', 'Ye'],
+  ['զ', 'z'],
+  ['Զ', 'Z'],
+  ['է', 'e'],
+  ['Է', 'E'],
+  ['ը', 'y'],
+  ['Ը', 'Y'],
+  ['թ', 't'],
+  ['Թ', 'T'],
+  ['ժ', 'zh'],
+  ['Ժ', 'Zh'],
+  ['ի', 'i'],
+  ['Ի', 'I'],
+  ['լ', 'l'],
+  ['Լ', 'L'],
+  ['խ', 'kh'],
+  ['Խ', 'Kh'],
+  ['ծ', 'ts'],
+  ['Ծ', 'Ts'],
+  ['կ', 'k'],
+  ['Կ', 'K'],
+  ['հ', 'h'],
+  ['Հ', 'H'],
+  ['ձ', 'dz'],
+  ['Ձ', 'Dz'],
+  ['ղ', 'gh'],
+  ['Ղ', 'Gh'],
+  ['ճ', 'tch'],
+  ['Ճ', 'Tch'],
+  ['մ', 'm'],
+  ['Մ', 'M'],
+  ['յ', 'y'],
+  ['Յ', 'Y'],
+  ['ն', 'n'],
+  ['Ն', 'N'],
+  ['շ', 'sh'],
+  ['Շ', 'Sh'],
+  ['ո', 'vo'],
+  ['Ո', 'Vo'],
+  ['չ', 'ch'],
+  ['Չ', 'Ch'],
+  ['պ', 'p'],
+  ['Պ', 'P'],
+  ['ջ', 'j'],
+  ['Ջ', 'J'],
+  ['ռ', 'r'],
+  ['Ռ', 'R'],
+  ['ս', 's'],
+  ['Ս', 'S'],
+  ['վ', 'v'],
+  ['Վ', 'V'],
+  ['տ', 't'],
+  ['Տ', 'T'],
+  ['ր', 'r'],
+  ['Ր', 'R'],
+  ['ց', 'c'],
+  ['Ց', 'C'],
+  ['ու', 'u'],
+  ['ՈՒ', 'U'],
+  ['Ու', 'U'],
+  ['փ', 'p'],
+  ['Փ', 'P'],
+  ['ք', 'q'],
+  ['Ք', 'Q'],
+  ['օ', 'o'],
+  ['Օ', 'O'],
+  ['ֆ', 'f'],
+  ['Ֆ', 'F'],
+  ['և', 'yev'],
+]);
+const farsiMapping: { [key: string]: string } = Object.fromEntries([
+  ['چ', 'ch'],
+  ['ک', 'k'],
+  ['گ', 'g'],
+  ['پ', 'p'],
+  ['ژ', 'zh'],
+  ['ی', 'y'],
+]);
+export const charMapping: { [key: string]: string } = {
+  ...cyrillicMapping,
+  ...greekMapping,
+  ...arabicMapping,
+  ...farsiMapping,
+  ...armenianMapping,
+};