[FIX] composer: prevent autocomplete on unknown characters in composer

Before this commit, the composer incorrectly triggered autocomplete suggestions
even when unknown or special characters (e.g., "é") were typed. This led to irrelevant
suggestions and formula errors when selecting an option.

This commit ensures that unknown characters are properly considered, preventing autocomplete
from being triggered in such cases, improving the reliability of the function autocomplete.

closes #6134

Task: 4652661
X-original-commit: 1beed80
Signed-off-by: Lucas Lefèvre (lul) <lul@odoo.com>
Signed-off-by: Mehdi Rachico (mera) <mera@odoo.com>

Author: Rachico

src/formulas/tokenizer.ts

@@ -152,6 +152,12 @@ function tokenizeString(chars: TokenizingChars): Token | null {
   return null;
 }
 
+/**
+  - \p{L} is for any letter (from any language)
+  - \p{N} is for any number
+  - the u flag at the end is for unicode, which enables the `\p{...}` syntax
+ */
+const unicodeSymbolCharRegexp = /\p{L}|\p{N}|_|\.|!|\$/u;
 const SYMBOL_CHARS = new Set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.!$");
 
 /**
@@ -193,7 +199,10 @@ function tokenizeSymbol(chars: TokenizingChars): Token | null {
       };
     }
   }
-  while (chars.current && SYMBOL_CHARS.has(chars.current)) {
+  while (
+    chars.current &&
+    (SYMBOL_CHARS.has(chars.current) || chars.current.match(unicodeSymbolCharRegexp))
+  ) {
     result += chars.shift();
   }
   if (result.length) {

tests/composer/composer_integration_component.test.ts

@@ -155,6 +155,18 @@ describe("Composer interactions", () => {
     expect(fixture.querySelector(".o-popover .o-autocomplete-dropdown")).toBeNull();
   });
 
+  test("autocomplete disappears when there is no match with an unknown character", async () => {
+    await typeInComposerGrid("=éSUM");
+    await nextTick();
+    expect(fixture.querySelector(".o-grid .o-autocomplete-dropdown")).toBeNull();
+  });
+
+  test("autocomplete disappear when typing an unknown character", async () => {
+    await typeInComposerGrid("=SéSUM");
+    await nextTick();
+    expect(fixture.querySelector(".o-grid .o-autocomplete-dropdown")).toBeNull();
+  });
+
   test("focus top bar composer does not resize grid composer when autocomplete is displayed", async () => {
     await keyDown({ key: "Enter" });
     const topBarComposer = document.querySelector(".o-spreadsheet-topbar .o-composer")!;

tests/evaluation/tokenizer.test.ts

@@ -287,10 +287,38 @@ describe("tokenizer", () => {
     ]);
   });
 
-  test("Unknown characters", () => {
+  test("non-ascii characters", () => {
     expect(tokenize("=ù4")).toEqual([
       { type: "OPERATOR", value: "=" },
-      { type: "UNKNOWN", value: "ù" },
+      { type: "SYMBOL", value: "ù4" },
+    ]);
+    expect(tokenize("=jai_nommé_mon_range")).toEqual([
+      { type: "OPERATOR", value: "=" },
+      { type: "SYMBOL", value: "jai_nommé_mon_range" },
+    ]);
+    expect(tokenize("=ßabc123")).toEqual([
+      { type: "OPERATOR", value: "=" },
+      { type: "SYMBOL", value: "ßabc123" },
+    ]);
+    expect(tokenize("=ぁ72")).toEqual([
+      { type: "OPERATOR", value: "=" },
+      { type: "SYMBOL", value: "ぁ72" },
+    ]);
+    expect(tokenize("=ñôtÁFñ(5, wr_öñg) + šymbøl +4")).toEqual([
+      { type: "OPERATOR", value: "=" },
+      { type: "SYMBOL", value: "ñôtÁFñ" },
+      { type: "LEFT_PAREN", value: "(" },
+      { type: "NUMBER", value: "5" },
+      { type: "ARG_SEPARATOR", value: "," },
+      { type: "SPACE", value: " " },
+      { type: "SYMBOL", value: "wr_öñg" },
+      { type: "RIGHT_PAREN", value: ")" },
+      { type: "SPACE", value: " " },
+      { type: "OPERATOR", value: "+" },
+      { type: "SPACE", value: " " },
+      { type: "SYMBOL", value: "šymbøl" },
+      { type: "SPACE", value: " " },
+      { type: "OPERATOR", value: "+" },
       { type: "NUMBER", value: "4" },
     ]);
   });