Don't allow number token to be immediately followed by a unicode letter

Fixes #651
sql-formatter-org · Oct 25, 2023 · 945b1f6 · 945b1f6
1 parent c95354c
commit 945b1f6
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/src/lexer/Tokenizer.ts b/src/lexer/Tokenizer.ts
@@ -46,7 +46,7 @@ export default class Tokenizer {
       {
         type: TokenType.NUMBER,
         regex:
-          /(?:0x[0-9a-fA-F]+|0b[01]+|(?:-\s*)?[0-9]+(?:\.[0-9]*)?(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?)(?!\w)/uy,
+          /(?:0x[0-9a-fA-F]+|0b[01]+|(?:-\s*)?[0-9]+(?:\.[0-9]*)?(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?)(?![\w\p{Alphabetic}])/uy,
       },
       // RESERVED_PHRASE is matched before all other keyword tokens
       // to e.g. prioritize matching "TIMESTAMP WITH TIME ZONE" phrase over "WITH" clause.

diff --git a/test/behavesLikeMariaDbFormatter.ts b/test/behavesLikeMariaDbFormatter.ts
@@ -51,6 +51,18 @@ export default function behavesLikeMariaDbFormatter(format: FormatFn) {
     );
   });
 
+  // regression test for sql-formatter#651
+  it('supports unicode identifiers that start with numbers', () => {
+    expect(format('SELECT 1ä FROM tbl')).toBe(
+      dedent`
+        SELECT
+          1ä
+        FROM
+          tbl
+      `
+    );
+  });
+
   it('supports @variables', () => {
     expect(format('SELECT @foo, @some_long.var$with$special.chars')).toBe(dedent`
       SELECT