From e1d106dbea96e076ef0160ccb5093f8bb52b5943 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 19 Sep 2021 14:02:02 +0200 Subject: [PATCH 1/2] Put identifier before repeatable symbols --- src/core/tokenizer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/tokenizer.ts b/src/core/tokenizer.ts index 0aabae53bb..07ff15c90e 100644 --- a/src/core/tokenizer.ts +++ b/src/core/tokenizer.ts @@ -42,9 +42,9 @@ const NUMBERS_REGEX = "(?<=[^.\\d]|^)\\d+\\.\\d+(?=[^.\\d]|$)"; // (not-dot/digi const REGEX = [ FIXED_TOKENS_REGEX, - REPEATABLE_SYMBOLS_REGEX, NUMBERS_REGEX, IDENTIFIERS_REGEX, + REPEATABLE_SYMBOLS_REGEX, SINGLE_SYMBOLS_REGEX, ].join("|"); From f20193cd3bbf6fb36e0553b00d52f25e935441b0 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Sun, 19 Sep 2021 14:52:07 +0200 Subject: [PATCH 2/2] Added test --- src/test/suite/tokenizer.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/suite/tokenizer.test.ts b/src/test/suite/tokenizer.test.ts index b9be64172c..b939d51c28 100644 --- a/src/test/suite/tokenizer.test.ts +++ b/src/test/suite/tokenizer.test.ts @@ -22,6 +22,7 @@ const tests: TestCase[] = [ ["my.variable", ["my", ".", "variable"]], ["my/variable", ["my", "/", "variable"]], ["my::variable", ["my", "::", "variable"]], + ["_a", ["_a"]], // Strings ['"my variable"', ['"', "my", "variable", '"']], ["'my variable'", ["'", "my", "variable", "'"]],