Merge pull request #4662 from kevinushey/feature/tokenizer-unicode-regex

allow tokenizer rules to opt-in to using unicode regexes
ajaxorg · Mar 22, 2022 · 07893b9 · 07893b9
2 parents 8c098ef + 73d04f1
commit 07893b9
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/lib/ace/tokenizer.js b/lib/ace/tokenizer.js
@@ -62,8 +62,10 @@ var Tokenizer = function(rules) {
             var rule = state[i];
             if (rule.defaultToken)
                 mapping.defaultToken = rule.defaultToken;
-            if (rule.caseInsensitive)
-                flag = "gi";
+            if (rule.caseInsensitive && flag.indexOf("i") === -1)
+                flag += "i";
+            if (rule.unicode && flag.indexOf("u") === -1)
+                flag += "u";
             if (rule.regex == null)
                 continue;