diff --git a/src/core/tokenizer.ts b/src/core/tokenizer.ts index fe37c7e3f2..7a22271e27 100644 --- a/src/core/tokenizer.ts +++ b/src/core/tokenizer.ts @@ -27,6 +27,7 @@ const FIXED_TOKENS = [ "=>", "->", "??", + '"""', "\\r", "\\n", "\\t", diff --git a/src/test/suite/tokenizer.test.ts b/src/test/suite/tokenizer.test.ts index d0da8c621c..fadd497add 100644 --- a/src/test/suite/tokenizer.test.ts +++ b/src/test/suite/tokenizer.test.ts @@ -38,6 +38,10 @@ const tests: TestCase[] = [ ["->", ["->"]], ["??", ["??"]], ["\\r\\n\\t", ["\\r", "\\n", "\\t"]], + ['"""hello"""', ['"""', "hello", '"""']], + ['""""""', ['"""', '"""']], + ['""', ['"', '"']], + ["''", ["'", "'"]], // Comments ["// Hello world", ["//", "Hello", "world"]], // Hex colors