From 8bbfcee3f07d805c517982426d77be37b22919f1 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 13 Sep 2021 18:01:27 +0200 Subject: [PATCH 1/2] Added triple quote to regular expression --- src/core/tokenizer.ts | 1 + src/test/suite/tokenizer.test.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/core/tokenizer.ts b/src/core/tokenizer.ts index fe37c7e3f2..7a22271e27 100644 --- a/src/core/tokenizer.ts +++ b/src/core/tokenizer.ts @@ -27,6 +27,7 @@ const FIXED_TOKENS = [ "=>", "->", "??", + '"""', "\\r", "\\n", "\\t", diff --git a/src/test/suite/tokenizer.test.ts b/src/test/suite/tokenizer.test.ts index d0da8c621c..5a51415f33 100644 --- a/src/test/suite/tokenizer.test.ts +++ b/src/test/suite/tokenizer.test.ts @@ -38,6 +38,8 @@ const tests: TestCase[] = [ ["->", ["->"]], ["??", ["??"]], ["\\r\\n\\t", ["\\r", "\\n", "\\t"]], + ['"""hello"""', ['"""', "hello", '"""']], + ['""""""', ['"""', '"""']], // Comments ["// Hello world", ["//", "Hello", "world"]], // Hex colors From cfa1f7d94b9a89ef8e81b86125162161188d9dd6 Mon Sep 17 00:00:00 2001 From: Andreas Arvidsson Date: Mon, 13 Sep 2021 18:17:39 +0200 Subject: [PATCH 2/2] Added additional tests --- src/test/suite/tokenizer.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/suite/tokenizer.test.ts b/src/test/suite/tokenizer.test.ts index 5a51415f33..fadd497add 100644 --- a/src/test/suite/tokenizer.test.ts +++ b/src/test/suite/tokenizer.test.ts @@ -40,6 +40,8 @@ const tests: TestCase[] = [ ["\\r\\n\\t", ["\\r", "\\n", "\\t"]], ['"""hello"""', ['"""', "hello", '"""']], ['""""""', ['"""', '"""']], + ['""', ['"', '"']], + ["''", ["'", "'"]], // Comments ["// Hello world", ["//", "Hello", "world"]], // Hex colors