#885: Support string escaping

eclipse-langium · Jan 30, 2023 · 75184b1 · 75184b1
1 parent 7a8338c
commit 75184b1
Show file tree

Hide file tree

Showing 19 changed files with 199 additions and 164 deletions.
diff --git a/examples/domainmodel/src/language-server/domain-model.langium b/examples/domainmodel/src/language-server/domain-model.langium
@@ -30,8 +30,6 @@ QualifiedName returns string:
 
 hidden terminal WS: /\s+/;
 terminal ID: /[_a-zA-Z][\w_]*/;
-terminal INT returns number: /[0-9]+/;
-terminal STRING: /"[^"]*"|'[^']*'/;
 
 hidden terminal ML_COMMENT: /\/\*[\s\S]*?\*\//;
 hidden terminal SL_COMMENT: /\/\/[^\n\r]*/;
diff --git a/examples/domainmodel/src/language-server/generated/grammar.ts b/examples/domainmodel/src/language-server/generated/grammar.ts
@@ -376,30 +376,6 @@ export const DomainModelGrammar = (): Grammar => loadedDomainModelGrammar ?? (lo
       "fragment": false,
       "hidden": false
     },
-    {
-      "$type": "TerminalRule",
-      "name": "INT",
-      "type": {
-        "$type": "ReturnType",
-        "name": "number"
-      },
-      "definition": {
-        "$type": "RegexToken",
-        "regex": "[0-9]+"
-      },
-      "fragment": false,
-      "hidden": false
-    },
-    {
-      "$type": "TerminalRule",
-      "name": "STRING",
-      "definition": {
-        "$type": "RegexToken",
-        "regex": "\\"[^\\"]*\\"|'[^']*'"
-      },
-      "fragment": false,
-      "hidden": false
-    },
     {
       "$type": "TerminalRule",
       "hidden": true,

diff --git a/examples/domainmodel/syntaxes/domain-model.tmLanguage.json b/examples/domainmodel/syntaxes/domain-model.tmLanguage.json
@@ -11,16 +11,6 @@
     {
       "name": "keyword.control.domain-model",
       "match": "\\b(datatype|entity|extends|many|package)\\b"
-    },
-    {
-      "name": "string.quoted.double.domain-model",
-      "begin": "\"",
-      "end": "\""
-    },
-    {
-      "name": "string.quoted.single.domain-model",
-      "begin": "'",
-      "end": "'"
     }
   ],
   "repository": {

diff --git a/examples/domainmodel/syntaxes/domainmodel.monarch.ts b/examples/domainmodel/syntaxes/domainmodel.monarch.ts
@@ -11,8 +11,6 @@ export default {
     tokenizer: {
         initial: [
             { regex: /[_a-zA-Z][\w_]*/, action: { cases: { '@keywords': {"token":"keyword"}, '@default': {"token":"ID"} }} },
-            { regex: /[0-9]+/, action: {"token":"number"} },
-            { regex: /"[^"]*"|'[^']*'/, action: {"token":"string"} },
             { include: '@whitespace' },
             { regex: /@symbols/, action: { cases: { '@operators': {"token":"operator"}, '@default': {"token":""} }} },
         ],

diff --git a/examples/requirements/src/language-server/common.langium b/examples/requirements/src/language-server/common.langium
@@ -3,7 +3,7 @@ Contact: 'contact' ':' user_name=STRING;
 hidden terminal WS: /\s+/;
 terminal ID: /[_a-zA-Z][\w_]*/;
 terminal INT returns number: /[0-9]+/;
-terminal STRING: /"[^"]*"|'[^']*'/;
+terminal STRING: /"(\\.|[^"\\])*"|'(\\.|[^'\\])*'/;
 
 hidden terminal ML_COMMENT: /\/\*[\s\S]*?\*\//;
 hidden terminal SL_COMMENT: /\/\/[^\n\r]*/;
diff --git a/examples/requirements/src/language-server/generated/grammar.ts b/examples/requirements/src/language-server/generated/grammar.ts
@@ -278,7 +278,7 @@ export const RequirementsGrammar = (): Grammar => loadedRequirementsGrammar ?? (
       "name": "STRING",
       "definition": {
         "$type": "RegexToken",
-        "regex": "\\"[^\\"]*\\"|'[^']*'"
+        "regex": "\\"(\\\\\\\\.|[^\\"\\\\\\\\])*\\"|'(\\\\\\\\.|[^'\\\\\\\\])*'"
       },
       "fragment": false,
       "hidden": false
@@ -590,7 +590,7 @@ export const TestsGrammar = (): Grammar => loadedTestsGrammar ?? (loadedTestsGra
       "name": "STRING",
       "definition": {
         "$type": "RegexToken",
-        "regex": "\\"[^\\"]*\\"|'[^']*'"
+        "regex": "\\"(\\\\\\\\.|[^\\"\\\\\\\\])*\\"|'(\\\\\\\\.|[^'\\\\\\\\])*'"
       },
       "fragment": false,
       "hidden": false

diff --git a/examples/requirements/syntaxes/requirements.tmLanguage.json b/examples/requirements/syntaxes/requirements.tmLanguage.json
@@ -15,12 +15,22 @@
     {
       "name": "string.quoted.double.requirements-lang",
       "begin": "\"",
-      "end": "\""
+      "end": "\"",
+      "patterns": [
+        {
+          "include": "#string-character-escape"
+        }
+      ]
     },
     {
       "name": "string.quoted.single.requirements-lang",
       "begin": "'",
-      "end": "'"
+      "end": "'",
+      "patterns": [
+        {
+          "include": "#string-character-escape"
+        }
+      ]
     }
   ],
   "repository": {
@@ -52,6 +62,10 @@
           "name": "comment.line.requirements-lang"
         }
       ]
+    },
+    "string-character-escape": {
+      "name": "constant.character.escape.requirements-lang",
+      "match": "\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\\{[0-9A-Fa-f]+\\}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.|$)"
     }
   }
-}
+}
diff --git a/examples/requirements/syntaxes/tests.tmLanguage.json b/examples/requirements/syntaxes/tests.tmLanguage.json
@@ -15,12 +15,22 @@
     {
       "name": "string.quoted.double.tests-lang",
       "begin": "\"",
-      "end": "\""
+      "end": "\"",
+      "patterns": [
+        {
+          "include": "#string-character-escape"
+        }
+      ]
     },
     {
       "name": "string.quoted.single.tests-lang",
       "begin": "'",
-      "end": "'"
+      "end": "'",
+      "patterns": [
+        {
+          "include": "#string-character-escape"
+        }
+      ]
     }
   ],
   "repository": {
@@ -52,6 +62,10 @@
           "name": "comment.line.tests-lang"
         }
       ]
+    },
+    "string-character-escape": {
+      "name": "constant.character.escape.tests-lang",
+      "match": "\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\\{[0-9A-Fa-f]+\\}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.|$)"
     }
   }
-}
+}
diff --git a/examples/statemachine/src/language-server/generated/grammar.ts b/examples/statemachine/src/language-server/generated/grammar.ts
@@ -303,30 +303,6 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
       "fragment": false,
       "hidden": false
     },
-    {
-      "$type": "TerminalRule",
-      "name": "INT",
-      "type": {
-        "$type": "ReturnType",
-        "name": "number"
-      },
-      "definition": {
-        "$type": "RegexToken",
-        "regex": "[0-9]+"
-      },
-      "fragment": false,
-      "hidden": false
-    },
-    {
-      "$type": "TerminalRule",
-      "name": "STRING",
-      "definition": {
-        "$type": "RegexToken",
-        "regex": "\\"[^\\"]*\\"|'[^']*'"
-      },
-      "fragment": false,
-      "hidden": false
-    },
     {
       "$type": "TerminalRule",
       "hidden": true,

diff --git a/examples/statemachine/src/language-server/statemachine.langium b/examples/statemachine/src/language-server/statemachine.langium
@@ -24,8 +24,6 @@ Transition:
 
 hidden terminal WS: /\s+/;
 terminal ID: /[_a-zA-Z][\w_]*/;
-terminal INT returns number: /[0-9]+/;
-terminal STRING: /"[^"]*"|'[^']*'/;
 
 hidden terminal ML_COMMENT: /\/\*[\s\S]*?\*\//;
 hidden terminal SL_COMMENT: /\/\/[^\n\r]*/;
diff --git a/examples/statemachine/syntaxes/statemachine.monarch.ts b/examples/statemachine/syntaxes/statemachine.monarch.ts
@@ -11,8 +11,6 @@ export default {
     tokenizer: {
         initial: [
             { regex: /[_a-zA-Z][\w_]*/, action: { cases: { '@keywords': {"token":"keyword"}, '@default': {"token":"ID"} }} },
-            { regex: /[0-9]+/, action: {"token":"number"} },
-            { regex: /"[^"]*"|'[^']*'/, action: {"token":"string"} },
             { include: '@whitespace' },
             { regex: /@symbols/, action: { cases: { '@operators': {"token":"operator"}, '@default': {"token":""} }} },
         ],

diff --git a/examples/statemachine/syntaxes/statemachine.tmLanguage.json b/examples/statemachine/syntaxes/statemachine.tmLanguage.json
@@ -11,16 +11,6 @@
     {
       "name": "keyword.control.statemachine",
       "match": "\\b(actions|commands|end|events|initialState|state|statemachine)\\b"
-    },
-    {
-      "name": "string.quoted.double.statemachine",
-      "begin": "\"",
-      "end": "\""
-    },
-    {
-      "name": "string.quoted.single.statemachine",
-      "begin": "'",
-      "end": "'"
     }
   ],
   "repository": {

diff --git a/packages/generator-langium/langium-template/src/language-server/language-id.langium b/packages/generator-langium/langium-template/src/language-server/language-id.langium
@@ -12,7 +12,7 @@ Greeting:
 hidden terminal WS: /\s+/;
 terminal ID: /[_a-zA-Z][\w_]*/;
 terminal INT returns number: /[0-9]+/;
-terminal STRING: /"[^"]*"|'[^']*'/;
+terminal STRING: /"(\\.|[^"\\])*"|'(\\.|[^'\\])*'/;
 
 hidden terminal ML_COMMENT: /\/\*[\s\S]*?\*\//;
 hidden terminal SL_COMMENT: /\/\/[^\n\r]*/;
diff --git a/packages/langium-cli/src/generator/highlighting/textmate-generator.ts b/packages/langium-cli/src/generator/highlighting/textmate-generator.ts
@@ -2,7 +2,7 @@
  * Copyright 2021-2022 TypeFox GmbH
  * This program and the accompanying materials are made available under the
  * terms of the MIT License, which is available in the project root.
- ******************************************************************************/
+******************************************************************************/
 
 import {
     escapeRegExp, getCaseInsensitivePattern, getTerminalParts, Grammar, GrammarAST, isCommentTerminal, stream
@@ -11,6 +11,8 @@ import { terminalRegex } from 'langium/lib/grammar/internal-grammar-util';
 import { LangiumLanguageConfig } from '../../package';
 import { collectKeywords } from '../util';
 
+/* eslint-disable dot-notation */
+
 export interface TextMateGrammar {
     repository: Repository;
     readonly scopeName: string;
@@ -71,7 +73,9 @@ function getPatterns(grammar: Grammar, config: LangiumLanguageConfig): Pattern[]
 }
 
 function getRepository(grammar: Grammar, config: LangiumLanguageConfig): Repository {
+    const repository: Repository = {};
     const commentPatterns: Pattern[] = [];
+    let stringEscapePattern: Pattern | undefined;
     for (const rule of grammar.rules) {
         if (GrammarAST.isTerminalRule(rule) && isCommentTerminal(rule)) {
             const parts = getTerminalParts(terminalRegex(rule));
@@ -105,14 +109,22 @@ function getRepository(grammar: Grammar, config: LangiumLanguageConfig): Reposit
                     });
                 }
             }
+        } else if (GrammarAST.isTerminalRule(rule) && rule.name.toLowerCase() === 'string') {
+            stringEscapePattern = {
+                'name': `constant.character.escape.${config.id}`,
+                'match': '\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\\{[0-9A-Fa-f]+\\}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.|$)'
+            };
         }
     }
-    const repository: Repository = {
-        'comments': {
-            'patterns': commentPatterns
-        }
-    };
 
+    if (commentPatterns.length > 0) {
+        repository['comments'] = {
+            'patterns': commentPatterns
+        };
+    }
+    if (stringEscapePattern) {
+        repository['string-character-escape'] = stringEscapePattern;
+    }
     return repository;
 }
 
@@ -170,7 +182,12 @@ function getStringPatterns(grammar: Grammar, pack: LangiumLanguageConfig): Patte
                 stringPatterns.push({
                     'name': `string.quoted.${delimiterName(part.start)}.${pack.id}`,
                     'begin': part.start,
-                    'end': part.end
+                    'end': part.end,
+                    'patterns': [
+                        {
+                            'include': '#string-character-escape'
+                        }
+                    ]
                 });
             }
         }

diff --git a/packages/langium-vscode/data/langium.tmLanguage.json b/packages/langium-vscode/data/langium.tmLanguage.json
@@ -26,12 +26,22 @@
         {
             "name": "string.quoted.double.langium",
             "begin": "\"",
-            "end": "\""
+            "end": "\"",
+            "patterns": [
+				{
+					"include": "#string-character-escape"
+				}
+			]
         },
         {
             "name": "string.quoted.single.langium",
             "begin": "'",
-            "end": "'"
+            "end": "'",
+            "patterns": [
+				{
+					"include": "#string-character-escape"
+				}
+			]
         }
     ],
     "repository": {
@@ -64,23 +74,27 @@
                 }
             ]
         },
+        "string-character-escape": {
+            "name": "constant.character.escape.langium",
+            "match": "\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\\{[0-9A-Fa-f]+\\}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.|$)"
+        },
         "regex": {
             "patterns": [
                 {
-                    "name": "string.regex.ts",
+                    "name": "string.regex.langium",
                     "begin": "/(?![/*])(?=(?:[^/\\\\\\[]|\\\\.|\\[([^\\]\\\\]|\\\\.)+\\])+/(?![/*]))",
                     "beginCaptures": {
                         "0": {
-                            "name": "punctuation.definition.string.begin.ts"
+                            "name": "punctuation.definition.string.begin.langium"
                         }
                     },
                     "end": "/",
                     "endCaptures": {
                         "1": {
-                            "name": "punctuation.definition.string.end.ts"
+                            "name": "punctuation.definition.string.end.langium"
                         },
                         "2": {
-                            "name": "keyword.other.ts"
+                            "name": "keyword.other.langium"
                         }
                     },
                     "patterns": [