dct: add basic tokenizer

pulsejet · Nov 3, 2023 · b6509e7 · b6509e7
1 parent 1dac383
commit b6509e7
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 2 deletions.
diff --git a/src/app/dct/dct.component.html b/src/app/dct/dct.component.html
@@ -1,4 +1,4 @@
-<app-editor [(code)]="schema" language="text"></app-editor>
+<app-editor [(code)]="schema" language="versec"></app-editor>
 
 <div class="actions">
     <button class="button is-danger is-light is-small mt-1" (click)="compile()">

diff --git a/src/app/editor/editor.component.ts b/src/app/editor/editor.component.ts
@@ -1,4 +1,5 @@
 import { Component, EventEmitter, Input, OnInit, Output, SimpleChange } from '@angular/core';
+import versecLang from './versec.lang';
 import * as userTypes from '../user-types';
 
 export const monacoConfig = {
@@ -19,7 +20,7 @@ export const monacoConfig = {
       url);
     }
 
-    // compiler options
+    // diagnostic options
     monaco.languages.typescript.javascriptDefaults.setDiagnosticsOptions({
       noSemanticValidation: false,
       noSyntaxValidation: false,
@@ -28,11 +29,16 @@ export const monacoConfig = {
       ]
     });
 
+    // compiler options
     monaco.languages.typescript.javascriptDefaults.setCompilerOptions({
       target: monaco.languages.typescript.ScriptTarget.ES2015,
       allowNonTsExtensions: true
     });
 
+    // add versec language
+    monaco.languages.register({ id: 'versec' });
+    monaco.languages.setMonarchTokensProvider('versec', versecLang)
+
     await injectLib('/assets/user-types.d.ts', 'ndn', Object.keys(userTypes.ext));
   }
 };

diff --git a/src/app/editor/versec.lang.ts b/src/app/editor/versec.lang.ts
@@ -0,0 +1,59 @@
+export default {
+  operators: [":", "<="],
+  symbols: /[=><!~?:&|+\-*\/\^%]+/,
+  escapes:
+    /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+  tokenizer: {
+    root: [
+      // whitespace
+      { include: "@whitespace" },
+
+      // types
+      [
+        /#(pubPrefix|msgsValidator|certValidator|pduValidator)\b/,
+        { token: "keyword" },
+      ],
+      [/#\w+\b/, { token: "type" }],
+      [/_\w*\b/, { token: "variable" }],
+
+      // defns
+      [/^(_|#)?\w+/, { token: "function" }],
+
+      // invocations
+      [/\w+\(\)/, { token: "function" }],
+
+      // keywords
+      [/&|<=|\//, { token: "keyword" }],
+
+      // delimiters and operators
+      [/[{}()\[\]]/, "@brackets"],
+      [/[<>](?!@symbols)/, "@brackets"],
+      [/@symbols/, { cases: { "@operators": "operator", "@default": "" } }],
+
+      // strings
+      [/"([^"\\]|\\.)*$/, "string.invalid"], // non-teminated string
+      [/"/, { token: "string.quote", bracket: "@open", next: "@string" }],
+    ],
+
+    comment: [
+      [/[^\/*]+/, "comment"],
+      [/\/\*/, "comment", "@push"], // nested comment
+      ["\\*/", "comment", "@pop"],
+      [/[\/*]/, "comment"],
+    ],
+
+    string: [
+      [/[^\\"]+/, "string"],
+      [/@escapes/, "string.escape"],
+      [/\\./, "string.escape.invalid"],
+      [/"/, { token: "string.quote", bracket: "@close", next: "@pop" }],
+    ],
+
+    whitespace: [
+      [/[ \t\r\n]+/, "white"],
+      [/\/\*/, "comment", "@comment"],
+      [/\/\/.*$/, "comment"],
+    ],
+  },
+};