fix(devin-lang): update parser and lexer tokens and rules for improve…

…d grammar and tokenization. #101
unit-mesh · Mar 11, 2024 · eda58cd · eda58cd
1 parent b7e6106
commit eda58cd
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 16 deletions.
diff --git a/exts/devin-lang/src/grammar/DevInLexer.flex b/exts/devin-lang/src/grammar/DevInLexer.flex
@@ -27,7 +27,7 @@ import com.intellij.psi.TokenType;
 
 IDENTIFIER=[a-zA-Z0-9]([_\-a-zA-Z0-9]*)
 REF_BLOCK=("@" {IDENTIFIER} )
-TEXT_SEGMENT=.*
+TEXT_SEGMENT=[^$/@]+
 NEWLINE=\n|\r\n
 
 %{

diff --git a/exts/devin-lang/src/grammar/DevInParser.bnf b/exts/devin-lang/src/grammar/DevInParser.bnf
@@ -13,6 +13,11 @@
   elementTypeHolderClass="cc.unitmesh.language.psi.DevInTypes"
   elementTypeClass="cc.unitmesh.language.psi.DevInElementType"
   tokenTypeClass="cc.unitmesh.language.lexer.DevInTokenType"
+
+  tokens=[
+  ]
 }
 
-DevInFile ::= (REF_BLOCK | TEXT_SEGMENT | NEWLINE)*
+DevInFile ::= (REF_BLOCK | TEXT_SEGMENT | CODE_FENCE | NEWLINE)*
+
+//used ::= (AGENT | COMMAND | VARIABLE) IDENTIFIER
diff --git a/exts/devin-lang/src/test/testData/parser/BasicTest.txt b/exts/devin-lang/src/test/testData/parser/BasicTest.txt
@@ -1,2 +1,4 @@
 FILE
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('你好 @hello-world sm')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('你好 ')
+  PsiElement(DevInTokenType.REF_BLOCK)('@hello-world')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)(' sm')
diff --git a/exts/devin-lang/src/test/testData/parser/JavaHelloWorld.txt b/exts/devin-lang/src/test/testData/parser/JavaHelloWorld.txt
@@ -1,14 +1,2 @@
 FILE
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('```java')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('public class Main {')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('    public static void main(String[] args) {')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('        System.out.println("Hello, world!");')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('    }')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('}')
-  PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('```')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)('```java\npublic class Main {\n    public static void main(String[] args) {\n        System.out.println("Hello, world!");\n    }\n}\n```')