feat(devin-lang): introduce support for agent commands and variables #…

…101 The commit introduces a new lexical and syntactic model for the DevInLanguage, including the support for agent commands and variables. The changes include the addition of new token types, grammar rules, and completion logic, as well as the modification of the lexer and syntax highlighting to accommodate these new features.
unit-mesh · Mar 11, 2024 · 03c6d92 · 03c6d92
1 parent 035f8b7
commit 03c6d92
Show file tree

Hide file tree

Showing 7 changed files with 41 additions and 14 deletions.
diff --git a/exts/devin-lang/src/grammar/DevInLexer.flex b/exts/devin-lang/src/grammar/DevInLexer.flex
@@ -23,7 +23,7 @@ import com.intellij.psi.TokenType;
 %eof}
 
 %s CONTEXT_BLOCK
-%s ID_SUFFIX
+%s AGENT_BLOCK
 
 IDENTIFIER=[a-zA-Z0-9]([_\-a-zA-Z0-9]*)
 REF_BLOCK=([$/@] {IDENTIFIER} )
@@ -35,21 +35,23 @@ NEWLINE=\n|\r\n
         yybegin(YYINITIAL);
 
         String text = yytext().toString();
-        System.out.println("contextBlock: " + text);
 
         return TEXT_SEGMENT;
     }
 %}
 
 %%
 <YYINITIAL> {
-  {REF_BLOCK}          { return REF_BLOCK; }
+  "@"                  { yybegin(AGENT_BLOCK); return AGENT_START; }
+  "/"                  { yybegin(AGENT_BLOCK); return COMMAND_START; }
+  "$"                  { yybegin(AGENT_BLOCK); return VARIABLE_START; }
+
   {TEXT_SEGMENT}       { return TEXT_SEGMENT; }
   {NEWLINE}            { return NEWLINE; }
   [^]                  { return TokenType.BAD_CHARACTER; }
 }
 
-<CONTEXT_BLOCK> {
-  [$/@]                { yybegin(YYINITIAL); return contextBlock(); }
+<AGENT_BLOCK> {
+  {IDENTIFIER}         { yybegin(YYINITIAL); return IDENTIFIER; }
   [^]                  { return TokenType.BAD_CHARACTER; }
 }
diff --git a/exts/devin-lang/src/grammar/DevInParser.bnf b/exts/devin-lang/src/grammar/DevInParser.bnf
@@ -15,9 +15,20 @@
   tokenTypeClass="cc.unitmesh.language.lexer.DevInTokenType"
 
   tokens=[
+    AGENT_START    = "AGENT_START"
+    COMMAND_START  = "COMMAND_START"
+    VARIABLE_START = "VARIABLE_START"
   ]
 }
 
-DevInFile ::= (REF_BLOCK | TEXT_SEGMENT | CODE_FENCE | NEWLINE)*
+DevInFile ::= (used | TEXT_SEGMENT | CODE_FENCE | NEWLINE)*
 
-//used ::= (AGENT | COMMAND | VARIABLE) IDENTIFIER
+used ::= (
+    AGENT_START agentId?
+    | COMMAND_START commandId?
+    | VARIABLE_START variableId?
+)
+
+agentId ::= IDENTIFIER
+commandId ::= IDENTIFIER
+variableId ::= IDENTIFIER
diff --git a/exts/devin-lang/src/main/kotlin/cc/unitmesh/language/DevInCompletionContributor.kt b/exts/devin-lang/src/main/kotlin/cc/unitmesh/language/DevInCompletionContributor.kt
@@ -1,17 +1,27 @@
 package cc.unitmesh.language
 
+import cc.unitmesh.language.completion.VariableProvider
+import cc.unitmesh.language.psi.DevInFile
 import cc.unitmesh.language.psi.DevInTypes
 import com.intellij.codeInsight.completion.CompletionContributor
+import com.intellij.codeInsight.completion.CompletionInitializationContext
 import com.intellij.codeInsight.completion.CompletionType
 import com.intellij.patterns.PlatformPatterns.psiElement
-import cc.unitmesh.language.completion.VariableProvider
 
 class DevInCompletionContributor : CompletionContributor() {
+    private val INPUT_DUMMY_IDENTIFIER = "AutoDevDummy"
+
     init {
         extend(
             CompletionType.BASIC,
-            psiElement(DevInTypes.REF_BLOCK),
+            psiElement(DevInTypes.VARIABLE_START),
             VariableProvider()
         )
     }
+
+    override fun beforeCompletion(context: CompletionInitializationContext) {
+        if ((context.file is DevInFile) && context.dummyIdentifier === INPUT_DUMMY_IDENTIFIER) {
+            context.dummyIdentifier = INPUT_DUMMY_IDENTIFIER
+        }
+    }
 }
diff --git a/exts/devin-lang/src/main/kotlin/cc/unitmesh/language/DevInParserDefinition.kt b/exts/devin-lang/src/main/kotlin/cc/unitmesh/language/DevInParserDefinition.kt
@@ -45,7 +45,7 @@ internal class DevInParserDefinition : ParserDefinition {
 
     @NotNull
     override fun createElement(node: ASTNode?): PsiElement {
-        TODO()
+        return DevInTypes.Factory.createElement(node)
     }
 
     companion object {

diff --git a/exts/devin-lang/src/main/kotlin/cc/unitmesh/language/DevInSyntaxHighlighter.kt b/exts/devin-lang/src/main/kotlin/cc/unitmesh/language/DevInSyntaxHighlighter.kt
@@ -19,7 +19,8 @@ class DevInSyntaxHighlighter : SyntaxHighlighter {
         private val ATTRIBUTES: MutableMap<IElementType, TextAttributesKey> = HashMap()
 
         init {
-            ATTRIBUTES[DevInTypes.REF_BLOCK] = DefaultLanguageHighlighterColors.KEYWORD
+            ATTRIBUTES[DevInTypes.VARIABLE_START] = DefaultLanguageHighlighterColors.KEYWORD
+            ATTRIBUTES[DevInTypes.VARIABLE_ID] = DefaultLanguageHighlighterColors.NUMBER
         }
     }
 

diff --git a/exts/devin-lang/src/test/testData/parser/BasicTest.txt b/exts/devin-lang/src/test/testData/parser/BasicTest.txt
@@ -1,4 +1,7 @@
-FILE
+DevInFile
   PsiElement(DevInTokenType.TEXT_SEGMENT)('你好 ')
-  PsiElement(DevInTokenType.REF_BLOCK)('@hello-world')
+  DevInUsedImpl(USED)
+    PsiElement(DevInTokenType.AGENT_START)('@')
+    DevInAgentIdImpl(AGENT_ID)
+      PsiElement(DevInTokenType.IDENTIFIER)('hello-world')
   PsiElement(DevInTokenType.TEXT_SEGMENT)(' sm')
diff --git a/exts/devin-lang/src/test/testData/parser/JavaHelloWorld.txt b/exts/devin-lang/src/test/testData/parser/JavaHelloWorld.txt
@@ -1,2 +1,2 @@
-FILE
+DevInFile
   PsiElement(DevInTokenType.TEXT_SEGMENT)('```java\npublic class Main {\n    public static void main(String[] args) {\n        System.out.println("Hello, world!");\n    }\n}\n```')