feat(devins-lang): add support for system calling with identifiers an…

…d colon-separated parameters #101 The commit introduces a new feature to the DevInParser and DevInLexer, enabling the parsing and lexing of system calling commands with identifiers and colon-separated parameters. This enhancement allows for more complex and expressive system interactions within the DevInLanguage.
unit-mesh · Mar 19, 2024 · 37b88cf · 37b88cf
1 parent 05d93c1
commit 37b88cf
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 8 deletions.
diff --git a/docs/devins/devins-language.md b/docs/devins/devins-language.md
@@ -22,7 +22,7 @@ Based on: [JetBrains' Markdown Util](https://github.com/JetBrains/intellij-commu
 - `/` Builtin Command, natural language command with IDE/editor, like read file, write file, etc.
 - `@` Agent, natural language custom function / system function name, the handler or command, 
 - `$` Variable, natural language variable name, like file name, file content, etc.
-- `#` Third-party system API for traditional, like `#kanban`, `#issue`, `#github`, etc. 
+- `#` Third-party system API for traditional, like `#kanban:afd`, `#issue:233`, `#github:111`, etc. 
 
 ## Language spec
 

diff --git a/exts/devins-lang/src/grammar/DevInLexer.flex b/exts/devins-lang/src/grammar/DevInLexer.flex
@@ -39,15 +39,17 @@ IDENTIFIER=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 VARIABLE_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 AGENT_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 COMMAND_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
-LANGUAGE_ID=[a-zA-Z0-9][_\-a-zA-Z0-9 .]*
-SYSTEM_ID=[a-zA-Z0-9][_\-a-zA-Z0-9 .]*
+LANGUAGE_ID=[a-zA-Z][_\-a-zA-Z0-9 .]*
+SYSTEM_ID=[a-zA-Z][_\-a-zA-Z0-9]*
+NUMBER=[0-9]+
 
-TEXT_SEGMENT=[^$/@\n]+
+TEXT_SEGMENT=[^$/@#\n]+
 COMMAND_PROP=[^:\ \t\r\n]*
 CODE_CONTENT=[^\n]+
 NEWLINE= \n | \r | \r\n
 
 COLON=:
+SHARP=#
 
 %{
     private boolean isCodeStart = false;
@@ -145,8 +147,10 @@ COLON=:
 }
 
 <SYSTEM_BLOCK> {
-  {SYSTEM_ID}        { yybegin(YYINITIAL); return SYSTEM_ID; }
-  [^]                  { return TokenType.BAD_CHARACTER; }
+  {SYSTEM_ID}          { return SYSTEM_ID; }
+  {COLON}              { return COLON; }
+  {NUMBER}             { return NUMBER; }
+  [^]                  { yybegin(YYINITIAL); yypushback(yylength()); }
 }
 
 <CODE_BLOCK> {

diff --git a/exts/devins-lang/src/grammar/DevInParser.bnf b/exts/devins-lang/src/grammar/DevInParser.bnf
@@ -31,6 +31,7 @@
     SYSTEM_ID          = "SYSTEM_ID"
     COLON              = "COLON"
     COMMAND_PROP       = "COMMAND_PROP"
+    SHARP              = "SHARP"
   ]
 }
 
@@ -40,7 +41,7 @@ used ::= (
     AGENT_START AGENT_ID
     | COMMAND_START COMMAND_ID (COLON COMMAND_PROP?)?
     | VARIABLE_START VARIABLE_ID
-    | SYSTEM_START SYSTEM_ID
+    | SYSTEM_START SYSTEM_ID COLON NUMBER
 )
 
 code ::=  CODE_BLOCK_START LANGUAGE_ID? NEWLINE? code_contents? CODE_BLOCK_END?

diff --git a/exts/devins-lang/src/test/kotlin/cc/unitmesh/devti/language/DevInParsingTest.kt b/exts/devins-lang/src/test/kotlin/cc/unitmesh/devti/language/DevInParsingTest.kt
@@ -39,4 +39,8 @@ class DevInParsingTest : ParsingTestCase("parser", "devin", DevInParserDefinitio
     fun testCommandAndSymbol() {
         doTest(true)
     }
+
+    fun testSystemCalling() {
+        doTest(true)
+    }
 }
diff --git a/exts/devins-lang/src/test/testData/parser/BasicTest.txt b/exts/devins-lang/src/test/testData/parser/BasicTest.txt
@@ -31,4 +31,9 @@ DevInFile
     PsiElement(DevInTokenType.COMMAND_PROP)('632372da')
   PsiElement(DevInTokenType.TEXT_SEGMENT)(' 从版本库中读取内容')
   PsiElement(DevInTokenType.NEWLINE)('\n')
-  PsiElement(DevInTokenType.TEXT_SEGMENT)('#system_id:51 传递参数到 story_id')
+  DevInUsedImpl(USED)
+    PsiElement(DevInTokenType.SYSTEM_START)('#')
+    PsiElement(DevInTokenType.SYSTEM_ID)('system_id')
+    PsiElement(DevInTokenType.COLON)(':')
+    PsiElement(DevInTokenType.NUMBER)('51')
+  PsiElement(DevInTokenType.TEXT_SEGMENT)(' 传递参数到 story_id')
diff --git a/exts/devins-lang/src/test/testData/parser/SystemCalling.devin b/exts/devins-lang/src/test/testData/parser/SystemCalling.devin
@@ -0,0 +1 @@
+#kanban:1
diff --git a/exts/devins-lang/src/test/testData/parser/SystemCalling.txt b/exts/devins-lang/src/test/testData/parser/SystemCalling.txt
@@ -0,0 +1,6 @@
+DevInFile
+  DevInUsedImpl(USED)
+    PsiElement(DevInTokenType.SYSTEM_START)('#')
+    PsiElement(DevInTokenType.SYSTEM_ID)('kanban')
+    PsiElement(DevInTokenType.COLON)(':')
+    PsiElement(DevInTokenType.NUMBER)('1')