From d524095392ac70cf1d47bda378baffcaa3ed86b8 Mon Sep 17 00:00:00 2001 From: Phodal Huang Date: Sun, 31 Mar 2024 15:20:49 +0800 Subject: [PATCH] feat(devins-lang): add support for line info in commands This commit introduces the concept of line info in DevIns commands, enhancing the language's capabilities. The line info feature allows for more precise control over AI agent actions by specifying the exact line and column numbers within a code block. This enhancement is achieved by adding a new token type 'LINE_INFO' to the DevIns lexer and parser, and corresponding highlighting in the syntax highlighter. --- exts/devins-lang/README.md | 19 ++++++++ exts/devins-lang/src/grammar/DevInLexer.flex | 43 ++++++++++++++++++- exts/devins-lang/src/grammar/DevInParser.bnf | 3 +- .../highlight/DevInSyntaxHighlighter.kt | 2 + .../src/test/testData/parser/AutoCommand.txt | 4 +- .../test/testData/parser/CommandAndSymbol.txt | 4 +- 6 files changed, 70 insertions(+), 5 deletions(-) diff --git a/exts/devins-lang/README.md b/exts/devins-lang/README.md index 72e10eed9a..cb73a6020e 100644 --- a/exts/devins-lang/README.md +++ b/exts/devins-lang/README.md @@ -24,3 +24,22 @@ interaction with LLM, we later renamed it to DevIns. This change was prompted by AI company, to avoid confusion. The name DevIns, derived from DevInstruction, succinctly captures the essence of the language in guiding the AI Agent's actions. +## AutoDev/Netscape Navigator + +Return Code: + +```devin +/symbol:cc.unitmesh.devti.language.lexer.DevInsLexer.parse +``` + +Hyperlink to the IDE: + +```devin +/link:cc.unitmesh.devti.language.lexer.DevInsLexer.parse +``` + +L1C2 = Line 1 Column 2 + +```devin +/file:cpp/src/main/kotlin/cc/unitmesh/cpp/context/CppFileContextBuilder.kt#L1C2-L23 +``` diff --git a/exts/devins-lang/src/grammar/DevInLexer.flex b/exts/devins-lang/src/grammar/DevInLexer.flex index 98483210f8..01985498a9 100644 --- a/exts/devins-lang/src/grammar/DevInLexer.flex +++ b/exts/devins-lang/src/grammar/DevInLexer.flex @@ -4,6 +4,8 @@ import com.intellij.lexer.FlexLexer; import com.intellij.psi.tree.IElementType; import static cc.unitmesh.devti.language.psi.DevInTypes.*; import com.intellij.psi.TokenType; +import java.util.regex.Matcher; +import java.util.regex.Pattern; %% @@ -33,6 +35,7 @@ import com.intellij.psi.TokenType; %s CODE_BLOCK %s COMMENT_BLOCK +%s LINE_BLOCK %s LANG_ID @@ -44,8 +47,11 @@ COMMAND_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]* LANGUAGE_ID=[a-zA-Z][_\-a-zA-Z0-9 .]* SYSTEM_ID=[a-zA-Z][_\-a-zA-Z0-9]* NUMBER=[0-9]+ - TEXT_SEGMENT=[^$/@#\n]+ + +// READ LINE FORMAT: L2C2-L0C100 or L1-L1 +LINE_INFO=L[0-9]+(C[0-9]+)?(-L[0-9]+(C[0-9]+)?)? +//LINE_INFO=[L][0-9]+[L][0-9]+ COMMAND_PROP=[^\ \t\r\n]* CODE_CONTENT=[^\n]+ COMMENTS=\[ ([^\]]+)? \] [^\t\r\n]* @@ -118,6 +124,33 @@ SHARP=# return TEXT_SEGMENT; } } + + private IElementType command_value() { + String text = yytext().toString().trim(); + String [] split = text.split("#"); + + if (split.length == 1) { + return COMMAND_PROP; + } + + // split by # if it is a line info + String last = split[split.length - 1]; + Pattern compile = Pattern.compile("L\\d+(C\\d+)?(-L\\d+(C\\d+)?)?"); + Matcher matcher = compile.matcher(last); + if (matcher.matches()) { + // before # is command prop, after # is line info + int number = last.length() + "#".length(); + if (number > 0) { + yypushback(number); + yybegin(LINE_BLOCK); + return COMMAND_PROP; + } else { + return COMMAND_PROP; + } + } + + return COMMAND_PROP; + } %} %% @@ -154,11 +187,17 @@ SHARP=# } { - {COMMAND_PROP} { return COMMAND_PROP; } + {COMMAND_PROP} { return command_value(); } " " { yypushback(1); yybegin(YYINITIAL); } [^] { yypushback(1); yybegin(YYINITIAL); } } + { + {LINE_INFO} { return LINE_INFO; } + {SHARP} { return SHARP; } + [^] { yypushback(yylength()); yybegin(COMMAND_VALUE_BLOCK); } +} + { {AGENT_ID} { yybegin(YYINITIAL); return AGENT_ID; } [^] { return TokenType.BAD_CHARACTER; } diff --git a/exts/devins-lang/src/grammar/DevInParser.bnf b/exts/devins-lang/src/grammar/DevInParser.bnf index 3455b39dfe..7869367a77 100644 --- a/exts/devins-lang/src/grammar/DevInParser.bnf +++ b/exts/devins-lang/src/grammar/DevInParser.bnf @@ -32,6 +32,7 @@ COLON = "COLON" COMMAND_PROP = "COMMAND_PROP" SHARP = "SHARP" + LINE_INFO = "LINE_INFO" ] } @@ -39,7 +40,7 @@ DevInsFile ::= (used | code | TEXT_SEGMENT | NEWLINE | COMMENTS)* used ::= ( AGENT_START AGENT_ID - | COMMAND_START COMMAND_ID (COLON COMMAND_PROP?)? + | COMMAND_START COMMAND_ID (COLON COMMAND_PROP (SHARP LINE_INFO)?)? | VARIABLE_START VARIABLE_ID | SYSTEM_START SYSTEM_ID COLON NUMBER ) diff --git a/exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/highlight/DevInSyntaxHighlighter.kt b/exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/highlight/DevInSyntaxHighlighter.kt index c074c2e63e..4f2423a5e8 100644 --- a/exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/highlight/DevInSyntaxHighlighter.kt +++ b/exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/highlight/DevInSyntaxHighlighter.kt @@ -32,6 +32,8 @@ class DevInSyntaxHighlighter : SyntaxHighlighter { ATTRIBUTES[DevInTypes.COMMAND_ID] = DefaultLanguageHighlighterColors.KEYWORD ATTRIBUTES[DevInTypes.COMMAND_PROP] = DefaultLanguageHighlighterColors.STRING + ATTRIBUTES[DevInTypes.LINE_INFO] = DefaultLanguageHighlighterColors.NUMBER + ATTRIBUTES[DevInTypes.CODE_BLOCK_START] = DefaultLanguageHighlighterColors.KEYWORD ATTRIBUTES[DevInTypes.CODE_BLOCK_END] = DefaultLanguageHighlighterColors.KEYWORD ATTRIBUTES[DevInTypes.LANGUAGE_ID] = DefaultLanguageHighlighterColors.CONSTANT diff --git a/exts/devins-lang/src/test/testData/parser/AutoCommand.txt b/exts/devins-lang/src/test/testData/parser/AutoCommand.txt index a63daf20bb..3c3d2df933 100644 --- a/exts/devins-lang/src/test/testData/parser/AutoCommand.txt +++ b/exts/devins-lang/src/test/testData/parser/AutoCommand.txt @@ -3,4 +3,6 @@ DevInFile PsiElement(DevInTokenType.COMMAND_START)('/') PsiElement(DevInTokenType.COMMAND_ID)('write') PsiElement(DevInTokenType.COLON)(':') - PsiElement(DevInTokenType.COMMAND_PROP)('Sample.file#L1-L12') \ No newline at end of file + PsiElement(DevInTokenType.COMMAND_PROP)('Sample.file') + PsiElement(DevInTokenType.SHARP)('#') + PsiElement(DevInTokenType.LINE_INFO)('L1-L12') \ No newline at end of file diff --git a/exts/devins-lang/src/test/testData/parser/CommandAndSymbol.txt b/exts/devins-lang/src/test/testData/parser/CommandAndSymbol.txt index f41d5972da..37d0d515c5 100644 --- a/exts/devins-lang/src/test/testData/parser/CommandAndSymbol.txt +++ b/exts/devins-lang/src/test/testData/parser/CommandAndSymbol.txt @@ -25,4 +25,6 @@ DevInFile PsiElement(DevInTokenType.COMMAND_START)('/') PsiElement(DevInTokenType.COMMAND_ID)('write') PsiElement(DevInTokenType.COLON)(':') - PsiElement(DevInTokenType.COMMAND_PROP)('presentation/VirtualFilePresentation.java#L1-L12') \ No newline at end of file + PsiElement(DevInTokenType.COMMAND_PROP)('presentation/VirtualFilePresentation.java') + PsiElement(DevInTokenType.SHARP)('#') + PsiElement(DevInTokenType.LINE_INFO)('L1-L12') \ No newline at end of file