feat(devins-lang): add support for line info in commands

This commit introduces the concept of line info in DevIns commands, enhancing the language's capabilities. The line info feature allows for more precise control over AI agent actions by specifying the exact line and column numbers within a code block. This enhancement is achieved by adding a new token type 'LINE_INFO' to the DevIns lexer and parser, and corresponding highlighting in the syntax highlighter.

exts/devins-lang/README.md

@@ -24,3 +24,22 @@ interaction with LLM, we later renamed it to DevIns. This change was prompted by
 AI company, to avoid confusion. The name DevIns, derived from DevInstruction, succinctly captures the essence of the
 language in guiding the AI Agent's actions.
 
+## AutoDev/Netscape Navigator
+
+Return Code:
+
+```devin
+/symbol:cc.unitmesh.devti.language.lexer.DevInsLexer.parse
+```
+
+Hyperlink to the IDE:
+
+```devin
+/link:cc.unitmesh.devti.language.lexer.DevInsLexer.parse
+```
+
+L1C2 = Line 1 Column 2
+
+```devin
+/file:cpp/src/main/kotlin/cc/unitmesh/cpp/context/CppFileContextBuilder.kt#L1C2-L23
+```

exts/devins-lang/src/grammar/DevInLexer.flex

@@ -4,6 +4,8 @@ import com.intellij.lexer.FlexLexer;
 import com.intellij.psi.tree.IElementType;
 import static cc.unitmesh.devti.language.psi.DevInTypes.*;
 import com.intellij.psi.TokenType;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 %%
 
@@ -33,6 +35,7 @@ import com.intellij.psi.TokenType;
 
 %s CODE_BLOCK
 %s COMMENT_BLOCK
+%s LINE_BLOCK
 
 %s LANG_ID
 
@@ -44,8 +47,11 @@ COMMAND_ID=[a-zA-Z0-9][_\-a-zA-Z0-9]*
 LANGUAGE_ID=[a-zA-Z][_\-a-zA-Z0-9 .]*
 SYSTEM_ID=[a-zA-Z][_\-a-zA-Z0-9]*
 NUMBER=[0-9]+
-
 TEXT_SEGMENT=[^$/@#\n]+
+
+// READ LINE FORMAT: L2C2-L0C100 or L1-L1
+LINE_INFO=L[0-9]+(C[0-9]+)?(-L[0-9]+(C[0-9]+)?)?
+//LINE_INFO=[L][0-9]+[L][0-9]+
 COMMAND_PROP=[^\ \t\r\n]*
 CODE_CONTENT=[^\n]+
 COMMENTS=\[ ([^\]]+)? \] [^\t\r\n]*
@@ -118,6 +124,33 @@ SHARP=#
             return TEXT_SEGMENT;
         }
     }
+
+    private IElementType command_value() {
+        String text = yytext().toString().trim();
+        String [] split = text.split("#");
+
+        if (split.length == 1) {
+            return COMMAND_PROP;
+        }
+
+        // split by # if it is a line info
+        String last = split[split.length - 1];
+        Pattern compile = Pattern.compile("L\\d+(C\\d+)?(-L\\d+(C\\d+)?)?");
+        Matcher matcher = compile.matcher(last);
+        if (matcher.matches()) {
+            // before # is command prop, after # is line info
+            int number = last.length() + "#".length();
+            if (number > 0) {
+                yypushback(number);
+                yybegin(LINE_BLOCK);
+                return COMMAND_PROP;
+            } else {
+                return COMMAND_PROP;
+            }
+        }
+
+        return COMMAND_PROP;
+    }
 %}
 
 %%
@@ -154,11 +187,17 @@ SHARP=#
 }
 
 <COMMAND_VALUE_BLOCK> {
-  {COMMAND_PROP}          { return COMMAND_PROP;  }
+  {COMMAND_PROP}          { return command_value();  }
   " "                     { yypushback(1); yybegin(YYINITIAL); }
   [^]                     { yypushback(1); yybegin(YYINITIAL); }
 }
 
+<LINE_BLOCK> {
+  {LINE_INFO}             { return LINE_INFO; }
+  {SHARP}                 { return SHARP; }
+  [^]                     { yypushback(yylength()); yybegin(COMMAND_VALUE_BLOCK); }
+}
+
 <AGENT_BLOCK> {
   {AGENT_ID}           { yybegin(YYINITIAL); return AGENT_ID; }
   [^]                  { return TokenType.BAD_CHARACTER; }

exts/devins-lang/src/grammar/DevInParser.bnf

@@ -32,14 +32,15 @@
     COLON              = "COLON"
     COMMAND_PROP       = "COMMAND_PROP"
     SHARP              = "SHARP"
+    LINE_INFO          = "LINE_INFO"
   ]
 }
 
 DevInsFile ::= (used | code | TEXT_SEGMENT | NEWLINE | COMMENTS)*
 
 used ::= (
     AGENT_START AGENT_ID
-    | COMMAND_START COMMAND_ID (COLON COMMAND_PROP?)?
+    | COMMAND_START COMMAND_ID (COLON COMMAND_PROP (SHARP LINE_INFO)?)?
     | VARIABLE_START VARIABLE_ID
     | SYSTEM_START SYSTEM_ID COLON NUMBER
 )

exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/highlight/DevInSyntaxHighlighter.kt

@@ -32,6 +32,8 @@ class DevInSyntaxHighlighter : SyntaxHighlighter {
             ATTRIBUTES[DevInTypes.COMMAND_ID] = DefaultLanguageHighlighterColors.KEYWORD
             ATTRIBUTES[DevInTypes.COMMAND_PROP] = DefaultLanguageHighlighterColors.STRING
 
+            ATTRIBUTES[DevInTypes.LINE_INFO] = DefaultLanguageHighlighterColors.NUMBER
+
             ATTRIBUTES[DevInTypes.CODE_BLOCK_START] = DefaultLanguageHighlighterColors.KEYWORD
             ATTRIBUTES[DevInTypes.CODE_BLOCK_END] = DefaultLanguageHighlighterColors.KEYWORD
             ATTRIBUTES[DevInTypes.LANGUAGE_ID] = DefaultLanguageHighlighterColors.CONSTANT

exts/devins-lang/src/test/testData/parser/AutoCommand.txt

@@ -3,4 +3,6 @@ DevInFile
     PsiElement(DevInTokenType.COMMAND_START)('/')
     PsiElement(DevInTokenType.COMMAND_ID)('write')
     PsiElement(DevInTokenType.COLON)(':')
-    PsiElement(DevInTokenType.COMMAND_PROP)('Sample.file#L1-L12')
+    PsiElement(DevInTokenType.COMMAND_PROP)('Sample.file')
+    PsiElement(DevInTokenType.SHARP)('#')
+    PsiElement(DevInTokenType.LINE_INFO)('L1-L12')

exts/devins-lang/src/test/testData/parser/CommandAndSymbol.txt

@@ -25,4 +25,6 @@ DevInFile
     PsiElement(DevInTokenType.COMMAND_START)('/')
     PsiElement(DevInTokenType.COMMAND_ID)('write')
     PsiElement(DevInTokenType.COLON)(':')
-    PsiElement(DevInTokenType.COMMAND_PROP)('presentation/VirtualFilePresentation.java#L1-L12')
+    PsiElement(DevInTokenType.COMMAND_PROP)('presentation/VirtualFilePresentation.java')
+    PsiElement(DevInTokenType.SHARP)('#')
+    PsiElement(DevInTokenType.LINE_INFO)('L1-L12')