Implemented basic macro expand functionality (#20579)

* Implemented level based macro expand functionality

- it can handle single macro call or expand whole function/proc/etc and it

- In addition, I have altered the parser to provide the endInfo for the node.
The usefulness of the `endInfo` is not limited to the `expandMacro`
functionality but also it is useful for `ideOutline` functionality and I have
altered the ideOutline functionality to use `endInfo`. Note `endInfo` most of
the time is lost during the AST transformation thus in `nimsuggest.nim` I am
using freshly parsed tree to get the location information.

* Make sure we stop expanding correctly

* Test CI

* Fix tv3_outline.nim

(cherry picked from commit 7031ea6)

compiler/ast.nim

@@ -777,6 +777,8 @@ type
       ident*: PIdent
     else:
       sons*: TNodeSeq
+    when defined(nimsuggest):
+      endInfo*: TLineInfo
 
   TStrTable* = object         # a table[PIdent] of PSym
     counter*: int
@@ -873,6 +875,8 @@ type
     typ*: PType
     name*: PIdent
     info*: TLineInfo
+    when defined(nimsuggest):
+      endInfo*: TLineInfo
     owner*: PSym
     flags*: TSymFlags
     ast*: PNode               # syntax tree of proc, iterator, etc.:
@@ -1657,6 +1661,8 @@ proc copyNode*(src: PNode): PNode =
   of nkIdent: result.ident = src.ident
   of nkStrLit..nkTripleStrLit: result.strVal = src.strVal
   else: discard
+  when defined(nimsuggest):
+    result.endInfo = src.endInfo
 
 template transitionNodeKindCommon(k: TNodeKind) =
   let obj {.inject.} = n[]
@@ -1705,6 +1711,8 @@ template copyNodeImpl(dst, src, processSonsStmt) =
   if src == nil: return
   dst = newNode(src.kind)
   dst.info = src.info
+  when defined(nimsuggest):
+    result.endInfo = src.endInfo
   dst.typ = src.typ
   dst.flags = src.flags * PersistentNodeFlags
   dst.comment = src.comment

compiler/lexer.nim

@@ -121,6 +121,8 @@ type
     cache*: IdentCache
     when defined(nimsuggest):
       previousToken: TLineInfo
+      tokenEnd*: TLineInfo
+      previousTokenEnd*: TLineInfo
     config*: ConfigRef
 
 proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} =
@@ -1216,6 +1218,10 @@ proc skip(L: var Lexer, tok: var Token) =
 proc rawGetTok*(L: var Lexer, tok: var Token) =
   template atTokenEnd() {.dirty.} =
     when defined(nimsuggest):
+      L.previousTokenEnd.line = L.tokenEnd.line
+      L.previousTokenEnd.col = L.tokenEnd.col
+      L.tokenEnd.line = tok.line.uint16
+      L.tokenEnd.col = getColNumber(L, L.bufpos).int16
       # we attach the cursor to the last *strong* token
       if tok.tokType notin weakTokens:
         L.previousToken.line = tok.line.uint16

compiler/options.nim

@@ -185,7 +185,7 @@ type
   IdeCmd* = enum
     ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideChkFile, ideMod,
     ideHighlight, ideOutline, ideKnown, ideMsg, ideProject, ideGlobalSymbols,
-    ideRecompile, ideChanged, ideType, ideDeclaration
+    ideRecompile, ideChanged, ideType, ideDeclaration, ideExpand
 
   Feature* = enum  ## experimental features; DO NOT RENAME THESE!
     implicitDeref,
@@ -264,6 +264,9 @@ type
     scope*, localUsages*, globalUsages*: int # more usages is better
     tokenLen*: int
     version*: int
+    endLine*: uint16
+    endCol*: int
+
   Suggestions* = seq[Suggest]
 
   ProfileInfo* = object
@@ -394,6 +397,11 @@ type
     nimMainPrefix*: string
     vmProfileData*: ProfileData
 
+    expandProgress*: bool
+    expandLevels*: int
+    expandNodeResult*: string
+    expandPosition*: TLineInfo
+
 proc parseNimVersion*(a: string): NimVer =
   # could be moved somewhere reusable
   if a.len > 0:
@@ -981,6 +989,9 @@ proc isDynlibOverride*(conf: ConfigRef; lib: string): bool =
   result = optDynlibOverrideAll in conf.globalOptions or
      conf.dllOverrides.hasKey(lib.canonDynlibName)
 
+proc expandDone*(conf: ConfigRef): bool =
+  result = conf.ideCmd == ideExpand and conf.expandLevels == 0 and conf.expandProgress
+
 proc parseIdeCmd*(s: string): IdeCmd =
   case s:
   of "sug": ideSug
@@ -1020,6 +1031,7 @@ proc `$`*(c: IdeCmd): string =
   of ideProject: "project"
   of ideGlobalSymbols: "globalSymbols"
   of ideDeclaration: "declaration"
+  of ideExpand: "expand"
   of ideRecompile: "recompile"
   of ideChanged: "changed"
   of ideType: "type"