closureiters: use case-statement-based dispatcher

Summary
=======

Use a case statement for the dispatcher generated by the `closureiters`
transformation, removing the need for both dedicated code generator
support and the `nkState` node kind. This is a significant step towards
closure iterator support the JavaScript and VM backend.

While neither officially supported nor documented, it was previously
possible to emit `nkGotoState` and `nkState` nodes from a macro, which
could be used for unsafe unstructured control-flow. With the removal of
the `nkState` node kind and `nkGotoState` being disallowed in AST
reaching semantic analysis, the aforementioned is no longer possible.

Details
=======

The AST generated by the `closureiters` transformation always looked
as follows:

```nim
while true:
  block :stateLoop: # the block's body is wrapped in a try-except, if
		    # necessary
    goto env.:state # goto the label corresponding to the run-time value
                    # of `:state`
    # declarations
    body
```

where `body` defines the code blocks associated with each state
value, like so:

```nim
state 0:
...
env.:state = 1
break :stateLoop
state 1:
...
state 2:
...
... # more labels (if used) follow
```

The control-flow described by the `goto` (`nkGotoState`) followed by
the labels (`nkState`) is equally representable by the use of a case
statement, which is what is now done after this commit.

To summarize the change:
- don't support `nkGotoState` nodes in AST reaching sem
- adjust `closureiters` to generate a case statement for the dispatcher
  and to not depend on `nkState`
- remove C code-generator support for `nkGotoState` and `nkState`
- remove the `nkState` node kind

Nodes with the `nkGotoState` kind now only exists *during* the
transformation, but never before or after.

compiler/ast/ast_types.nim

@@ -233,8 +233,8 @@ type
     nkPattern             ## a special pattern; used for matching
     nkHiddenTryStmt       ## a hidden try statement
     nkClosure             ## (prc, env)-pair (internally used for code gen)
-    nkGotoState           ## used for the state machine (for iterators)
-    nkState               ## give a label to a code section (for iterators)
+    nkGotoState           ## used only temporarily during closure iterator
+                          ## transformation
     nkFuncDef             ## a func
     nkTupleConstr         ## a tuple constructor
     nkError               ## erroneous AST node see `errorhandling`

compiler/ast/renderer.nim

@@ -1627,15 +1627,6 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
     initContext c
     putWithSpace g, tkSymbol, "goto"
     gsons(g, n, c)
-  of nkState:
-    var c: TContext
-    initContext c
-    putWithSpace g, tkSymbol, "state"
-    gsub(g, n[0], c)
-    putWithSpace(g, tkColon, ":")
-    indentNL(g)
-    gsons(g, n, c, 1)
-    dedent(g)
   of nkTypeClassTy:
     gTypeClassTy(g, n)
   of nkError:

compiler/backend/ccgexprs.nim

@@ -2967,9 +2967,6 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
           # by ensuring it's no inner proc (owner is a module).
           # Generate proc even if empty body, bugfix #11651.
           genProc(p.module, prc)
-  of nkState: genState(p, n)
-  of nkGotoState:
-    genGotoState(p, n)
   of nkMixinStmt, nkBindStmt: discard
   else:
     internalError(p.config, n.info, "expr(" & $n.kind & "); unknown node kind")

compiler/backend/ccgstmts.nim

@@ -155,15 +155,6 @@ template preserveBreakIdx(body: untyped): untyped =
   body
   p.breakIdx = oldBreakIdx
 
-proc genState(p: BProc, n: PNode) =
-  internalAssert p.config, n.len == 1
-  let n0 = n[0]
-  if n0.kind == nkIntLit:
-    let idx = n[0].intVal
-    linefmt(p, cpsStmts, "STATE$1: ;$n", [idx])
-  elif n0.kind == nkStrLit:
-    linefmt(p, cpsStmts, "$1: ;$n", [n0.strVal])
-
 proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
   # Called by return and break stmts.
   # Deals with issues faced when jumping out of try/except/finally stmts.
@@ -200,30 +191,6 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
     for i in countdown(howManyExcepts-1, 0):
       linefmt(p, cpsStmts, "#popCurrentException();$n", [])
 
-proc genGotoState(p: BProc, n: PNode) =
-  # we resist the temptation to translate it into duff's device as it later
-  # will be translated into computed gotos anyway for GCC at least:
-  # switch (x.state) {
-  #   case 0: goto STATE0;
-  # ...
-  var a: TLoc
-  initLocExpr(p, n[0], a)
-  lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)])
-  p.flags.incl beforeRetNeeded
-  lineF(p, cpsStmts, "case -1:$n", [])
-  blockLeaveActions(p,
-    howManyTrys    = p.nestedTryStmts.len,
-    howManyExcepts = p.inExceptBlockLen)
-  lineF(p, cpsStmts, " goto BeforeRet_;$n", [])
-  var statesCounter = lastOrd(p.config, n[0].typ)
-  if n.len >= 2 and n[1].kind == nkIntLit:
-    statesCounter = getInt(n[1])
-  let prefix = if n.len == 3 and n[2].kind == nkStrLit: n[2].strVal.rope
-               else: rope"STATE"
-  for i in 0i64..toInt64(statesCounter):
-    lineF(p, cpsStmts, "case $2: goto $1$2;$n", [prefix, rope(i)])
-  lineF(p, cpsStmts, "}$n", [])
-
 proc genBreakState(p: BProc, n: PNode, d: var TLoc) =
   ## Generates the code for the ``mFinished`` magic, which tests if a
   ## closure iterator is in the "finished" state (i.e. the internal

compiler/backend/cgen.nim

@@ -960,9 +960,6 @@ proc allPathsAsgnResult(n: PNode): InitResultEnum =
       if result == InitSkippable: result = Unknown
   of harmless:
     result = Unknown
-  of nkGotoState:
-    # give up for now.
-    result = InitRequired
   of nkSym:
     # some path reads from 'result' before it was written to!
     if n.sym.kind == skResult: result = InitRequired

compiler/backend/jsgen.nim

@@ -2640,10 +2640,6 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     if {sfExportc, sfCompilerProc} * s.flags == {sfExportc}:
       genSym(p, n[namePos], r)
       r.res = ""
-  of nkGotoState, nkState:
-    globalReport(p.config, n.info, BackendReport(
-      kind: rbackJsUnsupportedClosureIter))
-
   of nkPragmaBlock: gen(p, n.lastSon, r)
   else: internalError(p.config, n.info, "gen: unknown node type: " & $n.kind)
 

compiler/mir/mirgen.nim

@@ -1988,7 +1988,7 @@ proc gen(c: var TCtx, n: PNode) =
     if hasInteresting:
       c.stmts.add MirNode(kind: mnkPNode, node: n)
 
-  of nkGotoState, nkState, nkAsmStmt:
+  of nkAsmStmt:
     # these don't have a direct MIR counterpart
     c.stmts.add MirNode(kind: mnkPNode, node: n)
   of nkWhenStmt:

compiler/mir/mirtrees.nim

@@ -241,8 +241,8 @@ type
              ## If it appears as a statement, it is expected to not have any
              ## obsersvable effects
              ## XXX: eventually, everything that currently requires
-             ##      ``mnkPNode`` (for example, ``nkGotoState``, ``nkAsmStmt``,
-             ##      emit, etc.) should be expressable directly in the IR
+             ##      ``mnkPNode`` (for example, ``nkAsmStmt``, emit, etc.)
+             ##      should be expressable directly in the IR
 
   EffectKind* = enum
     ekMutate    ## the value in the location is mutated

compiler/sem/closureiters.nim

@@ -185,7 +185,7 @@ type
     unrollFinallySym: PSym # Indicates that we're unrolling finally states (either exception happened or premature return)
     curExcSym: PSym # Current exception
 
-    states: seq[PNode] # The resulting states. Every state is an nkState node.
+    states: seq[tuple[label: int, body: PNode]] # The resulting states
     blockLevel: int # Temp used to transform break and continue stmts
     stateLoopLabel: PSym # Label to break on, when jumping between states.
     exitStateIdx: int # index of the last state
@@ -266,9 +266,7 @@ proc newState(ctx: var Ctx, n, gotoOut: PNode): int =
   # Returns index of the newly created state
 
   result = ctx.states.len
-  let resLit = ctx.g.newIntLit(n.info, result)
-  let s = newTreeI(nkState, n.info): [resLit, n]
-  ctx.states.add(s)
+  ctx.states.add (result, n)
   ctx.exceptionTable.add(ctx.curExcHandlingState)
 
   if not gotoOut.isNil:
@@ -1128,7 +1126,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
     if label == -1:
       newLabel = ctx.exitStateIdx
     else:
-      let fs = skipStmtList(ctx, ctx.states[label][1])
+      let fs = skipStmtList(ctx, ctx.states[label].body)
       if fs.kind == nkGotoState:
         newLabel = fs[0].intVal.int
     if label == newLabel: break
@@ -1137,7 +1135,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
     if maxJumps == 0:
       assert(false, "Internal error")
 
-  result = ctx.states[stateIdx][0].intVal.int
+  result = ctx.states[stateIdx].label
 
 proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
   result = n
@@ -1255,7 +1253,6 @@ proc wrapIntoTryExcept(ctx: var Ctx, n: PNode): PNode {.inline.} =
 proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   # while true:
   #   block :stateLoop:
-  #     gotoState :state
   #     local vars decl (if needed)
   #     body # Might get wrapped in try-except
   let loopBody = newNodeI(nkStmtList, n.info)
@@ -1274,11 +1271,7 @@ proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   let blockStmt = newNodeI(nkBlockStmt, n.info)
   blockStmt.add(newSymNode(ctx.stateLoopLabel))
 
-  let gs = newNodeI(nkGotoState, n.info)
-  gs.add(ctx.newStateAccess())
-  gs.add(ctx.g.newIntLit(n.info, ctx.states.len - 1))
-
-  var blockBody = newTree(nkStmtList, gs, localVars, n)
+  var blockBody = newTree(nkStmtList, localVars, n)
   if ctx.hasExceptions:
     blockBody = ctx.wrapIntoTryExcept(blockBody)
 
@@ -1291,29 +1284,29 @@ proc deleteEmptyStates(ctx: var Ctx) =
 
   # Apply new state indexes and mark unused states with -1
   var iValid = 0
-  for i, s in ctx.states:
-    let body = skipStmtList(ctx, s[1])
-    if body.kind == nkGotoState and i != ctx.states.len - 1 and i != 0:
+  for i, s in ctx.states.mpairs:
+    let body = skipStmtList(ctx, s.body)
+    if body.kind == nkGotoState and i in 1..(ctx.states.len-1):
       # This is an empty state. Mark with -1.
-      s[0].intVal = -1
+      s.label = -1
     else:
-      s[0].intVal = iValid
+      s.label = iValid
       inc iValid
 
-  for i, s in ctx.states:
-    let body = skipStmtList(ctx, s[1])
+  for i, s in ctx.states.pairs:
+    let body = skipStmtList(ctx, s.body)
     if body.kind != nkGotoState or i == 0:
-      discard ctx.skipThroughEmptyStates(s)
+      discard ctx.skipThroughEmptyStates(s.body)
       let excHandlState = ctx.exceptionTable[i]
       if excHandlState < 0:
         ctx.exceptionTable[i] = -ctx.skipEmptyStates(-excHandlState)
       elif excHandlState != 0:
         ctx.exceptionTable[i] = ctx.skipEmptyStates(excHandlState)
 
-  var i = 0
+  # remove unused states except for the first (entry) and last one (exit)
+  var i = 1
   while i < ctx.states.len - 1:
-    let fs = skipStmtList(ctx, ctx.states[i][1])
-    if fs.kind == nkGotoState and i != 0:
+    if ctx.states[i].label == -1: # is it unused?
       ctx.states.delete(i)
       ctx.exceptionTable.delete(i)
     else:
@@ -1451,21 +1444,26 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n:
   # Optimize empty states away
   ctx.deleteEmptyStates()
 
-  # Make new body by concatenating the list of states
-  result = newNodeI(nkStmtList, n.info)
-  for s in ctx.states:
-    assert(s.len == 2)
-    let body = s[1]
-    s.sons.del(1)
-    result.add(s)
-    result.add(body)
-
-  result = ctx.transformStateAssignments(result)
-  result = ctx.wrapIntoStateLoop(result)
-
-  # echo "TRANSFORM TO STATES: "
-  # echo renderTree(result)
-
-  # echo "exception table:"
-  # for i, e in ctx.exceptionTable:
-  #   echo i, " -> ", e
+  # create the dispatcher:
+  #
+  #   case env.:state
+  #   of 0: ...
+  #   of 1: ...
+  #   ...
+  #   else: return
+  result = newNodeI(nkCaseStmt, n.info)
+  result.add(ctx.newStateAccess())
+  for s in ctx.states.items:
+    # transform the gotos into state assignments:
+    let body = ctx.transformStateAssignments(s.body)
+    # then add the block as a branch to the dispatcher:
+    result.add:
+      newTreeI(nkOfBranch, body.info):
+        [g.newIntLit(body.info, s.label), body]
+
+  # add the exit:
+  result.add:
+    newTreeI(nkElse, n.info):
+      newTreeI(nkReturnStmt, n.info, g.emptyNode)
+
+  result = ctx.wrapIntoStateLoop(result)

compiler/sem/semexprs.nim

@@ -3823,12 +3823,6 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     n[0] = semExpr(c, n[0])
     if not n[0].typ.isEmptyType and not implicitlyDiscardable(n[0]):
       localReport(c.config, n, reportSem rsemExpectedTypelessDeferBody)
-  of nkGotoState, nkState:
-    if n.len != 1 and n.len != 2:
-      semReportIllformedAst(c.config, n, "")
-
-    for i in 0..<n.len:
-      n[i] = semExpr(c, n[i])
   of nkMixinStmt: discard
   of nkBindStmt:
     if c.p != nil:

compiler/sem/semstmts.nim

@@ -3272,7 +3272,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
        sfNoReturn in x[0].sym.flags:
       for j in i + 1..<n.len:
         case n[j].kind
-        of nkPragma, nkCommentStmt, nkNilLit, nkEmpty, nkState:
+        of nkPragma, nkCommentStmt, nkNilLit, nkEmpty:
           discard
         else:
           localReport(c.config, n[j].info,

lib/core/macros.nim

@@ -22,7 +22,7 @@ import std/private/since
 # If you look for the implementation of the magic symbol
 # ``{.magic: "Foo".}``, search for `mFoo` and `opcFoo`.
 
-template skipEnumValue(define: untyped, predecessor: untyped): untyped =
+template skipEnumValue(define: untyped, predecessor: untyped; gap = 1): untyped =
   ## This template is used to keep the ordinal values of the ``TNodeKind``
   ## enum in sync with the ``NimNodeKind`` enum.
   ##
@@ -36,12 +36,14 @@ template skipEnumValue(define: untyped, predecessor: untyped): untyped =
   ## ``NimNodeKind`` are removed, the successor of the removed enum entry uses
   ## ``skipEnumValue`` to leave a gap in the case that `define`, which is used
   ## to indicate that the enum entry is not present in the compiler, is not
-  ## defined
+  ## defined.
+  ##
+  ## `gap` specifies the amount of enum fields to skip.
   when defined(define):
     ord(predecessor) + 1
   else:
-    # leave a gap where the removed node kind is located
-    ord(predecessor) + 2
+    # leave a gap where the removed node kinds are located
+    ord(predecessor) + gap + 1
 
 type
   NimNodeKind* = enum
@@ -106,8 +108,7 @@ type
     nnkHiddenTryStmt,
     nnkClosure,
     nnkGotoState,
-    nnkState,
-    nnkFuncDef = skipEnumValue(nimHasNkBreakStateNodeRemoved, nnkState)
+    nnkFuncDef = skipEnumValue(nimHasNkBreakStateNodeRemoved, nnkGotoState, 2),
     nnkTupleConstr,
     nnkError,  ## erroneous AST node
     nnkNimNodeLit