Showing with 3,355 additions and 779 deletions.
  1. +69 −10 changelog.md
  2. +161 −13 compiler/ast.nim
  3. +16 −9 compiler/ccgcalls.nim
  4. +15 −11 compiler/ccgexprs.nim
  5. +9 −1 compiler/cgen.nim
  6. +3 −1 compiler/condsyms.nim
  7. +3 −3 compiler/docgen.nim
  8. +4 −0 compiler/ic/ic.nim
  9. +14 −10 compiler/injectdestructors.nim
  10. +11 −5 compiler/liftdestructors.nim
  11. +2 −4 compiler/lineinfos.nim
  12. +60 −30 compiler/lookups.nim
  13. +1 −0 compiler/nim.cfg
  14. +2 −0 compiler/options.nim
  15. +9 −2 compiler/parampatterns.nim
  16. +1 −1 compiler/pragmas.nim
  17. +3 −1 compiler/renderer.nim
  18. +15 −10 compiler/reorder.nim
  19. +1 −60 compiler/sem.nim
  20. +2 −0 compiler/semdata.nim
  21. +208 −104 compiler/semexprs.nim
  22. +8 −3 compiler/semfold.nim
  23. +24 −9 compiler/semgnrc.nim
  24. +8 −3 compiler/seminst.nim
  25. +2 −0 compiler/semmagic.nim
  26. +1 −1 compiler/sempass2.nim
  27. +134 −14 compiler/semstmts.nim
  28. +41 −22 compiler/semtempl.nim
  29. +15 −11 compiler/semtypes.nim
  30. +5 −1 compiler/semtypinst.nim
  31. +191 −146 compiler/sigmatch.nim
  32. +13 −7 compiler/types.nim
  33. +15 −2 compiler/varpartitions.nim
  34. +1 −1 compiler/vm.nim
  35. +4 −4 config/nim.cfg
  36. +6 −6 doc/manual.md
  37. +94 −0 doc/manual_experimental.md
  38. +2 −2 koch.nim
  39. +3 −1 lib/core/macros.nim
  40. +11 −3 lib/nimbase.h
  41. +40 −25 lib/pure/concurrency/atomics.nim
  42. +1 −1 lib/pure/fenv.nim
  43. +1 −1 lib/pure/math.nim
  44. +3 −16 lib/pure/md5.nim
  45. +30 −36 lib/pure/memfiles.nim
  46. +2 −0 lib/pure/mimetypes.nim
  47. +3 −0 lib/pure/os.nim
  48. +2 −0 lib/pure/osproc.nim
  49. +2 −2 lib/pure/pegs.nim
  50. +4 −3 lib/pure/sugar.nim
  51. +10 −8 lib/std/private/ossymlinks.nim
  52. +1 −1 lib/std/syncio.nim
  53. +20 −4 lib/std/tasks.nim
  54. +14 −9 lib/system.nim
  55. +155 −57 lib/system/alloc.nim
  56. +1 −1 lib/system/compilation.nim
  57. +54 −0 tests/alloc/tmembug.nim
  58. +58 −0 tests/alloc/tmembug2.nim
  59. +3 −1 tests/arc/thard_alignment.nim
  60. +16 −0 tests/arc/topenarray.nim
  61. +1 −1 tests/arc/topt_no_cursor.nim
  62. +25 −0 tests/ccgbugs/t23796.nim
  63. +1 −2 tests/ccgbugs/tforward_decl_only.nim
  64. +2 −1 tests/config.nims
  65. +12 −0 tests/controlflow/tunreachable2.nim
  66. +6 −9 tests/converter/tconverter_unique_ptr.nim
  67. +4 −6 tests/destructor/const_smart_ptr.nim
  68. +2 −0 tests/destructor/tatomicptrs.nim
  69. +29 −2 tests/destructor/tnonvardestructor.nim
  70. +12 −40 tests/destructor/tv2_cast.nim
  71. +12 −0 tests/discard/t23677.nim
  72. +39 −0 tests/discard/tdiscardable.nim
  73. +19 −0 tests/discard/tfinallyerrmsg.nim
  74. +16 −0 tests/effects/thooks.nim
  75. +2 −2 tests/enum/tambiguousoverloads.nim
  76. +1 −0 tests/enum/tpure_enums_conflict.nim
  77. +2 −2 tests/errmsgs/t22097.nim
  78. +2 −1 tests/errmsgs/t8064.nim
  79. +1 −1 tests/errmsgs/tinconsistentgensym.nim
  80. +37 −0 tests/errmsgs/tuntypedoverload.nim
  81. +12 −2 tests/gc/closureleak.nim
  82. +34 −0 tests/generics/mopensymimport1.nim
  83. +16 −0 tests/generics/mopensymimport2.nim
  84. +8 −0 tests/generics/t22826.nim
  85. +14 −0 tests/generics/t23790.nim
  86. +91 −0 tests/generics/t23853.nim
  87. +8 −0 tests/generics/taliashijack.nim
  88. +60 −1 tests/generics/tmacroinjectedsym.nim
  89. +10 −1 tests/generics/tmacroinjectedsymwarning.nim
  90. +5 −0 tests/generics/topensymimport.nim
  91. +37 −0 tests/int/t1.nim
  92. +16 −0 tests/int/twrongexplicitvarconv.nim
  93. +9 −0 tests/int/twrongvarconv.nim
  94. +2 −0 tests/lookups/issue_23032/deep_scope.nim
  95. +13 −0 tests/lookups/t23032.nim
  96. +37 −0 tests/lookups/t23749.nim
  97. +1 −1 tests/lookups/tambiguousemit.nim
  98. +2 −2 tests/lookups/tambprocvar.nim
  99. +1 −1 tests/lookups/tambsym3.nim
  100. +22 −0 tests/lookups/tenumlocalsym.nim
  101. +19 −0 tests/macros/t23032_1.nim
  102. +20 −0 tests/macros/t23032_2.nim
  103. +5 −2 tests/macros/tgetimpl.nim
  104. +9 −0 tests/metatype/tmetatype_issues.nim
  105. +28 −0 tests/metatype/twrong_same_type.nim
  106. +4 −0 tests/modules/treorder.nim
  107. +19 −0 tests/openarray/tuncheckedarray.nim
  108. +18 −0 tests/osproc/twaitforexit.nim
  109. +145 −0 tests/overload/mvaruintconv.nim
  110. +2 −2 tests/overload/t8829.nim
  111. +24 −21 tests/overload/toverload_issues.nim
  112. +60 −0 tests/overload/toverload_various.nim
  113. +207 −0 tests/overload/tvaruintconv.nim
  114. +1 −0 tests/pragmas/monoff1.nim
  115. +8 −0 tests/pragmas/tonoff1.nim
  116. +14 −0 tests/pragmas/tonoff2.nim
  117. +6 −0 tests/pragmas/tpush.nim
  118. +2 −2 tests/range/tcompiletime_range_checks.nim
  119. +10 −0 tests/specialops/tsetter.nim
  120. +11 −0 tests/stdlib/tparseutils.nim
  121. +36 −0 tests/stdlib/ttasks.nim
  122. +6 −0 tests/system/tsystem_misc.nim
  123. +19 −0 tests/template/t24112.nim
  124. +12 −0 tests/template/tinnerouterproc.nim
  125. +209 −0 tests/template/topensym.nim
  126. +39 −0 tests/template/topensymoverride.nim
  127. +60 −0 tests/template/topensymwarning.nim
  128. +51 −0 tests/threads/tmembug.nim
  129. +14 −0 tests/tuples/t18125_1.nim
  130. +20 −0 tests/tuples/t18125_2.nim
  131. +4 −0 tests/types/tisopr.nim
  132. +6 −0 tests/types/ttopdowninference.nim
  133. +36 −0 tests/vm/tgenericcompiletimeproc.nim
  134. +1 −1 tests/vm/tvmmisc.nim
79 changes: 69 additions & 10 deletions changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@

## Changes affecting backward compatibility

- `-d:nimStrictDelete` becomes the default. An index error is produced when the index passed to `system.delete` was out of bounds. Use `-d:nimAuditDelete` to mimic the old behavior for backwards compatibility.
- The default user-agent in `std/httpclient` has been changed to `Nim-httpclient/<version>` instead of `Nim httpclient/<version>` which was incorrect according to the HTTP spec.
- With `-d:nimPreviewNonVarDestructor`, non-var destructors become the default.

## Standard library additions and changes

Expand All @@ -27,28 +30,37 @@ slots when enlarging a sequence.



- An experimental option `genericsOpenSym` has been added to allow captured
symbols in generic routine bodies to be replaced by symbols injected locally
by templates/macros at instantiation time. `bind` may be used to keep the
captured symbols over the injected ones regardless of enabling the option.
- The experimental option `--experimental:openSym` has been added to allow
captured symbols in generic routine and template bodies respectively to be
replaced by symbols injected locally by templates/macros at instantiation
time. `bind` may be used to keep the captured symbols over the injected ones
regardless of enabling the option, but other methods like renaming the
captured symbols should be used instead so that the code is not affected by
context changes.

Since this change may affect runtime behavior, the experimental switch
`genericsOpenSym` needs to be enabled, and a warning is given in the case
where an injected symbol would replace a captured symbol not bound by `bind`
and the experimental switch isn't enabled.
`openSym` needs to be enabled; and a warning is given in the case where an
injected symbol would replace a captured symbol not bound by `bind` and
the experimental switch isn't enabled.

```nim
const value = "captured"
template foo(x: int, body: untyped) =
template foo(x: int, body: untyped): untyped =
let value {.inject.} = "injected"
body
proc old[T](): string =
foo(123):
return value # warning: a new `value` has been injected, use `bind` or turn on `experimental:genericsOpenSym`
return value # warning: a new `value` has been injected, use `bind` or turn on `experimental:openSym`
echo old[int]() # "captured"
{.experimental: "genericsOpenSym".}
template oldTempl(): string =
block:
foo(123):
value # warning: a new `value` has been injected, use `bind` or turn on `experimental:openSym`
echo oldTempl() # "captured"
{.experimental: "openSym".}
proc bar[T](): string =
foo(123):
Expand All @@ -60,6 +72,53 @@ slots when enlarging a sequence.
foo(123):
return value
assert baz[int]() == "captured"
template barTempl(): string =
block:
foo(123):
value
assert barTempl() == "injected" # previously it would be "captured"
template bazTempl(): string =
bind value
block:
foo(123):
value
assert bazTempl() == "captured"
```

This option also generates a new node kind `nnkOpenSym` which contains
exactly 1 `nnkSym` node. In the future this might be merged with a slightly
modified `nnkOpenSymChoice` node but macros that want to support the
experimental feature should still handle `nnkOpenSym`, as the node kind would
simply not be generated as opposed to being removed.

Another experimental switch `genericsOpenSym` exists that enables this behavior
at instantiation time, meaning templates etc can enable it specifically when
they are being called. However this does not generate `nnkOpenSym` nodes
(unless the other switch is enabled) and so doesn't reflect the regular
behavior of the switch.

```nim
const value = "captured"
template foo(x: int, body: untyped): untyped =
let value {.inject.} = "injected"
{.push experimental: "genericsOpenSym".}
body
{.pop.}
proc bar[T](): string =
foo(123):
return value
echo bar[int]() # "injected"
template barTempl(): string =
block:
var res: string
foo(123):
res = value
res
assert barTempl() == "injected"
```

## Compiler changes
Expand Down
174 changes: 161 additions & 13 deletions compiler/ast.nim
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ type
nkModuleRef # for .rod file support: A (moduleId, itemId) pair
nkReplayAction # for .rod file support: A replay action
nkNilRodNode # for .rod file support: a 'nil' PNode
nkOpenSym # container for captured sym that can be overriden by local symbols

TNodeKinds* = set[TNodeKind]

Expand Down Expand Up @@ -517,7 +518,9 @@ type
nfFirstWrite # this node is a first write
nfHasComment # node has a comment
nfSkipFieldChecking # node skips field visable checking
nfOpenSym # node is a captured sym but can be overriden by local symbols
nfDisabledOpenSym # temporary: node should be nkOpenSym but cannot
# because openSym experimental switch is disabled
# gives warning instead

TNodeFlags* = set[TNodeFlag]
TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 47)
Expand Down Expand Up @@ -1091,7 +1094,7 @@ const
nfFromTemplate, nfDefaultRefsParam,
nfExecuteOnReload, nfLastRead,
nfFirstWrite, nfSkipFieldChecking,
nfOpenSym}
nfDisabledOpenSym}
namePos* = 0
patternPos* = 1 # empty except for term rewriting macros
genericParamsPos* = 2
Expand All @@ -1107,7 +1110,7 @@ const
nkCallKinds* = {nkCall, nkInfix, nkPrefix, nkPostfix,
nkCommand, nkCallStrLit, nkHiddenCallConv}
nkIdentKinds* = {nkIdent, nkSym, nkAccQuoted, nkOpenSymChoice,
nkClosedSymChoice}
nkClosedSymChoice, nkOpenSym}

nkPragmaCallKinds* = {nkExprColonExpr, nkCall, nkCallStrLit}
nkLiterals* = {nkCharLit..nkTripleStrLit}
Expand Down Expand Up @@ -1135,6 +1138,7 @@ proc getPIdent*(a: PNode): PIdent {.inline.} =
of nkSym: a.sym.name
of nkIdent: a.ident
of nkOpenSymChoice, nkClosedSymChoice: a.sons[0].sym.name
of nkOpenSym: getPIdent(a.sons[0])
else: nil

const
Expand Down Expand Up @@ -1254,8 +1258,11 @@ proc getDeclPragma*(n: PNode): PNode =

proc extractPragma*(s: PSym): PNode =
## gets the pragma node of routine/type/var/let/const symbol `s`
if s.kind in routineKinds:
result = s.ast[pragmasPos]
if s.kind in routineKinds: # bug #24167
if s.ast[pragmasPos] != nil and s.ast[pragmasPos].kind != nkEmpty:
result = s.ast[pragmasPos]
else:
result = nil
elif s.kind in {skType, skVar, skLet, skConst}:
if s.ast != nil and s.ast.len > 0:
if s.ast[0].kind == nkPragmaExpr and s.ast[0].len > 1:
Expand Down Expand Up @@ -1453,6 +1460,9 @@ proc newSymNode*(sym: PSym, info: TLineInfo): PNode =
result.typ = sym.typ
result.info = info

proc newOpenSym*(n: PNode): PNode {.inline.} =
result = newTreeI(nkOpenSym, n.info, n)

proc newIntNode*(kind: TNodeKind, intVal: BiggestInt): PNode =
result = newNode(kind)
result.intVal = intVal
Expand All @@ -1461,15 +1471,50 @@ proc newIntNode*(kind: TNodeKind, intVal: Int128): PNode =
result = newNode(kind)
result.intVal = castToInt64(intVal)

proc lastSon*(n: Indexable): Indexable = n.sons[^1]
proc lastSon*(n: Indexable): Indexable {.inline.} = n.sons[^1]
template setLastSon*(n: PNode, s: PNode) = n.sons[^1] = s

template firstSon*(n: PNode): PNode = n.sons[0]
template secondSon*(n: PNode): PNode = n.sons[1]

template hasSon*(n: PNode): bool = n.len > 0
template has2Sons*(n: PNode): bool = n.len > 1

proc replaceFirstSon*(n, newson: PNode) {.inline.} =
n.sons[0] = newson

proc replaceSon*(n: PNode; i: int; newson: PNode) {.inline.} =
n.sons[i] = newson

proc last*(n: PType): PType {.inline.} = n.sons[^1]

proc elementType*(n: PType): PType {.inline.} = n.sons[^1]
proc skipModifier*(n: PType): PType {.inline.} = n.sons[^1]

proc indexType*(n: PType): PType {.inline.} = n.sons[0]
proc baseClass*(n: PType): PType {.inline.} = n.sons[0]

proc base*(t: PType): PType {.inline.} =
result = t.sons[0]

proc returnType*(n: PType): PType {.inline.} = n.sons[0]
proc setReturnType*(n, r: PType) {.inline.} = n.sons[0] = r
proc setIndexType*(n, idx: PType) {.inline.} = n.sons[0] = idx

proc firstParamType*(n: PType): PType {.inline.} = n.sons[1]
proc firstGenericParam*(n: PType): PType {.inline.} = n.sons[1]

proc typeBodyImpl*(n: PType): PType {.inline.} = n.sons[^1]

proc genericHead*(n: PType): PType {.inline.} = n.sons[0]

proc skipTypes*(t: PType, kinds: TTypeKinds): PType =
## Used throughout the compiler code to test whether a type tree contains or
## doesn't contain a specific type/types - it is often the case that only the
## last child nodes of a type tree need to be searched. This is a really hot
## path within the compiler!
result = t
while result.kind in kinds: result = lastSon(result)
while result.kind in kinds: result = last(result)

proc newIntTypeNode*(intVal: BiggestInt, typ: PType): PNode =
let kind = skipTypes(typ, abstractVarRange).kind
Expand Down Expand Up @@ -1528,6 +1573,107 @@ proc `$`*(s: PSym): string =
else:
result = "<nil>"

when false:
iterator items*(t: PType): PType =
for i in 0..<t.sons.len: yield t.sons[i]

iterator pairs*(n: PType): tuple[i: int, n: PType] =
for i in 0..<n.sons.len: yield (i, n.sons[i])

when true:
proc len*(n: PType): int {.inline.} =
result = n.sons.len

proc sameTupleLengths*(a, b: PType): bool {.inline.} =
result = a.sons.len == b.sons.len

iterator tupleTypePairs*(a, b: PType): (int, PType, PType) =
for i in 0 ..< a.sons.len:
yield (i, a.sons[i], b.sons[i])

iterator underspecifiedPairs*(a, b: PType; start = 0; without = 0): (PType, PType) =
# XXX Figure out with what typekinds this is called.
for i in start ..< min(a.sons.len, b.sons.len) + without:
yield (a.sons[i], b.sons[i])

proc signatureLen*(t: PType): int {.inline.} =
result = t.sons.len

proc paramsLen*(t: PType): int {.inline.} =
result = t.sons.len - 1

proc genericParamsLen*(t: PType): int {.inline.} =
assert t.kind == tyGenericInst
result = t.sons.len - 2 # without 'head' and 'body'

proc genericInvocationParamsLen*(t: PType): int {.inline.} =
assert t.kind == tyGenericInvocation
result = t.sons.len - 1 # without 'head'

proc kidsLen*(t: PType): int {.inline.} =
result = t.sons.len

proc genericParamHasConstraints*(t: PType): bool {.inline.} = t.sons.len > 0

proc hasElementType*(t: PType): bool {.inline.} = t.sons.len > 0
proc isEmptyTupleType*(t: PType): bool {.inline.} = t.sons.len == 0
proc isSingletonTupleType*(t: PType): bool {.inline.} = t.sons.len == 1

proc genericConstraint*(t: PType): PType {.inline.} = t.sons[0]

iterator genericInstParams*(t: PType): (bool, PType) =
for i in 1..<t.sons.len-1:
yield (i!=1, t.sons[i])

iterator genericInstParamPairs*(a, b: PType): (int, PType, PType) =
for i in 1..<min(a.sons.len, b.sons.len)-1:
yield (i-1, a.sons[i], b.sons[i])

iterator genericInvocationParams*(t: PType): (bool, PType) =
for i in 1..<t.sons.len:
yield (i!=1, t.sons[i])

iterator genericInvocationAndBodyElements*(a, b: PType): (PType, PType) =
for i in 1..<a.sons.len:
yield (a.sons[i], b.sons[i-1])

iterator genericInvocationParamPairs*(a, b: PType): (bool, PType, PType) =
for i in 1..<a.sons.len:
if i >= b.sons.len:
yield (false, nil, nil)
else:
yield (true, a.sons[i], b.sons[i])

iterator genericBodyParams*(t: PType): (int, PType) =
for i in 0..<t.sons.len-1:
yield (i, t.sons[i])

iterator userTypeClassInstParams*(t: PType): (bool, PType) =
for i in 1..<t.sons.len-1:
yield (i!=1, t.sons[i])

iterator ikids*(t: PType): (int, PType) =
for i in 0..<t.sons.len: yield (i, t.sons[i])

const
FirstParamAt* = 1
FirstGenericParamAt* = 1

iterator paramTypes*(t: PType): (int, PType) =
for i in FirstParamAt..<t.sons.len: yield (i, t.sons[i])

iterator paramTypePairs*(a, b: PType): (PType, PType) =
for i in FirstParamAt..<a.sons.len: yield (a.sons[i], b.sons[i])

template paramTypeToNodeIndex*(x: int): int = x

iterator kids*(t: PType): PType =
for i in 0..<t.sons.len: yield t.sons[i]

iterator signature*(t: PType): PType =
# yields return type + parameter types
for i in 0..<t.sons.len: yield t.sons[i]

proc newType*(kind: TTypeKind, id: ItemId; owner: PSym): PType =
result = PType(kind: kind, owner: owner, size: defaultSize,
align: defaultAlignment, itemId: id,
Expand Down Expand Up @@ -1638,7 +1784,7 @@ proc skipTypes*(t: PType, kinds: TTypeKinds; maxIters: int): PType =
result = t
var i = maxIters
while result.kind in kinds:
result = lastSon(result)
result = last(result)
dec i
if i == 0: return nil

Expand All @@ -1647,7 +1793,7 @@ proc skipTypesOrNil*(t: PType, kinds: TTypeKinds): PType =
result = t
while result != nil and result.kind in kinds:
if result.len == 0: return nil
result = lastSon(result)
result = last(result)

proc isGCedMem*(t: PType): bool {.inline.} =
result = t.kind in {tyString, tyRef, tySequence} or
Expand Down Expand Up @@ -1916,13 +2062,15 @@ proc skipGenericOwner*(s: PSym): PSym =
## Generic instantiations are owned by their originating generic
## symbol. This proc skips such owners and goes straight to the owner
## of the generic itself (the module or the enclosing proc).
result = if s.kind in skProcKinds and sfFromGeneric in s.flags:
result = if s.kind == skModule:
s
elif s.kind in skProcKinds and sfFromGeneric in s.flags and s.owner.kind != skModule:
s.owner.owner
else:
s.owner

proc originatingModule*(s: PSym): PSym =
result = s.owner
result = s
while result.kind != skModule: result = result.owner

proc isRoutine*(s: PSym): bool {.inline.} =
Expand Down Expand Up @@ -1984,7 +2132,7 @@ proc toObject*(typ: PType): PType =
## cases should be a ``tyObject``).
## Otherwise ``typ`` is simply returned as-is.
let t = typ.skipTypes({tyAlias, tyGenericInst})
if t.kind == tyRef: t.lastSon
if t.kind == tyRef: t.elementType
else: typ

proc toObjectFromRefPtrGeneric*(typ: PType): PType =
Expand All @@ -2001,7 +2149,7 @@ proc toObjectFromRefPtrGeneric*(typ: PType): PType =
result = typ
while true:
case result.kind
of tyGenericBody: result = result.lastSon
of tyGenericBody: result = result.last
of tyRef, tyPtr, tyGenericInst, tyGenericInvocation, tyAlias: result = result[0]
# automatic dereferencing is deep, refs #18298.
else: break
Expand Down
Loading