.gitattributes

@@ -0,0 +1,6 @@
+# Avoids changelog conflicts by assuming additions-only, which is by far the common case.
+# In the rare case where branch b1 rebases against branch b2 and both branches
+# modified the same changelog entry, you'll end up with that changelog entry
+# duplicated, which is easily identifiable and fixable.
+/changelog.md merge=union
+

azure-pipelines.yml

@@ -30,8 +30,12 @@ jobs:
       Windows_amd64:
         vmImage: 'windows-2019'
         CPU: amd64
-      Windows_amd64_pkg:
-        vmImage: 'windows-2019'
+      Linux_amd64_pkg:
+        vmImage: 'ubuntu-16.04'
+        CPU: amd64
+        NIM_TEST_PACKAGES: true
+      OSX_amd64_pkg:
+        vmImage: 'macOS-10.15'
         CPU: amd64
         NIM_TEST_PACKAGES: true
 
@@ -53,8 +57,8 @@ jobs:
 
     - task: NodeTool@0
       inputs:
-        versionSpec: '8.x'
-      displayName: 'Install node.js 8.x'
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
 
     - bash: |
         sudo apt-fast update -qq
@@ -107,7 +111,7 @@ jobs:
       displayName: 'Install dependencies (i386 Linux)'
       condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
 
-    - bash: brew install boehmgc make sfml
+    - bash: brew install boehmgc make sfml gtk+3
       displayName: 'Install dependencies (OSX)'
       condition: eq(variables['Agent.OS'], 'Darwin')
 

changelog.md

@@ -4,9 +4,109 @@
 
 ## Standard library additions and changes
 
+  For `net` and `nativesockets`, an `inheritable` flag has been added to all
+  `proc`s that create sockets, allowing the user to control whether the
+  resulting socket is inheritable. This flag is provided to ease the writing of
+  multi-process servers, where sockets inheritance is desired.
+
+  For a transistion period, define `nimInheritHandles` to enable file handle
+  inheritance by default. This flag does **not** affect the `selectors` module
+  due to the differing semantics between operating systems.
+
+  `system.setInheritable` and `nativesockets.setInheritable` is also introduced
+  for setting file handle or socket inheritance. Not all platform have these
+  `proc`s defined.
+
+- The file descriptors created for internal bookkeeping by `ioselector_kqueue`
+  and `ioselector_epoll` will no longer be leaked to child processes.
+
+- `strutils.formatFloat` with `precision = 0` has been restored to the version
+  1 behaviour that produces a trailing dot, e.g. `formatFloat(3.14159, precision = 0)`
+  is now `3.`, not `3`.
+- `critbits` adds `commonPrefixLen`.
+
+- `relativePath(rel, abs)` and `relativePath(abs, rel)` used to silently give wrong results
+  (see #13222); instead they now use `getCurrentDir` to resolve those cases,
+  and this can now throw in edge cases where `getCurrentDir` throws.
+  `relativePath` also now works for js with `-d:nodejs`.
+
+- JavaScript and NimScript standard library changes: `streams.StringStream` is
+  now supported in JavaScript, with the limitation that any buffer `pointer`s
+  used must be castable to `ptr string`, any incompatible pointer type will not
+  work. The `lexbase` and `streams` modules used to fail to compile on
+  NimScript due to a bug, but this has been fixed.
+
+  The following modules now compile on both JS and NimScript: `parsecsv`,
+  `parsecfg`, `parsesql`, `xmlparser`, `htmlparser` and `ropes`. Additionally
+  supported for JS is `cstrutils.startsWith` and `cstrutils.endsWith`, for
+  NimScript: `json`, `parsejson`, `strtabs` and `unidecode`. 
+
+- Added `streams.readStr` and `streams.peekStr` overloads to
+  accept an existing string to modify, which avoids memory
+  allocations, similar to `streams.readLine` (#13857).
+
+- Added high-level `asyncnet.sendTo` and `asyncnet.recvFrom`. UDP functionality.
+
+- `paramCount` & `paramStr` are now defined in os.nim instead of nimscript.nim for nimscript/nimble.
+- `dollars.$` now works for unsigned ints with `nim js`
+
+- Improvements to the `bitops` module, including bitslices, non-mutating versions
+  of the original masking functions, `mask`/`masked`, and varargs support for
+  `bitand`, `bitor`, and `bitxor`.
+
+- `sugar.=>` and `sugar.->` changes: Previously `(x, y: int)` was transformed
+  into `(x: auto, y: int)`, it now becomes `(x: int, y: int)` in consistency
+  with regular proc definitions (although you cannot use semicolons).
+
+  Pragmas and using a name are now allowed on the lefthand side of `=>`. Here
+  is an aggregate example of these changes:
+  ```nim
+  import sugar
+
+  foo(x, y: int) {.noSideEffect.} => x + y
+
+  # is transformed into
+
+  proc foo(x: int, y: int): auto {.noSideEffect.} = x + y
+  ```
+- The fields of `times.DateTime` are now private, and are accessed with getters and deprecated setters.
+
+- The `times` module now handles the default value for `DateTime` more consistently. Most procs raise an assertion error when given
+  an uninitialized `DateTime`, the exceptions are `==` and `$` (which returns `"Uninitialized DateTime"`). The proc `times.isInitialized`
+  has been added which can be used to check if a `DateTime` has been initialized.
+
+- Fix a bug where calling `close` on io streams in osproc.startProcess was a noop and led to
+  hangs if a process had both reads from stdin and writes (eg to stdout).
 
 ## Language changes
+- In newruntime it is now allowed to assign discriminator field without restrictions as long as case object doesn't have custom destructor. Discriminator value doesn't have to be a constant either. If you have custom destructor for case object and you do want to freely assign discriminator fields, it is recommended to refactor object into 2 objects like this: 
+  ```nim
+  type
+    MyObj = object
+      case kind: bool
+        of true: y: ptr UncheckedArray[float]
+        of false: z: seq[int]
+
+  proc `=destroy`(x: MyObj) =
+    if x.kind and x.y != nil:
+      deallocShared(x.y)
+      x.y = nil
+  ```
+  Refactor into:
+  ```nim
+  type
+    MySubObj = object
+      val: ptr UncheckedArray[float]
+    MyObj = object
+      case kind: bool
+      of true: y: MySubObj
+      of false: z: seq[int]
 
+  proc `=destroy`(x: MySubObj) =
+    if x.val != nil:
+      deallocShared(x.val)
+      x.val = nil
+  ```
 
 ## Compiler changes
 

compiler/ast.nim

@@ -292,6 +292,7 @@ type
     sfInjectDestructors # whether the proc needs the 'injectdestructors' transformation
     sfNeverRaises     # proc can never raise an exception, not even OverflowError
                       # or out-of-memory
+    sfUsedInFinallyOrExcept  # symbol is used inside an 'except' or 'finally'
 
   TSymFlags* = set[TSymFlag]
 

compiler/astalgo.nim

@@ -1048,3 +1048,7 @@ proc listSymbolNames*(symbols: openArray[PSym]): string =
       result.add ", "
     result.add sym.name.s
 
+proc isDiscriminantField*(n: PNode): bool =
+  if n.kind == nkCheckedFieldExpr: sfDiscriminant in n[0][1].sym.flags
+  elif n.kind == nkDotExpr: sfDiscriminant in n[1].sym.flags
+  else: false

compiler/ccgcalls.nim

@@ -21,11 +21,39 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
     # we have to be *very* conservative:
     result = canRaiseConservative(n)
 
-proc leftAppearsOnRightSide(le, ri: PNode): bool =
+proc preventNrvo(p: BProc; le, ri: PNode): bool =
+  proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
+    var n = le
+    while true:
+      # do NOT follow nkHiddenDeref here!
+      case n.kind
+      of nkSym:
+        # we don't own the location so it escapes:
+        if n.sym.owner != p.prc:
+          return true
+        elif inTryStmt and sfUsedInFinallyOrExcept in n.sym.flags:
+          # it is also an observable store if the location is used
+          # in 'except' or 'finally'
+          return true
+        return false
+      of nkDotExpr, nkBracketExpr, nkObjUpConv, nkObjDownConv,
+          nkCheckedFieldExpr:
+        n = n[0]
+      of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+        n = n[1]
+      else:
+        # cannot analyse the location; assume the worst
+        return true
+
   if le != nil:
     for i in 1..<ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
+    # we use the weaker 'canRaise' here in order to prevent too many
+    # annoying warnings, see #14514
+    if canRaise(ri[0]) and
+        locationEscapes(p, le, p.nestedTryStmts.len > 0):
+      message(p.config, le.info, warnObservableStores, $le)
 
 proc hasNoInit(call: PNode): bool {.inline.} =
   result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
@@ -51,7 +79,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
     if isInvalidReturnType(p.config, typ[0]):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
@@ -113,7 +141,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
 
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
-proc openArrayLoc(p: BProc, n: PNode): Rope =
+proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
   var a: TLoc
 
   var q = skipConv(n)
@@ -149,8 +177,11 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     of tyOpenArray, tyVarargs, tyUncheckedArray, tyCString:
       result = "($4*)($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-          not compileToCpp(p.module):
+      let atyp = skipTypes(a.t, abstractInst)
+      if formalType.skipTypes(abstractInst).kind == tyVar and atyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if atyp.kind == tyVar and not compileToCpp(p.module):
         result = "($5*)(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
       else:
         result = "($5*)$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
@@ -162,8 +193,11 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     of tyOpenArray, tyVarargs:
       result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-            not compileToCpp(p.module):
+      let ntyp = skipTypes(n.typ, abstractInst)
+      if formalType.skipTypes(abstractInst).kind == tyVar and ntyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if ntyp.kind == tyVar and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
         result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
@@ -194,7 +228,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
     result = genArgStringToCString(p, n)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
     var n = if n.kind != nkHiddenAddr: n else: n[0]
-    result = openArrayLoc(p, n)
+    result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
     result = addrLoc(p.config, a)
@@ -286,7 +320,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     if isInvalidReturnType(p.config, typ[0]):
       if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
           getTemp(p, typ[0], d, needsInit=true)

compiler/ccgexprs.nim

@@ -1194,9 +1194,11 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) =
 proc genReset(p: BProc, n: PNode) =
   var a: TLoc
   initLocExpr(p, n[1], a)
-  linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
-          [addrLoc(p.config, a),
-          genTypeInfo(p.module, skipTypes(a.t, {tyVar}), n.info)])
+  specializeReset(p, a)
+  when false:
+    linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
+            [addrLoc(p.config, a),
+            genTypeInfo(p.module, skipTypes(a.t, {tyVar}), n.info)])
 
 proc genDefault(p: BProc; n: PNode; d: var TLoc) =
   if d.k == locNone: getTemp(p, n.typ, d, needsInit=true)
@@ -2156,7 +2158,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     const opr: array[mInc..mDec, string] = ["+=", "-="]
     const fun64: array[mInc..mDec, string] = ["nimAddInt64", "nimSubInt64"]
     const fun: array[mInc..mDec, string] = ["nimAddInt","nimSubInt"]
-    let underlying = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent, tyRange})
+    let underlying = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent, tyRange, tyDistinct})
     if optOverflowCheck notin p.options or underlying.kind in {tyUInt..tyUInt64}:
       binaryStmt(p, e, d, opr[op])
     else:
@@ -2787,6 +2789,21 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope =
   else:
     globalError(p.config, info, "cannot create null element for: " & $t.kind)
 
+proc caseObjDefaultBranch(obj: PNode; branch: Int128): int =
+  for i in 1 ..< obj.len:
+    for j in 0 .. obj[i].len - 2:
+      if obj[i][j].kind == nkRange:
+        let x = getOrdValue(obj[i][j][0])
+        let y = getOrdValue(obj[i][j][1])
+        if branch >= x and branch <= y:
+          return i
+      elif getOrdValue(obj[i][j]) == branch:
+        return i
+    if obj[i].len == 1:
+      # else branch
+      return i
+  assert(false, "unreachable")
+
 proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
                      result: var Rope; count: var int;
                      isConst: bool, info: TLineInfo) =
@@ -2809,31 +2826,14 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
           branch = getOrdValue(constOrNil[i])
           break
 
-    var selectedBranch = -1
-    block branchSelection:
-      for i in 1 ..< obj.len:
-        for j in 0 .. obj[i].len - 2:
-          if obj[i][j].kind == nkRange:
-              let x = getOrdValue(obj[i][j][0])
-              let y = getOrdValue(obj[i][j][1])
-              if branch >= x and branch <= y:
-                selectedBranch = i
-                break branchSelection
-          elif getOrdValue(obj[i][j]) == branch:
-            selectedBranch = i
-            break branchSelection
-        if obj[i].len == 1:
-          # else branch
-          selectedBranch = i
-    assert(selectedBranch >= 1)
-
+    let selectedBranch = caseObjDefaultBranch(obj, branch)
     result.add "{"
     var countB = 0
     let b = lastSon(obj[selectedBranch])
     # designated initilization is the only way to init non first element of unions
     # branches are allowed to have no members (b.len == 0), in this case they don't need initializer
-    if  b.kind == nkRecList and b.len > 0:
-      result.add "._" &  mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
+    if b.kind == nkRecList and b.len > 0:
+      result.add "._" & mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
       getNullValueAux(p, t,  b, constOrNil, result, countB, isConst, info)
       result.add "}"
     elif b.kind == nkSym: