azure-pipelines.yml

@@ -42,9 +42,10 @@ steps:
     condition: eq(variables['Agent.OS'], 'Windows_NT')
 
   - checkout: self
+    fetchDepth: 1
 
-  - bash: git clone --depth 1 https://github.com/nim-lang/csources.git
-    displayName: 'Checkout csources'
+  - bash: git clone --depth 1 https://github.com/nim-lang/csources
+    displayName: 'Checkout Nim csources'
 
   - task: NodeTool@0
     inputs:
@@ -62,19 +63,13 @@ steps:
   - bash: |
       sudo dpkg --add-architecture i386
 
-      # Downgrade llvm, libgcc and libstdc++:
+      # Downgrade llvm:
       # - llvm has to be downgraded to have 32bit version installed for sfml.
-      # - libgcc and libstdc++ have to be downgraded as an optimization to
-      #   prevent the use of the toolchain ppa, which has a terrible download
-      #   speed.
       cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
-      Package: libllvm6.0 libgcc1 libstdc++6
+      Package: libllvm6.0
       Pin: origin "azure.archive.ubuntu.com"
       Pin-Priority: 1001
 
-      Package: *
-      Pin: release o=LP-PPA-ubuntu-toolchain-r-test
-      Pin-Priority: 100
       EOF
 
       sudo apt-fast update -qq
@@ -128,8 +123,19 @@ steps:
       make -v
     displayName: 'System information'
 
+  - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+    displayName: 'Get csources version'
+
+  - task: Cache@2
+    inputs:
+      key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+      path: csources/bin
+    displayName: 'Restore built csources'
+
+
   - bash: |
       ncpu=
+      ext=
       case '$(Agent.OS)' in
       'Linux')
         ncpu=$(nproc)
@@ -139,19 +145,24 @@ steps:
         ;;
       'Windows_NT')
         ncpu=$NUMBER_OF_PROCESSORS
+        ext=.exe
         ;;
       esac
       [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-      make -C csources -j $ncpu CC=gcc ucpu=$(CPU)
-    displayName: 'Build csources'
+      if [[ -x csources/bin/nim$ext ]]; then
+        echo "Found cached compiler, skipping build"
+      else
+        make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+      fi
+      cp csources/bin/nim$ext bin
+    displayName: 'Build 1-stage compiler from csources'
 
   - bash: nim c koch
     displayName: 'Build koch'
 
     # set result to omit the "bash exited with error code '1'" message
-  - bash: |
-      ./koch runCI || echo '##vso[task.complete result=Failed]'
+  - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
     displayName: 'Run CI'
     env:
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)

compiler/ccgexprs.nim

@@ -1735,9 +1735,9 @@ template binaryExprIn(p: BProc, e: PNode, a, b, d: var TLoc, frmt: string) =
 
 proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) =
   case int(getSize(p.config, skipTypes(e.sons[1].typ, abstractVar)))
-  of 1: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&7U)))!=0)")
-  of 2: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&15U)))!=0)")
-  of 4: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&31U)))!=0)")
+  of 1: binaryExprIn(p, e, a, b, d, "(($1 &((NU8)1<<((NU)($2)&7U)))!=0)")
+  of 2: binaryExprIn(p, e, a, b, d, "(($1 &((NU16)1<<((NU)($2)&15U)))!=0)")
+  of 4: binaryExprIn(p, e, a, b, d, "(($1 &((NU32)1<<((NU)($2)&31U)))!=0)")
   of 8: binaryExprIn(p, e, a, b, d, "(($1 &((NU64)1<<((NU)($2)&63U)))!=0)")
   else: binaryExprIn(p, e, a, b, d, "(($1[(NU)($2)>>3] &(1U<<((NU)($2)&7U)))!=0)")
 

compiler/semfields.nim

@@ -23,7 +23,7 @@ proc instFieldLoopBody(c: TFieldInstCtx, n: PNode, forLoop: PNode): PNode =
     result = newNode(nkEmpty)
     return
   case n.kind
-  of nkEmpty..pred(nkIdent), succ(nkSym)..nkNilLit: result = n
+  of nkEmpty..pred(nkIdent), succ(nkSym)..nkNilLit: result = copyNode(n)
   of nkIdent, nkSym:
     result = n
     let ident = considerQuotedIdent(c.c, n)
@@ -53,8 +53,7 @@ proc instFieldLoopBody(c: TFieldInstCtx, n: PNode, forLoop: PNode): PNode =
     if n.kind == nkContinueStmt:
       localError(c.c.config, n.info,
                  "'continue' not supported in a 'fields' loop")
-    result = copyNode(n)
-    newSons(result, len(n))
+    result = shallowCopy(n)
     for i in 0 ..< len(n):
       result.sons[i] = instFieldLoopBody(c, n.sons[i], forLoop)
 

compiler/sigmatch.nim

@@ -1089,7 +1089,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
       let x = typeRel(c, f, branch, flags + {trDontBind})
       if x == isNone: return isNone
       if x < result: result = x
-    return
+    return result
 
   of tyAnd:
     # XXX: deal with the current dual meaning of tyGenericParam
@@ -1423,7 +1423,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
           let ff = rootf.sons[i]
           let aa = roota.sons[i]
           let res = typeRel(c, ff, aa, nextFlags)
-          if res != isEqual: result = isGeneric
+          if res != isNone and res != isEqual: result = isGeneric
           if res notin {isEqual, isGeneric}:
             if trNoCovariance notin flags and ff.kind == aa.kind:
               let paramFlags = rootf.base.sons[i-1].flags

compiler/sizealignoffsetimpl.nim

@@ -138,12 +138,13 @@ proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, packed: bool, a
       accum.offset  = szUnknownSize
       accum.maxAlign = szUnknownSize
     else:
-      # the union neds to be aligned first, before the offsets can be assigned
+      # the union needs to be aligned first, before the offsets can be assigned
       accum.align(maxChildAlign)
       let accumRoot = accum # copy, because each branch should start af the same offset
       for i in 1 ..< len(n):
-        var branchAccum = accumRoot
+        var branchAccum = OffsetAccum(offset: accumRoot.offset, maxAlign: 1)
         computeObjectOffsetsFoldFunction(conf, n.sons[i].lastSon, packed, branchAccum)
+        discard finish(branchAccum)
         accum.mergeBranch(branchAccum)
   of nkRecList:
     for i, child in n.sons:
@@ -171,9 +172,10 @@ proc computeUnionObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode; accum: var
     localError(conf, n.info, "Illegal use of ``case`` in union type.")
   of nkRecList:
     let accumRoot = accum # copy, because each branch should start af the same offset
-    for i, child in n.sons:
-      var branchAccum = accumRoot
+    for child in n.sons:
+      var branchAccum = OffsetAccum(offset: accumRoot.offset, maxAlign: 1)
       computeUnionObjectOffsetsFoldFunction(conf, child, branchAccum)
+      discard finish(branchAccum)
       accum.mergeBranch(branchAccum)
   of nkSym:
     var size = szUnknownSize

compiler/transf.nim

@@ -1104,15 +1104,15 @@ proc liftDeferAux(n: PNode) =
         if n.sons[i].kind == nkDefer:
           let deferPart = newNodeI(nkFinally, n.sons[i].info)
           deferPart.add n.sons[i].sons[0]
-          var tryStmt = newNodeI(nkTryStmt, n.sons[i].info)
-          var body = newNodeI(n.kind, n.sons[i].info)
+          var tryStmt = newNodeIT(nkTryStmt, n.sons[i].info, n.typ)
+          var body = newNodeIT(n.kind, n.sons[i].info, n.typ)
           if i < last:
             body.sons = n.sons[(i+1)..last]
           tryStmt.addSon(body)
           tryStmt.addSon(deferPart)
           n.sons[i] = tryStmt
           n.sons.setLen(i+1)
-          n.typ = n.sons[i].typ
+          n.typ = tryStmt.typ
           goOn = true
           break
   for i in 0..n.safeLen-1:

compiler/vmgen.nim

@@ -247,6 +247,8 @@ proc freeTemp(c: PCtx; r: TRegister) =
 proc getTempRange(cc: PCtx; n: int; kind: TSlotKind): TRegister =
   # if register pressure is high, we re-use more aggressively:
   let c = cc.prc
+  # we could also customize via the following (with proper caching in ConfigRef):
+  # let highRegisterPressure = cc.config.getConfigVar("vm.highRegisterPressure", "40").parseInt
   if c.maxSlots >= HighRegisterPressure or c.maxSlots+n >= high(TRegister):
     for i in 0 .. c.maxSlots-n:
       if not c.slots[i].inUse:
@@ -1533,14 +1535,14 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
     let tmp = c.genx(ri)
     c.preventFalseAlias(le[0], opcWrObj, objR, idx, tmp)
     c.freeTemp(tmp)
-    c.freeTemp(idx)
+    # c.freeTemp(idx) # BUGFIX, see nkDotExpr
     c.freeTemp(objR)
   of nkDotExpr:
     let dest = c.genx(le.sons[0], {gfNode})
     let idx = genField(c, le.sons[1])
     let tmp = c.genx(ri)
     c.preventFalseAlias(le, opcWrObj, dest, idx, tmp)
-    c.freeTemp(idx)
+    # c.freeTemp(idx) # BUGFIX: idx is an immediate (field position), not a register
     c.freeTemp(tmp)
     c.freeTemp(dest)
   of nkDerefExpr, nkHiddenDeref:

koch.nim

@@ -10,7 +10,7 @@
 #
 
 const
-  NimbleStableCommit = "4007b2a778429a978e12307bf13a038029b4c4d9" # master
+  NimbleStableCommit = "8f7af860c5ce9634af880a7081c6435e1f2a5148" # master
 
 when defined(gcc) and defined(windows):
   when defined(x86):
@@ -27,6 +27,7 @@ import
   os, strutils, parseopt, osproc, streams
 
 import tools / kochdocs
+import tools / deps
 
 const VersionAsString = system.NimVersion
 
@@ -115,52 +116,25 @@ proc copyExe(source, dest: string) =
 
 const
   compileNimInst = "tools/niminst/niminst"
+  distDir = "dist"
 
 proc csource(args: string) =
   nimexec(("cc $1 -r $3 --var:version=$2 --var:mingw=none csource " &
            "--main:compiler/nim.nim compiler/installer.ini $1") %
        [args, VersionAsString, compileNimInst])
 
 proc bundleC2nim(args: string) =
-  if not dirExists("dist/c2nim/.git"):
-    exec("git clone https://github.com/nim-lang/c2nim.git dist/c2nim")
+  cloneDependency(distDir, "https://github.com/nim-lang/c2nim.git")
   nimCompile("dist/c2nim/c2nim",
              options = "--noNimblePath --path:. " & args)
 
 proc bundleNimbleExe(latest: bool, args: string) =
-  if not dirExists("dist/nimble/.git"):
-    exec("git clone https://github.com/nim-lang/nimble.git dist/nimble")
-  if not latest:
-    withDir("dist/nimble"):
-      exec("git fetch")
-      exec("git checkout " & NimbleStableCommit)
+  let commit = if latest: "HEAD" else: NimbleStableCommit
+  cloneDependency(distDir, "https://github.com/nim-lang/nimble.git",
+                  commit = commit, allowBundled = true)
   # installer.ini expects it under $nim/bin
   nimCompile("dist/nimble/src/nimble.nim",
-             options = "-d:release --nilseqs:on " & args)
-
-proc buildNimble(latest: bool, args: string) =
-  # if koch is used for a tar.xz, build the dist/nimble we shipped
-  # with the tarball:
-  var installDir = "dist/nimble"
-  if not latest and dirExists(installDir) and not dirExists("dist/nimble/.git"):
-    discard "don't do the git dance"
-  else:
-    if not dirExists("dist/nimble/.git"):
-      if dirExists(installDir):
-        var id = 0
-        while dirExists("dist/nimble" & $id):
-          inc id
-        installDir = "dist/nimble" & $id
-      exec("git clone https://github.com/nim-lang/nimble.git " & installDir)
-    withDir(installDir):
-      if latest:
-        exec("git checkout -f master")
-        exec("git pull")
-      else:
-        exec("git fetch")
-        exec("git checkout " & NimbleStableCommit)
-  nimCompile(installDir / "src/nimble.nim",
-             options = "--noNimblePath --nilseqs:on -d:release " & args)
+             options = "-d:release --noNimblePath --nilseqs:on " & args)
 
 proc bundleNimsuggest(args: string) =
   nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim",
@@ -318,10 +292,10 @@ proc boot(args: string) =
     # in order to use less memory, we split the build into two steps:
     # --compileOnly produces a $project.json file and does not run GCC/Clang.
     # jsonbuild then uses the $project.json file to build the Nim binary.
-    exec "$# $# $# $# --nimcache:$# --compileOnly compiler" / "nim.nim" %
-      [nimi, bootOptions, extraOption, args, smartNimcache]
-    exec "$# jsonscript $# --nimcache:$# compiler" / "nim.nim" %
-      [nimi, args, smartNimcache]
+    exec "$# $# $# --nimcache:$# $# --compileOnly compiler" / "nim.nim" %
+      [nimi, bootOptions, extraOption, smartNimcache, args]
+    exec "$# jsonscript --nimcache:$# $# compiler" / "nim.nim" %
+      [nimi, smartNimcache, args]
 
     if sameFileContent(output, i.thVersion):
       copyExe(output, finalDest)
@@ -479,7 +453,7 @@ proc runCI(cmd: string) =
   when defined(posix): # appveyor (on windows) didn't run this
     kochExecFold("Boot", "boot")
   # boot without -d:nimHasLibFFI to make sure this still works
-  kochExecFold("Boot in release mode", "boot -d:release -d:danger")
+  kochExecFold("Boot in release mode", "boot -d:release")
 
   ## build nimble early on to enable remainder to depend on it if needed
   kochExecFold("Build Nimble", "nimble")
@@ -491,7 +465,7 @@ proc runCI(cmd: string) =
   if getEnv("NIM_TEST_PACKAGES", "false") == "true":
     execFold("Test selected Nimble packages", "nim c -r testament/testament cat nimble-packages")
   else:
-    buildTools() # altenatively, kochExec "tools --toolsNoNimble"
+    buildTools()
 
     ## run tests
     execFold("Test nimscript", "nim e tests/test_nimscript.nims")
@@ -628,13 +602,13 @@ when isMainModule:
       of "temp": temp(op.cmdLineRest)
       of "xtemp": xtemp(op.cmdLineRest)
       of "wintools": bundleWinTools(op.cmdLineRest)
-      of "nimble": buildNimble(latest, op.cmdLineRest)
+      of "nimble": bundleNimbleExe(latest, op.cmdLineRest)
       of "nimsuggest": bundleNimsuggest(op.cmdLineRest)
       of "toolsnonimble":
         buildTools(op.cmdLineRest)
       of "tools":
         buildTools(op.cmdLineRest)
-        buildNimble(latest, op.cmdLineRest)
+        bundleNimbleExe(latest, op.cmdLineRest)
       of "pushcsource", "pushcsources": pushCsources()
       of "valgrind": valgrind(op.cmdLineRest)
       of "c2nim": bundleC2nim(op.cmdLineRest)

lib/pure/asyncnet.nim

@@ -689,8 +689,12 @@ elif defined(nimdoc):
 
 proc close*(socket: AsyncSocket) =
   ## Closes the socket.
+  if socket.closed: return
+
   defer:
     socket.fd.AsyncFD.closeSocket()
+    socket.closed = true # TODO: Add extra debugging checks for this.
+
   when defineSsl:
     if socket.isSsl:
       let res = SSL_shutdown(socket.sslHandle)
@@ -699,7 +703,6 @@ proc close*(socket: AsyncSocket) =
         discard
       elif res != 1:
         raiseSSLError()
-  socket.closed = true # TODO: Add extra debugging checks for this.
 
 when defineSsl:
   proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) =

lib/pure/collections/tables.nim

@@ -2546,7 +2546,7 @@ proc `[]`*[A](t: CountTableRef[A], key: A): int =
   ## See also:
   ## * `getOrDefault<#getOrDefault,CountTableRef[A],A,int>`_ to return
   ##   a custom value if the key doesn't exist
-  ## * `mget proc<#mget,CountTableRef[A],A>`_
+  ## * `inc proc<#inc,CountTableRef[A],A>`_ to inc even if missing
   ## * `[]= proc<#[]%3D,CountTableRef[A],A,int>`_ for inserting a new
   ##   (key, value) pair in the table
   ## * `hasKey proc<#hasKey,CountTableRef[A],A>`_ for checking if a key

lib/pure/net.nim

@@ -1703,4 +1703,5 @@ proc getPrimaryIPAddr*(dest = parseIpAddress("8.8.8.8")): IpAddress =
     else:
       newSocket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP)
   socket.connect($dest, 80.Port)
-  socket.getLocalAddr()[0].parseIpAddress()
+  result = socket.getLocalAddr()[0].parseIpAddress()
+  socket.close()

lib/pure/parsexml.nim

@@ -666,6 +666,9 @@ proc parseAttribute(my: var XmlParser) =
         parseEntity(my, my.b)
         my.kind = xmlAttribute # parseEntity overwrites my.kind!
         pos = my.bufpos
+      elif c == '/':
+        pos = lexbase.handleRefillChar(my, pos)
+        add(my.b, '/')
       else:
         add(my.b, c)
         inc(pos)

lib/pure/terminal.nim

@@ -687,10 +687,9 @@ template styledEchoProcessArg(f: File, color: Color) =
 template styledEchoProcessArg(f: File, cmd: TerminalCmd) =
   when cmd == resetStyle:
     resetAttributes(f)
-  when cmd == fgColor:
-    fgSetColor = true
-  when cmd == bgColor:
-    fgSetColor = false
+  elif cmd in {fgColor, bgColor}:
+    let term = getTerminal()
+    term.fgSetColor = cmd == fgColor
 
 macro styledWrite*(f: File, m: varargs[typed]): untyped =
   ## Similar to ``write``, but treating terminal style arguments specially.