58 changes: 51 additions & 7 deletions compiler/depends.nim
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@

# This module implements a dependency file generator.

import
options, ast, ropes, idents, passes, modulepaths, pathutils
import options, ast, ropes, passes, pathutils, msgs, lineinfos

from modulegraphs import ModuleGraph, PPassContext
import modulegraphs

import std/[os, strutils, parseutils]
import std/private/globs

type
TGen = object of PPassContext
Expand All @@ -28,18 +30,60 @@ proc addDependencyAux(b: Backend; importing, imported: string) =
b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)])
# s1 -> s2_4[label="[0-9]"];

proc toNimblePath(s: string, isStdlib: bool): string =
const stdPrefix = "std/"
const pkgPrefix = "pkg/"
if isStdlib:
let sub = "lib/"
var start = s.find(sub)
if start < 0:
doAssert false
else:
start += sub.len
let base = s[start..^1]

if base.startsWith("system") or base.startsWith("std"):
result = base
else:
for dir in stdlibDirs:
if base.startsWith(dir):
return stdPrefix & base.splitFile.name

result = stdPrefix & base
else:
var sub = getEnv("NIMBLE_DIR")
if sub.len == 0:
sub = ".nimble/pkgs/"
else:
sub.add "/pkgs/"
var start = s.find(sub)
if start < 0:
result = s
else:
start += sub.len
start += skipUntil(s, '/', start)
start += 1
result = pkgPrefix & s[start..^1]

proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) =
doAssert n.kind == nkSym, $n.kind

let path = splitFile(toProjPath(g.config, n.sym.position.FileIndex))
let modulePath = splitFile(toProjPath(g.config, g.module.position.FileIndex))
let parent = nativeToUnixPath(modulePath.dir / modulePath.name).toNimblePath(belongsToStdlib(g.graph, g.module))
let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym))
addDependencyAux(b, parent, child)

proc addDotDependency(c: PPassContext, n: PNode): PNode =
result = n
let g = PGen(c)
let b = Backend(g.graph.backend)
case n.kind
of nkImportStmt:
for i in 0..<n.len:
var imported = getModuleName(g.config, n[i])
addDependencyAux(b, g.module.name.s, imported)
addDependency(c, g, b, n[i])
of nkFromStmt, nkImportExceptStmt:
var imported = getModuleName(g.config, n[0])
addDependencyAux(b, g.module.name.s, imported)
addDependency(c, g, b, n[0])
of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
for i in 0..<n.len: discard addDotDependency(c, n[i])
else:
Expand Down
16 changes: 7 additions & 9 deletions compiler/docgen.nim
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import
packages/docutils/rst, packages/docutils/rstgen,
json, xmltree, trees, types,
typesrenderer, astalgo, lineinfos, intsets,
pathutils, tables, nimpaths, renderverbatim, osproc
pathutils, tables, nimpaths, renderverbatim, osproc, packages
import packages/docutils/rstast except FileIndex, TLineInfo

from uri import encodeUrl
Expand Down Expand Up @@ -395,9 +395,6 @@ proc getPlainDocstring(n: PNode): string =
result = getPlainDocstring(n[i])
if result.len > 0: return

proc belongsToPackage(conf: ConfigRef; module: PSym): bool =
result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId

proc externalDep(d: PDoc; module: PSym): string =
if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0:
let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position)
Expand Down Expand Up @@ -453,7 +450,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var string;
"\\spanIdentifier{$1}", [escLit, procLink])
elif s != nil and s.kind in {skType, skVar, skLet, skConst} and
sfExported in s.flags and s.owner != nil and
belongsToPackage(d.conf, s.owner) and d.target == outHtml:
belongsToProjectPackage(d.conf, s.owner) and d.target == outHtml:
let external = externalDep(d, s.owner)
result.addf "<a href=\"$1#$2\"><span class=\"Identifier\">$3</span></a>",
[changeFileExt(external, "html"), literal,
Expand Down Expand Up @@ -1041,7 +1038,7 @@ proc traceDeps(d: PDoc, it: PNode) =
for x in it[2]:
a[2] = x
traceDeps(d, a)
elif it.kind == nkSym and belongsToPackage(d.conf, it.sym):
elif it.kind == nkSym and belongsToProjectPackage(d.conf, it.sym):
let external = externalDep(d, it.sym)
if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
dispA(d.conf, d.section[k].finalMarkup,
Expand All @@ -1051,7 +1048,7 @@ proc traceDeps(d: PDoc, it: PNode) =

proc exportSym(d: PDoc; s: PSym) =
const k = exportSection
if s.kind == skModule and belongsToPackage(d.conf, s):
if s.kind == skModule and belongsToProjectPackage(d.conf, s):
let external = externalDep(d, s)
if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
dispA(d.conf, d.section[k].finalMarkup,
Expand All @@ -1060,7 +1057,7 @@ proc exportSym(d: PDoc; s: PSym) =
changeFileExt(external, "html")])
elif s.kind != skModule and s.owner != nil:
let module = originatingModule(s)
if belongsToPackage(d.conf, module):
if belongsToProjectPackage(d.conf, module):
let
complexSymbol = complexName(s.kind, s.ast, s.name.s)
symbolOrId = d.newUniquePlainSymbol(complexSymbol)
Expand Down Expand Up @@ -1227,9 +1224,10 @@ proc finishGenerateDoc*(d: var PDoc) =
var str: string
renderRstToOut(d[], resolved, str)
entry.json[entry.rstField] = %str
d.jEntriesFinal.add entry.json
d.jEntriesPre[i].rst = nil

d.jEntriesFinal.add entry.json # generates docs

proc add(d: PDoc; j: JsonItem) =
if j.json != nil or j.rst != nil: d.jEntriesPre.add j

Expand Down
4 changes: 2 additions & 2 deletions compiler/docgen2.nim
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# semantic checking.

import
options, ast, msgs, passes, docgen, lineinfos, pathutils
options, ast, msgs, passes, docgen, lineinfos, pathutils, packages

from modulegraphs import ModuleGraph, PPassContext

Expand All @@ -23,7 +23,7 @@ type
PGen = ref TGen

proc shouldProcess(g: PGen): bool =
(optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
(optWholeProject in g.doc.conf.globalOptions and g.doc.conf.belongsToProjectPackage(g.module)) or
sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex

template closeImpl(body: untyped) {.dirty.} =
Expand Down
8 changes: 4 additions & 4 deletions compiler/evalffi.nim
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI
else:
globalError(conf, info, "cannot map calling convention to FFI")

template rd(T, p: untyped): untyped = (cast[ptr T](p))[]
template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v
template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[]
template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v
template `+!`(x, y: untyped): untyped =
cast[pointer](cast[ByteAddress](x) + y)

Expand Down Expand Up @@ -177,8 +177,8 @@ const maxPackDepth = 20
var packRecCheck = 0

proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
template awr(T, v: untyped): untyped =
wr(T, res, v)
template awr(typ, v: untyped): untyped =
wr(typ, res, v)

case typ.kind
of tyBool: awr(bool, v.intVal != 0)
Expand Down
63 changes: 28 additions & 35 deletions compiler/extccomp.nim
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
# from a lineinfos file, to provide generalized procedures to compile
# nim files.

import ropes, platform, condsyms, options, msgs, lineinfos, pathutils
import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths

import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar]
import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils]

type
TInfoCCProp* = enum # properties of the C compiler:
Expand Down Expand Up @@ -367,6 +367,7 @@ proc initVars*(conf: ConfigRef) =

proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile,
createSubDir: bool = true): AbsoluteFile =
## Generate the absolute file path to the generated modules.
result = completeGeneratedFilePath(conf, cfile, createSubDir)

proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile =
Expand All @@ -377,7 +378,7 @@ proc addFileToCompile*(conf: ConfigRef; cf: Cfile) =
conf.toCompile.add(cf)

proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) =
let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string
let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string
var value = conf.cfileSpecificOptions.getOrDefault(key)
if strutils.find(value, option, 0) < 0:
addOpt(value, option)
Expand Down Expand Up @@ -483,22 +484,25 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
(conf.target.hostOS == osWindows)

proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
conf.backend == backendCpp and not cfile.string.endsWith(".c")
# List of possible file extensions taken from gcc
for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]:
if cfile.string.endsWith(ext): return true
false

proc envFlags(conf: ConfigRef): string =
result = if conf.backend == backendCpp:
getEnv("CXXFLAGS")
else:
getEnv("CFLAGS")

proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string =
proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string =
if compiler == ccEnv:
result = if useCpp(conf, cfile):
result = if isCpp:
getEnv("CXX")
else:
getEnv("CC")
else:
result = if useCpp(conf, cfile):
result = if isCpp:
CC[compiler].cppCompiler
else:
CC[compiler].compilerExe
Expand All @@ -512,47 +516,36 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool =
result = false # assume an old or crappy GCC
var exe = getConfigVar(conf, conf.cCompiler, ".exe")
if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
# NOTE: should we need the full version, use -dumpfullversion
let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except: ("", 1)
if exitCode == 0:
var i = 0
var j = 0
# the version is the last part of the first line:
while i < s.len and s[i] != '\n':
if s[i] in {' ', '\t'}: j = i+1
inc i
if j > 0:
var major = 0
while j < s.len and s[j] in {'0'..'9'}:
major = major * 10 + (ord(s[j]) - ord('0'))
inc j
if i < s.len and s[j] == '.': inc j
while j < s.len and s[j] in {'0'..'9'}:
inc j
if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
# we found a third version number, chances are high
# we really parsed the version:
result = major >= 5
var major: int
discard parseInt(s, major)
result = major >= 5
else:
result = conf.cCompiler == ccCLang

proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
else: getCompilerExe(conf, compiler, AbsoluteFile"")
else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp)

proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
isMainFile = false; produceOutput = false): string =
let c = conf.cCompiler
let
c = conf.cCompiler
isCpp = useCpp(conf, cfile.cname)
# We produce files like module.nim.cpp, so the absolute Nim filename is not
# cfile.name but `cfile.cname.changeFileExt("")`:
var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string)
if useCpp(conf, cfile.cname):
if isCpp:
# needs to be prepended so that --passc:-std=c++17 can override default.
# we could avoid allocation by making cFileSpecificOptions inplace
options = CC[c].cppXsupport & ' ' & options
# If any C++ file was compiled, we need to use C++ driver for linking as well
incl conf.globalOptions, optMixedMode

var exe = getConfigVar(conf, c, ".exe")
if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
if exe.len == 0: exe = getCompilerExe(conf, c, isCpp)

if needsExeExt(conf): exe = addFileExt(exe, "exe")
if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and
Expand All @@ -572,7 +565,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,

compilePattern = joinPath(conf.cCompilerPath, exe)
else:
compilePattern = getCompilerExe(conf, c, cfile.cname)
compilePattern = getCompilerExe(conf, c, isCpp)

includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))

Expand Down Expand Up @@ -634,7 +627,7 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good

let hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1")
let currentHash = footprint(conf, cfile)
var f: File
if open(f, hashFile.string, fmRead):
Expand Down Expand Up @@ -842,9 +835,9 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu
proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
if conf.hasHint(hintCC):
if optListCmd in conf.globalOptions or conf.verbosity > 1:
result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd)
result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd)
else:
result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name)
result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name)

proc callCCompiler*(conf: ConfigRef) =
var
Expand Down
8 changes: 4 additions & 4 deletions compiler/ic/bitabs.nim
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,13 @@ proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId =
t.vals.add v


proc `[]`*[T](t: var BiTable[T]; LitId: LitId): var T {.inline.} =
let idx = idToIdx LitId
proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} =
let idx = idToIdx litId
assert idx < t.vals.len
result = t.vals[idx]

proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} =
let idx = idToIdx LitId
proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} =
let idx = idToIdx litId
assert idx < t.vals.len
result = t.vals[idx]

Expand Down
5 changes: 3 additions & 2 deletions compiler/ic/cbackend.nim
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import std/packedsets, algorithm, tables

import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen,
pathutils, extccomp, msgs]
pathutils, extccomp, msgs, modulepaths]

import packed_ast, ic, dce, rodfiles

Expand Down Expand Up @@ -61,7 +61,8 @@ proc addFileToLink(config: ConfigRef; m: PSym) =
if config.backend == backendCpp: ".nim.cpp"
elif config.backend == backendObjc: ".nim.m"
else: ".nim.c"
let cfile = changeFileExt(completeCfilePath(config, withPackageName(config, filename)), ext)
let cfile = changeFileExt(completeCfilePath(config,
mangleModuleName(config, filename).AbsoluteFile), ext)
let objFile = completeCfilePath(config, toObjFile(config, cfile))
if fileExists(objFile):
var cf = Cfile(nimname: m.name.s, cname: cfile,
Expand Down
19 changes: 6 additions & 13 deletions compiler/ic/ic.nim
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import hashes, tables, intsets, std/sha1
import packed_ast, bitabs, rodfiles
import ".." / [ast, idents, lineinfos, msgs, ropes, options,
pathutils, condsyms]
pathutils, condsyms, packages, modulepaths]
#import ".." / [renderer, astalgo]
from os import removeFile, isAbsolute

Expand Down Expand Up @@ -548,6 +548,10 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
rawMessage(config, warnCannotOpenFile, filename.string & " reason: " & $err)
#echo "Error: ", $err, " loading file: ", filename.string

proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile =
result = changeFileExt(completeGeneratedFilePath(conf,
mangleModuleName(conf, f).AbsoluteFile), ext)

proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef;
ignoreConfig = false): RodFileError =
var f = rodfiles.open(filename.string)
Expand Down Expand Up @@ -927,17 +931,6 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
result = g[si].types[t.item]
assert result.itemId.item > 0

proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym =
let filename = AbsoluteFile toFullPath(config, fileIdx)
let name = getIdent(cache, splitFile(filename).name)
let info = newLineInfo(fileIdx, 1, 1)
let
pck = getPackageName(config, filename.string)
pck2 = if pck.len > 0: pck else: "unknown"
pack = getIdent(cache, pck2)
result = newSym(skPackage, getIdent(cache, pck2),
ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info)

proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
fileIdx: FileIndex; m: var LoadedModule) =
m.iface = initTable[PIdent, seq[PackedItemId]]()
Expand Down Expand Up @@ -965,7 +958,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
name: getIdent(cache, splitFile(filename).name),
info: newLineInfo(fileIdx, 1, 1),
position: int(fileIdx))
m.module.owner = newPackage(conf, cache, fileIdx)
m.module.owner = getPackage(conf, cache, fileIdx)
m.module.flags = m.fromDisk.moduleFlags

proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
Expand Down
3 changes: 1 addition & 2 deletions compiler/ic/rodfiles.nim
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@ from typetraits import supportsCopyMem
##
## Now read the bits below to understand what's missing.
##
## Issues with the Example
## ```````````````````````
## ### Issues with the Example
## Missing Sections:
## This is a low level API, so headers and sections need to be stored and
## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` &
Expand Down
29 changes: 26 additions & 3 deletions compiler/injectdestructors.nim
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,7 @@ proc processScope(c: var Con; s: var Scope; ret: PNode): PNode =

template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: untyped): PNode =
assert not ret.typ.isEmptyType
var result = newNodeI(nkStmtListExpr, ret.info)
var result = newNodeIT(nkStmtListExpr, ret.info, ret.typ)
# There is a possibility to do this check: s.wasMoved.len > 0 or s.final.len > 0
# later and use it to eliminate the temporary when theres no need for it, but its
# tricky because you would have to intercept moveOrCopy at a certain point
Expand Down Expand Up @@ -1018,6 +1018,26 @@ proc sameLocation*(a, b: PNode): bool =
of nkHiddenStdConv, nkHiddenSubConv: sameLocation(a[1], b)
else: false

proc genFieldAccessSideEffects(c: var Con; dest, ri: PNode, isDecl: bool): PNode =
# with side effects
var temp = newSym(skLet, getIdent(c.graph.cache, "bracketTmp"), nextSymId c.idgen, c.owner, ri[1].info)
temp.typ = ri[1].typ
var v = newNodeI(nkLetSection, ri[1].info)
let tempAsNode = newSymNode(temp)

var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
vpart[0] = tempAsNode
vpart[1] = newNodeI(nkEmpty, tempAsNode.info)
vpart[2] = ri[1]
v.add(vpart)

var newAccess = copyNode(ri)
newAccess.add ri[0]
newAccess.add tempAsNode

var snk = c.genSink(dest, newAccess, isDecl)
result = newTree(nkStmtList, v, snk, c.genWasMoved(newAccess))

proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, isDecl = false): PNode =
if sameLocation(dest, ri):
# rule (self-assignment-removal):
Expand All @@ -1041,8 +1061,11 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, isDecl = false): PNod
elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c):
if aliases(dest, ri) == no:
# Rule 3: `=sink`(x, z); wasMoved(z)
var snk = c.genSink(dest, ri, isDecl)
result = newTree(nkStmtList, snk, c.genWasMoved(ri))
if isAtom(ri[1]):
var snk = c.genSink(dest, ri, isDecl)
result = newTree(nkStmtList, snk, c.genWasMoved(ri))
else:
result = genFieldAccessSideEffects(c, dest, ri, isDecl)
else:
result = c.genSink(dest, destructiveMoveVar(ri, c, s), isDecl)
else:
Expand Down
3 changes: 3 additions & 0 deletions compiler/liftdestructors.nim
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,12 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
# the value needs to be destroyed before we assign the selector
# or the value is lost
let prevKind = c.kind
let prevAddMemReset = c.addMemReset
c.kind = attachedDestructor
c.addMemReset = true
fillBodyObj(c, n, body, x, y, enforceDefaultOp = false)
c.kind = prevKind
c.addMemReset = prevAddMemReset
localEnforceDefaultOp = true

if c.kind != attachedDestructor:
Expand Down
71 changes: 42 additions & 29 deletions compiler/linter.nim
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
import std/strutils
from std/sugar import dup

import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs
import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs, semdata, packages
export packages

const
Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'}
Expand Down Expand Up @@ -85,24 +86,32 @@ proc differ*(line: string, a, b: int, x: string): string =
result = y

proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
# operators stay as they are:
if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
if k in {skType, skGenericParam} and sfAnon in s.flags: return
if s.typ != nil and s.typ.kind == tyTypeDesc: return
if {sfImportc, sfExportc} * s.flags != {}: return
if optStyleCheck notin s.options: return
let beau = beautifyName(s.name.s, k)
if s.name.s != beau:
lintReport(conf, info, beau, s.name.s)

template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
if {optStyleHint, optStyleError} * conf.globalOptions != {} and optStyleUsages notin conf.globalOptions:
nep1CheckDefImpl(conf, info, s, k)

template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =
styleCheckDef(conf, info, s, s.kind)
template styleCheckDef*(conf: ConfigRef; s: PSym) =
styleCheckDef(conf, s.info, s, s.kind)
template styleCheckDef*(ctx: PContext; info: TLineInfo; sym: PSym; k: TSymKind) =
## Check symbol definitions adhere to NEP1 style rules.
if optStyleCheck in ctx.config.options and # ignore if styleChecks are off
hintName in ctx.config.notes and # ignore if name checks are not requested
ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages
optStyleUsages notin ctx.config.globalOptions and # ignore if requested to only check name usage
sym.kind != skResult and # ignore `result`
sym.kind != skTemp and # ignore temporary variables created by the compiler
sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
k notin {skType, skGenericParam} and # ignore types and generic params
(sym.typ == nil or sym.typ.kind != tyTypeDesc) and # ignore `typedesc`
{sfImportc, sfExportc} * sym.flags == {} and # ignore FFI
sfAnon notin sym.flags: # ignore if created by compiler
nep1CheckDefImpl(ctx.config, info, sym, k)

template styleCheckDef*(ctx: PContext; info: TLineInfo; s: PSym) =
## Check symbol definitions adhere to NEP1 style rules.
styleCheckDef(ctx, info, s, s.kind)

template styleCheckDef*(ctx: PContext; s: PSym) =
## Check symbol definitions adhere to NEP1 style rules.
styleCheckDef(ctx, s.info, s, s.kind)

proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
let line = sourceLine(conf, info)
Expand All @@ -116,23 +125,27 @@ proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
let last = first+identLen(line, first)-1
result = differ(line, first, last, newName)

proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
if info.fileIndex.int < 0: return
# we simply convert it to what it looks like in the definition
# for consistency

# operators stay as they are:
if s.kind == skTemp or s.name.s[0] notin Letters or sfAnon in s.flags:
return

proc styleCheckUseImpl(conf: ConfigRef; info: TLineInfo; s: PSym) =
let newName = s.name.s
let badName = differs(conf, info, newName)
if badName.len > 0:
# special rules for historical reasons
let forceHint = badName == "nnkArgList" and newName == "nnkArglist" or badName == "nnkArglist" and newName == "nnkArgList"
lintReport(conf, info, newName, badName, forceHint = forceHint, extraMsg = "".dup(addDeclaredLoc(conf, s)))

proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
lintReport(conf, info, newName, badName, "".dup(addDeclaredLoc(conf, s)))

template styleCheckUse*(ctx: PContext; info: TLineInfo; sym: PSym) =
## Check symbol uses match their definition's style.
if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off
hintName in ctx.config.notes and # ignore if name checks are not requested
ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages
sym.kind != skTemp and # ignore temporary variables created by the compiler
sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
sfAnon notin sym.flags: # ignore temporary variables created by the compiler
styleCheckUseImpl(ctx.config, info, sym)

proc checkPragmaUseImpl(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
let wanted = $w
if pragmaName != wanted:
lintReport(conf, info, wanted, pragmaName)

template checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
if {optStyleHint, optStyleError} * conf.globalOptions != {}:
checkPragmaUseImpl(conf, info, w, pragmaName)
114 changes: 88 additions & 26 deletions compiler/modulegraphs.nim
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
## represents a complete Nim project. Single modules can either be kept in RAM
## or stored in a rod-file.

import intsets, tables, hashes, md5
import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils
import intsets, tables, hashes, md5, sequtils
import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages
import ic / [packed_ast, ic]

type
Expand Down Expand Up @@ -50,6 +50,10 @@ type
concreteTypes*: seq[FullId]
inst*: PInstantiation

SymInfoPair* = object
sym*: PSym
info*: TLineInfo

ModuleGraph* {.acyclic.} = ref object
ifaces*: seq[Iface] ## indexed by int32 fileIdx
packed*: PackedModuleGraph
Expand All @@ -64,7 +68,6 @@ type

startupPackedConfig*: PackedConfig
packageSyms*: TStrTable
modulesPerPackage*: Table[ItemId, TStrTable]
deps*: IntSet # the dependency graph or potentially its transitive closure.
importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
suggestMode*: bool # whether we are in nimsuggest mode or not.
Expand All @@ -81,6 +84,8 @@ type
doStopCompile*: proc(): bool {.closure.}
usageSym*: PSym # for nimsuggest
owners*: seq[PSym]
suggestSymbols*: Table[FileIndex, seq[SymInfoPair]]
suggestErrors*: Table[FileIndex, seq[Suggest]]
methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization!
systemModule*: PSym
sysTypes*: array[TTypeKind, PType]
Expand Down Expand Up @@ -368,20 +373,9 @@ template getPContext(): untyped =
when c is PContext: c
else: c.c

when defined(nimfind):
template onUse*(info: TLineInfo; s: PSym) =
let c = getPContext()
if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info)

template onDef*(info: TLineInfo; s: PSym) =
let c = getPContext()
if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info)

template onDefResolveForward*(info: TLineInfo; s: PSym) =
let c = getPContext()
if c.graph.onDefinitionResolveForward != nil:
c.graph.onDefinitionResolveForward(c.graph, s, info)

when defined(nimsuggest):
template onUse*(info: TLineInfo; s: PSym) = discard
template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
else:
template onUse*(info: TLineInfo; s: PSym) = discard
template onDef*(info: TLineInfo; s: PSym) = discard
Expand Down Expand Up @@ -432,8 +426,7 @@ proc initOperators*(g: ModuleGraph): Operators =
result.opNot = createMagic(g, "not", mNot)
result.opContains = createMagic(g, "contains", mInSet)

proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
result = ModuleGraph()
proc initModuleGraphFields(result: ModuleGraph) =
# A module ID of -1 means that the symbol is not attached to a module at all,
# but to the module graph:
result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32)
Expand All @@ -443,9 +436,9 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
result.ifaces = @[]
result.importStack = @[]
result.inclToMod = initTable[FileIndex, FileIndex]()
result.config = config
result.cache = cache
result.owners = @[]
result.suggestSymbols = initTable[FileIndex, seq[SymInfoPair]]()
result.suggestErrors = initTable[FileIndex, seq[Suggest]]()
result.methods = @[]
initStrTable(result.compilerprocs)
initStrTable(result.exposed)
Expand All @@ -459,6 +452,12 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
result.operators = initOperators(result)
result.emittedTypeInfo = initTable[string, FileIndex]()

proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
result = ModuleGraph()
result.config = config
result.cache = cache
initModuleGraphFields(result)

proc resetAllModules*(g: ModuleGraph) =
initStrTable(g.packageSyms)
g.deps = initIntSet()
Expand All @@ -470,6 +469,7 @@ proc resetAllModules*(g: ModuleGraph) =
g.methods = @[]
initStrTable(g.compilerprocs)
initStrTable(g.exposed)
initModuleGraphFields(g)

proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
if fileIdx.int32 >= 0:
Expand Down Expand Up @@ -548,7 +548,19 @@ proc transitiveClosure(g: var IntSet; n: int) =

proc markDirty*(g: ModuleGraph; fileIdx: FileIndex) =
let m = g.getModule fileIdx
if m != nil: incl m.flags, sfDirty
if m != nil:
g.suggestSymbols.del(fileIdx)
g.suggestErrors.del(fileIdx)
incl m.flags, sfDirty

proc unmarkAllDirty*(g: ModuleGraph) =
for i in 0i32..<g.ifaces.len.int32:
let m = g.ifaces[i].module
if m != nil:
m.flags.excl sfDirty

proc isDirty*(g: ModuleGraph; m: PSym): bool =
result = g.suggestMode and sfDirty in m.flags

proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
# we need to mark its dependent modules D as dirty right away because after
Expand All @@ -558,14 +570,33 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
g.invalidTransitiveClosure = false
transitiveClosure(g.deps, g.ifaces.len)

# every module that *depends* on this file is also dirty:
for i in 0i32..<g.ifaces.len.int32:
if g.deps.contains(i.dependsOn(fileIdx.int)):
let
fi = FileIndex(i)
module = g.getModule(fi)
if module != nil and not g.isDirty(module):
g.markDirty(fi)
g.markClientsDirty(fi)

proc needsCompilation*(g: ModuleGraph): bool =
# every module that *depends* on this file is also dirty:
for i in 0i32..<g.ifaces.len.int32:
let m = g.ifaces[i].module
if m != nil and g.deps.contains(i.dependsOn(fileIdx.int)):
incl m.flags, sfDirty
if m != nil:
if sfDirty in m.flags:
return true

proc isDirty*(g: ModuleGraph; m: PSym): bool =
result = g.suggestMode and sfDirty in m.flags
proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool =
let module = g.getModule(fileIdx)
if module != nil and g.isDirty(module):
return true

for i in 0i32..<g.ifaces.len.int32:
let m = g.ifaces[i].module
if m != nil and g.isDirty(m) and g.deps.contains(fileIdx.int32.dependsOn(i)):
return true

proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
result = s.ast[bodyPos]
Expand Down Expand Up @@ -594,3 +625,34 @@ proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string,
let fromModule2 = if fromModule != nil: $fromModule.name.s else: "(toplevel)"
let mode = if isNimscript: "(nims) " else: ""
rawMessage(conf, hintProcessing, "$#$# $#: $#: $#" % [mode, indent, fromModule2, moduleStatus, path])

proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
## Returns a package symbol for yet to be defined module for fileIdx.
## The package symbol is added to the graph if it doesn't exist.
let pkgSym = getPackage(graph.config, graph.cache, fileIdx)
# check if the package is already in the graph
result = graph.packageSyms.strTableGet(pkgSym.name)
if result == nil:
# the package isn't in the graph, so create and add it
result = pkgSym
graph.packageSyms.strTableAdd(pkgSym)

func belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool =
## Check if symbol belongs to the 'stdlib' package.
sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId

proc `==`*(a, b: SymInfoPair): bool =
result = a.sym == b.sym and a.info.exactEquals(b.info)

proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): seq[SymInfoPair] =
result = graph.suggestSymbols.getOrDefault(fileIdx, @[]).deduplicate

iterator suggestSymbolsIter*(g: ModuleGraph): SymInfoPair =
for xs in g.suggestSymbols.values:
for x in xs.deduplicate:
yield x

iterator suggestErrorsIter*(g: ModuleGraph): Suggest =
for xs in g.suggestErrors.values:
for x in xs:
yield x
109 changes: 15 additions & 94 deletions compiler/modulepaths.nim
Original file line number Diff line number Diff line change
Expand Up @@ -10,100 +10,6 @@
import ast, renderer, strutils, msgs, options, idents, os, lineinfos,
pathutils

when false:
const
considerParentDirs = not defined(noParentProjects)
considerNimbleDirs = not defined(noNimbleDirs)

proc findInNimbleDir(pkg, subdir, dir: string): string =
var best = ""
var bestv = ""
for k, p in os.walkDir(dir, relative=true):
if k == pcDir and p.len > pkg.len+1 and
p[pkg.len] == '-' and p.startsWith(pkg):
let (_, a, _) = getPathVersionChecksum(p)
if bestv.len == 0 or bestv < a:
bestv = a
best = dir / p

if best.len > 0:
var f: File
if open(f, best / changeFileExt(pkg, ".nimble-link")):
# the second line contains what we're interested in, see:
# https://github.com/nim-lang/nimble#nimble-link
var override = ""
discard readLine(f, override)
discard readLine(f, override)
close(f)
if not override.isAbsolute():
best = best / override
else:
best = override
let f = if subdir.len == 0: pkg else: subdir
let res = addFileExt(best / f, "nim")
if best.len > 0 and fileExists(res):
result = res

when false:
proc resolveDollar(project, source, pkg, subdir: string; info: TLineInfo): string =
template attempt(a) =
let x = addFileExt(a, "nim")
if fileExists(x): return x

case pkg
of "stdlib":
if subdir.len == 0:
return options.libpath
else:
for candidate in stdlibDirs:
attempt(options.libpath / candidate / subdir)
of "root":
let root = project.splitFile.dir
if subdir.len == 0:
return root
else:
attempt(root / subdir)
else:
when considerParentDirs:
var p = parentDir(source.splitFile.dir)
# support 'import $karax':
let f = if subdir.len == 0: pkg else: subdir

while p.len > 0:
let dir = p / pkg
if dirExists(dir):
attempt(dir / f)
# 2nd attempt: try to use 'karax/karax'
attempt(dir / pkg / f)
# 3rd attempt: try to use 'karax/src/karax'
attempt(dir / "src" / f)
attempt(dir / "src" / pkg / f)
p = parentDir(p)

when considerNimbleDirs:
if not options.gNoNimblePath:
var nimbleDir = getEnv("NIMBLE_DIR")
if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
result = findInNimbleDir(pkg, subdir, nimbleDir / "pkgs")
if result.len > 0: return result
when not defined(windows):
result = findInNimbleDir(pkg, subdir, "/opt/nimble/pkgs")
if result.len > 0: return result

proc scriptableImport(pkg, sub: string; info: TLineInfo): string =
resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)

proc lookupPackage(pkg, subdir: PNode): string =
let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
case pkg.kind
of nkStrLit, nkRStrLit, nkTripleStrLit:
result = scriptableImport(pkg.strVal, sub, pkg.info)
of nkIdent:
result = scriptableImport(pkg.ident.s, sub, pkg.info)
else:
localError(pkg.info, "package name must be an identifier or string literal")
result = ""

proc getModuleName*(conf: ConfigRef; n: PNode): string =
# This returns a short relative module name without the nim extension
# e.g. like "system", "importer" or "somepath/module"
Expand Down Expand Up @@ -163,3 +69,18 @@ proc checkModuleName*(conf: ConfigRef; n: PNode; doLocalError=true): FileIndex =
result = InvalidFileIdx
else:
result = fileInfoIdx(conf, fullPath)

proc mangleModuleName*(conf: ConfigRef; path: AbsoluteFile): string =
## Mangle a relative module path to avoid path and symbol collisions.
##
## Used by backends that need to generate intermediary files from Nim modules.
## This is needed because the compiler uses a flat cache file hierarchy.
##
## Example:
## `foo-#head/../bar` becomes `@foo-@hhead@s..@sbar`
"@m" & relativeTo(path, conf.projectPath).string.multiReplace(
{$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"})

proc demangleModuleName*(path: string): string =
## Demangle a relative module path.
result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"})
51 changes: 3 additions & 48 deletions compiler/modules.nim
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import
ast, astalgo, magicsys, msgs, options,
idents, lexer, passes, syntaxes, llstream, modulegraphs,
lineinfos, pathutils, tables
lineinfos, pathutils, tables, packages

import ic / replayer

Expand All @@ -22,56 +22,11 @@ proc resetSystemArtifacts*(g: ModuleGraph) =
template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent =
getIdent(graph.cache, splitFile(filename).name)

template packageId(): untyped {.dirty.} = ItemId(module: PackageModuleId, item: int32(fileIdx))

proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym =
## returns package symbol (skPackage) for yet to be defined module for fileIdx
let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
let name = getModuleIdent(graph, filename)
let info = newLineInfo(fileIdx, 1, 1)
let
pck = getPackageName(graph.config, filename.string)
pck2 = if pck.len > 0: pck else: "unknown"
pack = getIdent(graph.cache, pck2)
result = graph.packageSyms.strTableGet(pack)
if result == nil:
result = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info)
#initStrTable(packSym.tab)
graph.packageSyms.strTableAdd(result)
else:
let modules = graph.modulesPerPackage.getOrDefault(result.itemId)
let existing = if modules.data.len > 0: strTableGet(modules, name) else: nil
if existing != nil and existing.info.fileIndex != info.fileIndex:
when false:
# we used to produce an error:
localError(graph.config, info,
"module names need to be unique per Nimble package; module clashes with " &
toFullPath(graph.config, existing.info.fileIndex))
else:
# but starting with version 0.20 we now produce a fake Nimble package instead
# to resolve the conflicts:
let pck3 = fakePackageName(graph.config, filename)
# this makes the new `result`'s owner be the original `result`
result = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), result, info)
#initStrTable(packSym.tab)
graph.packageSyms.strTableAdd(result)

proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
let packSym = getPackage(graph, fileIdx)
result.owner = packSym
result.position = int fileIdx

#initStrTable(result.tab(graph))
when false:
strTableAdd(result.tab, result) # a module knows itself
# This is now implemented via
# c.moduleScope.addSym(module) # a module knows itself
# in sem.nim, around line 527

if graph.modulesPerPackage.getOrDefault(packSym.itemId).data.len == 0:
graph.modulesPerPackage[packSym.itemId] = newStrTable()
graph.modulesPerPackage[packSym.itemId].strTableAdd(result)

proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
# We cannot call ``newSym`` here, because we have to circumvent the ID
Expand Down Expand Up @@ -133,7 +88,7 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
# localError(result.info, errAttemptToRedefine, result.name.s)
# restore the notes for outer module:
graph.config.notes =
if s.getnimblePkgId == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
else: graph.config.foreignPackageNotes

proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
Expand Down Expand Up @@ -168,7 +123,7 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
conf.projectMainIdx2 = projectFile

let packSym = getPackage(graph, projectFile)
graph.config.mainPackageId = packSym.getnimblePkgId
graph.config.mainPackageId = packSym.getPackageId
graph.importStack.add projectFile

if projectFile == systemFileIdx:
Expand Down
4 changes: 2 additions & 2 deletions compiler/msgs.nim
Original file line number Diff line number Diff line change
Expand Up @@ -618,9 +618,9 @@ template internalAssert*(conf: ConfigRef, e: bool) =
let arg = info2.toFileLineCol
internalErrorImpl(conf, unknownLineInfo, arg, info2)

template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, forceHint = false, extraMsg = "") =
template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, extraMsg = "") =
let m = "'$1' should be: '$2'$3" % [got, beau, extraMsg]
let msg = if optStyleError in conf.globalOptions and not forceHint: errGenerated else: hintName
let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName
liMessage(conf, info, msg, m, doNothing, instLoc())

proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
Expand Down
1 change: 1 addition & 0 deletions compiler/nim.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ define:nimPreviewFloatRoundtrip

@if windows:
cincludes: "$lib/wrappers/libffi/common"
tlsEmulation:off
@end

define:useStdoutAsStdmsg
Expand Down
24 changes: 16 additions & 8 deletions compiler/options.nim
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,9 @@ type
# as far as usesWriteBarrier() is concerned

IdeCmd* = enum
ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod,
ideHighlight, ideOutline, ideKnown, ideMsg, ideProject
ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideChkFile, ideMod,
ideHighlight, ideOutline, ideKnown, ideMsg, ideProject, ideGlobalSymbols,
ideRecompile, ideChanged, ideType

Feature* = enum ## experimental features; DO NOT RENAME THESE!
implicitDeref,
Expand Down Expand Up @@ -801,18 +802,15 @@ proc toGeneratedFile*(conf: ConfigRef; path: AbsoluteFile,

proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile,
createSubDir: bool = true): AbsoluteFile =
## Return an absolute path of a generated intermediary file.
## Optionally creates the cache directory if `createSubDir` is `true`.
let subdir = getNimcacheDir(conf)
if createSubDir:
try:
createDir(subdir.string)
except OSError:
conf.quitOrRaise "cannot create directory: " & subdir.string
result = subdir / RelativeFile f.string.splitPath.tail
#echo "completeGeneratedFilePath(", f, ") = ", result

proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile =
result = changeFileExt(completeGeneratedFilePath(conf,
withPackageName(conf, f)), ext)

proc rawFindFile(conf: ConfigRef; f: RelativeFile; suppressStdlib: bool): AbsoluteFile =
for it in conf.searchPaths:
Expand Down Expand Up @@ -849,7 +847,7 @@ when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo):
let ret = relativePath(path, base)
result = path.len > 0 and not ret.startsWith ".."

const stdlibDirs = [
const stdlibDirs* = [
"pure", "core", "arch",
"pure/collections",
"pure/concurrency",
Expand Down Expand Up @@ -991,12 +989,17 @@ proc parseIdeCmd*(s: string): IdeCmd =
of "use": ideUse
of "dus": ideDus
of "chk": ideChk
of "chkFile": ideChkFile
of "mod": ideMod
of "highlight": ideHighlight
of "outline": ideOutline
of "known": ideKnown
of "msg": ideMsg
of "project": ideProject
of "globalSymbols": ideGlobalSymbols
of "recompile": ideRecompile
of "changed": ideChanged
of "type": ideType
else: ideNone

proc `$`*(c: IdeCmd): string =
Expand All @@ -1007,13 +1010,18 @@ proc `$`*(c: IdeCmd): string =
of ideUse: "use"
of ideDus: "dus"
of ideChk: "chk"
of ideChkFile: "chkFile"
of ideMod: "mod"
of ideNone: "none"
of ideHighlight: "highlight"
of ideOutline: "outline"
of ideKnown: "known"
of ideMsg: "msg"
of ideProject: "project"
of ideGlobalSymbols: "globalSymbols"
of ideRecompile: "recompile"
of ideChanged: "changed"
of ideType: "type"

proc floatInt64Align*(conf: ConfigRef): int16 =
## Returns either 4 or 8 depending on reasons.
Expand Down
23 changes: 3 additions & 20 deletions compiler/packagehandling.nim
Original file line number Diff line number Diff line change
Expand Up @@ -37,24 +37,7 @@ proc getNimbleFile*(conf: ConfigRef; path: string): string =
proc getPackageName*(conf: ConfigRef; path: string): string =
## returns nimble package name, e.g.: `cligen`
let path = getNimbleFile(conf, path)
result = path.splitFile.name

proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string =
# Convert `path` so that 2 modules with same name
# in different directory get different name and they can be
# placed in a directory.
# foo-#head/../bar becomes @foo-@hhead@s..@sbar
result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace(
{$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"})

proc demanglePackageName*(path: string): string =
result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"})

proc withPackageName*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile =
let x = getPackageName(conf, path.string)
let (p, file, ext) = path.splitFile
if x == "stdlib":
# Hot code reloading now relies on 'stdlib_system' names etc.
result = p / RelativeFile((x & '_' & file) & ext)
if path.len > 0:
return path.splitFile.name
else:
result = p / RelativeFile(fakePackageName(conf, path))
return "unknown"
49 changes: 49 additions & 0 deletions compiler/packages.nim
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#
#
# The Nim Compiler
# (c) Copyright 2022 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#

## Package related procs.
##
## See Also:
## * `packagehandling` for package path handling
## * `modulegraphs.getPackage`
## * `modulegraphs.belongsToStdlib`

import "." / [options, ast, lineinfos, idents, pathutils, msgs]

proc getPackage*(conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym =
## Return a new package symbol.
##
## See Also:
## * `modulegraphs.getPackage`
let
filename = AbsoluteFile toFullPath(conf, fileIdx)
name = getIdent(cache, splitFile(filename).name)
info = newLineInfo(fileIdx, 1, 1)
pkgName = getPackageName(conf, filename.string)
pkgIdent = getIdent(cache, pkgName)
newSym(skPackage, pkgIdent, ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info)

func getPackageSymbol*(sym: PSym): PSym =
## Return the owning package symbol.
assert sym != nil
result = sym
while result.kind != skPackage:
result = result.owner
assert result != nil, repr(sym.info)

func getPackageId*(sym: PSym): int =
## Return the owning package ID.
sym.getPackageSymbol.id

func belongsToProjectPackage*(conf: ConfigRef, sym: PSym): bool =
## Return whether the symbol belongs to the project's package.
##
## See Also:
## * `modulegraphs.belongsToStdlib`
conf.mainPackageId == sym.getPackageId
18 changes: 9 additions & 9 deletions compiler/passes.nim
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import
options, ast, llstream, msgs,
idents,
syntaxes, modulegraphs, reorder,
lineinfos, pathutils
lineinfos, pathutils, std/sha1, packages


type
TPassData* = tuple[input: PNode, closeOutput: PNode]
Expand Down Expand Up @@ -101,7 +102,7 @@ const

proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
# don't be verbose unless the module belongs to the main package:
if module.getnimblePkgId == graph.config.mainPackageId:
if graph.config.belongsToProjectPackage(module):
graph.config.notes = graph.config.mainPackageNotes
else:
if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
Expand All @@ -111,12 +112,6 @@ proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
result = true
#module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")

proc partOfStdlib(x: PSym): bool =
var it = x.owner
while it != nil and it.kind == skPackage and it.owner != nil:
it = it.owner
result = it != nil and it.name.s == "stdlib"

proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
stream: PLLStream): bool {.discardable.} =
if graph.stopCompile(): return true
Expand All @@ -135,10 +130,15 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
return false
else:
s = stream

when defined(nimsuggest):
let filename = toFullPathConsiderDirty(graph.config, fileIdx).string
msgs.setHash(graph.config, fileIdx, $sha1.secureHashFile(filename))

while true:
openParser(p, fileIdx, s, graph.cache, graph.config)

if not partOfStdlib(module) or module.name.s == "distros":
if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"):
# XXX what about caching? no processing then? what if I change the
# modules to include between compilation runs? we'd need to track that
# in ROD files. I think we should enable this feature only
Expand Down
2 changes: 1 addition & 1 deletion compiler/plugins/locals.nim
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import ".." / [ast, astalgo,
proc semLocals*(c: PContext, n: PNode): PNode =
var counter = 0
var tupleType = newTypeS(tyTuple, c)
result = newNodeIT(nkPar, n.info, tupleType)
result = newNodeIT(nkTupleConstr, n.info, tupleType)
tupleType.n = newNodeI(nkRecList, n.info)
let owner = getCurrOwner(c)
# for now we skip openarrays ...
Expand Down
11 changes: 5 additions & 6 deletions compiler/pragmas.nim
Original file line number Diff line number Diff line change
Expand Up @@ -635,12 +635,13 @@ proc pragmaLine(c: PContext, n: PNode) =
n.info = getInfoContext(c.config, -1)

proc processPragma(c: PContext, n: PNode, i: int) =
## Create and add a new custom pragma `{.pragma: name.}` node to the module's context.
let it = n[i]
if it.kind notin nkPragmaCallKinds and it.safeLen == 2: invalidPragma(c, n)
elif it.safeLen != 2 or it[0].kind != nkIdent or it[1].kind != nkIdent:
invalidPragma(c, n)

var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), nil, it.info, c.config.options)
var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), c.module, it.info, c.config.options)
userPragma.ast = newTreeI(nkPragma, n.info, n.sons[i+1..^1])
strTableAdd(c.userPragmas, userPragma)

Expand Down Expand Up @@ -822,8 +823,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
let ident = considerQuotedIdent(c, key)
var userPragma = strTableGet(c.userPragmas, ident)
if userPragma != nil:
if {optStyleHint, optStyleError} * c.config.globalOptions != {}:
styleCheckUse(c.config, key.info, userPragma)
styleCheckUse(c, key.info, userPragma)

# number of pragmas increase/decrease with user pragma expansion
inc c.instCounter
Expand All @@ -837,8 +837,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
else:
let k = whichKeyword(ident)
if k in validPragmas:
if {optStyleHint, optStyleError} * c.config.globalOptions != {}:
checkPragmaUse(c.config, key.info, k, ident.s)
checkPragmaUse(c.config, key.info, k, ident.s)
case k
of wExportc, wExportCpp:
makeExternExport(c, sym, getOptionalStr(c, it, "$1"), it.info)
Expand Down Expand Up @@ -1243,7 +1242,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
elif comesFromPush and whichKeyword(ident) != wInvalid:
discard "ignore the .push pragma; it doesn't apply"
else:
if sym == nil or (sym.kind in {skVar, skLet, skParam,
if sym == nil or (sym.kind in {skVar, skLet, skParam, skIterator,
skField, skProc, skFunc, skConverter, skMethod, skType}):
n[i] = semCustomPragma(c, it)
elif sym != nil:
Expand Down
2 changes: 1 addition & 1 deletion compiler/semexprs.nim
Original file line number Diff line number Diff line change
Expand Up @@ -2584,7 +2584,7 @@ proc semBlock(c: PContext, n: PNode; flags: TExprFlags): PNode =
labl.owner = c.p.owner
n[0] = newSymNode(labl, n[0].info)
suggestSym(c.graph, n[0].info, labl, c.graph.usageSym)
styleCheckDef(c.config, labl)
styleCheckDef(c, labl)
onDef(n[0].info, labl)
n[1] = semExpr(c, n[1], flags)
n.typ = n[1].typ
Expand Down
2 changes: 1 addition & 1 deletion compiler/semfold.nim
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ proc foldConv(n, a: PNode; idgen: IdGenerator; g: ModuleGraph; check = false): P
rangeCheck(n, getInt(result), g)
of tyFloat..tyFloat64:
case srcTyp.kind
of tyInt..tyInt64, tyEnum, tyBool, tyChar:
of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyBool, tyChar:
result = newFloatNodeT(toFloat64(getOrdValue(a)), n, g)
else:
result = a
Expand Down
2 changes: 1 addition & 1 deletion compiler/semgnrc.nim
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) =
let s = newSymS(skUnknown, getIdentNode(c, n), c)
addPrelimDecl(c, s)
styleCheckDef(c.config, n.info, s, kind)
styleCheckDef(c, n.info, s, kind)
onDef(n.info, s)

proc semGenericStmt(c: PContext, n: PNode,
Expand Down
2 changes: 1 addition & 1 deletion compiler/semmagic.nim
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ proc semQuantifier(c: PContext; n: PNode): PNode =
let op = considerQuotedIdent(c, it[0])
if op.id == ord(wIn):
let v = newSymS(skForVar, it[1], c)
styleCheckDef(c.config, v)
styleCheckDef(c, v)
onDef(it[1].info, v)
let domain = semExprWithType(c, it[2], {efWantIterator})
v.typ = domain.typ
Expand Down
16 changes: 8 additions & 8 deletions compiler/semstmts.nim
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@ proc semUsing(c: PContext; n: PNode): PNode =
let typ = semTypeNode(c, a[^2], nil)
for j in 0..<a.len-2:
let v = semIdentDef(c, a[j], skParam)
styleCheckDef(c.config, v)
styleCheckDef(c, v)
onDef(a[j].info, v)
v.typ = typ
strTableIncl(c.signatures, v)
Expand Down Expand Up @@ -597,7 +597,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
addToVarSection(c, result, n, a)
continue
var v = semIdentDef(c, a[j], symkind, false)
styleCheckDef(c.config, v)
styleCheckDef(c, v)
onDef(a[j].info, v)
if sfGenSym notin v.flags:
if not isDiscardUnderscore(v): addInterfaceDecl(c, v)
Expand Down Expand Up @@ -722,7 +722,7 @@ proc semConst(c: PContext, n: PNode): PNode =
var v = semIdentDef(c, a[j], skConst)
if sfGenSym notin v.flags: addInterfaceDecl(c, v)
elif v.owner == nil: v.owner = getCurrOwner(c)
styleCheckDef(c.config, v)
styleCheckDef(c, v)
onDef(a[j].info, v)

if a.kind != nkVarTuple:
Expand All @@ -747,7 +747,7 @@ include semfields
proc symForVar(c: PContext, n: PNode): PSym =
let m = if n.kind == nkPragmaExpr: n[0] else: n
result = newSymG(skForVar, m, c)
styleCheckDef(c.config, result)
styleCheckDef(c, result)
onDef(n.info, result)
if n.kind == nkPragmaExpr:
pragma(c, result, n[1], forVarPragmas)
Expand Down Expand Up @@ -978,7 +978,7 @@ proc semCase(c: PContext, n: PNode; flags: TExprFlags): PNode =
var typ = commonTypeBegin
var hasElse = false
let caseTyp = skipTypes(n[0].typ, abstractVar-{tyTypeDesc})
const shouldChckCovered = {tyInt..tyInt64, tyChar, tyEnum, tyUInt..tyUInt32, tyBool}
const shouldChckCovered = {tyInt..tyInt64, tyChar, tyEnum, tyUInt..tyUInt64, tyBool}
case caseTyp.kind
of shouldChckCovered:
chckCovered = true
Expand Down Expand Up @@ -1365,7 +1365,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
let name = typeSectionTypeName(c, a[0])
var s = name.sym
# check the style here after the pragmas have been processed:
styleCheckDef(c.config, s)
styleCheckDef(c, s)
# compute the type's size and check for illegal recursions:
if a[1].kind == nkEmpty:
var x = a[2]
Expand Down Expand Up @@ -1979,7 +1979,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
("'" & proto.name.s & "' from " & c.config$proto.info &
" '" & s.name.s & "' from " & c.config$s.info))

styleCheckDef(c.config, s)
styleCheckDef(c, s)
if hasProto:
onDefResolveForward(n[namePos].info, proto)
else:
Expand Down Expand Up @@ -2056,7 +2056,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
trackProc(c, s, s.ast[bodyPos])
else:
if (s.typ[0] != nil and s.kind != skIterator):
addDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), nextSymId c.idgen, nil, n.info))
addDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), nextSymId c.idgen, s, n.info))

openScope(c)
n[bodyPos] = semGenericStmt(c, n[bodyPos])
Expand Down
20 changes: 13 additions & 7 deletions compiler/semtempl.nim
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
if n.kind != nkSym:
let local = newGenSym(k, ident, c)
addPrelimDecl(c.c, local)
styleCheckDef(c.c.config, n.info, local)
styleCheckDef(c.c, n.info, local)
onDef(n.info, local)
replaceIdentBySym(c.c, n, newSymNode(local, n.info))
if k == skParam and c.inTemplateHeader > 0:
Expand All @@ -250,8 +250,14 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode =
of skUnknown:
# Introduced in this pass! Leave it as an identifier.
result = n
of OverloadableSyms-{skEnumField}:
of OverloadableSyms-{skEnumField, skTemplate, skMacro}:
result = symChoice(c, n, s, scOpen, isField)
of skTemplate, skMacro:
result = symChoice(c, n, s, scOpen, isField)
if result.kind == nkSym:
# template/macro symbols might need to be semchecked again
# prepareOperand etc don't do this without setting the type to nil
result.typ = nil
of skGenericParam:
if isField and sfGenSym in s.flags: result = n
else: result = newSymNodeTypeDesc(s, c.idgen, n.info)
Expand All @@ -271,8 +277,8 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode =
when defined(nimsuggest):
suggestSym(c.graph, n.info, s, c.graph.usageSym, false)
# field access (dot expr) will be handled by builtinFieldAccess
if not isField and {optStyleHint, optStyleError} * c.config.globalOptions != {}:
styleCheckUse(c.config, n.info, s)
if not isField:
styleCheckUse(c, n.info, s)

proc semRoutineInTemplName(c: var TemplCtx, n: PNode): PNode =
result = n
Expand All @@ -297,7 +303,7 @@ proc semRoutineInTemplBody(c: var TemplCtx, n: PNode, k: TSymKind): PNode =
var s = newGenSym(k, ident, c)
s.ast = n
addPrelimDecl(c.c, s)
styleCheckDef(c.c.config, n.info, s)
styleCheckDef(c.c, n.info, s)
onDef(n.info, s)
n[namePos] = newSymNode(s, n[namePos].info)
else:
Expand Down Expand Up @@ -431,7 +437,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
# labels are always 'gensym'ed:
let s = newGenSym(skLabel, n[0], c)
addPrelimDecl(c.c, s)
styleCheckDef(c.c.config, s)
styleCheckDef(c.c, s)
onDef(n[0].info, s)
n[0] = newSymNode(s, n[0].info)
n[1] = semTemplBody(c, n[1])
Expand Down Expand Up @@ -619,7 +625,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
s.owner.name.s == "vm" and s.name.s == "stackTrace":
incl(s.flags, sfCallsite)

styleCheckDef(c.config, s)
styleCheckDef(c, s)
onDef(n[namePos].info, s)
# check parameter list:
#s.scope = c.currentScope
Expand Down
8 changes: 4 additions & 4 deletions compiler/semtypes.nim
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
e.flags.incl {sfUsed, sfExported}

result.n.add symNode
styleCheckDef(c.config, e)
styleCheckDef(c, e)
onDef(e.info, e)
if sfGenSym notin e.flags:
if not isPure:
Expand Down Expand Up @@ -476,7 +476,7 @@ proc semTuple(c: PContext, n: PNode, prev: PType): PType =
else:
result.n.add newSymNode(field)
addSonSkipIntLit(result, typ, c.idgen)
styleCheckDef(c.config, a[j].info, field)
styleCheckDef(c, a[j].info, field)
onDef(field.info, field)
if result.n.len == 0: result.n = nil
if isTupleRecursive(result):
Expand Down Expand Up @@ -808,7 +808,7 @@ proc semRecordNodeAux(c: PContext, n: PNode, check: var IntSet, pos: var int,
localError(c.config, info, "attempt to redefine: '" & f.name.s & "'")
if a.kind == nkEmpty: father.add newSymNode(f)
else: a.add newSymNode(f)
styleCheckDef(c.config, f)
styleCheckDef(c, f)
onDef(f.info, f)
if a.kind != nkEmpty: father.add a
of nkSym:
Expand Down Expand Up @@ -1315,7 +1315,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
result.n.add newSymNode(arg)
rawAddSon(result, finalType)
addParamOrResult(c, arg, kind)
styleCheckDef(c.config, a[j].info, arg)
styleCheckDef(c, a[j].info, arg)
onDef(a[j].info, arg)
if {optNimV1Emulation, optNimV12Emulation} * c.config.globalOptions == {}:
a[j] = newSymNode(arg)
Expand Down
58 changes: 39 additions & 19 deletions compiler/suggest.nim
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int
elif sourceIdent != ident:
result = 0

proc symToSuggest(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
quality: range[0..100]; prefix: PrefixMatch;
inTypeContext: bool; scope: int;
useSuppliedInfo = false): Suggest =
Expand Down Expand Up @@ -157,19 +157,25 @@ proc symToSuggest(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info:
result.forth = ""
when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler):
result.doc = extractDocComment(g, s)
let infox =
if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}:
info
else:
s.info
result.filePath = toFullPath(g.config, infox)
result.line = toLinenumber(infox)
result.column = toColumn(infox)
if s.kind == skModule and s.ast.len != 0 and section != ideHighlight:
result.filePath = toFullPath(g.config, s.ast[0].info)
result.line = 1
result.column = 0
result.tokenLen = 0
else:
let infox =
if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}:
info
else:
s.info
result.filePath = toFullPath(g.config, infox)
result.line = toLinenumber(infox)
result.column = toColumn(infox)
result.tokenLen = if section != ideHighlight:
s.name.s.len
else:
getTokenLenFromSource(g.config, s.name.s, infox)
result.version = g.config.suggestVersion
result.tokenLen = if section != ideHighlight:
s.name.s.len
else:
getTokenLenFromSource(g.config, s.name.s, infox)

proc `$`*(suggest: Suggest): string =
result = $suggest.section
Expand Down Expand Up @@ -203,14 +209,14 @@ proc `$`*(suggest: Suggest): string =
result.add(sep)
when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler):
result.add(suggest.doc.escape)
if suggest.version == 0:
if suggest.version in {0, 3}:
result.add(sep)
result.add($suggest.quality)
if suggest.section == ideSug:
result.add(sep)
result.add($suggest.prefix)

proc suggestResult(conf: ConfigRef; s: Suggest) =
proc suggestResult*(conf: ConfigRef; s: Suggest) =
if not isNil(conf.suggestionResultHook):
conf.suggestionResultHook(s)
else:
Expand Down Expand Up @@ -424,7 +430,7 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
t = skipTypes(t[0], skipPtrs)
elif typ.kind == tyTuple and typ.n != nil:
suggestSymList(c, typ.n, field, n.info, outputs)

suggestOperations(c, n, field, orig, outputs)
if typ != orig:
suggestOperations(c, n, field, typ, outputs)
Expand Down Expand Up @@ -482,7 +488,7 @@ proc findDefinition(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym
if s.isNil: return
if isTracked(info, g.config.m.trackPos, s.name.s.len) or (s == usageSym and sfForward notin s.flags):
suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0, useSuppliedInfo = s == usageSym))
if sfForward notin s.flags:
if sfForward notin s.flags and g.config.suggestVersion != 3:
suggestQuit()
else:
usageSym = s
Expand All @@ -497,6 +503,8 @@ proc suggestSym*(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym; i
## misnamed: should be 'symDeclared'
let conf = g.config
when defined(nimsuggest):
g.suggestSymbols.mgetOrPut(info.fileIndex, @[]).add SymInfoPair(sym: s, info: info)

if conf.suggestVersion == 0:
if s.allUsages.len == 0:
s.allUsages = @[info]
Expand Down Expand Up @@ -596,8 +604,7 @@ proc markUsed(c: PContext; info: TLineInfo; s: PSym) =
if sfError in s.flags: userError(conf, info, s)
when defined(nimsuggest):
suggestSym(c.graph, info, s, c.graph.usageSym, false)
if {optStyleHint, optStyleError} * conf.globalOptions != {}:
styleCheckUse(conf, info, s)
styleCheckUse(c, info, s)
markOwnerModuleAsUsed(c, s)

proc safeSemExpr*(c: PContext, n: PNode): PNode =
Expand Down Expand Up @@ -692,3 +699,16 @@ proc suggestSentinel*(c: PContext) =

dec(c.compilesContextId)
produceOutput(outputs, c.config)

when defined(nimsuggest):
proc onDef(graph: ModuleGraph, s: PSym, info: TLineInfo) =
if graph.config.suggestVersion == 3 and info.exactEquals(s.info):
suggestSym(graph, info, s, graph.usageSym)

template getPContext(): untyped =
when c is PContext: c
else: c.c

template onDef*(info: TLineInfo; s: PSym) =
let c = getPContext()
onDef(c.graph, s, info)
2 changes: 1 addition & 1 deletion compiler/varpartitions.nim
Original file line number Diff line number Diff line change
Expand Up @@ -848,7 +848,7 @@ proc computeLiveRanges(c: var Partitions; n: PNode) =
# connect(graph, cursorVar)
inc c.inLoop
for child in n: computeLiveRanges(c, child)
inc c.inLoop
dec c.inLoop
of nkElifBranch, nkElifExpr, nkElse, nkOfBranch:
inc c.inConditional
for child in n: computeLiveRanges(c, child)
Expand Down
56 changes: 34 additions & 22 deletions compiler/vm.nim
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,9 @@ proc bailOut(c: PCtx; tos: PStackFrame) =
when not defined(nimComputedGoto):
{.pragma: computedGoto.}

proc ensureKind(n: var TFullReg, kind: TRegisterKind) =
if n.kind != kind:
n = TFullReg(kind: kind)
proc ensureKind(n: var TFullReg, k: TRegisterKind) {.inline.} =
if n.kind != k:
n = TFullReg(kind: k)

template ensureKind(k: untyped) {.dirty.} =
ensureKind(regs[ra], k)
Expand Down Expand Up @@ -119,13 +119,13 @@ template move(a, b: untyped) {.dirty.} = system.shallowCopy(a, b)

proc derefPtrToReg(address: BiggestInt, typ: PType, r: var TFullReg, isAssign: bool): bool =
# nim bug: `isAssign: static bool` doesn't work, giving odd compiler error
template fun(field, T, rkind) =
template fun(field, typ, rkind) =
if isAssign:
cast[ptr T](address)[] = T(r.field)
cast[ptr typ](address)[] = typ(r.field)
else:
r.ensureKind(rkind)
let val = cast[ptr T](address)[]
when T is SomeInteger | char:
let val = cast[ptr typ](address)[]
when typ is SomeInteger | char:
r.field = BiggestInt(val)
else:
r.field = val
Expand Down Expand Up @@ -521,6 +521,10 @@ template maybeHandlePtr(node2: PNode, reg: TFullReg, isAssign2: bool): bool =
when not defined(nimHasSinkInference):
{.pragma: nosinks.}

template takeAddress(reg, source) =
reg.nodeAddr = addr source
GC_ref source

proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
var pc = start
var tos = tos
Expand Down Expand Up @@ -637,7 +641,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
of opcNodeToReg:
let ra = instr.regA
let rb = instr.regB
# opcDeref might already have loaded it into a register. XXX Let's hope
# opcLdDeref might already have loaded it into a register. XXX Let's hope
# this is still correct this way:
if regs[rb].kind != rkNode:
regs[ra] = regs[rb]
Expand Down Expand Up @@ -679,7 +683,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
let idx = regs[rc].intVal.int
let src = if regs[rb].kind == rkNode: regs[rb].node else: regs[rb].nodeAddr[]
if src.kind notin {nkEmpty..nkTripleStrLit} and idx <% src.len:
regs[ra].nodeAddr = addr src.sons[idx]
takeAddress regs[ra], src.sons[idx]
else:
stackTrace(c, tos, pc, formatErrorIndexBound(idx, src.safeLen-1))
of opcLdStrIdx:
Expand Down Expand Up @@ -747,11 +751,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
of nkObjConstr:
let n = src.sons[rc + 1]
if n.kind == nkExprColonExpr:
regs[ra].nodeAddr = addr n.sons[1]
takeAddress regs[ra], n.sons[1]
else:
regs[ra].nodeAddr = addr src.sons[rc + 1]
takeAddress regs[ra], src.sons[rc + 1]
else:
regs[ra].nodeAddr = addr src.sons[rc]
takeAddress regs[ra], src.sons[rc]
of opcWrObj:
# a.b = c
decodeBC(rkNode)
Expand All @@ -778,7 +782,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
decodeB(rkNodeAddr)
case regs[rb].kind
of rkNode:
regs[ra].nodeAddr = addr(regs[rb].node)
takeAddress regs[ra], regs[rb].node
of rkNodeAddr: # bug #14339
regs[ra].nodeAddr = regs[rb].nodeAddr
else:
Expand Down Expand Up @@ -1006,6 +1010,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
decodeBC(rkInt)
template getTyp(n): untyped =
n.typ.skipTypes(abstractInst)
template skipRegisterAddr(n: TFullReg): TFullReg =
var tmp = n
while tmp.kind == rkRegisterAddr:
tmp = tmp.regAddr[]
tmp

proc ptrEquality(n1: ptr PNode, n2: PNode): bool =
## true if n2.intVal represents a ptr equal to n1
let p1 = cast[int](n1)
Expand All @@ -1019,16 +1029,19 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
return t2.kind in PtrLikeKinds and n2.intVal == p1
else: return false

if regs[rb].kind == rkNodeAddr:
if regs[rc].kind == rkNodeAddr:
ret = regs[rb].nodeAddr == regs[rc].nodeAddr
let rbReg = skipRegisterAddr(regs[rb])
let rcReg = skipRegisterAddr(regs[rc])

if rbReg.kind == rkNodeAddr:
if rcReg.kind == rkNodeAddr:
ret = rbReg.nodeAddr == rcReg.nodeAddr
else:
ret = ptrEquality(regs[rb].nodeAddr, regs[rc].node)
elif regs[rc].kind == rkNodeAddr:
ret = ptrEquality(regs[rc].nodeAddr, regs[rb].node)
ret = ptrEquality(rbReg.nodeAddr, rcReg.node)
elif rcReg.kind == rkNodeAddr:
ret = ptrEquality(rcReg.nodeAddr, rbReg.node)
else:
let nb = regs[rb].node
let nc = regs[rc].node
let nb = rbReg.node
let nc = rcReg.node
if nb.kind != nc.kind: discard
elif (nb == nc) or (nb.kind == nkNilLit): ret = true # intentional
elif nb.kind in {nkSym, nkTupleConstr, nkClosure} and nb.typ != nil and nb.typ.kind == tyProc and sameConstant(nb, nc):
Expand Down Expand Up @@ -2266,7 +2279,6 @@ proc setupMacroParam(x: PNode, typ: PType): TFullReg =
else:
var n = x
if n.kind in {nkHiddenSubConv, nkHiddenStdConv}: n = n[1]
n = n.canonValue
n.flags.incl nfIsRef
n.typ = x.typ
result = TFullReg(kind: rkNode, node: n)
Expand Down
10 changes: 2 additions & 8 deletions compiler/vmgen.nim
Original file line number Diff line number Diff line change
Expand Up @@ -441,14 +441,11 @@ proc genAndOr(c: PCtx; n: PNode; opc: TOpcode; dest: var TDest) =
c.gABC(n, opcAsgnInt, dest, tmp)
freeTemp(c, tmp)

proc canonValue*(n: PNode): PNode =
result = n

proc rawGenLiteral(c: PCtx; n: PNode): int =
result = c.constants.len
#assert(n.kind != nkCall)
n.flags.incl nfAllConst
c.constants.add n.canonValue
c.constants.add n
internalAssert c.config, result < regBxMax

proc sameConstant*(a, b: PNode): bool =
Expand Down Expand Up @@ -1403,9 +1400,6 @@ proc unneededIndirection(n: PNode): bool =
n.typ.skipTypes(abstractInstOwned-{tyTypeDesc}).kind == tyRef

proc canElimAddr(n: PNode): PNode =
if n[0].typ.skipTypes(abstractInst).kind in {tyObject, tyTuple, tyArray}:
# objects are reference types in the VM
return n[0]
case n[0].kind
of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64:
var m = n[0][0]
Expand Down Expand Up @@ -1875,7 +1869,7 @@ proc genVarSection(c: PCtx; n: PNode) =
else:
let sa = getNullValue(s.typ, a.info, c.config)
#if s.ast.isNil: getNullValue(s.typ, a.info)
#else: canonValue(s.ast)
#else: s.ast
assert sa.kind != nkCall
c.globals.add(sa)
s.position = c.globals.len
Expand Down
64 changes: 32 additions & 32 deletions compiler/vmops.nim
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,13 @@ template macrosop(op) {.dirty.} =
template md5op(op) {.dirty.} =
registerCallback(c, "stdlib.md5." & astToStr(op), `op Wrapper`)

template wrap1f_math(op) {.dirty.} =
template wrap1fMath(op) {.dirty.} =
proc `op Wrapper`(a: VmArgs) {.nimcall.} =
doAssert a.numArgs == 1
setResult(a, op(getFloat(a, 0)))
mathop op

template wrap2f_math(op) {.dirty.} =
template wrap2fMath(op) {.dirty.} =
proc `op Wrapper`(a: VmArgs) {.nimcall.} =
setResult(a, op(getFloat(a, 0), getFloat(a, 1)))
mathop op
Expand Down Expand Up @@ -170,40 +170,40 @@ proc registerAdditionalOps*(c: PCtx) =
proc getProjectPathWrapper(a: VmArgs) =
setResult a, c.config.projectPath.string

wrap1f_math(sqrt)
wrap1f_math(cbrt)
wrap1f_math(ln)
wrap1f_math(log10)
wrap1f_math(log2)
wrap1f_math(exp)
wrap1f_math(arccos)
wrap1f_math(arcsin)
wrap1f_math(arctan)
wrap1f_math(arcsinh)
wrap1f_math(arccosh)
wrap1f_math(arctanh)
wrap2f_math(arctan2)
wrap1f_math(cos)
wrap1f_math(cosh)
wrap2f_math(hypot)
wrap1f_math(sinh)
wrap1f_math(sin)
wrap1f_math(tan)
wrap1f_math(tanh)
wrap2f_math(pow)
wrap1f_math(trunc)
wrap1f_math(floor)
wrap1f_math(ceil)
wrap1f_math(erf)
wrap1f_math(erfc)
wrap1f_math(gamma)
wrap1f_math(lgamma)
wrap1fMath(sqrt)
wrap1fMath(cbrt)
wrap1fMath(ln)
wrap1fMath(log10)
wrap1fMath(log2)
wrap1fMath(exp)
wrap1fMath(arccos)
wrap1fMath(arcsin)
wrap1fMath(arctan)
wrap1fMath(arcsinh)
wrap1fMath(arccosh)
wrap1fMath(arctanh)
wrap2fMath(arctan2)
wrap1fMath(cos)
wrap1fMath(cosh)
wrap2fMath(hypot)
wrap1fMath(sinh)
wrap1fMath(sin)
wrap1fMath(tan)
wrap1fMath(tanh)
wrap2fMath(pow)
wrap1fMath(trunc)
wrap1fMath(floor)
wrap1fMath(ceil)
wrap1fMath(erf)
wrap1fMath(erfc)
wrap1fMath(gamma)
wrap1fMath(lgamma)

when declared(copySign):
wrap2f_math(copySign)
wrap2fMath(copySign)

when declared(signbit):
wrap1f_math(signbit)
wrap1fMath(signbit)

registerCallback c, "stdlib.math.round", proc (a: VmArgs) {.nimcall.} =
let n = a.numArgs
Expand Down
14 changes: 7 additions & 7 deletions compiler/wordrecg.nim
Original file line number Diff line number Diff line change
Expand Up @@ -89,23 +89,23 @@ type
wLiftLocals = "liftlocals", wEnforceNoRaises = "enforceNoRaises",

wAuto = "auto", wBool = "bool", wCatch = "catch", wChar = "char",
wClass = "class", wCompl = "compl", wConst_cast = "const_cast", wDefault = "default",
wDelete = "delete", wDouble = "double", wDynamic_cast = "dynamic_cast",
wClass = "class", wCompl = "compl", wConstCast = "const_cast", wDefault = "default",
wDelete = "delete", wDouble = "double", wDynamicCast = "dynamic_cast",
wExplicit = "explicit", wExtern = "extern", wFalse = "false", wFloat = "float",
wFriend = "friend", wGoto = "goto", wInt = "int", wLong = "long", wMutable = "mutable",
wNamespace = "namespace", wNew = "new", wOperator = "operator", wPrivate = "private",
wProtected = "protected", wPublic = "public", wRegister = "register",
wReinterpret_cast = "reinterpret_cast", wRestrict = "restrict", wShort = "short",
wSigned = "signed", wSizeof = "sizeof", wStatic_cast = "static_cast", wStruct = "struct",
wReinterpretCast = "reinterpret_cast", wRestrict = "restrict", wShort = "short",
wSigned = "signed", wSizeof = "sizeof", wStaticCast = "static_cast", wStruct = "struct",
wSwitch = "switch", wThis = "this", wThrow = "throw", wTrue = "true", wTypedef = "typedef",
wTypeid = "typeid", wTypeof = "typeof", wTypename = "typename",
wUnion = "union", wPacked = "packed", wUnsigned = "unsigned", wVirtual = "virtual",
wVoid = "void", wVolatile = "volatile", wWchar_t = "wchar_t",
wVoid = "void", wVolatile = "volatile", wWchar = "wchar_t",

wAlignas = "alignas", wAlignof = "alignof", wConstexpr = "constexpr", wDecltype = "decltype",
wNullptr = "nullptr", wNoexcept = "noexcept",
wThread_local = "thread_local", wStatic_assert = "static_assert",
wChar16_t = "char16_t", wChar32_t = "char32_t",
wThreadLocal = "thread_local", wStaticAssert = "static_assert",
wChar16 = "char16_t", wChar32 = "char32_t",

wStdIn = "stdin", wStdOut = "stdout", wStdErr = "stderr",

Expand Down
5 changes: 5 additions & 0 deletions config/config.nims
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,9 @@ when defined(nimStrictMode):
switch("hintAsError", "ConvFromXtoItselfNotNeeded")
# future work: XDeclaredButNotUsed

when defined(windows) and not defined(booting):
# Avoid some rare stack corruption while using exceptions with a SEH-enabled
# toolchain: https://github.com/nim-lang/Nim/pull/19197
switch("define", "nimRawSetjmp")

switch("define", "nimVersion:" & NimVersion)
3 changes: 3 additions & 0 deletions config/nim.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,9 @@ nimblepath="$home/.nimble/pkgs/"
# Configuration for the GNU C/C++ compiler:
@if windows:
#gcc.path = r"$nim\dist\mingw\bin"
@if gcc or tcc:
tlsEmulation:on
@end
@end

gcc.maxerrorsimpl = "-fmax-errors=3"
Expand Down
29 changes: 21 additions & 8 deletions doc/manual.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1616,7 +1616,7 @@ type conversions in this context:

myWriteln(stdout, 123, "abc", 4.0)
# is transformed to:
myWriteln(stdout, [$123, $"def", $4.0])
myWriteln(stdout, [$123, $"abc", $4.0])

In this example `$` is applied to any argument that is passed to the
parameter `a`. (Note that `$` applied to strings is a nop.)
Expand Down Expand Up @@ -6488,6 +6488,19 @@ iterator in which case the overloading resolution takes place:
write(stdout, x) # not ambiguous: uses the module C's x


Packages
--------
A collection of modules in a file tree with an ``identifier.nimble`` file in the
root of the tree is called a Nimble package. A valid package name can only be a
valid Nim identifier and thus its filename is ``identifier.nimble`` where
``identifier`` is the desired package name. A module without a ``.nimble`` file
is assigned the package identifier: `unknown`.

The distinction between packages allows diagnostic compiler messages to be
scoped to the current project's package vs foreign packages.



Compiler Messages
=================

Expand Down Expand Up @@ -7123,7 +7136,7 @@ The `link` pragma can be used to link an additional file with the project:
{.link: "myfile.o".}


PassC pragma
passc pragma
------------
The `passc` pragma can be used to pass additional parameters to the C
compiler like one would using the command-line switch `--passc`:option:\:
Expand Down Expand Up @@ -7151,20 +7164,20 @@ the pragma resides in:
{.localPassc: "-Wall -Werror".} # Passed when compiling A.nim.cpp


PassL pragma
passl pragma
------------
The `passL` pragma can be used to pass additional parameters to the linker
like one would be using the command-line switch `--passL`:option:\:
The `passl` pragma can be used to pass additional parameters to the linker
like one would be using the command-line switch `--passl`:option:\:

.. code-block:: Nim
{.passL: "-lSDLmain -lSDL".}
{.passl: "-lSDLmain -lSDL".}

Note that one can use `gorge` from the `system module <system.html>`_ to
embed parameters from an external command that will be executed
during semantic analysis:

.. code-block:: Nim
{.passL: gorge("pkg-config --libs sdl").}
{.passl: gorge("pkg-config --libs sdl").}


Emit pragma
Expand Down Expand Up @@ -7478,7 +7491,7 @@ allows *sloppy* interfacing with libraries written in Objective C:
.. code-block:: Nim
# horrible example of how to interface with GNUStep ...

{.passL: "-lobjc".}
{.passl: "-lobjc".}
{.emit: """
#include <objc/Object.h>
@interface Greeter:Object
Expand Down
4 changes: 2 additions & 2 deletions doc/testament.rst
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ Options
(for debugging)
--failing Only show failing/ignored tests
--targets:"c cpp js objc"
Run tests for specified targets (default: all)
Run tests for specified targets (default: c)
--nim:path Use a particular nim executable (default: $PATH/nim)
--directory:dir Change to directory dir before reading the tests
or doing anything else.
Expand Down Expand Up @@ -164,7 +164,7 @@ Example "template" **to edit** and write a Testament unittest:
# Timeout seconds to run the test. Fractional values are supported.
timeout: 1.5
# Targets to run the test into (c, cpp, objc, js).
# Targets to run the test into (c, cpp, objc, js). Defaults to c.
targets: "c js"
# flags with which to run the test, delimited by `;`
Expand Down
4 changes: 2 additions & 2 deletions koch.nim
Original file line number Diff line number Diff line change
Expand Up @@ -332,9 +332,9 @@ proc boot(args: string) =
# in order to use less memory, we split the build into two steps:
# --compileOnly produces a $project.json file and does not run GCC/Clang.
# jsonbuild then uses the $project.json file to build the Nim binary.
exec "$# $# $# --nimcache:$# $# --compileOnly compiler" / "nim.nim" %
exec "$# $# $# --nimcache:$# $# --noNimblePath --compileOnly compiler" / "nim.nim" %
[nimi, bootOptions, extraOption, smartNimcache, args]
exec "$# jsonscript --nimcache:$# $# compiler" / "nim.nim" %
exec "$# jsonscript --noNimblePath --nimcache:$# $# compiler" / "nim.nim" %
[nimi, smartNimcache, args]

if sameFileContent(output, i.thVersion):
Expand Down
4 changes: 3 additions & 1 deletion lib/core/macros.nim
Original file line number Diff line number Diff line change
Expand Up @@ -1352,7 +1352,9 @@ proc `$`*(node: NimNode): string =
of nnkOpenSymChoice, nnkClosedSymChoice:
result = $node[0]
of nnkAccQuoted:
result = $node[0]
result = ""
for i in 0 ..< node.len:
result.add(repr(node[i]))
else:
badNodeKind node, "$"

Expand Down
23 changes: 11 additions & 12 deletions lib/impure/db_mysql.nim
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ when false:
discard mysql_stmt_close(stmt)

proc dbQuote*(s: string): string =
## DB quotes the string.
## DB quotes the string. Note that this doesn't escape `%` and `_`.
result = newStringOfCap(s.len + 2)
result.add "'"
for c in items(s):
Expand All @@ -132,7 +132,6 @@ proc dbQuote*(s: string): string =
of '"': result.add "\\\""
of '\'': result.add "\\'"
of '\\': result.add "\\\\"
of '_': result.add "\\_"
else: result.add c
add(result, '\'')

Expand All @@ -150,25 +149,25 @@ proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {.
tags: [ReadDbEffect, WriteDbEffect].} =
## tries to execute the query and returns true if successful, false otherwise.
var q = dbFormat(query, args)
return mysql.realQuery(PMySQL db, q, q.len) == 0'i32
return mysql.real_query(PMySQL db, q, q.len) == 0'i32

proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) =
var q = dbFormat(query, args)
if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db)
if mysql.real_query(PMySQL db, q, q.len) != 0'i32: dbError(db)

proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
tags: [ReadDbEffect, WriteDbEffect].} =
## executes the query and raises EDB if not successful.
var q = dbFormat(query, args)
if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db)
if mysql.real_query(PMySQL db, q, q.len) != 0'i32: dbError(db)

proc newRow(L: int): Row =
newSeq(result, L)
for i in 0..L-1: result[i] = ""

proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) =
if row != nil:
while mysql.fetchRow(sqlres) != nil: discard
while mysql.fetch_row(sqlres) != nil: discard
mysql.freeResult(sqlres)

iterator fastRows*(db: DbConn, query: SqlQuery,
Expand All @@ -190,7 +189,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
backup: Row
newSeq(result, L)
while true:
row = mysql.fetchRow(sqlres)
row = mysql.fetch_row(sqlres)
if row == nil: break
for i in 0..L-1:
setLen(result[i], 0)
Expand All @@ -209,7 +208,7 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
let L = int(mysql.numFields(sqlres))
var row: cstringArray
while true:
row = mysql.fetchRow(sqlres)
row = mysql.fetch_row(sqlres)
if row == nil: break
yield InstantRow(row: row, len: L)
properFreeResult(sqlres, row)
Expand Down Expand Up @@ -290,7 +289,7 @@ iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery;
setColumnInfo(columns, sqlres, L)
var row: cstringArray
while true:
row = mysql.fetchRow(sqlres)
row = mysql.fetch_row(sqlres)
if row == nil: break
yield InstantRow(row: row, len: L)
properFreeResult(sqlres, row)
Expand All @@ -317,7 +316,7 @@ proc getRow*(db: DbConn, query: SqlQuery,
if sqlres != nil:
var L = int(mysql.numFields(sqlres))
result = newRow(L)
var row = mysql.fetchRow(sqlres)
var row = mysql.fetch_row(sqlres)
if row != nil:
for i in 0..L-1:
setLen(result[i], 0)
Expand All @@ -335,7 +334,7 @@ proc getAllRows*(db: DbConn, query: SqlQuery,
var row: cstringArray
var j = 0
while true:
row = mysql.fetchRow(sqlres)
row = mysql.fetch_row(sqlres)
if row == nil: break
setLen(result, j+1)
newSeq(result[j], L)
Expand All @@ -361,7 +360,7 @@ proc tryInsertId*(db: DbConn, query: SqlQuery,
## executes the query (typically "INSERT") and returns the
## generated ID for the row or -1 in case of an error.
var q = dbFormat(query, args)
if mysql.realQuery(PMySQL db, q, q.len) != 0'i32:
if mysql.real_query(PMySQL db, q, q.len) != 0'i32:
result = -1'i64
else:
result = mysql.insertId(PMySQL db)
Expand Down
7 changes: 6 additions & 1 deletion lib/packages/docutils/rst.nim
Original file line number Diff line number Diff line change
Expand Up @@ -1864,11 +1864,16 @@ proc countTitles(s: PRstSharedState, n: PRstNode) =
if s.hTitleCnt >= 2:
break

proc isMarkdownCodeBlock(p: RstParser, idx: int): bool =
roSupportMarkdown in p.s.options and p.tok[idx].symbol == "```"

proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool =
## check that underline/overline length is enough for the heading.
## No support for Unicode.
if p.tok[adornmentIdx].symbol in ["::", "..", "|"]:
return false
if isMarkdownCodeBlock(p, adornmentIdx):
return false
var headlineLen = 0
var failure = ""
if p.idx < adornmentIdx: # check for underline
Expand Down Expand Up @@ -1946,7 +1951,7 @@ proc findPipe(p: RstParser, start: int): bool =
proc whichSection(p: RstParser): RstNodeKind =
if currentTok(p).kind in {tkAdornment, tkPunct}:
# for punctuation sequences that can be both tkAdornment and tkPunct
if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```":
if isMarkdownCodeBlock(p, p.idx):
return rnCodeBlock
elif currentTok(p).symbol == "::":
return rnLiteralBlock
Expand Down
19 changes: 14 additions & 5 deletions lib/posix/epoll.nim
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,28 @@ const
EPOLL_CTL_DEL* = 2 # Remove a file descriptor from the interface.
EPOLL_CTL_MOD* = 3 # Change file descriptor epoll_event structure.

# https://github.com/torvalds/linux/blob/ff6992735ade75aae3e35d16b17da1008d753d28/include/uapi/linux/eventpoll.h#L77
when defined(linux) and defined(amd64):
{.pragma: epollPacked, packed.}
else:
{.pragma: epollPacked.}

type
EpollData* {.importc: "union epoll_data",
header: "<sys/epoll.h>", pure, final.} = object # TODO: This is actually a union.
EpollData* {.importc: "epoll_data_t",
header: "<sys/epoll.h>", pure, final, union.} = object
`ptr`* {.importc: "ptr".}: pointer
fd* {.importc: "fd".}: cint
u32* {.importc: "u32".}: uint32
u64* {.importc: "u64".}: uint64

EpollEvent* {.importc: "struct epoll_event", header: "<sys/epoll.h>", pure, final.} = object
EpollEvent* {.importc: "struct epoll_event", header: "<sys/epoll.h>", pure, final, epollPacked.} = object
events*: uint32 # Epoll events
data*: EpollData # User data variable

proc epoll_create*(size: cint): cint {.importc: "epoll_create",
header: "<sys/epoll.h>".}
## Creates an epoll instance. Returns an fd for the new instance.
##
##
## The "size" parameter is a hint specifying the number of file
## descriptors to be associated with the new instance. The fd
## returned by epoll_create() should be closed with close().
Expand All @@ -59,7 +68,7 @@ proc epoll_ctl*(epfd: cint; op: cint; fd: cint | SocketHandle; event: ptr EpollE
importc: "epoll_ctl", header: "<sys/epoll.h>".}
## Manipulate an epoll instance "epfd". Returns `0` in case of success,
## `-1` in case of error (the "errno" variable will contain the specific error code).
##
##
## The "op" parameter is one of the `EPOLL_CTL_*`
## constants defined above. The "fd" parameter is the target of the
## operation. The "event" parameter describes which events the caller
Expand Down
2 changes: 1 addition & 1 deletion lib/posix/posix.nim
Original file line number Diff line number Diff line change
Expand Up @@ -1012,7 +1012,7 @@ proc endhostent*() {.importc, header: "<netdb.h>".}
proc endnetent*() {.importc, header: "<netdb.h>".}
proc endprotoent*() {.importc, header: "<netdb.h>".}
proc endservent*() {.importc, header: "<netdb.h>".}
proc freeaddrinfo*(a1: ptr AddrInfo) {.importc, header: "<netdb.h>".}
proc freeAddrInfo*(a1: ptr AddrInfo) {.importc: "freeaddrinfo", header: "<netdb.h>".}

proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "<netdb.h>".}

Expand Down
2 changes: 1 addition & 1 deletion lib/posix/posix_linux_amd64.nim
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ type
Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = cuint
Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>",
pure, final.} = object
abi: array[48 div sizeof(clong), clong]
abi: array[40 div sizeof(clong), clong]
Pthread_mutexattr* {.importc: "pthread_mutexattr_t",
header: "<sys/types.h>", pure, final.} = object
abi: array[4 div sizeof(cint), cint]
Expand Down
6 changes: 3 additions & 3 deletions lib/pure/asyncdispatch.nim
Original file line number Diff line number Diff line change
Expand Up @@ -1791,7 +1791,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped,
curAddrInfo = curAddrInfo.ai_next

if curAddrInfo == nil:
freeaddrinfo(addrInfo)
freeAddrInfo(addrInfo)
when shouldCreateFd:
closeUnusedFds()
if lastException != nil:
Expand All @@ -1807,7 +1807,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped,
try:
curFd = createAsyncNativeSocket(domain, sockType, protocol)
except:
freeaddrinfo(addrInfo)
freeAddrInfo(addrInfo)
closeUnusedFds()
raise getCurrentException()
when defined(windows):
Expand All @@ -1817,7 +1817,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped,
doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo
curAddrInfo = curAddrInfo.ai_next
else:
freeaddrinfo(addrInfo)
freeAddrInfo(addrInfo)
when shouldCreateFd:
closeUnusedFds(ord(domain))
retFuture.complete(curFd)
Expand Down
6 changes: 3 additions & 3 deletions lib/pure/asyncnet.nim
Original file line number Diff line number Diff line change
Expand Up @@ -648,9 +648,9 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {.

var aiList = getAddrInfo(realaddr, port, socket.domain)
if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32:
freeaddrinfo(aiList)
freeAddrInfo(aiList)
raiseOSError(osLastError())
freeaddrinfo(aiList)
freeAddrInfo(aiList)

proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} =
## Determines whether an AsyncSocket has data buffered.
Expand Down Expand Up @@ -859,7 +859,7 @@ proc sendTo*(socket: AsyncSocket, address: string, port: Port, data: string,

it = it.ai_next

freeaddrinfo(aiList)
freeAddrInfo(aiList)

if not success:
if lastException != nil:
Expand Down
2 changes: 1 addition & 1 deletion lib/pure/collections/sequtils.nim
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,7 @@ template foldl*(sequence, operation, first): untyped =
##
## The `operation` parameter should be an expression which uses the variables
## `a` and `b` for each step of the fold. The `first` parameter is the
## start value (the first `a`) and therefor defines the type of the result.
## start value (the first `a`) and therefore defines the type of the result.
##
## **See also:**
## * `foldr template<#foldr.t,untyped,untyped>`_
Expand Down
2 changes: 1 addition & 1 deletion lib/pure/cookies.nim
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,4 @@ proc setCookie*(key, value: string, expires: DateTime|Time,
## `Set-Cookie: key=value; Domain=...; ...`
result = setCookie(key, value, domain, path,
format(expires.utc, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"),
noname, secure, httpOnly, maxAge, sameSite)
noName, secure, httpOnly, maxAge, sameSite)
2 changes: 1 addition & 1 deletion lib/pure/ioselects/ioselectors_epoll.nim
Original file line number Diff line number Diff line change
Expand Up @@ -526,4 +526,4 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1,
body2

proc getFd*[T](s: Selector[T]): int =
return s.epollFd.int
return s.epollFD.int
16 changes: 8 additions & 8 deletions lib/pure/ioselects/ioselectors_select.nim
Original file line number Diff line number Diff line change
Expand Up @@ -26,27 +26,27 @@ else:
#include <sys/types.h>
#include <unistd.h>"""
type
Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object
FdSet {.importc: "fd_set", header: platformHeaders, pure, final.} = object
var
FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint

proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset)
proc IOFD_SET(fd: SocketHandle, fdset: ptr FdSet)
{.cdecl, importc: "FD_SET", header: platformHeaders, inline.}
proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset)
proc IOFD_CLR(fd: SocketHandle, fdset: ptr FdSet)
{.cdecl, importc: "FD_CLR", header: platformHeaders, inline.}
proc IOFD_ZERO(fdset: ptr Fdset)
proc IOFD_ZERO(fdset: ptr FdSet)
{.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.}

when defined(windows):
proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint
proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint
{.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.}
proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset,
proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet,
timeout: ptr Timeval): cint
{.stdcall, importc: "select", header: platformHeaders.}
else:
proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint
proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint
{.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.}
proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset,
proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet,
timeout: ptr Timeval): cint
{.cdecl, importc: "select", header: platformHeaders.}

Expand Down
12 changes: 6 additions & 6 deletions lib/pure/json.nim
Original file line number Diff line number Diff line change
Expand Up @@ -955,12 +955,12 @@ proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats

when defined(js):
from math import `mod`
from std/jsffi import JSObject, `[]`, to
from std/jsffi import JsObject, `[]`, to
from std/private/jsutils import getProtoName, isInteger, isSafeInteger

proc parseNativeJson(x: cstring): JSObject {.importjs: "JSON.parse(#)".}
proc parseNativeJson(x: cstring): JsObject {.importjs: "JSON.parse(#)".}

proc getVarType(x: JSObject, isRawNumber: var bool): JsonNodeKind =
proc getVarType(x: JsObject, isRawNumber: var bool): JsonNodeKind =
result = JNull
case $getProtoName(x) # TODO: Implicit returns fail here.
of "[object Array]": return JArray
Expand All @@ -979,12 +979,12 @@ when defined(js):
of "[object String]": return JString
else: assert false

proc len(x: JSObject): int =
proc len(x: JsObject): int =
asm """
`result` = `x`.length;
"""

proc convertObject(x: JSObject): JsonNode =
proc convertObject(x: JsObject): JsonNode =
var isRawNumber = false
case getVarType(x, isRawNumber)
of JArray:
Expand All @@ -997,7 +997,7 @@ when defined(js):
if (`x`.hasOwnProperty(property)) {
"""
var nimProperty: cstring
var nimValue: JSObject
var nimValue: JsObject
asm "`nimProperty` = property; `nimValue` = `x`[property];"
result[$nimProperty] = nimValue.convertObject()
asm "}}"
Expand Down
4 changes: 2 additions & 2 deletions lib/pure/nativesockets.nim
Original file line number Diff line number Diff line change
Expand Up @@ -633,8 +633,8 @@ when useNimNetLite:
INET_ADDRSTRLEN = 16
INET6_ADDRSTRLEN = 46 # it's actually 46 in both cases

proc sockAddrToStr(sa: ptr Sockaddr): string {.noinit.} =
let af_family = sa.sa_family
proc sockAddrToStr(sa: ptr SockAddr): string {.noinit.} =
let af_family = sa.sa_family
var nl, v4Slice: cint
var si_addr: ptr InAddr

Expand Down
53 changes: 35 additions & 18 deletions lib/pure/net.nim
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,30 @@ type
when defined(nimHasStyleChecks):
{.pop.}


when defined(posix) and not defined(lwip):
from posix import TPollfd, POLLIN, POLLPRI, POLLOUT, POLLWRBAND, Tnfds

template monitorPollEvent(x: var SocketHandle, y: cint, timeout: int): int =
var tpollfd: TPollfd
tpollfd.fd = cast[cint](x)
tpollfd.events = y
posix.poll(addr(tpollfd), Tnfds(1), timeout)

proc timeoutRead(fd: var SocketHandle, timeout = 500): int =
when defined(windows) or defined(lwip):
var fds = @[fd]
selectRead(fds, timeout)
else:
monitorPollEvent(fd, POLLIN or POLLPRI, timeout)

proc timeoutWrite(fd: var SocketHandle, timeout = 500): int =
when defined(windows) or defined(lwip):
var fds = @[fd]
selectWrite(fds, timeout)
else:
monitorPollEvent(fd, POLLOUT or POLLWRBAND, timeout)

proc socketError*(socket: Socket, err: int = -1, async = false,
lastError = (-1).OSErrorCode,
flags: set[SocketFlag] = {}) {.gcsafe.}
Expand Down Expand Up @@ -984,11 +1008,11 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {.

var aiList = getAddrInfo(realaddr, port, socket.domain)
if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32:
freeaddrinfo(aiList)
freeAddrInfo(aiList)
var address2: string
address2.addQuoted address
raiseOSError(osLastError(), "address: $# port: $#" % [address2, $port])
freeaddrinfo(aiList)
freeAddrInfo(aiList)

proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string,
flags = {SocketFlag.SafeDisconn},
Expand Down Expand Up @@ -1308,14 +1332,6 @@ proc hasDataBuffered*(s: Socket): bool =
if s.isSsl and not result:
result = s.sslHasPeekChar

proc select(readfd: Socket, timeout = 500): int =
## Used for socket operation timeouts.
if readfd.hasDataBuffered:
return 1

var fds = @[readfd.fd]
result = selectRead(fds, timeout)

proc isClosed(socket: Socket): bool =
socket.fd == osInvalidSocket

Expand Down Expand Up @@ -1429,7 +1445,9 @@ proc waitFor(socket: Socket, waited: var Duration, timeout, size: int,
return min(sslPending, size)

var startTime = getMonoTime()
let selRet = select(socket, (timeout - waited.inMilliseconds).int)
let selRet = if socket.hasDataBuffered: 1
else:
timeoutRead(socket.fd, (timeout - waited.inMilliseconds).int)
if selRet < 0: raiseOSError(osLastError())
if selRet != 1:
raise newException(TimeoutError, "Call to '" & funcName & "' timed out.")
Expand Down Expand Up @@ -1790,7 +1808,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer,
it = it.ai_next

let osError = osLastError()
freeaddrinfo(aiList)
freeAddrInfo(aiList)

if not success:
raiseOSError(osError)
Expand Down Expand Up @@ -2006,7 +2024,7 @@ proc dial*(address: string, port: Port,
# network system problem (e.g. not enough FDs), and not an unreachable
# address.
let err = osLastError()
freeaddrinfo(aiList)
freeAddrInfo(aiList)
closeUnusedFds()
raiseOSError(err)
fdPerDomain[ord(domain)] = lastFd
Expand All @@ -2015,7 +2033,7 @@ proc dial*(address: string, port: Port,
break
lastError = osLastError()
it = it.ai_next
freeaddrinfo(aiList)
freeAddrInfo(aiList)
closeUnusedFds(ord(domain))

if success:
Expand Down Expand Up @@ -2045,7 +2063,7 @@ proc connect*(socket: Socket, address: string,
else: lastError = osLastError()
it = it.ai_next

freeaddrinfo(aiList)
freeAddrInfo(aiList)
if not success: raiseOSError(lastError)

when defineSsl:
Expand Down Expand Up @@ -2097,7 +2115,7 @@ proc connectAsync(socket: Socket, name: string, port = Port(0),

it = it.ai_next

freeaddrinfo(aiList)
freeAddrInfo(aiList)
if not success: raiseOSError(lastError)

proc connect*(socket: Socket, address: string, port = Port(0),
Expand All @@ -2109,8 +2127,7 @@ proc connect*(socket: Socket, address: string, port = Port(0),
socket.fd.setBlocking(false)

socket.connectAsync(address, port, socket.domain)
var s = @[socket.fd]
if selectWrite(s, timeout) != 1:
if timeoutWrite(socket.fd, timeout) != 1:
raise newException(TimeoutError, "Call to 'connect' timed out.")
else:
let res = getSockOptInt(socket.fd, SOL_SOCKET, SO_ERROR)
Expand Down
Loading