Skip to content

Commit

Permalink
Introduce pedantic mode for tester
Browse files Browse the repository at this point in the history
Motivation
----------
External tools need to know whether or not any test in suite failed. For
example buildbot, or packaging tool would like to stop the execution and
mark it as failed if some tests does not pass.

Modification
------------
Add `--pedantic` switch to tester program which will force it to quit
with non-zero exit code if at least one failure detected. Also update
`tests()` proc in koch to inspect result from tester and propagate it to
user.

Result
------
Nothing has changed in default behaviour. But following invocations will
exit with non-zero code if there failed tests:

    ./koch tests --pedantic all
    ./tests/testament/tester --pedantic all
  • Loading branch information
avsej committed May 25, 2015
1 parent 71561be commit e00fb7c
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 2 deletions.
4 changes: 3 additions & 1 deletion koch.nim
Expand Up @@ -333,8 +333,10 @@ proc tests(args: string) =
# taint mode test :-)
exec "nim cc --taintMode:on tests/testament/tester"
let tester = quoteShell(getCurrentDir() / "tests/testament/tester".exe)
exec tester & " " & (args|"all")
let success = tryExec tester & " " & (args|"all")
exec tester & " html"
if not success:
quit("tests failed", QuitFailure)

proc temp(args: string) =
var output = "compiler" / "nim".exe
Expand Down
7 changes: 6 additions & 1 deletion tests/testament/tester.nim
Expand Up @@ -31,6 +31,7 @@ Arguments:
Options:
--print also print results to the console
--failing only show failing/ignored tests
--pedantic return non-zero status code if there are failures
""" % resultsFile

type
Expand Down Expand Up @@ -310,12 +311,14 @@ proc main() =
backend.open()
var optPrintResults = false
var optFailing = false
var optPedantic = false
var p = initOptParser()
p.next()
while p.kind == cmdLongoption:
case p.key.string.normalize
of "print", "verbose": optPrintResults = true
of "failing": optFailing = true
of "pedantic": optPedantic = true
else: quit Usage
p.next()
if p.kind != cmdArgument: quit Usage
Expand Down Expand Up @@ -348,8 +351,10 @@ proc main() =
if action == "html": openDefaultBrowser(resultsFile)
else: echo r, r.data
backend.close()
if optPedantic:
var failed = r.total - r.passed - r.skipped
if failed > 0 : quit(failed)

if paramCount() == 0:
quit Usage
main()

0 comments on commit e00fb7c

Please sign in to comment.