Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Test runner improvements #124

Merged
merged 6 commits into from

3 participants

@justindujardin
Collaborator

The Problem
As a user of Docco, I want to be able to have the test suite print more useful success/failure information, and proper process return codes.

The Solution
Update the test runner to wrap the node js assert module properly, remove custom equality code, aggregate and report errors without failure.

Testing
Continuous integration on my fork verifies that proper process return codes are being supplied, and visual inspection of test output verifies more detailed information is reported.

 --------------------------------------------------------
  Testing completed in 1.27 seconds
    4 tests passed, 0 failed
    31 asserts passed, 0 failed
 --------------------------------------------------------
@keithamus keithamus merged commit ea9049a into from
@justindujardin
Collaborator

:thumbsup: :)

@keithamus
Collaborator

I think @jashkenas needs to actually enable Docco builds in Travis, but this is something I'd definitely push for ;)

@justindujardin
Collaborator

Yeah, I guess that is up to him, really. I know some people that dislike CI builds because they can produce email spam sometimes, and since it's his repo, he'd get the emails. I can still take advantage of it, so it's all good.

@jashkenas
Owner

I'm really not a fan of Travis -- because it's way overkill for small + simple projects like this one, and because of false negatives. So I'd rather leave it out, unless y'all really, really want it.

@keithamus
Collaborator

I agree that the project is really small, but I don't think that detracts from the usefulness of having Travis green light PRs.

@justindujardin
Collaborator

Like I said, I'm satisfied to just have it running on my fork.

@keithamus The nice thing is that Travis will green light PRs if you have it running on your own fork. e.g. my PRs to the official fork will show green status, because my fork uses Travis. It's not needed on the official one for that!

@keithamus
Collaborator

@justindujardin yeah that is really nifty. I'd be sweet if we could get everyone's PRs running on Travis though, especially as not everyone will enable travis for their forks.

@justindujardin
Collaborator

We could try adding some text to the README about development practices, and recommend that people use it. Travis has a pretty streamlined integration, it's literally a few clicks away, as it uses your github user account, and configures the post-service hook for you. It's also totally free. I don't think I had to enter any information to get it running on my fork.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
Showing with 58 additions and 47 deletions.
  1. +47 −37 Cakefile
  2. +11 −10 test/tests.coffee
View
84 Cakefile
@@ -33,68 +33,77 @@ task 'doc', 'rebuild the Docco documentation', ->
)
task 'test', 'run the Docco test suite', ->
- runTests Docco
+ runTests()
-# Simple test runner, borrowed from [CoffeeScript](http://coffeescript.org/).
-runTests = (Docco) ->
- startTime = Date.now()
- currentFile = null
- passedTests = 0
- failures = []
+# Simple test runner, adapted from [CoffeeScript](http://coffeescript.org/).
+runTests = () ->
+ startTime = Date.now()
+ currentFile = null
+ currentTest = null
+ currentSource = null
+ passedTests = 0
+ passedAssert = 0
+ failedAssert = 0
+ failures = []
+ done = false
- global[name] = func for name, func of require 'assert'
+ # Wrap each assert function in a try/catch block to report passed/failed assertions.
+ wrapAssert = (func,name) ->
+ return () ->
+ try
+ result = func.apply this, arguments
+ ++passedAssert
+ catch e
+ ++failedAssert
+ e.description = arguments[2] if arguments.length == 3
+ e.source = currentSource
+ e.testName = currentTest
+ failures.push
+ filename: currentFile
+ error: e
+ result
- # Convenience alias.
+ global[name] = wrapAssert(func,name) for name, func of require 'assert'
global.Docco = Docco
# Our test helper function for delimiting different test cases.
global.test = (description, fn) ->
try
fn.test = {description, currentFile}
+ currentTest = description
+ currentSource = fn.toString() if fn.toString?
fn.call(fn)
++passedTests
catch e
+ e.testName = currentTest
e.description = description if description?
e.source = fn.toString() if fn.toString?
failures.push filename: currentFile, error: e
-
- # See http://wiki.ecmascript.org/doku.php?id=harmony:egal
- egal = (a, b) ->
- if a is b
- a isnt 0 or 1/a is 1/b
- else
- a isnt a and b isnt b
-
- # A recursive functional equivalence helper; uses egal for testing equivalence.
- arrayEgal = (a, b) ->
- if egal a, b then yes
- else if a instanceof Array and b instanceof Array
- return no unless a.length is b.length
- return no for el, idx in a when not arrayEgal el, b[idx]
- yes
-
- global.eq = (a, b, msg) -> ok egal(a, b), msg
- global.arrayEq = (a, b, msg) -> ok arrayEgal(a,b), msg
# When all the tests have run, collect and print errors.
# If a stacktrace is available, output the compiled function source.
process.on 'exit', ->
+ return if done
+ done = true
time = ((Date.now() - startTime) / 1000).toFixed(2)
- message = "passed #{passedTests} tests in #{time} seconds"
- return console.log(message) unless failures.length
- console.log "failed #{failures.length} and #{message}"
for fail in failures
{error, filename} = fail
jsFilename = filename.replace(/\.coffee$/,'.js')
- match = error.stack?.match(new RegExp(fail.file+":(\\d+):(\\d+)"))
+ match = error.stack?.match(new RegExp(fail.filename+":(\\d+):(\\d+)"))
match = error.stack?.match(/on line (\d+):/) unless match
[match, line, col] = match if match
- console.log ''
- console.log " #{error.description}"
- console.log " #{error.stack}"
- console.log " #{jsFilename}: line #{line ? 'unknown'}, column #{col ? 'unknown'}"
- console.log " #{error.source}" if error.source
- return
+ console.log "\n--------------------------------------------------------"
+ console.log " FAILED: #{error.testName}\n" if error.testName
+ console.log " FILE :#{jsFilename}: line #{line ? 'unknown'}, column #{col ? 'unknown'}"
+ console.log " ERROR : #{error.description}" if error.description
+ console.log " STACK : #{error.stack}" if error.stack
+ console.log " SOURCE: #{error.source}" if error.source
+ console.log "--------------------------------------------------------"
+ console.log "Testing completed in #{time} seconds"
+ console.log " #{passedTests} tests passed, #{failures.length} failed"
+ console.log " #{passedAssert} asserts passed, #{failedAssert} failed"
+ console.log "--------------------------------------------------------"
+ process.exit if failures.length > 0 then 1 else 0
# Run every test in the `test` folder, recording failures.
files = fs.readdirSync 'test'
@@ -104,5 +113,6 @@ runTests = (Docco) ->
try
CoffeeScript.run code.toString(), {filename}
catch error
+ error.description = currentTest
failures.push {filename, error}
return !failures.length
View
21 test/tests.coffee
@@ -14,14 +14,14 @@ testDoccoRun = (testName,sources,options=null,callback=null) ->
destPath = path.join dataPath, testName
cleanup = (callback) -> rimraf destPath, callback
cleanup (error) ->
- eq not error, true, "path cleaned up properly"
+ equal not error, true, "path cleaned up properly"
options?.output = destPath
Docco.document sources, options, ->
files = []
files = files.concat(Docco.resolveSource(src)) for src in sources
expected = files.length + 1
found = fs.readdirSync(destPath).length
- eq found, expected, "find expected output (#{expected} files) - (#{found})"
+ equal found, expected, "find expected output (#{expected} files) - (#{found})"
callback() if callback?
# **Custom jst template files should be supported**
@@ -63,16 +63,16 @@ test "single line comment parsing", ->
return testNextLanguage(keys, callback) if not path.existsSync languageExample
testDoccoRun languageTest, [languageExample], options, ->
- eq true, path.existsSync(languageOutput), "#{languageOutput} -> output file created properly"
+ equal true, path.existsSync(languageOutput), "#{languageOutput} -> output file created properly"
content = fs.readFileSync(languageOutput).toString()
comments = (c.trim() for c in content.split(',') when c.trim() != '')
- eq true, comments.length >= 1, 'expect at least the descriptor comment'
+ equal true, comments.length >= 1, 'expect at least the descriptor comment'
expected = parseInt(comments[0])
- eq comments.length, expected, [
+ equal comments.length, expected, [
""
"#{path.basename(languageOutput)} comments"
"------------------------"
@@ -84,7 +84,7 @@ test "single line comment parsing", ->
# *Kick off the first language test*
testNextLanguage languageKeys.slice()
-
+
# **URL references should resolve across sections**
#
# Resolves [Issue 100](https://github.com/jashkenas/docco/issues/100)
@@ -101,11 +101,12 @@ test "url references", ->
outPath = path.join dataPath, "_urlreferences"
outFile = "#{outPath}/_urlref.html"
rimraf outPath, (error) ->
- eq not error, true
+ equal not error, true
Docco.document [sourceFile], output: outPath, ->
contents = fs.readFileSync(outFile).toString()
count = contents.match ///<a\shref="http://www.google.com">Google</a>///g
- eq count.length, 2, "find expected (2) resolved url references"
+ equal count.length, 2, "find expected (2) resolved url references"
+
# **Paths should be recursively created if needed**
#
# ensureDirectory should properly create complex output paths.
@@ -113,8 +114,8 @@ test "create complex paths that do not exist", ->
exist = fs.existsSync or path.existsSync
outputPath = path.join dataPath, 'complex/path/that/doesnt/exist'
rimraf outputPath, (error) ->
- eq not error, true
+ equal not error, true
Docco.ensureDirectory outputPath, ->
equal exist(outputPath), true, 'created output path'
stat = fs.statSync outputPath
- equal stat.isDirectory(), true, "target is directory"
+ equal stat.isDirectory(), true, "target is directory"
Something went wrong with that request. Please try again.