Skip to content

Commit

Permalink
[New] show full error stack on failure
Browse files Browse the repository at this point in the history
  • Loading branch information
Tristan Davies authored and ljharb committed Dec 4, 2016
1 parent 5ec88e7 commit 9302682
Show file tree
Hide file tree
Showing 13 changed files with 295 additions and 29 deletions.
9 changes: 7 additions & 2 deletions lib/results.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
var defined = require('defined');
var EventEmitter = require('events').EventEmitter;
var inherits = require('inherits');
var through = require('through');
Expand Down Expand Up @@ -157,8 +158,12 @@ function encodeResult (res, count) {
if (res.at) {
output += inner + 'at: ' + res.at + '\n';
}
if (res.operator === 'error' && res.actual && res.actual.stack) {
var lines = String(res.actual.stack).split('\n');

var actualStack = res.actual && res.actual.stack;
var errorStack = res.error && res.error.stack;
var stack = defined(actualStack, errorStack);
if (stack) {
var lines = String(stack).split('\n');
output += inner + 'stack: |-\n';
for (var i = 0; i < lines.length; i++) {
output += inner + ' ' + lines[i] + '\n';
Expand Down
9 changes: 8 additions & 1 deletion test/circular-things.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@ var tape = require('../');
var tap = require('tap');
var concat = require('concat-stream');

var stripFullStack = require('./common').stripFullStack;

tap.test('circular test', function (assert) {
var test = tape.createHarness({ exit : false });
assert.plan(1);

test.createStream().pipe(concat(function (body) {
assert.equal(
body.toString('utf8'),
stripFullStack(body.toString('utf8')),
'TAP version 13\n'
+ '# circular\n'
+ 'not ok 1 should be equal\n'
Expand All @@ -18,6 +20,11 @@ tap.test('circular test', function (assert) {
+ ' {}\n'
+ ' actual: |-\n'
+ ' { circular: [Circular] }\n'
+ ' stack: |-\n'
+ ' Error: should be equal\n'
+ ' [... stack stripped ...]\n'
+ ' at Test.<anonymous> ($TEST/circular-things.js:$LINE:$COL)\n'
+ ' [... stack stripped ...]\n'
+ ' ...\n'
+ '\n'
+ '1..1\n'
Expand Down
56 changes: 56 additions & 0 deletions test/common.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
var yaml = require('js-yaml');

module.exports.getDiag = function (body) {
var yamlStart = body.indexOf(' ---');
var yamlEnd = body.indexOf(' ...\n');
var diag = body.slice(yamlStart, yamlEnd).split('\n').map(function (line) {
return line.slice(2);
}).join('\n');

// The stack trace will vary depending on where the code is run, so just
// strip it out.
var withStack = yaml.safeLoad(diag);
delete withStack.stack;
return withStack;
}

// There are three challenges associated with checking the stack traces included
// in errors:
// 1) The base checkout directory of tape might change. Because stack traces
// include absolute paths, the stack traces will change depending on the
// checkout path. We handle this by replacing the base test directory with a
// placeholder $TEST variable.
// 2) Line positions within the file might change. We handle this by replacing
// line and column markers with placeholder $LINE and $COL "variables"
// 3) Stacks themselves change frequently with refactoring. We've even run into
// issues with node library refactorings "breaking" stack traces. Most of
// these changes are irrelevant to the tests themselves. To counter this, we
// strip out all stack frames that aren't directly under our test directory,
// and replace them with placeholders.
module.exports.stripFullStack = function (output) {
var stripped = ' [... stack stripped ...]';
var withDuplicates = output.split('\n').map(function (line) {
var m = line.match(/[ ]{8}at .*\((.*)\)/);

var stripChangingData = function (line) {
var withoutDirectory = line.replace(__dirname, '$TEST');
var withoutLineNumbers = withoutDirectory.replace(/:\d+:\d+/g, ':$LINE:$COL');
return withoutLineNumbers;
}

if (m) {
if (m[1].slice(0, __dirname.length) === __dirname) {
return stripChangingData(line);
}
return stripped;
}
return stripChangingData(line);
})

var deduped = withDuplicates.filter(function (line, ix) {
var hasPrior = line === stripped && withDuplicates[ix - 1] === stripped;
return !hasPrior;
});

return deduped.join('\n');
}
39 changes: 25 additions & 14 deletions test/deep-equal-failure.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ var tape = require('../');
var tap = require('tap');
var concat = require('concat-stream');
var tapParser = require('tap-parser');
var yaml = require('js-yaml');
var common = require('./common');

var getDiag = common.getDiag;
var stripFullStack = common.stripFullStack;

tap.test('deep equal failure', function (assert) {
var test = tape.createHarness({ exit : false });
Expand All @@ -13,7 +16,7 @@ tap.test('deep equal failure', function (assert) {
stream.pipe(parser);
stream.pipe(concat(function (body) {
assert.equal(
body.toString('utf8'),
stripFullStack(body.toString('utf8')),
'TAP version 13\n'
+ '# deep equal\n'
+ 'not ok 1 should be equal\n'
Expand All @@ -23,6 +26,11 @@ tap.test('deep equal failure', function (assert) {
+ ' { b: 2 }\n'
+ ' actual: |-\n'
+ ' { a: 1 }\n'
+ ' stack: |-\n'
+ ' Error: should be equal\n'
+ ' [... stack stripped ...]\n'
+ ' at Test.<anonymous> ($TEST/deep-equal-failure.js:$LINE:$COL)\n'
+ ' [... stack stripped ...]\n'
+ ' ...\n'
+ '\n'
+ '1..1\n'
Expand All @@ -39,6 +47,7 @@ tap.test('deep equal failure', function (assert) {
}));

parser.once('assert', function (data) {
delete data.diag.stack;
assert.deepEqual(data, {
ok: false,
id: 1,
Expand Down Expand Up @@ -66,7 +75,7 @@ tap.test('deep equal failure, depth 6, with option', function (assert) {
stream.pipe(parser);
stream.pipe(concat(function (body) {
assert.equal(
body.toString('utf8'),
stripFullStack(body.toString('utf8')),
'TAP version 13\n'
+ '# deep equal\n'
+ 'not ok 1 should be equal\n'
Expand All @@ -76,6 +85,11 @@ tap.test('deep equal failure, depth 6, with option', function (assert) {
+ ' { a: { a1: { a2: { a3: { a4: { a5: 2 } } } } } }\n'
+ ' actual: |-\n'
+ ' { a: { a1: { a2: { a3: { a4: { a5: 1 } } } } } }\n'
+ ' stack: |-\n'
+ ' Error: should be equal\n'
+ ' [... stack stripped ...]\n'
+ ' at Test.<anonymous> ($TEST/deep-equal-failure.js:$LINE:$COL)\n'
+ ' [... stack stripped ...]\n'
+ ' ...\n'
+ '\n'
+ '1..1\n'
Expand All @@ -92,6 +106,7 @@ tap.test('deep equal failure, depth 6, with option', function (assert) {
}));

parser.once('assert', function (data) {
delete data.diag.stack;
assert.deepEqual(data, {
ok: false,
id: 1,
Expand Down Expand Up @@ -119,7 +134,7 @@ tap.test('deep equal failure, depth 6, without option', function (assert) {
stream.pipe(parser);
stream.pipe(concat(function (body) {
assert.equal(
body.toString('utf8'),
stripFullStack(body.toString('utf8')),
'TAP version 13\n'
+ '# deep equal\n'
+ 'not ok 1 should be equal\n'
Expand All @@ -129,6 +144,11 @@ tap.test('deep equal failure, depth 6, without option', function (assert) {
+ ' { a: { a1: { a2: { a3: { a4: [Object] } } } } }\n'
+ ' actual: |-\n'
+ ' { a: { a1: { a2: { a3: { a4: [Object] } } } } }\n'
+ ' stack: |-\n'
+ ' Error: should be equal\n'
+ ' [... stack stripped ...]\n'
+ ' at Test.<anonymous> ($TEST/deep-equal-failure.js:$LINE:$COL)\n'
+ ' [... stack stripped ...]\n'
+ ' ...\n'
+ '\n'
+ '1..1\n'
Expand All @@ -145,6 +165,7 @@ tap.test('deep equal failure, depth 6, without option', function (assert) {
}));

parser.once('assert', function (data) {
delete data.diag.stack;
assert.deepEqual(data, {
ok: false,
id: 1,
Expand All @@ -162,13 +183,3 @@ tap.test('deep equal failure, depth 6, without option', function (assert) {
t.equal({ a: { a1: { a2: { a3: { a4: { a5: 1 } } } } } }, { a: { a1: { a2: { a3: { a4: { a5: 2 } } } } } });
});
})

function getDiag (body) {
var yamlStart = body.indexOf(' ---');
var yamlEnd = body.indexOf(' ...\n');
var diag = body.slice(yamlStart, yamlEnd).split('\n').map(function (line) {
return line.slice(2);
}).join('\n');

return yaml.safeLoad(diag);
}
31 changes: 30 additions & 1 deletion test/double_end.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,49 @@ var path = require('path');
var concat = require('concat-stream');
var spawn = require('child_process').spawn;

var stripFullStack = require('./common').stripFullStack;

test(function (t) {
t.plan(2);
var ps = spawn(process.execPath, [path.join(__dirname, 'double_end', 'double.js')]);
ps.on('exit', function (code) {
t.equal(code, 1);
});
ps.stdout.pipe(concat(function (body) {
t.equal(body.toString('utf8'), [
// The implementation of node's timer library has changed over time. We
// need to reverse engineer the error we expect to see.

// This code is unfortunately by necessity highly coupled to node
// versions, and may require tweaking with future versions of the timers
// library.
function doEnd() { throw new Error() };
var to = setTimeout(doEnd, 5000);
clearTimeout(to);
to._onTimeout = doEnd;

var stackExpected;
try {
to._onTimeout();
}
catch (e) {
stackExpected = stripFullStack(e.stack).split('\n')[1];
stackExpected = stackExpected.replace('double_end.js', 'double_end/double.js');
stackExpected = stackExpected.trim();
}

var stripped = stripFullStack(body.toString('utf8'));
t.equal(stripped, [
'TAP version 13',
'# double end',
'ok 1 should be equal',
'not ok 2 .end() called twice',
' ---',
' operator: fail',
' stack: |-',
' Error: .end() called twice',
' [... stack stripped ...]',
' ' + stackExpected,
' [... stack stripped ...]',
' ...',
'',
'1..2',
Expand Down
8 changes: 5 additions & 3 deletions test/double_end/double.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
var test = require('../../');

test('double end', function (t) {
function doEnd() {
t.end();
}

t.equal(1 + 1, 2);
t.end();
setTimeout(function () {
t.end();
}, 5);
setTimeout(doEnd, 5);
});
22 changes: 19 additions & 3 deletions test/exit.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ var path = require('path');
var spawn = require('child_process').spawn;
var concat = require('concat-stream');

var stripFullStack = require('./common').stripFullStack;

tap.test('exit ok', function (t) {
t.plan(2);

Expand Down Expand Up @@ -38,7 +40,7 @@ tap.test('exit fail', function (t) {
t.plan(2);

var tc = function (rows) {
t.same(rows.toString('utf8'), [
t.same(stripFullStack(rows.toString('utf8')), [
'TAP version 13',
'# array',
'ok 1 should be equivalent',
Expand All @@ -50,6 +52,14 @@ tap.test('exit fail', function (t) {
' operator: deepEqual',
' expected: [ [ 1, 2, [ 3, 4444 ] ], [ 5, 6 ] ]',
' actual: [ [ 1, 2, [ 3, 4 ] ], [ 5, 6 ] ]',
' stack: |-',
' Error: should be equivalent',
' [... stack stripped ...]',
' at $TEST/exit/fail.js:$LINE:$COL',
' at eval (eval at <anonymous> ($TEST/exit/fail.js:$LINE:$COL), <anonymous>:$LINE:$COL)',
' at eval (eval at <anonymous> ($TEST/exit/fail.js:$LINE:$COL), <anonymous>:$LINE:$COL)',
' at Test.<anonymous> ($TEST/exit/fail.js:$LINE:$COL)',
' [... stack stripped ...]',
' ...',
'',
'1..5',
Expand All @@ -70,7 +80,7 @@ tap.test('too few exit', function (t) {
t.plan(2);

var tc = function (rows) {
t.same(rows.toString('utf8'), [
t.same(stripFullStack(rows.toString('utf8')), [
'TAP version 13',
'# array',
'ok 1 should be equivalent',
Expand All @@ -83,6 +93,9 @@ tap.test('too few exit', function (t) {
' operator: fail',
' expected: 6',
' actual: 5',
' stack: |-',
' Error: plan != count',
' [... stack stripped ...]',
' ...',
'',
'1..6',
Expand All @@ -103,7 +116,7 @@ tap.test('more planned in a second test', function (t) {
t.plan(2);

var tc = function (rows) {
t.same(rows.toString('utf8'), [
t.same(stripFullStack(rows.toString('utf8')), [
'TAP version 13',
'# first',
'ok 1 should be truthy',
Expand All @@ -114,6 +127,9 @@ tap.test('more planned in a second test', function (t) {
' operator: fail',
' expected: 2',
' actual: 1',
' stack: |-',
' Error: plan != count',
' [... stack stripped ...]',
' ...',
'',
'1..3',
Expand Down
12 changes: 11 additions & 1 deletion test/fail.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@ var tape = require('../');
var tap = require('tap');
var concat = require('concat-stream');

var stripFullStack = require('./common').stripFullStack;

tap.test('array test', function (tt) {
tt.plan(1);

var test = tape.createHarness({ exit : false });
var tc = function (rows) {
tt.same(rows.toString('utf8'), [
tt.same(stripFullStack(rows.toString('utf8')), [
'TAP version 13',
'# array',
'ok 1 should be equivalent',
Expand All @@ -20,6 +22,14 @@ tap.test('array test', function (tt) {
' operator: deepEqual',
' expected: [ [ 1, 2, [ 3, 4444 ] ], [ 5, 6 ] ]',
' actual: [ [ 1, 2, [ 3, 4 ] ], [ 5, 6 ] ]',
' stack: |-',
' Error: should be equivalent',
' [... stack stripped ...]',
' at $TEST/fail.js:$LINE:$COL',
' at eval (eval at <anonymous> ($TEST/fail.js:$LINE:$COL), <anonymous>:$LINE:$COL)',
' at eval (eval at <anonymous> ($TEST/fail.js:$LINE:$COL), <anonymous>:$LINE:$COL)',
' at Test.<anonymous> ($TEST/fail.js:$LINE:$COL)',
' [... stack stripped ...]',
' ...',
'',
'1..5',
Expand Down
Loading

0 comments on commit 9302682

Please sign in to comment.