Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Bug 816688 Output test messages in the tinderbox standard #669

Merged
merged 4 commits into from

2 participants

@Mossop
Owner

I wasn't totally sure the best way to approach this, but this replaces the console for tests with a custom one that outputs lines in the expected format for tinderbox. Every test starts with TEST-START and ends with TEST-END, with TEST-INFO, TEST-PASS, TEST-KNOWN-FAIL, TEST-UNEXPECTED-FAIL and TEST-UNEXPECTED-PASS in between. Each line also includes the running test where possible.

@Gozala
Owner

This looks fine, only thing is I would rather prefer if we had argument for a logging mode instead, so for humans output will remain same, but for tinderbox we could do something like cfx testall --parsable or something along that lines.

Also I could live with this output too.

@Mossop
Owner

@Gozala I've updated this to add a --parseable flag to cfx that switches to the new logging format.

@Gozala
Owner

@Mossop Looks good to me, only thing I'm not sure about is that in --parsable mode no stack traces are printed. Is that intentional ? Ability to see stacks traces on failures in tinderbox had being pretty useful, specially on ones that are hard to reproduce on local machine.

If printing stack traces makes sense please do so, otherwise feel free to land. Either way I don't think we need any more iterations on this.

@Mossop
Owner

Oh that's not intentional, I'll make it output stacks before landing.

@Mossop Mossop merged commit 02e0eef into mozilla:master
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Nov 30, 2012
  1. @Mossop
Commits on Dec 20, 2012
  1. @Mossop
  2. @Mossop

    Merge branch 'master' into bug816688

    Mossop authored
    Conflicts:
    	lib/sdk/test/harness.js
Commits on Jan 3, 2013
  1. @Mossop
This page is out of date. Refresh to see the latest.
View
1  app-extension/bootstrap.js
@@ -173,6 +173,7 @@ function startup(data, reasonCode) {
profileMemory: options.profileMemory,
stopOnError: options.stopOnError,
verbose: options.verbose,
+ parseable: options.parseable,
}
}
});
View
49 lib/sdk/deprecated/unit-test.js
@@ -11,6 +11,7 @@ module.metadata = {
const memory = require('./memory');
var timer = require("../timers");
+var cfxArgs = require("@test/options");
exports.findAndRunTests = function findAndRunTests(options) {
var TestFinder = require("./unit-test-finder").TestFinder;
@@ -50,35 +51,53 @@ TestRunner.prototype = {
if (!(why in this.test.errors))
this.test.errors[why] = 0;
this.test.errors[why]++;
- if (!this.testFailureLogged) {
- this.console.error("TEST FAILED: " + this.test.name + " (" + why + ")");
- this.testFailureLogged = true;
- }
},
pass: function pass(message) {
if(!this.expectFailure) {
- this.console.info("pass:", message);
+ if ("testMessage" in this.console)
+ this.console.testMessage(true, true, this.test.name, message);
+ else
+ this.console.info("pass:", message);
this.passed++;
this.test.passed++;
}
else {
this.expectFailure = false;
- this.fail('Failure Expected: ' + message);
+ this._logTestFailed("failure");
+ if ("testMessage" in this.console) {
+ this.console.testMessage(true, false, this.test.name, message);
+ }
+ else {
+ this.console.error("fail:", 'Failure Expected: ' + message)
+ this.console.trace();
+ }
+ this.failed++;
+ this.test.failed++;
}
},
fail: function fail(message) {
if(!this.expectFailure) {
this._logTestFailed("failure");
- this.console.error("fail:", message);
- this.console.trace();
+ if ("testMessage" in this.console) {
+ this.console.testMessage(false, false, this.test.name, message);
+ }
+ else {
+ this.console.error("fail:", message)
+ this.console.trace();
+ }
this.failed++;
this.test.failed++;
}
else {
this.expectFailure = false;
- this.pass(message);
+ if ("testMessage" in this.console)
+ this.console.testMessage(false, true, this.test.name, message);
+ else
+ this.console.info("pass:", message);
+ this.passed++;
+ this.test.passed++;
}
},
@@ -431,12 +450,18 @@ TestRunner.prototype = {
this.test.errors = {};
this.isDone = false;
- this.onDone = options.onDone;
+ this.onDone = function(self) {
+ if (cfxArgs.parseable)
+ self.console.print("TEST-END | " + self.test.name + "\n");
+ options.onDone(self);
+ }
this.waitTimeout = null;
- this.testFailureLogged = false;
try {
- this.console.info("executing '" + this.test.name + "'");
+ if (cfxArgs.parseable)
+ this.console.print("TEST-START | " + this.test.name + "\n");
+ else
+ this.console.info("executing '" + this.test.name + "'");
if(this.test.setup) {
this.test.setup(this);
View
98 lib/sdk/test/harness.js
@@ -26,6 +26,9 @@ const url = require("../url");
var cService = Cc['@mozilla.org/consoleservice;1'].getService()
.QueryInterface(Ci.nsIConsoleService);
+// The console used to log messages
+var testConsole;
+
// Cuddlefish loader in which we load and execute tests.
var loader;
@@ -279,7 +282,7 @@ var consoleListener = {
var pointless = [err for each (err in POINTLESS_ERRORS)
if (message.indexOf(err) >= 0)];
if (pointless.length == 0 && message)
- print("console: " + message + "\n");
+ testConsole.log(message);
}
};
@@ -305,6 +308,93 @@ function TestRunnerConsole(base, options) {
};
}
+function stringify(arg) {
+ try {
+ return String(arg);
+ }
+ catch(ex) {
+ return "<toString() error>";
+ }
+}
+
+function stringifyArgs(args) {
+ return Array.map(args, stringify).join(" ");
+}
+
+function TestRunnerTinderboxConsole(options) {
+ this.print = options.print;
+ this.verbose = options.verbose;
+ this.errorsLogged = 0;
+
+ // Binding all the public methods to an instance so that they can be used
+ // as callback / listener functions straightaway.
+ this.log = this.log.bind(this);
+ this.info = this.info.bind(this);
+ this.warn = this.warn.bind(this);
+ this.error = this.error.bind(this);
+ this.debug = this.debug.bind(this);
+ this.exception = this.exception.bind(this);
+ this.trace = this.trace.bind(this);
+};
+
+TestRunnerTinderboxConsole.prototype = {
+ testMessage: function testMessage(pass, expected, test, message) {
+ let type = "TEST-";
+ if (expected) {
+ if (pass)
+ type += "PASS";
+ else
+ type += "KNOWN-FAIL";
+ }
+ else {
+ this.errorsLogged++;
+ if (pass)
+ type += "UNEXPECTED-PASS";
+ else
+ type += "UNEXPECTED-FAIL";
+ }
+
+ this.print(type + " | " + test + " | " + message + "\n");
+ if (!expected)
+ this.trace();
+ },
+
+ log: function log() {
+ this.print("TEST-INFO | " + stringifyArgs(arguments) + "\n");
+ },
+
+ info: function info(first) {
+ this.print("TEST-INFO | " + stringifyArgs(arguments) + "\n");
+ },
+
+ warn: function warn() {
+ this.errorsLogged++;
+ this.print("TEST-UNEXPECTED-FAIL | " + stringifyArgs(arguments) + "\n");
+ },
+
+ error: function error() {
+ this.errorsLogged++;
+ this.print("TEST-UNEXPECTED-FAIL | " + stringifyArgs(arguments) + "\n");
+ },
+
+ debug: function debug() {
+ this.print("TEST-INFO | " + stringifyArgs(arguments) + "\n");
+ },
+
+ exception: function exception(e) {
+ var fullString = ("An exception occurred.\n" +
+ require("../console/traceback").format(e) + "\n" + e);
+ this.error(fullString);
+ },
+
+ trace: function trace() {
+ var traceback = require("../console/traceback");
+ var stack = traceback.get();
+ stack.splice(-1, 1);
+ this.print("TEST-INFO | " + stringify(traceback.format(stack)) + "\n");
+ }
+};
+
var runTests = exports.runTests = function runTests(options) {
iterationsLeft = options.iterations;
profileMemory = options.profileMemory;
@@ -320,9 +410,13 @@ var runTests = exports.runTests = function runTests(options) {
system.id + ") under " +
system.platform + "/" + system.architecture + ".\n");
+ if (options.parseable)
+ testConsole = new TestRunnerTinderboxConsole(options);
+ else
+ testConsole = new TestRunnerConsole(new PlainTextConsole(print), options);
loader = Loader(module, {
- console: new TestRunnerConsole(new PlainTextConsole(print), options)
+ console: testConsole
});
nextIteration();
View
1  lib/sdk/test/runner.js
@@ -43,6 +43,7 @@ function runTests(findAndRunTests) {
profileMemory: cfxArgs.profileMemory,
stopOnError: cfxArgs.stopOnError,
verbose: cfxArgs.verbose,
+ parseable: cfxArgs.parseable,
print: stdout.write,
onDone: onDone
});
View
8 python-lib/cuddlefish/__init__.py
@@ -138,6 +138,12 @@
metavar=None,
default="{}",
cmds=['run', 'xpi'])),
+ (("", "--parseable",), dict(dest="parseable",
+ help="display test output in a parseable format",
+ action="store_true",
+ default=False,
+ cmds=['test', 'testex', 'testpkgs',
+ 'testall'])),
]
),
@@ -632,7 +638,7 @@ def run(arguments=sys.argv[1:], target_cfg=None, pkg_cfg=None,
if 'tests' not in target_cfg:
target_cfg['tests'] = []
inherited_options.extend(['iterations', 'filter', 'profileMemory',
- 'stopOnError'])
+ 'stopOnError', 'parseable'])
enforce_timeouts = True
elif command == "run":
use_main = True
View
33 test/test-unit-test.js
@@ -121,9 +121,7 @@ exports.testWaitUntilTimeoutInCallback = function(test) {
calls: 0,
error: function(msg) {
this.calls++;
- if (this.calls == 1)
- test.assertEqual(arguments[0], "TEST FAILED: wait4ever (timed out)");
- else if (this.calls == 2) {
+ if (this.calls == 2) {
test.assertEqual(arguments[0], "test assertion never became true:\n");
test.assertEqual(arguments[1], "assertion failed, value is false\n");
// We could additionally check that arguments[1] contains the correct
@@ -140,9 +138,34 @@ exports.testWaitUntilTimeoutInCallback = function(test) {
}
},
info: function (msg) {
- test.assertEqual(msg, "executing 'wait4ever'");
+ this.calls++;
+ if (require("@test/options").parseable) {
+ test.fail("We got unexpected console.info() calls: " + msg)
+ }
+ else if (this.calls == 1) {
+ test.assertEqual(arguments[0], "executing 'wait4ever'");
+ }
+ else {
+ test.fail("We got unexpected console.info() calls: " + msg);
+ }
},
- trace: function () {}
+ trace: function () {},
+ exception: function () {},
+ print: function (str) {
+ this.calls++;
+ if (!require("@test/options").parseable) {
+ test.fail("We got unexpected console.print() calls: " + str)
+ }
+ else if (this.calls == 1) {
+ test.assertEqual(str, "TEST-START | wait4ever\n");
+ }
+ else if (this.calls == 3) {
+ test.assertEqual(str, "TEST-END | wait4ever\n");
+ }
+ else {
+ test.fail("We got unexpected console.print() calls: " + str);
+ }
+ }
}
});
Something went wrong with that request. Please try again.