Skip to content

Bug 816688 Output test messages in the tinderbox standard #669

Merged
merged 4 commits into from Jan 3, 2013
View
1 app-extension/bootstrap.js
@@ -173,6 +173,7 @@ function startup(data, reasonCode) {
profileMemory: options.profileMemory,
stopOnError: options.stopOnError,
verbose: options.verbose,
+ parseable: options.parseable,
}
}
});
View
49 lib/sdk/deprecated/unit-test.js
@@ -11,6 +11,7 @@ module.metadata = {
const memory = require('./memory');
var timer = require("../timers");
+var cfxArgs = require("@test/options");
exports.findAndRunTests = function findAndRunTests(options) {
var TestFinder = require("./unit-test-finder").TestFinder;
@@ -50,35 +51,53 @@ TestRunner.prototype = {
if (!(why in this.test.errors))
this.test.errors[why] = 0;
this.test.errors[why]++;
- if (!this.testFailureLogged) {
- this.console.error("TEST FAILED: " + this.test.name + " (" + why + ")");
- this.testFailureLogged = true;
- }
},
pass: function pass(message) {
if(!this.expectFailure) {
- this.console.info("pass:", message);
+ if ("testMessage" in this.console)
+ this.console.testMessage(true, true, this.test.name, message);
+ else
+ this.console.info("pass:", message);
this.passed++;
this.test.passed++;
}
else {
this.expectFailure = false;
- this.fail('Failure Expected: ' + message);
+ this._logTestFailed("failure");
+ if ("testMessage" in this.console) {
+ this.console.testMessage(true, false, this.test.name, message);
+ }
+ else {
+ this.console.error("fail:", 'Failure Expected: ' + message)
+ this.console.trace();
+ }
+ this.failed++;
+ this.test.failed++;
}
},
fail: function fail(message) {
if(!this.expectFailure) {
this._logTestFailed("failure");
- this.console.error("fail:", message);
- this.console.trace();
+ if ("testMessage" in this.console) {
+ this.console.testMessage(false, false, this.test.name, message);
+ }
+ else {
+ this.console.error("fail:", message)
+ this.console.trace();
+ }
this.failed++;
this.test.failed++;
}
else {
this.expectFailure = false;
- this.pass(message);
+ if ("testMessage" in this.console)
+ this.console.testMessage(false, true, this.test.name, message);
+ else
+ this.console.info("pass:", message);
+ this.passed++;
+ this.test.passed++;
}
},
@@ -431,12 +450,18 @@ TestRunner.prototype = {
this.test.errors = {};
this.isDone = false;
- this.onDone = options.onDone;
+ this.onDone = function(self) {
+ if (cfxArgs.parseable)
+ self.console.print("TEST-END | " + self.test.name + "\n");
+ options.onDone(self);
+ }
this.waitTimeout = null;
- this.testFailureLogged = false;
try {
- this.console.info("executing '" + this.test.name + "'");
+ if (cfxArgs.parseable)
+ this.console.print("TEST-START | " + this.test.name + "\n");
+ else
+ this.console.info("executing '" + this.test.name + "'");
if(this.test.setup) {
this.test.setup(this);
View
98 lib/sdk/test/harness.js
@@ -26,6 +26,9 @@ const url = require("../url");
var cService = Cc['@mozilla.org/consoleservice;1'].getService()
.QueryInterface(Ci.nsIConsoleService);
+// The console used to log messages
+var testConsole;
+
// Cuddlefish loader in which we load and execute tests.
var loader;
@@ -279,7 +282,7 @@ var consoleListener = {
var pointless = [err for each (err in POINTLESS_ERRORS)
if (message.indexOf(err) >= 0)];
if (pointless.length == 0 && message)
- print("console: " + message + "\n");
+ testConsole.log(message);
}
};
@@ -305,6 +308,93 @@ function TestRunnerConsole(base, options) {
};
}
+function stringify(arg) {
+ try {
+ return String(arg);
+ }
+ catch(ex) {
+ return "<toString() error>";
+ }
+}
+
+function stringifyArgs(args) {
+ return Array.map(args, stringify).join(" ");
+}
+
+function TestRunnerTinderboxConsole(options) {
+ this.print = options.print;
+ this.verbose = options.verbose;
+ this.errorsLogged = 0;
+
+ // Binding all the public methods to an instance so that they can be used
+ // as callback / listener functions straightaway.
+ this.log = this.log.bind(this);
+ this.info = this.info.bind(this);
+ this.warn = this.warn.bind(this);
+ this.error = this.error.bind(this);
+ this.debug = this.debug.bind(this);
+ this.exception = this.exception.bind(this);
+ this.trace = this.trace.bind(this);
+};
+
+TestRunnerTinderboxConsole.prototype = {
+ testMessage: function testMessage(pass, expected, test, message) {
+ let type = "TEST-";
+ if (expected) {
+ if (pass)
+ type += "PASS";
+ else
+ type += "KNOWN-FAIL";
+ }
+ else {
+ this.errorsLogged++;
+ if (pass)
+ type += "UNEXPECTED-PASS";
+ else
+ type += "UNEXPECTED-FAIL";
+ }
+
+ this.print(type + " | " + test + " | " + message + "\n");
+ if (!expected)
+ this.trace();
+ },
+
+ log: function log() {
+ this.print("TEST-INFO | " + stringifyArgs(arguments) + "\n");
+ },
+
+ info: function info(first) {
+ this.print("TEST-INFO | " + stringifyArgs(arguments) + "\n");
+ },
+
+ warn: function warn() {
+ this.errorsLogged++;
+ this.print("TEST-UNEXPECTED-FAIL | " + stringifyArgs(arguments) + "\n");
+ },
+
+ error: function error() {
+ this.errorsLogged++;
+ this.print("TEST-UNEXPECTED-FAIL | " + stringifyArgs(arguments) + "\n");
+ },
+
+ debug: function debug() {
+ this.print("TEST-INFO | " + stringifyArgs(arguments) + "\n");
+ },
+
+ exception: function exception(e) {
+ var fullString = ("An exception occurred.\n" +
+ require("../console/traceback").format(e) + "\n" + e);
+ this.error(fullString);
+ },
+
+ trace: function trace() {
+ var traceback = require("../console/traceback");
+ var stack = traceback.get();
+ stack.splice(-1, 1);
+ this.print("TEST-INFO | " + stringify(traceback.format(stack)) + "\n");
+ }
+};
+
var runTests = exports.runTests = function runTests(options) {
iterationsLeft = options.iterations;
profileMemory = options.profileMemory;
@@ -320,9 +410,13 @@ var runTests = exports.runTests = function runTests(options) {
system.id + ") under " +
system.platform + "/" + system.architecture + ".\n");
+ if (options.parseable)
+ testConsole = new TestRunnerTinderboxConsole(options);
+ else
+ testConsole = new TestRunnerConsole(new PlainTextConsole(print), options);
loader = Loader(module, {
- console: new TestRunnerConsole(new PlainTextConsole(print), options)
+ console: testConsole
});
nextIteration();
View
1 lib/sdk/test/runner.js
@@ -43,6 +43,7 @@ function runTests(findAndRunTests) {
profileMemory: cfxArgs.profileMemory,
stopOnError: cfxArgs.stopOnError,
verbose: cfxArgs.verbose,
+ parseable: cfxArgs.parseable,
print: stdout.write,
onDone: onDone
});
View
8 python-lib/cuddlefish/__init__.py
@@ -138,6 +138,12 @@
metavar=None,
default="{}",
cmds=['run', 'xpi'])),
+ (("", "--parseable",), dict(dest="parseable",
+ help="display test output in a parseable format",
+ action="store_true",
+ default=False,
+ cmds=['test', 'testex', 'testpkgs',
+ 'testall'])),
]
),
@@ -632,7 +638,7 @@ def run(arguments=sys.argv[1:], target_cfg=None, pkg_cfg=None,
if 'tests' not in target_cfg:
target_cfg['tests'] = []
inherited_options.extend(['iterations', 'filter', 'profileMemory',
- 'stopOnError'])
+ 'stopOnError', 'parseable'])
enforce_timeouts = True
elif command == "run":
use_main = True
View
33 test/test-unit-test.js
@@ -121,9 +121,7 @@ exports.testWaitUntilTimeoutInCallback = function(test) {
calls: 0,
error: function(msg) {
this.calls++;
- if (this.calls == 1)
- test.assertEqual(arguments[0], "TEST FAILED: wait4ever (timed out)");
- else if (this.calls == 2) {
+ if (this.calls == 2) {
test.assertEqual(arguments[0], "test assertion never became true:\n");
test.assertEqual(arguments[1], "assertion failed, value is false\n");
// We could additionally check that arguments[1] contains the correct
@@ -140,9 +138,34 @@ exports.testWaitUntilTimeoutInCallback = function(test) {
}
},
info: function (msg) {
- test.assertEqual(msg, "executing 'wait4ever'");
+ this.calls++;
+ if (require("@test/options").parseable) {
+ test.fail("We got unexpected console.info() calls: " + msg)
+ }
+ else if (this.calls == 1) {
+ test.assertEqual(arguments[0], "executing 'wait4ever'");
+ }
+ else {
+ test.fail("We got unexpected console.info() calls: " + msg);
+ }
},
- trace: function () {}
+ trace: function () {},
+ exception: function () {},
+ print: function (str) {
+ this.calls++;
+ if (!require("@test/options").parseable) {
+ test.fail("We got unexpected console.print() calls: " + str)
+ }
+ else if (this.calls == 1) {
+ test.assertEqual(str, "TEST-START | wait4ever\n");
+ }
+ else if (this.calls == 3) {
+ test.assertEqual(str, "TEST-END | wait4ever\n");
+ }
+ else {
+ test.fail("We got unexpected console.print() calls: " + str);
+ }
+ }
}
});
Something went wrong with that request. Please try again.