Spec: Define "error" event for global failures

Ref qunitjs/qunit#1633.

lib/reporters/ConsoleReporter.js

@@ -5,6 +5,7 @@ module.exports = class ConsoleReporter {
     // https://github.com/js-reporters/js-reporters/issues/125
     this.log = options.log || console.log.bind(console);
 
+    runner.on('error', this.onError.bind(this));
     runner.on('runStart', this.onRunStart.bind(this));
     runner.on('testStart', this.onTestStart.bind(this));
     runner.on('testEnd', this.onTestEnd.bind(this));
@@ -15,6 +16,10 @@ module.exports = class ConsoleReporter {
     return new ConsoleReporter(runner);
   }
 
+  onError (error) {
+    this.log('error', error);
+  }
+
   onRunStart (runStart) {
     this.log('runStart', runStart);
   }

lib/reporters/SummaryReporter.js

@@ -21,6 +21,10 @@ module.exports = class SummaryReporter extends EventEmitter {
 
     this.summary = null;
 
+    runner.once('error', (error) => {
+      this.emit('error', error);
+    });
+
     runner.on('suiteEnd', (suiteEnd) => {
       const ownFull = suiteEnd.fullName.join('>');
       const suiteName = suiteEnd.fullName.length >= 2 ? suiteEnd.fullName.slice(-2, -1)[0] : null;

lib/reporters/TapReporter.js

@@ -169,6 +169,7 @@ module.exports = class TapReporter {
 
     this.testCount = 0;
 
+    runner.on('error', this.onError.bind(this));
     runner.on('runStart', this.onRunStart.bind(this));
     runner.on('testEnd', this.onTestEnd.bind(this));
     runner.on('runEnd', this.onRunEnd.bind(this));
@@ -182,6 +183,23 @@ module.exports = class TapReporter {
     this.log('TAP version 13');
   }
 
+  onError (error) {
+    if (typeof error === 'object' && error !== null) {
+      let out = '  ---';
+      const errYaml = prettyYamlValue(error);
+      if (errYaml !== '{}') {
+        out += `\n  error: ${errYaml}`;
+      }
+      if (error.stack) {
+        // This property is non-enumerable by default, so include it manually.
+        out += `\n  stack: ${prettyYamlValue(error.stack + '\n')}`;
+      }
+      out += '\n  ...';
+      this.log(out);
+    }
+    this.log('Bail out! ' + error);
+  }
+
   onTestEnd (test) {
     this.testCount = this.testCount + 1;
 

spec/cri-draft.adoc

@@ -189,6 +189,26 @@ Callback parameters:
 producer.on('testEnd', (testEnd) => { … });
 ----
 
+=== `error` event
+
+The **error** event indicates a global failure. It may be emitted at any time, including before a <<runstart-event>> or <<runend-event>>.
+
+Reporters must interpret error events as the <<run>> having failed. Reporters should not wait for or expect other events to be emitted afterwards. For example, any pending <<testend-event>> or <<runend-event>> might not be delivered. If other events do get emitted after this, reporters should doubt their accuracy or ignore the event entirely. For example, if a `runEnd` event is emitted _after_ an `error` event and the `runEnd` reports no failures, then reporters must still consider the run as having failed.
+
+Callback parameters:
+
+* <<error>> **error**.
+
+[source,javascript]
+----
+producer.on('error', (error) => { … });
+----
+
+[TIP]
+=====
+The "error" event is analogous to "Bail out!" lines as defined in https://testanything.org/tap-version-13-specification.html#directives[TAP specification].
+=====
+
 == Event data
 
 The following data structures must be implemented as objects that have the specified fields as own properties. The objects are not required to be an instance of any specific class. They may be null-inherited objects, plain objects, or an instance of any public or private class.
@@ -286,6 +306,16 @@ Producers may set additional (non-standard) properties on `Assertion` objects.
 The properties of the Assertion object was decided in https://github.com/js-reporters/js-reporters/issues/79[issue #79], and later revised by https://github.com/js-reporters/js-reporters/issues/105[issue #105].
 =====
 
+=== Error
+
+The **Error** value may be any JavaScript value. It is recommended to pass an object, such as an instance of the [ECMAScript Error Constructor](https://github.com/js-reporters/js-reporters/issues/123) (or a subclass of that). But it may also be a non-object, or a plain object with the following properties:
+
+`Error` object:
+
+* `string` **name**: Error class name, or other prefix.
+* `string` **message**: Error message.
+* `string|undefined|null` **stack**: Optional stack trace.
+
 == Producer API
 
 The object on which the Producer API is implemented does not need to be exclusive or otherwise limited to the Producer API. Producers are encouraged to implement the API as transparently as possible.

test/unit/console-reporter.js

@@ -39,4 +39,9 @@ QUnit.module('ConsoleReporter', hooks => {
     emitter.emit('testEnd', {});
     assert.equal(con.log.callCount, 1);
   });
+
+  test('Event "error"', assert => {
+    emitter.emit('error', {});
+    assert.equal(con.log.callCount, 1);
+  });
 });

test/unit/summary-reporter.js

@@ -210,4 +210,16 @@ QUnit.module('SummaryReporter', hooks => {
       done();
     });
   });
+
+  test('forward "error" event', assert => {
+    const done = assert.async();
+    const obj = { message: 'Boo' };
+
+    reporter.once('error', (err) => {
+      assert.strictEqual(err, obj);
+      done();
+    });
+
+    emitter.emit('error', obj);
+  });
 });

test/unit/tap-reporter.js

@@ -67,6 +67,28 @@ QUnit.module('TapReporter', hooks => {
     }
   });
 
+  test('output global failure (string)', assert => {
+    emitter.emit('error', 'Boo');
+
+    assert.true(spy.calledWith('Bail out! Boo'));
+  });
+
+  test('output global failure (Error)', assert => {
+    const err = new ReferenceError('Boo is not defined');
+    err.stack = `ReferenceError: Boo is not defined
+    at foo (foo.js:1:2)
+    at bar (bar.js:1:2)`;
+    emitter.emit('error', err);
+
+    assert.true(spy.calledWith(`  ---
+  stack: |
+    ReferenceError: Boo is not defined
+        at foo (foo.js:1:2)
+        at bar (bar.js:1:2)
+  ...`));
+    assert.true(spy.calledWith('Bail out! ReferenceError: Boo is not defined'));
+  });
+
   test('output actual assertion value of undefined', assert => {
     emitter.emit('testEnd', data.actualUndefinedTest);
     assert.true(spy.calledWithMatch(/^ {2}actual {2}: undefined$/m));