Merge pull request #47 from goyakin/expected-exception-reporting

Report the expected error indicated by the "negative" frontmatter when it isn't thrown
bterlson · Sep 24, 2015 · 3191698 · 3191698
2 parents 65aa9f3 + 038cc9c
commit 3191698
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 13 deletions.
diff --git a/lib/runner.js b/lib/runner.js
@@ -157,7 +157,7 @@ var errorLogRe = /^test262\/error (.*)$/;
 // errorStack: stack trace of error thrown (used for debugging purposes)
 Runner.prototype.validateResult = function(test, result) {
     var expectingStack = false;
-    var isNegative = test.attrs.flags.negative || test.attrs.negative;
+    var negative = test.attrs.negative;
     // parse result from log
     (result.log || []).forEach(function(log) {
         var errorMatch = log.match(errorLogRe);
@@ -203,17 +203,13 @@ Runner.prototype.validateResult = function(test, result) {
         test.errorMessage = result.errorMessage;
         test.errorStack = result.errorStack;
 
-        if(isNegative) {
-            if(test.attrs.negative) {
-                // failure can either match against error name, or an exact match
-                // against error message (the latter case is thus far only to support
-                // NotEarlyError thrown errors which have an error type of "Error").
-                test.pass =
-                    !!result.errorName.match(new RegExp(test.attrs.negative)) ||
-                    result.errorMessage === test.attrs.negative;
-            } else {
-                test.pass = true
-            }
+        if(negative) {
+            // failure can either match against error name, or an exact match
+            // against error message (the latter case is thus far only to support
+            // NotEarlyError thrown errors which have an error type of "Error").
+            test.pass =
+                !!result.errorName.match(new RegExp(negative)) ||
+                result.errorMessage === negative;
         } else {
             test.pass = false
         }
@@ -223,8 +219,10 @@ Runner.prototype.validateResult = function(test, result) {
             test.pass = false;
             test.errorName = "Test262 Error";
             test.errorMessage = "Test did not run to completion ($DONE not called)";
-        } else if(isNegative) {
+        } else if(negative) {
             test.pass = false;
+            test.errorName = "Error Expected";
+            test.errorMessage = "'" + negative + "' is expected, but was not thrown";
         } else {
             test.pass = true;
         }

diff --git a/test/collateral/negativeMessage.js b/test/collateral/negativeMessage.js
@@ -0,0 +1,4 @@
+/*---
+description: Should report the expected error indicated by the "negative" frontmatter
+negative: ExpectedError
+---*/
diff --git a/test/expected.js b/test/expected.js
@@ -15,6 +15,8 @@ var all = [
     { file: 'test/collateral/error.js', strictMode: true, pass: false, errorMessage: 'failure message', errorName: 'Test262Error' },
     { file: 'test/collateral/thrownError.js', strictMode: false, pass: false, errorMessage: 'failure message', errorName: 'Error', topOfStack: "foo" },
     { file: 'test/collateral/thrownError.js', strictMode: true, pass: false, errorMessage: 'failure message', errorName: 'Error', topOfStack: "foo" },
+    { file: 'test/collateral/negativeMessage.js', strictMode: false, pass: false, errorMessage: "'ExpectedError' is expected, but was not thrown", errorName: 'Error Expected'},
+    { file: 'test/collateral/negativeMessage.js', strictMode: true, pass: false, errorMessage: "'ExpectedError' is expected, but was not thrown", errorName: 'Error Expected'},
 ]
 
 var seen = {};