fail wpt runner if expected failures does not match actual (#3236)

nodejs · May 10, 2024 · 3b1b2fb · 3b1b2fb
1 parent 0a66d30
commit 3b1b2fb
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 26 deletions.
diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs
@@ -218,7 +218,7 @@ export class WPTRunner extends EventEmitter {
           if (message.type === 'result') {
             this.handleIndividualTestCompletion(message, status, test, meta, result)
           } else if (message.type === 'completion') {
-            this.handleTestCompletion(worker)
+            this.handleTestCompletion(worker, status, test)
           } else if (message.type === 'error') {
             this.#uncaughtExceptions.push({ error: message.error, test })
             this.#stats.failedTests += 1
@@ -318,8 +318,36 @@ export class WPTRunner extends EventEmitter {
    * Called after all the tests in a worker are completed.
    * @param {Worker} worker
    */
-  handleTestCompletion (worker) {
+  handleTestCompletion (worker, status, path) {
     worker.terminate()
+
+    const { file } = status
+    const hasExpectedFailures = !!file.fail
+    const testHasFailures = !!this.#statusOutput?.[path]
+    const failed = this.#statusOutput?.[path] ?? []
+
+    if (hasExpectedFailures !== testHasFailures) {
+      console.log({ expected: file.fail, failed })
+
+      if (failed.length === 0) {
+        console.log(colors('Tests are marked as failure but did not fail, yay!', 'red'))
+      } else if (!hasExpectedFailures) {
+        console.log(colors('Test failed but there were no expected errors.', 'red'))
+      }
+
+      process.exitCode = 1
+    } else if (hasExpectedFailures && testHasFailures) {
+      const diff = [
+        ...file.fail.filter(x => !failed.includes(x)),
+        ...failed.filter(x => !file.fail.includes(x))
+      ]
+
+      if (diff.length) {
+        console.log({ diff })
+        console.log(colors('Expected failures did not match actual failures', 'red'))
+        process.exitCode = 1
+      }
+    }
   }
 
   /**

diff --git a/test/wpt/runner/util.mjs b/test/wpt/runner/util.mjs
@@ -168,5 +168,5 @@ export function resolveStatusPath (path, status) {
     }
   }
 
-  return { topLevel: status ?? {}, file: status?.[paths.at(-1)] ?? {} }
+  return { fullPath: path, topLevel: status ?? {}, file: status?.[paths.at(-1)] ?? {} }
 }
diff --git a/test/wpt/status/FileAPI.status.json b/test/wpt/status/FileAPI.status.json
@@ -11,7 +11,8 @@
       "skip": true
     },
     "Blob-stream.any.js": {
-      "fail": [
+      "note": "fails in node v18",
+      "flaky": [
         "Reading Blob.stream() with BYOB reader"
       ]
     }

diff --git a/test/wpt/status/fetch.status.json b/test/wpt/status/fetch.status.json
@@ -3,10 +3,7 @@
     "abort": {
       "general.any.js": {
         "fail": [
-          "Already aborted signal rejects immediately",
-          "Underlying connection is closed when aborting after receiving response - no-cors",
-          "Stream errors once aborted. Underlying connection closed.",
-          "Readable stream synchronously cancels with AbortError if aborted before reading"
+          "Already aborted signal rejects immediately"
         ]
       },
       "cache.https.any.js": {
@@ -52,9 +49,6 @@
       },
       "referrer.any.js": {
         "fail": [
-          "origin-when-cross-origin policy on a cross-origin URL",
-          "origin-when-cross-origin policy on a cross-origin URL after same-origin redirection",
-          "origin-when-cross-origin policy on a same-origin URL after cross-origin redirection",
           "origin-when-cross-origin policy on a same-origin URL"
         ]
       },
@@ -100,7 +94,7 @@
       },
       "stream-safe-creation.any.js": {
         "note": "tests are very finnicky",
-        "fail": [
+        "flaky": [
           "throwing Object.prototype.type accessor should not affect stream creation by 'fetch'",
           "Object.prototype.type accessor returning invalid value should not affect stream creation by 'fetch'",
           "throwing Object.prototype.highWaterMark accessor should not affect stream creation by 'fetch'",
@@ -344,7 +338,6 @@
           "Adding invalid request header \"Accept-Encoding: KO\"",
           "Adding invalid request header \"Access-Control-Request-Headers: KO\"",
           "Adding invalid request header \"Access-Control-Request-Method: KO\"",
-          "Adding invalid request header \"Access-Control-Request-Private-Network: KO\"",
           "Adding invalid request header \"Connection: KO\"",
           "Adding invalid request header \"Content-Length: KO\"",
           "Adding invalid request header \"Cookie: KO\"",
@@ -394,7 +387,8 @@
         ]
       },
       "response-consume-stream.any.js": {
-        "fail": [
+        "note": "only fail in node v18",
+        "flaky": [
           "Read blob response's body as readableStream with mode=byob",
           "Read text response's body as readableStream with mode=byob",
           "Read URLSearchParams response's body as readableStream with mode=byob",

diff --git a/test/wpt/status/websockets.status.json b/test/wpt/status/websockets.status.json
@@ -16,12 +16,6 @@
       }
     }
   },
-  "Create-blocked-port.any.js": {
-    "note": "TODO(@KhafraDev): investigate failure",
-    "fail": [
-      "Basic check"
-    ]
-  },
   "Send-binary-arraybufferview-float32.any.js": {
     "note": "TODO(@KhafraDev): investigate failure",
     "fail": [
@@ -58,12 +52,6 @@
       "Send binary data on a WebSocket - ArrayBufferView - Uint32Array with offset - Connection should be closed"
     ]
   },
-  "basic-auth.any.js": {
-    "note": "TODO(@KhafraDev): investigate failure",
-    "fail": [
-      "HTTP basic authentication should work with WebSockets"
-    ]
-  },
   "Create-on-worker-shutdown.any.js": {
     "skip": true,
     "//": "Node.js workers are different from web workers & don't work with blob: urls"