diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs index 4673c8fc0be..cbd5ea540d8 100644 --- a/test/wpt/runner/runner.mjs +++ b/test/wpt/runner/runner.mjs @@ -58,11 +58,13 @@ export class WPTRunner extends EventEmitter { #reportPath #stats = { - completed: 0, - failed: 0, - success: 0, + completedTests: 0, + failedTests: 0, + passedTests: 0, expectedFailures: 0, - skipped: 0 + failedFiles: 0, + passedFiles: 0, + skippedFiles: 0 } constructor (folder, url, { appendReport = false, reportPath } = {}) { @@ -158,7 +160,7 @@ export class WPTRunner extends EventEmitter { const status = resolveStatusPath(test, this.#status) if (status.file.skip || status.topLevel.skip) { - this.#stats.skipped += 1 + this.#stats.skippedFiles += 1 console.log(colors(`[${finishedFiles}/${total}] SKIPPED - ${test}`, 'yellow')) console.log('='.repeat(96)) @@ -187,19 +189,19 @@ export class WPTRunner extends EventEmitter { } }) - let result, report + const fileUrl = new URL(`/${this.#folderName}${test.slice(this.#folderPath.length)}`, 'http://wpt') + fileUrl.pathname = fileUrl.pathname.replace(/\.js$/, '.html') + fileUrl.search = variant + const result = { + test: fileUrl.href.slice(fileUrl.origin.length), + subtests: [], + status: '' + } + + let report if (this.#appendReport) { report = JSON.parse(readFileSync(this.#reportPath)) - - const fileUrl = new URL(`/${this.#folderName}${test.slice(this.#folderPath.length)}`, 'http://wpt') - fileUrl.pathname = fileUrl.pathname.replace(/\.js$/, '.html') - fileUrl.search = variant - - result = { - test: fileUrl.href.slice(fileUrl.origin.length), - subtests: [], - status: 'OK' - } + result.status = 'OK' report.results.push(result) } @@ -214,8 +216,8 @@ export class WPTRunner extends EventEmitter { this.handleTestCompletion(worker) } else if (message.type === 'error') { this.#uncaughtExceptions.push({ error: message.error, test }) - this.#stats.failed += 1 - this.#stats.success -= 1 + this.#stats.failedTests += 1 + this.#stats.passedTests -= 1 } }) @@ -224,14 +226,27 @@ export class WPTRunner extends EventEmitter { signal: AbortSignal.timeout(timeout) }) - console.log(colors(`[${finishedFiles}/${total}] PASSED - ${test}`, 'green')) + if (result.subtests.some((subtest) => subtest?.isExpectedFailure === false)) { + this.#stats.failedFiles += 1 + console.log(colors(`[${finishedFiles}/${total}] FAILED - ${test}`, 'red')) + } else { + this.#stats.passedFiles += 1 + console.log(colors(`[${finishedFiles}/${total}] PASSED - ${test}`, 'green')) + } + if (variant) console.log('Variant:', variant) - console.log(`Test took ${(performance.now() - start).toFixed(2)}ms`) + console.log(`File took ${(performance.now() - start).toFixed(2)}ms`) console.log('='.repeat(96)) } catch (e) { - console.log(`${test} timed out after ${timeout}ms`) + // If the worker is terminated by the timeout signal, the test is marked as failed + this.#stats.failedFiles += 1 + console.log(colors(`[${finishedFiles}/${total}] FAILED - ${test}`, 'red')) + + if (variant) console.log('Variant:', variant) + console.log(`File timed out after ${timeout}ms`) + console.log('='.repeat(96)) } finally { - if (result?.subtests.length > 0) { + if (this.#appendReport && result?.subtests.length > 0) { writeFileSync(this.#reportPath, JSON.stringify(report)) } @@ -248,50 +263,49 @@ export class WPTRunner extends EventEmitter { * Called after a test has succeeded or failed. */ handleIndividualTestCompletion (message, status, path, meta, wptResult) { - const { file, topLevel } = status - - if (message.type === 'result') { - this.#stats.completed += 1 + this.#stats.completedTests += 1 - if (message.result.status === 1) { - let expectedFailure = false - this.#stats.failed += 1 + const { file, topLevel } = status + const isFailure = message.result.status === 1 - wptResult?.subtests.push({ - status: 'FAIL', - name: sanitizeUnpairedSurrogates(message.result.name), - message: sanitizeUnpairedSurrogates(message.result.message) - }) + const testResult = { + status: isFailure ? 'FAIL' : 'PASS', + name: sanitizeUnpairedSurrogates(message.result.name) + } - const name = normalizeName(message.result.name) - - if (file.flaky?.includes(name)) { - expectedFailure = true - this.#stats.expectedFailures += 1 - } else if (file.allowUnexpectedFailures || topLevel.allowUnexpectedFailures || file.fail?.includes(name)) { - if (!file.allowUnexpectedFailures && !topLevel.allowUnexpectedFailures) { - if (Array.isArray(file.fail)) { - this.#statusOutput[path] ??= [] - this.#statusOutput[path].push(name) - } + if (isFailure) { + let isExpectedFailure = false + this.#stats.failedTests += 1 + + const name = normalizeName(message.result.name) + const sanitizedMessage = sanitizeUnpairedSurrogates(message.result.message) + + if (file.flaky?.includes(name)) { + isExpectedFailure = true + this.#stats.expectedFailures += 1 + wptResult?.subtests.push({ ...testResult, message: sanitizedMessage, isExpectedFailure }) + } else if (file.allowUnexpectedFailures || topLevel.allowUnexpectedFailures || file.fail?.includes(name)) { + if (!file.allowUnexpectedFailures && !topLevel.allowUnexpectedFailures) { + if (Array.isArray(file.fail)) { + this.#statusOutput[path] ??= [] + this.#statusOutput[path].push(name) } - - expectedFailure = true - this.#stats.expectedFailures += 1 - } else { - process.exitCode = 1 - console.error(message.result) - } - if (!expectedFailure) { - process._rawDebug(`Failed test: ${path}`) } + + isExpectedFailure = true + this.#stats.expectedFailures += 1 + wptResult?.subtests.push({ ...testResult, message: sanitizedMessage, isExpectedFailure }) } else { - wptResult?.subtests.push({ - status: 'PASS', - name: sanitizeUnpairedSurrogates(message.result.name) - }) - this.#stats.success += 1 + wptResult?.subtests.push({ ...testResult, message: sanitizedMessage, isExpectedFailure }) + process.exitCode = 1 + console.error(message.result) + } + if (!isExpectedFailure) { + process._rawDebug(`Failed test: ${path}`) } + } else { + this.#stats.passedTests += 1 + wptResult?.subtests.push(testResult) } } @@ -313,16 +327,23 @@ export class WPTRunner extends EventEmitter { } this.emit('completion') - const { completed, failed, success, expectedFailures, skipped } = this.#stats + + const { passedFiles, failedFiles, skippedFiles } = this.#stats + console.log( + `File results for folder [${this.#folderName}]: ` + + `completed: ${this.#files.length}, passed: ${passedFiles}, failed: ${failedFiles}, ` + + `skipped: ${skippedFiles}` + ) + + const { completedTests, failedTests, passedTests, expectedFailures } = this.#stats console.log( - `[${this.#folderName}]: ` + - `completed: ${completed}, failed: ${failed}, success: ${success}, ` + + `Test results for folder [${this.#folderName}]: ` + + `completed: ${completedTests}, failed: ${failedTests}, passed: ${passedTests}, ` + `expected failures: ${expectedFailures}, ` + - `unexpected failures: ${failed - expectedFailures}, ` + - `skipped: ${skipped}` + `unexpected failures: ${failedTests - expectedFailures}` ) - process.exit(failed - expectedFailures ? 1 : process.exitCode) + process.exit(failedTests - expectedFailures ? 1 : process.exitCode) } addInitScript (code) {