Skip to content

Commit

Permalink
wip [skip ci]
Browse files Browse the repository at this point in the history
  • Loading branch information
@panva
panva committed Feb 3, 2023
1 parent 9fafb0a commit 1e5d25e
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 59 deletions.
6 changes: 6 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,12 @@ test-message: test-build
test-wpt: all
$(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt

.PHONY: test-wpt-report
test-wpt-report: all
$(RM) -r out/wpt
mkdir -p out/wpt
WPTREPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt

.PHONY: test-simple
test-simple: | cctest # Depends on 'all'.
$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential
Expand Down
195 changes: 137 additions & 58 deletions test/common/wpt.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,67 @@ const events = require('events');
const { inspect } = require('util');
const { Worker } = require('worker_threads');

class WPTReport {
constructor() {
this.results = [];
this.time_start = Date.now();
}

addResult(name, status) {
const result = {
test: name,
status,
subtests: [],
addSubtest(name, status, message) {
const subtest = {
status,
name,
};
if (message) subtest.message = message;
this.subtests.push(subtest);
return subtest;
},
};
this.results.push(result);
return result;
}

finalize() {
this.time_end = Date.now();
for (const result of this.results) {
result.test = result.test.replace(/\.js(:\?|$)/, '.html');
}
this.results = this.results.filter((result) => {
return result.status === 'SKIP' || result.subtests.length !== 0;
});
}

write() {
this.finalize();
this.run_info = {
product: 'Node.js',
browser_channel: process.release.sourceUrl ? 'stable' : 'experimental',
browser_version: process.version,
};
if (fs.existsSync('out/wpt/wptreport.json')) {
const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json'));
this.results = [...prev.results, ...this.results];
this.time_start = Math.min(this.time_start, prev.time_start);
this.time_end = Math.max(this.time_end, prev.time_end);
}
const { results, time_start, time_end } = this;
delete this.results;
delete this.time_start;
delete this.time_end;

this.time_start = time_start;
this.time_end = time_end;
this.results = results;

fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this, null, 2));
}
}

// https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
// TODO: get rid of this half-baked harness in favor of the one
// pulled from WPT
Expand Down Expand Up @@ -313,6 +374,10 @@ class WPTRunner {
this.unexpectedFailures = [];

this.scriptsModifier = null;

if (process.env.WPTREPORT) {
this.report = new WPTReport();
}
}

/**
Expand All @@ -339,18 +404,23 @@ class WPTRunner {
this.scriptsModifier = modifier;
}

get fullInitScript() {
fullInitScript(locationSearchString) {
let { initScript } = this;
if (locationSearchString) {
initScript = `${initScript}\n\n//===\nglobalThis.location &&= { search: "${locationSearchString}" };`
}

if (this.globalThisInitScripts.length === null) {
return this.initScript;
return initScript;
}

const globalThisInitScript = this.globalThisInitScripts.join('\n\n//===\n');

if (this.initScript === null) {
if (initScript === null) {
return globalThisInitScript;
}

return `${globalThisInitScript}\n\n//===\n${this.initScript}`;
return `${globalThisInitScript}\n\n//===\n${initScript}`;
}

/**
Expand Down Expand Up @@ -455,7 +525,7 @@ class WPTRunner {
for (const spec of queue) {
const testFileName = spec.filename;
const content = spec.getContent();
const meta = spec.title = this.getMeta(content);
const meta = spec.meta = this.getMeta(content);

const absolutePath = spec.getAbsolutePath();
const relativePath = spec.getRelativePath();
Expand All @@ -480,54 +550,58 @@ class WPTRunner {
this.scriptsModifier?.(obj);
scriptsToRun.push(obj);

const workerPath = path.join(__dirname, 'wpt/worker.js');
const worker = new Worker(workerPath, {
execArgv: this.flags,
workerData: {
testRelativePath: relativePath,
wptRunner: __filename,
wptPath: this.path,
initScript: this.fullInitScript,
harness: {
code: fs.readFileSync(harnessPath, 'utf8'),
filename: harnessPath,
for (const variant of meta.variant || ['']) {
const workerPath = path.join(__dirname, 'wpt/worker.js');
const worker = new Worker(workerPath, {
execArgv: this.flags,
workerData: {
testRelativePath: relativePath,
wptRunner: __filename,
wptPath: this.path,
initScript: this.fullInitScript(variant),
harness: {
code: fs.readFileSync(harnessPath, 'utf8'),
filename: harnessPath,
},
scriptsToRun,
},
scriptsToRun,
},
});
this.workers.set(testFileName, worker);

worker.on('message', (message) => {
switch (message.type) {
case 'result':
return this.resultCallback(testFileName, message.result);
case 'completion':
return this.completionCallback(testFileName, message.status);
default:
throw new Error(`Unexpected message from worker: ${message.type}`);
}
});
});
this.workers.set(testFileName, worker);

let reportResult;
worker.on('message', (message) => {
switch (message.type) {
case 'result':
reportResult ||= this.report?.addResult(`/${relativePath}${variant}`, 'OK');
return this.resultCallback(testFileName, message.result, reportResult);
case 'completion':
return this.completionCallback(testFileName, message.status);
default:
throw new Error(`Unexpected message from worker: ${message.type}`);
}
});

worker.on('error', (err) => {
if (!this.inProgress.has(testFileName)) {
// The test is already finished. Ignore errors that occur after it.
// This can happen normally, for example in timers tests.
return;
}
this.fail(
testFileName,
{
status: NODE_UNCAUGHT,
name: 'evaluation in WPTRunner.runJsTests()',
message: err.message,
stack: inspect(err),
},
kUncaught,
);
this.inProgress.delete(testFileName);
});
worker.on('error', (err) => {
if (!this.inProgress.has(testFileName)) {
// The test is already finished. Ignore errors that occur after it.
// This can happen normally, for example in timers tests.
return;
}
this.fail(
testFileName,
{
status: NODE_UNCAUGHT,
name: 'evaluation in WPTRunner.runJsTests()',
message: err.message,
stack: inspect(err),
},
kUncaught,
);
this.inProgress.delete(testFileName);
});

await events.once(worker, 'exit').catch(() => {});
await events.once(worker, 'exit').catch(() => {});
}
}

process.on('exit', () => {
Expand Down Expand Up @@ -587,6 +661,8 @@ class WPTRunner {
}
}

this.report?.write();

const ran = queue.length;
const total = ran + skipped;
const passed = ran - expectedFailures - failures.length;
Expand All @@ -611,8 +687,7 @@ class WPTRunner {

getTestTitle(filename) {
const spec = this.specMap.get(filename);
const title = spec.meta && spec.meta.title;
return title ? `${filename} : ${title}` : filename;
return spec.meta?.title || filename;
}

// Map WPT test status to strings
Expand All @@ -638,14 +713,14 @@ class WPTRunner {
* @param {string} filename
* @param {Test} test The Test object returned by WPT harness
*/
resultCallback(filename, test) {
resultCallback(filename, test, reportResult) {
const status = this.getTestStatus(test.status);
const title = this.getTestTitle(filename);
console.log(`---- ${title} ----`);
if (status !== kPass) {
this.fail(filename, test, status);
this.fail(filename, test, status, reportResult);
} else {
this.succeed(filename, test, status);
this.succeed(filename, test, status, reportResult);
}
}

Expand Down Expand Up @@ -693,11 +768,12 @@ class WPTRunner {
}
}

succeed(filename, test, status) {
succeed(filename, test, status, reportResult) {
console.log(`[${status.toUpperCase()}] ${test.name}`);
reportResult?.addSubtest(test.name, 'PASS');
}

fail(filename, test, status) {
fail(filename, test, status, reportResult) {
const spec = this.specMap.get(filename);
const expected = spec.failedTests.includes(test.name);
if (expected) {
Expand All @@ -713,6 +789,9 @@ class WPTRunner {
const command = `${process.execPath} ${process.execArgv}` +
` ${require.main.filename} ${filename}`;
console.log(`Command: ${command}\n`);

reportResult?.addSubtest(test.name, 'FAIL', test.message);

this.addTestResult(filename, {
name: test.name,
expected,
Expand Down Expand Up @@ -742,7 +821,7 @@ class WPTRunner {
const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
const key = parts[1];
const value = parts[2];
if (key === 'script') {
if (key === 'script' || key === 'variant') {
if (result[key]) {
result[key].push(value);
} else {
Expand Down
2 changes: 1 addition & 1 deletion test/wpt/test-webcrypto.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const { WPTRunner } = require('../common/wpt');
const runner = new WPTRunner('WebCryptoAPI');

runner.setInitScript(`
global.location = {};
globalThis.location ||= {};
`);

runner.pretendGlobalThisAs('Window');
Expand Down

0 comments on commit 1e5d25e

Please sign in to comment.