From 3fafff964789ee0550bdaa363b7a996961fa3e6e Mon Sep 17 00:00:00 2001 From: Domenic Denicola Date: Sun, 19 Nov 2017 17:12:26 -0500 Subject: [PATCH] Start web platform tests server before tests run This is an attempt to address recent flaky builds, which we think might be caused because the server is starting in the middle of a JSDOM API test, causing a timeout for the currently-running JSDOM API test. The previous design started booting the server the first time it was needed, and made all tests wait for the server-creation promise before commencing their actual logic. Now, server-creation runs in a before() block. In the process, it removes the fallback to w3c-test.org, as that configuration isn't tested and generally can cause issues. --- test/web-platform-tests/run-single-wpt.js | 91 +-------------------- test/web-platform-tests/run-tuwpts.js | 12 ++- test/web-platform-tests/run-wpts.js | 12 ++- test/web-platform-tests/start-wpt-server.js | 74 +++++++++++++++++ 4 files changed, 97 insertions(+), 92 deletions(-) create mode 100644 test/web-platform-tests/start-wpt-server.js diff --git a/test/web-platform-tests/run-single-wpt.js b/test/web-platform-tests/run-single-wpt.js index 3c8f002b3a..68f949d6d6 100644 --- a/test/web-platform-tests/run-single-wpt.js +++ b/test/web-platform-tests/run-single-wpt.js @@ -1,95 +1,20 @@ "use strict"; -/* eslint-disable no-console, no-process-exit, global-require */ +/* eslint-disable no-console */ const fs = require("fs"); const path = require("path"); -const dns = require("dns"); -const childProcess = require("child_process"); -const { EventEmitter } = require("events"); -const q = require("q"); const { specify } = require("mocha-sugar-free"); const { inBrowserContext, nodeResolverPromise } = require("../util.js"); -const requestHead = require("request-promise-native").head; const jsdom = require("../../lib/old-api.js"); -const wptDir = path.resolve(__dirname, "tests"); - -const configPaths = { - default: path.resolve(__dirname, "wpt-config.json"), - toUpstream: path.resolve(__dirname, "tuwpt-config.json") -}; - -const configs = { - default: require(configPaths.default), - toUpstream: require(configPaths.toUpstream) -}; - const globalPool = { maxSockets: 6 }; -module.exports = ({ toUpstream = false } = {}) => { +module.exports = urlPrefixFactory => { if (inBrowserContext()) { return () => { // TODO: browser support for running WPT }; } - const configType = toUpstream ? "toUpstream" : "default"; - const configPath = configPaths[configType]; - const config = configs[configType]; - - const server = new EventEmitter(); - - let serverHasStarted; - server.started = new Promise(resolve => { - serverHasStarted = resolve; - }); - server.isStarted = false; - - let urlPrefix = `http://${config.host}:${config.ports.http[0]}/`; - - dns.lookup("web-platform.test", err => { - if (err) { - console.warn(); - console.warn("Host entries not present for web platform tests."); - console.warn("See https://github.com/w3c/web-platform-tests#running-the-tests"); - - if (!toUpstream) { - console.warn("Falling back to hosted versions at w3c-test.org"); - urlPrefix = "http://w3c-test.org/"; - } - serverHasStarted(); - return; - } - - const configArg = path.relative(path.resolve(wptDir), configPath); - const args = ["./wpt.py", "serve", "--config", configArg]; - const python = childProcess.spawn("python", args, { - cwd: wptDir, - stdio: "inherit" - }); - - python.on("error", e => { - console.warn(); - console.warn("Error starting python server process:", e.message); - - if (toUpstream) { - console.error("Cannot proceed with running the tests."); - process.exit(1); - } else { - console.warn("Falling back to hosted versions at w3ctest.org"); - urlPrefix = "http://w3c-test.org/"; - serverHasStarted(); - } - }); - - pollForServer(() => urlPrefix).then(serverHasStarted); - - process.on("exit", () => { - // Python doesn't register a default handler for SIGTERM and it doesn't run __exit__() methods of context managers - // when it gets that signal. Using SIGINT avoids this problem - python.kill("SIGINT"); - }); - }); - return (testPath, title = testPath) => { specify({ title, @@ -99,22 +24,12 @@ module.exports = ({ toUpstream = false } = {}) => { slow: 10000, skipIfBrowser: true, fn() { - return server.started.then(() => createJSDOM(urlPrefix, testPath)); + return createJSDOM(urlPrefixFactory(), testPath); } }); }; }; -function pollForServer(urlGetter) { - console.log("Checking if the web platform tests server is up"); - return requestHead(urlGetter()) - .then(() => console.log("Server is up!")) - .catch(err => { - console.log(`Server is not up yet (${err.message}); trying again`); - return q.delay(500).then(() => pollForServer(urlGetter)); - }); -} - function createJSDOM(urlPrefix, testPath) { const reporterPathname = "/resources/testharnessreport.js"; const unhandledExceptions = []; diff --git a/test/web-platform-tests/run-tuwpts.js b/test/web-platform-tests/run-tuwpts.js index f7b69cea8a..65a99fc0fc 100644 --- a/test/web-platform-tests/run-tuwpts.js +++ b/test/web-platform-tests/run-tuwpts.js @@ -1,13 +1,13 @@ "use strict"; const path = require("path"); -const { describe } = require("mocha-sugar-free"); +const { describe, before } = require("mocha-sugar-free"); const { spawnSync } = require("child_process"); const { readManifest, getPossibleTestFilePaths } = require("./wpt-manifest-utils.js"); +const startWPTServer = require("./start-wpt-server.js"); const wptPath = path.resolve(__dirname, "tests"); const testsPath = path.resolve(__dirname, "to-upstream"); const manifestFilename = path.resolve(__dirname, "tuwpt-manifest.json"); -const runSingleWPT = require("./run-single-wpt.js")({ toUpstream: true }); // We can afford to re-generate the manifest each time; we have few enough files that it's cheap. const testsRootArg = path.relative(wptPath, testsPath); @@ -18,6 +18,14 @@ spawnSync("python", args, { cwd: wptPath, stdio: "inherit" }); const manifest = readManifest(manifestFilename); const possibleTestFilePaths = getPossibleTestFilePaths(manifest); +let wptServerURL; +const runSingleWPT = require("./run-single-wpt.js")(() => wptServerURL); +before({ timeout: 30 * 1000 }, () => { + return startWPTServer({ toUpstream: true }).then(url => { + wptServerURL = url; + }); +}); + describe("Local tests in web-platform-test format (to-upstream)", () => { for (const test of possibleTestFilePaths) { runSingleWPT(test); diff --git a/test/web-platform-tests/run-wpts.js b/test/web-platform-tests/run-wpts.js index d9b475ad7a..3aaaf05b54 100644 --- a/test/web-platform-tests/run-wpts.js +++ b/test/web-platform-tests/run-wpts.js @@ -3,9 +3,9 @@ const path = require("path"); const fs = require("fs"); const jsYAML = require("js-yaml"); const { Minimatch } = require("minimatch"); -const { describe, specify } = require("mocha-sugar-free"); +const { describe, specify, before } = require("mocha-sugar-free"); const { readManifest, getPossibleTestFilePaths, stripPrefix } = require("./wpt-manifest-utils.js"); -const runSingleWPT = require("./run-single-wpt.js")({ toUpstream: false }); +const startWPTServer = require("./start-wpt-server.js"); const validReasons = new Set(["fail", "timeout", "needs-await", "needs-node8"]); @@ -30,6 +30,14 @@ const minimatchers = new Map(); checkToRun(); +let wptServerURL; +const runSingleWPT = require("./run-single-wpt.js")(() => wptServerURL); +before({ timeout: 30 * 1000 }, () => { + return startWPTServer({ toUpstream: false }).then(url => { + wptServerURL = url; + }); +}); + describe("web-platform-tests", () => { for (const toRunDoc of toRunDocs) { describe(toRunDoc.DIR, () => { diff --git a/test/web-platform-tests/start-wpt-server.js b/test/web-platform-tests/start-wpt-server.js new file mode 100644 index 0000000000..e4c3731b1c --- /dev/null +++ b/test/web-platform-tests/start-wpt-server.js @@ -0,0 +1,74 @@ +"use strict"; +/* eslint-disable no-console, global-require */ +const path = require("path"); +const dns = require("dns"); +const childProcess = require("child_process"); +const q = require("q"); +const { inBrowserContext } = require("../util.js"); +const requestHead = require("request-promise-native").head; +const dnsLookup = q.denodeify(dns.lookup); + +const wptDir = path.resolve(__dirname, "tests"); + +const configPaths = { + default: path.resolve(__dirname, "wpt-config.json"), + toUpstream: path.resolve(__dirname, "tuwpt-config.json") +}; + +const configs = { + default: require(configPaths.default), + toUpstream: require(configPaths.toUpstream) +}; + +module.exports = ({ toUpstream = false } = {}) => { + if (inBrowserContext()) { + return Promise.resolve(); + } + + const configType = toUpstream ? "toUpstream" : "default"; + const configPath = configPaths[configType]; + const config = configs[configType]; + + const urlPrefix = `http://${config.host}:${config.ports.http[0]}/`; + + return dnsLookup("web-platform.test").then( + () => { + const configArg = path.relative(path.resolve(wptDir), configPath); + const args = ["./wpt.py", "serve", "--config", configArg]; + const python = childProcess.spawn("python", args, { + cwd: wptDir, + stdio: "inherit" + }); + + return new Promise((resolve, reject) => { + python.on("error", e => { + reject(new Error("Error starting python server process:", e.message)); + }); + + resolve(pollForServer(urlPrefix)); + + process.on("exit", () => { + // Python doesn't register a default handler for SIGTERM and it doesn't run __exit__() methods of context + // managers when it gets that signal. Using SIGINT avoids this problem. + python.kill("SIGINT"); + }); + }); + }, + () => { + throw new Error("Host entries not present for web platform tests. See " + + "https://github.com/w3c/web-platform-tests#running-the-tests"); + } + ); +}; + +function pollForServer(url) { + return requestHead(url) + .then(() => { + console.log(`WPT server at ${url} is up!`); + return url; + }) + .catch(err => { + console.log(`WPT server at ${url} is not up yet (${err.message}); trying again`); + return q.delay(500).then(() => pollForServer(url)); + }); +}