diff --git a/examples/simple_benchmark/best.prod.config.js b/examples/simple_benchmark/best.prod.config.js index 32345e76..7c51610b 100644 --- a/examples/simple_benchmark/best.prod.config.js +++ b/examples/simple_benchmark/best.prod.config.js @@ -15,6 +15,15 @@ module.exports = { "options": { path: '/best' }, "remoteRunner": "@best/runner-headless" } + }, + { + "runner": '@best/runner-remote', + "name": "remote", + "config": { + "host": "http://best-agent-pool.lwcjs.org/", + "options": { path: '/best' }, + "remoteRunner": "@best/runner-headless" + } } ], }; diff --git a/packages/best-config/src/defaults.js b/packages/best-config/src/defaults.js index 02f9c473..98bd0750 100644 --- a/packages/best-config/src/defaults.js +++ b/packages/best-config/src/defaults.js @@ -10,7 +10,7 @@ export default { runner: "default", runnerConfig: [{ runner: '@best/runner-headless', config: {} }], benchmarkEnvironment: 'production', - benchmarkMaxDuration: 1000 * 10, // 10s + benchmarkMaxDuration: 1000 * 15, // 15s benchmarkMinIterations: 30, benchmarkOnClient: false, benchmarkIterations: undefined, diff --git a/packages/best-runner-headless/src/__tests__/runner-headless.spec.js b/packages/best-runner-headless/src/__tests__/runner-headless.spec.js index 1cae16b6..f88f5956 100644 --- a/packages/best-runner-headless/src/__tests__/runner-headless.spec.js +++ b/packages/best-runner-headless/src/__tests__/runner-headless.spec.js @@ -79,7 +79,7 @@ describe('errors', () => { }; return expect(run(benchmarkConfig, PROJECT_CONFIG, GLOBAL_CONFIG, MOCK_MESSAGER)).rejects.toThrow( - /BEST is not defined/, + /Benchmark parse error/, ); }); diff --git a/packages/best-runner-headless/src/index.js b/packages/best-runner-headless/src/index.js index c7fc339b..fe4359d1 100644 --- a/packages/best-runner-headless/src/index.js +++ b/packages/best-runner-headless/src/index.js @@ -62,6 +62,12 @@ async function runServerIterations(page, state, opts, messager) { } async function runIterations(page, state, opts, messager) { + // TODO: Throw on timeouts, current logic is + // currently non-existant for clientside iteration mode + // if (state.executedTime > opts.maxDuration) { + // throw new Error('Benchmark timmed out'); + // } + if (state.iterateOnClient) { return runClientIterations(page, state, opts, messager); } @@ -130,6 +136,7 @@ export async function run({ benchmarkName, benchmarkEntry }, projectConfig, glob const { projectName } = projectConfig; let browser; + let parseError; try { browser = await puppeteer.launch(PUPPETEER_OPTIONS); const environment = await normalizeEnvironment(browser, projectConfig, globalConfig); @@ -137,8 +144,17 @@ export async function run({ benchmarkName, benchmarkEntry }, projectConfig, glob messager.onBenchmarkStart(benchmarkName, projectName); const page = await browser.newPage(); + page.on('pageerror', (err) => (parseError = err)); await page.goto('file:///' + benchmarkEntry); + // page.goto() will wait for the onload + // if we caught something that throws there is a parsing error in the benchmark code + if (parseError) { + messager.onBenchmarkError(benchmarkName, projectName); + parseError.message = 'Benchmark parse error.\n' + parseError.message; + throw parseError; + } + const { results } = await runIterations(page, state, opts, messager); return { results, environment }; } catch (e) { diff --git a/packages/best-runtime/src/runner.js b/packages/best-runtime/src/runner.js index 53329959..ffe0702f 100644 --- a/packages/best-runtime/src/runner.js +++ b/packages/best-runtime/src/runner.js @@ -1,6 +1,7 @@ import { getBenchmarkRootNode } from './state'; import { runBenchmarkIteration } from './run_iteration'; import { normalizeResults } from './results'; +import { validateState } from "./utils/validate"; function collectResults(node) { const { name, duration, startedAt, run } = node; @@ -38,6 +39,7 @@ async function runIterations(config) { } export async function runBenchmark(benchmarkState) { + validateState(benchmarkState); if (benchmarkState.benchmarkDefinitionError) { throw benchmarkState.benchmarkDefinitionError; } diff --git a/packages/best-runtime/src/utils/validate.js b/packages/best-runtime/src/utils/validate.js new file mode 100644 index 00000000..f928f8db --- /dev/null +++ b/packages/best-runtime/src/utils/validate.js @@ -0,0 +1,10 @@ +export function validateState(benchmarkState) { + const { rootDescribeBlock, currentDescribeBlock } = benchmarkState; + if (rootDescribeBlock !== currentDescribeBlock) { + benchmarkState.benchmarkDefinitionError = 'Benchmark parsing error'; + } + + if (rootDescribeBlock.children === 0) { + benchmarkState.benchmarkDefinitionError = 'No benchmarks to run'; + } +}