fix(testing): better error reporting

package.json

@@ -108,7 +108,7 @@
     "path-browserify": "0.0.0",
     "pixelmatch": "4.0.2",
     "postcss": "7.0.17",
-    "puppeteer": "~1.18.0",
+    "puppeteer": "~1.17.0",
     "rollup": "1.16.2",
     "rollup-plugin-commonjs": "10.0.0",
     "rollup-plugin-json": "^4.0.0",
@@ -131,7 +131,7 @@
     "jest": "~24.8.0",
     "jest-cli": "~24.8.0",
     "pixelmatch": "4.0.2",
-    "puppeteer": "~1.18.0",
+    "puppeteer": "~1.17.0",
     "workbox-build": "~4.3.1"
   },
   "engines": {

scripts/packages/internal/index.d.ts

@@ -14,4 +14,5 @@ export {
   PluginTransformResults,
   PrintLine
 } from '../dist/declarations';
+export * from '../dist/declarations/screenshots';
 export * from '../dist/declarations/docs';

src/compiler/config/validate-testing.ts

@@ -16,6 +16,10 @@ export function validateTesting(config: d.Config, diagnostics: d.Diagnostic[]) {
     testing.browserHeadless = true;
   }
 
+  if (!testing.browserWaitUntil) {
+    testing.browserWaitUntil = 'load';
+  }
+
   testing.browserArgs = testing.browserArgs || [];
   addOption(testing.browserArgs, '--disable-gpu');
   addOption(testing.browserArgs, '--disable-canvas-aa');
@@ -147,6 +151,15 @@ export function validateTesting(config: d.Config, diagnostics: d.Diagnostic[]) {
     );
   }
 
+  if (typeof testing.waitBeforeScreenshot === 'number') {
+    if (testing.waitBeforeScreenshot < 0) {
+      const err = buildError(diagnostics);
+      err.messageText = `waitBeforeScreenshot must be a value that is 0 or greater`;
+    }
+  } else {
+    testing.waitBeforeScreenshot = 10;
+  }
+
   if (!Array.isArray(testing.emulate) || testing.emulate.length === 0) {
     testing.emulate = [
       {

src/declarations/screenshots.ts

@@ -66,7 +66,7 @@ export interface ScreenshotConnectorOptions {
   allowableMismatchedPixels?: number;
   allowableMismatchedRatio?: number;
   pixelmatchThreshold?: number;
-  timeoutBeforeScreenshot?: number;
+  waitBeforeScreenshot?: number;
   pixelmatchModulePath?: string;
 }
 

src/declarations/testing.ts

@@ -186,6 +186,7 @@ export interface E2EProcessEnv {
   __STENCIL_BROWSER_URL__?: string;
   __STENCIL_APP_URL__?: string;
   __STENCIL_BROWSER_WS_ENDPOINT__?: string;
+  __STENCIL_BROWSER_WAIT_UNTIL?: string;
 
   __STENCIL_SCREENSHOT__?: 'true';
   __STENCIL_SCREENSHOT_BUILD__?: string;
@@ -403,6 +404,12 @@ export interface TestingConfig extends JestConfig {
    */
   browserSlowMo?: number;
 
+  /**
+   * By default, all E2E pages wait until the "load" event, this global setting can be used
+   * to change the default `waitUntil` behaviour.
+   */
+  browserWaitUntil?: 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2';
+
   /**
    * Whether to auto-open a DevTools panel for each tab.
    * If this option is true, the headless option will be set false
@@ -419,8 +426,12 @@ export interface TestingConfig extends JestConfig {
    * Path to the Screenshot Connector module.
    */
   screenshotConnector?: string;
-}
 
+  /**
+   * Amount of time in milliseconds to wait before a screenshot is taken.
+   */
+  waitBeforeScreenshot?: number;
+}
 
 export interface EmulateConfig {
   /**

src/screenshot/connector-base.ts

@@ -31,7 +31,7 @@ export class ScreenshotConnector implements d.ScreenshotConnector {
   allowableMismatchedRatio: number;
   allowableMismatchedPixels: number;
   pixelmatchThreshold: number;
-  timeoutBeforeScreenshot: number;
+  waitBeforeScreenshot: number;
   pixelmatchModulePath: string;
 
   async initBuild(opts: d.ScreenshotConnectorOptions) {
@@ -47,7 +47,7 @@ export class ScreenshotConnector implements d.ScreenshotConnector {
     this.packageDir = opts.packageDir;
     this.rootDir = opts.rootDir;
     this.appNamespace = opts.appNamespace;
-    this.timeoutBeforeScreenshot = typeof opts.timeoutBeforeScreenshot === 'number' ? opts.timeoutBeforeScreenshot : 4;
+    this.waitBeforeScreenshot = opts.waitBeforeScreenshot;
     this.pixelmatchModulePath = opts.pixelmatchModulePath;
 
     if (!opts.logger) {
@@ -300,7 +300,7 @@ export class ScreenshotConnector implements d.ScreenshotConnector {
       allowableMismatchedPixels: this.allowableMismatchedPixels,
       allowableMismatchedRatio: this.allowableMismatchedRatio,
       pixelmatchThreshold: this.pixelmatchThreshold,
-      timeoutBeforeScreenshot: this.timeoutBeforeScreenshot,
+      timeoutBeforeScreenshot: this.waitBeforeScreenshot,
       pixelmatchModulePath: this.pixelmatchModulePath
     };
 

src/testing/jest/jest-config.ts

@@ -9,14 +9,15 @@ export function buildJestArgv(config: d.Config) {
     ...config.flags.knownArgs.slice()
   ];
 
-  if (config.flags.e2e && (config.flags.ci || config.flags.devtools)) {
-    if (!args.some(a => a.startsWith('--max-workers') || a.startsWith('--maxWorkers'))) {
-      args.push('--max-workers=1');
-    }
+  if (!args.some(a => a.startsWith('--max-workers') || a.startsWith('--maxWorkers'))) {
+    args.push(`--max-workers=${config.maxConcurrentWorkers}`);
+  }
+
+  if (config.flags.devtools) {
     args.push('--runInBand');
   }
 
-  config.logger.debug(`jest args: ${args.join(' ')}`);
+  config.logger.info(config.logger.magenta(`jest args: ${args.join(' ')}`));
 
   const { options } = require('jest-cli/build/cli/args');
   const jestArgv = yargs(args).options(options).argv as d.JestArgv;

src/testing/jest/jest-screenshot.ts

@@ -9,7 +9,6 @@ export async function runJestScreenshot(config: d.Config, env: d.E2EProcessEnv)
   const connector: d.ScreenshotConnector = new ScreenshotConnector();
 
   // for CI, let's wait a little longer than locally before taking the screenshot
-  const timeoutBeforeScreenshot = config.flags.ci ? 30 : 10;
   const pixelmatchModulePath = config.sys.path.join(config.sys.compiler.packageDir, 'screenshot', 'pixel-match.js');
   config.logger.debug(`pixelmatch module: ${pixelmatchModulePath}`);
 
@@ -27,7 +26,7 @@ export async function runJestScreenshot(config: d.Config, env: d.E2EProcessEnv)
     allowableMismatchedPixels: config.testing.allowableMismatchedPixels,
     allowableMismatchedRatio: config.testing.allowableMismatchedRatio,
     pixelmatchThreshold: config.testing.pixelmatchThreshold,
-    timeoutBeforeScreenshot: timeoutBeforeScreenshot,
+    waitBeforeScreenshot: config.testing.waitBeforeScreenshot,
     pixelmatchModulePath: pixelmatchModulePath
   });
 

src/testing/puppeteer/puppeteer-browser.ts

@@ -13,6 +13,7 @@ export async function startPuppeteerBrowser(config: d.Config) {
   const puppeteerModulePath = config.sys.lazyRequire.getModulePath('puppeteer');
   const puppeteer = require(puppeteerModulePath);
   env.__STENCIL_PUPPETEER_MODULE__ = puppeteerModulePath;
+  env.__STENCIL_BROWSER_WAIT_UNTIL = config.testing.browserWaitUntil;
 
   if (config.flags.devtools) {
     config.testing.browserDevtools = true;
@@ -47,12 +48,14 @@ export async function startPuppeteerBrowser(config: d.Config) {
     launchOpts.executablePath = config.testing.browserExecutablePath;
   }
 
-  const browser = (config.testing.browserWSEndpoint)
-    ? await puppeteer.connect({
+  const browser = await ((config.testing.browserWSEndpoint)
+    ? puppeteer.connect({
         ...launchOpts,
         browserWSEndpoint: config.testing.browserWSEndpoint
       })
-    : await puppeteer.launch(launchOpts);
+    : puppeteer.launch({
+        ...launchOpts,
+      }));
 
   env.__STENCIL_BROWSER_WS_ENDPOINT__ = browser.wsEndpoint();
 

src/testing/puppeteer/puppeteer-page.ts

@@ -10,15 +10,15 @@ import * as puppeteer from 'puppeteer';
 declare const global: d.JestEnvironmentGlobal;
 
 
+const env: d.E2EProcessEnv = process.env;
 export async function newE2EPage(opts: pd.NewE2EPageOptions = {}): Promise<pd.E2EPage> {
   if (!global.__NEW_TEST_PAGE__) {
-    console.error(`newE2EPage() is only available from E2E tests, and ran with the --e2e cmd line flag.`);
-    return failedPage();
+    throw new Error(`newE2EPage() is only available from E2E tests, and ran with the --e2e cmd line flag.`);
   }
 
-  try {
-    const page: pd.E2EPageInternal = await global.__NEW_TEST_PAGE__();
+  const page: pd.E2EPageInternal = await global.__NEW_TEST_PAGE__();
 
+  try {
     page._e2eElements = [];
 
     page._e2eGoto = page.goto;
@@ -43,7 +43,7 @@ export async function newE2EPage(opts: pd.NewE2EPageOptions = {}): Promise<pd.E2
     };
 
     page.debugger = () => {
-      if ((process.env as d.E2EProcessEnv).__STENCIL_E2E_DEVTOOLS__ !== 'true') {
+      if (env.__STENCIL_E2E_DEVTOOLS__ !== 'true') {
         throw new Error('Set the --devtools flag in order to use E2EPage.debugger()');
       }
       return page.evaluate(() => {
@@ -86,23 +86,12 @@ export async function newE2EPage(opts: pd.NewE2EPageOptions = {}): Promise<pd.E2
       page.goto = e2eGoTo.bind(null, page);
       page.setContent = e2eSetContent.bind(null, page);
     }
-
-    return page;
-
   } catch (e) {
-    console.error(e);
+    if (page) {
+      await closePage(page);
+    }
+    throw e;
   }
-
-  return failedPage();
-}
-
-function failedPage() {
-  const page: pd.E2EPageInternal = {
-    isClosed: () => true,
-    setContent: () => Promise.resolve(),
-    waitForChanges: () => Promise.resolve(),
-    find: () => Promise.resolve(null)
-  } as any;
   return page;
 }
 
@@ -125,7 +114,7 @@ async function e2eGoTo(page: pd.E2EPageInternal, url: string, options: puppeteer
     return 'gotoTest() url must start with /';
   }
 
-  const browserUrl = (process.env as d.E2EProcessEnv).__STENCIL_BROWSER_URL__;
+  const browserUrl = env.__STENCIL_BROWSER_URL__;
   if (typeof browserUrl !== 'string') {
     await closePage(page);
     return 'invalid gotoTest() browser url';
@@ -134,7 +123,7 @@ async function e2eGoTo(page: pd.E2EPageInternal, url: string, options: puppeteer
   const fullUrl = browserUrl + url.substring(1);
 
   if (!options.waitUntil) {
-    options.waitUntil = DEFAULT_WAIT_FOR;
+    options.waitUntil = env.__STENCIL_BROWSER_WAIT_UNTIL as any;
   }
   const rsp = await page._e2eGoto(fullUrl, options);
 
@@ -168,13 +157,13 @@ async function e2eSetContent(page: pd.E2EPageInternal, html: string, options: pu
     return 'invalid e2eSetContent() html';
   }
 
-  const appUrl = (process.env as d.E2EProcessEnv).__STENCIL_APP_URL__;
+  const appUrl = env.__STENCIL_APP_URL__;
   if (typeof appUrl !== 'string') {
     await closePage(page);
     return 'invalid e2eSetContent() app script url';
   }
 
-  const url = (process.env as d.E2EProcessEnv).__STENCIL_BROWSER_URL__;
+  const url = env.__STENCIL_BROWSER_URL__;
   const body = [
     `<script type="module" src="${appUrl}"></script>`,
     html
@@ -197,7 +186,7 @@ async function e2eSetContent(page: pd.E2EPageInternal, html: string, options: pu
   });
 
   if (!options.waitUntil) {
-    options.waitUntil = DEFAULT_WAIT_FOR;
+    options.waitUntil = env.__STENCIL_BROWSER_WAIT_UNTIL as any;
   }
   const rsp = await page._e2eGoto(url, options);
 
@@ -227,8 +216,6 @@ async function setPageEmulate(page: puppeteer.Page) {
     return;
   }
 
-  const env = (process.env) as d.E2EProcessEnv;
-
   const emulateJsonContent = env.__STENCIL_EMULATE__;
   if (!emulateJsonContent) {
     return;
@@ -328,5 +315,3 @@ function pageError(e: any) {
 function requestFailed(req: puppeteer.Request) {
   console.error('requestfailed', req.url());
 }
-
-const DEFAULT_WAIT_FOR = 'load';

src/testing/testing.ts

@@ -147,7 +147,7 @@ export class Testing implements d.Testing {
 function setupTestingConfig(config: d.Config) {
   config.buildEs5 = false;
   config.devMode = true;
-  config.maxConcurrentWorkers = 1;
+  config.maxConcurrentWorkers = Math.min(config.maxConcurrentWorkers, 6);
   config.validateTypes = false;
   config._isTesting = true;