core(fr): add navigation runner (#11975)

lighthouse-core/fraggle-rock/api.js

@@ -7,8 +7,10 @@
 
 const {snapshot} = require('./gather/snapshot-runner.js');
 const {startTimespan} = require('./gather/timespan-runner.js');
+const {navigation} = require('./gather/navigation-runner.js');
 
 module.exports = {
   snapshot,
   startTimespan,
+  navigation,
 };

lighthouse-core/fraggle-rock/config/config.js

@@ -61,7 +61,11 @@ function resolveArtifactsToDefns(artifacts, configDir) {
 
   const coreGathererList = Runner.getGathererList();
   const artifactDefns = artifacts.map(artifactJson => {
-    const gatherer = resolveGathererToDefn(artifactJson.gatherer, coreGathererList, configDir);
+    /** @type {LH.Config.GathererJson} */
+    // @ts-expect-error FR-COMPAT - eventually move the config-helpers to support new types
+    const gathererJson = artifactJson.gatherer;
+
+    const gatherer = resolveGathererToDefn(gathererJson, coreGathererList, configDir);
     if (!isFRGathererDefn(gatherer)) {
       throw new Error(`${gatherer.instance.name} gatherer does not support Fraggle Rock`);
     }

lighthouse-core/fraggle-rock/gather/navigation-runner.js

@@ -0,0 +1,211 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const Driver = require('./driver.js');
+const Runner = require('../../runner.js');
+const {defaultNavigationConfig} = require('../../config/constants.js');
+const {initializeConfig} = require('../config/config.js');
+const {getBaseArtifacts} = require('./base-artifacts.js');
+
+/**
+ * @typedef NavigationContext
+ * @property {Driver} driver
+ * @property {LH.Config.NavigationDefn} navigation
+ * @property {string} requestedUrl
+ */
+
+/** @typedef {Record<string, Promise<any>>} IntermediateArtifacts */
+
+/**
+ * @param {{driver: Driver, config: LH.Config.FRConfig, requestedUrl: string}} args
+ */
+async function _setup({driver, config, requestedUrl}) {
+  await driver.connect();
+  // TODO(FR-COMPAT): use frameNavigated-based navigation
+  await driver._page.goto(defaultNavigationConfig.blankPage);
+
+  // TODO(FR-COMPAT): setupDriver
+
+  const baseArtifacts = getBaseArtifacts(config);
+  baseArtifacts.URL.requestedUrl = requestedUrl;
+
+  return {baseArtifacts};
+}
+
+/**
+ * @param {NavigationContext} navigationContext
+ */
+async function _setupNavigation({driver, navigation}) {
+  // TODO(FR-COMPAT): use frameNavigated-based navigation
+  await driver._page.goto(navigation.blankPage);
+
+  // TODO(FR-COMPAT): setup network conditions (throttling & cache state)
+}
+
+/**
+ * @param {NavigationContext} navigationContext
+ * @param {IntermediateArtifacts} artifacts
+ */
+async function _beforeTimespanPhase(navigationContext, artifacts) {
+  for (const artifactDefn of navigationContext.navigation.artifacts) {
+    const gatherer = artifactDefn.gatherer.instance;
+    if (!gatherer.meta.supportedModes.includes('timespan')) continue;
+
+    const artifactPromise = Promise.resolve().then(() =>
+      gatherer.beforeTimespan({driver: navigationContext.driver, gatherMode: 'navigation'})
+    );
+    artifacts[artifactDefn.id] = artifactPromise;
+    await artifactPromise.catch(() => {});
+  }
+}
+
+/**
+ * @param {NavigationContext} navigationContext
+ */
+async function _navigate(navigationContext) {
+  const {driver, requestedUrl} = navigationContext;
+  // TODO(FR-COMPAT): use waitForCondition-based navigation
+  await driver._page.goto(requestedUrl, {waitUntil: ['load', 'networkidle2']});
+
+  // TODO(FR-COMPAT): disable all throttling settings
+  // TODO(FR-COMPAT): capture page load errors
+}
+
+/**
+ * @param {NavigationContext} navigationContext
+ * @param {IntermediateArtifacts} artifacts
+ */
+async function _afterTimespanPhase(navigationContext, artifacts) {
+  for (const artifactDefn of navigationContext.navigation.artifacts) {
+    const gatherer = artifactDefn.gatherer.instance;
+    if (!gatherer.meta.supportedModes.includes('timespan')) continue;
+
+    const artifactPromise = (artifacts[artifactDefn.id] || Promise.resolve()).then(() =>
+      gatherer.afterTimespan({driver: navigationContext.driver, gatherMode: 'navigation'})
+    );
+    artifacts[artifactDefn.id] = artifactPromise;
+    await artifactPromise.catch(() => {});
+  }
+}
+
+/**
+ * @param {NavigationContext} navigationContext
+ * @param {IntermediateArtifacts} artifacts
+ */
+async function _snapshotPhase(navigationContext, artifacts) {
+  for (const artifactDefn of navigationContext.navigation.artifacts) {
+    const gatherer = artifactDefn.gatherer.instance;
+    if (!gatherer.meta.supportedModes.includes('snapshot')) continue;
+
+    const artifactPromise = Promise.resolve().then(() =>
+      gatherer.snapshot({driver: navigationContext.driver, gatherMode: 'navigation'})
+    );
+    artifacts[artifactDefn.id] = artifactPromise;
+    await artifactPromise.catch(() => {});
+  }
+}
+
+/**
+ * @param {IntermediateArtifacts} timespanArtifacts
+ * @param {IntermediateArtifacts} snapshotArtifacts
+ * @return {Promise<Partial<LH.GathererArtifacts>>}
+ */
+async function _mergeArtifacts(timespanArtifacts, snapshotArtifacts) {
+  /** @type {IntermediateArtifacts} */
+  const artifacts = {};
+  for (const [id, promise] of Object.entries({...timespanArtifacts, ...snapshotArtifacts})) {
+    artifacts[id] = await promise.catch(err => err);
+  }
+
+  return artifacts;
+}
+
+/**
+ * @param {NavigationContext} navigationContext
+ */
+async function _navigation(navigationContext) {
+  /** @type {IntermediateArtifacts} */
+  const timespanArtifacts = {};
+  /** @type {IntermediateArtifacts} */
+  const snapshotArtifacts = {};
+
+  await _setupNavigation(navigationContext);
+  await _beforeTimespanPhase(navigationContext, timespanArtifacts);
+  await _navigate(navigationContext);
+  await _afterTimespanPhase(navigationContext, timespanArtifacts);
+  await _snapshotPhase(navigationContext, snapshotArtifacts);
+
+  const artifacts = await _mergeArtifacts(timespanArtifacts, snapshotArtifacts);
+  return {artifacts};
+}
+
+/**
+ * @param {{driver: Driver, config: LH.Config.FRConfig, requestedUrl: string}} args
+ */
+async function _navigations({driver, config, requestedUrl}) {
+  if (!config.navigations) throw new Error('No navigations configured');
+
+  /** @type {Partial<LH.GathererArtifacts>} */
+  const artifacts = {};
+
+  for (const navigation of config.navigations) {
+    const navigationContext = {
+      driver,
+      navigation,
+      requestedUrl,
+    };
+
+    const navigationResult = await _navigation(navigationContext);
+    Object.assign(artifacts, navigationResult.artifacts);
+  }
+
+  return {artifacts};
+}
+
+/**
+ * @param {{driver: Driver}} args
+ */
+async function _cleanup({driver}) { // eslint-disable-line no-unused-vars
+  // TODO(FR-COMPAT): clear storage if necessary
+}
+
+/**
+ * @param {{url: string, page: import('puppeteer').Page, config?: LH.Config.Json}} options
+ * @return {Promise<LH.RunnerResult|undefined>}
+ */
+async function navigation(options) {
+  const {url: requestedUrl, page} = options;
+  const {config} = initializeConfig(options.config, {gatherMode: 'navigation'});
+
+  return Runner.run(
+    async () => {
+      const driver = new Driver(page);
+      const {baseArtifacts} = await _setup({driver, config, requestedUrl});
+      const {artifacts} = await _navigations({driver, config, requestedUrl});
+      await _cleanup({driver});
+
+      return /** @type {LH.Artifacts} */ ({...baseArtifacts, ...artifacts}); // Cast to drop Partial<>
+    },
+    {
+      url: requestedUrl,
+      config,
+    }
+  );
+}
+
+module.exports = {
+  navigation,
+  _setup,
+  _setupNavigation,
+  _beforeTimespanPhase,
+  _afterTimespanPhase,
+  _snapshotPhase,
+  _navigate,
+  _navigation,
+  _navigations,
+  _cleanup,
+};

lighthouse-core/test/fixtures/fraggle-rock/navigation-basic/index.html

@@ -0,0 +1,32 @@
+<!doctype html>
+<!--
+ * Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+-->
+<html lang="en">
+<head>
+<title></title>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0">
+</head>
+<body>
+  Hello, Fraggle Rock!
+
+  This page has some basic violations when the page has loaded.
+
+  <!-- FAIL(label) -->
+  <section>
+    <form>
+      <input
+        id="label"
+        type="text">
+    </form>
+  </section>
+
+  <script>
+    // FAIL(errors-in-console)
+    console.error('Accessibility violations added!');
+  </script>
+</body>
+</html>

lighthouse-core/test/fraggle-rock/api-test-pptr.js

@@ -132,4 +132,23 @@ describe('Fraggle Rock API', () => {
       if (!matchingLog) expect(errorLogs).toContain({description: /violations added/});
     });
   });
+
+  describe('navigation', () => {
+    beforeEach(() => {
+      server.baseDir = path.join(__dirname, '../fixtures/fraggle-rock/navigation-basic');
+    });
+
+    it('should compute both Accessibility & ConsoleMessage results', async () => {
+      const result = await lighthouse.navigation({page, url: `${serverBaseUrl}/index.html`});
+      if (!result) throw new Error('Lighthouse failed to produce a result');
+
+      const {lhr} = result;
+      const {failedAudits, erroredAudits} = getAuditsBreakdown(lhr);
+      expect(erroredAudits).toHaveLength(0);
+
+      const failedAuditIds = failedAudits.map(audit => audit.id);
+      expect(failedAuditIds).toContain('label');
+      expect(failedAuditIds).toContain('errors-in-console');
+    });
+  });
 });

lighthouse-core/test/fraggle-rock/config/config-test.js

@@ -147,4 +147,5 @@ describe('Fraggle Rock Config', () => {
   it.todo('should filter configuration by inclusive settings');
   it.todo('should filter configuration by exclusive settings');
   it.todo('should validate audit/gatherer interdependencies');
+  it.todo('should validate gatherers do not support all 3 modes');
 });

lighthouse-core/test/fraggle-rock/gather/mock-driver.js

@@ -0,0 +1,75 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-env jest */
+
+const {
+  createMockOnFn,
+  createMockOnceFn,
+  createMockSendCommandFn,
+} = require('../../gather/mock-commands.js');
+
+/**
+ * @fileoverview Mock fraggle rock driver for testing.
+ */
+
+/** @typedef {import('../../../fraggle-rock/gather/driver.js')} Driver */
+/** @typedef {import('../../../gather/driver/execution-context.js')} ExecutionContext */
+
+function createMockSession() {
+  return {
+    sendCommand: createMockSendCommandFn({useSessionId: false}),
+    once: createMockOnceFn(),
+    on: createMockOnFn(),
+    off: jest.fn(),
+  };
+}
+
+function createMockPage() {
+  return {
+    goto: jest.fn(),
+    target: () => ({createCDPSession: () => createMockSession()}),
+
+    /** @return {import('puppeteer').Page} */
+    asPage() {
+      // @ts-expect-error - We'll rely on the tests passing to know this matches.
+      return this;
+    },
+  };
+}
+
+function createMockExecutionContext() {
+  const context = /** @type {ExecutionContext} */ ({});
+  return {...context, evaluate: jest.fn(), evaluateAsync: jest.fn()};
+}
+
+function createMockDriver() {
+  const session = createMockSession();
+  const context = createMockExecutionContext();
+
+  return {
+    _page: createMockPage(),
+    _executionContext: context,
+    _session: session,
+    defaultSession: session,
+    connect: jest.fn(),
+    evaluate: context.evaluate,
+    evaluateAsync: context.evaluateAsync,
+
+    /** @return {Driver} */
+    asDriver() {
+      // @ts-expect-error - We'll rely on the tests passing to know this matches.
+      return this;
+    },
+  };
+}
+
+module.exports = {
+  createMockDriver,
+  createMockPage,
+  createMockSession,
+};