core(fr): convert main-document-content gatherer (#12470)

lighthouse-core/fraggle-rock/config/default-config.js

@@ -28,6 +28,7 @@ const artifacts = {
   InstallabilityErrors: '',
   JsUsage: '',
   LinkElements: '',
+  MainDocumentContent: '',
   MetaElements: '',
   NetworkUserAgent: '',
   PasswordInputsWithPreventedPaste: '',
@@ -70,6 +71,7 @@ const defaultConfig = {
     {id: artifacts.InstallabilityErrors, gatherer: 'installability-errors'},
     {id: artifacts.JsUsage, gatherer: 'js-usage'},
     {id: artifacts.LinkElements, gatherer: 'link-elements'},
+    {id: artifacts.MainDocumentContent, gatherer: 'main-document-content'},
     {id: artifacts.MetaElements, gatherer: 'meta-elements'},
     {id: artifacts.NetworkUserAgent, gatherer: 'network-user-agent'},
     {id: artifacts.PasswordInputsWithPreventedPaste, gatherer: 'dobetterweb/password-inputs-with-prevented-paste'},
@@ -110,6 +112,7 @@ const defaultConfig = {
         artifacts.InstallabilityErrors,
         artifacts.JsUsage,
         artifacts.LinkElements,
+        artifacts.MainDocumentContent,
         artifacts.MetaElements,
         artifacts.NetworkUserAgent,
         artifacts.PasswordInputsWithPreventedPaste,

lighthouse-core/gather/driver.js

@@ -8,7 +8,7 @@
 const Fetcher = require('./fetcher.js');
 const ExecutionContext = require('./driver/execution-context.js');
 const LHError = require('../lib/lh-error.js');
-const NetworkRequest = require('../lib/network-request.js');
+const {fetchResponseBodyFromCache} = require('../gather/driver/network.js');
 const EventEmitter = require('events').EventEmitter;
 
 const log = require('lighthouse-logger');
@@ -384,13 +384,7 @@ class Driver {
    * @return {Promise<string>}
    */
   async getRequestContent(requestId, timeout = 1000) {
-    requestId = NetworkRequest.getRequestIdForBackend(requestId);
-
-    // Encoding issues may lead to hanging getResponseBody calls: https://github.com/GoogleChrome/lighthouse/pull/4718
-    // driver.sendCommand will handle timeout after 1s.
-    this.setNextProtocolTimeout(timeout);
-    const result = await this.sendCommand('Network.getResponseBody', {requestId});
-    return result.body;
+    return fetchResponseBodyFromCache(this.defaultSession, requestId, timeout);
   }
 
   /**

lighthouse-core/gather/driver/network.js

@@ -0,0 +1,28 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const NetworkRequest = require('../../lib/network-request.js');
+
+/**
+ * Return the body of the response with the given ID. Rejects if getting the
+ * body times out.
+ * @param {LH.Gatherer.FRProtocolSession} session
+ * @param {string} requestId
+ * @param {number} [timeout]
+ * @return {Promise<string>}
+ */
+async function fetchResponseBodyFromCache(session, requestId, timeout = 1000) {
+  requestId = NetworkRequest.getRequestIdForBackend(requestId);
+
+  // Encoding issues may lead to hanging getResponseBody calls: https://github.com/GoogleChrome/lighthouse/pull/4718
+  // session.sendCommand will handle timeout after 1s.
+  session.setNextProtocolTimeout(timeout);
+  const result = await session.sendCommand('Network.getResponseBody', {requestId});
+  return result.body;
+}
+
+module.exports = {fetchResponseBodyFromCache};

lighthouse-core/gather/gatherers/main-document-content.js

@@ -5,23 +5,51 @@
  */
 'use strict';
 
-const Gatherer = require('./gatherer.js');
+const FRGatherer = require('../../fraggle-rock/gather/base-gatherer.js');
 const NetworkAnalyzer = require('../../lib/dependency-graph/simulator/network-analyzer.js');
+const NetworkRecords = require('../../computed/network-records.js');
+const DevtoolsLog = require('./devtools-log.js');
+const {fetchResponseBodyFromCache} = require('../driver/network.js');
 
 /**
  * Collects the content of the main html document.
  */
-class MainDocumentContent extends Gatherer {
+class MainDocumentContent extends FRGatherer {
+  /** @type {LH.Gatherer.GathererMeta<'DevtoolsLog'>} */
+  meta = {
+    supportedModes: ['navigation'],
+    dependencies: {DevtoolsLog: DevtoolsLog.symbol},
+  }
+
+  /**
+   *
+   * @param {LH.Gatherer.FRTransitionalContext} context
+   * @param {LH.Artifacts.NetworkRequest[]} networkRecords
+   * @return {Promise<LH.Artifacts['MainDocumentContent']>}
+   */
+  async _getArtifact(context, networkRecords) {
+    const mainResource = NetworkAnalyzer.findMainDocument(networkRecords, context.url);
+    const session = context.driver.defaultSession;
+    return fetchResponseBodyFromCache(session, mainResource.requestId);
+  }
+  /**
+   *
+   * @param {LH.Gatherer.FRTransitionalContext<'DevtoolsLog'>} context
+   * @return {Promise<LH.Artifacts['MainDocumentContent']>}
+   */
+  async getArtifact(context) {
+    const devtoolsLog = context.dependencies.DevtoolsLog;
+    const networkRecords = await NetworkRecords.request(devtoolsLog, context);
+    return this._getArtifact(context, networkRecords);
+  }
+
   /**
    * @param {LH.Gatherer.PassContext} passContext
    * @param {LH.Gatherer.LoadData} loadData
    * @return {Promise<LH.Artifacts['MainDocumentContent']>}
    */
   async afterPass(passContext, loadData) {
-    const mainResource = NetworkAnalyzer.findMainDocument(loadData.networkRecords, passContext.url);
-
-    const driver = passContext.driver;
-    return driver.getRequestContent(mainResource.requestId);
+    return this._getArtifact({...passContext, dependencies: {}}, loadData.networkRecords);
   }
 }
 

lighthouse-core/test/fraggle-rock/gather/mock-driver.js

@@ -127,11 +127,34 @@ function mockDriverModule(driverProvider) {
   };
 }
 
+function createMockContext() {
+  return {
+    driver: createMockDriver(),
+    url: 'https://example.com',
+    gatherMode: 'navigation',
+    computedCache: new Map(),
+    dependencies: {},
+
+    /** @return {LH.Gatherer.FRTransitionalContext} */
+    asContext() {
+      // @ts-expect-error - We'll rely on the tests passing to know this matches.
+      return this;
+    },
+
+    /** @return {LH.Gatherer.PassContext} */
+    asLegacyContext() {
+      // @ts-expect-error - We'll rely on the tests passing to know this matches.
+      return this;
+    },
+  };
+}
+
 module.exports = {
   mockRunnerModule,
   mockDriverModule,
   createMockDriver,
   createMockPage,
   createMockSession,
   createMockGathererInstance,
+  createMockContext,
 };

lighthouse-core/test/gather/gatherers/main-document-content-test.js

@@ -0,0 +1,49 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-env jest */
+
+const MainDocumentContent = require('../../../gather/gatherers/main-document-content.js');
+const NetworkRecorder = require('../../../lib/network-recorder.js');
+const {createMockContext} = require('../../fraggle-rock/gather/mock-driver.js');
+
+const devtoolsLog
+  = /** @type {LH.DevtoolsLog} */ (require('../../fixtures/traces/lcp-m78.devtools.log.json'));
+
+describe('FR compat', () => {
+  it('uses loadData in legacy mode', async () => {
+    const gatherer = new MainDocumentContent();
+    const networkRecords = NetworkRecorder.recordsFromLogs(devtoolsLog);
+    const mockContext = createMockContext();
+    mockContext.driver.defaultSession.sendCommand
+      .mockResponse('Network.getResponseBody', {body: 'RESPONSE'});
+
+    const artifact = await gatherer.afterPass(
+      mockContext.asLegacyContext(),
+      {devtoolsLog, networkRecords}
+    );
+
+    expect(artifact).toEqual('RESPONSE');
+  });
+
+  it('uses dependencies for FR', async () => {
+    const gatherer = new MainDocumentContent();
+    const mockContext = createMockContext();
+    mockContext.driver.defaultSession.sendCommand
+      .mockResponse('Network.getResponseBody', {body: 'RESPONSE'});
+
+    /** @type {LH.Gatherer.FRTransitionalContext<'DevtoolsLog'>} */
+    const context = {
+      ...mockContext.asContext(),
+      dependencies: {DevtoolsLog: devtoolsLog},
+    };
+
+    const artifact = await gatherer.getArtifact(context);
+
+    expect(artifact).toEqual('RESPONSE');
+  });
+});

types/artifacts.d.ts

@@ -25,7 +25,6 @@ declare global {
       | 'HTTPRedirect'
       | 'ImageElements'
       | 'InspectorIssues'
-      | 'MainDocumentContent'
       | 'Manifest'
       | 'MixedContent'
       | 'OptimizedImages'