From 217d2d182cfc2536867bfd464630e612f7668316 Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Wed, 28 Apr 2021 16:31:54 -0400 Subject: [PATCH 1/8] start --- lighthouse-core/gather/fetcher.js | 49 ++++++++++--------- .../gather/gatherers/seo/robots-txt.js | 27 ++-------- .../gather/gatherers/source-maps.js | 8 +-- 3 files changed, 36 insertions(+), 48 deletions(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 87fb7784b50e..8d9c5361593a 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -13,6 +13,8 @@ /* global document */ +/** @typedef {{content: string|null, status: number|null}} FetchResponse */ + const log = require('lighthouse-logger'); const {getBrowserVersion} = require('./driver/environment.js'); @@ -97,7 +99,7 @@ class Fetcher { * * @param {string} url * @param {{timeout: number}=} options timeout is in ms - * @return {Promise} + * @return {Promise} */ async fetchResource(url, options = {timeout: 500}) { if (!this._enabled) { @@ -141,7 +143,7 @@ class Fetcher { /** * @param {string} url - * @return {Promise} + * @return {Promise<{stream: LH.Crdp.IO.StreamHandle|null, status: number|null}>} */ async _loadNetworkResource(url) { await this.session.sendCommand('Network.enable'); @@ -156,18 +158,16 @@ class Fetcher { }); await this.session.sendCommand('Network.disable'); - if (!networkResponse.resource.success || !networkResponse.resource.stream) { - const statusCode = networkResponse.resource.httpStatusCode || ''; - throw new Error(`Loading network resource failed (${statusCode})`); - } - - return networkResponse.resource.stream; + return { + stream: networkResponse.resource.success ? (networkResponse.resource.stream || null) : null, + status: networkResponse.resource.httpStatusCode || null, + }; } /** * @param {string} url * @param {{timeout: number}} options timeout is in ms - * @return {Promise} + * @return {Promise} */ async _fetchResourceOverProtocol(url, options) { const startTime = Date.now(); @@ -180,21 +180,29 @@ class Fetcher { }, options.timeout); }); - const fetchStreamPromise = this._loadNetworkResource(url); - const stream = await Promise.race([fetchStreamPromise, timeoutPromise]) + const responsePromise = this._loadNetworkResource(url); + + /** @type {{stream: LH.Crdp.IO.StreamHandle|null, status: number|null}} */ + const response = await Promise.race([responsePromise, timeoutPromise]) .finally(() => clearTimeout(timeoutHandle)); - return await this._readIOStream(stream, {timeout: options.timeout - (Date.now() - startTime)}); + if (response.stream && response.status && response.status >= 200 && response.status <= 299) { + const content = await this._readIOStream(response.stream, { + timeout: options.timeout - (Date.now() - startTime), + }); + return {content, status: response.status}; + } + return {content: null, status: response.status}; } /** * Fetches resource by injecting an iframe into the page. * @param {string} url * @param {{timeout: number}} options timeout is in ms - * @return {Promise} + * @return {Promise} */ async _fetchResourceIframe(url, options) { - /** @type {Promise} */ + /** @type {Promise} */ const requestInterceptionPromise = new Promise((resolve, reject) => { /** @param {LH.Crdp.Fetch.RequestPausedEvent} event */ const handlerAsync = async event => { @@ -216,17 +224,14 @@ class Fetcher { return; } - // Now in the response stage, but the request failed. - if (!(responseStatusCode >= 200 && responseStatusCode < 300)) { - reject(new Error(`Invalid response status code: ${responseStatusCode}`)); - return; - } - const responseBody = await this.session.sendCommand('Fetch.getResponseBody', {requestId}); if (responseBody.base64Encoded) { - resolve(Buffer.from(responseBody.body, 'base64').toString()); + resolve({ + content: Buffer.from(responseBody.body, 'base64').toString(), + status: responseStatusCode, + }); } else { - resolve(responseBody.body); + resolve({content: responseBody.body, status: responseStatusCode}); } // Fail the request (from the page's perspective) so that the iframe never loads. diff --git a/lighthouse-core/gather/gatherers/seo/robots-txt.js b/lighthouse-core/gather/gatherers/seo/robots-txt.js index 89d967b1fd54..eb912ab40be2 100644 --- a/lighthouse-core/gather/gatherers/seo/robots-txt.js +++ b/lighthouse-core/gather/gatherers/seo/robots-txt.js @@ -7,25 +7,6 @@ const FRGatherer = require('../../../fraggle-rock/gather/base-gatherer.js'); -/* global fetch, location */ - -/** @return {Promise} */ -/* c8 ignore start */ -async function getRobotsTxtContent() { - try { - const response = await fetch(new URL('/robots.txt', location.href).href); - if (!response.ok) { - return {status: response.status, content: null}; - } - - const content = await response.text(); - return {status: response.status, content}; - } catch (_) { - return {status: null, content: null}; - } -} -/* c8 ignore stop */ - class RobotsTxt extends FRGatherer { /** @type {LH.Gatherer.GathererMeta} */ meta = { @@ -37,10 +18,10 @@ class RobotsTxt extends FRGatherer { * @return {Promise} */ snapshot(passContext) { - return passContext.driver.executionContext.evaluate(getRobotsTxtContent, { - args: [], - useIsolation: true, - }); + const robotsUrl = new URL(passContext.url); + robotsUrl.pathname = '/robots.txt'; + passContext.driver.fetcher.enable(); + return passContext.driver.fetcher.fetchResource(robotsUrl.toString()); } } diff --git a/lighthouse-core/gather/gatherers/source-maps.js b/lighthouse-core/gather/gatherers/source-maps.js index 3c1566f161a3..fad4425ba255 100644 --- a/lighthouse-core/gather/gatherers/source-maps.js +++ b/lighthouse-core/gather/gatherers/source-maps.js @@ -27,9 +27,11 @@ class SourceMaps extends Gatherer { * @return {Promise} */ async fetchSourceMap(driver, sourceMapUrl) { - /** @type {string} */ - const sourceMapJson = await driver.fetcher.fetchResource(sourceMapUrl, {timeout: 1500}); - return JSON.parse(sourceMapJson); + const response = await driver.fetcher.fetchResource(sourceMapUrl, {timeout: 1500}); + if (response.content === null) { + throw new Error(`Source map could not be fetched (${response.status})`); + } + return JSON.parse(response.content); } /** From a28b66c807f2ed1cde0d31194724c93dea98d25a Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Wed, 28 Apr 2021 16:47:22 -0400 Subject: [PATCH 2/8] update smoke --- .../test-definitions/csp/csp-expectations.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/lighthouse-cli/test/smokehouse/test-definitions/csp/csp-expectations.js b/lighthouse-cli/test/smokehouse/test-definitions/csp/csp-expectations.js index 51b4fdad0099..1214ce4ffa07 100644 --- a/lighthouse-cli/test/smokehouse/test-definitions/csp/csp-expectations.js +++ b/lighthouse-cli/test/smokehouse/test-definitions/csp/csp-expectations.js @@ -48,19 +48,11 @@ module.exports = [ { artifacts: { RobotsTxt: { - status: null, + status: 404, content: null, }, InspectorIssues: { contentSecurityPolicy: [ - { - // TODO: Fix connect-src violation when fetching robots.txt. - // https://github.com/GoogleChrome/lighthouse/issues/10225 - blockedURL: 'http://localhost:10200/robots.txt', - violatedDirective: 'connect-src', - isReportOnly: false, - contentSecurityPolicyViolationType: 'kURLViolation', - }, { // TODO: Fix style-src-elem violation when checking tap targets. // https://github.com/GoogleChrome/lighthouse/issues/11862 From 362cd602463deb39d3f4b7313a60634eab74bc69 Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Wed, 28 Apr 2021 17:32:00 -0400 Subject: [PATCH 3/8] tests --- .../gather/gatherers/source-maps.js | 2 +- lighthouse-core/test/gather/fetcher-test.js | 8 ++--- .../test/gather/gatherers/source-maps-test.js | 35 +++++++++++++++++-- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/lighthouse-core/gather/gatherers/source-maps.js b/lighthouse-core/gather/gatherers/source-maps.js index fad4425ba255..0b9d81d2009d 100644 --- a/lighthouse-core/gather/gatherers/source-maps.js +++ b/lighthouse-core/gather/gatherers/source-maps.js @@ -29,7 +29,7 @@ class SourceMaps extends Gatherer { async fetchSourceMap(driver, sourceMapUrl) { const response = await driver.fetcher.fetchResource(sourceMapUrl, {timeout: 1500}); if (response.content === null) { - throw new Error(`Source map could not be fetched (${response.status})`); + throw new Error(`Failed fetching source map (${response.status})`); } return JSON.parse(response.content); } diff --git a/lighthouse-core/test/gather/fetcher-test.js b/lighthouse-core/test/gather/fetcher-test.js index 17960359d36f..139201c39ba8 100644 --- a/lighthouse-core/test/gather/fetcher-test.js +++ b/lighthouse-core/test/gather/fetcher-test.js @@ -133,10 +133,10 @@ describe('._fetchResourceOverProtocol', () => { .mockResponse('Network.disable'); const data = await fetcher._fetchResourceOverProtocol('https://example.com', {timeout: 500}); - expect(data).toEqual(streamContents); + expect(data).toEqual({content: streamContents, status: 200}); }); - it('throws when resource could not be fetched', async () => { + it('returns null when resource could not be fetched', async () => { connectionStub.sendCommand = createMockSendCommandFn() .mockResponse('Network.enable') .mockResponse('Page.getFrameTree', {frameTree: {frame: {id: 'FRAME'}}}) @@ -145,8 +145,8 @@ describe('._fetchResourceOverProtocol', () => { }) .mockResponse('Network.disable'); - const dataPromise = fetcher._fetchResourceOverProtocol('https://example.com', {timeout: 500}); - await expect(dataPromise).rejects.toThrowError(/Loading network resource failed/); + const data = await fetcher._fetchResourceOverProtocol('https://example.com', {timeout: 500}); + expect(data).toEqual({content: null, status: 404}); }); it('throws on timeout', async () => { diff --git a/lighthouse-core/test/gather/gatherers/source-maps-test.js b/lighthouse-core/test/gather/gatherers/source-maps-test.js index 8ee4f4199b25..8f27f41d5bff 100644 --- a/lighthouse-core/test/gather/gatherers/source-maps-test.js +++ b/lighthouse-core/test/gather/gatherers/source-maps-test.js @@ -30,7 +30,7 @@ describe('SourceMaps gatherer', () => { * `resolvedSourceMapUrl` is used to assert that the SourceMaps gatherer is using the expected * url to fetch the source map. * `fetchError` mocks an error that happens in the page. Only fetch error message make sense. - * @param {Array<{scriptParsedEvent: LH.Crdp.Debugger.ScriptParsedEvent, map: string, resolvedSourceMapUrl?: string, fetchError: string}>} mapsAndEvents + * @param {Array<{scriptParsedEvent: LH.Crdp.Debugger.ScriptParsedEvent, map: string, status?: number, resolvedSourceMapUrl?: string, fetchError: string}>} mapsAndEvents * @return {Promise} */ async function runSourceMaps(mapsAndEvents) { @@ -50,7 +50,14 @@ describe('SourceMaps gatherer', () => { .mockResponse('Fetch.disable', {}); const fetchMock = jest.fn(); - for (const {scriptParsedEvent, map, resolvedSourceMapUrl, fetchError} of mapsAndEvents) { + for (const mapAndEvents of mapsAndEvents) { + const { + scriptParsedEvent, + map, + status = null, + resolvedSourceMapUrl, + fetchError, + } = mapAndEvents; onMock.mockEvent('protocolevent', { method: 'Debugger.scriptParsed', params: scriptParsedEvent, @@ -71,7 +78,7 @@ describe('SourceMaps gatherer', () => { throw new Error(fetchError); } - return map; + return {content: map, status}; }); } const connectionStub = new Connection(); @@ -173,6 +180,28 @@ describe('SourceMaps gatherer', () => { ]); }); + it('throws an error message when fetching map returns bad status code', async () => { + const mapsAndEvents = [ + { + scriptParsedEvent: { + url: 'http://www.example.com/bundle.js', + sourceMapURL: 'http://www.example.com/bundle.js.map', + }, + status: 404, + map: null, + }, + ]; + const artifact = await runSourceMaps(mapsAndEvents); + expect(artifact).toEqual([ + { + scriptUrl: mapsAndEvents[0].scriptParsedEvent.url, + sourceMapUrl: mapsAndEvents[0].scriptParsedEvent.sourceMapURL, + errorMessage: 'Error: Failed fetching source map (404)', + map: undefined, + }, + ]); + }); + it('generates an error message when fetching map fails', async () => { const mapsAndEvents = [ { From 4581bb77df714ed2b7da85861598e17f4a6693fd Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Thu, 29 Apr 2021 15:11:34 -0400 Subject: [PATCH 4/8] fix smoke --- .../test-definitions/csp/csp-expectations.js | 44 ++++++++++++++++++- lighthouse-core/gather/fetcher.js | 18 +++++--- .../gather/gatherers/seo/robots-txt.js | 35 ++++++++++++++- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/lighthouse-cli/test/smokehouse/test-definitions/csp/csp-expectations.js b/lighthouse-cli/test/smokehouse/test-definitions/csp/csp-expectations.js index 1214ce4ffa07..2ee5607860fd 100644 --- a/lighthouse-cli/test/smokehouse/test-definitions/csp/csp-expectations.js +++ b/lighthouse-cli/test/smokehouse/test-definitions/csp/csp-expectations.js @@ -47,12 +47,22 @@ module.exports = [ }, { artifacts: { + _maxChromiumMilestone: 91, RobotsTxt: { - status: 404, + status: null, content: null, }, InspectorIssues: { contentSecurityPolicy: [ + { + // https://github.com/GoogleChrome/lighthouse/issues/10225 + // + // Fixed with new fetcher using M92. + blockedURL: 'http://localhost:10200/robots.txt', + violatedDirective: 'connect-src', + isReportOnly: false, + contentSecurityPolicyViolationType: 'kURLViolation', + }, { // TODO: Fix style-src-elem violation when checking tap targets. // https://github.com/GoogleChrome/lighthouse/issues/11862 @@ -67,7 +77,6 @@ module.exports = [ // https://github.com/GoogleChrome/lighthouse/pull/12044#issuecomment-788274938 // // Fixed with new fetcher using M92. - _maxChromiumMilestone: 91, sourceMapUrl: 'http://localhost:10200/source-map/script.js.map', errorMessage: 'Error: Timed out fetching resource.', map: undefined, @@ -79,4 +88,35 @@ module.exports = [ audits: {}, }, }, + { + // Same CSP as above, but verifies correct behavior for M92. + artifacts: { + _minChromiumMilestone: 92, + RobotsTxt: { + status: 404, + content: null, + }, + InspectorIssues: { + contentSecurityPolicy: [ + { + // TODO: Fix style-src-elem violation when checking tap targets. + // https://github.com/GoogleChrome/lighthouse/issues/11862 + violatedDirective: 'style-src-elem', + isReportOnly: false, + contentSecurityPolicyViolationType: 'kInlineViolation', + }, + ], + }, + SourceMaps: [{ + sourceMapUrl: 'http://localhost:10200/source-map/script.js.map', + map: {}, + errorMessage: undefined, + }], + }, + lhr: { + requestedUrl: 'http://localhost:10200/csp.html?' + blockAllExceptInlineScriptCsp, + finalUrl: 'http://localhost:10200/csp.html?' + blockAllExceptInlineScriptCsp, + audits: {}, + }, + }, ]; diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 8d9c5361593a..c3f1aa4a8064 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -224,14 +224,18 @@ class Fetcher { return; } - const responseBody = await this.session.sendCommand('Fetch.getResponseBody', {requestId}); - if (responseBody.base64Encoded) { - resolve({ - content: Buffer.from(responseBody.body, 'base64').toString(), - status: responseStatusCode, - }); + if (responseStatusCode >= 200 && responseStatusCode <= 299) { + const responseBody = await this.session.sendCommand('Fetch.getResponseBody', {requestId}); + if (responseBody.base64Encoded) { + resolve({ + content: Buffer.from(responseBody.body, 'base64').toString(), + status: responseStatusCode, + }); + } else { + resolve({content: responseBody.body, status: responseStatusCode}); + } } else { - resolve({content: responseBody.body, status: responseStatusCode}); + resolve({content: null, status: responseStatusCode}); } // Fail the request (from the page's perspective) so that the iframe never loads. diff --git a/lighthouse-core/gather/gatherers/seo/robots-txt.js b/lighthouse-core/gather/gatherers/seo/robots-txt.js index eb912ab40be2..e1280825b3c2 100644 --- a/lighthouse-core/gather/gatherers/seo/robots-txt.js +++ b/lighthouse-core/gather/gatherers/seo/robots-txt.js @@ -6,6 +6,26 @@ 'use strict'; const FRGatherer = require('../../../fraggle-rock/gather/base-gatherer.js'); +const {getBrowserVersion} = require('../../driver/environment.js'); + +/* global fetch, location */ + +/** @return {Promise} */ +/* c8 ignore start */ +async function getRobotsTxtContent() { + try { + const response = await fetch(new URL('/robots.txt', location.href).href); + if (!response.ok) { + return {status: response.status, content: null}; + } + + const content = await response.text(); + return {status: response.status, content}; + } catch (_) { + return {status: null, content: null}; + } +} +/* c8 ignore stop */ class RobotsTxt extends FRGatherer { /** @type {LH.Gatherer.GathererMeta} */ @@ -17,7 +37,20 @@ class RobotsTxt extends FRGatherer { * @param {LH.Gatherer.FRTransitionalContext} passContext * @return {Promise} */ - snapshot(passContext) { + async snapshot(passContext) { + const milestone + = await getBrowserVersion(passContext.driver.defaultSession).then(v => v.milestone); + + // TODO: Remove when 92 hits stable. + // Iframe fetcher still has issues with CSPs. + // Only use the fetcher if we are fetching over the CDP. + if (milestone < 92) { + return passContext.driver.executionContext.evaluate(getRobotsTxtContent, { + args: [], + useIsolation: true, + }); + } + const robotsUrl = new URL(passContext.url); robotsUrl.pathname = '/robots.txt'; passContext.driver.fetcher.enable(); From 0fa55a31c743117c4a979dc442968aef9c05f863 Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Thu, 29 Apr 2021 17:07:27 -0400 Subject: [PATCH 5/8] pr --- lighthouse-core/gather/fetcher.js | 42 +++++++++++-------- .../gather/gatherers/seo/robots-txt.js | 3 +- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index c3f1aa4a8064..51eaa023ac79 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -109,7 +109,7 @@ class Fetcher { // `Network.loadNetworkResource` was introduced in M88. // The long timeout bug with `IO.read` was fixed in M92: // https://bugs.chromium.org/p/chromium/issues/detail?id=1191757 - const milestone = await getBrowserVersion(this.session).then(v => v.milestone); + const {milestone} = await getBrowserVersion(this.session); if (milestone >= 92) { return await this._fetchResourceOverProtocol(url, options); } @@ -164,6 +164,18 @@ class Fetcher { }; } + /** + * @param {string} requestId + * @return {Promise} + */ + async _resolveResponseBody(requestId) { + const responseBody = await this.session.sendCommand('Fetch.getResponseBody', {requestId}); + if (responseBody.base64Encoded) { + return Buffer.from(responseBody.body, 'base64').toString(); + } + return responseBody.body; + } + /** * @param {string} url * @param {{timeout: number}} options timeout is in ms @@ -186,13 +198,12 @@ class Fetcher { const response = await Promise.race([responsePromise, timeoutPromise]) .finally(() => clearTimeout(timeoutHandle)); - if (response.stream && response.status && response.status >= 200 && response.status <= 299) { - const content = await this._readIOStream(response.stream, { - timeout: options.timeout - (Date.now() - startTime), - }); - return {content, status: response.status}; - } - return {content: null, status: response.status}; + const isOk = response.status && response.status >= 200 && response.status <= 299; + if (!response.stream || !isOk) return {status: response.status, content: null}; + + const timeout = options.timeout - (Date.now() - startTime); + const content = await this._readIOStream(response.stream, {timeout}); + return {status: response.status, content}; } /** @@ -225,17 +236,12 @@ class Fetcher { } if (responseStatusCode >= 200 && responseStatusCode <= 299) { - const responseBody = await this.session.sendCommand('Fetch.getResponseBody', {requestId}); - if (responseBody.base64Encoded) { - resolve({ - content: Buffer.from(responseBody.body, 'base64').toString(), - status: responseStatusCode, - }); - } else { - resolve({content: responseBody.body, status: responseStatusCode}); - } + resolve({ + status: responseStatusCode, + content: await this._resolveResponseBody(requestId), + }); } else { - resolve({content: null, status: responseStatusCode}); + resolve({status: responseStatusCode, content: null}); } // Fail the request (from the page's perspective) so that the iframe never loads. diff --git a/lighthouse-core/gather/gatherers/seo/robots-txt.js b/lighthouse-core/gather/gatherers/seo/robots-txt.js index e1280825b3c2..3b0b664a247f 100644 --- a/lighthouse-core/gather/gatherers/seo/robots-txt.js +++ b/lighthouse-core/gather/gatherers/seo/robots-txt.js @@ -38,8 +38,7 @@ class RobotsTxt extends FRGatherer { * @return {Promise} */ async snapshot(passContext) { - const milestone - = await getBrowserVersion(passContext.driver.defaultSession).then(v => v.milestone); + const {milestone} = await getBrowserVersion(passContext.driver.defaultSession); // TODO: Remove when 92 hits stable. // Iframe fetcher still has issues with CSPs. From aa0eafa404c57ad58a80f7a18c247502d12ca191 Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Fri, 30 Apr 2021 11:37:56 -0400 Subject: [PATCH 6/8] case --- lighthouse-core/gather/gatherers/seo/robots-txt.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lighthouse-core/gather/gatherers/seo/robots-txt.js b/lighthouse-core/gather/gatherers/seo/robots-txt.js index 3b0b664a247f..971ec88cd0c7 100644 --- a/lighthouse-core/gather/gatherers/seo/robots-txt.js +++ b/lighthouse-core/gather/gatherers/seo/robots-txt.js @@ -50,10 +50,9 @@ class RobotsTxt extends FRGatherer { }); } - const robotsUrl = new URL(passContext.url); - robotsUrl.pathname = '/robots.txt'; + const robotsUrl = new URL('/robots.txt', passContext.url).href; passContext.driver.fetcher.enable(); - return passContext.driver.fetcher.fetchResource(robotsUrl.toString()); + return passContext.driver.fetcher.fetchResource(robotsUrl); } } From f41b130619f985e85ebb82008f43e90704deed0c Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Fri, 30 Apr 2021 11:57:10 -0400 Subject: [PATCH 7/8] catch --- lighthouse-core/gather/gatherers/seo/robots-txt.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lighthouse-core/gather/gatherers/seo/robots-txt.js b/lighthouse-core/gather/gatherers/seo/robots-txt.js index e3a041a0052c..c6958a831d64 100644 --- a/lighthouse-core/gather/gatherers/seo/robots-txt.js +++ b/lighthouse-core/gather/gatherers/seo/robots-txt.js @@ -51,8 +51,9 @@ class RobotsTxt extends FRGatherer { } const robotsUrl = new URL('/robots.txt', passContext.url).href; - passContext.driver.fetcher.enable(); - return passContext.driver.fetcher.fetchResource(robotsUrl); + await passContext.driver.fetcher.enable(); + return passContext.driver.fetcher.fetchResource(robotsUrl) + .catch(() => ({status: null, content: null})); } } From 2b28ddeea2a94b08dd3465e5ad1c486b8720282f Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Mon, 3 May 2021 16:53:13 -0400 Subject: [PATCH 8/8] wow nice one --- .../devtools/lighthouse/lighthouse-successful-run-expected.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-run-expected.txt b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-run-expected.txt index b790273b8e30..ad9396418743 100644 --- a/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-run-expected.txt +++ b/third-party/chromium-webtests/webtests/http/tests/devtools/lighthouse/lighthouse-successful-run-expected.txt @@ -112,6 +112,7 @@ Gathering: TagsBlockingFirstPaint Gathering: FontSize Gathering: EmbeddedContent Gathering: RobotsTxt +Getting browser version Gathering: TapTargets Gathering: Accessibility Gathering: TraceElements