Skip to content

Commit

Permalink
tests(devtools): sync e2e tests (#14426)
Browse files Browse the repository at this point in the history
  • Loading branch information
adamraine committed Oct 5, 2022
1 parent 944a2cf commit 6e9467c
Show file tree
Hide file tree
Showing 7 changed files with 249 additions and 98 deletions.
3 changes: 2 additions & 1 deletion third-party/devtools-tests/e2e/lighthouse/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@ import("../../../third_party/typescript/typescript.gni")

node_ts_library("lighthouse") {
sources = [
"devtools-settings_test.ts",
"navigation_test.ts",
"request_blocking_test.ts",
"snapshot_test.ts",
"start-view_test.ts",
"timespan_test.ts",
]

deps = [
Expand Down
120 changes: 120 additions & 0 deletions third-party/devtools-tests/e2e/lighthouse/devtools-settings_test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
// Copyright 2022 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import {assert} from 'chai';
import * as path from 'path';

import {getBrowserAndPages, waitFor, waitForAria, waitForElementWithTextContent} from '../../shared/helper.js';
import {describe, it} from '../../shared/mocha-extensions.js';
import {openDeviceToolbar, reloadDockableFrontEnd, selectDevice} from '../helpers/emulation-helpers.js';
import {
clickStartButton,
getTargetViewport,
navigateToLighthouseTab,
selectCategories,
waitForResult,
} from '../helpers/lighthouse-helpers.js';

// This test will fail (by default) in headful mode, as the target page never gets painted.
// To resolve this when debugging, just make sure the target page is visible during the lighthouse run.

const IPAD_MINI_LANDSCAPE_VIEWPORT_DIMENSIONS = {
innerHeight: 768,
innerWidth: 1024,
outerHeight: 768,
outerWidth: 1024,
devicePixelRatio: 2,
};

describe('DevTools', function() {
// The tests in this suite are particularly slow
this.timeout(60_000);

describe('request blocking', () => {
// Start blocking *.css
// Ideally this would be done with UI manipulation, but it'd be less reliable AND
// the designated tests in network-request-blocking-panel_test.ts are skipped by default due to flakiness.
beforeEach(async () => {
const {frontend} = getBrowserAndPages();
await frontend.evaluate(() => {
// @ts-ignore layout test global
const networkManager = self.SDK.multitargetNetworkManager;
networkManager.setBlockingEnabled(true);
networkManager.setBlockedPatterns([{enabled: true, url: '*.css'}]);
});
});

// Reset request blocking state
afterEach(async () => {
const {frontend} = getBrowserAndPages();
await frontend.evaluate(() => {
// @ts-ignore layout test global
const networkManager = globalThis.SDK.multitargetNetworkManager;
networkManager.setBlockingEnabled(false);
networkManager.setBlockedPatterns([]);
});
});

it('is respected during a lighthouse run', async () => {
await navigateToLighthouseTab('lighthouse/hello.html');

await selectCategories(['performance']);

await clickStartButton();

const {lhr} = await waitForResult();

const requests = lhr.audits['network-requests'].details.items;

const trimmedRequests = requests.map((item: Record<string, unknown>) => {
return {
url: typeof item.url === 'string' && path.basename(item.url),
statusCode: item.statusCode,
};
});
assert.deepEqual(trimmedRequests, [
{url: 'hello.html', statusCode: 200},
{url: 'basic.css', statusCode: -1}, // statuCode === -1 means the request failed
]);
});
});

describe('device emulation', () => {
beforeEach(async function() {
await reloadDockableFrontEnd();
await waitFor('.tabbed-pane-left-toolbar');
await openDeviceToolbar();
});

it('is restored after a lighthouse run', async () => {
// Use iPad Mini in landscape mode and custom zoom.
await selectDevice('iPad Mini');
const rotateButton = await waitForAria('Rotate');
await rotateButton.click();
const zoomButton = await waitForAria('Zoom');
await zoomButton.click();
const zoom75 = await waitForElementWithTextContent('75%');
await zoom75.click();

assert.deepStrictEqual(await getTargetViewport(), IPAD_MINI_LANDSCAPE_VIEWPORT_DIMENSIONS);

await navigateToLighthouseTab('lighthouse/hello.html');
await selectCategories(['performance']);
await clickStartButton();

const {artifacts} = await waitForResult();
assert.deepStrictEqual(artifacts.ViewportDimensions, {
innerHeight: 640,
innerWidth: 360,
outerHeight: 640,
outerWidth: 360,
devicePixelRatio: 3,
});

const zoomText = await zoomButton.evaluate(zoomButtonEl => zoomButtonEl.textContent);
assert.strictEqual(zoomText, '75%');
assert.deepStrictEqual(await getTargetViewport(), IPAD_MINI_LANDSCAPE_VIEWPORT_DIMENSIONS);
});
});
});
54 changes: 37 additions & 17 deletions third-party/devtools-tests/e2e/lighthouse/navigation_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,14 @@ describe('Navigation', async function() {
await navigateToLighthouseTab('lighthouse/hello.html');

await setLegacyNavigation(mode === 'legacy');
await selectCategories([
'performance',
'accessibility',
'best-practices',
'seo',
'pwa',
'lighthouse-plugin-publisher-ads',
]);

await clickStartButton();

Expand All @@ -54,35 +62,41 @@ describe('Navigation', async function() {
// 9.6.x is forked so Lighthouse ToT is still using 9.5.0 as the version.
// assert.strictEqual(lhr.lighthouseVersion, '9.6.6');
assert.match(lhr.finalUrl, /^https:\/\/localhost:[0-9]+\/test\/e2e\/resources\/lighthouse\/hello.html/);

assert.strictEqual(lhr.configSettings.throttlingMethod, 'simulate');
assert.strictEqual(lhr.configSettings.disableStorageReset, false);
assert.strictEqual(lhr.configSettings.formFactor, 'mobile');
assert.strictEqual(lhr.configSettings.throttling.rttMs, 150);
assert.strictEqual(lhr.configSettings.screenEmulation.disabled, true);
assert.include(lhr.configSettings.emulatedUserAgent, 'Mobile');

// A bug in FR caused `networkUserAgent` to be excluded from the LHR.
// https://github.com/GoogleChrome/lighthouse/pull/14392
// TODO: Reenable once the fix lands in DT.
if (mode === 'legacy') {
assert.include(lhr.environment.networkUserAgent, 'Mobile');
}

const {innerWidth, innerHeight, outerWidth, outerHeight, devicePixelRatio} = artifacts.ViewportDimensions;
// This value can vary slightly, depending on the display.
// https://bugs.chromium.org/p/chromium/issues/detail?id=1346355
assert.approximately(innerHeight, 1742, 1);
assert.strictEqual(innerWidth, 980);
assert.strictEqual(outerWidth, 360);
assert.strictEqual(outerHeight, 640);
assert.strictEqual(devicePixelRatio, 3);
assert.deepStrictEqual(artifacts.ViewportDimensions, {
innerHeight: 640,
innerWidth: 360,
outerHeight: 640,
outerWidth: 360,
devicePixelRatio: 3,
});

const {auditResults, erroredAudits, failedAudits} = getAuditsBreakdown(lhr);
assert.strictEqual(auditResults.length, 150);
assert.strictEqual(auditResults.length, 173);
assert.strictEqual(erroredAudits.length, 0);
assert.deepStrictEqual(failedAudits.map(audit => audit.id), [
'service-worker',
'viewport',
'installable-manifest',
'splash-screen',
'themed-omnibox',
'maskable-icon',
'content-width',
'document-title',
'html-has-lang',
'meta-description',
'font-size',
'tap-targets',
]);

const viewTraceText = await reportEl.$eval('.lh-button--trace', viewTraceEl => {
Expand Down Expand Up @@ -114,17 +128,13 @@ describe('Navigation', async function() {
assert.strictEqual(erroredAudits.length, 0);
assert.deepStrictEqual(failedAudits.map(audit => audit.id), [
'service-worker',
'viewport',
'installable-manifest',
'splash-screen',
'themed-omnibox',
'maskable-icon',
'content-width',
'document-title',
'html-has-lang',
'meta-description',
'font-size',
'tap-targets',
]);

const viewTraceText = await reportEl.$eval('.lh-button--trace', viewTraceEl => {
Expand Down Expand Up @@ -158,6 +168,16 @@ describe('Navigation', async function() {
assert.deepStrictEqual(Object.keys(lhr.categories), ['performance', 'best-practices']);
assert.strictEqual(lhr.configSettings.disableStorageReset, true);
assert.strictEqual(lhr.configSettings.formFactor, 'desktop');
assert.strictEqual(lhr.configSettings.throttling.rttMs, 40);
assert.strictEqual(lhr.configSettings.screenEmulation.disabled, true);
assert.notInclude(lhr.configSettings.emulatedUserAgent, 'Mobile');

// A bug in FR caused `networkUserAgent` to be excluded from the LHR.
// https://github.com/GoogleChrome/lighthouse/pull/14392
// TODO: Reenable once the fix lands in DT.
if (mode === 'legacy') {
assert.notInclude(lhr.environment.networkUserAgent, 'Mobile');
}

const viewTraceText = await reportEl.$eval('.lh-button--trace', viewTraceEl => {
return viewTraceEl.textContent;
Expand Down
70 changes: 0 additions & 70 deletions third-party/devtools-tests/e2e/lighthouse/request_blocking_test.ts

This file was deleted.

17 changes: 7 additions & 10 deletions third-party/devtools-tests/e2e/lighthouse/snapshot_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,25 +52,22 @@ describe('Snapshot', async function() {

assert.strictEqual(lhr.gatherMode, 'snapshot');

const {innerWidth, innerHeight, outerWidth, outerHeight, devicePixelRatio} = artifacts.ViewportDimensions;
// This value can vary slightly, depending on the display.
// https://bugs.chromium.org/p/chromium/issues/detail?id=1346355
assert.approximately(innerHeight, 1742, 1);
assert.strictEqual(innerWidth, 980);
assert.strictEqual(outerWidth, 360);
assert.strictEqual(outerHeight, 640);
assert.strictEqual(devicePixelRatio, 3);
assert.deepStrictEqual(artifacts.ViewportDimensions, {
innerHeight: 640,
innerWidth: 360,
outerHeight: 640,
outerWidth: 360,
devicePixelRatio: 3,
});

const {auditResults, erroredAudits, failedAudits} = getAuditsBreakdown(lhr);
assert.strictEqual(auditResults.length, 72);
assert.strictEqual(erroredAudits.length, 0);
assert.deepStrictEqual(failedAudits.map(audit => audit.id), [
'viewport',
'document-title',
'html-has-lang',
'label',
'meta-description',
'font-size',
'tap-targets',
]);

Expand Down
Loading

0 comments on commit 6e9467c

Please sign in to comment.