diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6776f641..46400b0a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,15 @@ jobs: uses: nrwl/nx-set-shas@v4 - name: Install dependencies run: npm ci + - name: Set custom Chrome path for Windows only + if: matrix.os == 'windows-latest' + # This path is considered in `testing/setup/src/lib/chrome-path-setup.ts` and used in different test configurations + run: | + echo "CUSTOM_CHROME_PATH=C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + shell: pwsh + #- name: Log all environment variables + # run: | + # printenv - name: Integration test affected projects run: npx nx affected -t integration-test --parallel=3 --coverage.enabled diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 206c32abf..340646b68 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,25 @@ Make sure to install dependencies: npm install ``` +## Environment Variables + +This table provides a quick overview of the environmental setup, with detailed explanations in the corresponding sections. + +| Feature | Local Default | CI Default | Description | +| -------------------------------- | ------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `env.INCLUDE_SLOW_TESTS` **❗️** | `false` | `true` | Controls inclusion of long-running tests. Overridden by setting. Details in the [Testing](#Testing) section. | +| `env.CUSTOM_CHROME_PATH` | N/A | Windows **❗️❗️** | Path to Chrome executable. See [plugin-lighthouse/CONTRIBUTING.md](./packages/plugin-lighthouse/CONTRIBUTING.md#chrome-path). | +| Quality Pipeline | Off | On | Runs all plugins against the codebase. | + +**❗️** Test Inclusion Logic + +- `INCLUDE_SLOW_TESTS='false'` skips long tests. +- without `INCLUDE_SLOW_TESTS`, tests run if `CI` is set. + +**❗️❗️** Windows specific path set only in CI + +- some setups also require this setting locally + ## Development Refer to docs on [how to run tasks in Nx](https://nx.dev/core-features/run-tasks). @@ -41,6 +60,15 @@ npx nx affected:lint npx nx code-pushup -- collect ``` +## Testing + +Some of the plugins have a longer runtime. In order to ensure better DX, longer tests are excluded by default when executing tests locally. + +You can control the execution of long-running tests over the `INCLUDE_SLOW_TESTS` environment variable. + +To change this setup, open (or create) the `.env` file in the root folder. +Edit or add the environment variable there as follows: `INCLUDE_SLOW_TESTS=true`. + ## Git Commit messages must follow [conventional commits](https://conventionalcommits.org/) format. diff --git a/packages/plugin-lighthouse/CONTRIBUTING.md b/packages/plugin-lighthouse/CONTRIBUTING.md new file mode 100644 index 000000000..1efb5d87f --- /dev/null +++ b/packages/plugin-lighthouse/CONTRIBUTING.md @@ -0,0 +1,124 @@ +# Contributing + +## Setup + +Make sure to install dependencies: + +```sh +npm install +``` + +### Chrome path + +In this plugin we provide Lighthouse functionality exposed over the `lighthousePlugin`. +To test lighthouse properly we work with a predefined testing setup. + +On some OS there could be a problem finding the path to Chrome. + +We try to detect it automatically in the set-setup script. + +If no chrome path is detected the error looks like this: `Runtime error encountered: No Chrome installations found.` + +To prevent this from happening you have to provide the path manually in your `.env`: + +```bash +CUSTOM_CHROME_PATH=/Applications/Google Chrome.app/Contents/MacOS/Google Chrome +``` + +In the CI you can set the env variable like this: + +```yml +# ... +- name: Set custom Chrome path for Windows only + if: matrix.os == 'windows-latest' + run: | + echo "CUSTOM_CHROME_PATH=C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + shell: pwsh + +# Optional debug log +- name: Log all environment variables + run: printenv +# ... +``` + +We added consider this path in our `beforeAll` hook. + +```ts +beforeEach(() => { + try { + vi.stubEnv('CHROME_PATH', getChromePath()); + } catch (e) { + const customChromePath = process.env['CUSTOM_CHROME_PATH']; + if (customChromePath == null || customChromePath === '') { + throw new Error('Chrome path not found. Please read the in the packages CONTRIBUTING.md/#trouble-shooting section.'); + } + vi.stubEnv('CHROME_PATH', customChromePath); + } +}); +``` + +### Testing chrome flags + +1. run `npx chrome-debug --` to pass terminal arguments to Chrome. E.g. `npx chrome-debug --headless=shell`. + `npx chrome-debug --headless=shell --@TODO-PUT-OTHER-EXAMPLE-FOR-FLAG` + +For a full list of available flags check out [this document](https://peter.sh/experiments/chromium-command-line-switches/). + +> [!NOTE] +> To pass chrome flags to lighthouse you have to provide them under `--chrome-flags=""`. +> E.g. `lighthouse https://example.com --chrome-flage="--headless=shell"` + +2. Check if the flag got accepted. This is quite unintuitive as we would expect the passed flag to be visible under `chrome://flags/` but as you can see in the screenshot it is not visible. + chrome-flags + Instead open `chrome://version/` and look under the "Command Line" section. + chrome-chrome-version + +### Chrome User Data + +To bootstrap Chrome with a predefined for setting we have to provide a couple of config files that we located under `/mock/chromium-user-data`. +When executing Lighthouse we provide the path to this folder over the `Flag` object. + +To generate initialise or edit the file structure under `chromium-user-data` do the following steps: + +1. Spin up Chrome by running `npx chrome-debug --user-data-dir=./packages/plugin-lighthouse/mock/chromium-user-data` + chrome-blank-screen + +2. If you do this the first time you should already see content under `/mock/chromium-user-data` +3. Edit the configuration over the Chrome UI. E.g. adding a profile +4. Close chromium and open it again, and you should see chromium bootstraps as the configured user + chrome-blank-screen-pre-configured + +To reset the above just delete the folder and apply the settings again. + +_A helpful chromium setup is preconfigured with the following settings:_ + +- A user profile is set up. This enables certain debugging related options as well as help to visually distinguish between test setups as the header bar is colored. + chrome-settings-manage-profile + +#### Resources + +- [chromium flags guide](https://www.chromium.org/developers/how-tos/run-chromium-with-flags/) + +## Troubleshooting + +1. Verify Chrome Installation + Ensure Chrome is correctly installed and accessible to the Lighthouse process. + Run `npx chrome-debug` to test it. Read further under [chrome-path](#chrome-path) + +2. Increase Timeout + Lighthouse has a longer runtime which can time out in different environments. + **Try increasing the test timeout** in `lighthouse-plugin.integration.test.ts` for `runner creation and execution` test suite. + +3. Turn on debug mode + Show debug logs of Lighthouse. Set the following environment variable: `DEBUG='*'` + +4. Understand error messages (⏳ could also be because of timeout problems :D ) + +- Could not find `report.json` (⏳) + ![lighthouse-error-2.png](./docs/images/lighthouse-error-2.png) +- Lighthouse Error - `Could Not Connect to Chrome` (⏳) + ![lighthouse-error-1.png](./docs/images/lighthouse-error-1.png) + Your Chrome path is set incorrectly. Read further under [chrome-path](#chrome-path) +- Lighthouse Error - `start lh::" performance mark has not been set` (⏳) + ![lighthouse-error-3.png](./docs/images/lighthouse-error-3.png) + If this error pops up you are able to launch Chrome but had problems to communicate over the ports. diff --git a/packages/plugin-lighthouse/README.md b/packages/plugin-lighthouse/README.md index 2e1cbda30..9a79eb8dc 100644 --- a/packages/plugin-lighthouse/README.md +++ b/packages/plugin-lighthouse/README.md @@ -1,3 +1,121 @@ # @code-pushup/lighthouse-plugin -TODO: docs +[![npm](https://img.shields.io/npm/v/%40code-pushup%2Flighthouse-plugin.svg)](https://www.npmjs.com/package/@code-pushup/lighthouse-plugin) +[![downloads](https://img.shields.io/npm/dm/%40code-pushup%2Flighthouse-plugin)](https://npmtrends.com/@code-pushup/lighthouse-plugin) +[![dependencies](https://img.shields.io/librariesio/release/npm/%40code-pushup/lighthouse-plugin)](https://www.npmjs.com/package/@code-pushup/lighthouse-plugin?activeTab=dependencies) + +🕵️ **Code PushUp plugin for measuring web performance and quality with Lighthouse.** 🔥 + +--- + +The plugin parses your Lighthouse configuration and lints all audits of the official [Lighthouse](https://github.com/GoogleChrome/lighthouse/blob/main/readme.md#lighthouse-------) CLI. + +Detected Lighthouse audits are mapped to Code PushUp audits. Audit reports are calculated based on the [original implementation](https://googlechrome.github.io/lighthouse/scorecalc/). +Additionally, Lighthouse categories are mapped to Code PushUp groups which can make it easier to assemble the categories. + +For more infos visit the [official docs](https://developer.chrome.com/docs/lighthouse/overview). + +## Getting started + +1. If you haven't already, install [@code-pushup/cli](../cli/README.md) and create a configuration file. + +2. Install as a dev dependency with your package manager: + + ```sh + npm install --save-dev @code-pushup/lighthouse-plugin + ``` + + ```sh + yarn add --dev @code-pushup/lighthouse-plugin + ``` + + ```sh + pnpm add --save-dev @code-pushup/lighthouse-plugin + ``` + +3. Add this plugin to the `plugins` array in your Code PushUp CLI config file (e.g. `code-pushup.config.ts`). + + Pass in the URL you want to measure, along with optional [flags](#flags) and [config](#config) data. + + ```ts + import lighthousePlugin from '@code-pushup/lighthouse-plugin'; + + export default { + // ... + plugins: [ + // ... + await lighthousePlugin('https://example.com'), + ], + }; + ``` + +4. Run the CLI with `npx code-pushup collect` and view or upload the report (refer to [CLI docs](../cli/README.md)). + +### Optionally set up categories + +@TODO + +## Flags + +The plugin accepts a second optional argument, `flags`. + +`flags` is the Lighthouse [CLI flags](https://github.com/GoogleChrome/lighthouse/blob/7d80178c37a1b600ea8f092fc0b098029799a659/cli/cli-flags.js#L80) as a JS object. + +Within the flags object a couple of other external configuration files can be referenced. E.g. `configPath` , `preset` or `budgetPath` reference external `json` or JavaScript files. + +For a complete list the [official documentation of CLI flags](https://github.com/GoogleChrome/lighthouse/blob/main/readme.md#cli-options) + +> [!TIP] +> If you are not used to work with the Lighthouse CLI you would pass flags like this: +> `lighthouse https://example.com --output=json --chromeFlags='--headless=shell'` +> +> Now with the plugin it would look like this: +> +> ```ts +> // code-pushup.config.ts +> ... +> lighthousePlugin('https://example.com', { output: 'json', chromeFlags: ['--headless=shell']}); +> ``` + +## Config + +The plugin accepts a third optional argument, `config`. + +`config` is the Lighthouse [configuration](https://github.com/GoogleChrome/lighthouse/blob/7d80178c37a1b600ea8f092fc0b098029799a659/types/config.d.ts#L21) as a JS object. + +For a complete guide on Lighthouse configuration read the [official documentation on configuring](https://github.com/GoogleChrome/lighthouse/blob/main/docs/configuration.md) + +> [!TIP] +> If you are not used to work with the Lighthouse CLI you would pass a config like this: +> `lighthouse --config-path=path/to/custom-config.js https://example.com` +> +> And in a separate file you would place the following object: +> +> ```typescript +> // custom-config.js file +> export default { +> extends: 'lighthouse:default', +> settings: { +> onlyAudits: ['first-meaningful-paint', 'speed-index', 'interactive'], +> }, +> }; +> ``` +> +> Now with the plugin it would look like this: +> +> ```ts +> // code-pushup.config.ts +> ... +> lighthousePlugin('https://example.com', undefined, { +> extends: 'lighthouse:default', +> settings: { +> onlyAudits: [ +> 'first-meaningful-paint', +> 'speed-index', +> 'interactive', +> ], +> } +> }) +> ``` + +If you want to contribute, please refer to [CONTRIBUTING.md](./CONTRIBUTING.md). diff --git a/packages/plugin-lighthouse/docs/images/chrome-blank-screen-pre-configure.png b/packages/plugin-lighthouse/docs/images/chrome-blank-screen-pre-configure.png new file mode 100644 index 000000000..8873ee240 Binary files /dev/null and b/packages/plugin-lighthouse/docs/images/chrome-blank-screen-pre-configure.png differ diff --git a/packages/plugin-lighthouse/docs/images/chrome-blank-screen.png b/packages/plugin-lighthouse/docs/images/chrome-blank-screen.png new file mode 100644 index 000000000..f3b9d7893 Binary files /dev/null and b/packages/plugin-lighthouse/docs/images/chrome-blank-screen.png differ diff --git a/packages/plugin-lighthouse/docs/images/chrome-flags.png b/packages/plugin-lighthouse/docs/images/chrome-flags.png new file mode 100644 index 000000000..46fce6bf4 Binary files /dev/null and b/packages/plugin-lighthouse/docs/images/chrome-flags.png differ diff --git a/packages/plugin-lighthouse/docs/images/chrome-settings-manage-profile.png b/packages/plugin-lighthouse/docs/images/chrome-settings-manage-profile.png new file mode 100644 index 000000000..a0c805145 Binary files /dev/null and b/packages/plugin-lighthouse/docs/images/chrome-settings-manage-profile.png differ diff --git a/packages/plugin-lighthouse/docs/images/chrome-version.png b/packages/plugin-lighthouse/docs/images/chrome-version.png new file mode 100644 index 000000000..483a4842c Binary files /dev/null and b/packages/plugin-lighthouse/docs/images/chrome-version.png differ diff --git a/packages/plugin-lighthouse/docs/images/lighthouse-error-1.png b/packages/plugin-lighthouse/docs/images/lighthouse-error-1.png new file mode 100644 index 000000000..ad918016a Binary files /dev/null and b/packages/plugin-lighthouse/docs/images/lighthouse-error-1.png differ diff --git a/packages/plugin-lighthouse/docs/images/lighthouse-error-2.png b/packages/plugin-lighthouse/docs/images/lighthouse-error-2.png new file mode 100644 index 000000000..6efe0b9a1 Binary files /dev/null and b/packages/plugin-lighthouse/docs/images/lighthouse-error-2.png differ diff --git a/packages/plugin-lighthouse/docs/images/lighthouse-error-3.png b/packages/plugin-lighthouse/docs/images/lighthouse-error-3.png new file mode 100644 index 000000000..9d52d734d Binary files /dev/null and b/packages/plugin-lighthouse/docs/images/lighthouse-error-3.png differ diff --git a/packages/plugin-lighthouse/package.json b/packages/plugin-lighthouse/package.json index e469e861d..fe2a1b0a3 100644 --- a/packages/plugin-lighthouse/package.json +++ b/packages/plugin-lighthouse/package.json @@ -5,6 +5,8 @@ "dependencies": { "@code-pushup/models": "*", "lighthouse": "^11.0.0", - "@code-pushup/utils": "*" + "@code-pushup/utils": "*", + "lighthouse-logger": "2.0.1", + "chalk": "^5.3.0" } } diff --git a/packages/plugin-lighthouse/src/index.ts b/packages/plugin-lighthouse/src/index.ts index 691d67cdc..761f845ec 100644 --- a/packages/plugin-lighthouse/src/index.ts +++ b/packages/plugin-lighthouse/src/index.ts @@ -1,3 +1,11 @@ import { lighthousePlugin } from './lib/lighthouse-plugin'; +export { lighthousePlugin, LighthouseCliFlags } from './lib/lighthouse-plugin'; +export { + LIGHTHOUSE_REPORT_NAME, + LIGHTHOUSE_PLUGIN_SLUG, + LIGHTHOUSE_AUDITS, + LIGHTHOUSE_GROUPS, +} from './lib/constants'; + export default lighthousePlugin; diff --git a/packages/plugin-lighthouse/src/lib/constants.ts b/packages/plugin-lighthouse/src/lib/constants.ts index 2485b5174..b328ceda6 100644 --- a/packages/plugin-lighthouse/src/lib/constants.ts +++ b/packages/plugin-lighthouse/src/lib/constants.ts @@ -1,4 +1,5 @@ import { + type CliFlags, type Config, type IcuMessage, Audit as LHAudit, @@ -11,7 +12,7 @@ export const LIGHTHOUSE_REPORT_NAME = 'lighthouse-report.json'; const { audits, categories } = defaultConfig; -export const GROUPS: Group[] = Object.entries(categories ?? {}).map( +export const LIGHTHOUSE_GROUPS: Group[] = Object.entries(categories ?? {}).map( ([id, category]) => ({ slug: id, title: getMetaString(category.title), @@ -22,7 +23,7 @@ export const GROUPS: Group[] = Object.entries(categories ?? {}).map( }), ); -export const AUDITS: Audit[] = await Promise.all( +export const LIGHTHOUSE_AUDITS: Audit[] = await Promise.all( (audits ?? []).map(async value => { const audit = await loadLighthouseAudit(value); return { @@ -63,3 +64,21 @@ async function loadLighthouseAudit( }; return module.default; } + +export const DEFAULT_CLI_FLAGS: Partial = { + // default values extracted from + // https://github.com/GoogleChrome/lighthouse/blob/7d80178c37a1b600ea8f092fc0b098029799a659/cli/cli-flags.js#L80 + verbose: false, + quiet: false, + saveAssets: false, + // needed to pass CI on linux and windows (locally it works without headless too) + chromeFlags: '--headless=shell', + port: 0, + hostname: '127.0.0.1', + view: false, + channel: 'cli', + chromeIgnoreDefaultFlags: false, + // custom overwrites in favour of the plugin + output: ['json'], + outputPath: LIGHTHOUSE_REPORT_NAME, +}; diff --git a/packages/plugin-lighthouse/src/lib/lighthouse-plugin.integration.test.ts b/packages/plugin-lighthouse/src/lib/lighthouse-plugin.integration.test.ts index 19f619fc1..51949be6f 100644 --- a/packages/plugin-lighthouse/src/lib/lighthouse-plugin.integration.test.ts +++ b/packages/plugin-lighthouse/src/lib/lighthouse-plugin.integration.test.ts @@ -1,12 +1,77 @@ -import { expect } from 'vitest'; -import { pluginConfigSchema } from '@code-pushup/models'; -import { lighthousePlugin } from './lighthouse-plugin'; +import { rm } from 'node:fs/promises'; +import { join } from 'node:path'; +import { afterEach, expect } from 'vitest'; +import { AuditOutput, pluginConfigSchema } from '@code-pushup/models'; +import { + getLogMessages, + shouldSkipLongRunningTests, +} from '@code-pushup/test-utils'; +import { ui } from '@code-pushup/utils'; +import { createRunnerFunction, lighthousePlugin } from './lighthouse-plugin'; + +const lighthousePluginTestFolder = join('tmp', 'plugin-lighthouse'); describe('lighthousePlugin', () => { it('should create valid plugin config', () => { - const pluginConfig = lighthousePlugin('https://code-pushup-portal.com'); + const pluginConfig = lighthousePlugin('https://www.google.com/'); expect(() => pluginConfigSchema.parse(pluginConfig)).not.toThrow(); expect(pluginConfig.audits.length).toBeGreaterThan(100); expect(pluginConfig.groups).toHaveLength(5); }); }); + +// eslint-disable-next-line vitest/no-disabled-tests +describe.skip('runner creation and execution', () => { + const getRunnerTestFolder = join(lighthousePluginTestFolder, 'get-runner'); + + afterEach(async () => { + await rm(getRunnerTestFolder, { recursive: true, force: true }); + }); + + afterAll(async () => { + await rm(lighthousePluginTestFolder, { recursive: true, force: true }); + }); + + it.skipIf(shouldSkipLongRunningTests())( + 'should create and execute runner correctly', + async () => { + const runner = createRunnerFunction('https://www.google.com/', { + // onlyAudits is used to reduce test time + onlyAudits: ['is-on-https'], + outputPath: + 'tmp/plugin-lighthouse/get-runner/should-create/lh-report.json', + chromeFlags: ['--headless=shell'], + }); + await expect(runner(undefined)).resolves.toEqual([ + expect.objectContaining({ + slug: 'is-on-https', + score: 1, + value: 0, + } satisfies AuditOutput), + ]); + }, + ); + + it.skipIf(shouldSkipLongRunningTests())( + 'should log about unsupported precomputedLanternDataPath flag', + async () => { + const precomputedLanternDataPath = join( + 'path', + 'to', + 'latern-data-folder', + ); + const runner = createRunnerFunction('https://www.google.com/', { + precomputedLanternDataPath, + // onlyAudits is used to reduce test time + onlyAudits: ['is-on-https'], + outputPath: + 'tmp/plugin-lighthouse/get-runner/no-latern-data/lh-report.json', + chromeFlags: ['--headless=shell'], + }); + await expect(runner(undefined)).resolves.toBeTruthy(); + expect(getLogMessages(ui().logger).at(0)).toMatch( + `Parsing precomputedLanternDataPath "${precomputedLanternDataPath}" is skipped as not implemented.`, + ); + }, + ); +}, 45_000); diff --git a/packages/plugin-lighthouse/src/lib/lighthouse-plugin.ts b/packages/plugin-lighthouse/src/lib/lighthouse-plugin.ts index 741f3ee73..7a1a024c9 100644 --- a/packages/plugin-lighthouse/src/lib/lighthouse-plugin.ts +++ b/packages/plugin-lighthouse/src/lib/lighthouse-plugin.ts @@ -1,20 +1,51 @@ +import { type CliFlags, type RunnerResult } from 'lighthouse'; +import { runLighthouse } from 'lighthouse/cli/run.js'; +import { dirname } from 'node:path'; import { - type Config as LighthouseConfig, - type CliFlags as LighthouseFlags, -} from 'lighthouse'; -import { PluginConfig } from '@code-pushup/models'; -import { AUDITS, GROUPS, LIGHTHOUSE_PLUGIN_SLUG } from './constants'; -import { filterAuditsAndGroupsByOnlyOptions } from './utils'; + AuditOutputs, + PluginConfig, + RunnerFunction, +} from '@code-pushup/models'; +import { ensureDirectoryExists, ui } from '@code-pushup/utils'; +import { + DEFAULT_CLI_FLAGS, + LIGHTHOUSE_AUDITS, + LIGHTHOUSE_GROUPS, + LIGHTHOUSE_PLUGIN_SLUG, +} from './constants'; +import { + filterAuditsAndGroupsByOnlyOptions, + getBudgets, + getConfig, + setLogLevel, + toAuditOutputs, + validateFlags, +} from './utils'; +export type LighthouseCliFlags = Partial< + Omit +>; + +// No error reporting implemented as in the source Sentry was involved +/* +if (cliFlags.enableErrorReporting) { + await Sentry.init({ + url: urlUnderTest, + flags: cliFlags, + environmentData: { + serverName: 'redacted', // prevent sentry from using hostname + environment: isDev() ? 'development' : 'production', + release: pkg.version, + }, + }); + */ export function lighthousePlugin( url: string, - flags?: Partial, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - config?: Partial, + flags?: LighthouseCliFlags, ): PluginConfig { const { audits, groups } = filterAuditsAndGroupsByOnlyOptions( - AUDITS, - GROUPS, + LIGHTHOUSE_AUDITS, + LIGHTHOUSE_GROUPS, flags, ); return { @@ -23,6 +54,58 @@ export function lighthousePlugin( icon: 'lighthouse', audits, groups, - runner: () => audits.map(({ slug }) => ({ slug, value: 0, score: 0 })), + runner: createRunnerFunction(url, flags), + }; +} + +export function createRunnerFunction( + urlUnderTest: string, + flags: LighthouseCliFlags = {}, +): RunnerFunction { + return async (): Promise => { + const { + precomputedLanternDataPath, + budgetPath, + budgets = [], + outputPath, + ...parsedFlags + } = validateFlags({ + ...DEFAULT_CLI_FLAGS, + ...flags, + }); + + setLogLevel(parsedFlags); + + const config = await getConfig(parsedFlags); + + const budgetsJson = budgetPath ? await getBudgets(budgetPath) : budgets; + + if (outputPath) { + await ensureDirectoryExists(dirname(outputPath)); + } + + const flagsWithDefaults = { + ...parsedFlags, + budgets: budgetsJson, + outputPath, + }; + + if (precomputedLanternDataPath) { + ui().logger.info( + `Parsing precomputedLanternDataPath "${precomputedLanternDataPath}" is skipped as not implemented.`, + ); + } + + const runnerResult: unknown = await runLighthouse( + urlUnderTest, + flagsWithDefaults, + config, + ); + + if (runnerResult == null) { + throw new Error('Lighthouse did not produce a result.'); + } + const { lhr } = runnerResult as RunnerResult; + return toAuditOutputs(Object.values(lhr.audits)); }; } diff --git a/packages/plugin-lighthouse/src/lib/lighthouse-plugin.unit.test.ts b/packages/plugin-lighthouse/src/lib/lighthouse-plugin.unit.test.ts index 5e73302ec..5f0a25c3f 100644 --- a/packages/plugin-lighthouse/src/lib/lighthouse-plugin.unit.test.ts +++ b/packages/plugin-lighthouse/src/lib/lighthouse-plugin.unit.test.ts @@ -1,18 +1,148 @@ -import { expect } from 'vitest'; +import { type Config } from 'lighthouse'; +import { runLighthouse } from 'lighthouse/cli/run.js'; +import { Result } from 'lighthouse/types/lhr/audit-result'; +import { expect, vi } from 'vitest'; import { auditSchema, groupSchema, pluginConfigSchema, } from '@code-pushup/models'; -import { AUDITS, GROUPS } from './constants'; -import { lighthousePlugin } from './lighthouse-plugin'; +import { LIGHTHOUSE_AUDITS, LIGHTHOUSE_GROUPS } from './constants'; +import { + LighthouseCliFlags, + createRunnerFunction, + lighthousePlugin, +} from './lighthouse-plugin'; +import { getBudgets, getConfig, setLogLevel } from './utils'; + +vi.mock('./utils', async () => { + // Import the actual 'lighthouse' module + const actual = await vi.importActual('./utils'); + + // Return the mocked module, merging the actual module with overridden parts + return { + ...actual, + setLogLevel: vi.fn(), + getBudgets: vi.fn().mockImplementation((path: string) => [{ path }]), + getConfig: vi.fn(), + }; +}); + +vi.mock('lighthouse/cli/run.js', async () => { + // Import the actual 'lighthouse' module + const actual = await import('lighthouse/cli/run.js'); + // Define the mock implementation + const mockRunLighthouse = vi.fn( + (url: string, flags: LighthouseCliFlags, config: Config) => + url.includes('fail') + ? undefined + : { + flags, + config, + lhr: { + audits: { + ['cumulative-layout-shift']: { + id: 'cumulative-layout-shift', + title: 'Cumulative Layout Shift', + description: + 'Cumulative Layout Shift measures the movement of visible elements within the viewport.', + scoreDisplayMode: 'numeric', + numericValue: 1200, + displayValue: '1.2 s', + score: 0.9, + } satisfies Result, + }, + }, + }, + ); + + // Return the mocked module, merging the actual module with overridden parts + return { + ...actual, + runLighthouse: mockRunLighthouse, // Mock the default export if 'lighthouse' is imported as default + }; +}); + +describe('createRunnerFunction', () => { + it('should return AuditOutputs if executed correctly', async () => { + const runner = createRunnerFunction('https://localhost:8080'); + await expect(runner(undefined)).resolves.toEqual( + expect.arrayContaining([ + { + slug: 'cumulative-layout-shift', + value: 1200, + displayValue: '1.2 s', + score: 0.9, + }, + ]), + ); + + expect(setLogLevel).toHaveBeenCalledWith( + expect.objectContaining({ verbose: false, quiet: false }), + ); + expect(getBudgets).not.toHaveBeenCalled(); + expect(getConfig).toHaveBeenCalledWith(expect.objectContaining({})); + }); + + it('should return verbose and quiet flags for logging', async () => { + await createRunnerFunction('https://localhost:8080', { + verbose: true, + quiet: true, + })(undefined); + expect(setLogLevel).toHaveBeenCalledWith( + expect.objectContaining({ verbose: true, quiet: true }), + ); + }); + + it('should return configPath', async () => { + await createRunnerFunction('https://localhost:8080', { + configPath: 'lh-config.js', + })(undefined); + expect(getConfig).toHaveBeenCalledWith( + expect.objectContaining({ configPath: 'lh-config.js' }), + ); + }); + + it('should return budgets from the budgets object directly', async () => { + await createRunnerFunction('https://localhost:8080', { + budgets: [{ path: '*/xyz/' }], + })(undefined); + expect(getBudgets).not.toHaveBeenCalled(); + expect(runLighthouse).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ budgets: [{ path: '*/xyz/' }] }), + undefined, + ); + }); + + it('should return budgets maintained in the file specified over budgetPath', async () => { + await createRunnerFunction('https://localhost:8080', { + budgetPath: 'lh-budgets.js', + } as LighthouseCliFlags)(undefined); + expect(getBudgets).toHaveBeenCalledWith('lh-budgets.js'); + expect(runLighthouse).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ budgets: [{ path: 'lh-budgets.js' }] }), + undefined, + ); + }); + + it('should throw if lighthouse returns an empty result', async () => { + const runner = createRunnerFunction('fail'); + await expect(runner(undefined)).rejects.toThrow( + 'Lighthouse did not produce a result.', + ); + }); +}); describe('lighthousePlugin-config-object', () => { it('should create valid plugin config', () => { const pluginConfig = lighthousePlugin('https://code-pushup-portal.com'); expect(() => pluginConfigSchema.parse(pluginConfig)).not.toThrow(); - expect(pluginConfig.audits.length).toBeGreaterThan(0); - expect(pluginConfig.groups?.length).toBeGreaterThan(0); + + const { audits, groups } = pluginConfig; + expect(audits.length).toBeGreaterThan(0); + expect(groups?.length).toBeGreaterThan(0); }); it('should filter audits by onlyAudits string "first-contentful-paint"', () => { @@ -51,7 +181,7 @@ describe('lighthousePlugin-config-object', () => { }); describe('constants', () => { - it.each(AUDITS.map(a => [a.slug, a]))( + it.each(LIGHTHOUSE_AUDITS.map(a => [a.slug, a]))( 'should parse audit "%s" correctly', (slug, audit) => { expect(() => auditSchema.parse(audit)).not.toThrow(); @@ -59,7 +189,7 @@ describe('constants', () => { }, ); - it.each(GROUPS.map(g => [g.slug, g]))( + it.each(LIGHTHOUSE_GROUPS.map(g => [g.slug, g]))( 'should parse group "%s" correctly', (slug, group) => { expect(() => groupSchema.parse(group)).not.toThrow(); diff --git a/packages/plugin-lighthouse/src/lib/utils.ts b/packages/plugin-lighthouse/src/lib/utils.ts index f0538c091..ab22ebdf2 100644 --- a/packages/plugin-lighthouse/src/lib/utils.ts +++ b/packages/plugin-lighthouse/src/lib/utils.ts @@ -1,13 +1,22 @@ -import { type CliFlags } from 'lighthouse'; +import chalk from 'chalk'; +import { type Budget, type CliFlags, type Config } from 'lighthouse'; +import log from 'lighthouse-logger'; +import desktopConfig from 'lighthouse/core/config/desktop-config.js'; +import experimentalConfig from 'lighthouse/core/config/experimental-config.js'; +import perfConfig from 'lighthouse/core/config/perf-config.js'; import { Result } from 'lighthouse/types/lhr/audit-result'; +import path from 'node:path'; import { Audit, AuditOutput, AuditOutputs, Group } from '@code-pushup/models'; import { filterItemRefsBy, + importEsmModule, objectToCliArgs, + readJsonFile, toArray, ui, } from '@code-pushup/utils'; import { LIGHTHOUSE_REPORT_NAME } from './constants'; +import { type LighthouseCliFlags } from './lighthouse-plugin'; type RefinedLighthouseOption = { url: CliFlags['_']; @@ -87,7 +96,7 @@ export function toAuditOutputs(lhrAudits: Result[]): AuditOutputs { const auditOutput: AuditOutput = { slug, score: score ?? 1, // score can be null - value, + value: Number.parseInt(value.toString(), 10), displayValue, }; @@ -166,3 +175,90 @@ export function filterAuditsAndGroupsByOnlyOptions( groups, }; } + +export async function getConfig( + flags: Pick = {}, +): Promise { + const { configPath: filepath, preset } = flags; + + if (filepath != null) { + // Resolve the config file path relative to where cli was called. + + if (filepath.endsWith('.json')) { + return readJsonFile(filepath); + } else if (/\.(ts|js|mjs)$/.test(filepath)) { + return importEsmModule({ filepath }); + } + } else if (typeof preset === 'string') { + switch (preset) { + case 'desktop': + return desktopConfig; + case 'perf': + return perfConfig as Config; + case 'experimental': + return experimentalConfig as Config; + default: + // as preset is a string literal the default case here is normally caught by TS and not possible to happen. Now in reality it can happen and preset could be a string not included in the literal. + // Therefor we have to use `as string` is used. Otherwise, it will consider preset as type never + ui().logger.info(`Preset "${preset as string}" is not supported`); + } + } + return undefined; +} + +export async function getBudgets( + budgetPath?: string | null, +): Promise { + if (budgetPath) { + /** @type {Array} */ + return await readJsonFile( + path.resolve(process.cwd(), budgetPath), + ); + } + return []; +} + +export function setLogLevel({ + verbose, + quiet, +}: { + verbose?: boolean; + quiet?: boolean; +} = {}) { + // set logging preferences + if (verbose) { + log.setLevel('verbose'); + } else if (quiet) { + log.setLevel('silent'); + } else { + log.setLevel('info'); + } +} + +const excludedFlags = new Set([ + // lighthouse CLI specific debug logs + 'list-all-audits', // Prints a list of all available audits and exits. + 'list-locales', // Prints a list of all supported locales and exits. + 'list-trace-categories', // Prints a list of all required trace categories and exits. +]); + +export function validateFlags( + flags: LighthouseCliFlags = {}, +): LighthouseCliFlags { + const unsupportedFlagsInUse = Object.keys(flags).filter(flag => + excludedFlags.has(flag), + ); + + if (unsupportedFlagsInUse.length > 0) { + ui().logger.info( + `${chalk.yellow( + '⚠', + )} The following used flags are not supported: ${chalk.bold( + unsupportedFlagsInUse.join(', '), + )}`, + ); + } + return Object.fromEntries( + Object.entries(flags).filter(([flagName]) => !excludedFlags.has(flagName)), + ); +} diff --git a/packages/plugin-lighthouse/src/lib/utils.unit.test.ts b/packages/plugin-lighthouse/src/lib/utils.unit.test.ts index f28655135..ff02cde4d 100644 --- a/packages/plugin-lighthouse/src/lib/utils.unit.test.ts +++ b/packages/plugin-lighthouse/src/lib/utils.unit.test.ts @@ -1,42 +1,58 @@ +import chalk from 'chalk'; +import debug from 'debug'; +import { type Budget } from 'lighthouse'; +import log from 'lighthouse-logger'; import Details from 'lighthouse/types/lhr/audit-details'; -import { describe, expect, it } from 'vitest'; +import { vol } from 'memfs'; +import { join } from 'node:path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { Audit, - AuditOutput, + CoreConfig, Group, PluginConfig, + auditOutputsSchema, pluginConfigSchema, } from '@code-pushup/models'; +import { MEMFS_VOLUME, getLogMessages } from '@code-pushup/test-utils'; +import { ui } from '@code-pushup/utils'; +import { LighthouseCliFlags } from './lighthouse-plugin'; import { AuditsNotImplementedError, CategoriesNotImplementedError, filterAuditsAndGroupsByOnlyOptions, - getLighthouseCliArguments, + getBudgets, + getConfig, + setLogLevel, toAuditOutputs, + validateFlags, validateOnlyAudits, validateOnlyCategories, } from './utils'; -describe('getLighthouseCliArguments', () => { - it('should parse valid options', () => { - expect( - getLighthouseCliArguments({ - url: ['https://code-pushup-portal.com'], +// mock bundleRequire inside importEsmModule used for fetching config +vi.mock('bundle-require', async () => { + const { CORE_CONFIG_MOCK }: Record = + await vi.importActual('@code-pushup/test-utils'); + + return { + bundleRequire: vi + .fn() + .mockImplementation((options: { filepath: string }) => { + const project = options.filepath.split('.').at(-2); + return { + mod: { + default: { + ...CORE_CONFIG_MOCK, + upload: { + ...CORE_CONFIG_MOCK?.upload, + project, // returns loaded file extension to check in test + }, + }, + }, + }; }), - ).toEqual(expect.arrayContaining(['https://code-pushup-portal.com'])); - }); - - it('should parse chrome-flags options correctly', () => { - const args = getLighthouseCliArguments({ - url: ['https://code-pushup-portal.com'], - chromeFlags: { headless: 'new', 'user-data-dir': 'test' }, - }); - expect(args).toEqual( - expect.arrayContaining([ - '--chromeFlags="--headless=new --user-data-dir=test"', - ]), - ); - }); + }; }); describe('validateOnlyAudits', () => { @@ -365,6 +381,26 @@ describe('filterAuditsAndGroupsByOnlyOptions to be used in plugin config', () => describe('toAuditOutputs', () => { it('should parse valid lhr details', () => { + expect(() => + auditOutputsSchema.parse( + toAuditOutputs([ + { + id: 'first-contentful-paint', + title: 'First Contentful Paint', + description: + 'First Contentful Paint marks the time at which the first text or image is painted. [Learn more about the First Contentful Paint metric](https://developer.chrome.com/docs/lighthouse/performance/first-contentful-paint/).', + score: 0.55, + scoreDisplayMode: 'numeric', + numericValue: 2838.974, + numericUnit: 'millisecond', + displayValue: '2.8 s', + }, + ]), + ), + ).not.toThrow(); + }); + + it('should parse valid lhr float value to integer', () => { expect( toAuditOutputs([ { @@ -379,14 +415,7 @@ describe('toAuditOutputs', () => { displayValue: '2.8 s', }, ]), - ).toStrictEqual([ - { - displayValue: '2.8 s', - score: 0.55, - slug: 'first-contentful-paint', - value: 2838.974, - }, - ]); + ).toStrictEqual([expect.objectContaining({ value: 2838 })]); }); it('should convert null score to 1', () => { @@ -587,3 +616,189 @@ describe('toAuditOutputs', () => { expect(outputs[0]?.details).toBeUndefined(); }); }); + +describe('getConfig', () => { + it('should return undefined if no path is specified', async () => { + await expect(getConfig()).resolves.toBeUndefined(); + }); + + it.each([ + [ + 'desktop', + expect.objectContaining({ + settings: expect.objectContaining({ formFactor: 'desktop' }), + }), + ], + [ + 'perf', + expect.objectContaining({ + settings: expect.objectContaining({ onlyCategories: ['performance'] }), + }), + ], + [ + 'experimental', + expect.objectContaining({ + audits: expect.arrayContaining(['autocomplete']), + }), + ], + ] satisfies readonly ['desktop' | 'perf' | 'experimental', object][])( + 'should load config from lighthouse preset if %s preset is specified', + async (preset, config) => { + await expect(getConfig({ preset })).resolves.toEqual(config); + }, + ); + + it('should return undefined if preset is specified wrong', async () => { + await expect( + getConfig({ preset: 'wrong' as 'desktop' }), + ).resolves.toBeUndefined(); + expect(getLogMessages(ui().logger).at(0)).toMatch( + 'Preset "wrong" is not supported', + ); + }); + + it('should load config from json file if configPath is specified', async () => { + vol.fromJSON( + { + 'lh-config.json': JSON.stringify( + { extends: 'lighthouse:default' }, + null, + 2, + ), + }, + MEMFS_VOLUME, + ); + await expect(getConfig({ configPath: 'lh-config.json' })).resolves.toEqual({ + extends: 'lighthouse:default', + }); + }); + + it('should load config from lh-config.js file if configPath is specified', async () => { + await expect(getConfig({ configPath: 'lh-config.js' })).resolves.toEqual( + expect.objectContaining({ + upload: expect.objectContaining({ + project: expect.stringContaining('lh-config'), + }), + }), + ); + }); + + it('should return undefined if configPath is specified wrong', async () => { + await expect( + getConfig({ configPath: join('wrong.xyz') }), + ).resolves.toBeUndefined(); + }); +}); + +describe('getBudgets', () => { + it('should return and empty array if no path is specified', async () => { + await expect(getBudgets()).resolves.toStrictEqual([]); + }); + + it('should load budgets from specified path', async () => { + const budgets: Budget[] = [ + { + path: '*', + resourceCounts: [ + { + budget: 3, + resourceType: 'media', + }, + ], + }, + ]; + vol.fromJSON( + { + 'lh-budgets.json': JSON.stringify(budgets, null, 2), + }, + MEMFS_VOLUME, + ); + await expect(getBudgets('lh-budgets.json')).resolves.toEqual(budgets); + }); + + it('should throw if path is specified wrong', async () => { + await expect(getBudgets('wrong.xyz')).rejects.toThrow( + 'ENOENT: no such file or directory', + ); + }); +}); + +describe('setLogLevel', () => { + const debugLib = debug as { enabled: (flag: string) => boolean }; + beforeEach(() => { + log.setLevel('info'); + }); + + /** + * + * case 'silent': + * debug.enable('-LH:*'); + * break; + * case 'verbose': + * debug.enable('LH:*'); + * break; + * case 'warn': + * debug.enable('-LH:*, LH:*:warn, LH:*:error'); + * break; + * case 'error': + * debug.enable('-LH:*, LH:*:error'); + * break; + * default: // 'info' + * debug.enable('LH:*, -LH:*:verbose'); + */ + + it('should set log level to info if no options are given', () => { + setLogLevel(); + expect(log.isVerbose()).toBe(false); + expect(debugLib.enabled('LH:*')).toBe(true); + expect(debugLib.enabled('LH:*:verbose')).toBe(false); + }); + + it('should set log level to verbose', () => { + setLogLevel({ verbose: true }); + expect(log.isVerbose()).toBe(true); + expect(debugLib.enabled('LH:*')).toBe(true); + expect(debugLib.enabled('LH:*:verbose')).toBe(false); + }); + + it('should set log level to quiet', () => { + setLogLevel({ quiet: true }); + expect(log.isVerbose()).toBe(false); + expect(debugLib.enabled('LH:*')).toBe(true); + expect(debugLib.enabled('-LH:*')).toBe(true); + expect(debugLib.enabled('LH:*:verbose')).toBe(false); + }); + + it('should set log level to verbose if verbose and quiet are given', () => { + setLogLevel({ verbose: true, quiet: true }); + expect(log.isVerbose()).toBe(true); + expect(debugLib.enabled('LH:*')).toBe(true); + expect(debugLib.enabled('LH:*:verbose')).toBe(false); + }); +}); + +describe('validateFlags', () => { + it('should work with empty flags', () => { + expect(validateFlags()).toStrictEqual({}); + }); + + it('should forward supported entries', () => { + expect(validateFlags({ verbose: true })).toStrictEqual({ verbose: true }); + }); + + it('should remove unsupported entries and log', () => { + expect( + validateFlags({ + 'list-all-audits': true, + verbose: true, + } as LighthouseCliFlags), + ).toStrictEqual({ verbose: true }); + expect(getLogMessages(ui().logger).at(0)).toBe( + `[ blue(info) ] ${chalk.yellow( + '⚠', + )} The following used flags are not supported: ${chalk.bold( + 'list-all-audits', + )}`, + ); + }); +}); diff --git a/packages/plugin-lighthouse/vite.config.integration.ts b/packages/plugin-lighthouse/vite.config.integration.ts index a24f19c9f..94d318a94 100644 --- a/packages/plugin-lighthouse/vite.config.integration.ts +++ b/packages/plugin-lighthouse/vite.config.integration.ts @@ -22,8 +22,9 @@ export default defineConfig({ include: ['src/**/*.integration.test.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'], globalSetup: ['../../global-setup.ts'], setupFiles: [ - '../../testing/test-setup/src/lib/console.mock.ts', + '../../testing/test-setup/src/lib/cliui.mock.ts', '../../testing/test-setup/src/lib/reset.mocks.ts', + '../../testing/test-setup/src/lib/chrome-path.setup.ts', ], }, }); diff --git a/packages/plugin-lighthouse/vite.config.unit.ts b/packages/plugin-lighthouse/vite.config.unit.ts index 55a1513bd..b7788a2c1 100644 --- a/packages/plugin-lighthouse/vite.config.unit.ts +++ b/packages/plugin-lighthouse/vite.config.unit.ts @@ -22,6 +22,7 @@ export default defineConfig({ include: ['src/**/*.unit.test.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'], globalSetup: ['../../global-setup.ts'], setupFiles: [ + '../../testing/test-setup/src/lib/cliui.mock.ts', '../../testing/test-setup/src/lib/fs.mock.ts', '../../testing/test-setup/src/lib/console.mock.ts', '../../testing/test-setup/src/lib/reset.mocks.ts', diff --git a/packages/utils/src/lib/file-system.ts b/packages/utils/src/lib/file-system.ts index 9903a83ff..86f7df727 100644 --- a/packages/utils/src/lib/file-system.ts +++ b/packages/utils/src/lib/file-system.ts @@ -81,7 +81,9 @@ export class NoExportError extends Error { } } -export async function importEsmModule(options: Options): Promise { +export async function importEsmModule( + options: Options, +): Promise { const { mod } = await bundleRequire({ format: 'esm', ...options, @@ -90,7 +92,7 @@ export async function importEsmModule(options: Options): Promise { if (!('default' in mod)) { throw new NoExportError(options.filepath); } - return mod.default; + return mod.default as T; } export function pluginWorkDir(slug: string): string { diff --git a/testing/test-setup/src/lib/chrome-path.setup.ts b/testing/test-setup/src/lib/chrome-path.setup.ts new file mode 100644 index 000000000..5a4011715 --- /dev/null +++ b/testing/test-setup/src/lib/chrome-path.setup.ts @@ -0,0 +1,13 @@ +import { getChromePath } from 'chrome-launcher'; +import * as process from 'node:process'; +import { beforeEach, vi } from 'vitest'; + +beforeEach(() => { + const customChromePath = process.env['CUSTOM_CHROME_PATH']; + + if (customChromePath == null) { + vi.stubEnv('CHROME_PATH', getChromePath()); + } else { + vi.stubEnv('CHROME_PATH', customChromePath); + } +}); diff --git a/testing/test-utils/src/index.ts b/testing/test-utils/src/index.ts index 0d0b42300..640567d05 100644 --- a/testing/test-utils/src/index.ts +++ b/testing/test-utils/src/index.ts @@ -2,6 +2,7 @@ export * from './lib/constants'; export * from './lib/utils/execute-process-helper.mock'; export * from './lib/utils/os-agnostic-paths'; export * from './lib/utils/logging'; +export * from './lib/utils/env'; // static mocks export * from './lib/utils/commit.mock'; diff --git a/testing/test-utils/src/lib/utils/env.ts b/testing/test-utils/src/lib/utils/env.ts new file mode 100644 index 000000000..e05b20c44 --- /dev/null +++ b/testing/test-utils/src/lib/utils/env.ts @@ -0,0 +1,6 @@ +export function shouldSkipLongRunningTests(): boolean { + if (process.env['INCLUDE_SLOW_TESTS']) { + return process.env['INCLUDE_SLOW_TESTS'] === 'false'; + } + return !process.env['CI']; +}