From 9e091e74bd9ee2dbbcb42f2a2c516df22df85556 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 1 May 2024 11:22:08 -0700 Subject: [PATCH] =?UTF-8?q?cherry-pick(#30611):=20chore:=20add=20common=20?= =?UTF-8?q?env=20vars=20for=20junit=20and=20json=20re=E2=80=A6=20(#30624)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …porters --- docs/src/test-reporters-js.md | 10 +++-- packages/playwright/src/reporters/base.ts | 47 ++++++++++++++++++++ packages/playwright/src/reporters/blob.ts | 34 +++++--------- packages/playwright/src/reporters/json.ts | 35 +++++++-------- packages/playwright/src/reporters/junit.ts | 18 +++----- tests/playwright-test/reporter-blob.spec.ts | 6 +++ tests/playwright-test/reporter-json.spec.ts | 38 ++++++++++++++++ tests/playwright-test/reporter-junit.spec.ts | 38 ++++++++++++++++ 8 files changed, 167 insertions(+), 59 deletions(-) diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 6c85dba15d162..b841e5029f894 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -231,7 +231,7 @@ Blob report supports following configuration options and environment variables: |---|---|---|---| | `PLAYWRIGHT_BLOB_OUTPUT_DIR` | `outputDir` | Directory to save the output. Existing content is deleted before writing the new report. | `blob-report` | `PLAYWRIGHT_BLOB_OUTPUT_NAME` | `fileName` | Report file name. | `report---.zip` -| `PLAYWRIGHT_BLOB_OUTPUT_FILE` | `outputFile` | Full path for the output. If defined, `outputDir` and `fileName` will be ignored. | `undefined` +| `PLAYWRIGHT_BLOB_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `outputDir` and `fileName` will be ignored. | `undefined` ### JSON reporter @@ -267,7 +267,9 @@ JSON report supports following configuration options and environment variables: | Environment Variable Name | Reporter Config Option| Description | Default |---|---|---|---| -| `PLAYWRIGHT_JUNIT_OUTPUT_NAME` | `outputFile` | Report file path. | JSON report is printed to stdout. +| `PLAYWRIGHT_JSON_OUTPUT_DIR` | | Directory to save the output file. Ignored if output file is specified. | `cwd` or config directory. +| `PLAYWRIGHT_JSON_OUTPUT_NAME` | `outputFile` | Base file name for the output, relative to the output dir. | JSON report is printed to the stdout. +| `PLAYWRIGHT_JSON_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `PLAYWRIGHT_JSON_OUTPUT_DIR` and `PLAYWRIGHT_JSON_OUTPUT_NAME` will be ignored. | JSON report is printed to the stdout. ### JUnit reporter @@ -303,7 +305,9 @@ JUnit report supports following configuration options and environment variables: | Environment Variable Name | Reporter Config Option| Description | Default |---|---|---|---| -| `PLAYWRIGHT_JUNIT_OUTPUT_NAME` | `outputFile` | Report file path. | JUnit report is printed to stdout. +| `PLAYWRIGHT_JUNIT_OUTPUT_DIR` | | Directory to save the output file. Ignored if output file is not specified. | `cwd` or config directory. +| `PLAYWRIGHT_JUNIT_OUTPUT_NAME` | `outputFile` | Base file name for the output, relative to the output dir. | JUnit report is printed to the stdout. +| `PLAYWRIGHT_JUNIT_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `PLAYWRIGHT_JUNIT_OUTPUT_DIR` and `PLAYWRIGHT_JUNIT_OUTPUT_NAME` will be ignored. | JUnit report is printed to the stdout. | | `stripANSIControlSequences` | Whether to remove ANSI control sequences from the text before writing it in the report. | By default output text is added as is. | | `includeProjectInTestName` | Whether to include Playwright project name in every test case as a name prefix. | By default not included. | `PLAYWRIGHT_JUNIT_SUITE_ID` | | Value of the `id` attribute on the root `` report entry. | Empty string. diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index f4da4a74a0f2b..0c023e876138b 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -19,6 +19,7 @@ import path from 'path'; import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location } from '../../types/testReporter'; import { getPackageManagerExecCommand } from 'playwright-core/lib/utils'; import type { ReporterV2 } from './reporterV2'; +import { resolveReporterOutputPath } from '../util'; export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' }; export const kOutputSymbol = Symbol('output'); @@ -547,3 +548,49 @@ function fitToWidth(line: string, width: number, prefix?: string): string { function belongsToNodeModules(file: string) { return file.includes(`${path.sep}node_modules${path.sep}`); } + +function resolveFromEnv(name: string): string | undefined { + const value = process.env[name]; + if (value) + return path.resolve(process.cwd(), value); + return undefined; +} + +// In addition to `outputFile` the function returns `outputDir` which should +// be cleaned up if present by some reporters contract. +export function resolveOutputFile(reporterName: string, options: { + configDir: string, + outputDir?: string, + fileName?: string, + outputFile?: string, + default?: { + fileName: string, + outputDir: string, + } + }): { outputFile: string, outputDir?: string } |undefined { + const name = reporterName.toUpperCase(); + let outputFile; + if (options.outputFile) + outputFile = path.resolve(options.configDir, options.outputFile); + if (!outputFile) + outputFile = resolveFromEnv(`PLAYWRIGHT_${name}_OUTPUT_FILE`); + // Return early to avoid deleting outputDir. + if (outputFile) + return { outputFile }; + + let outputDir; + if (options.outputDir) + outputDir = path.resolve(options.configDir, options.outputDir); + if (!outputDir) + outputDir = resolveFromEnv(`PLAYWRIGHT_${name}_OUTPUT_DIR`); + if (!outputDir && options.default) + outputDir = resolveReporterOutputPath(options.default.outputDir, options.configDir, undefined); + + if (!outputFile) { + const reportName = options.fileName ?? process.env[`PLAYWRIGHT_${name}_OUTPUT_NAME`] ?? options.default?.fileName; + if (!reportName) + return undefined; + outputFile = path.resolve(outputDir ?? process.cwd(), reportName); + } + return { outputFile, outputDir }; +} diff --git a/packages/playwright/src/reporters/blob.ts b/packages/playwright/src/reporters/blob.ts index f2db23e1d3b50..837fe55be0e9c 100644 --- a/packages/playwright/src/reporters/blob.ts +++ b/packages/playwright/src/reporters/blob.ts @@ -24,7 +24,7 @@ import type { FullConfig, FullResult, TestResult } from '../../types/testReporte import type { JsonAttachment, JsonEvent } from '../isomorphic/teleReceiver'; import { TeleReporterEmitter } from './teleEmitter'; import { yazl } from 'playwright-core/lib/zipBundle'; -import { resolveReporterOutputPath } from '../util'; +import { resolveOutputFile } from './base'; type BlobReporterOptions = { configDir: string; @@ -107,17 +107,15 @@ export class BlobReporter extends TeleReporterEmitter { } private async _prepareOutputFile() { - let outputFile = reportOutputFileFromEnv(); - if (!outputFile && this._options.outputFile) - outputFile = path.resolve(this._options.configDir, this._options.outputFile); - // Explicit `outputFile` overrides `outputDir` and `fileName` options. - if (!outputFile) { - const reportName = this._options.fileName || process.env[`PLAYWRIGHT_BLOB_OUTPUT_NAME`] || this._defaultReportName(this._config); - const outputDir = resolveReporterOutputPath('blob-report', this._options.configDir, this._options.outputDir ?? reportOutputDirFromEnv()); - if (!process.env.PWTEST_BLOB_DO_NOT_REMOVE) - await removeFolders([outputDir]); - outputFile = path.resolve(outputDir, reportName); - } + const { outputFile, outputDir } = resolveOutputFile('BLOB', { + ...this._options, + default: { + fileName: this._defaultReportName(this._config), + outputDir: 'blob-report', + } + })!; + if (!process.env.PWTEST_BLOB_DO_NOT_REMOVE) + await removeFolders([outputDir!]); await fs.promises.mkdir(path.dirname(outputFile), { recursive: true }); return outputFile; } @@ -149,15 +147,3 @@ export class BlobReporter extends TeleReporterEmitter { }); } } - -function reportOutputDirFromEnv(): string | undefined { - if (process.env[`PLAYWRIGHT_BLOB_OUTPUT_DIR`]) - return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_BLOB_OUTPUT_DIR`]); - return undefined; -} - -function reportOutputFileFromEnv(): string | undefined { - if (process.env[`PLAYWRIGHT_BLOB_OUTPUT_FILE`]) - return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_BLOB_OUTPUT_FILE`]); - return undefined; -} diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts index 36ba67825813e..62a625db5d937 100644 --- a/packages/playwright/src/reporters/json.ts +++ b/packages/playwright/src/reporters/json.ts @@ -17,24 +17,29 @@ import fs from 'fs'; import path from 'path'; import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep, JSONReportError } from '../../types/testReporter'; -import { formatError, prepareErrorStack } from './base'; -import { MultiMap, assert, toPosixPath } from 'playwright-core/lib/utils'; +import { formatError, prepareErrorStack, resolveOutputFile } from './base'; +import { MultiMap, toPosixPath } from 'playwright-core/lib/utils'; import { getProjectId } from '../common/config'; import EmptyReporter from './empty'; +type JSONOptions = { + outputFile?: string, + configDir: string, +}; + class JSONReporter extends EmptyReporter { config!: FullConfig; suite!: Suite; private _errors: TestError[] = []; - private _outputFile: string | undefined; + private _resolvedOutputFile: string | undefined; - constructor(options: { outputFile?: string } = {}) { + constructor(options: JSONOptions) { super(); - this._outputFile = options.outputFile || reportOutputNameFromEnv(); + this._resolvedOutputFile = resolveOutputFile('JSON', options)?.outputFile; } override printsToStdio() { - return !this._outputFile; + return !this._resolvedOutputFile; } override onConfigure(config: FullConfig) { @@ -50,7 +55,7 @@ class JSONReporter extends EmptyReporter { } override async onEnd(result: FullResult) { - await outputReport(this._serializeReport(result), this.config, this._outputFile); + await outputReport(this._serializeReport(result), this._resolvedOutputFile); } private _serializeReport(result: FullResult): JSONReport { @@ -228,13 +233,11 @@ class JSONReporter extends EmptyReporter { } } -async function outputReport(report: JSONReport, config: FullConfig, outputFile: string | undefined) { +async function outputReport(report: JSONReport, resolvedOutputFile: string | undefined) { const reportString = JSON.stringify(report, undefined, 2); - if (outputFile) { - assert(config.configFile || path.isAbsolute(outputFile), 'Expected fully resolved path if not using config file.'); - outputFile = config.configFile ? path.resolve(path.dirname(config.configFile), outputFile) : outputFile; - await fs.promises.mkdir(path.dirname(outputFile), { recursive: true }); - await fs.promises.writeFile(outputFile, reportString); + if (resolvedOutputFile) { + await fs.promises.mkdir(path.dirname(resolvedOutputFile), { recursive: true }); + await fs.promises.writeFile(resolvedOutputFile, reportString); } else { console.log(reportString); } @@ -250,12 +253,6 @@ function removePrivateFields(config: FullConfig): FullConfig { return Object.fromEntries(Object.entries(config).filter(([name, value]) => !name.startsWith('_'))) as FullConfig; } -function reportOutputNameFromEnv(): string | undefined { - if (process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]) - return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]); - return undefined; -} - export function serializePatterns(patterns: string | RegExp | (string | RegExp)[]): string[] { if (!Array.isArray(patterns)) patterns = [patterns]; diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts index 99e3fe6ba13d5..1f0cbf6362f66 100644 --- a/packages/playwright/src/reporters/junit.ts +++ b/packages/playwright/src/reporters/junit.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import type { FullConfig, FullResult, Suite, TestCase } from '../../types/testReporter'; -import { formatFailure, stripAnsiEscapes } from './base'; +import { formatFailure, resolveOutputFile, stripAnsiEscapes } from './base'; import EmptyReporter from './empty'; type JUnitOptions = { @@ -25,7 +25,7 @@ type JUnitOptions = { stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, - configDir?: string, + configDir: string, }; class JUnitReporter extends EmptyReporter { @@ -40,14 +40,12 @@ class JUnitReporter extends EmptyReporter { private stripANSIControlSequences = false; private includeProjectInTestName = false; - constructor(options: JUnitOptions = {}) { + constructor(options: JUnitOptions) { super(); this.stripANSIControlSequences = options.stripANSIControlSequences || false; this.includeProjectInTestName = options.includeProjectInTestName || false; - this.configDir = options.configDir || ''; - const outputFile = options.outputFile || reportOutputNameFromEnv(); - if (outputFile) - this.resolvedOutputFile = path.resolve(this.configDir, outputFile); + this.configDir = options.configDir; + this.resolvedOutputFile = resolveOutputFile('JUNIT', options)?.outputFile; } override printsToStdio() { @@ -261,10 +259,4 @@ function escape(text: string, stripANSIControlSequences: boolean, isCharacterDat return text; } -function reportOutputNameFromEnv(): string | undefined { - if (process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]) - return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]); - return undefined; -} - export default JUnitReporter; diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 6ed8fdf9e7341..0e5757bd01983 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -1292,12 +1292,18 @@ test('support PLAYWRIGHT_BLOB_OUTPUT_FILE environment variable', async ({ runInl test('math 1 @smoke', async ({}) => {}); `, }; + const defaultDir = test.info().outputPath('blob-report'); + fs.mkdirSync(defaultDir, { recursive: true }); + const file = path.join(defaultDir, 'some.file'); + fs.writeFileSync(file, 'content'); await runInlineTest(files, { shard: `1/2` }, { PLAYWRIGHT_BLOB_OUTPUT_FILE: 'subdir/report-one.zip' }); await runInlineTest(files, { shard: `2/2` }, { PLAYWRIGHT_BLOB_OUTPUT_FILE: test.info().outputPath('subdir/report-two.zip') }); const reportDir = test.info().outputPath('subdir'); const reportFiles = await fs.promises.readdir(reportDir); expect(reportFiles.sort()).toEqual(['report-one.zip', 'report-two.zip']); + + expect(fs.existsSync(file), 'Default directory should not be cleaned up if output file is specified.').toBe(true); }); test('keep projects with same name different bot name separate', async ({ runInlineTest, mergeReports, showReport, page }) => { diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts index 904fc927597de..313cd0e71e18f 100644 --- a/tests/playwright-test/reporter-json.spec.ts +++ b/tests/playwright-test/reporter-json.spec.ts @@ -288,4 +288,42 @@ test.describe('report location', () => { expect(result.passed).toBe(1); expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true); }); + + test('support PLAYWRIGHT_JSON_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'foo/package.json': `{ "name": "foo" }`, + // unused config along "search path" + 'foo/bar/playwright.config.js': ` + module.exports = { projects: [ {} ] }; + `, + 'foo/bar/baz/tests/a.spec.js': ` + import { test, expect } from '@playwright/test'; + const fs = require('fs'); + test('pass', ({}, testInfo) => { + }); + ` + }, { 'reporter': 'json' }, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PLAYWRIGHT_JSON_OUTPUT_FILE': '../my-report.json' }, { + cwd: 'foo/bar/baz/tests', + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true); + }); + + test('support PLAYWRIGHT_JSON_OUTPUT_DIR and PLAYWRIGHT_JSON_OUTPUT_NAME', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { projects: [ {} ] }; + `, + 'tests/a.spec.js': ` + import { test, expect } from '@playwright/test'; + const fs = require('fs'); + test('pass', ({}, testInfo) => { + }); + ` + }, { 'reporter': 'json' }, { 'PLAYWRIGHT_JSON_OUTPUT_DIR': 'foo/bar', 'PLAYWRIGHT_JSON_OUTPUT_NAME': 'baz/my-report.json' }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true); + }); }); diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts index 644db548b7f36..6db49cbba00ad 100644 --- a/tests/playwright-test/reporter-junit.spec.ts +++ b/tests/playwright-test/reporter-junit.spec.ts @@ -504,6 +504,44 @@ for (const useIntermediateMergeReport of [false, true] as const) { expect(result.passed).toBe(1); expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true); }); + + test('support PLAYWRIGHT_JUNIT_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'foo/package.json': `{ "name": "foo" }`, + // unused config along "search path" + 'foo/bar/playwright.config.js': ` + module.exports = { projects: [ {} ] }; + `, + 'foo/bar/baz/tests/a.spec.js': ` + import { test, expect } from '@playwright/test'; + const fs = require('fs'); + test('pass', ({}, testInfo) => { + }); + ` + }, { 'reporter': 'junit,line' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_FILE': '../my-report.xml' }, { + cwd: 'foo/bar/baz/tests', + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true); + }); + + test('support PLAYWRIGHT_JUNIT_OUTPUT_DIR and PLAYWRIGHT_JUNIT_OUTPUT_NAME', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { projects: [ {} ] }; + `, + 'tests/a.spec.js': ` + import { test, expect } from '@playwright/test'; + const fs = require('fs'); + test('pass', ({}, testInfo) => { + }); + ` + }, { 'reporter': 'junit,line' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_DIR': 'foo/bar', 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': 'baz/my-report.xml' }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true); + }); }); test('testsuites time is test run wall time', async ({ runInlineTest }) => {