chore: allow updating har while routing (#15197)

docs/src/api/class-browsercontext.md

@@ -1044,8 +1044,13 @@ Path to a [HAR](http://www.softwareishard.com/blog/har-12-spec) file with prerec
 
 Defaults to abort.
 
+### option: BrowserContext.routeFromHAR.update
+- `update` ?<boolean>
+
+If specified, updates the given HAR with the actual network information instead of serving from file.
+
 ### option: BrowserContext.routeFromHAR.url
-- `url` <[string]|[RegExp]|[function]\([URL]\):[boolean]>
+- `url` <[string]|[RegExp]>
 
 A glob pattern, regular expression or predicate to match the request URL. Only requests with URL matching the pattern will be served from the HAR file. If not specified, all requests are served from the HAR file.
 

docs/src/api/class-page.md

@@ -2751,8 +2751,13 @@ Path to a [HAR](http://www.softwareishard.com/blog/har-12-spec) file with prerec
 
 Defaults to abort.
 
+### option: Page.routeFromHAR.update
+- `update` ?<boolean>
+
+If specified, updates the given HAR with the actual network information instead of serving from file.
+
 ### option: Page.routeFromHAR.url
-- `url` <[string]|[RegExp]|[function]\([URL]\):[boolean]>
+- `url` <[string]|[RegExp]>
 
 A glob pattern, regular expression or predicate to match the request URL. Only requests with URL matching the pattern will be served from the HAR file. If not specified, all requests are served from the HAR file.
 

docs/src/api/params.md

@@ -561,8 +561,8 @@ Logger sink for Playwright logging.
 - `recordHar` <[Object]>
   - `omitContent` ?<[boolean]> Optional setting to control whether to omit request content from the HAR. Defaults to
     `false`. Deprecated, use `content` policy instead.
-  - `content` ?<[HarContentPolicy]<"omit"|"embed"|"attach">> Optional setting to control resource content management. If `omit` is specified, content is not persisted. If `attach` is specified, resources are persistet as separate files and all of these files are archived along with the HAR file. Defaults to `embed`, which stores content inline the HAR file as per HAR specification.
-  - `path` <[path]> Path on the filesystem to write the HAR file to. If the file name ends with `.zip`, `attach` mode is used by default.
+  - `content` ?<[HarContentPolicy]<"omit"|"embed"|"attach">> Optional setting to control resource content management. If `omit` is specified, content is not persisted. If `attach` is specified, resources are persistet as separate files or entries in the ZIP archive. If `embed` is specified, content is stored inline the HAR file as per HAR specification. Defaults to `attach` for `.zip` output files and to `embed` for all other file extensions.
+  - `path` <[path]> Path on the filesystem to write the HAR file to. If the file name ends with `.zip`, `content: 'attach'` is used by default.
   - `mode` ?<[HarMode]<"full"|"minimal">> When set to `minimal`, only record information necessary for routing from HAR. This omits sizes, timing, page, cookies, security and other types of HAR information that are not used when replaying from HAR. Defaults to `full`.
   - `urlFilter` ?<[string]|[RegExp]> A glob or regex pattern to filter requests that are stored in the HAR. When a [`option: baseURL`] via the context options was provided and the passed URL is a path, it gets merged via the [`new URL()`](https://developer.mozilla.org/en-US/docs/Web/API/URL/URL) constructor.
 

packages/playwright-core/src/client/browserContext.ts

@@ -58,6 +58,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
   readonly _backgroundPages = new Set<Page>();
   readonly _serviceWorkers = new Set<Worker>();
   readonly _isChromium: boolean;
+  private _harRecorders = new Map<string, { path: string, content: 'embed' | 'attach' | 'omit' | undefined }>();
 
   static from(context: channels.BrowserContextChannel): BrowserContext {
     return (context as any)._object;
@@ -100,6 +101,8 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
   _setBrowserType(browserType: BrowserType) {
     this._browserType = browserType;
     browserType._contexts.add(this);
+    if (this._options.recordHar)
+      this._harRecorders.set('', { path: this._options.recordHar.path, content: this._options.recordHar.content });
   }
 
   private _onPage(page: Page): void {
@@ -270,7 +273,24 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
       await this._channel.setNetworkInterceptionEnabled({ enabled: true });
   }
 
-  async routeFromHAR(har: string, options: { url?: URLMatch, notFound?: 'abort' | 'fallback' } = {}): Promise<void> {
+  async _recordIntoHAR(har: string, page: Page | null, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback', update?: boolean } = {}): Promise<void> {
+    const { harId } = await this._channel.harStart({
+      page: page?._channel,
+      options: prepareRecordHarOptions({
+        path: har,
+        content: 'attach',
+        mode: 'minimal',
+        urlFilter: options.url
+      })!
+    });
+    this._harRecorders.set(harId, { path: har, content: 'attach' });
+  }
+
+  async routeFromHAR(har: string, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback', update?: boolean } = {}): Promise<void> {
+    if (options.update) {
+      await this._recordIntoHAR(har, null, options);
+      return;
+    }
     const harRouter = await HarRouter.create(this._connection.localUtils(), har, options.notFound || 'abort', { urlMatch: options.url });
     harRouter.addContextRoute(this);
   }
@@ -340,10 +360,18 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
     try {
       await this._wrapApiCall(async () => {
         await this._browserType?._onWillCloseContext?.(this);
-        if (this._options.recordHar)  {
-          const har = await this._channel.harExport();
+        for (const [harId, harParams] of this._harRecorders) {
+          const har = await this._channel.harExport({ harId });
           const artifact = Artifact.from(har.artifact);
-          await artifact.saveAs(this._options.recordHar.path);
+          // Server side will compress artifact if content is attach or if file is .zip.
+          const isCompressed = harParams.content === 'attach' || harParams.path.endsWith('.zip');
+          const needCompressed = harParams.path.endsWith('.zip');
+          if (isCompressed && !needCompressed) {
+            await artifact.saveAs(harParams.path + '.tmp');
+            await this._connection.localUtils()._channel.harUnzip({ zipFile: harParams.path + '.tmp', harFile: harParams.path });
+          } else {
+            await artifact.saveAs(harParams.path);
+          }
           await artifact.delete();
         }
       }, true);

packages/playwright-core/src/client/page.ts

@@ -468,7 +468,11 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
       await this._channel.setNetworkInterceptionEnabled({ enabled: true });
   }
 
-  async routeFromHAR(har: string, options: { url?: URLMatch, notFound?: 'abort' | 'fallback' } = {}): Promise<void> {
+  async routeFromHAR(har: string, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback', update?: boolean } = {}): Promise<void> {
+    if (options.update) {
+      await this._browserContext._recordIntoHAR(har, this, options);
+      return;
+    }
     const harRouter = await HarRouter.create(this._connection.localUtils(), har, options.notFound || 'abort', { urlMatch: options.url });
     harRouter.addPageRoute(this);
   }

packages/playwright-core/src/protocol/channels.ts

@@ -382,6 +382,7 @@ export interface LocalUtilsChannel extends LocalUtilsEventTarget, Channel {
   harOpen(params: LocalUtilsHarOpenParams, metadata?: Metadata): Promise<LocalUtilsHarOpenResult>;
   harLookup(params: LocalUtilsHarLookupParams, metadata?: Metadata): Promise<LocalUtilsHarLookupResult>;
   harClose(params: LocalUtilsHarCloseParams, metadata?: Metadata): Promise<LocalUtilsHarCloseResult>;
+  harUnzip(params: LocalUtilsHarUnzipParams, metadata?: Metadata): Promise<LocalUtilsHarUnzipResult>;
 }
 export type LocalUtilsZipParams = {
   zipFile: string,
@@ -427,6 +428,14 @@ export type LocalUtilsHarCloseOptions = {
 
 };
 export type LocalUtilsHarCloseResult = void;
+export type LocalUtilsHarUnzipParams = {
+  zipFile: string,
+  harFile: string,
+};
+export type LocalUtilsHarUnzipOptions = {
+
+};
+export type LocalUtilsHarUnzipResult = void;
 
 export interface LocalUtilsEvents {
 }
@@ -1119,7 +1128,8 @@ export interface BrowserContextChannel extends BrowserContextEventTarget, EventT
   pause(params?: BrowserContextPauseParams, metadata?: Metadata): Promise<BrowserContextPauseResult>;
   recorderSupplementEnable(params: BrowserContextRecorderSupplementEnableParams, metadata?: Metadata): Promise<BrowserContextRecorderSupplementEnableResult>;
   newCDPSession(params: BrowserContextNewCDPSessionParams, metadata?: Metadata): Promise<BrowserContextNewCDPSessionResult>;
-  harExport(params?: BrowserContextHarExportParams, metadata?: Metadata): Promise<BrowserContextHarExportResult>;
+  harStart(params: BrowserContextHarStartParams, metadata?: Metadata): Promise<BrowserContextHarStartResult>;
+  harExport(params: BrowserContextHarExportParams, metadata?: Metadata): Promise<BrowserContextHarExportResult>;
   createTempFile(params: BrowserContextCreateTempFileParams, metadata?: Metadata): Promise<BrowserContextCreateTempFileResult>;
 }
 export type BrowserContextBindingCallEvent = {
@@ -1325,8 +1335,22 @@ export type BrowserContextNewCDPSessionOptions = {
 export type BrowserContextNewCDPSessionResult = {
   session: CDPSessionChannel,
 };
-export type BrowserContextHarExportParams = {};
-export type BrowserContextHarExportOptions = {};
+export type BrowserContextHarStartParams = {
+  page?: PageChannel,
+  options: RecordHarOptions,
+};
+export type BrowserContextHarStartOptions = {
+  page?: PageChannel,
+};
+export type BrowserContextHarStartResult = {
+  harId: string,
+};
+export type BrowserContextHarExportParams = {
+  harId?: string,
+};
+export type BrowserContextHarExportOptions = {
+  harId?: string,
+};
 export type BrowserContextHarExportResult = {
   artifact: ArtifactChannel,
 };

packages/playwright-core/src/protocol/protocol.yml

@@ -520,6 +520,11 @@ LocalUtils:
       parameters:
         harId: string
 
+    harUnzip:
+      parameters:
+        zipFile: string
+        harFile: string
+
 Root:
   type: interface
 
@@ -926,7 +931,16 @@ BrowserContext:
       returns:
         session: CDPSession
 
+    harStart:
+      parameters:
+        page: Page?
+        options: RecordHarOptions
+      returns:
+        harId: string
+
     harExport:
+      parameters:
+        harId: string?
       returns:
         artifact: Artifact
 

packages/playwright-core/src/protocol/validator.ts

@@ -220,6 +220,10 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
   scheme.LocalUtilsHarCloseParams = tObject({
     harId: tString,
   });
+  scheme.LocalUtilsHarUnzipParams = tObject({
+    zipFile: tString,
+    harFile: tString,
+  });
   scheme.RootInitializeParams = tObject({
     sdkLanguage: tString,
   });
@@ -527,7 +531,13 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     page: tOptional(tChannel('Page')),
     frame: tOptional(tChannel('Frame')),
   });
-  scheme.BrowserContextHarExportParams = tOptional(tObject({}));
+  scheme.BrowserContextHarStartParams = tObject({
+    page: tOptional(tChannel('Page')),
+    options: tType('RecordHarOptions'),
+  });
+  scheme.BrowserContextHarExportParams = tObject({
+    harId: tOptional(tString),
+  });
   scheme.BrowserContextCreateTempFileParams = tObject({
     name: tString,
   });

packages/playwright-core/src/server/browserContext.ts

@@ -17,7 +17,7 @@
 
 import * as os from 'os';
 import { TimeoutSettings } from '../common/timeoutSettings';
-import { debugMode } from '../utils';
+import { createGuid, debugMode } from '../utils';
 import { mkdirIfNeeded } from '../utils/fileUtils';
 import type { Browser, BrowserOptions } from './browser';
 import type { Download } from './download';
@@ -40,6 +40,7 @@ import { HarRecorder } from './har/harRecorder';
 import { Recorder } from './recorder';
 import * as consoleApiSource from '../generated/consoleApiSource';
 import { BrowserContextAPIRequestContext } from './fetch';
+import type { Artifact } from './artifact';
 
 export abstract class BrowserContext extends SdkObject {
   static Events = {
@@ -67,7 +68,7 @@ export abstract class BrowserContext extends SdkObject {
   readonly _browserContextId: string | undefined;
   private _selectors?: Selectors;
   private _origins = new Set<string>();
-  readonly _harRecorder: HarRecorder | undefined;
+  readonly _harRecorders = new Map<string, HarRecorder>();
   readonly tracing: Tracing;
   readonly fetchRequest: BrowserContextAPIRequestContext;
   private _customCloseHandler?: () => Promise<any>;
@@ -87,7 +88,7 @@ export abstract class BrowserContext extends SdkObject {
     this.fetchRequest = new BrowserContextAPIRequestContext(this);
 
     if (this._options.recordHar)
-      this._harRecorder = new HarRecorder(this, this._options.recordHar);
+      this._harRecorders.set('', new HarRecorder(this, null, this._options.recordHar));
 
     this.tracing = new Tracing(this, browser.options.tracesDir);
   }
@@ -316,7 +317,8 @@ export abstract class BrowserContext extends SdkObject {
       this.emit(BrowserContext.Events.BeforeClose);
       this._closedStatus = 'closing';
 
-      await this._harRecorder?.flush();
+      for (const harRecorder of this._harRecorders.values())
+        await harRecorder.flush();
       await this.tracing.flush();
 
       // Cleanup.
@@ -442,6 +444,17 @@ export abstract class BrowserContext extends SdkObject {
     this.on(BrowserContext.Events.Page, installInPage);
     return Promise.all(this.pages().map(installInPage));
   }
+
+  async _harStart(page: Page | null, options: channels.RecordHarOptions): Promise<string> {
+    const harId = createGuid();
+    this._harRecorders.set(harId, new HarRecorder(this, page, options));
+    return harId;
+  }
+
+  async _harExport(harId: string | undefined): Promise<Artifact> {
+    const recorder = this._harRecorders.get(harId || '')!;
+    return recorder.export();
+  }
 }
 
 export function assertBrowserContextIsNotOwned(context: BrowserContext) {

packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts

@@ -213,8 +213,13 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
     return { session: new CDPSessionDispatcher(this._scope, await crBrowserContext.newCDPSession((params.page ? params.page as PageDispatcher : params.frame as FrameDispatcher)._object)) };
   }
 
+  async harStart(params: channels.BrowserContextHarStartParams): Promise<channels.BrowserContextHarStartResult> {
+    const harId = await this._context._harStart(params.page ? (params.page as PageDispatcher)._object : null, params.options);
+    return { harId };
+  }
+
   async harExport(params: channels.BrowserContextHarExportParams): Promise<channels.BrowserContextHarExportResult> {
-    const artifact = await this._context._harRecorder?.export();
+    const artifact = await this._context._harExport(params.harId);
     if (!artifact)
       throw new Error('No HAR artifact. Ensure record.harPath is set.');
     return { artifact: new ArtifactDispatcher(this._scope, artifact) };

packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts

@@ -124,6 +124,20 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.
       harBackend.dispose();
     }
   }
+
+  async harUnzip(params: channels.LocalUtilsHarUnzipParams, metadata?: channels.Metadata): Promise<void> {
+    const dir = path.dirname(params.zipFile);
+    const zipFile = new ZipFile(params.zipFile);
+    for (const entry of await zipFile.entries()) {
+      const buffer = await zipFile.read(entry);
+      if (entry === 'har.har')
+        await fs.promises.writeFile(params.harFile, buffer);
+      else
+        await fs.promises.writeFile(path.join(dir, entry), buffer);
+    }
+    zipFile.close();
+    await fs.promises.unlink(params.zipFile);
+  }
 }
 
 const redirectStatus = [301, 302, 303, 307, 308];

packages/playwright-core/src/server/har/harRecorder.ts

@@ -26,6 +26,7 @@ import type { ZipFile } from '../../zipBundle';
 import { ManualPromise } from '../../utils/manualPromise';
 import type EventEmitter from 'events';
 import { createGuid } from '../../utils';
+import type { Page } from '../page';
 
 export class HarRecorder {
   private _artifact: Artifact;
@@ -35,12 +36,12 @@ export class HarRecorder {
   private _zipFile: ZipFile | null = null;
   private _writtenZipEntries = new Set<string>();
 
-  constructor(context: BrowserContext, options: channels.RecordHarOptions) {
+  constructor(context: BrowserContext, page: Page | null, options: channels.RecordHarOptions) {
     this._artifact = new Artifact(context, path.join(context._browser.options.artifactsDir, `${createGuid()}.har`));
     const urlFilterRe = options.urlRegexSource !== undefined && options.urlRegexFlags !== undefined ? new RegExp(options.urlRegexSource, options.urlRegexFlags) : undefined;
     const expectsZip = options.path.endsWith('.zip');
     const content = options.content || (expectsZip ? 'attach' : 'embed');
-    this._tracer = new HarTracer(context, this, {
+    this._tracer = new HarTracer(context, page, this, {
       content,
       slimMode: options.mode === 'minimal',
       includeTraceInfo: false,