fix(har): record request overrides to har (#17027)

packages/playwright-core/src/server/har/harRecorder.ts

@@ -45,6 +45,7 @@ export class HarRecorder {
       content,
       slimMode: options.mode === 'minimal',
       includeTraceInfo: false,
+      recordRequestOverrides: true,
       waitForContentOnStop: true,
       skipScripts: false,
       urlFilter: urlFilterRe ?? options.urlGlob,

packages/playwright-core/src/server/har/harTracer.ts

@@ -30,7 +30,7 @@ import { ManualPromise } from '../../utils/manualPromise';
 import { getPlaywrightVersion } from '../../common/userAgent';
 import { urlMatches } from '../../common/netUtils';
 import { Frame } from '../frames';
-import type { LifecycleEvent } from '../types';
+import type { HeadersArray, LifecycleEvent } from '../types';
 import { isTextualMimeType } from '../../utils/mimeType';
 
 const FALLBACK_HTTP_VERSION = 'HTTP/1.1';
@@ -45,6 +45,7 @@ type HarTracerOptions = {
   content: 'omit' | 'attach' | 'embed';
   skipScripts: boolean;
   includeTraceInfo: boolean;
+  recordRequestOverrides: boolean;
   waitForContentOnStop: boolean;
   urlFilter?: string | RegExp;
   slimMode?: boolean;
@@ -248,6 +249,7 @@ export class HarTracer {
     const harEntry = createHarEntry(request.method(), url, request.frame()?.guid, this._options);
     if (pageEntry)
       harEntry.pageref = pageEntry.id;
+    this._recordRequestHeadersAndCookies(harEntry, request.headers());
     harEntry.request.postData = this._postDataForRequest(request, this._options.content);
     if (!this._options.omitSizes)
       harEntry.request.bodySize = request.bodySize();
@@ -261,6 +263,24 @@ export class HarTracer {
     this._delegate.onEntryStarted(harEntry);
   }
 
+  private _recordRequestHeadersAndCookies(harEntry: har.Entry, headers: HeadersArray) {
+    if (!this._options.omitCookies) {
+      harEntry.request.cookies = [];
+      for (const header of headers.filter(header => header.name.toLowerCase() === 'cookie'))
+        harEntry.request.cookies.push(...header.value.split(';').map(parseCookie));
+    }
+    harEntry.request.headers = headers;
+  }
+
+  private _recordRequestOverrides(harEntry: har.Entry, request: network.Request) {
+    if (!request._hasOverrides() || !this._options.recordRequestOverrides)
+      return;
+    harEntry.request.method = request.method();
+    harEntry.request.url = request.url();
+    harEntry.request.postData = this._postDataForRequest(request, this._options.content);
+    this._recordRequestHeadersAndCookies(harEntry, request.headers());
+  }
+
   private async _onRequestFinished(request: network.Request, response: network.Response | null) {
     if (!response)
       return;
@@ -330,6 +350,7 @@ export class HarTracer {
 
     if (request._failureText !== null)
       harEntry.response._failureText = request._failureText;
+    this._recordRequestOverrides(harEntry, request);
     if (this._started)
       this._delegate.onEntryFinished(harEntry);
   }
@@ -423,12 +444,9 @@ export class HarTracer {
           harEntry._securityDetails = details;
       }));
     }
+    this._recordRequestOverrides(harEntry, request);
     this._addBarrier(page || request.serviceWorker(), request.rawRequestHeaders().then(headers => {
-      if (!this._options.omitCookies) {
-        for (const header of headers.filter(header => header.name.toLowerCase() === 'cookie'))
-          harEntry.request.cookies.push(...header.value.split(';').map(parseCookie));
-      }
-      harEntry.request.headers = headers;
+      this._recordRequestHeadersAndCookies(harEntry, headers);
     }));
     this._addBarrier(page || request.serviceWorker(), response.rawResponseHeaders().then(headers => {
       if (!this._options.omitCookies) {

packages/playwright-core/src/server/network.ts

@@ -22,8 +22,9 @@ import type * as channels from '../protocol/channels';
 import { assert } from '../utils';
 import { ManualPromise } from '../utils/manualPromise';
 import { SdkObject } from './instrumentation';
-import type { NameValue } from '../common/types';
+import type { HeadersArray, NameValue } from '../common/types';
 import { APIRequestContext } from './fetch';
+import type { NormalizedContinueOverrides } from './types';
 
 export function filterCookies(cookies: channels.NetworkCookie[], urls: string[]): channels.NetworkCookie[] {
   const parsedURLs = urls.map(s => new URL(s));
@@ -97,17 +98,18 @@ export class Request extends SdkObject {
   private _resourceType: string;
   private _method: string;
   private _postData: Buffer | null;
-  readonly _headers: types.HeadersArray;
+  readonly _headers: HeadersArray;
   private _headersMap = new Map<string, string>();
   readonly _frame: frames.Frame | null = null;
   readonly _serviceWorker: pages.Worker | null = null;
   readonly _context: contexts.BrowserContext;
-  private _rawRequestHeadersPromise = new ManualPromise<types.HeadersArray>();
+  private _rawRequestHeadersPromise = new ManualPromise<HeadersArray>();
   private _waitForResponsePromise = new ManualPromise<Response | null>();
   _responseEndTiming = -1;
+  private _overrides: NormalizedContinueOverrides | undefined;
 
   constructor(context: contexts.BrowserContext, frame: frames.Frame | null, serviceWorker: pages.Worker | null, redirectedFrom: Request | null, documentId: string | undefined,
-    url: string, resourceType: string, method: string, postData: Buffer | null, headers: types.HeadersArray) {
+    url: string, resourceType: string, method: string, postData: Buffer | null, headers: HeadersArray) {
     super(frame || context, 'request');
     assert(!url.startsWith('data:'), 'Data urls should not fire requests');
     this._context = context;
@@ -122,8 +124,7 @@ export class Request extends SdkObject {
     this._method = method;
     this._postData = postData;
     this._headers = headers;
-    for (const { name, value } of this._headers)
-      this._headersMap.set(name.toLowerCase(), value);
+    this._updateHeadersMap();
     this._isFavicon = url.endsWith('/favicon.ico') || !!redirectedFrom?._isFavicon;
   }
 
@@ -132,38 +133,52 @@ export class Request extends SdkObject {
     this._waitForResponsePromise.resolve(null);
   }
 
+  _setOverrides(overrides: types.NormalizedContinueOverrides) {
+    this._overrides = overrides;
+    this._updateHeadersMap();
+  }
+
+  private _updateHeadersMap() {
+    for (const { name, value } of this.headers())
+      this._headersMap.set(name.toLowerCase(), value);
+  }
+
+  _hasOverrides() {
+    return !!this._overrides;
+  }
+
   url(): string {
-    return this._url;
+    return this._overrides?.url || this._url;
   }
 
   resourceType(): string {
     return this._resourceType;
   }
 
   method(): string {
-    return this._method;
+    return this._overrides?.method || this._method;
   }
 
   postDataBuffer(): Buffer | null {
-    return this._postData;
+    return this._overrides?.postData || this._postData;
   }
 
-  headers(): types.HeadersArray {
-    return this._headers;
+  headers(): HeadersArray {
+    return this._overrides?.headers || this._headers;
   }
 
   headerValue(name: string): string | undefined {
     return this._headersMap.get(name);
   }
 
   // "null" means no raw headers available - we'll use provisional headers as raw headers.
-  setRawRequestHeaders(headers: types.HeadersArray | null) {
+  setRawRequestHeaders(headers: HeadersArray | null) {
     if (!this._rawRequestHeadersPromise.isDone())
       this._rawRequestHeadersPromise.resolve(headers || this._headers);
   }
 
-  async rawRequestHeaders(): Promise<NameValue[]> {
-    return this._rawRequestHeadersPromise;
+  async rawRequestHeaders(): Promise<HeadersArray> {
+    return this._overrides?.headers || this._rawRequestHeadersPromise;
   }
 
   response(): PromiseLike<Response | null> {
@@ -303,6 +318,7 @@ export class Route extends SdkObject {
       if (oldUrl.protocol !== newUrl.protocol)
         throw new Error('New URL must have same protocol as overridden URL');
     }
+    this._request._setOverrides(overrides);
     await this._delegate.continue(this._request, overrides);
     this._endHandling();
   }
@@ -360,20 +376,20 @@ export class Response extends SdkObject {
   private _status: number;
   private _statusText: string;
   private _url: string;
-  private _headers: types.HeadersArray;
+  private _headers: HeadersArray;
   private _headersMap = new Map<string, string>();
   private _getResponseBodyCallback: GetResponseBodyCallback;
   private _timing: ResourceTiming;
   private _serverAddrPromise = new ManualPromise<RemoteAddr | undefined>();
   private _securityDetailsPromise = new ManualPromise<SecurityDetails | undefined>();
-  private _rawResponseHeadersPromise = new ManualPromise<types.HeadersArray>();
+  private _rawResponseHeadersPromise = new ManualPromise<HeadersArray>();
   private _httpVersion: string | undefined;
   private _fromServiceWorker: boolean;
   private _encodedBodySizePromise = new ManualPromise<number | null>();
   private _transferSizePromise = new ManualPromise<number | null>();
   private _responseHeadersSizePromise = new ManualPromise<number | null>();
 
-  constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, fromServiceWorker: boolean, httpVersion?: string) {
+  constructor(request: Request, status: number, statusText: string, headers: HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback, fromServiceWorker: boolean, httpVersion?: string) {
     super(request.frame() || request._context, 'response');
     this._request = request;
     this._timing = timing;
@@ -418,7 +434,7 @@ export class Response extends SdkObject {
     return this._statusText;
   }
 
-  headers(): types.HeadersArray {
+  headers(): HeadersArray {
     return this._headers;
   }
 
@@ -431,7 +447,7 @@ export class Response extends SdkObject {
   }
 
   // "null" means no raw headers available - we'll use provisional headers as raw headers.
-  setRawResponseHeaders(headers: types.HeadersArray | null) {
+  setRawResponseHeaders(headers: HeadersArray | null) {
     if (!this._rawResponseHeadersPromise.isDone())
       this._rawResponseHeadersPromise.resolve(headers || this._headers);
   }
@@ -658,11 +674,11 @@ export const STATUS_TEXTS: { [status: string]: string } = {
   '511': 'Network Authentication Required',
 };
 
-export function singleHeader(name: string, value: string): types.HeadersArray {
+export function singleHeader(name: string, value: string): HeadersArray {
   return [{ name, value }];
 }
 
-export function mergeHeaders(headers: (types.HeadersArray | undefined | null)[]): types.HeadersArray {
+export function mergeHeaders(headers: (HeadersArray | undefined | null)[]): HeadersArray {
   const lowerCaseToValue = new Map<string, string>();
   const lowerCaseToOriginalCase = new Map<string, string>();
   for (const h of headers) {
@@ -674,7 +690,7 @@ export function mergeHeaders(headers: (types.HeadersArray | undefined | null)[])
       lowerCaseToValue.set(lower, value);
     }
   }
-  const result: types.HeadersArray = [];
+  const result: HeadersArray = [];
   for (const [lower, value] of lowerCaseToValue)
     result.push({ name: lowerCaseToOriginalCase.get(lower)!, value });
   return result;

packages/playwright-core/src/server/trace/recorder/tracing.ts

@@ -87,6 +87,7 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
     this._harTracer = new HarTracer(context, null, this, {
       content: 'attach',
       includeTraceInfo: true,
+      recordRequestOverrides: false,
       waitForContentOnStop: false,
       skipScripts: true,
     });

packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts

@@ -34,7 +34,7 @@ export class InMemorySnapshotter extends BaseSnapshotStorage implements Snapshot
   constructor(context: BrowserContext) {
     super();
     this._snapshotter = new Snapshotter(context, this);
-    this._harTracer = new HarTracer(context, null, this, { content: 'attach', includeTraceInfo: true, waitForContentOnStop: false, skipScripts: true });
+    this._harTracer = new HarTracer(context, null, this, { content: 'attach', includeTraceInfo: true, recordRequestOverrides: false, waitForContentOnStop: false, skipScripts: true });
   }
 
   async initialize(): Promise<void> {

packages/playwright-core/src/server/types.ts

@@ -15,8 +15,8 @@
  * limitations under the License.
  */
 
-import type { Size, Point, TimeoutOptions } from '../common/types';
-export type { Size, Point, Rect, Quad, URLMatch, TimeoutOptions } from '../common/types';
+import type { Size, Point, TimeoutOptions, HeadersArray } from '../common/types';
+export type { Size, Point, Rect, Quad, URLMatch, TimeoutOptions, HeadersArray } from '../common/types';
 import type * as channels from '../protocol/channels';
 
 export type StrictOptions = {
@@ -129,8 +129,6 @@ export type MouseMultiClickOptions = PointerActionOptions & {
 
 export type World = 'main' | 'utility';
 
-export type HeadersArray = { name: string, value: string }[];
-
 export type GotoOptions = NavigateOptions & {
   referer?: string,
 };

tests/library/har.spec.ts

@@ -256,6 +256,32 @@ it('should include secure set-cookies', async ({ contextFactory, httpsServer },
   expect(cookies[0]).toEqual({ name: 'name1', value: 'value1', secure: true });
 });
 
+it('should record request overrides', async ({ contextFactory, server }, testInfo) => {
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  page.route('**/foo', route => {
+    route.fallback({
+      url: server.EMPTY_PAGE,
+      method: 'POST',
+      headers: {
+        ...route.request().headers(),
+        'content-type': 'text/plain',
+        'cookie': 'foo=bar',
+        'custom': 'value'
+      },
+      postData: 'Hi!'
+    });
+  });
+
+  await page.goto(server.PREFIX + '/foo');
+  const log = await getLog();
+  const request = log.entries[0].request;
+  expect(request.url).toBe(server.EMPTY_PAGE);
+  expect(request.method).toBe('POST');
+  expect(request.headers).toContainEqual({ name: 'custom', value: 'value' });
+  expect(request.cookies).toContainEqual({ name: 'foo', value: 'bar' });
+  expect(request.postData).toEqual({ 'mimeType': 'text/plain', 'params': [], 'text': 'Hi!' });
+});
+
 it('should include content @smoke', async ({ contextFactory, server }, testInfo) => {
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   await page.goto(server.PREFIX + '/har.html');
@@ -409,7 +435,7 @@ it('should have -1 _transferSize when its a failed request', async ({ contextFac
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   server.setRoute('/one-style.css', (req, res) => {
     res.setHeader('Content-Type', 'text/css');
-    res.connection.destroy();
+    res.socket.destroy();
   });
   const failedRequests = [];
   page.on('requestfailed', request => failedRequests.push(request));
@@ -419,6 +445,49 @@ it('should have -1 _transferSize when its a failed request', async ({ contextFac
   expect(log.entries[1].response._transferSize).toBe(-1);
 });
 
+it('should record failed request headers', async ({ contextFactory, server }, testInfo) => {
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  server.setRoute('/har.html', (req, res) => {
+    res.socket.destroy();
+  });
+  await page.goto(server.PREFIX + '/har.html').catch(() => {});
+  const log = await getLog();
+  expect(log.entries[0].response._failureText).toBeTruthy();
+  const request = log.entries[0].request;
+  expect(request.url.endsWith('/har.html')).toBe(true);
+  expect(request.method).toBe('GET');
+  expect(request.headers).toContainEqual(expect.objectContaining({ name: 'User-Agent' }));
+});
+
+it('should record failed request overrides', async ({ contextFactory, server }, testInfo) => {
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  server.setRoute('/empty.html', (req, res) => {
+    res.socket.destroy();
+  });
+  await page.route('**/foo', route => {
+    route.fallback({
+      url: server.EMPTY_PAGE,
+      method: 'POST',
+      headers: {
+        ...route.request().headers(),
+        'content-type': 'text/plain',
+        'cookie': 'foo=bar',
+        'custom': 'value'
+      },
+      postData: 'Hi!'
+    });
+  });
+  await page.goto(server.PREFIX + '/foo').catch(() => {});
+  const log = await getLog();
+  expect(log.entries[0].response._failureText).toBeTruthy();
+  const request = log.entries[0].request;
+  expect(request.url).toBe(server.EMPTY_PAGE);
+  expect(request.method).toBe('POST');
+  expect(request.headers).toContainEqual({ name: 'custom', value: 'value' });
+  expect(request.cookies).toContainEqual({ name: 'foo', value: 'bar' });
+  expect(request.postData).toEqual({ 'mimeType': 'text/plain', 'params': [], 'text': 'Hi!' });
+});
+
 it('should report the correct request body size', async ({ contextFactory, server }, testInfo) => {
   server.setRoute('/api', (req, res) => res.end());
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
@@ -556,7 +625,7 @@ it('should have connection details for redirects', async ({ contextFactory, serv
 it('should have connection details for failed requests', async ({ contextFactory, server, browserName, platform, mode }, testInfo) => {
   server.setRoute('/one-style.css', (_, res) => {
     res.setHeader('Content-Type', 'text/css');
-    res.connection.destroy();
+    res.socket.destroy();
   });
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   await page.goto(server.PREFIX + '/one-style.html');