fix(mitm): http2 session frame emulator data

hero/core/lib/CorePlugins.ts

@@ -24,13 +24,16 @@ import ICorePluginCreateOptions from '@secret-agent/interfaces/ICorePluginCreate
 import IBrowserEngine from '@secret-agent/interfaces/IBrowserEngine';
 import { PluginTypes } from '@secret-agent/interfaces/IPluginTypes';
 import requirePlugins from '@secret-agent/plugin-utils/lib/utils/requirePlugins';
+import IHttp2ConnectSettings from '@secret-agent/interfaces/IHttp2ConnectSettings';
+import IDeviceProfile from '@secret-agent/interfaces/IDeviceProfile';
 import Core from '../index';
 
 const DefaultBrowserEmulatorId = 'default-browser-emulator';
 const DefaultHumanEmulatorId = 'default-human-emulator';
 
 interface IOptionsCreate {
   userAgentSelector?: string;
+  deviceProfile?: IDeviceProfile;
   humanEmulatorId?: string;
   browserEmulatorId?: string;
   selectBrowserMeta?: ISelectBrowserMeta;
@@ -87,7 +90,13 @@ export default class CorePlugins implements ICorePlugins {
 
     const { browserEngine, userAgentOption } =
       options.selectBrowserMeta || BrowserEmulator.selectBrowserMeta(userAgentSelector);
-    this.createOptions = { browserEngine, userAgentOption, logger, corePlugins: this };
+    this.createOptions = {
+      browserEngine,
+      userAgentOption,
+      logger,
+      corePlugins: this,
+      deviceProfile: options.deviceProfile,
+    };
     this.browserEngine = browserEngine;
     this.logger = logger;
 
@@ -139,6 +148,17 @@ export default class CorePlugins implements ICorePlugins {
     );
   }
 
+  public async onHttp2SessionConnect(
+    resource: IHttpResourceLoadDetails,
+    settings: IHttp2ConnectSettings,
+  ): Promise<void> {
+    await Promise.all(
+      this.instances
+        .filter(p => p.onHttp2SessionConnect)
+        .map(p => p.onHttp2SessionConnect(resource, settings)),
+    );
+  }
+
   public async beforeHttpRequest(resource: IHttpResourceLoadDetails): Promise<void> {
     await Promise.all(
       this.instances.filter(p => p.beforeHttpRequest).map(p => p.beforeHttpRequest(resource)),

hero/core/lib/Session.ts

@@ -92,22 +92,30 @@ export default class Session extends TypedEventEmitter<{
     this.awaitedEventListener = new AwaitedEventListener(this);
 
     const {
-      userAgent: userAgentSelector,
       browserEmulatorId,
       humanEmulatorId,
       dependencyMap,
       corePluginPaths,
+      userProfile,
+      userAgent,
     } = options;
+
+    const userAgentSelector = userAgent ?? userProfile?.userAgentString;
     this.plugins = new CorePlugins(
-      { userAgentSelector, browserEmulatorId, humanEmulatorId, dependencyMap, corePluginPaths },
+      {
+        userAgentSelector,
+        browserEmulatorId,
+        humanEmulatorId,
+        dependencyMap,
+        corePluginPaths,
+        deviceProfile: userProfile?.deviceProfile,
+      },
       this.logger,
     );
 
     this.browserEngine = this.plugins.browserEngine;
 
-    if (options.userProfile) {
-      this.userProfile = options.userProfile;
-    }
+    this.userProfile = options.userProfile;
     this.upstreamProxyUrl = options.upstreamProxyUrl;
     this.geolocation = options.geolocation;
 

hero/core/lib/UserProfile.ts

@@ -30,6 +30,8 @@ export default class UserProfile {
     return {
       cookies,
       storage,
+      userAgentString: session.plugins.browserEmulator.userAgentString,
+      deviceProfile: session.plugins.browserEmulator.deviceProfile,
     } as IUserProfile;
   }
 

hero/core/test/user-profile.test.ts

@@ -55,13 +55,18 @@ describe('UserProfile cookie tests', () => {
     await tab2.waitForLoad('PaintingStable');
     expect(cookie).not.toBeTruthy();
 
+    expect(profile.userAgentString).toBeTruthy();
+
     const meta3 = await connection.createSession({
       userProfile: profile,
     });
     const tab3 = Session.getTab(meta3);
     Helpers.needsClosing.push(tab3.session);
     const cookiesBefore = await connection.exportUserProfile(meta3);
     expect(cookiesBefore.cookies).toHaveLength(1);
+    expect(cookiesBefore.userAgentString).toBe(profile.userAgentString);
+    expect(tab3.session.plugins.browserEmulator.userAgentString).toBe(profile.userAgentString);
+    expect(cookiesBefore.deviceProfile).toEqual(profile.deviceProfile);
 
     await tab3.goto(`${koaServer.baseUrl}/cookie2`);
     await tab3.waitForLoad('PaintingStable');

hero/full-client/test/emulate.test.ts

@@ -206,6 +206,44 @@ describe('geolocation', () => {
 describe('user agent and platform', () => {
   const propsToGet = `appVersion, platform, userAgent, deviceMemory`.split(',').map(x => x.trim());
 
+  it('should be able to configure a userAgent', async () => {
+    const userAgent =
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4472.124 Safari/537.36';
+    const agent = await handler.createAgent({
+      userAgent,
+    });
+    Helpers.needsClosing.push(agent);
+
+    const agentMeta = await agent.meta;
+    expect(agentMeta.userAgentString).toBe(userAgent);
+  });
+
+  it('should be able to configure a userAgent with a range', async () => {
+    const agent = await handler.createAgent({
+      userAgent: '~ chrome >= 88 && chrome < 89',
+    });
+    Helpers.needsClosing.push(agent);
+
+    const agentMeta = await agent.meta;
+    const chromeMatch = agentMeta.userAgentString.match(/Chrome\/(\d+)/);
+    expect(chromeMatch).toBeTruthy();
+    const version = Number(chromeMatch[1]);
+    expect(version).toBe(88);
+  });
+
+  it('should be able to configure a userAgent with a wildcard', async () => {
+    const agent = await handler.createAgent({
+      userAgent: '~ chrome = 88.x',
+    });
+    Helpers.needsClosing.push(agent);
+
+    const agentMeta = await agent.meta;
+    const chromeMatch = agentMeta.userAgentString.match(/Chrome\/(\d+)/);
+    expect(chromeMatch).toBeTruthy();
+    const version = Number(chromeMatch[1]);
+    expect(version).toBe(88);
+  });
+
   it('should add user agent and platform to window & frames', async () => {
     const agent = await handler.createAgent();
     Helpers.needsClosing.push(agent);

hero/interfaces/ICorePlugin.ts

@@ -14,8 +14,13 @@ import { IPuppetPage } from './IPuppetPage';
 import { IPuppetWorker } from './IPuppetWorker';
 import IViewport from './IViewport';
 import IGeolocation from './IGeolocation';
+import IDeviceProfile from './IDeviceProfile';
+import IHttp2ConnectSettings from './IHttp2ConnectSettings';
 
-export default interface ICorePlugin extends ICorePluginMethods, IBrowserEmulatorMethods, IHumanEmulatorMethods {
+export default interface ICorePlugin
+  extends ICorePluginMethods,
+    IBrowserEmulatorMethods,
+    IHumanEmulatorMethods {
   id: string;
 }
 
@@ -94,6 +99,7 @@ export interface IBrowserEmulator extends ICorePlugin {
   operatingSystemVersion: IVersion;
 
   userAgentString: string;
+  deviceProfile: IDeviceProfile;
 }
 
 export interface IBrowserEmulatorMethods {
@@ -103,6 +109,10 @@ export interface IBrowserEmulatorMethods {
   onTcpConfiguration?(settings: ITcpSettings): Promise<any> | void;
   onTlsConfiguration?(settings: ITlsSettings): Promise<any> | void;
 
+  onHttp2SessionConnect?(
+    request: IHttpResourceLoadDetails,
+    settings: IHttp2ConnectSettings,
+  ): Promise<any> | void;
   beforeHttpRequest?(request: IHttpResourceLoadDetails): Promise<any> | void;
   beforeHttpResponse?(resource: IHttpResourceLoadDetails): Promise<any> | void;
 

hero/interfaces/ICorePluginCreateOptions.ts

@@ -2,10 +2,12 @@ import { IBoundLog } from './ILog';
 import IBrowserEngine from './IBrowserEngine';
 import ICorePlugins from './ICorePlugins';
 import IUserAgentOption from './IUserAgentOption';
+import IDeviceProfile from './IDeviceProfile';
 
 export default interface ICorePluginCreateOptions {
   userAgentOption: IUserAgentOption;
   browserEngine: IBrowserEngine;
   corePlugins: ICorePlugins;
   logger: IBoundLog;
+  deviceProfile?: IDeviceProfile;
 }

hero/interfaces/IDeviceProfile.ts

@@ -0,0 +1,8 @@
+export default interface IDeviceProfile {
+  deviceMemory?: number;
+  videoDevice?: {
+    deviceId: string;
+    groupId: string;
+  };
+  webGlParameters?: Record<string, string | number | boolean>;
+}

hero/interfaces/IHttp2ConnectSettings.ts

@@ -0,0 +1,6 @@
+import * as http2 from 'http2';
+
+export default interface IHttp2ConnectSettings {
+  settings: http2.Settings;
+  localWindowSize: number;
+}

hero/interfaces/IUserProfile.ts

@@ -1,7 +1,10 @@
 import { ICookie } from './ICookie';
 import IDomStorage from './IDomStorage';
+import IDeviceProfile from './IDeviceProfile';
 
 export default interface IUserProfile {
   cookies?: ICookie[];
   storage?: IDomStorage;
+  userAgentString?: string;
+  deviceProfile?: IDeviceProfile;
 }

hero/mitm-socket/go/emulate_tls.go

@@ -44,7 +44,7 @@ func EmulateTls(dialConn net.Conn, addr string, sessionArgs SessionArgs, connect
 
 	if connectArgs.KeylogPath != "" {
 		var keylog io.Writer
-		keylog, err = os.OpenFile(connectArgs.KeylogPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+		keylog, err = os.OpenFile(connectArgs.KeylogPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
 		if err != nil {
 			return nil, err
 		}

hero/mitm/handlers/HeadersHandler.ts

@@ -1,5 +1,3 @@
-// @ts-ignore
-import * as nodeCommon from '_http_common';
 import IResourceHeaders from '@secret-agent/interfaces/IResourceHeaders';
 import * as http from 'http';
 import * as http2 from 'http2';
@@ -81,8 +79,6 @@ export default class HeadersHandler {
   ): IResourceHeaders {
     const headers: IResourceHeaders = {};
     for (const [headerName, value] of Object.entries(originalRawHeaders)) {
-      if (nodeCommon._checkInvalidHeaderChar(value)) continue;
-
       const canonizedKey = headerName.trim();
 
       const lowerHeaderName = toLowerCase(canonizedKey);
@@ -137,6 +133,21 @@ export default class HeadersHandler {
     }
   }
 
+  public static prepareHttp2RequestHeadersForSave(
+    headers: IMitmRequestContext['requestHeaders'],
+  ): IMitmRequestContext['requestHeaders'] {
+    const order: string[] = [];
+    for (const key of Object.keys(headers)) {
+      if (key.startsWith(':')) order.unshift(key);
+      else order.push(key);
+    }
+    const newHeaders = {};
+    for (const key of order) {
+      newHeaders[key] = headers[key];
+    }
+    return newHeaders;
+  }
+
   public static prepareRequestHeadersForHttp2(ctx: IMitmRequestContext): void {
     const url = ctx.url;
     const oldHeaders = ctx.requestHeaders;

hero/mitm/lib/MitmRequestAgent.ts

@@ -8,6 +8,8 @@ import MitmSocketSession from '@secret-agent/mitm-socket/lib/MitmSocketSession';
 import IResourceHeaders from '@secret-agent/interfaces/IResourceHeaders';
 import ITcpSettings from '@secret-agent/interfaces/ITcpSettings';
 import ITlsSettings from '@secret-agent/interfaces/ITlsSettings';
+import Resolvable from '@secret-agent/commons/Resolvable';
+import IHttp2ConnectSettings from '@secret-agent/interfaces/IHttp2ConnectSettings';
 import IMitmRequestContext from '../interfaces/IMitmRequestContext';
 import MitmRequestContext from './MitmRequestContext';
 import RequestSession from '../handlers/RequestSession';
@@ -283,15 +285,15 @@ export default class MitmRequestAgent {
 
   /////// ////////// Http2 helpers //////////////////////////////////////////////////////////////////
 
-  private http2Request(ctx: IMitmRequestContext): http2.ClientHttp2Stream {
-    const client = this.createHttp2Session(ctx);
+  private async http2Request(ctx: IMitmRequestContext): Promise<http2.ClientHttp2Stream> {
+    const client = await this.createHttp2Session(ctx);
     ctx.setState(ResourceState.CreateProxyToServerRequest);
     const weight = (ctx.clientToProxyRequest as Http2ServerRequest).stream?.state?.weight;
 
-    return client.request(ctx.requestHeaders, { waitForTrailers: true, weight });
+    return client.request(ctx.requestHeaders, { waitForTrailers: true, weight, exclusive: true });
   }
 
-  private createHttp2Session(ctx: IMitmRequestContext): ClientHttp2Session {
+  private async createHttp2Session(ctx: IMitmRequestContext): Promise<ClientHttp2Session> {
     const origin = ctx.url.origin;
     let originSocketPool: SocketPool;
     let resolvedHost: string;
@@ -312,11 +314,32 @@ export default class MitmRequestAgent {
     const session = (ctx.clientToProxyRequest as Http2ServerRequest).stream?.session;
 
     ctx.setState(ResourceState.CreateH2Session);
-    const clientSettings = session?.remoteSettings;
-    const proxyToServerH2Client = http2.connect(origin, {
-      settings: clientSettings,
-      createConnection: () => ctx.proxyToServerMitmSocket.socket,
-    });
+
+    const settings: IHttp2ConnectSettings = {
+      settings: session?.remoteSettings,
+      localWindowSize: session?.state.localWindowSize,
+    };
+    if (ctx.requestSession.plugins.onHttp2SessionConnect) {
+      await ctx.requestSession.plugins.onHttp2SessionConnect(ctx, settings);
+    }
+
+    const connectPromise = new Resolvable<void>();
+    const proxyToServerH2Client = http2.connect(
+      origin,
+      {
+        settings: settings.settings,
+        createConnection: () => ctx.proxyToServerMitmSocket.socket,
+      },
+      async remoteSession => {
+        if ('setLocalWindowSize' in remoteSession && settings.localWindowSize) {
+          // @ts-ignore
+          remoteSession.setLocalWindowSize(settings.localWindowSize);
+          await new Promise(setImmediate);
+        }
+        connectPromise.resolve();
+      },
+    );
+
     if (session) {
       session.on('ping', bytes => {
         proxyToServerH2Client.ping(bytes, () => null);
@@ -352,11 +375,11 @@ export default class MitmRequestAgent {
       if (session && !session.destroyed) session.destroy();
     });
 
-    proxyToServerH2Client.on('remoteSettings', settings => {
+    proxyToServerH2Client.on('remoteSettings', remoteSettings => {
       log.stats('Http2Client.remoteSettings', {
         sessionId: this.session.sessionId,
         origin,
-        settings,
+        settings: remoteSettings,
       });
     });
 
@@ -402,6 +425,7 @@ export default class MitmRequestAgent {
 
     originSocketPool.registerHttp2Session(proxyToServerH2Client, ctx.proxyToServerMitmSocket);
 
+    await connectPromise;
     return proxyToServerH2Client;
   }
 }