feat(replay): single install of replay

fix(puppet): improve start issues
chore(puppet): share common dir for engines/replay
feat(replay): register scripts over http instead of launch args

client/package.json

@@ -9,14 +9,14 @@
   },
   "dependencies": {
     "@secret-agent/commons": "1.2.0-alpha.4",
-    "@secret-agent/core": "1.2.0-alpha.5",
     "@secret-agent/core-interfaces": "1.2.0-alpha.4",
     "@secret-agent/replay": "1.2.0-alpha.4",
     "awaited-dom": "^1.1.10",
     "ws": "^7.4.2",
     "uuid": "^8.1.0"
   },
   "devDependencies": {
-    "@secret-agent/testing": "1.2.0-alpha.5"
+    "@secret-agent/testing": "1.2.0-alpha.5",
+    "@secret-agent/core": "1.2.0-alpha.5"
   }
 }

core/lib/GlobalPool.ts

@@ -46,7 +46,8 @@ export default class GlobalPool {
 
     for (const emulatorId of browserEmulatorIds) {
       const browserEmulator = BrowserEmulators.getClass(emulatorId);
-      this.addPuppet(browserEmulator.engine);
+      const puppet = await this.addPuppet(browserEmulator.engine);
+      await puppet.isReady;
     }
 
     this.resolveWaitingConnection();
@@ -89,16 +90,21 @@ export default class GlobalPool {
     log.stats('CompletedGlobalPoolShutdown', { parentLogId: logId, sessionId: null });
   }
 
-  private static addPuppet(engine: IBrowserEngine) {
+  private static async addPuppet(engine: IBrowserEngine): Promise<Puppet> {
     const existing = this.getPuppet(engine);
-    if (existing) return existing;
+    if (existing) return Promise.resolve(existing);
 
     const puppet = new Puppet(engine);
     this.puppets.push(puppet);
 
     const showBrowser = !!process.env.SHOW_BROWSER;
     const showBrowserLogs = !!process.env.DEBUG;
-    puppet.start({ proxyPort: this.mitmServer.port, showBrowser, pipeBrowserIo: showBrowserLogs });
+    const browserOrError = await puppet.start({
+      proxyPort: this.mitmServer.port,
+      showBrowser,
+      pipeBrowserIo: showBrowserLogs,
+    });
+    if (browserOrError instanceof Error) throw browserOrError;
     return puppet;
   }
 
@@ -121,7 +127,8 @@ export default class GlobalPool {
       const session = new Session(options);
       session.on('closing', this.releaseConnection.bind(this));
 
-      puppet = this.getPuppet(session.browserEngine) ?? this.addPuppet(session.browserEngine);
+      puppet =
+        this.getPuppet(session.browserEngine) ?? (await this.addPuppet(session.browserEngine));
 
       const browserContext = await puppet.newContext(
         session.getBrowserEmulation(),

full-client/test/interact.test.ts

@@ -41,7 +41,7 @@ describe('basic Interact tests', () => {
 
     await agent.close();
     await httpServer.close();
-  }, 20e3);
+  }, 30e3);
 
   it('should be able to get multiple entries out of the pool', async () => {
     const httpServer = await Helpers.runHttpServer({

mitm-socket/index.ts

@@ -186,6 +186,7 @@ export default class MitmSocket extends TypedEventEmitter<{
     if (this.connectError)
       this.socket.destroy(buildConnectError(this.connectError, this.callStack));
     this.socket.end();
+    this.socket.unref();
     this.isConnected = false;
     unlink(this.socketPath, () => null);
     delete this.socket;

mitm/lib/MitmRequestAgent.ts

@@ -92,7 +92,11 @@ export default class MitmRequestAgent {
   }
 
   public close(): void {
-    this.http2Sessions.map(x => x.client.destroy());
+    for (const session of this.http2Sessions) {
+      session.mitmSocket.close();
+      session.client.destroy();
+      session.client.unref();
+    }
     this.http2Sessions.length = 0;
     for (const socket of this.sockets) {
       socket.close();

puppet-chrome/index.ts

@@ -54,6 +54,27 @@ const PuppetLauncher: IPuppetLauncher = {
       throw error;
     }
   },
+  translateLaunchError(error: Error): Error {
+    // These error messages are taken from Chromium source code as of July, 2020:
+    // https://github.com/chromium/chromium/blob/70565f67e79f79e17663ad1337dc6e63ee207ce9/content/browser/zygote_host/zygote_host_impl_linux.cc
+    if (
+      !error.message.includes('crbug.com/357670') &&
+      !error.message.includes('No usable sandbox!') &&
+      !error.message.includes('crbug.com/638180')
+    ) {
+      return error;
+    }
+    error.stack += [
+      `\nChromium sandboxing failed!`,
+      `================================`,
+      `To workaround sandboxing issues, do either of the following:`,
+      `  - (preferred): Configure environment to support sandboxing (eg: in Docker, use custom seccomp profile + non-root user + --ipc=host)`,
+      `  - (alternative): Launch Chromium without sandbox using 'chromiumSandbox: false' option`,
+      `================================`,
+      ``,
+    ].join('\n');
+    return error;
+  },
 };
 export default PuppetLauncher;
 

puppet-interfaces/IPuppetLauncher.ts

@@ -2,6 +2,7 @@ import ILaunchedProcess from './ILaunchedProcess';
 import IPuppetBrowser from './IPuppetBrowser';
 
 export default interface IPuppetLauncher {
-  getLaunchArgs(options: { proxyPort?: number; showBrowser?: boolean });
+  getLaunchArgs(options: { proxyPort?: number; showBrowser?: boolean }): string[];
   createPuppet(process: ILaunchedProcess, revision: string): Promise<IPuppetBrowser>;
+  translateLaunchError(error: Error): Error;
 }

puppet/index.ts

@@ -5,7 +5,9 @@ import IPuppetLauncher from '@secret-agent/puppet-interfaces/IPuppetLauncher';
 import IPuppetBrowser from '@secret-agent/puppet-interfaces/IPuppetBrowser';
 import IBrowserEmulationSettings from '@secret-agent/puppet-interfaces/IBrowserEmulationSettings';
 import IBrowserEngine from '@secret-agent/core-interfaces/IBrowserEngine';
+import { existsSync } from 'fs';
 import launchProcess from './lib/launchProcess';
+import { getExecutablePath } from './lib/browserPaths';
 
 const { log } = Log(module);
 
@@ -14,49 +16,46 @@ export default class Puppet {
   public readonly id: number;
   public readonly engine: IBrowserEngine;
   public isShuttingDown: boolean;
-  private browser: Promise<IPuppetBrowser>;
+  private browserOrError: Promise<IPuppetBrowser | Error>;
+
+  public get isReady(): Promise<IPuppetBrowser> {
+    return this.browserOrError.then(x => {
+      if (x instanceof Error) throw x;
+      return x;
+    });
+  }
 
   constructor(engine: IBrowserEngine) {
     this.engine = engine;
     this.isShuttingDown = false;
     this.id = puppBrowserCounter;
-    this.browser = null;
+    this.browserOrError = null;
     puppBrowserCounter += 1;
   }
 
   public start(
-    args: {
-      proxyPort?: number;
-      showBrowser?: boolean;
-      pipeBrowserIo?: boolean;
-    } = {
+    args: ILaunchArgs = {
       showBrowser: false,
       pipeBrowserIo: false,
     },
-  ) {
-    if (this.browser) {
-      return;
+  ): Promise<IPuppetBrowser | Error> {
+    if (this.browserOrError) {
+      return this.browserOrError;
     }
-    const { proxyPort, showBrowser, pipeBrowserIo } = args;
     this.isShuttingDown = false;
 
     let launcher: IPuppetLauncher;
     if (this.engine.browser === 'chrome' || this.engine.browser === 'chromium') {
       launcher = PuppetChrome;
     }
 
-    const launchArgs = launcher.getLaunchArgs({ proxyPort, showBrowser });
-    const launchedProcess = launchProcess(
-      this.engine.executablePath,
-      launchArgs,
-      {},
-      pipeBrowserIo,
-    );
-    this.browser = launcher.createPuppet(launchedProcess, this.engine.revision);
+    this.browserOrError = this.launchEngine(launcher, args).catch(err => err);
+    return this.browserOrError;
   }
 
   public async newContext(emulation: IBrowserEmulationSettings, logger: IBoundLog) {
-    const browser = await this.browser;
+    const browser = await this.browserOrError;
+    if (browser instanceof Error) throw browser;
     if (this.isShuttingDown) throw new Error('Shutting down');
     return browser.newContext(emulation, logger);
   }
@@ -66,14 +65,50 @@ export default class Puppet {
     this.isShuttingDown = true;
     log.stats('Puppet.Closing');
 
-    const browserPromise = this.browser;
-    this.browser = null;
+    const browserPromise = this.browserOrError;
+    this.browserOrError = null;
 
     try {
       const browser = await browserPromise;
-      if (browser) await browser.close();
+      if (browser && !(browser instanceof Error)) await browser.close();
     } catch (error) {
       log.error('Puppet.Closing:Error', { sessionId: null, error });
     }
   }
+
+  private async launchEngine(
+    launcher: IPuppetLauncher,
+    args: ILaunchArgs,
+  ): Promise<IPuppetBrowser> {
+    const executablePath = this.engine.executablePath;
+
+    if (!existsSync(executablePath)) {
+      const errorMessageLines = [
+        `Failed to launch ${this.engine.browser}@${this.engine.revision} because executable doesn't exist at ${executablePath}`,
+      ];
+
+      const packagedPath = getExecutablePath(this.engine.browser, this.engine.revision);
+      // If we tried using stock downloaded browser, suggest re-installing SecretAgent.
+      if (executablePath === packagedPath)
+        errorMessageLines.push(
+          `Try re-installing SecretAgent with "npm install secret-agent" or re-install any custom BrowserEmulators.`,
+        );
+      throw new Error(errorMessageLines.join('\n'));
+    }
+
+    try {
+      const { pipeBrowserIo, proxyPort, showBrowser } = args;
+      const launchArgs = launcher.getLaunchArgs({ showBrowser, proxyPort });
+      const launchedProcess = await launchProcess(executablePath, launchArgs, {}, pipeBrowserIo);
+      return launcher.createPuppet(launchedProcess, this.engine.revision);
+    } catch (err) {
+      throw launcher.translateLaunchError(err);
+    }
+  }
+}
+
+interface ILaunchArgs {
+  proxyPort?: number;
+  showBrowser?: boolean;
+  pipeBrowserIo?: boolean;
 }

puppet/lib/BrowserFetcher.ts

@@ -15,10 +15,9 @@
  */
 
 import * as os from 'os';
-import * as fs from 'fs';
+import { createWriteStream, existsSync, promises as fs } from 'fs';
 import * as path from 'path';
 import * as util from 'util';
-import { promisify } from 'util';
 import * as childProcess from 'child_process';
 import * as https from 'https';
 import * as http from 'http';
@@ -64,15 +63,11 @@ function downloadURL(platform: Platform, host: string, revision: string): string
   return util.format(downloadURLs[platform], host, revision, archiveName(platform, revision));
 }
 
-const readdirAsync = promisify(fs.readdir.bind(fs));
-const mkdirAsync = promisify(fs.mkdir.bind(fs));
-const unlinkAsync = promisify(fs.unlink.bind(fs));
-const chmodAsync = promisify(fs.chmod.bind(fs));
-
 function existsAsync(filePath: string): Promise<boolean> {
-  return new Promise(resolve => {
-    fs.access(filePath, err => resolve(!err));
-  });
+  return fs
+    .access(filePath)
+    .then(() => true)
+    .catch(() => false);
 }
 
 /**
@@ -183,18 +178,19 @@ export class BrowserFetcher {
     const archivePath = path.join(this._downloadsFolder, fileName);
     const outputPath = this._getFolderPath(revision);
     if (await existsAsync(outputPath)) return this.revisionInfo(revision);
-    if (!(await existsAsync(this._downloadsFolder))) await mkdirAsync(this._downloadsFolder);
+    if (!(await existsAsync(this._downloadsFolder)))
+      await fs.mkdir(this._downloadsFolder, { recursive: true });
     if (os.arch() === 'arm64') {
       throw new Error('The chromium binary is not available for arm64');
     }
     try {
       await downloadFile(url, archivePath, progressCallback);
       await install(archivePath, outputPath);
     } finally {
-      if (await existsAsync(archivePath)) await unlinkAsync(archivePath);
+      if (await existsAsync(archivePath)) await fs.unlink(archivePath);
     }
     const revisionInfo = this.revisionInfo(revision);
-    if (revisionInfo) await chmodAsync(revisionInfo.executablePath, 0o755);
+    if (revisionInfo) await fs.chmod(revisionInfo.executablePath, 0o755);
     return revisionInfo;
   }
 
@@ -206,7 +202,7 @@ export class BrowserFetcher {
    */
   public async localRevisions(): Promise<string[]> {
     if (!(await existsAsync(this._downloadsFolder))) return [];
-    const fileNames = await readdirAsync(this._downloadsFolder);
+    const fileNames = await fs.readdir(this._downloadsFolder);
     return fileNames
       .map(fileName => parseFolderPath(fileName))
       .filter(entry => entry && entry.platform === this._platform)
@@ -252,7 +248,7 @@ export class BrowserFetcher {
     else throw new Error(`Unsupported platform: ${this._platform}`);
 
     const url = downloadURL(this._platform, this._downloadHost, revision);
-    const local = fs.existsSync(folderPath);
+    const local = existsSync(folderPath);
     const revisionInfo = {
       revision,
       executablePath,
@@ -325,7 +321,7 @@ function downloadFile(
       downloadReject(error);
       return;
     }
-    const file = fs.createWriteStream(destinationPath);
+    const file = createWriteStream(destinationPath);
     file.on('finish', () => downloadResolve());
     file.on('error', error => downloadReject(error));
     response.pipe(file);
@@ -345,7 +341,9 @@ function install(archivePath: string, folderPath: string): Promise<unknown> {
   npmlog(`Installing ${archivePath} to ${folderPath}`);
   if (archivePath.endsWith('.zip')) return extractZip(archivePath, { dir: folderPath });
   if (archivePath.endsWith('.dmg')) {
-    return mkdirAsync(folderPath).then(() => installDMG(archivePath, folderPath));
+    return fs
+      .mkdir(folderPath, { recursive: true })
+      .then(() => installDMG(archivePath, folderPath));
   }
   throw new Error(`Unsupported archive format: ${archivePath}`);
 }
@@ -364,7 +362,7 @@ async function installDMG(dmgPath: string, folderPath: string): Promise<void> {
     if (!volumes) throw new Error(`Could not find volume path in ${stdout}`);
     mountPath = volumes[0];
 
-    const fileNames = await readdirAsync(mountPath);
+    const fileNames = await fs.readdir(mountPath);
 
     const appName = fileNames.filter(item => typeof item === 'string' && item.endsWith('.app'))[0];
     if (!appName) throw new Error(`Cannot find app in ${mountPath}`);

puppet/lib/browserPaths.ts

@@ -20,7 +20,7 @@ export function getExecutablePath(browser: string, revision: string) {
 }
 
 export function getInstallDirectory(browser: string, revision: string) {
-  return `${getCacheDirectory()}/${browser}-${revision}`;
+  return `${getCacheDirectory()}/secret-agent/${browser}-${revision}`;
 }
 
 function getCacheDirectory() {