devops: downloading ffmpeg during install step (#5249)

This patch starts downloading FFMPEG like we download our browsers
instead of bundling it in the NPM package.

With this patch, NPM size is reduced from 8.8MB to 1.7MB.

Consequences:
- `npx playwright` is drastically faster now
- playwright driver for language bindings is way smaller
- projects that bundle Playwright can pass Apple Notorization

Fixes #5193

browsers.json

@@ -15,6 +15,11 @@
       "name": "webkit",
       "revision": "1428",
       "download": true
+    },
+    {
+      "name": "ffmpeg",
+      "revision": "1004",
+      "download": true
     }
   ]
 }

packages/build_package.js

@@ -29,14 +29,13 @@ const SCRIPT_NAME = path.basename(__filename);
 const ROOT_PATH = path.join(__dirname, '..');
 
 const PLAYWRIGHT_CORE_FILES = ['bin/PrintDeps.exe', 'lib', 'types', 'NOTICE', 'LICENSE'];
-const FFMPEG_FILES = ['third_party/ffmpeg'];
 
 const PACKAGES = {
   'playwright': {
     description: 'A high-level API to automate web browsers',
-    browsers: ['chromium', 'firefox', 'webkit'],
+    browsers: ['chromium', 'firefox', 'webkit', 'ffmpeg'],
     // We copy README.md additionally for Playwright so that it looks nice on NPM.
-    files: [...PLAYWRIGHT_CORE_FILES, ...FFMPEG_FILES, 'README.md'],
+    files: [...PLAYWRIGHT_CORE_FILES, 'README.md'],
   },
   'playwright-core': {
     description: 'A high-level API to automate web browsers',
@@ -55,20 +54,20 @@ const PACKAGES = {
   },
   'playwright-chromium': {
     description: 'A high-level API to automate Chromium',
-    browsers: ['chromium'],
-    files: [...PLAYWRIGHT_CORE_FILES, ...FFMPEG_FILES],
+    browsers: ['chromium', 'ffmpeg'],
+    files: [...PLAYWRIGHT_CORE_FILES],
   },
   'playwright-electron': {
     version: '0.4.0', // Manually manage playwright-electron version.
     description: 'A high-level API to automate Electron',
-    browsers: [],
-    files: [...PLAYWRIGHT_CORE_FILES, ...FFMPEG_FILES],
+    browsers: ['ffmpeg'],
+    files: [...PLAYWRIGHT_CORE_FILES],
   },
   'playwright-android': {
     version: '0.0.8', // Manually manage playwright-android version.
     description: 'A high-level API to automate Chrome for Android',
-    browsers: [],
-    files: [...PLAYWRIGHT_CORE_FILES, ...FFMPEG_FILES, 'bin/android-driver.apk', 'bin/android-driver-target.apk'],
+    browsers: ['ffmpeg'],
+    files: [...PLAYWRIGHT_CORE_FILES, 'bin/android-driver.apk', 'bin/android-driver-target.apk'],
   },
 };
 

src/install/browserFetcher.ts

@@ -47,6 +47,7 @@ function getDownloadHost(browserName: BrowserName, revision: number): string {
     chromium: 'PLAYWRIGHT_CHROMIUM_DOWNLOAD_HOST',
     firefox: 'PLAYWRIGHT_FIREFOX_DOWNLOAD_HOST',
     webkit: 'PLAYWRIGHT_WEBKIT_DOWNLOAD_HOST',
+    ffmpeg: 'PLAYWRIGHT_FFMPEG_DOWNLOAD_HOST',
   };
   return getFromENV(envDownloadHost[browserName]) ||
          getFromENV('PLAYWRIGHT_DOWNLOAD_HOST') ||
@@ -95,6 +96,20 @@ function getDownloadUrl(browserName: BrowserName, revision: number, platform: Br
       ['win64', '%s/builds/webkit/%s/webkit-win64.zip'],
     ]).get(platform);
   }
+
+  if (browserName === 'ffmpeg') {
+    return new Map<BrowserPlatform, string | undefined>([
+      ['ubuntu18.04', '%s/builds/ffmpeg/%s/ffmpeg-linux.zip'],
+      ['ubuntu20.04', '%s/builds/ffmpeg/%s/ffmpeg-linux.zip'],
+      ['mac10.13', '%s/builds/ffmpeg/%s/ffmpeg-mac.zip'],
+      ['mac10.14', '%s/builds/ffmpeg/%s/ffmpeg-mac.zip'],
+      ['mac10.15', '%s/builds/ffmpeg/%s/ffmpeg-mac.zip'],
+      ['mac11', '%s/builds/ffmpeg/%s/ffmpeg-mac.zip'],
+      ['mac11-arm64', '%s/builds/ffmpeg/%s/ffmpeg-mac.zip'],
+      ['win32', '%s/builds/ffmpeg/%s/ffmpeg-win32.zip'],
+      ['win64', '%s/builds/ffmpeg/%s/ffmpeg-win64.zip'],
+    ]).get(platform);
+  }
 }
 
 function revisionURL(browser: BrowserDescriptor, platform = browserPaths.hostPlatform): string {

src/server/android/android.ts

@@ -22,6 +22,7 @@ import * as stream from 'stream';
 import * as util from 'util';
 import * as ws from 'ws';
 import { createGuid, makeWaitForNextTask } from '../../utils/utils';
+import * as browserPaths from '../../utils/browserPaths';
 import { BrowserOptions, BrowserProcess, PlaywrightOptions } from '../browser';
 import { BrowserContext, validateBrowserContextOptions } from '../browserContext';
 import { ProgressController } from '../progress';
@@ -56,10 +57,14 @@ export interface SocketBackend extends EventEmitter {
 export class Android {
   private _backend: Backend;
   private _devices = new Map<string, AndroidDevice>();
+  readonly _ffmpegPath: string | null;
   readonly _timeoutSettings: TimeoutSettings;
   readonly _playwrightOptions: PlaywrightOptions;
 
-  constructor(backend: Backend, playwrightOptions: PlaywrightOptions) {
+  constructor(packagePath: string, backend: Backend, playwrightOptions: PlaywrightOptions, ffmpeg: browserPaths.BrowserDescriptor) {
+    const browsersPath = browserPaths.browsersPath(packagePath);
+    const browserPath = browserPaths.browserDirectory(browsersPath, ffmpeg);
+    this._ffmpegPath = browserPaths.executablePath(browserPath, ffmpeg) || null;
     this._backend = backend;
     this._playwrightOptions = playwrightOptions;
     this._timeoutSettings = new TimeoutSettings();
@@ -270,7 +275,7 @@ export class AndroidDevice extends EventEmitter {
     };
     validateBrowserContextOptions(options, browserOptions);
 
-    const browser = await CRBrowser.connect(androidBrowser, browserOptions);
+    const browser = await CRBrowser.connect(androidBrowser, browserOptions, this._android._ffmpegPath);
     const controller = new ProgressController();
     const defaultContext = browser._defaultContext!;
     await controller.run(async progress => {

src/server/chromium/chromium.ts

@@ -22,17 +22,22 @@ import { kBrowserCloseMessageId } from './crConnection';
 import { rewriteErrorMessage } from '../../utils/stackTrace';
 import { BrowserType } from '../browserType';
 import { ConnectionTransport, ProtocolRequest } from '../transport';
-import type { BrowserDescriptor } from '../../utils/browserPaths';
+import * as browserPaths from '../../utils/browserPaths';
 import { CRDevTools } from './crDevTools';
 import { BrowserOptions, PlaywrightOptions } from '../browser';
 import * as types from '../types';
 import { isDebugMode } from '../../utils/utils';
 
 export class Chromium extends BrowserType {
   private _devtools: CRDevTools | undefined;
+  private _ffmpegPath: string | null;
 
-  constructor(packagePath: string, browser: BrowserDescriptor, playwrightOptions: PlaywrightOptions) {
+  constructor(packagePath: string, browser: browserPaths.BrowserDescriptor, ffmpeg: browserPaths.BrowserDescriptor, playwrightOptions: PlaywrightOptions) {
     super(packagePath, browser, playwrightOptions);
+
+    const browsersPath = browserPaths.browsersPath(packagePath);
+    const browserPath = browserPaths.browserDirectory(browsersPath, ffmpeg);
+    this._ffmpegPath = browserPaths.executablePath(browserPath, ffmpeg) || null;
     if (isDebugMode())
       this._devtools = this._createDevTools();
   }
@@ -47,7 +52,7 @@ export class Chromium extends BrowserType {
       devtools = this._createDevTools();
       await (options as any).__testHookForDevTools(devtools);
     }
-    return CRBrowser.connect(transport, options, devtools);
+    return CRBrowser.connect(transport, options, this._ffmpegPath, devtools);
   }
 
   _rewriteStartupError(error: Error): Error {

src/server/chromium/crBrowser.ts

@@ -40,14 +40,15 @@ export class CRBrowser extends Browser {
   _devtools?: CRDevTools;
   _isMac = false;
   private _version = '';
+  readonly _ffmpegPath: string | null;
 
   private _tracingRecording = false;
   private _tracingPath: string | null = '';
   private _tracingClient: CRSession | undefined;
 
-  static async connect(transport: ConnectionTransport, options: BrowserOptions, devtools?: CRDevTools): Promise<CRBrowser> {
+  static async connect(transport: ConnectionTransport, options: BrowserOptions, ffmpegPath: string | null, devtools?: CRDevTools): Promise<CRBrowser> {
     const connection = new CRConnection(transport, options.protocolLogger, options.browserLogsCollector);
-    const browser = new CRBrowser(connection, options);
+    const browser = new CRBrowser(connection, options, ffmpegPath);
     browser._devtools = devtools;
     const session = connection.rootSession;
     const version = await session.send('Browser.getVersion');
@@ -88,8 +89,9 @@ export class CRBrowser extends Browser {
     return browser;
   }
 
-  constructor(connection: CRConnection, options: BrowserOptions) {
+  constructor(connection: CRConnection, options: BrowserOptions, ffmpegPath: string | null) {
     super(options);
+    this._ffmpegPath = ffmpegPath;
     this._connection = connection;
     this._session = this._connection.rootSession;
     this._connection.on(ConnectionEvents.Disconnected, () => this._didClose());

src/server/chromium/crPage.ts

@@ -788,7 +788,10 @@ class FrameSession {
 
   async _startScreencast(screencastId: string, options: types.PageScreencastOptions): Promise<void> {
     assert(!this._screencastId);
-    this._videoRecorder = await VideoRecorder.launch(options);
+    const ffmpegPath = this._crPage._browserContext._browser._ffmpegPath;
+    if (!ffmpegPath)
+      throw new Error('ffmpeg executable was not found');
+    this._videoRecorder = await VideoRecorder.launch(ffmpegPath, options);
     this._screencastId = screencastId;
     const gotFirstFrame = new Promise(f => this._client.once('Page.screencastFrame', f));
     await this._client.send('Page.startScreencast', {

src/server/chromium/videoRecorder.ts

@@ -15,7 +15,6 @@
  */
 
 import { ChildProcess } from 'child_process';
-import { ffmpegExecutable } from '../../utils/binaryPaths';
 import { assert, monotonicTime } from '../../utils/utils';
 import { launchProcess } from '../processLauncher';
 import { Progress, ProgressController } from '../progress';
@@ -33,22 +32,24 @@ export class VideoRecorder {
   private readonly _progress: Progress;
   private _frameQueue: Buffer[] = [];
   private _isStopped = false;
+  private _ffmpegPath: string;
 
-  static async launch(options: types.PageScreencastOptions): Promise<VideoRecorder> {
+  static async launch(ffmpegPath: string, options: types.PageScreencastOptions): Promise<VideoRecorder> {
     if (!options.outputFile.endsWith('.webm'))
       throw new Error('File must have .webm extension');
 
     const controller = new ProgressController();
     controller.setLogName('browser');
     return await controller.run(async progress => {
-      const recorder = new VideoRecorder(progress);
+      const recorder = new VideoRecorder(ffmpegPath, progress);
       await recorder._launch(options);
       return recorder;
     });
   }
 
-  private constructor(progress: Progress) {
+  private constructor(ffmpegPath: string, progress: Progress) {
     this._progress = progress;
+    this._ffmpegPath = ffmpegPath;
   }
 
   private async _launch(options: types.PageScreencastOptions) {
@@ -87,11 +88,8 @@ export class VideoRecorder {
     args.push(options.outputFile);
     const progress = this._progress;
 
-    const executablePath = ffmpegExecutable();
-    if (!executablePath)
-      throw new Error('ffmpeg executable was not found');
     const { launchedProcess, gracefullyClose } = await launchProcess({
-      executablePath,
+      executablePath: this._ffmpegPath,
       args,
       stdio: 'stdin',
       log: (message: string) => progress.log(message),

src/server/electron/electron.ts

@@ -21,6 +21,7 @@ import { CRExecutionContext } from '../chromium/crExecutionContext';
 import * as js from '../javascript';
 import { Page } from '../page';
 import { TimeoutSettings } from '../../utils/timeoutSettings';
+import * as browserPaths from '../../utils/browserPaths';
 import { WebSocketTransport } from '../transport';
 import * as types from '../types';
 import { launchProcess, envArrayToObject } from '../processLauncher';
@@ -123,8 +124,12 @@ export class ElectronApplication extends EventEmitter {
 
 export class Electron  {
   private _playwrightOptions: PlaywrightOptions;
+  private _ffmpegPath: string | null;
 
-  constructor(playwrightOptions: PlaywrightOptions) {
+  constructor(packagePath: string, playwrightOptions: PlaywrightOptions, ffmpeg: browserPaths.BrowserDescriptor) {
+    const browsersPath = browserPaths.browsersPath(packagePath);
+    const browserPath = browserPaths.browserDirectory(browsersPath, ffmpeg);
+    this._ffmpegPath = browserPaths.executablePath(browserPath, ffmpeg) || null;
     this._playwrightOptions = playwrightOptions;
   }
 
@@ -182,7 +187,7 @@ export class Electron  {
         protocolLogger: helper.debugProtocolLogger(),
         browserLogsCollector,
       };
-      const browser = await CRBrowser.connect(chromeTransport, browserOptions);
+      const browser = await CRBrowser.connect(chromeTransport, browserOptions, this._ffmpegPath);
       app = new ElectronApplication(browser, nodeConnection);
       await app._init();
       return app;

src/server/playwright.ts

@@ -47,16 +47,17 @@ export class Playwright {
       ]
     };
     const chromium = browsers.find(browser => browser.name === 'chromium');
-    this.chromium = new Chromium(packagePath, chromium!, this.options);
+    const ffmpeg = browsers.find(browser => browser.name === 'ffmpeg');
+    this.chromium = new Chromium(packagePath, chromium!, ffmpeg!, this.options);
 
     const firefox = browsers.find(browser => browser.name === 'firefox');
     this.firefox = new Firefox(packagePath, firefox!, this.options);
 
     const webkit = browsers.find(browser => browser.name === 'webkit');
     this.webkit = new WebKit(packagePath, webkit!, this.options);
 
-    this.electron = new Electron(this.options);
-    this.android = new Android(new AdbBackend(), this.options);
+    this.electron = new Electron(packagePath, this.options, ffmpeg!);
+    this.android = new Android(packagePath, new AdbBackend(), this.options, ffmpeg!);
   }
 }
 

src/server/validateDependencies.ts

@@ -42,6 +42,7 @@ const DL_OPEN_LIBRARIES = {
   webkit: ['libGLESv2.so.2', 'libx264.so'],
   firefox: [],
   clank: [],
+  ffmpeg: [],
 };
 
 function isSupportedWindowsVersion(): boolean {

src/utils/binaryPaths.ts

@@ -15,24 +15,12 @@
  */
 
 import * as fs from 'fs';
-import * as os from 'os';
 import * as path from 'path';
 
 export function printDepsWindowsExecutable(): string | undefined {
   return pathToExecutable(['bin', 'PrintDeps.exe']);
 }
 
-export function ffmpegExecutable(): string | undefined {
-  let ffmpegName;
-  if (process.platform === 'win32')
-    ffmpegName = os.arch() === 'x64' ? 'ffmpeg-win64.exe' : 'ffmpeg-win32.exe';
-  else if (process.platform === 'darwin')
-    ffmpegName = 'ffmpeg-mac';
-  else
-    ffmpegName = 'ffmpeg-linux';
-  return pathToExecutable(['third_party', 'ffmpeg', ffmpegName]);
-}
-
 function pathToExecutable(relative: string[]): string | undefined {
   try {
     const defaultPath = path.join(__dirname, '..', '..', ...relative);

src/utils/browserPaths.ts

@@ -21,7 +21,7 @@ import * as path from 'path';
 import { getUbuntuVersionSync } from './ubuntuVersion';
 import { getFromENV } from './utils';
 
-export type BrowserName = 'chromium'|'webkit'|'firefox';
+export type BrowserName = 'chromium'|'webkit'|'firefox'|'ffmpeg';
 export type BrowserPlatform = 'win32'|'win64'|'mac10.13'|'mac10.14'|'mac10.15'|'mac11'|'mac11-arm64'|'ubuntu18.04'|'ubuntu20.04';
 export type BrowserDescriptor = {
   name: BrowserName,
@@ -130,6 +130,19 @@ export function executablePath(browserPath: string, browser: BrowserDescriptor):
       ['win64', ['Playwright.exe']],
     ]).get(hostPlatform);
   }
+  if (browser.name === 'ffmpeg') {
+    tokens = new Map<BrowserPlatform, string[] | undefined>([
+      ['ubuntu18.04', ['ffmpeg-linux']],
+      ['ubuntu20.04', ['ffmpeg-linux']],
+      ['mac10.13', ['ffmpeg-mac']],
+      ['mac10.14', ['ffmpeg-mac']],
+      ['mac10.15', ['ffmpeg-mac']],
+      ['mac11', ['ffmpeg-mac']],
+      ['mac11-arm64', ['ffmpeg-mac']],
+      ['win32', ['ffmpeg-win32.exe']],
+      ['win64', ['ffmpeg-win64.exe']],
+    ]).get(hostPlatform);
+  }
   return tokens ? path.join(browserPath, ...tokens) : undefined;
 }
 
@@ -166,5 +179,5 @@ export function markerFilePath(browsersPath: string, browser: BrowserDescriptor)
 
 export function isBrowserDirectory(browserPath: string): boolean {
   const baseName = path.basename(browserPath);
-  return baseName.startsWith('chromium-') || baseName.startsWith('firefox-') || baseName.startsWith('webkit-');
+  return baseName.startsWith('chromium-') || baseName.startsWith('firefox-') || baseName.startsWith('webkit-') || baseName.startsWith('ffmpeg-');
 }

test/screencast.spec.ts

@@ -19,15 +19,14 @@ import fs from 'fs';
 import path from 'path';
 import { spawnSync } from 'child_process';
 import { PNG } from 'pngjs';
+import * as browserPaths from '../src/utils/browserPaths';
 
-let ffmpegName = '';
-if (process.platform === 'win32')
-  ffmpegName = process.arch === 'ia32' ? 'ffmpeg-win32' : 'ffmpeg-win64';
-else if (process.platform === 'darwin')
-  ffmpegName = 'ffmpeg-mac';
-else if (process.platform === 'linux')
-  ffmpegName = 'ffmpeg-linux';
-const ffmpeg = path.join(__dirname, '..', 'third_party', 'ffmpeg', ffmpegName);
+
+const browsersJSON = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'browsers.json'), 'utf8'));
+const ffmpegDescriptor = browsersJSON.browsers.find(({name}) => name === 'ffmpeg');
+const browsersPath = browserPaths.browsersPath(path.join(__dirname, '..'));
+const browserPath = browserPaths.browserDirectory(browsersPath, ffmpegDescriptor);
+const ffmpeg = browserPaths.executablePath(browserPath, ffmpegDescriptor) || '';
 
 export class VideoPlayer {
   fileName: string;