From af436ef7be74806def82ef2cf25335c3361ed3d1 Mon Sep 17 00:00:00 2001
From: Kiko Beats <josefrancisco.verdu@gmail.com>
Date: Tue, 19 Mar 2024 12:10:47 +0100
Subject: [PATCH 1/2] feat: revamped screencast

---
 packages/screencast/examples/ffmpeg.js | 59 ++++++++++++++++
 packages/screencast/examples/server.js | 76 ++++++++++++++++++++
 packages/screencast/package.json       |  7 +-
 packages/screencast/src/index.js       | 98 +++++---------------------
 packages/screencast/src/utils.js       | 23 ------
 packages/screencast/test/index.js      | 90 ++++++-----------------
 6 files changed, 174 insertions(+), 179 deletions(-)
 create mode 100644 packages/screencast/examples/ffmpeg.js
 create mode 100644 packages/screencast/examples/server.js
 delete mode 100644 packages/screencast/src/utils.js

diff --git a/packages/screencast/examples/ffmpeg.js b/packages/screencast/examples/ffmpeg.js
new file mode 100644
index 0000000000..45aa9994ff
--- /dev/null
+++ b/packages/screencast/examples/ffmpeg.js
@@ -0,0 +1,59 @@
+'use strict'
+
+const { Readable } = require('stream')
+const $ = require('tinyspawn')
+
+module.exports = (buffer, { width, height, frameRate = 30 }) =>
+  new Promise(resolve => {
+    const subprocess = $('ffmpeg', [
+      '-loglevel',
+      'error',
+      // Reduces general buffering.
+      // '-avioflags',
+      // 'direct',
+      // Reduces initial buffering while analyzing input fps and other stats.
+      '-fpsprobesize',
+      '0',
+      '-probesize',
+      '32',
+      '-analyzeduration',
+      '0',
+      // '-fflags',
+      // 'nobuffer',
+      // Forces input to be read from standard input, and forces png input
+      // image format.
+      '-f',
+      'image2pipe',
+      '-c:v',
+      'png',
+      '-i',
+      'pipe:0',
+      // Overwrite output and no audio.
+      '-y',
+      '-an',
+      // This drastically reduces stalling when cpu is overbooked. By default
+      // VP9 tries to use all available threads?
+      '-threads',
+      '1',
+      // Specifies the frame rate we are giving ffmpeg.
+      '-framerate',
+      `${frameRate}`,
+      // Specifies the encoding and format we are using.
+      '-vf',
+      'fps=5,split[s0][s1];[s0]palettegen=stats_mode=diff[p];[s1][p]paletteuse',
+      '-f',
+      'gif',
+      // Disable bitrate.
+      '-b:v',
+      '0',
+      // Filters to ensure the images are piped correctly.
+      '-vf',
+      `scale=${width}:${height}`,
+      'pipe:1'
+    ])
+
+    const chunks = []
+    subprocess.stdout.on('data', data => chunks.push(data))
+    subprocess.stdout.on('end', () => resolve(Buffer.concat(chunks)))
+    Readable.from(buffer).pipe(subprocess.stdin)
+  })
diff --git a/packages/screencast/examples/server.js b/packages/screencast/examples/server.js
new file mode 100644
index 0000000000..49f031d880
--- /dev/null
+++ b/packages/screencast/examples/server.js
@@ -0,0 +1,76 @@
+'use strict'
+
+const { createCanvas, Image } = require('canvas')
+const createBrowser = require('browserless')
+const GIFEncoder = require('gifencoder')
+const http = require('http')
+
+const createScreencast = require('..')
+const ffmpeg = require('./ffmpeg')
+
+const browser = createBrowser({
+  timeout: 25000,
+  lossyDeviceName: true,
+  ignoreHTTPSErrors: true
+})
+
+const CACHE = Object.create(null)
+
+const server = http.createServer(async (req, res) => {
+  if (req.url === '/favicon.ico') return res.end()
+  const url = req.url.slice(1)
+
+  if (CACHE[url]) {
+    const pngBuffer = CACHE[url].toBuffer('image/png')
+    res.setHeader('Content-Type', 'image/png')
+    res.write(pngBuffer)
+    return res.end()
+  }
+
+  const browserless = await browser.createContext()
+  const page = await browserless.page()
+  let lastCanvas = null
+
+  res.setHeader('Content-Type', 'image/gif')
+
+  const width = 1280
+  const height = 800
+  const resizeRatio = 0.5
+
+  const outputSize = { width: width * resizeRatio, height: height * resizeRatio }
+
+  const encoder = new GIFEncoder(outputSize.width, outputSize.height)
+  const canvas = createCanvas(outputSize.width, outputSize.height)
+  const ctx = canvas.getContext('2d')
+
+  encoder.createWriteStream({ repeat: -1, delay: 0 }).pipe(res)
+
+  const screencast = createScreencast(page, {
+    quality: 0,
+    format: 'png',
+    everyNthFrame: 1
+  })
+
+  screencast.onFrame(async data => {
+    const frame = Buffer.from(data, 'base64')
+    const img = new Image()
+    img.src = await ffmpeg(frame, outputSize)
+    ctx.drawImage(img, 0, 0, img.width, img.height)
+    encoder.addFrame(ctx)
+    lastCanvas = canvas
+  })
+
+  screencast.start()
+  encoder.start()
+  await browserless.goto(page, { url, viewport: { width, height, deviceScaleFactor: 1 } })
+  encoder.finish()
+  await screencast.stop()
+
+  CACHE[url] = lastCanvas
+})
+
+server.listen(3000, () =>
+  console.log(`
+  Listen: http://localhost:3000/{URL}
+ Example: http://localhost:3000/https://browserless.js.org\n`)
+)
diff --git a/packages/screencast/package.json b/packages/screencast/package.json
index 4a6477cede..06a54d4aaa 100644
--- a/packages/screencast/package.json
+++ b/packages/screencast/package.json
@@ -32,13 +32,12 @@
     "screencast",
     "video"
   ],
-  "dependencies": {
-    "tinyspawn": "~1.2.6"
-  },
   "devDependencies": {
     "@browserless/test": "^10.3.0",
     "ava": "5",
-    "file-type": "16"
+    "canvas": "latest",
+    "gifencoder": "latest",
+    "tinyspawn": "latest"
   },
   "engines": {
     "node": ">= 12"
diff --git a/packages/screencast/src/index.js b/packages/screencast/src/index.js
index a7c4a4e522..735a878b53 100644
--- a/packages/screencast/src/index.js
+++ b/packages/screencast/src/index.js
@@ -1,86 +1,20 @@
 'use strict'
 
-const { unlink, readFile } = require('fs/promises')
-const { randomUUID } = require('crypto')
-const { Readable } = require('stream')
-const { tmpdir } = require('os')
-const $ = require('tinyspawn')
-const path = require('path')
-
-const { startScreencast } = require('./utils')
-
-// Inspired by https://github.com/microsoft/playwright/blob/37b3531a1181c99990899c15000925a98f035eb7/packages/playwright-core/src/server/chromium/videoRecorder.ts#L101
-const ffmpegArgs = format => {
-  // `-an` disables audio
-  // `-b:v 0` disables video bitrate control
-  // `-c:v` alias for -vcodec
-  // `-avioflags direct` reduces buffering
-  // `-probesize 32` size of the data to analyze to get stream information
-  // `-analyzeduration 0` specify how many microseconds are analyzed to probe the input
-  // `-fpsprobesize 0` set number of frames used to probe fps
-  // `-fflags nobuffer` disables buffering when reading or writing multimedia data
-  const args =
-    '-loglevel error -an -b:v 0 -avioflags direct -probesize 32 -analyzeduration 0 -fpsprobesize 0 -fflags nobuffer'
-
-  if (format === 'mp4') {
-    // ffmpeg -h encoder=h264
-    return `${args} -c:v libx264 -pix_fmt yuv420p -preset ultrafast -realtime true`
-  }
-
-  if (format === 'gif') {
-    return args
+const getCDPClient = page => page._client()
+
+module.exports = (page, opts) => {
+  const client = getCDPClient(page)
+  let onFrame
+
+  client.on('Page.screencastFrame', ({ data, metadata, sessionId }) => {
+    client.send('Page.screencastFrameAck', { sessionId }).catch(() => {})
+    if (metadata.timestamp) onFrame(data, metadata)
+  })
+
+  return {
+    // https://chromedevtools.github.io/devtools-protocol/tot/Page/#method-startScreencast
+    start: () => client.send('Page.startScreencast', opts),
+    onFrame: fn => (onFrame = fn),
+    stop: async () => client.send('Page.stopScreencast')
   }
-
-  if (format === 'webm') {
-    // ffmpeg -h encoder=vp9
-    return `${args} -c:v libvpx-vp9 -quality realtime`
-  }
-
-  throw new TypeError(`Format '${format}' not supported`)
-}
-
-module.exports = async ({
-  ffmpegPath,
-  format = 'webm',
-  frameRate = 25,
-  frames: framesOpts = {},
-  getBrowserless,
-  gotoOpts,
-  timeout,
-  tmpPath = tmpdir(),
-  withPage
-} = {}) => {
-  const browserless = await getBrowserless()
-
-  const fn = (page, goto) => async gotoOpts => {
-    await goto(page, gotoOpts)
-    const screencastStop = await startScreencast(page, framesOpts)
-    await withPage(page)
-    const frames = await screencastStop()
-
-    const interpolatedFrames = frames.reduce((acc, { data, metadata }, index) => {
-      const previousIndex = index - 1
-      const previousFrame = index > 0 ? frames[previousIndex] : undefined
-      const durationSeconds = previousFrame
-        ? metadata.timestamp - previousFrame.metadata.timestamp
-        : 0
-      const numFrames = Math.max(1, Math.round(durationSeconds * frameRate))
-      for (let i = 0; i < numFrames; i++) acc.push(data)
-      return acc
-    }, [])
-
-    const filepath = path.join(tmpPath, `${randomUUID()}.${format}`)
-    const subprocess = $(
-      `${ffmpegPath} -f image2pipe -i pipe:0 -r ${frameRate} ${ffmpegArgs(format)} ${filepath}`
-    )
-    Readable.from(interpolatedFrames).pipe(subprocess.stdin)
-
-    await subprocess
-    const buffer = await readFile(filepath)
-    await unlink(filepath)
-
-    return buffer
-  }
-
-  return browserless.withPage(fn, { timeout })(gotoOpts)
 }
diff --git a/packages/screencast/src/utils.js b/packages/screencast/src/utils.js
deleted file mode 100644
index 71d1ca13fa..0000000000
--- a/packages/screencast/src/utils.js
+++ /dev/null
@@ -1,23 +0,0 @@
-'use strict'
-
-const getCDPClient = page => page._client()
-
-const startScreencast = async (page, { onFrame, ...opts }) => {
-  const client = getCDPClient(page)
-  const frames = []
-
-  client.on('Page.screencastFrame', async ({ data, metadata, sessionId }) => {
-    if (metadata.timestamp) frames.push({ data: Buffer.from(data, 'base64'), metadata })
-    client.send('Page.screencastFrameAck', { sessionId }).catch(() => {})
-  })
-
-  // https://chromedevtools.github.io/devtools-protocol/tot/Page/#method-startScreencast
-  await client.send('Page.startScreencast', opts)
-
-  return async () => {
-    await client.send('Page.stopScreencast')
-    return frames.sort((a, b) => a.metadata.timestamp - b.metadata.timestamp)
-  }
-}
-
-module.exports = { getCDPClient, startScreencast }
diff --git a/packages/screencast/test/index.js b/packages/screencast/test/index.js
index d1d13d2aed..344f4bf0f9 100644
--- a/packages/screencast/test/index.js
+++ b/packages/screencast/test/index.js
@@ -1,83 +1,33 @@
 'use strict'
 
 const { getBrowserContext } = require('@browserless/test/util')
-const { writeFile } = require('fs/promises')
-const { randomUUID } = require('crypto')
-const FileType = require('file-type')
-const { unlinkSync } = require('fs')
-const { tmpdir } = require('os')
-const $ = require('tinyspawn')
-const path = require('path')
 const test = require('ava')
 
-const screencast = require('..')
+const createScreencast = require('..')
 
-const isCI = !!process.env.CI
+test('capture frames', async t => {
+  const frames = []
 
-test('get a webm video', async t => {
-  const [browserless, { stdout: ffmpegPath }] = await Promise.all([
-    getBrowserContext(t),
-    $('which ffmpeg')
-  ])
+  const browserless = await getBrowserContext(t)
+  const page = await browserless.page()
 
-  const buffer = await screencast({
-    getBrowserless: () => browserless,
-    ffmpegPath,
-    frames: {
-      everyNthFrame: 2
-    },
-    gotoOpts: {
-      url: 'https://vercel.com',
-      animations: true,
-      abortTypes: [],
-      waitUntil: 'load'
-    },
-    withPage: async page => {
-      const TOTAL_TIME = 7_000 * isCI ? 0.5 : 1
-
-      const timing = {
-        topToQuarter: (TOTAL_TIME * 1.5) / 7,
-        quarterToQuarter: (TOTAL_TIME * 0.3) / 7,
-        quarterToBottom: (TOTAL_TIME * 1) / 7,
-        bottomToTop: (TOTAL_TIME * 2) / 7
-      }
-
-      const scrollTo = (partial, ms) =>
-        page.evaluate(
-          (partial, ms) =>
-            new Promise(resolve => {
-              window.requestAnimationFrame(() => {
-                window.scrollTo({
-                  top: document.scrollingElement.scrollHeight * partial,
-                  behavior: 'smooth'
-                })
-                setTimeout(resolve, ms)
-              })
-            }),
-          partial,
-          ms
-        )
-
-      await scrollTo(1 / 3, timing.topToQuarter)
-      await scrollTo(2 / 3, timing.quarterToQuarter)
-      await scrollTo(3 / 3, timing.quarterToBottom)
-      await scrollTo(0, timing.bottomToTop)
-    }
+  const screencast = createScreencast(page, {
+    quality: 0,
+    format: 'png',
+    everyNthFrame: 1
   })
 
-  const { ext, mime } = await FileType.fromBuffer(buffer)
-  t.is(ext, 'webm')
-  t.is(mime, 'video/webm')
-
-  const filepath = path.join(tmpdir(), randomUUID())
-  t.teardown(() => unlinkSync(filepath))
-  await writeFile(filepath, buffer)
+  screencast.onFrame((data, metadata) => {
+    frames.push({ data, metadata })
+  })
 
-  const { stdout: ffprobe } = await $.json(
-    `ffprobe ${filepath} -print_format json -v quiet -show_format -show_streams -show_error`
-  )
+  screencast.start()
+  await page.goto('https://example.com', { waitUntil: 'domcontentloaded' })
+  await screencast.stop()
 
-  t.is(ffprobe.streams[0].codec_name, 'vp9')
-  t.is(ffprobe.streams[0].pix_fmt, 'yuv420p')
-  t.is(ffprobe.streams[0].avg_frame_rate, '25/1')
+  frames.forEach(({ data, metadata }) => {
+    t.truthy(data)
+    t.is(typeof metadata, 'object')
+    t.truthy(metadata.timestamp)
+  })
 })

From 2ff7ad4651fb9db44dd2127b88c916d553b4a2cf Mon Sep 17 00:00:00 2001
From: Kiko Beats <josefrancisco.verdu@gmail.com>
Date: Tue, 19 Mar 2024 13:04:26 +0100
Subject: [PATCH 2/2] refactor: use sharp instead of ffmpeg

---
 packages/screencast/examples/ffmpeg.js | 59 --------------------------
 packages/screencast/examples/server.js | 34 +++++++++------
 packages/screencast/package.json       |  4 +-
 3 files changed, 24 insertions(+), 73 deletions(-)
 delete mode 100644 packages/screencast/examples/ffmpeg.js

diff --git a/packages/screencast/examples/ffmpeg.js b/packages/screencast/examples/ffmpeg.js
deleted file mode 100644
index 45aa9994ff..0000000000
--- a/packages/screencast/examples/ffmpeg.js
+++ /dev/null
@@ -1,59 +0,0 @@
-'use strict'
-
-const { Readable } = require('stream')
-const $ = require('tinyspawn')
-
-module.exports = (buffer, { width, height, frameRate = 30 }) =>
-  new Promise(resolve => {
-    const subprocess = $('ffmpeg', [
-      '-loglevel',
-      'error',
-      // Reduces general buffering.
-      // '-avioflags',
-      // 'direct',
-      // Reduces initial buffering while analyzing input fps and other stats.
-      '-fpsprobesize',
-      '0',
-      '-probesize',
-      '32',
-      '-analyzeduration',
-      '0',
-      // '-fflags',
-      // 'nobuffer',
-      // Forces input to be read from standard input, and forces png input
-      // image format.
-      '-f',
-      'image2pipe',
-      '-c:v',
-      'png',
-      '-i',
-      'pipe:0',
-      // Overwrite output and no audio.
-      '-y',
-      '-an',
-      // This drastically reduces stalling when cpu is overbooked. By default
-      // VP9 tries to use all available threads?
-      '-threads',
-      '1',
-      // Specifies the frame rate we are giving ffmpeg.
-      '-framerate',
-      `${frameRate}`,
-      // Specifies the encoding and format we are using.
-      '-vf',
-      'fps=5,split[s0][s1];[s0]palettegen=stats_mode=diff[p];[s1][p]paletteuse',
-      '-f',
-      'gif',
-      // Disable bitrate.
-      '-b:v',
-      '0',
-      // Filters to ensure the images are piped correctly.
-      '-vf',
-      `scale=${width}:${height}`,
-      'pipe:1'
-    ])
-
-    const chunks = []
-    subprocess.stdout.on('data', data => chunks.push(data))
-    subprocess.stdout.on('end', () => resolve(Buffer.concat(chunks)))
-    Readable.from(buffer).pipe(subprocess.stdin)
-  })
diff --git a/packages/screencast/examples/server.js b/packages/screencast/examples/server.js
index 49f031d880..764549ae23 100644
--- a/packages/screencast/examples/server.js
+++ b/packages/screencast/examples/server.js
@@ -1,12 +1,13 @@
 'use strict'
 
+const timeSpan = require('@kikobeats/time-span')({ format: n => `${n.toFixed(2)}ms` })
 const { createCanvas, Image } = require('canvas')
+const { GifEncoder } = require('@skyra/gifenc')
 const createBrowser = require('browserless')
-const GIFEncoder = require('gifencoder')
+const sharp = require('sharp')
 const http = require('http')
 
 const createScreencast = require('..')
-const ffmpeg = require('./ffmpeg')
 
 const browser = createBrowser({
   timeout: 25000,
@@ -18,6 +19,10 @@ const CACHE = Object.create(null)
 
 const server = http.createServer(async (req, res) => {
   if (req.url === '/favicon.ico') return res.end()
+
+  const duration = timeSpan()
+  let firstFrame = true
+
   const url = req.url.slice(1)
 
   if (CACHE[url]) {
@@ -35,37 +40,40 @@ const server = http.createServer(async (req, res) => {
 
   const width = 1280
   const height = 800
-  const resizeRatio = 0.5
+  const deviceScaleFactor = 0.5
 
-  const outputSize = { width: width * resizeRatio, height: height * resizeRatio }
+  const outputSize = { width: width * deviceScaleFactor, height: height * deviceScaleFactor }
 
-  const encoder = new GIFEncoder(outputSize.width, outputSize.height)
   const canvas = createCanvas(outputSize.width, outputSize.height)
   const ctx = canvas.getContext('2d')
 
-  encoder.createWriteStream({ repeat: -1, delay: 0 }).pipe(res)
+  const encoder = new GifEncoder(outputSize.width, outputSize.height)
+  encoder.createReadStream().pipe(res)
 
-  const screencast = createScreencast(page, {
-    quality: 0,
-    format: 'png',
-    everyNthFrame: 1
-  })
+  const screencast = createScreencast(page, { maxWidth: width, maxHeight: height })
 
   screencast.onFrame(async data => {
     const frame = Buffer.from(data, 'base64')
+    const buffer = await sharp(frame).resize(outputSize).toBuffer()
+
     const img = new Image()
-    img.src = await ffmpeg(frame, outputSize)
+    img.src = buffer
     ctx.drawImage(img, 0, 0, img.width, img.height)
     encoder.addFrame(ctx)
+
+    if (firstFrame === true) firstFrame = duration()
+
     lastCanvas = canvas
   })
 
   screencast.start()
   encoder.start()
-  await browserless.goto(page, { url, viewport: { width, height, deviceScaleFactor: 1 } })
+  await browserless.goto(page, { url })
   encoder.finish()
   await screencast.stop()
 
+  console.log(`\n  Resolved ${url}; first frame ${firstFrame}, total ${duration()}`)
+
   CACHE[url] = lastCanvas
 })
 
diff --git a/packages/screencast/package.json b/packages/screencast/package.json
index 06a54d4aaa..6fee24bdfe 100644
--- a/packages/screencast/package.json
+++ b/packages/screencast/package.json
@@ -34,9 +34,11 @@
   ],
   "devDependencies": {
     "@browserless/test": "^10.3.0",
+    "@kikobeats/time-span": "latest",
+    "@skyra/gifenc": "latest",
     "ava": "5",
     "canvas": "latest",
-    "gifencoder": "latest",
+    "sharp": "latest",
     "tinyspawn": "latest"
   },
   "engines": {