feat(replay): Allow to treeshake & configure compression worker URL (#…

…9409) This PR does two things: 1. Allow to configure a `workerUrl` in replay config, which is expected to be an URL of a self-hosted worker script. a. Added an example worker script, which is a built version of the pako-based compression worker a. Users can basically host this file themselves and point to it in `workerUrl`, as long as it is on the same origin as the website itself. a. We can eventually document this in docs 1. Allows to configure `__SENTRY_EXCLUDE_REPLAY_WORKER__` in your build to strip the default included web worker. You can configure this if you're disabling compression anyhow, or if you want to configure a custom web worker as in the above step. Fixes #6739, and allows to reduce bundle size further. Once merged/released we can also add this to the bundler plugins `bundleSizeOptimizations` options. Note that we _do not recommend_ to disable the web worker completely. We only recommend to tree shake the worker code if you provide a custom worker URL - else, replay payloads will not be compressed, resulting in much larger payloads sent over the network, which is bad for your applications performance. Also note that when providing a custom worker, it is your own responsibility to keep it up to date - we try to keep the worker interface stable, and the worker is generally not updated often, but you should still check regularly when updating the SDK if the example worker has changed. --------- Co-authored-by: Billy Vong <billyvg@users.noreply.github.com>
getsentry · Oct 31, 2023 · 2ec3582 · 2ec3582
1 parent 1005925
commit 2ec3582
Show file tree

Hide file tree

Showing 19 changed files with 4,484 additions and 23 deletions.
diff --git a/.size-limit.js b/.size-limit.js
@@ -21,6 +21,7 @@ module.exports = [
           __RRWEB_EXCLUDE_CANVAS__: true,
           __RRWEB_EXCLUDE_SHADOW_DOM__: true,
           __RRWEB_EXCLUDE_IFRAME__: true,
+          __SENTRY_EXCLUDE_REPLAY_WORKER__: true,
         }),
       );
       return config;

diff --git a/packages/browser-integration-tests/suites/replay/compression/subject.js b/packages/browser-integration-tests/suites/replay/compression/subject.js
diff --git a/packages/browser-integration-tests/suites/replay/compression/test.ts b/packages/browser-integration-tests/suites/replay/compression/test.ts
@@ -5,11 +5,12 @@ import { getExpectedReplayEvent } from '../../../utils/replayEventTemplates';
 import {
   getFullRecordingSnapshots,
   getReplayEvent,
+  replayEnvelopeIsCompressed,
   shouldSkipReplayTest,
   waitForReplayRequest,
 } from '../../../utils/replayHelpers';
 
-sentryTest('replay recording should be compressed by default', async ({ getLocalTestPath, page }) => {
+sentryTest('replay recording should be compressed by default', async ({ getLocalTestPath, page, forceFlushReplay }) => {
   if (shouldSkipReplayTest()) {
     sentryTest.skip();
   }
@@ -27,10 +28,16 @@ sentryTest('replay recording should be compressed by default', async ({ getLocal
   const url = await getLocalTestPath({ testDir: __dirname });
 
   await page.goto(url);
-  const replayEvent0 = getReplayEvent(await reqPromise0);
+  await forceFlushReplay();
+
+  const req0 = await reqPromise0;
+
+  const replayEvent0 = getReplayEvent(req0);
   expect(replayEvent0).toEqual(getExpectedReplayEvent());
 
-  const snapshots = getFullRecordingSnapshots(await reqPromise0);
+  expect(replayEnvelopeIsCompressed(req0)).toEqual(true);
+
+  const snapshots = getFullRecordingSnapshots(req0);
   expect(snapshots.length).toEqual(1);
 
   const stringifiedSnapshot = JSON.stringify(snapshots[0]);

diff --git a/packages/browser-integration-tests/suites/replay/compressionDisabled/init.js b/packages/browser-integration-tests/suites/replay/compressionDisabled/init.js
@@ -0,0 +1,18 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+window.Replay = new Sentry.Replay({
+  flushMinDelay: 200,
+  flushMaxDelay: 200,
+  minReplayDuration: 0,
+  useCompression: false,
+});
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  sampleRate: 0,
+  replaysSessionSampleRate: 1.0,
+  replaysOnErrorSampleRate: 0.0,
+
+  integrations: [window.Replay],
+});
diff --git a/packages/browser-integration-tests/suites/replay/compressionDisabled/template.html b/packages/browser-integration-tests/suites/replay/compressionDisabled/template.html
@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+  </head>
+  <body>
+    <button id="go-background">New Tab</button>
+  </body>
+</html>
diff --git a/packages/browser-integration-tests/suites/replay/compressionDisabled/test.ts b/packages/browser-integration-tests/suites/replay/compressionDisabled/test.ts
@@ -0,0 +1,52 @@
+import { expect } from '@playwright/test';
+
+import { sentryTest } from '../../../utils/fixtures';
+import { getExpectedReplayEvent } from '../../../utils/replayEventTemplates';
+import {
+  getFullRecordingSnapshots,
+  getReplayEvent,
+  replayEnvelopeIsCompressed,
+  shouldSkipReplayTest,
+  waitForReplayRequest,
+} from '../../../utils/replayHelpers';
+
+sentryTest(
+  'replay recording should allow to disable compression',
+  async ({ getLocalTestPath, page, forceFlushReplay }) => {
+    if (shouldSkipReplayTest()) {
+      sentryTest.skip();
+    }
+
+    const reqPromise0 = waitForReplayRequest(page, 0);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
+    });
+
+    const url = await getLocalTestPath({ testDir: __dirname });
+
+    await page.goto(url);
+    await forceFlushReplay();
+
+    const req0 = await reqPromise0;
+
+    const replayEvent0 = getReplayEvent(req0);
+    expect(replayEvent0).toEqual(getExpectedReplayEvent());
+
+    expect(replayEnvelopeIsCompressed(req0)).toEqual(false);
+
+    const snapshots = getFullRecordingSnapshots(req0);
+    expect(snapshots.length).toEqual(1);
+
+    const stringifiedSnapshot = JSON.stringify(snapshots[0]);
+    expect(stringifiedSnapshot).toContain('"tagName":"body"');
+    expect(stringifiedSnapshot).toContain('"tagName":"html"');
+    expect(stringifiedSnapshot).toContain('"tagName":"button"');
+    expect(stringifiedSnapshot).toContain('"textContent":"*** ***"');
+    expect(stringifiedSnapshot).toContain('"id":"go-background"');
+  },
+);
diff --git a/packages/browser-integration-tests/suites/replay/compressionWorkerUrl/init.js b/packages/browser-integration-tests/suites/replay/compressionWorkerUrl/init.js
@@ -0,0 +1,19 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+window.Replay = new Sentry.Replay({
+  flushMinDelay: 200,
+  flushMaxDelay: 200,
+  minReplayDuration: 0,
+  useCompression: true,
+  workerUrl: `${window.location.origin}/my-test-worker.js`,
+});
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  sampleRate: 0,
+  replaysSessionSampleRate: 1.0,
+  replaysOnErrorSampleRate: 0.0,
+
+  integrations: [window.Replay],
+});
diff --git a/packages/browser-integration-tests/suites/replay/compressionWorkerUrl/template.html b/packages/browser-integration-tests/suites/replay/compressionWorkerUrl/template.html
@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+  </head>
+  <body>
+    <button id="go-background">New Tab</button>
+  </body>
+</html>
diff --git a/packages/browser-integration-tests/suites/replay/compressionWorkerUrl/test.ts b/packages/browser-integration-tests/suites/replay/compressionWorkerUrl/test.ts
@@ -0,0 +1,66 @@
+import { expect } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+
+import { sentryTest, TEST_HOST } from '../../../utils/fixtures';
+import { getExpectedReplayEvent } from '../../../utils/replayEventTemplates';
+import {
+  getFullRecordingSnapshots,
+  getReplayEvent,
+  replayEnvelopeIsCompressed,
+  shouldSkipReplayTest,
+  waitForReplayRequest,
+} from '../../../utils/replayHelpers';
+
+sentryTest(
+  'replay recording should be compressed if using custom workerUrl',
+  async ({ getLocalTestUrl, page, forceFlushReplay }) => {
+    if (shouldSkipReplayTest()) {
+      sentryTest.skip();
+    }
+
+    const reqPromise0 = waitForReplayRequest(page, 0);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
+    });
+
+    const url = await getLocalTestUrl({ testDir: __dirname });
+
+    let customCompressCalled = 0;
+
+    // Ensure to register this _after_ getLocalTestUrl is called, as that also registers a default route for TEST_HOST
+    await page.route(`${TEST_HOST}/my-test-worker.js`, route => {
+      const filePath = path.resolve(__dirname, '../../../../replay-worker/examples/worker.min.js');
+
+      customCompressCalled++;
+
+      return fs.existsSync(filePath) ? route.fulfill({ path: filePath }) : route.continue();
+    });
+
+    await page.goto(url);
+    await forceFlushReplay();
+
+    const req0 = await reqPromise0;
+
+    const replayEvent0 = getReplayEvent(req0);
+    expect(replayEvent0).toEqual(getExpectedReplayEvent());
+
+    expect(replayEnvelopeIsCompressed(req0)).toEqual(true);
+    expect(customCompressCalled).toBe(1);
+
+    const snapshots = getFullRecordingSnapshots(req0);
+    expect(snapshots.length).toEqual(1);
+
+    const stringifiedSnapshot = JSON.stringify(snapshots[0]);
+    expect(stringifiedSnapshot).toContain('"tagName":"body"');
+    expect(stringifiedSnapshot).toContain('"tagName":"html"');
+    expect(stringifiedSnapshot).toContain('"tagName":"button"');
+    expect(stringifiedSnapshot).toContain('"textContent":"*** ***"');
+    expect(stringifiedSnapshot).toContain('"id":"go-background"');
+  },
+);
diff --git a/packages/browser-integration-tests/utils/replayEventTemplates.ts b/packages/browser-integration-tests/utils/replayEventTemplates.ts
@@ -8,7 +8,7 @@ const DEFAULT_REPLAY_EVENT = {
   timestamp: expect.any(Number),
   error_ids: [],
   trace_ids: [],
-  urls: [expect.stringContaining('/dist/index.html')],
+  urls: [expect.stringContaining('/index.html')],
   replay_id: expect.stringMatching(/\w{32}/),
   replay_start_timestamp: expect.any(Number),
   segment_id: 0,
@@ -31,7 +31,7 @@ const DEFAULT_REPLAY_EVENT = {
     name: 'sentry.javascript.browser',
   },
   request: {
-    url: expect.stringContaining('/dist/index.html'),
+    url: expect.stringContaining('/index.html'),
     headers: {
       'User-Agent': expect.stringContaining(''),
     },

diff --git a/packages/browser-integration-tests/utils/replayHelpers.ts b/packages/browser-integration-tests/utils/replayHelpers.ts
@@ -302,6 +302,38 @@ const replayEnvelopeRequestParser = (request: Request | null, envelopeIndex = 2)
   return envelope[envelopeIndex] as Event;
 };
 
+export function replayEnvelopeIsCompressed(resOrReq: Request | Response): boolean {
+  const request = getRequest(resOrReq);
+
+  // https://develop.sentry.dev/sdk/envelopes/
+  const envelopeBytes = request.postDataBuffer() || '';
+
+  // first, we convert the bugger to string to split and go through the uncompressed lines
+  const envelopeString = envelopeBytes.toString();
+
+  const lines: boolean[] = envelopeString.split('\n').map(line => {
+    try {
+      JSON.parse(line);
+    } catch (error) {
+      // If we fail to parse a line, we _might_ have found a compressed payload,
+      // so let's check if this is actually the case.
+      // This is quite hacky but we can't go through `line` because the prior operations
+      // seem to have altered its binary content. Hence, we take the raw envelope and
+      // look up the place where the zlib compression header(0x78 0x9c) starts
+      for (let i = 0; i < envelopeBytes.length; i++) {
+        if (envelopeBytes[i] === 0x78 && envelopeBytes[i + 1] === 0x9c) {
+          // We found a zlib-compressed payload
+          return true;
+        }
+      }
+    }
+
+    return false;
+  });
+
+  return lines.some(line => line);
+}
+
 export const replayEnvelopeParser = (request: Request | null): unknown[] => {
   // https://develop.sentry.dev/sdk/envelopes/
   const envelopeBytes = request?.postDataBuffer() || '';