feature(crx-tests): run tests inside chrome extension

We send the test body to run in a chrome extension service worker.

For now, only tests depending on basic server info and page are supported (all others are skipped).

Also, if the test closure includes external objects other that `test` and `except`, it will fail.

packages/crx/package.json

@@ -69,6 +69,7 @@
     "@types/stack-utils": "^2.0.1",
     "@types/ws": "8.2.2",
     "@types/yazl": "^2.4.2",
+    "vite-plugin-inspect": "^0.7.28",
     "vite-plugin-node-polyfills": "^0.8.2"
   }
 }

packages/crx/src/background.ts

@@ -22,9 +22,6 @@ import { setUnderTest } from '@playwright-core/utils';
 import { runTest } from './test/runTest';
 import { expect } from '@playwright-test/matchers/expect';
 
-// @ts-ignore
-self.runTest = runTest; self.expect = expect;
-
 chrome.action.onClicked.addListener(async () => {
   runTest({ server: {} } as any, async ({ page }) => {
     await page.goto('https://demo.playwright.dev/todomvc/');
@@ -34,7 +31,6 @@ chrome.action.onClicked.addListener(async () => {
     await page.getByRole('link', { name: 'All' }).click();
     await page.getByRole('checkbox', { name: 'Toggle Todo' }).check();
     await page.getByTestId('todo-title').click();
-
     await expect(page.getByPlaceholder('What needs to be done?')).toContainText('World');
   });
 });

packages/crx/src/shims/transform.ts

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const noop = () => {};
+
+export const wrapFunctionWithLocation = noop;

packages/crx/src/shims/util.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-nocheck
+
+import { promisify as _promisify, inspect, types, isString, inherits, format } from '_util';
+export { inspect, types, isString, inherits, format } from '_util';
+
+export const promisify = function(first, ...rest) {
+  if (first !== 'function') return Promise.resolve();
+  return _promisify(first, ...rest);
+};
+
+export default { promisify, inspect, types, isString, inherits, format };

packages/crx/src/test/runTest.ts

@@ -20,6 +20,17 @@
 import type { PageTestFixtures } from 'tests/page/pageTestApi';
 import { getOrCreatePage } from '../crx/crxPlaywright';
 import { expect } from '@playwright-test/matchers/expect';
+import { rootTestType } from '@playwright-test/common/testType';
+import { setCurrentTestInfo, setCurrentlyLoadingFileSuite } from '@playwright-test/common/globals';
+import { Suite } from '@playwright-test/common/test';
+import { TestInfoImpl } from '@playwright-test/worker/testInfo';
+import type { FullConfigInternal, FullProjectInternal } from '@playwright-test/common/config';
+import type { SerializedConfig } from '@playwright-test/common/ipc';
+
+const test = rootTestType.test;
+
+// @ts-ignore
+self.runTest = runTest; self.expect = expect; self.test = test;
 
 type LightServerFixtures = {
   server: {
@@ -39,11 +50,41 @@ export async function runTest(serverFixtures: LightServerFixtures, fn: (fixtures
   });
 
   if (!tab.id) throw new Error(`Failed to create a new tab`);
-  const page = await getOrCreatePage(tab.id!);
 
-  const fixtures = { ...serverFixtures, tab, page };
-  await fn(fixtures);
+  const suite = new Suite('test', 'file');
+
+
+  try {
+    setCurrentlyLoadingFileSuite(suite);
+    test('test', fn);
+    setCurrentlyLoadingFileSuite(undefined);
+
+    const [testCase] = suite.tests;
+    const noop = () => {};
+    const testInfo = new TestInfoImpl(
+        { config: {} } as unknown as FullConfigInternal,
+        { project: { snapshotDir: '.', testDir: '.', outputDir: '.' } } as FullProjectInternal,
+        { workerIndex: 0, parallelIndex: 0, projectId: 'crx', repeatEachIndex: 0, config: { } as SerializedConfig },
+        testCase,
+        0,
+        noop,
+        noop,
+        noop,
+    );
+    setCurrentTestInfo(testInfo);
+
+    const page = await getOrCreatePage(tab.id!);
+
+    const fixtures = { ...serverFixtures, tab, page };
+    await fn(fixtures);
+
+    await page.close();
+    await chrome.tabs.remove(tab.id);
+
+  } finally {
+    setCurrentTestInfo(null);
 
-  await page.close();
-  await chrome.tabs.remove(tab.id);
+    // just to ensure we don't leak
+    setCurrentlyLoadingFileSuite(undefined);
+  }
 }

packages/crx/vite.config.ts

@@ -28,10 +28,10 @@ export default defineConfig({
       'playwright-core/lib/zipBundle': path.resolve(__dirname, '../playwright-core/bundles/zip/src/zipBundleImpl'),
       // more generic alias must be below specific ones
       'playwright-core/lib': path.resolve(__dirname, '../playwright-core/src'),
-      'playwright-test/lib/common/expectBundle': path.resolve(__dirname, '../playwright-test/bundles/expect/src/expectBundleImpl'),
-      'playwright-test/lib/utilsBundle': path.resolve(__dirname, '../playwright-test/bundles/utils/src/utilsBundleImpl'),
-      // more generic alias must be below specific ones
       'playwright-test/lib': path.resolve(__dirname, '../playwright-test/src'),
+      // we must keep this one relative to avoid problems with tests
+      '../common/expectBundle': path.resolve(__dirname, '../playwright-test/bundles/expect/src/expectBundleImpl'),
+      '../utilsBundle': path.resolve(__dirname, '../playwright-test/bundles/utils/src/utilsBundleImpl'),
 
       'child_process': path.resolve(__dirname, './src/shims/child_process'),
       'dns': path.resolve(__dirname, './src/shims/dns'),
@@ -55,13 +55,16 @@ export default defineConfig({
       'https': 'https-browserify',
       'os': 'os-browserify/browser',
       'url': 'url',
-      'util': 'util',
+      '_util': path.resolve(__dirname, '../../node_modules/util'),
+      'util/': path.resolve(__dirname, './src/shims/util'),
+      'util': path.resolve(__dirname, './src/shims/util'),
       'zlib': 'browserify-zlib',
     },
   },
   define: {
     // we need this one because of PLAYWRIGHT_CORE_PATH (it checks the actual version of playwright-core)
     'require.resolve': '((s) => s)',
+    'process.geteuid': '(() => "crx")',
     'process.platform': '"browser"',
     'process.versions.node': '"18.16"',
   },

packages/playwright-test/src/worker/crxTestRunner.ts

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Worker } from 'playwright-core';
+import { getRequiredFixtureNames } from './fixtureRunner';
+import type { TestInfoImpl } from './testInfo';
+
+export default class CrxTestRunner {
+  private _testInfo: TestInfoImpl;
+
+  constructor(testInfo: TestInfoImpl) {
+    this._testInfo = testInfo;
+  }
+
+  isCrxTest() {
+    return process.env.PWPAGE_IMPL === 'crx';
+  }
+
+  isToSkip() {
+    const names = getRequiredFixtureNames(this._testInfo.fn);
+    return !names.every(name => ['page', 'server', 'browserName'].includes(name));
+  }
+
+  async run(testFunctionParams: object | null) {
+    const { page, server: serverObj, browserName } = testFunctionParams as any;
+    const worker = page?.extensionServiceWorker as Worker;
+
+    if (!worker) throw new Error(`could not find extensionServiceWorker0`);
+
+    let server;
+    if (serverObj) {
+      const { PORT, PREFIX, CROSS_PROCESS_PREFIX, EMPTY_PAGE } = serverObj;
+      server = { PORT, PREFIX, CROSS_PROCESS_PREFIX, EMPTY_PAGE };
+    }
+    const fn = this._testInfo.fn;
+    const fnBody = fn.toString()
+        .replaceAll(/\w+\.expect/g, 'expect')
+        .replaceAll(/\_\w+Test\.test/g, 'test');
+    await worker.evaluate(new Function(`return async (fixtures) => {
+      const { test, expect } = self;
+      await runTest(fixtures, ${fnBody});
+    }`)(), { server, browserName });
+  }
+}

packages/playwright-test/src/worker/fixtureRunner.ts

@@ -297,7 +297,7 @@ export class FixtureRunner {
   }
 }
 
-function getRequiredFixtureNames(fn: Function, location?: Location) {
+export function getRequiredFixtureNames(fn: Function, location?: Location) {
   return fixtureParameterNames(fn, location ?? { file: '<unknown>', line: 1, column: 1 }, e => {
     throw new Error(`${formatLocation(e.location!)}: ${e.message}`);
   });

packages/playwright-test/src/worker/workerMain.ts

@@ -31,7 +31,7 @@ import { loadTestFile } from '../common/testLoader';
 import { buildFileSuiteForProject, filterTestsRemoveEmptySuites } from '../common/suiteUtils';
 import { PoolBuilder } from '../common/poolBuilder';
 import type { TestInfoError } from '../../types/test';
-import type {  Worker } from 'playwright-core';
+import CrxTestRunner from './crxTestRunner';
 
 type WorkerMainEventMap = {
   stdOut: TestOutputPayload,
@@ -281,6 +281,10 @@ export class WorkerMain extends ProcessRunner<WorkerMainEventMap> {
       }
     };
 
+    const crxTestRunner = new CrxTestRunner(testInfo);
+    if (crxTestRunner.isCrxTest() && crxTestRunner.isToSkip())
+      testInfo.expectedStatus = 'skipped';
+
     if (!this._isStopped)
       this._fixtureRunner.setPool(test._pool!);
 
@@ -385,14 +389,11 @@ export class WorkerMain extends ProcessRunner<WorkerMainEventMap> {
       await testInfo._runAndFailOnError(async () => {
         // Now run the test itself.
         debugTest(`test function started`);
-        const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]").
 
-        if (process.env.PWPAGE_IMPL === 'crx') {
-          const worker = (testFunctionParams as any).page.extensionServiceWorker as Worker;
-          await worker.evaluate(new Function(`return async (fixtures) => {
-            await runTest(fixtures, ${fn.toString().replaceAll(/(\w+)\.expect/g, 'expect')});
-          }`)(), { server: {} });
+        if (crxTestRunner.isCrxTest()) {
+          await crxTestRunner.run(testFunctionParams);
         } else {
+          const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]").
           await fn(testFunctionParams, testInfo);
         }
         debugTest(`test function finished`);

tests/config/browserTest.ts

@@ -86,6 +86,7 @@ const test = baseTest.extend<BrowserTestTestFixtures, BrowserTestWorkerFixtures>
   isAndroid: [false, { scope: 'worker' }],
   isElectron: [false, { scope: 'worker' }],
   isWebView2: [false, { scope: 'worker' }],
+  isCrx: [({ }, run) => run(process.env.PWPAGE_IMPL === 'crx'), { scope: 'worker' }],
 
   contextFactory: async ({ _contextFactory }: any, run) => {
     await run(_contextFactory);

tests/page/locator-frame.spec.ts

@@ -18,6 +18,8 @@
 import type { Page } from 'playwright-core';
 import { test as it, expect } from './pageTest';
 
+it.skip(({ isCrx }) => isCrx);
+
 async function routeIframe(page: Page) {
   await page.route('**/empty.html', route => {
     route.fulfill({

tests/page/pageTestApi.ts

@@ -35,4 +35,5 @@ export type PageWorkerFixtures = {
   isAndroid: boolean;
   isElectron: boolean;
   isWebView2: boolean;
+  isCrx: boolean;
 };