feature(cli): add custom inspector proxy based on `metro-inspector-pr…

…oxy` (#21449)

# Why

Fixes ENG-7467

Related #21265

This is an initial draft to extend the CDP functionality of
`metro-inspector-proxy`.

# How

The implementation is slightly wonky around the `ExpoInspectorDevice`.
We want to reuse as much as possible from `metro-inspector-proxy`, but
we need to add stateful data per device.

In order to achieve that, we generate a new class type, based on the
user's installed `metro-inspector-proxy`. This makes everything less
readable but should include future updates in these classes.

As for the `ExpoInspectorProxy`, to avoid having to do the same thing,
we just wrap the whole inspector class and reuse what we can. The device
map is "linked" within the original inspector proxy instance, making the
data available to all methods that need it.

# Test Plan

Enable this feature with `EXPO_USE_CUSTOM_INSPECTOR_PROXY=1`

- [x] See tests for the actual CDP events we handle.
- [ ] See tests on the "bootstrapping code" to create the inspector and
devices.

# Checklist

<!--
Please check the appropriate items below if they apply to your diff.
This is required for changes to Expo modules.
-->

- [ ] Documentation is up to date to reflect these changes (eg:
https://docs.expo.dev and README.md).
- [ ] Conforms with the [Documentation Writing Style
Guide](https://github.com/expo/expo/blob/main/guides/Expo%20Documentation%20Writing%20Style%20Guide.md)
- [ ] This diff will work correctly for `expo prebuild` & EAS Build (eg:
updated a module plugin).

---------

Co-authored-by: Evan Bacon <bacon@expo.io>

docs/pages/workflow/expo-cli.mdx

@@ -369,6 +369,7 @@ From here, you can choose to generate basic project files like:
 - `EXPO_TUNNEL_SUBDOMAIN` (**boolean**) **Experimental** disable using `exp.direct` as the hostname for `--tunnel` connections. This enables **https://** forwarding which can be used to test universal links on iOS. This may cause unexpected issues with `expo-linking` and Expo Go. Select the exact subdomain to use by passing a `string` value that is not one of: `true`, `false`, `1`, `0`.
 - `EXPO_METRO_NO_MAIN_FIELD_OVERRIDE` (**boolean**) force Expo CLI to use the [`resolver.resolverMainFields`](https://facebook.github.io/metro/docs/configuration/#resolvermainfields) from the project's **metro.config.js** for all platforms. By default, Expo CLI will use `['browser', 'module', 'main']`, which is the default for webpack, for the web and the user-defined main fields for other platforms.
 - `EXPO_USE_PATH_ALIASES` (**boolean**) **Experimental:** _SDK 49+_ Allow Metro to use the `compilerOptions.paths` and `compilerOptions.baseUrl` features from `tsconfig.json` (or `jsconfig.json`) to enable import aliases and absolute imports.
+- `EXPO_USE_CUSTOM_INSPECTOR_PROXY` (**boolean**) **Experimental:** _SDK 49+_ Use a customized inspector proxy with improved support for the Chrome DevTools protocol. This includes support for the network inspector.
 
 ## Telemetry
 

packages/@expo/cli/CHANGELOG.md

@@ -15,6 +15,7 @@
 - Reduce install prompt. ([#21264](https://github.com/expo/expo/pull/21264) by [@EvanBacon](https://github.com/EvanBacon))
 - Improve multi-target iOS scheme resolution for `expo run:ios`. ([#21240](https://github.com/expo/expo/pull/21240) by [@EvanBacon](https://github.com/EvanBacon))
 - Added experimental react-devtools integration. ([#21462](https://github.com/expo/expo/pull/21462) by [@kudo](https://github.com/kudo))
+- Add experimental inspector proxy to handle more CDP requests. ([#21449](https://github.com/expo/expo/pull/21449) by [@byCedric](https://github.com/byCedric))
 
 ### 🐛 Bug fixes
 

packages/@expo/cli/package.json

@@ -133,6 +133,8 @@
     "@types/webpack": "~4.41.32",
     "@types/webpack-dev-server": "3.11.0",
     "@types/wrap-ansi": "^8.0.1",
+    "@types/ws": "^8.5.4",
+    "devtools-protocol": "^0.0.1113120",
     "expo-module-scripts": "^3.0.0",
     "klaw-sync": "^6.0.0",
     "nock": "~13.2.2",

packages/@expo/cli/src/start/server/metro/inspector-proxy/__tests__/device.test.ts

@@ -0,0 +1,122 @@
+import { createInspectorDeviceClass } from '../device';
+import { InspectorHandler } from '../messages/types';
+
+describe('ExpoInspectorDevice', () => {
+  it('initializes with default handlers', () => {
+    const { device } = createTestDevice();
+    expect(device.handlers).toHaveLength(1);
+  });
+
+  describe('device', () => {
+    it('handles known device messages', () => {
+      const { device, MetroDevice } = createTestDevice();
+      const handler: InspectorHandler = { onDeviceMessage: jest.fn().mockReturnValue(true) };
+      device.handlers = [handler];
+
+      device._processMessageFromDevice(
+        { method: 'Network.responseReceived', params: { requestId: 420 } },
+        jest.fn() // debugger info mock
+      );
+
+      expect(handler.onDeviceMessage).toBeCalled();
+      // Expect the message is NOT propagated to original handlers
+      expect(MetroDevice.prototype._processMessageFromDevice).not.toBeCalled();
+    });
+
+    it('does not handle unknown device messages', () => {
+      const { device, MetroDevice } = createTestDevice();
+      const handler: InspectorHandler = { onDeviceMessage: jest.fn().mockReturnValue(false) };
+      device.handlers = [handler];
+
+      device._processMessageFromDevice(
+        { method: 'Network.responseReceived', params: { requestId: 420 } },
+        jest.fn() // debugger info mock
+      );
+
+      expect(handler.onDeviceMessage).toBeCalled();
+      // Expect the message is propagated to original handlers
+      expect(MetroDevice.prototype._processMessageFromDevice).toBeCalled();
+    });
+
+    it('does not handle without handlers', () => {
+      const { device, MetroDevice } = createTestDevice();
+      device.handlers = [];
+
+      device._processMessageFromDevice(
+        { method: 'Network.responseReceived', params: { requestId: 420 } },
+        jest.fn() // debugger info mock
+      );
+
+      // Expect the message is propagated to original handlers
+      expect(MetroDevice.prototype._processMessageFromDevice).toBeCalled();
+    });
+  });
+
+  describe('debugger', () => {
+    it('intercepts known debugger messages', () => {
+      const { device, MetroDevice } = createTestDevice();
+      const handler: InspectorHandler = { onDebuggerMessage: jest.fn().mockReturnValue(true) };
+      device.handlers = [handler];
+
+      const handled = device._interceptMessageFromDebugger(
+        { id: 420, method: 'Network.getResponseBody', params: { requestId: 420 } },
+        jest.fn(), // debugger info mock
+        jest.fn() as any // socket mock
+      );
+
+      expect(handled).toBe(true);
+      expect(handler.onDebuggerMessage).toBeCalled();
+      // Expect the message is NOT propagated to original handlers
+      expect(MetroDevice.prototype._interceptMessageFromDebugger).not.toBeCalled();
+    });
+
+    it('does not intercept unknown debugger messages', () => {
+      const { device, MetroDevice } = createTestDevice();
+      const handler: InspectorHandler = { onDebuggerMessage: jest.fn().mockReturnValue(false) };
+      device.handlers = [handler];
+
+      const handled = device._interceptMessageFromDebugger(
+        { id: 420, method: 'Network.getResponseBody', params: { requestId: 420 } },
+        jest.fn(), // debugger info mock
+        jest.fn() as any // socket mock
+      );
+
+      expect(handled).not.toBe(true);
+      expect(handler.onDebuggerMessage).toBeCalled();
+      // Expect the message is propagated to original handlers
+      expect(MetroDevice.prototype._interceptMessageFromDebugger).toBeCalled();
+    });
+
+    it('does not intercept without handlers', () => {
+      const { device, MetroDevice } = createTestDevice();
+      device.handlers = [];
+
+      const handled = device._interceptMessageFromDebugger(
+        { id: 420, method: 'Network.getResponseBody', params: { requestId: 420 } },
+        jest.fn(), // debugger info mock
+        jest.fn() as any // socket mock
+      );
+
+      expect(handled).not.toBe(true);
+      // Expect the message is propagated to original handlers
+      expect(MetroDevice.prototype._interceptMessageFromDebugger).toBeCalled();
+    });
+  });
+});
+
+/** Create a test device instance without extending the Metro device */
+function createTestDevice() {
+  class MetroDevice {
+    _processMessageFromDevice() {}
+    _interceptMessageFromDebugger() {}
+  }
+
+  // Dynamically replace these functions with mocks, doesn't work from class declaration
+  MetroDevice.prototype._processMessageFromDevice = jest.fn();
+  MetroDevice.prototype._interceptMessageFromDebugger = jest.fn();
+
+  const ExpoDevice = createInspectorDeviceClass(MetroDevice);
+  const device = new ExpoDevice();
+
+  return { ExpoDevice, MetroDevice, device };
+}

packages/@expo/cli/src/start/server/metro/inspector-proxy/__tests__/proxy.test.ts

@@ -0,0 +1,178 @@
+import connect from 'connect';
+import http from 'http';
+import { InspectorProxy as MetroProxy } from 'metro-inspector-proxy';
+import fetch from 'node-fetch';
+import { parse } from 'url';
+import WS from 'ws';
+
+import { ExpoInspectorProxy as ExpoProxy } from '../proxy';
+
+it('shares devices with metro proxy', () => {
+  const { expoProxy, metroProxy } = createTestProxy();
+  expect(metroProxy._devices).toBe(expoProxy.devices);
+});
+
+it('responds to `/json` and `/json/list` endpoint', async () => {
+  const { expoProxy } = createTestProxy();
+  const app = connect();
+  const { server, serverUrl } = await createTestServer(app);
+
+  app.use(expoProxy.processRequest);
+
+  try {
+    const [jsonResponse, listResponse] = await Promise.all([
+      fetch(`${serverUrl}/json`),
+      fetch(`${serverUrl}/json/list`),
+    ]);
+
+    expect(jsonResponse.ok).toBe(true);
+    expect(listResponse.ok).toBe(true);
+    expect(await jsonResponse.json()).toBeDefined();
+    expect(await listResponse.json()).toBeDefined();
+  } finally {
+    server.close();
+  }
+});
+
+it('creates websocket listeners for device and debugger', async () => {
+  const { expoProxy } = createTestProxy();
+  const { server } = await createTestServer();
+
+  const listeners = expoProxy.createWebSocketListeners(server);
+  server.close();
+
+  expect(listeners['/inspector/device']).toBeInstanceOf(WS.Server);
+  expect(listeners['/inspector/debug']).toBeInstanceOf(WS.Server);
+});
+
+// The sourcemap references are relying on the server address
+// Without proper failure, this could lead to unexpected sourcemap issues
+it('fails when creating websocket listeners without server address', async () => {
+  const { expoProxy } = createTestProxy();
+  const { server } = await createTestServer();
+
+  await new Promise<void>((resolve, reject) => {
+    server.close((error) => (error ? reject(error) : resolve()));
+  });
+
+  expect(() => expoProxy.createWebSocketListeners(server)).toThrowError(
+    'could not resolve the server address'
+  );
+});
+
+it('creates a new device when a client connects', async () => {
+  const { expoProxy } = createTestProxy();
+  const { server, deviceWebSocketUrl } = await createTestServer();
+  useWebsockets(server, expoProxy.createWebSocketListeners(server));
+
+  const device = new WS(deviceWebSocketUrl);
+
+  try {
+    await new Promise<void>((resolve) => device.on('open', resolve));
+
+    expect(device.readyState).toBe(device.OPEN);
+    expect(expoProxy.devices.size).toBe(1);
+  } finally {
+    server.close();
+    device.close();
+  }
+});
+
+it('removes device when client disconnects', async () => {
+  const { expoProxy } = createTestProxy();
+  const { server, deviceWebSocketUrl } = await createTestServer();
+  useWebsockets(server, expoProxy.createWebSocketListeners(server));
+
+  const device = new WS(deviceWebSocketUrl);
+
+  try {
+    await new Promise<void>((resolve) => device.on('open', resolve));
+    expect(expoProxy.devices.size).toBe(1);
+
+    device.close();
+
+    await new Promise<void>((resolve) => device.on('close', resolve));
+    expect(expoProxy.devices.size).toBe(0);
+  } finally {
+    server.close();
+    device.close();
+  }
+});
+
+it('accepts debugger connections when device is connected', async () => {
+  const { expoProxy } = createTestProxy();
+  const { server, deviceWebSocketUrl, debuggerWebSocketUrl } = await createTestServer();
+  useWebsockets(server, expoProxy.createWebSocketListeners(server));
+
+  let deviceWs: WS | null = null;
+  let debuggerWs: WS | null = null;
+
+  try {
+    deviceWs = new WS(deviceWebSocketUrl);
+    await new Promise<void>((resolve) => deviceWs?.on('open', resolve));
+
+    const device = expoProxy.devices.values().next().value;
+    expect(device).toBeDefined();
+
+    const deviceDebugHandler = jest.spyOn(device, 'handleDebuggerConnection');
+
+    debuggerWs = new WS(`${debuggerWebSocketUrl}?device=${device.id}&page=1`);
+    await new Promise<void>((resolve) => debuggerWs?.on('open', resolve));
+
+    expect(debuggerWs.readyState).toBe(debuggerWs.OPEN);
+    expect(deviceDebugHandler).toBeCalled();
+  } finally {
+    server.close();
+    deviceWs?.close();
+    debuggerWs?.close();
+  }
+});
+
+function createTestProxy() {
+  class ExpoDevice {
+    constructor(public readonly id: number) {}
+    handleDebuggerConnection() {}
+  }
+
+  const metroProxy = new MetroProxy();
+  const expoProxy = new ExpoProxy(metroProxy, ExpoDevice);
+
+  return { ExpoDevice, metroProxy, expoProxy };
+}
+
+async function createTestServer(app?: http.RequestListener) {
+  const server = http.createServer(app);
+
+  await new Promise<void>((resolve, reject) => {
+    server.listen((error) => (error ? reject(error) : resolve()));
+  });
+
+  const address = server.address();
+  if (!address || typeof address === 'string') {
+    throw new Error('Test server has no proper address');
+  }
+
+  const serverLocation =
+    address.family === 'IPv6'
+      ? `[${address.address}]:${address.port}`
+      : `${address.address}:${address.port}`;
+
+  return {
+    server,
+    serverUrl: `http://${serverLocation}`,
+    deviceWebSocketUrl: `ws://${serverLocation}/inspector/device`,
+    debuggerWebSocketUrl: `ws://${serverLocation}/inspector/debug`,
+  };
+}
+
+function useWebsockets(server: http.Server, websockets: Record<string, WS.Server>) {
+  server.on('upgrade', (request, socket, head) => {
+    const { pathname } = parse(request.url!);
+
+    if (pathname !== null && websockets[pathname]) {
+      websockets[pathname].handleUpgrade(request, socket, head, (ws) => {
+        websockets[pathname].emit('connection', ws, request);
+      });
+    }
+  });
+}

packages/@expo/cli/src/start/server/metro/inspector-proxy/device.ts

@@ -0,0 +1,41 @@
+import type { DebuggerInfo, Device as MetroDevice } from 'metro-inspector-proxy';
+import type WS from 'ws';
+
+import { NetworkResponseHandler } from './handlers/NetworkResponse';
+import { DeviceRequest, InspectorHandler, DebuggerRequest } from './handlers/types';
+
+export function createInspectorDeviceClass(MetroDeviceClass: typeof MetroDevice) {
+  return class ExpoInspectorDevice extends MetroDeviceClass implements InspectorHandler {
+    /** All handlers that should be used to intercept or reply to CDP events */
+    public handlers: InspectorHandler[] = [new NetworkResponseHandler()];
+
+    onDeviceMessage(message: any, info: DebuggerInfo): boolean {
+      return this.handlers.some((handler) => handler.onDeviceMessage?.(message, info) ?? false);
+    }
+
+    onDebuggerMessage(message: any, info: DebuggerInfo): boolean {
+      return this.handlers.some((handler) => handler.onDebuggerMessage?.(message, info) ?? false);
+    }
+
+    /** Hook into the message life cycle to answer more complex CDP messages */
+    async _processMessageFromDevice(message: DeviceRequest<any>, info: DebuggerInfo) {
+      if (!this.onDeviceMessage(message, info)) {
+        await super._processMessageFromDevice(message, info);
+      }
+    }
+
+    /** Hook into the message life cycle to answer more complex CDP messages */
+    _interceptMessageFromDebugger(
+      request: DebuggerRequest,
+      info: DebuggerInfo,
+      socket: WS
+    ): boolean {
+      // Note, `socket` is the exact same as `info.socket`
+      if (this.onDebuggerMessage(request, info)) {
+        return true;
+      }
+
+      return super._interceptMessageFromDebugger(request, info, socket);
+    }
+  };
+}