From 76673eed42432433cd28605a9efa694a38f347ec Mon Sep 17 00:00:00 2001 From: John McLear Date: Sat, 16 May 2026 06:39:40 +0100 Subject: [PATCH] feat(scaling): engine.io WS transport-level packing (#7756 lever 8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue: engine.io's WebSocket transport sends one WS frame per engine.io packet, even when the engine.io socket has multiple packets buffered (see ether/etherpad#7767). The polling transport already coalesces — `transport.send(packets)` goes through `encodePayload(packets)` and writes one HTTP response containing the whole payload. At high emit rate the WS path is dominated by per-frame syscalls on the server and per-message callback overhead on the client; that's why dropping the polling fallback (lever 4) made things sharply worse. This patch monkey-patches engine.io's WebSocket transport prototype so `send(packets)` with N > 1 packets goes through `encodePayload` and emits ONE WS frame containing the multi-packet payload — the same wire bytes the polling transport already uses. Single-packet sends keep the legacy fast path including the pre-encoded-frame optimisation, so steady-state quiet-pad behaviour is identical to upstream. Gated by settings.enginePacking (default false). Receiving clients must recognise payload-encoded frames (split on `\x1e` and decodePayload). The etherpad-cli-client patch in ether/etherpad-load-test#TBD is forward-compatible; production deployments enabling the flag MUST also ship a forward-compatible browser bundle. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/node/hooks/express/socketio.ts | 10 ++++ src/node/utils/EnginePacking.ts | 86 ++++++++++++++++++++++++++++++ src/node/utils/Settings.ts | 18 +++++++ 3 files changed, 114 insertions(+) create mode 100644 src/node/utils/EnginePacking.ts diff --git a/src/node/hooks/express/socketio.ts b/src/node/hooks/express/socketio.ts index 79ef892760b..2b6187e5b46 100644 --- a/src/node/hooks/express/socketio.ts +++ b/src/node/hooks/express/socketio.ts @@ -69,6 +69,16 @@ const socketSessionMiddleware = (args: any) => (socket: any, next: Function) => }; export const expressCreateServer = (hookName:string, args:ArgsExpressType, cb:Function) => { + // Engine.io WebSocket transport-level packing (ether/etherpad#7756 lever 8). + // Apply BEFORE constructing the socket.io Server so the patched transport + // prototype is in effect when the Server instantiates its engine. + if (settings.enginePacking === true) { + // Require lazily so production deployments that don't set the flag + // don't even load the patch module. + // eslint-disable-next-line @typescript-eslint/no-require-imports + require('../../utils/EnginePacking').installEngineWsPacking(); + } + // init socket.io and redirect all requests to the MessageHandler // there shouldn't be a browser that isn't compatible to all // transports in this list at once diff --git a/src/node/utils/EnginePacking.ts b/src/node/utils/EnginePacking.ts new file mode 100644 index 00000000000..281db4c17b8 --- /dev/null +++ b/src/node/utils/EnginePacking.ts @@ -0,0 +1,86 @@ +// Engine.io WebSocket transport-level packing — #7756 lever 8 prototype. +// +// Issue (see ether/etherpad#7767): engine.io's WebSocket transport sends one +// WS frame per engine.io packet, even when the engine.io socket has multiple +// packets buffered. The polling transport already coalesces — `send(packets)` +// goes through `encodePayload(packets)` and writes one HTTP response +// containing the whole payload. Under high emit rate the WS path is dominated +// by per-frame syscall overhead on the server and per-message callback +// overhead on the client. +// +// This module monkey-patches engine.io's WebSocket transport prototype so +// `send(packets)` with N > 1 packets goes through `encodePayload` and emits +// ONE WS frame containing the multi-packet payload. The frame contents are +// the same wire bytes the polling transport already uses, just delivered as +// a single WebSocket message instead of one frame per packet. +// +// Server-side only. The receiving client (engine.io-client, or anything +// reading the WS frames) must detect the engine.io-parser record separator +// (`\x1e`, U+001E) and call `decodePayload` instead of `decodePacket` when +// it's present. Newly-built clients (browser bundle + etherpad-cli-client +// patched separately) are forward-compatible: a single-packet frame never +// contains a raw `\x1e` (JSON escapes it to ``, and engine.io packet +// type bytes are '0'-'6' or empty for binary). +// +// Gated by settings.enginePacking. Production deployments are not affected +// by default. Enabling it without a forward-compatible client will silently +// break clients that receive a payload-encoded frame. + +import log4js from 'log4js'; + +const logger = log4js.getLogger('engine-packing'); + +let installed = false; + +/** Apply the patch once. Subsequent calls are no-ops. Idempotent so the + * module can be required from multiple boot paths without double-wrapping. */ +export const installEngineWsPacking = (): void => { + if (installed) return; + installed = true; + + let WebSocketTransport: any; + let encodePayload: any; + try { + // Resolve from inside engine.io's own dependency closure so we pick up + // exactly the engine.io-parser the transport uses, not a duplicate copy. + WebSocketTransport = require('engine.io/build/transports/websocket').WebSocket; + encodePayload = require('engine.io-parser').encodePayload; + } catch (err: any) { + logger.warn(`Unable to install engine.io WS packing (modules not found): ${err && err.message || err}`); + return; + } + if (typeof WebSocketTransport !== 'function' || + typeof WebSocketTransport.prototype !== 'object' || + typeof encodePayload !== 'function') { + logger.warn('engine.io shape is unexpected; skipping WS packing patch'); + return; + } + + const originalSend = WebSocketTransport.prototype.send; + + WebSocketTransport.prototype.send = function (packets: any[]) { + // Single-packet sends keep the legacy fast path: per-frame encoding + // including the pre-encoded-frame optimisation. Only fan-out bursts + // (writeBuffer accumulated more than one packet between flushes) are + // packed — for the steady state of a quiet pad, behaviour is identical + // to the upstream implementation. + if (!Array.isArray(packets) || packets.length < 2) { + return originalSend.call(this, packets); + } + + this.writable = false; + const self = this; + encodePayload(packets, (data: string) => { + // Send the whole payload as ONE WS frame and fire the drain/ready + // callbacks the upstream transport sends on the last packet. The + // socket.io socket relies on `drain` to start its next flush. + try { + self.socket.send(data, self._onSentLast); + } catch (err: any) { + self.onError('write error', err && err.stack ? err.stack : err); + } + }); + }; + + logger.info('engine.io WebSocket transport-level packing enabled (#7756 lever 8)'); +}; diff --git a/src/node/utils/Settings.ts b/src/node/utils/Settings.ts index 97413004100..c2478385194 100644 --- a/src/node/utils/Settings.ts +++ b/src/node/utils/Settings.ts @@ -272,6 +272,7 @@ export type SettingsType = { automaticReconnectionTimeout: number, loadTest: boolean, scalingDiveMetrics: boolean, + enginePacking: boolean, dumpOnUncleanExit: boolean, indentationOnNewLine: boolean, logconfig: any | null, @@ -658,6 +659,23 @@ const settings: SettingsType = { * production deployments aren't paying for instrumentation they don't use. */ scalingDiveMetrics: false, + /** + * Pack multiple engine.io packets into a single WebSocket frame + * (ether/etherpad#7756 lever 8). engine.io's WS transport otherwise + * sends one frame per packet, while the polling transport already + * batches via encodePayload. Enabling this matches the polling + * coalescing behaviour on the WS path; at high fan-out rates it cuts + * the WS frame count proportionally to packets-per-flush. + * + * WARNING: enabling this requires connected clients to recognise + * payload-encoded WS frames (split on the `\x1e` record separator + * and decodePayload). Clients that pass each frame straight to + * decodePacket will fail to parse a multi-packet frame and silently + * miss those messages. The etherpad browser bundle and + * etherpad-cli-client are forward-compatible (they detect the + * separator); third-party clients may not be. Coordinate rollouts. + */ + enginePacking: false, /** * Disable dump of objects preventing a clean exit */