From 6ec484d24e7e530f2bb574fa50688d4eab27dae9 Mon Sep 17 00:00:00 2001 From: Alan Barker Date: Thu, 27 Mar 2025 16:26:18 -0400 Subject: [PATCH 1/2] feat: Option to use gzip to compress event --- contract-tests/index.js | 2 + contract-tests/sdkClientEntity.js | 1 + .../__tests__/platform/NodeRequests.test.ts | 151 +++++++++++------- .../server-node/src/platform/NodePlatform.ts | 7 +- .../server-node/src/platform/NodeRequests.ts | 45 ++++-- .../internal/events/EventSender.test.ts | 3 + .../common/src/api/platform/Requests.ts | 5 + .../common/src/internal/events/EventSender.ts | 1 + .../sdk-server/src/api/options/LDOptions.ts | 10 ++ .../sdk-server/src/options/Configuration.ts | 5 + .../src/options/ValidatedOptions.ts | 1 + 11 files changed, 164 insertions(+), 67 deletions(-) diff --git a/contract-tests/index.js b/contract-tests/index.js index 9f1b829dcf..9b6937fab6 100644 --- a/contract-tests/index.js +++ b/contract-tests/index.js @@ -39,6 +39,8 @@ app.get('/', (req, res) => { 'evaluation-hooks', 'wrapper', 'client-prereq-events', + 'event-gzip', + 'optional-event-gzip', ], }); }); diff --git a/contract-tests/sdkClientEntity.js b/contract-tests/sdkClientEntity.js index 7301ffacfa..f3f10fbf2c 100644 --- a/contract-tests/sdkClientEntity.js +++ b/contract-tests/sdkClientEntity.js @@ -45,6 +45,7 @@ export function makeSdkConfig(options, tag) { cf.diagnosticOptOut = !options.events.enableDiagnostics; cf.flushInterval = maybeTime(options.events.flushIntervalMs); cf.privateAttributes = options.events.globalPrivateAttributes; + cf.enableEventCompression = options.events.enableGzip; } if (options.tags) { cf.application = { diff --git a/packages/sdk/server-node/__tests__/platform/NodeRequests.test.ts b/packages/sdk/server-node/__tests__/platform/NodeRequests.test.ts index ab4b7168f8..608c9cdc29 100644 --- a/packages/sdk/server-node/__tests__/platform/NodeRequests.test.ts +++ b/packages/sdk/server-node/__tests__/platform/NodeRequests.test.ts @@ -8,73 +8,76 @@ const TEXT_RESPONSE = 'Test Text'; const JSON_RESPONSE = '{"text": "value"}'; interface TestRequestData { - body: string; + body: string | Buffer; method: string | undefined; headers: http.IncomingHttpHeaders; } -describe('given a default instance of NodeRequests', () => { - let resolve: (value: TestRequestData | PromiseLike) => void; - let promise: Promise; - let server: http.Server; - let resetResolve: () => void; - let resetPromise: Promise; - - beforeEach(() => { - resetPromise = new Promise((res) => { - resetResolve = res; - }); +let resolve: (value: TestRequestData | PromiseLike) => void; +let promise: Promise; +let server: http.Server; +let resetResolve: () => void; +let resetPromise: Promise; - promise = new Promise((res) => { - resolve = res; +beforeEach(() => { + resetPromise = new Promise((res) => { + resetResolve = res; + }); + + promise = new Promise((res) => { + resolve = res; + }); + server = http.createServer({ keepAlive: false }, (req, res) => { + const chunks: any[] = []; + req.on('data', (chunk) => { + chunks.push(chunk); }); - server = http.createServer({ keepAlive: false }, (req, res) => { - const chunks: any[] = []; - req.on('data', (chunk) => { - chunks.push(chunk); - }); - req.on('end', () => { - resolve({ - method: req.method, - body: Buffer.concat(chunks).toString(), - headers: req.headers, - }); + req.on('end', () => { + resolve({ + method: req.method, + body: + req.headers['content-encoding'] === 'gzip' + ? Buffer.concat(chunks) + : Buffer.concat(chunks).toString(), + headers: req.headers, }); + }); + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Connection', 'close'); + if ((req.url?.indexOf('json') || -1) >= 0) { + res.end(JSON_RESPONSE); + } else if ((req.url?.indexOf('interrupt') || -1) >= 0) { + res.destroy(); + } else if ((req.url?.indexOf('404') || -1) >= 0) { + res.statusCode = 404; + res.end(); + } else if ((req.url?.indexOf('reset') || -1) >= 0) { res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.setHeader('Connection', 'close'); - if ((req.url?.indexOf('json') || -1) >= 0) { - res.end(JSON_RESPONSE); - } else if ((req.url?.indexOf('interrupt') || -1) >= 0) { + res.flushHeaders(); + res.write('potato'); + setTimeout(() => { res.destroy(); - } else if ((req.url?.indexOf('404') || -1) >= 0) { - res.statusCode = 404; - res.end(); - } else if ((req.url?.indexOf('reset') || -1) >= 0) { - res.statusCode = 200; - res.flushHeaders(); - res.write('potato'); - setTimeout(() => { - res.destroy(); - resetResolve(); - }, 0); - } else if ((req.url?.indexOf('gzip') || -1) >= 0) { - res.setHeader('Content-Encoding', 'gzip'); - res.end(zlib.gzipSync(Buffer.from(JSON_RESPONSE, 'utf8'))); - } else { - res.end(TEXT_RESPONSE); - } - }); - server.listen(PORT); + resetResolve(); + }, 0); + } else if ((req.url?.indexOf('gzip') || -1) >= 0) { + res.setHeader('Content-Encoding', 'gzip'); + res.end(zlib.gzipSync(Buffer.from(JSON_RESPONSE, 'utf8'))); + } else { + res.end(TEXT_RESPONSE); + } }); + server.listen(PORT); +}); - afterEach( - async () => - new Promise((resolveClose) => { - server.close(resolveClose); - }), - ); +afterEach( + async () => + new Promise((resolveClose) => { + server.close(resolveClose); + }), +); +describe('given a default instance of NodeRequests', () => { const requests = new NodeRequests(); it('can make a basic get request', async () => { const res = await requests.fetch(`http://localhost:${PORT}`); @@ -120,6 +123,17 @@ describe('given a default instance of NodeRequests', () => { expect(serverResult.body).toEqual('BODY TEXT'); }); + it('can make a basic post ignoring compressBodyIfPossible', async () => { + await requests.fetch(`http://localhost:${PORT}`, { + method: 'POST', + body: 'BODY TEXT', + compressBodyIfPossible: true, + }); + const serverResult = await promise; + expect(serverResult.method).toEqual('POST'); + expect(serverResult.body).toEqual('BODY TEXT'); + }); + it('can make a request with headers', async () => { await requests.fetch(`http://localhost:${PORT}`, { method: 'POST', @@ -166,3 +180,30 @@ describe('given a default instance of NodeRequests', () => { expect(serverResult.body).toEqual(''); }); }); + +describe('given an instance of NodeRequests with enableEventCompression turned on', () => { + const requests = new NodeRequests(undefined, undefined, undefined, true); + it('can make a basic post with compressBodyIfPossible enabled', async () => { + await requests.fetch(`http://localhost:${PORT}`, { + method: 'POST', + body: 'BODY TEXT', + compressBodyIfPossible: true, + }); + const serverResult = await promise; + expect(serverResult.method).toEqual('POST'); + expect(serverResult.headers['content-encoding']).toEqual('gzip'); + expect(serverResult.body).toEqual(zlib.gzipSync('BODY TEXT')); + }); + + it('can make a basic post with compressBodyIfPossible disabled', async () => { + await requests.fetch(`http://localhost:${PORT}`, { + method: 'POST', + body: 'BODY TEXT', + compressBodyIfPossible: false, + }); + const serverResult = await promise; + expect(serverResult.method).toEqual('POST'); + expect(serverResult.headers['content-encoding']).toBeUndefined(); + expect(serverResult.body).toEqual('BODY TEXT'); + }); +}); diff --git a/packages/sdk/server-node/src/platform/NodePlatform.ts b/packages/sdk/server-node/src/platform/NodePlatform.ts index 7a5fed5cbd..d72c6e79cd 100644 --- a/packages/sdk/server-node/src/platform/NodePlatform.ts +++ b/packages/sdk/server-node/src/platform/NodePlatform.ts @@ -16,6 +16,11 @@ export default class NodePlatform implements platform.Platform { constructor(options: LDOptions) { this.info = new NodeInfo(options); - this.requests = new NodeRequests(options.tlsParams, options.proxyOptions, options.logger); + this.requests = new NodeRequests( + options.tlsParams, + options.proxyOptions, + options.logger, + options.enableEventCompression, + ); } } diff --git a/packages/sdk/server-node/src/platform/NodeRequests.ts b/packages/sdk/server-node/src/platform/NodeRequests.ts index eb95a64623..69ae26c6af 100644 --- a/packages/sdk/server-node/src/platform/NodeRequests.ts +++ b/packages/sdk/server-node/src/platform/NodeRequests.ts @@ -5,6 +5,8 @@ import { HttpsProxyAgentOptions } from 'https-proxy-agent'; // No types for the event source. // @ts-ignore import { EventSource as LDEventSource } from 'launchdarkly-eventsource'; +import { promisify } from 'util'; +import * as zlib from 'zlib'; import { EventSourceCapabilities, @@ -16,6 +18,8 @@ import { import NodeResponse from './NodeResponse'; +const gzip = promisify(zlib.gzip); + function processTlsOptions(tlsOptions: LDTLSOptions): https.AgentOptions { const options: https.AgentOptions & { [index: string]: any } = { ca: tlsOptions.ca, @@ -101,25 +105,44 @@ export default class NodeRequests implements platform.Requests { private _hasProxyAuth: boolean = false; - constructor(tlsOptions?: LDTLSOptions, proxyOptions?: LDProxyOptions, logger?: LDLogger) { + private _enableEventCompression: boolean = false; + + constructor( + tlsOptions?: LDTLSOptions, + proxyOptions?: LDProxyOptions, + logger?: LDLogger, + enableEventCompression?: boolean, + ) { this._agent = createAgent(tlsOptions, proxyOptions, logger); this._hasProxy = !!proxyOptions; this._hasProxyAuth = !!proxyOptions?.auth; + this._enableEventCompression = !!enableEventCompression; } - fetch(url: string, options: platform.Options = {}): Promise { + async fetch(url: string, options: platform.Options = {}): Promise { const isSecure = url.startsWith('https://'); const impl = isSecure ? https : http; + const headers = { ...options.headers }; + let bodyData: String | Buffer | undefined = options.body; + // For get requests we are going to automatically support compressed responses. // Note this does not affect SSE as the event source is not using this fetch implementation. - const headers = - options.method?.toLowerCase() === 'get' - ? { - ...options.headers, - 'accept-encoding': 'gzip', - } - : options.headers; + if (options.method?.toLowerCase() === 'get') { + headers['accept-encoding'] = 'gzip'; + } + // For post requests we are going to support compressed post bodies if the + // enableEventCompression config setting is true and the compressBodyIfPossible + // option is true. + else if ( + this._enableEventCompression && + !!options.compressBodyIfPossible && + options.method?.toLowerCase() === 'post' && + options.body + ) { + headers['content-encoding'] = 'gzip'; + bodyData = await gzip(Buffer.from(options.body, 'utf8')); + } return new Promise((resolve, reject) => { const req = impl.request( @@ -133,8 +156,8 @@ export default class NodeRequests implements platform.Requests { (res) => resolve(new NodeResponse(res)), ); - if (options.body) { - req.write(options.body); + if (bodyData) { + req.write(bodyData); } req.on('error', (err) => { diff --git a/packages/shared/common/__tests__/internal/events/EventSender.test.ts b/packages/shared/common/__tests__/internal/events/EventSender.test.ts index 63a6130dc8..65c6d06866 100644 --- a/packages/shared/common/__tests__/internal/events/EventSender.test.ts +++ b/packages/shared/common/__tests__/internal/events/EventSender.test.ts @@ -133,6 +133,7 @@ describe('given an event sender', () => { expect(mockFetch).toHaveBeenCalledTimes(1); expect(mockFetch).toHaveBeenCalledWith(`${basicConfig.serviceEndpoints.events}/bulk`, { body: JSON.stringify(testEventData1), + compressBodyIfPossible: true, headers: analyticsHeaders(uuid), method: 'POST', keepalive: true, @@ -150,6 +151,7 @@ describe('given an event sender', () => { expect(mockFetch).toHaveBeenCalledTimes(2); expect(mockFetch).toHaveBeenNthCalledWith(1, `${basicConfig.serviceEndpoints.events}/bulk`, { body: JSON.stringify(testEventData1), + compressBodyIfPossible: true, headers: analyticsHeaders(uuid), method: 'POST', keepalive: true, @@ -159,6 +161,7 @@ describe('given an event sender', () => { `${basicConfig.serviceEndpoints.events}/diagnostic`, { body: JSON.stringify(testEventData2), + compressBodyIfPossible: true, headers: diagnosticHeaders, method: 'POST', keepalive: true, diff --git a/packages/shared/common/src/api/platform/Requests.ts b/packages/shared/common/src/api/platform/Requests.ts index 8b0438d20f..6eea3b8e89 100644 --- a/packages/shared/common/src/api/platform/Requests.ts +++ b/packages/shared/common/src/api/platform/Requests.ts @@ -75,6 +75,11 @@ export interface Options { headers?: Record; method?: string; body?: string; + /** + * Gzip compress the post body only if the underlying SDK framework supports it + * and the config option enableEventCompression is set to true. + */ + compressBodyIfPossible?: boolean; timeout?: number; /** * For use in browser environments. Platform support will be best effort for this field. diff --git a/packages/shared/common/src/internal/events/EventSender.ts b/packages/shared/common/src/internal/events/EventSender.ts index dbb46a4304..0c16752403 100644 --- a/packages/shared/common/src/internal/events/EventSender.ts +++ b/packages/shared/common/src/internal/events/EventSender.ts @@ -64,6 +64,7 @@ export default class EventSender implements LDEventSender { const { status, headers: resHeaders } = await this._requests.fetch(uri, { headers, body: JSON.stringify(events), + compressBodyIfPossible: true, method: 'POST', // When sending events from browser environments the request should be completed even // if the user is navigating away from the page. diff --git a/packages/shared/sdk-server/src/api/options/LDOptions.ts b/packages/shared/sdk-server/src/api/options/LDOptions.ts index 7697839730..f31f644aa2 100644 --- a/packages/shared/sdk-server/src/api/options/LDOptions.ts +++ b/packages/shared/sdk-server/src/api/options/LDOptions.ts @@ -304,4 +304,14 @@ export interface LDOptions { * ``` */ hooks?: Hook[]; + + /** + * Set to true to opt in to compressing event payloads if the SDK supports it, since the + * compression library may not be supported in the underlying SDK framework. If the compression + * library is not supported then event payloads will not be compressed even if this option + * is enabled. + * + * Defaults to false. + */ + enableEventCompression?: boolean; } diff --git a/packages/shared/sdk-server/src/options/Configuration.ts b/packages/shared/sdk-server/src/options/Configuration.ts index 77baf0de39..96d7494143 100644 --- a/packages/shared/sdk-server/src/options/Configuration.ts +++ b/packages/shared/sdk-server/src/options/Configuration.ts @@ -57,6 +57,7 @@ const validations: Record = { application: TypeValidators.Object, payloadFilterKey: TypeValidators.stringMatchingRegex(/^[a-zA-Z0-9](\w|\.|-)*$/), hooks: TypeValidators.createTypeArray('Hook[]', {}), + enableEventCompression: TypeValidators.Boolean, }; /** @@ -82,6 +83,7 @@ export const defaultValues: ValidatedOptions = { diagnosticOptOut: false, diagnosticRecordingInterval: 900, featureStore: () => new InMemoryFeatureStore(), + enableEventCompression: false, }; function validateTypesAndNames(options: LDOptions): { @@ -215,6 +217,8 @@ export default class Configuration { public readonly hooks?: Hook[]; + public readonly enableEventCompression: boolean; + constructor(options: LDOptions = {}, internalOptions: internal.LDInternalOptions = {}) { // The default will handle undefined, but not null. // Because we can be called from JS we need to be extra defensive. @@ -283,5 +287,6 @@ export default class Configuration { } this.hooks = validatedOptions.hooks; + this.enableEventCompression = validatedOptions.enableEventCompression; } } diff --git a/packages/shared/sdk-server/src/options/ValidatedOptions.ts b/packages/shared/sdk-server/src/options/ValidatedOptions.ts index ce9a58de9b..1ec31c6c4a 100644 --- a/packages/shared/sdk-server/src/options/ValidatedOptions.ts +++ b/packages/shared/sdk-server/src/options/ValidatedOptions.ts @@ -41,4 +41,5 @@ export interface ValidatedOptions { [index: string]: any; bigSegments?: LDBigSegmentsOptions; hooks?: Hook[]; + enableEventCompression: boolean; } From 7c107504ce1f495435e39ca8a190560f363de0e3 Mon Sep 17 00:00:00 2001 From: Alan Barker Date: Mon, 7 Apr 2025 16:47:29 -0400 Subject: [PATCH 2/2] chore: rename enableEventCompression to enableBodyCompression for internal usage in NodeRequests --- packages/sdk/server-node/src/platform/NodeRequests.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/sdk/server-node/src/platform/NodeRequests.ts b/packages/sdk/server-node/src/platform/NodeRequests.ts index 69ae26c6af..da7b4931b7 100644 --- a/packages/sdk/server-node/src/platform/NodeRequests.ts +++ b/packages/sdk/server-node/src/platform/NodeRequests.ts @@ -105,7 +105,7 @@ export default class NodeRequests implements platform.Requests { private _hasProxyAuth: boolean = false; - private _enableEventCompression: boolean = false; + private _enableBodyCompression: boolean = false; constructor( tlsOptions?: LDTLSOptions, @@ -116,7 +116,7 @@ export default class NodeRequests implements platform.Requests { this._agent = createAgent(tlsOptions, proxyOptions, logger); this._hasProxy = !!proxyOptions; this._hasProxyAuth = !!proxyOptions?.auth; - this._enableEventCompression = !!enableEventCompression; + this._enableBodyCompression = !!enableEventCompression; } async fetch(url: string, options: platform.Options = {}): Promise { @@ -135,7 +135,7 @@ export default class NodeRequests implements platform.Requests { // enableEventCompression config setting is true and the compressBodyIfPossible // option is true. else if ( - this._enableEventCompression && + this._enableBodyCompression && !!options.compressBodyIfPossible && options.method?.toLowerCase() === 'post' && options.body