From e4e8ec5e97e1366bb9828e5411af2a8c2e120033 Mon Sep 17 00:00:00 2001 From: Clifford Tawiah Date: Mon, 3 Feb 2025 16:47:01 -0600 Subject: [PATCH 1/3] feat: Add support for versioned metrics for AI Configs --- .../__tests__/LDAIConfigTrackerImpl.test.ts | 89 ++++++++++--------- packages/sdk/server-ai/src/LDAIClientImpl.ts | 3 + .../server-ai/src/LDAIConfigTrackerImpl.ts | 4 +- 3 files changed, 51 insertions(+), 45 deletions(-) diff --git a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts index 0c26f00296..655cd30b50 100644 --- a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts @@ -14,25 +14,26 @@ const mockLdClient: LDClientMin = { const testContext: LDContext = { kind: 'user', key: 'test-user' }; const configKey = 'test-config'; const variationKey = 'v1'; +const version = 1; beforeEach(() => { jest.clearAllMocks(); }); it('tracks duration', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); tracker.trackDuration(1000); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1000, ); }); it('tracks duration of async function', async () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); const result = await tracker.trackDurationOf(async () => 'test-result'); @@ -41,61 +42,61 @@ it('tracks duration of async function', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1000, ); }); it('tracks time to first token', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); tracker.trackTimeToFirstToken(1000); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:ttf', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1000, ); }); it('tracks positive feedback', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); tracker.trackFeedback({ kind: LDFeedbackKind.Positive }); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:feedback:user:positive', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); }); it('tracks negative feedback', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); tracker.trackFeedback({ kind: LDFeedbackKind.Negative }); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:feedback:user:negative', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); }); it('tracks success', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); tracker.trackSuccess(); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); }); it('tracks OpenAI usage', async () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); const TOTAL_TOKENS = 100; @@ -113,41 +114,41 @@ it('tracks OpenAI usage', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, COMPLETION_TOKENS, ); }); it('tracks error when OpenAI metrics function throws', async () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); const error = new Error('OpenAI API error'); @@ -160,27 +161,27 @@ it('tracks error when OpenAI metrics function throws', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1000, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); }); it('tracks Bedrock conversation with successful response', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); const TOTAL_TOKENS = 100; const PROMPT_TOKENS = 49; @@ -201,41 +202,41 @@ it('tracks Bedrock conversation with successful response', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 500, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, COMPLETION_TOKENS, ); }); it('tracks Bedrock conversation with error response', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); const response = { $metadata: { httpStatusCode: 400 }, @@ -247,20 +248,20 @@ it('tracks Bedrock conversation with error response', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); }); it('tracks tokens', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); const TOTAL_TOKENS = 100; const PROMPT_TOKENS = 49; @@ -275,27 +276,27 @@ it('tracks tokens', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:total', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, TOTAL_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, PROMPT_TOKENS, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:output', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, COMPLETION_TOKENS, ); }); it('only tracks non-zero token counts', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); tracker.trackTokens({ total: 0, @@ -313,7 +314,7 @@ it('only tracks non-zero token counts', () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:tokens:input', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 50, ); @@ -326,7 +327,7 @@ it('only tracks non-zero token counts', () => { }); it('returns empty summary when no metrics tracked', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); const summary = tracker.getSummary(); @@ -334,7 +335,7 @@ it('returns empty summary when no metrics tracked', () => { }); it('summarizes tracked metrics', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); tracker.trackDuration(1000); tracker.trackTokens({ @@ -362,7 +363,7 @@ it('summarizes tracked metrics', () => { }); it('tracks duration when async function throws', async () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); const error = new Error('test error'); @@ -375,26 +376,26 @@ it('tracks duration when async function throws', async () => { expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:duration:total', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1000, ); }); it('tracks error', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext); + const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); tracker.trackError(); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:generation:error', testContext, - { configKey, variationKey }, + { configKey, variationKey, version }, 1, ); }); diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 965c49ea7d..9d9402b6dd 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -13,6 +13,7 @@ import { LDClientMin } from './LDClientMin'; interface LDMeta { variationKey: string; enabled: boolean; + version: number; } /** @@ -45,6 +46,8 @@ export class LDAIClientImpl implements LDAIClient { key, // eslint-disable-next-line no-underscore-dangle value._ldMeta?.variationKey ?? '', + // eslint-disable-next-line no-underscore-dangle + value._ldMeta?.version ?? 1, context, ); // eslint-disable-next-line no-underscore-dangle diff --git a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts index b7884ff70b..0972a5eee5 100644 --- a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts +++ b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts @@ -13,13 +13,15 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { private _ldClient: LDClientMin, private _configKey: string, private _variationKey: string, + private _version: number, private _context: LDContext, ) {} - private _getTrackData(): { variationKey: string; configKey: string } { + private _getTrackData(): { variationKey: string; configKey: string; version: number } { return { variationKey: this._variationKey, configKey: this._configKey, + version: this._version, }; } From dc4c97f4121328655dc9e985d0ce11a1afb26fbc Mon Sep 17 00:00:00 2001 From: Clifford Tawiah Date: Mon, 3 Feb 2025 16:55:56 -0600 Subject: [PATCH 2/3] fix an issue with prettier --- .../__tests__/LDAIConfigTrackerImpl.test.ts | 128 +++++++++++++++--- 1 file changed, 112 insertions(+), 16 deletions(-) diff --git a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts index 655cd30b50..483d097889 100644 --- a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts @@ -21,7 +21,13 @@ beforeEach(() => { }); it('tracks duration', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); tracker.trackDuration(1000); expect(mockTrack).toHaveBeenCalledWith( @@ -33,7 +39,13 @@ it('tracks duration', () => { }); it('tracks duration of async function', async () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); const result = await tracker.trackDurationOf(async () => 'test-result'); @@ -48,7 +60,13 @@ it('tracks duration of async function', async () => { }); it('tracks time to first token', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); tracker.trackTimeToFirstToken(1000); expect(mockTrack).toHaveBeenCalledWith( @@ -60,7 +78,13 @@ it('tracks time to first token', () => { }); it('tracks positive feedback', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); tracker.trackFeedback({ kind: LDFeedbackKind.Positive }); expect(mockTrack).toHaveBeenCalledWith( @@ -72,7 +96,13 @@ it('tracks positive feedback', () => { }); it('tracks negative feedback', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); tracker.trackFeedback({ kind: LDFeedbackKind.Negative }); expect(mockTrack).toHaveBeenCalledWith( @@ -84,7 +114,13 @@ it('tracks negative feedback', () => { }); it('tracks success', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); tracker.trackSuccess(); expect(mockTrack).toHaveBeenCalledWith( @@ -96,7 +132,13 @@ it('tracks success', () => { }); it('tracks OpenAI usage', async () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); const TOTAL_TOKENS = 100; @@ -148,7 +190,13 @@ it('tracks OpenAI usage', async () => { }); it('tracks error when OpenAI metrics function throws', async () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); const error = new Error('OpenAI API error'); @@ -181,7 +229,13 @@ it('tracks error when OpenAI metrics function throws', async () => { }); it('tracks Bedrock conversation with successful response', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); const TOTAL_TOKENS = 100; const PROMPT_TOKENS = 49; @@ -236,7 +290,13 @@ it('tracks Bedrock conversation with successful response', () => { }); it('tracks Bedrock conversation with error response', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); const response = { $metadata: { httpStatusCode: 400 }, @@ -261,7 +321,13 @@ it('tracks Bedrock conversation with error response', () => { }); it('tracks tokens', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); const TOTAL_TOKENS = 100; const PROMPT_TOKENS = 49; @@ -296,7 +362,13 @@ it('tracks tokens', () => { }); it('only tracks non-zero token counts', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); tracker.trackTokens({ total: 0, @@ -327,7 +399,13 @@ it('only tracks non-zero token counts', () => { }); it('returns empty summary when no metrics tracked', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); const summary = tracker.getSummary(); @@ -335,7 +413,13 @@ it('returns empty summary when no metrics tracked', () => { }); it('summarizes tracked metrics', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); tracker.trackDuration(1000); tracker.trackTokens({ @@ -363,7 +447,13 @@ it('summarizes tracked metrics', () => { }); it('tracks duration when async function throws', async () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000); const error = new Error('test error'); @@ -382,7 +472,13 @@ it('tracks duration when async function throws', async () => { }); it('tracks error', () => { - const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext); + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); tracker.trackError(); expect(mockTrack).toHaveBeenCalledWith( From 88ed4d46a54450b8a13dd1dd8ffd2e56dc3020ea Mon Sep 17 00:00:00 2001 From: Cliff Tawiah <82856282+ctawiah@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:29:48 -0600 Subject: [PATCH 3/3] Update packages/sdk/server-ai/src/LDAIClientImpl.ts Co-authored-by: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> --- packages/sdk/server-ai/src/LDAIClientImpl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 9d9402b6dd..bca8431cce 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -13,7 +13,7 @@ import { LDClientMin } from './LDClientMin'; interface LDMeta { variationKey: string; enabled: boolean; - version: number; + version?: number; } /**