From cc4a975a3f020648d183ce620e244ba0a09c286d Mon Sep 17 00:00:00 2001 From: Bob Evans Date: Mon, 18 Dec 2023 15:13:16 -0500 Subject: [PATCH] feat: removed api.setLlmMetadata(#1918) * It will be assigned via api.addCustomAttribute with a prefix of `llm.` * The conversation_id will now be assigned from `llm.conversation_id` custom attribute --- api.js | 24 ----------------- lib/llm-events/openai/event.js | 2 +- test/unit/api/api-llm.test.js | 25 ----------------- test/unit/api/stub.test.js | 2 +- .../openai/chat-completion-message.test.js | 2 +- .../openai/chat-completion-summary.test.js | 4 +-- test/versioned/openai/chat-completions.tap.js | 27 ------------------- test/versioned/openai/embeddings.tap.js | 27 ------------------- 8 files changed, 5 insertions(+), 108 deletions(-) diff --git a/api.js b/api.js index 3385a82f9f..8e9fa6fb9e 100644 --- a/api.js +++ b/api.js @@ -1869,28 +1869,4 @@ API.prototype.setErrorGroupCallback = function setErrorGroupCallback(callback) { this.agent.errors.errorGroupCallback = callback } -/** - * Function for assigning metadata to all LLM events. This should only - * be called once in your application and before executing any openai, bedrock, - * langchain, generative AI methods. - * - * If passing in metadata via `api.recordLlmFeedbackEvent`, it will take precedence - * over what is assigned via this method. - * - * @param {object} metadata key/value metadata to be assigned to all LLM Events on creation - * @example - * newrelic.setLlmMetadata({ type: 'openai', framework: 'express', region: 'us' }) - */ -API.prototype.setLlmMetadata = function setLlmMetadata(metadata) { - const metric = this.agent.metrics.getOrCreateMetric(NAMES.SUPPORTABILITY.API + '/setLlmMetadata') - metric.incrementCallCount() - - if (!isSimpleObject(metadata)) { - logger.warn('metadata must be an object, not assigning LLM metadata.') - return - } - - this.agent.llm.metadata = metadata -} - module.exports = API diff --git a/lib/llm-events/openai/event.js b/lib/llm-events/openai/event.js index bb7ed8ecb4..d194c23d1c 100644 --- a/lib/llm-events/openai/event.js +++ b/lib/llm-events/openai/event.js @@ -51,6 +51,6 @@ module.exports = class LlmEvent { conversationId(agent) { const transaction = agent.tracer.getTransaction() const attrs = transaction?.trace?.custom.get(DESTINATIONS.TRANS_SCOPE) - return attrs?.conversation_id + return attrs?.['llm.conversation_id'] } } diff --git a/test/unit/api/api-llm.test.js b/test/unit/api/api-llm.test.js index 1fd5c89a0d..dba48de360 100644 --- a/test/unit/api/api-llm.test.js +++ b/test/unit/api/api-llm.test.js @@ -9,7 +9,6 @@ const tap = require('tap') const sinon = require('sinon') const proxyquire = require('proxyquire') const helper = require('../../lib/agent_helper') -const NAMES = require('../../../lib/metrics/names') tap.test('Agent API LLM methods', (t) => { t.autoend() @@ -36,30 +35,6 @@ tap.test('Agent API LLM methods', (t) => { helper.unloadAgent(t.context.api.agent) }) - t.test('should assign llm metadata when it is an object', (t) => { - const { api } = t.context - const meta = { user: 'bob', env: 'prod', random: 'data' } - api.setLlmMetadata(meta) - - t.equal(loggerMock.warn.callCount, 0, 'should not log warnings when successful') - t.equal( - api.agent.metrics.getOrCreateMetric(NAMES.SUPPORTABILITY.API + '/setLlmMetadata').callCount, - 1, - 'should increment the API tracking metric' - ) - t.same(api.agent.llm.metadata, meta) - t.end() - }) - ;['string', 10, true, null, undefined, [1, 2, 3, 4], [{ collection: true }]].forEach((meta) => { - t.test(`should not assign llm metadata when ${meta} is not an object`, (t) => { - const { api } = t.context - api.setLlmMetadata(meta) - t.equal(loggerMock.warn.callCount, 1, 'should log warning when metadata is not an object') - t.same(api.agent.llm, {}) - t.end() - }) - }) - t.test('getLlmMessageIds is no-op when ai_monitoring is disabled', async (t) => { const { api } = t.context api.agent.config.ai_monitoring.enabled = false diff --git a/test/unit/api/stub.test.js b/test/unit/api/stub.test.js index c9db639575..bc58a8bc3b 100644 --- a/test/unit/api/stub.test.js +++ b/test/unit/api/stub.test.js @@ -8,7 +8,7 @@ const tap = require('tap') const API = require('../../../stub_api') -const EXPECTED_API_COUNT = 36 +const EXPECTED_API_COUNT = 35 tap.test('Agent API - Stubbed Agent API', (t) => { t.autoend() diff --git a/test/unit/llm-events/openai/chat-completion-message.test.js b/test/unit/llm-events/openai/chat-completion-message.test.js index 8cb978abb2..12a3e92e29 100644 --- a/test/unit/llm-events/openai/chat-completion-message.test.js +++ b/test/unit/llm-events/openai/chat-completion-message.test.js @@ -74,7 +74,7 @@ tap.test('LlmChatCompletionMessage', (t) => { const api = helper.getAgentApi() const conversationId = 'convo-id' helper.runInTransaction(agent, () => { - api.addCustomAttribute('conversation_id', conversationId) + api.addCustomAttribute('llm.conversation_id', conversationId) const chatMessageEvent = new LlmChatCompletionMessage({ agent, segment: {}, diff --git a/test/unit/llm-events/openai/chat-completion-summary.test.js b/test/unit/llm-events/openai/chat-completion-summary.test.js index 0ddc69955b..0ab9f9e314 100644 --- a/test/unit/llm-events/openai/chat-completion-summary.test.js +++ b/test/unit/llm-events/openai/chat-completion-summary.test.js @@ -45,7 +45,7 @@ tap.test('LlmChatCompletionSummary', (t) => { const api = helper.getAgentApi() const conversationId = 'convo-id' helper.runInTransaction(agent, () => { - api.addCustomAttribute('conversation_id', conversationId) + api.addCustomAttribute('llm.conversation_id', conversationId) const chatSummaryEvent = new LlmChatCompletionSummary({ agent, segment: null, @@ -61,7 +61,7 @@ tap.test('LlmChatCompletionSummary', (t) => { const api = helper.getAgentApi() const conversationId = 'convo-id' helper.runInTransaction(agent, () => { - api.addCustomAttribute('conversation_id', conversationId) + api.addCustomAttribute('llm.conversation_id', conversationId) const chatSummaryEvent = new LlmChatCompletionSummary({ agent, segment: null, diff --git a/test/versioned/openai/chat-completions.tap.js b/test/versioned/openai/chat-completions.tap.js index 3723b01f46..2fe984ca7b 100644 --- a/test/versioned/openai/chat-completions.tap.js +++ b/test/versioned/openai/chat-completions.tap.js @@ -276,33 +276,6 @@ tap.test('OpenAI instrumentation - chat completions', (t) => { }) } - t.test('should spread metadata across events if present on agent.llm.metadata', (test) => { - const { client, agent } = t.context - const api = helper.getAgentApi() - helper.runInTransaction(agent, async (tx) => { - const meta = { key: 'value', extended: true, vendor: 'overwriteMe', id: 'bogus' } - api.setLlmMetadata(meta) - - await client.chat.completions.create({ - messages: [{ role: 'user', content: 'You are a mathematician.' }] - }) - - const events = agent.customEventAggregator.events.toArray() - events.forEach(([, testEvent]) => { - test.equal(testEvent.key, 'value') - test.equal(testEvent.extended, true) - test.equal( - testEvent.vendor, - 'openAI', - 'should not override properties of message with metadata' - ) - test.not(testEvent.id, 'bogus', 'should not override properties of message with metadata') - }) - tx.end() - test.end() - }) - }) - t.test('should not create llm events when not in a transaction', async (test) => { const { client, agent } = t.context await client.chat.completions.create({ diff --git a/test/versioned/openai/embeddings.tap.js b/test/versioned/openai/embeddings.tap.js index 5fe7e0ef94..db6a101b5c 100644 --- a/test/versioned/openai/embeddings.tap.js +++ b/test/versioned/openai/embeddings.tap.js @@ -114,33 +114,6 @@ tap.test('OpenAI instrumentation - embedding', (t) => { }) }) - t.test('should spread metadata across events if present on agent.llm.metadata', (test) => { - const { client, agent } = t.context - const api = helper.getAgentApi() - helper.runInTransaction(agent, async (tx) => { - const meta = { key: 'value', extended: true, vendor: 'overwriteMe', id: 'bogus' } - api.setLlmMetadata(meta) - - await client.embeddings.create({ - input: 'This is an embedding test.', - model: 'text-embedding-ada-002' - }) - - const events = agent.customEventAggregator.events.toArray() - const [[, testEvent]] = events - test.equal(testEvent.key, 'value') - test.equal(testEvent.extended, true) - test.equal( - testEvent.vendor, - 'openAI', - 'should not override properties of message with metadata' - ) - test.not(testEvent.id, 'bogus', 'should not override properties of message with metadata') - tx.end() - test.end() - }) - }) - t.test('embedding invalid payload errors should be tracked', (test) => { const { client, agent } = t.context helper.runInTransaction(agent, async (tx) => {