feat: removed api.setLlmMetadata(#1918)

* It will be assigned via api.addCustomAttribute with a prefix of `llm.` * The conversation_id will now be assigned from `llm.conversation_id` custom attribute
newrelic · Dec 18, 2023 · cc4a975 · cc4a975
1 parent 2d666b6
commit cc4a975
Show file tree

Hide file tree

Showing 8 changed files with 5 additions and 108 deletions.
diff --git a/api.js b/api.js
@@ -1869,28 +1869,4 @@ API.prototype.setErrorGroupCallback = function setErrorGroupCallback(callback) {
   this.agent.errors.errorGroupCallback = callback
 }
 
-/**
- * Function for assigning metadata to all LLM events. This should only
- * be called once in your application and before executing any openai, bedrock,
- * langchain, generative AI methods.
- *
- * If passing in metadata via `api.recordLlmFeedbackEvent`, it will take precedence
- * over what is assigned via this method.
- *
- * @param {object} metadata key/value metadata to be assigned to all LLM Events on creation
- * @example
- * newrelic.setLlmMetadata({ type: 'openai', framework: 'express', region: 'us' })
- */
-API.prototype.setLlmMetadata = function setLlmMetadata(metadata) {
-  const metric = this.agent.metrics.getOrCreateMetric(NAMES.SUPPORTABILITY.API + '/setLlmMetadata')
-  metric.incrementCallCount()
-
-  if (!isSimpleObject(metadata)) {
-    logger.warn('metadata must be an object, not assigning LLM metadata.')
-    return
-  }
-
-  this.agent.llm.metadata = metadata
-}
-
 module.exports = API
diff --git a/lib/llm-events/openai/event.js b/lib/llm-events/openai/event.js
@@ -51,6 +51,6 @@ module.exports = class LlmEvent {
   conversationId(agent) {
     const transaction = agent.tracer.getTransaction()
     const attrs = transaction?.trace?.custom.get(DESTINATIONS.TRANS_SCOPE)
-    return attrs?.conversation_id
+    return attrs?.['llm.conversation_id']
   }
 }
diff --git a/test/unit/api/api-llm.test.js b/test/unit/api/api-llm.test.js
@@ -9,7 +9,6 @@ const tap = require('tap')
 const sinon = require('sinon')
 const proxyquire = require('proxyquire')
 const helper = require('../../lib/agent_helper')
-const NAMES = require('../../../lib/metrics/names')
 
 tap.test('Agent API LLM methods', (t) => {
   t.autoend()
@@ -36,30 +35,6 @@ tap.test('Agent API LLM methods', (t) => {
     helper.unloadAgent(t.context.api.agent)
   })
 
-  t.test('should assign llm metadata when it is an object', (t) => {
-    const { api } = t.context
-    const meta = { user: 'bob', env: 'prod', random: 'data' }
-    api.setLlmMetadata(meta)
-
-    t.equal(loggerMock.warn.callCount, 0, 'should not log warnings when successful')
-    t.equal(
-      api.agent.metrics.getOrCreateMetric(NAMES.SUPPORTABILITY.API + '/setLlmMetadata').callCount,
-      1,
-      'should increment the API tracking metric'
-    )
-    t.same(api.agent.llm.metadata, meta)
-    t.end()
-  })
-  ;['string', 10, true, null, undefined, [1, 2, 3, 4], [{ collection: true }]].forEach((meta) => {
-    t.test(`should not assign llm metadata when ${meta} is not an object`, (t) => {
-      const { api } = t.context
-      api.setLlmMetadata(meta)
-      t.equal(loggerMock.warn.callCount, 1, 'should log warning when metadata is not an object')
-      t.same(api.agent.llm, {})
-      t.end()
-    })
-  })
-
   t.test('getLlmMessageIds is no-op when ai_monitoring is disabled', async (t) => {
     const { api } = t.context
     api.agent.config.ai_monitoring.enabled = false

diff --git a/test/unit/api/stub.test.js b/test/unit/api/stub.test.js
@@ -8,7 +8,7 @@
 const tap = require('tap')
 const API = require('../../../stub_api')
 
-const EXPECTED_API_COUNT = 36
+const EXPECTED_API_COUNT = 35
 
 tap.test('Agent API - Stubbed Agent API', (t) => {
   t.autoend()

diff --git a/test/unit/llm-events/openai/chat-completion-message.test.js b/test/unit/llm-events/openai/chat-completion-message.test.js
@@ -74,7 +74,7 @@ tap.test('LlmChatCompletionMessage', (t) => {
     const api = helper.getAgentApi()
     const conversationId = 'convo-id'
     helper.runInTransaction(agent, () => {
-      api.addCustomAttribute('conversation_id', conversationId)
+      api.addCustomAttribute('llm.conversation_id', conversationId)
       const chatMessageEvent = new LlmChatCompletionMessage({
         agent,
         segment: {},

diff --git a/test/unit/llm-events/openai/chat-completion-summary.test.js b/test/unit/llm-events/openai/chat-completion-summary.test.js
@@ -45,7 +45,7 @@ tap.test('LlmChatCompletionSummary', (t) => {
     const api = helper.getAgentApi()
     const conversationId = 'convo-id'
     helper.runInTransaction(agent, () => {
-      api.addCustomAttribute('conversation_id', conversationId)
+      api.addCustomAttribute('llm.conversation_id', conversationId)
       const chatSummaryEvent = new LlmChatCompletionSummary({
         agent,
         segment: null,
@@ -61,7 +61,7 @@ tap.test('LlmChatCompletionSummary', (t) => {
     const api = helper.getAgentApi()
     const conversationId = 'convo-id'
     helper.runInTransaction(agent, () => {
-      api.addCustomAttribute('conversation_id', conversationId)
+      api.addCustomAttribute('llm.conversation_id', conversationId)
       const chatSummaryEvent = new LlmChatCompletionSummary({
         agent,
         segment: null,

diff --git a/test/versioned/openai/chat-completions.tap.js b/test/versioned/openai/chat-completions.tap.js
@@ -276,33 +276,6 @@ tap.test('OpenAI instrumentation - chat completions', (t) => {
     })
   }
 
-  t.test('should spread metadata across events if present on agent.llm.metadata', (test) => {
-    const { client, agent } = t.context
-    const api = helper.getAgentApi()
-    helper.runInTransaction(agent, async (tx) => {
-      const meta = { key: 'value', extended: true, vendor: 'overwriteMe', id: 'bogus' }
-      api.setLlmMetadata(meta)
-
-      await client.chat.completions.create({
-        messages: [{ role: 'user', content: 'You are a mathematician.' }]
-      })
-
-      const events = agent.customEventAggregator.events.toArray()
-      events.forEach(([, testEvent]) => {
-        test.equal(testEvent.key, 'value')
-        test.equal(testEvent.extended, true)
-        test.equal(
-          testEvent.vendor,
-          'openAI',
-          'should not override properties of message with metadata'
-        )
-        test.not(testEvent.id, 'bogus', 'should not override properties of message with metadata')
-      })
-      tx.end()
-      test.end()
-    })
-  })
-
   t.test('should not create llm events when not in a transaction', async (test) => {
     const { client, agent } = t.context
     await client.chat.completions.create({

diff --git a/test/versioned/openai/embeddings.tap.js b/test/versioned/openai/embeddings.tap.js
@@ -114,33 +114,6 @@ tap.test('OpenAI instrumentation - embedding', (t) => {
     })
   })
 
-  t.test('should spread metadata across events if present on agent.llm.metadata', (test) => {
-    const { client, agent } = t.context
-    const api = helper.getAgentApi()
-    helper.runInTransaction(agent, async (tx) => {
-      const meta = { key: 'value', extended: true, vendor: 'overwriteMe', id: 'bogus' }
-      api.setLlmMetadata(meta)
-
-      await client.embeddings.create({
-        input: 'This is an embedding test.',
-        model: 'text-embedding-ada-002'
-      })
-
-      const events = agent.customEventAggregator.events.toArray()
-      const [[, testEvent]] = events
-      test.equal(testEvent.key, 'value')
-      test.equal(testEvent.extended, true)
-      test.equal(
-        testEvent.vendor,
-        'openAI',
-        'should not override properties of message with metadata'
-      )
-      test.not(testEvent.id, 'bogus', 'should not override properties of message with metadata')
-      tx.end()
-      test.end()
-    })
-  })
-
   t.test('embedding invalid payload errors should be tracked', (test) => {
     const { client, agent } = t.context
     helper.runInTransaction(agent, async (tx) => {