add embedding tracking

lib/instrumentation/openai.js

@@ -36,6 +36,7 @@ function shouldSkipInstrumentation(config, shim) {
   return semver.lt(pkgVersion, MIN_VERSION)
 }
 
+// eslint-disable-next-line sonarjs/cognitive-complexity
 module.exports = function initialize(agent, openai, moduleName, shim) {
   if (shouldSkipInstrumentation(agent.config, shim)) {
     shim.logger.debug(
@@ -224,6 +225,11 @@ module.exports = function initialize(agent, openai, moduleName, shim) {
         promise: true,
         // eslint-disable-next-line max-params
         after(_shim, _fn, _name, err, response, segment) {
+          if (!response) {
+            // If we get an error, it is possible that `response = null`.
+            // In that case, we define it to be an empty object.
+            response = {}
+          }
           response.headers = segment[openAiHeaders]
           response.api_key = segment[openAiApiKey]
           const embedding = new LlmEmbedding({
@@ -234,6 +240,12 @@ module.exports = function initialize(agent, openai, moduleName, shim) {
           })
 
           recordEvent('LlmEmbedding', embedding)
+
+          if (err) {
+            const llmError = new OpenAiLlmError({ cause: err, embedding, response })
+            shim.agent.errors.add(segment.transaction, llmError)
+          }
+
           // cleanup keys on response before returning to user code
           delete response.api_key
           delete response.headers

lib/llm-events/openai/llm-error.js

@@ -27,9 +27,6 @@ class OpenAiLlmError extends Error {
       code: cause.error.code,
       param: cause.error.param
     }
-    // this.statusCode = cause.status
-    // this.code = cause.error.code
-    // this.param = cause.error.param
     this.completion_id = summary?.id
     this.embedding_id = embedding?.id
   }

test/lib/openai-responses.js

@@ -177,3 +177,19 @@ responses.set('Invalid role.', {
     }
   }
 })
+
+responses.set('Embedding not allowed.', {
+  headers: {
+    'content-type': 'application/json; charset=utf-8',
+    'x-request-id': '25e22d5a7af9e40b7d5b1828593b52aa'
+  },
+  code: 403,
+  body: {
+    error: {
+      message: 'You are not allowed to generate embeddings from this model',
+      type: 'invalid_request_error',
+      param: null,
+      code: null
+    }
+  }
+})

test/versioned/openai/openai.tap.js

@@ -412,15 +412,53 @@ tap.test('OpenAI instrumentation', (t) => {
             code: null,
             param: null
           },
-          completion_id: /[\w\d]{32}/,
+          completion_id: /\w{32}/,
           cause: {
             message: /'bad-role' is not one of/,
             type: 'invalid_request_error'
           }
         },
         customAttributes: {},
         agentAttributes: {
-          spanId: /[\w\d]+/
+          spanId: /\w+/
+        }
+      })
+
+      tx.end()
+      test.end()
+    })
+  })
+
+  t.test('embedding invalid payload errors should be tracked', (test) => {
+    const { client, agent } = t.context
+    helper.runInTransaction(agent, async (tx) => {
+      try {
+        await client.embeddings.create({
+          model: 'gpt-4',
+          input: 'Embedding not allowed.'
+        })
+      } catch {}
+
+      t.equal(tx.exceptions.length, 1)
+      t.match(tx.exceptions[0], {
+        error: {
+          http: {
+            statusCode: 403
+          },
+          error: {
+            code: null,
+            param: null
+          },
+          completion_id: undefined,
+          embedding_id: /\w{32}/,
+          cause: {
+            message: 'You are not allowed to generate embeddings from this model',
+            type: 'invalid_request_error'
+          }
+        },
+        customAttributes: {},
+        agentAttributes: {
+          spanId: /\w+/
         }
       })