chore(llmobs): add span.finished telemetry metric (#5444)

* Add span.finished telemetry metric

* Add decorator tag

* address comments

* Fix root parent bool

* Add error tag

* Fix boolean coalescing

Author: Yun-Kim

packages/dd-trace/src/llmobs/constants/tags.js

@@ -4,6 +4,8 @@ module.exports = {
   SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval'],
   SPAN_KIND: '_ml_obs.meta.span.kind',
   SESSION_ID: '_ml_obs.session_id',
+  DECORATOR: '_ml_obs.decorator',
+  INTEGRATION: '_ml_obs.integration',
   METADATA: '_ml_obs.meta.metadata',
   METRICS: '_ml_obs.metrics',
   ML_APP: '_ml_obs.meta.ml_app',

packages/dd-trace/src/llmobs/noop.js

@@ -43,14 +43,14 @@ class NoopLLMObs {
         const ctx = ctxOrPropertyKey
         if (ctx.kind !== 'method') return target
 
-        return llmobs.wrap({ name: ctx.name, ...options }, target)
+        return llmobs.wrap({ name: ctx.name, _decorator: true, ...options }, target)
       } else {
         const propertyKey = ctxOrPropertyKey
         if (descriptor) {
           if (typeof descriptor.value !== 'function') return descriptor
 
           const original = descriptor.value
-          descriptor.value = llmobs.wrap({ name: propertyKey, ...options }, original)
+          descriptor.value = llmobs.wrap({ name: propertyKey, _decorator: true, ...options }, original)
 
           return descriptor
         } else {
@@ -59,7 +59,7 @@ class NoopLLMObs {
           const original = target[propertyKey]
           Object.defineProperty(target, propertyKey, {
             ...Object.getOwnPropertyDescriptor(target, propertyKey),
-            value: llmobs.wrap({ name: propertyKey, ...options }, original)
+            value: llmobs.wrap({ name: propertyKey, _decorator: true, ...options }, original)
           })
 
           return target

packages/dd-trace/src/llmobs/sdk.js

@@ -430,6 +430,7 @@ class LLMObs extends NoopLLMObs {
       modelProvider,
       sessionId,
       mlApp,
+      _decorator,
       ...spanOptions
     } = options
 
@@ -438,6 +439,7 @@ class LLMObs extends NoopLLMObs {
       modelName,
       modelProvider,
       sessionId,
+      _decorator,
       spanOptions
     }
   }

packages/dd-trace/src/llmobs/span_processor.js

@@ -27,6 +27,8 @@ const {
   ERROR_STACK
 } = require('../constants')
 
+const telemetry = require('./telemetry')
+
 const LLMObsTagger = require('./tagger')
 
 const tracerVersion = require('../../../../package.json').version
@@ -49,6 +51,7 @@ class LLMObsSpanProcessor {
 
     try {
       const formattedEvent = this.format(span)
+      telemetry.incrementLLMObsSpanFinishedCount(span)
       this._writer.append(formattedEvent)
     } catch (e) {
       // this should be a rare case

packages/dd-trace/src/llmobs/tagger.js

@@ -10,7 +10,6 @@ const {
   INPUT_VALUE,
   OUTPUT_DOCUMENTS,
   INPUT_DOCUMENTS,
-  INTEGRATION,
   OUTPUT_VALUE,
   METADATA,
   METRICS,
@@ -23,7 +22,9 @@ const {
   ROOT_PARENT_ID,
   INPUT_TOKENS_METRIC_KEY,
   OUTPUT_TOKENS_METRIC_KEY,
-  TOTAL_TOKENS_METRIC_KEY
+  TOTAL_TOKENS_METRIC_KEY,
+  INTEGRATION,
+  DECORATOR
 } = require('./constants/tags')
 
 // global registry of LLMObs spans
@@ -53,7 +54,8 @@ class LLMObsTagger {
     parent,
     kind,
     name,
-    integration
+    integration,
+    _decorator
   } = {}) {
     if (!this._config.llmobs.enabled) return
     if (!kind) return // do not register it in the map if it doesn't have an llmobs span kind
@@ -69,6 +71,7 @@ class LLMObsTagger {
     sessionId = sessionId || registry.get(parent)?.[SESSION_ID]
     if (sessionId) this._setTag(span, SESSION_ID, sessionId)
     if (integration) this._setTag(span, INTEGRATION, integration)
+    if (_decorator) this._setTag(span, DECORATOR, _decorator)
 
     if (!mlApp) mlApp = registry.get(parent)?.[ML_APP] || this._config.llmobs.mlApp
     this._setTag(span, ML_APP, mlApp)

packages/dd-trace/src/llmobs/telemetry.js

@@ -1,12 +1,59 @@
 'use strict'
 
+const {
+  SPAN_KIND,
+  MODEL_PROVIDER,
+  PARENT_ID_KEY,
+  SESSION_ID,
+  ROOT_PARENT_ID,
+  INTEGRATION,
+  DECORATOR
+} = require('./constants/tags')
+
+const ERROR_TYPE = require('../constants')
+
 const telemetryMetrics = require('../telemetry/metrics')
+
+const LLMObsTagger = require('./tagger')
+
 const llmobsMetrics = telemetryMetrics.manager.namespace('mlobs')
 
 function incrementLLMObsSpanStartCount (tags, value = 1) {
   llmobsMetrics.count('span.start', tags).inc(value)
 }
 
+function incrementLLMObsSpanFinishedCount (span, value = 1) {
+  const mlObsTags = LLMObsTagger.tagMap.get(span)
+  const spanTags = span.context()._tags
+
+  const isRootSpan = mlObsTags[PARENT_ID_KEY] === ROOT_PARENT_ID
+  const hasSessionId = mlObsTags[SESSION_ID] != null
+  const integration = mlObsTags[INTEGRATION]
+  const autoInstrumented = integration != null
+  const decorator = !!mlObsTags[DECORATOR]
+  const spanKind = mlObsTags[SPAN_KIND]
+  const modelProvider = mlObsTags[MODEL_PROVIDER]
+  const error = spanTags.error || spanTags[ERROR_TYPE]
+
+  const tags = {
+    autoinstrumented: Number(autoInstrumented),
+    has_session_id: Number(hasSessionId),
+    is_root_span: Number(isRootSpan),
+    span_kind: spanKind,
+    integration: integration || 'N/A',
+    error: error ? 1 : 0
+  }
+  if (!autoInstrumented) {
+    tags.decorator = Number(decorator)
+  }
+  if (modelProvider) {
+    tags.model_provider = modelProvider
+  }
+
+  llmobsMetrics.count('span.finished', tags).inc(value)
+}
+
 module.exports = {
-  incrementLLMObsSpanStartCount
+  incrementLLMObsSpanStartCount,
+  incrementLLMObsSpanFinishedCount
 }