chore(llmobs): add span size telemetry metrics (#5468)

* Add raw span size metric

* Add processed span event size metrics

* Refactor

* Remove unnecessary check

Author: Yun-Kim

packages/dd-trace/src/llmobs/telemetry.js

@@ -18,6 +18,27 @@ const LLMObsTagger = require('./tagger')
 
 const llmobsMetrics = telemetryMetrics.manager.namespace('mlobs')
 
+function extractIntegrationFromTags (tags) {
+  if (!Array.isArray(tags)) return null
+  const integrationTag = tags.find(tag => tag.startsWith('integration:'))
+  if (!integrationTag) return null
+  return integrationTag.split(':')[1] || null
+}
+
+function extractTagsFromSpanEvent (event) {
+  const spanKind = event.meta?.['span.kind'] || ''
+  const integration = extractIntegrationFromTags(event.tags)
+  const error = event.status === 'error'
+  const autoinstrumented = integration != null
+
+  return {
+    span_kind: spanKind,
+    autoinstrumented: Number(autoinstrumented),
+    error: error ? 1 : 0,
+    integration: integration || 'N/A'
+  }
+}
+
 function incrementLLMObsSpanStartCount (tags, value = 1) {
   llmobsMetrics.count('span.start', tags).inc(value)
 }
@@ -53,7 +74,20 @@ function incrementLLMObsSpanFinishedCount (span, value = 1) {
   llmobsMetrics.count('span.finished', tags).inc(value)
 }
 
+function recordLLMObsRawSpanSize (event, rawEventSize) {
+  const tags = extractTagsFromSpanEvent(event)
+  llmobsMetrics.distribution('span.raw_size', tags).track(rawEventSize)
+}
+
+function recordLLMObsSpanSize (event, eventSize, shouldTruncate) {
+  const tags = extractTagsFromSpanEvent(event)
+  tags.truncated = Number(shouldTruncate)
+  llmobsMetrics.distribution('span.size', tags).track(eventSize)
+}
+
 module.exports = {
   incrementLLMObsSpanStartCount,
-  incrementLLMObsSpanFinishedCount
+  incrementLLMObsSpanFinishedCount,
+  recordLLMObsRawSpanSize,
+  recordLLMObsSpanSize
 }

packages/dd-trace/src/llmobs/writers/spans/base.js

@@ -4,6 +4,7 @@ const { EVP_EVENT_SIZE_LIMIT, EVP_PAYLOAD_SIZE_LIMIT } = require('../../constant
 const { DROPPED_VALUE_TEXT } = require('../../constants/text')
 const { DROPPED_IO_COLLECTION_ERROR } = require('../../constants/tags')
 const BaseWriter = require('../base')
+const telemetry = require('../../telemetry')
 const logger = require('../../../log')
 
 const tracerVersion = require('../../../../../../package.json').version
@@ -18,11 +19,19 @@ class LLMObsSpanWriter extends BaseWriter {
 
   append (event) {
     const eventSizeBytes = Buffer.from(JSON.stringify(event)).byteLength
-    if (eventSizeBytes > EVP_EVENT_SIZE_LIMIT) {
+    telemetry.recordLLMObsRawSpanSize(event, eventSizeBytes)
+
+    const shouldTruncate = eventSizeBytes > EVP_EVENT_SIZE_LIMIT
+    let processedEventSizeBytes = eventSizeBytes
+
+    if (shouldTruncate) {
       logger.warn(`Dropping event input/output because its size (${eventSizeBytes}) exceeds the 1MB event size limit`)
       event = this._truncateSpanEvent(event)
+      processedEventSizeBytes = Buffer.from(JSON.stringify(event)).byteLength
     }
 
+    telemetry.recordLLMObsSpanSize(event, processedEventSizeBytes, shouldTruncate)
+
     if (this._bufferSize + eventSizeBytes > EVP_PAYLOAD_SIZE_LIMIT) {
       logger.debug('Flusing queue because queing next event will exceed EvP payload limit')
       this.flush()