feat(llmobs): add boolean evaluation metric support (#6919)

Author: sabrenner

docs/test.ts

@@ -662,6 +662,12 @@ llmobs.trace({ kind: 'llm', name: 'myLLM' }, (span) => {
     tags: {},
     timestampMs: Date.now()
   })
+
+  llmobs.submitEvaluation(llmobsSpanCtx, {
+    label: 'toxicity',
+    metricType: 'boolean',
+    value: 'true'
+  })
 })
 
 // annotate a span

index.d.ts

@@ -3011,15 +3011,15 @@ declare namespace tracer {
       label: string,
 
       /**
-       * The type of evaluation metric, one of 'categorical' or 'score'
+       * The type of evaluation metric, one of 'categorical', 'score', or 'boolean'
        */
-      metricType: 'categorical' | 'score',
+      metricType: 'categorical' | 'score' | 'boolean',
 
       /**
        * The value of the evaluation metric.
-       * Must be string for 'categorical' metrics and number for 'score' metrics.
+       * Must be string for 'categorical' metrics, number for 'score' metrics, and boolean for 'boolean' metrics.
        */
-      value: string | number,
+      value: string | number | boolean,
 
       /**
        * An object of string key-value pairs to tag the evaluation metric with.

packages/dd-trace/src/llmobs/sdk.js

@@ -369,7 +369,7 @@ class LLMObs extends NoopLLMObs {
         err = 'invalid_metric_label'
         throw new Error('label must be the specified name of the evaluation metric')
       }
-      if (!metricType || !['categorical', 'score'].includes(metricType)) {
+      if (!metricType || !['categorical', 'score', 'boolean'].includes(metricType)) {
         err = 'invalid_metric_type'
         throw new Error('metricType must be one of "categorical" or "score"')
       }
@@ -381,6 +381,10 @@ class LLMObs extends NoopLLMObs {
         err = 'invalid_metric_value'
         throw new Error('value must be a number for a score metric.')
       }
+      if (metricType === 'boolean' && typeof value !== 'boolean') {
+        err = 'invalid_metric_value'
+        throw new Error('value must be a boolean for a boolean metric')
+      }
 
       const evaluationTags = {
         'ddtrace.version': tracerVersion,

packages/dd-trace/test/llmobs/sdk/index.spec.js

@@ -4,6 +4,7 @@ const { expect } = require('chai')
 const { channel } = require('dc-polyfill')
 const { describe, it, beforeEach, afterEach, before, after } = require('mocha')
 const sinon = require('sinon')
+const assert = require('node:assert')
 
 const { getConfigFresh } = require('../../helpers/config')
 
@@ -1210,6 +1211,36 @@ describe('sdk', () => {
       expect(LLMObsEvalMetricsWriter.prototype.append.getCall(0).args[0]).to.have.property('timestamp_ms', 1234)
       Date.now.restore()
     })
+
+    it('submits a boolean evaluation metric', () => {
+      llmobs.submitEvaluation(spanCtx, {
+        label: 'has_toxicity',
+        metricType: 'boolean',
+        value: true,
+        timestampMs: 1234
+      })
+
+      const evalMetric = LLMObsEvalMetricsWriter.prototype.append.getCall(0).args[0]
+
+      assert.deepEqual(evalMetric, {
+        span_id: '5678',
+        trace_id: '1234',
+        label: 'has_toxicity',
+        metric_type: 'boolean',
+        ml_app: 'mlApp',
+        boolean_value: true,
+        timestamp_ms: 1234,
+        tags: [`ddtrace.version:${tracerVersion}`, 'ml_app:mlApp']
+      })
+    })
+
+    it('throws an error when submitting a non-boolean boolean evaluation metric', () => {
+      assert.throws(() => llmobs.submitEvaluation(spanCtx, {
+        label: 'has_toxicity',
+        metricType: 'boolean',
+        value: 'it is super toxic!'
+      }), { message: 'value must be a boolean for a boolean metric' })
+    })
   })
 
   describe('flush', () => {