Update OTel parts

gateway/src/api/anthropic.ts

@@ -66,9 +66,9 @@ function mapParts(content: string | BetaContentBlockParam[] | BetaContentBlock[]
       } else if (part.type === 'thinking') {
         parts.push({ type: 'thinking', content: part.thinking })
       } else if (part.type === 'image' && part.source.type === 'base64') {
-        parts.push({ type: 'blob', mime_type: part.source.media_type, data: part.source.data })
+        parts.push({ type: 'blob', mime_type: part.source.media_type, content: part.source.data, modality: 'image' })
       } else if (part.type === 'image' && part.source.type === 'url') {
-        parts.push({ type: 'file_data', file_uri: part.source.url })
+        parts.push({ type: 'uri', uri: part.source.url, modality: 'image' })
         // TODO(Marcelo): Currently, there's no semantic convention for built-in tools: https://github.com/open-telemetry/semantic-conventions/issues/2585
       } else if (part.type === 'tool_use' || part.type === 'server_tool_use') {
         mapToolCallIdToName[part.id] = part.name
@@ -95,7 +95,7 @@ function mapParts(content: string | BetaContentBlockParam[] | BetaContentBlock[]
           builtin: !(part.type === 'tool_result'),
         })
       } else {
-        parts.push({ ...part })
+        parts.push({ type: 'unknown', part: { ...part } })
       }
     }
   }

gateway/src/api/chat.ts

@@ -63,15 +63,18 @@ function mapInputParts(content: ChatCompletionMessageParam['content']): MessageP
       if (part.type === 'text') {
         parts.push({ type: 'text', content: part.text })
       } else if (part.type === 'image_url') {
-        parts.push({ type: 'file_data', file_uri: part.image_url.url })
+        parts.push({ type: 'uri', uri: part.image_url.url, modality: 'image' })
       } else if (part.type === 'input_audio') {
         const mimeType = mime.contentType(part.input_audio.format) || undefined
-        parts.push({ type: 'blob', mime_type: mimeType, data: part.input_audio.data })
+        parts.push({ type: 'blob', mime_type: mimeType, content: part.input_audio.data, modality: 'audio' })
       } else if (part.type === 'file' && part.file.file_data) {
+        const mimeType = _extractMimeTypeFromBase64(part.file.file_data)
+        parts.push({ type: 'blob', mime_type: mimeType, content: part.file.file_data, modality: 'document' })
+      } else if (part.type === 'file' && part.file.file_id) {
         const mimeType = part.file.filename ? mime.contentType(part.file.filename) || undefined : undefined
-        parts.push({ type: 'blob', mime_type: mimeType, data: part.file.file_data })
+        parts.push({ type: 'file', file_id: part.file.file_id, mime_type: mimeType, modality: 'unknown' })
       } else {
-        parts.push({ ...part })
+        parts.push({ type: 'unknown', part: { ...part } })
       }
     }
   }
@@ -101,3 +104,9 @@ function mapOutputParts(message: ChatCompletion.Choice['message']): MessagePart[
   }
   return parts
 }
+
+function _extractMimeTypeFromBase64(base64: string): string | undefined {
+  // The format is always "data:image/png;base64,{base64}"
+  const match = base64.match(/^data:([^;]+);base64,/)
+  return match ? match[1] : undefined
+}

gateway/src/api/google.ts

@@ -83,9 +83,9 @@ function mapContent(content: Content): ChatMessage {
         result: part.functionResponse.response,
       })
     } else if (part.fileData) {
-      parts.push({ type: 'file_data', file_uri: part.fileData.fileUri, mime_type: part.fileData.mimeType })
+      parts.push({ type: 'file', file_uri: part.fileData.fileUri, mime_type: part.fileData.mimeType })
     } else if (part.inlineData) {
-      parts.push({ type: 'blob', mime_type: part.inlineData.mimeType, data: part.inlineData.data })
+      parts.push({ type: 'blob', mime_type: part.inlineData.mimeType, content: part.inlineData.data })
     } else if (part.thought) {
       parts.push({ type: 'thinking', content: part.thought })
     } else if (part.thoughtSignature) {

gateway/src/otel/genai.d.ts

@@ -27,37 +27,49 @@ export interface ToolCallResponsePart {
   builtin?: boolean
 }
 
+type Modality = 'audio' | 'image' | 'text' | string
+
 // https://github.com/open-telemetry/semantic-conventions/pull/2754/
 export interface BlobPart {
   type: 'blob'
-  mime_type: string
-  data: string
+  content: string
+  mime_type?: string
+  modality: Modality
+}
+
+export interface FilePart {
+  type: 'file'
+  file_id: string
+  mime_type?: string
+  modality: Modality
 }
 
-export interface FileDataPart {
-  type: 'file_data'
+export interface UriPart {
+  type: 'uri'
+  uri: string
+  modality: Modality
   mime_type?: string
-  file_uri: string
 }
 
 export interface ThinkingPart {
   type: 'thinking'
   content?: string
 }
 
-export interface GenericPart {
-  type: string
-  [key: string]: unknown
+export interface UnknownPart {
+  type: 'unknown'
+  part: { [key: string]: unknown }
 }
 
 export type MessagePart =
   | TextPart
   | ToolCallPart
   | ToolCallResponsePart
   | BlobPart
-  | FileDataPart
+  | FilePart
+  | UriPart
   | ThinkingPart
-  | GenericPart
+  | UnknownPart
 
 export type Role = 'system' | 'user' | 'assistant' | 'tool'