[pull] main from tensorzero:main

.github/workflows/slash-command-merge-queue.yml

@@ -34,8 +34,8 @@ jobs:
           set -e
           gh workflow run merge-queue.yml --ref ${{ github.event.client_payload.pull_request.head.ref }}
 
-          # Wait a moment for the workflow to be created
-          sleep 3
+          # Wait 30s for the workflow to be created
+          sleep 30
 
           # Get the most recent workflow run for this branch
           RUN_ID=$(gh run list --workflow=merge-queue.yml --branch=${{ github.event.client_payload.pull_request.head.ref }} --limit=1 --json databaseId --jq '.[0].databaseId')

clients/rust/src/input_handling.rs

@@ -39,8 +39,8 @@ fn resolved_input_message_content_to_client_input_message_content(
     resolved_input_message_content: ResolvedInputMessageContent,
 ) -> ClientInputMessageContent {
     match resolved_input_message_content {
-        ResolvedInputMessageContent::Text { text } => {
-            ClientInputMessageContent::Text(TextKind::Text { text })
+        ResolvedInputMessageContent::Text(text) => {
+            ClientInputMessageContent::Text(TextKind::Text { text: text.text })
         }
         ResolvedInputMessageContent::Template(template) => {
             ClientInputMessageContent::Template(template)

evaluations/src/evaluators/exact_match.rs

@@ -76,7 +76,7 @@ mod tests {
         },
         inference::types::{
             ContentBlockChatOutput, JsonInferenceOutput, StoredInput, StoredInputMessage,
-            StoredInputMessageContent, Text, Usage,
+            StoredInputMessageContent, TemplateInput, Text, Usage,
         },
     };
     use uuid::Uuid;
@@ -90,9 +90,9 @@ mod tests {
                 system: None,
                 messages: vec![StoredInputMessage {
                     role: Role::User,
-                    content: vec![StoredInputMessageContent::Text {
-                        value: json!("Hello, world!"),
-                    }],
+                    content: vec![StoredInputMessageContent::Text(Text {
+                        text: "Hello, world!".to_string(),
+                    })],
                 }],
             },
             dataset_name: "test".to_string(),
@@ -153,9 +153,9 @@ mod tests {
                 system: None,
                 messages: vec![StoredInputMessage {
                     role: Role::User,
-                    content: vec![StoredInputMessageContent::Text {
-                        value: json!("Hello, world!"),
-                    }],
+                    content: vec![StoredInputMessageContent::Text(Text {
+                        text: "Hello, world!".to_string(),
+                    })],
                 }],
             },
             name: None,
@@ -185,9 +185,10 @@ mod tests {
                 system: None,
                 messages: vec![StoredInputMessage {
                     role: Role::User,
-                    content: vec![StoredInputMessageContent::Text {
-                        value: json!({"foo": "bar"}),
-                    }],
+                    content: vec![StoredInputMessageContent::Template(TemplateInput {
+                        name: "user".to_string(),
+                        arguments: serde_json::from_value(json!({"foo": "bar"})).unwrap(),
+                    })],
                 }],
             },
             dataset_name: "test".to_string(),
@@ -258,9 +259,10 @@ mod tests {
                 system: None,
                 messages: vec![StoredInputMessage {
                     role: Role::User,
-                    content: vec![StoredInputMessageContent::Text {
-                        value: json!({"foo": "bar"}),
-                    }],
+                    content: vec![StoredInputMessageContent::Template(TemplateInput {
+                        name: "user".to_string(),
+                        arguments: serde_json::from_value(json!({"foo": "bar"})).unwrap(),
+                    })],
                 }],
             },
             dataset_name: "test".to_string(),
@@ -294,9 +296,10 @@ mod tests {
                 system: None,
                 messages: vec![StoredInputMessage {
                     role: Role::User,
-                    content: vec![StoredInputMessageContent::Text {
-                        value: json!({"foo": "bar"}),
-                    }],
+                    content: vec![StoredInputMessageContent::Template(TemplateInput {
+                        name: "user".to_string(),
+                        arguments: serde_json::from_value(json!({"foo": "bar"})).unwrap(),
+                    })],
                 }],
             },
             dataset_name: "test".to_string(),

evaluations/tests/tests.rs

@@ -17,7 +17,7 @@ use tensorzero_core::endpoints::datasets::Datapoint;
 use tensorzero_core::evaluations::{LLMJudgeConfig, LLMJudgeInputFormat, LLMJudgeOutputType};
 use tensorzero_core::function::{FunctionConfig, FunctionConfigJson};
 use tensorzero_core::inference::types::{
-    StoredInput, StoredInputMessage, StoredInputMessageContent, TemplateInput, Text,
+    StoredInput, StoredInputMessage, StoredInputMessageContent, Text,
 };
 use tokio::time::sleep;
 use url::Url;
@@ -386,22 +386,8 @@ async fn run_exact_match_evaluation_chat() {
         };
         let clickhouse_input: StoredInput =
             serde_json::from_str(clickhouse_inference["input"].as_str().unwrap()).unwrap();
-        // The fixture is parsed from the old-style template, so convert it in place
-        let mut parsed_input = parsed.datapoint.input().clone();
-        for message in &mut parsed_input.messages {
-            for content in &mut message.content {
-                if let StoredInputMessageContent::Text { value } = content {
-                    if value.is_object() {
-                        *content = StoredInputMessageContent::Template(TemplateInput {
-                            name: message.role.implicit_template_name().to_string(),
-                            arguments: value.as_object().unwrap().clone(),
-                        });
-                    }
-                }
-            }
-        }
         // Check the input to the inference is the same as the input to the datapoint
-        assert_eq!(&clickhouse_input, &parsed_input);
+        assert_eq!(&clickhouse_input, parsed.datapoint.input());
         let clickhouse_output: Vec<ContentBlockChatOutput> =
             serde_json::from_str(clickhouse_inference["output"].as_str().unwrap()).unwrap();
         // Check the output to the inference is the same as the output in the response
@@ -525,22 +511,8 @@ async fn run_llm_judge_evaluation_chat() {
         };
         let clickhouse_input: StoredInput =
             serde_json::from_str(clickhouse_inference["input"].as_str().unwrap()).unwrap();
-        // The fixture is parsed from the old-style template, so convert it in place
-        let mut parsed_input = parsed.datapoint.input().clone();
-        for message in &mut parsed_input.messages {
-            for content in &mut message.content {
-                if let StoredInputMessageContent::Text { value } = content {
-                    if value.is_object() {
-                        *content = StoredInputMessageContent::Template(TemplateInput {
-                            name: message.role.implicit_template_name().to_string(),
-                            arguments: value.as_object().unwrap().clone(),
-                        });
-                    }
-                }
-            }
-        }
         // Check the input to the inference is the same as the input to the datapoint
-        assert_eq!(&clickhouse_input, &parsed_input);
+        assert_eq!(&clickhouse_input, parsed.datapoint.input());
         let clickhouse_output: Vec<ContentBlockChatOutput> =
             serde_json::from_str(clickhouse_inference["output"].as_str().unwrap()).unwrap();
         // Check the output to the inference is the same as the output in the response
@@ -1311,9 +1283,9 @@ async fn test_run_llm_judge_evaluator_chat() {
             system: None,
             messages: vec![StoredInputMessage {
                 role: Role::User,
-                content: vec![StoredInputMessageContent::Text {
-                    value: json!("Hello, world!"),
-                }],
+                content: vec![StoredInputMessageContent::Text(Text {
+                    text: "Hello, world!".to_string(),
+                })],
             }],
         },
         auxiliary: String::new(),
@@ -1420,9 +1392,9 @@ async fn test_run_llm_judge_evaluator_chat() {
             system: None,
             messages: vec![StoredInputMessage {
                 role: Role::User,
-                content: vec![StoredInputMessageContent::Text {
-                    value: json!("Hello, world!"),
-                }],
+                content: vec![StoredInputMessageContent::Text(Text {
+                    text: "Hello, world!".to_string(),
+                })],
             }],
         },
         auxiliary: String::new(),
@@ -1486,9 +1458,9 @@ async fn test_run_llm_judge_evaluator_json() {
             system: None,
             messages: vec![StoredInputMessage {
                 role: Role::User,
-                content: vec![StoredInputMessageContent::Text {
-                    value: json!("Hello, world!"),
-                }],
+                content: vec![StoredInputMessageContent::Text(Text {
+                    text: "Hello, world!".to_string(),
+                })],
             }],
         },
         auxiliary: String::new(),
@@ -1596,9 +1568,9 @@ async fn test_run_llm_judge_evaluator_json() {
             system: None,
             messages: vec![StoredInputMessage {
                 role: Role::User,
-                content: vec![StoredInputMessageContent::Text {
-                    value: json!("Hello, world!"),
-                }],
+                content: vec![StoredInputMessageContent::Text(Text {
+                    text: "Hello, world!".to_string(),
+                })],
             }],
         },
         auxiliary: String::new(),

examples/integrations/cursor/feedback/src/cursor.rs

@@ -3,7 +3,7 @@ use regex::Regex;
 use serde_json::Value;
 use std::path::PathBuf;
 use tensorzero_core::inference::types::{
-    ContentBlockChatOutput, StoredInput, StoredInputMessage, StoredInputMessageContent,
+    ContentBlockChatOutput, StoredInput, StoredInputMessage, StoredInputMessageContent, Text,
 };
 /*
 This file handles the outputs of inferences from Cursor. We handle two cases:
@@ -138,12 +138,7 @@ fn parse_cursor_edit_output(
     }
     let second_message = &messages[1];
     let second_message_text = match second_message.content.as_slice() {
-        [StoredInputMessageContent::Text { value: text }] => {
-            let Value::String(text) = text else {
-                return Err(anyhow::anyhow!("Expected text in second user message"));
-            };
-            text
-        }
+        [StoredInputMessageContent::Text(Text { text })] => text,
         _ => {
             return Err(anyhow::anyhow!("Expected text in second user message"));
         }
@@ -211,12 +206,7 @@ fn parse_cursor_insert_output(
     // Extract workspace path from the first message
     let first_message = &messages[0];
     let first_message_text = match first_message.content.as_slice() {
-        [StoredInputMessageContent::Text { value }] => {
-            let Value::String(s) = value else {
-                return Err(anyhow::anyhow!("Expected text in first user message"));
-            };
-            s
-        }
+        [StoredInputMessageContent::Text(Text { text })] => text,
         _ => {
             return Err(anyhow::anyhow!(
                 "Expected the first user message to contain a plain-text block"

internal/tensorzero-node/lib/bindings/ResolvedInputMessageContent.ts

@@ -1,13 +1,14 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { FileWithPath } from "./FileWithPath";
 import type { TemplateInput } from "./TemplateInput";
+import type { Text } from "./Text";
 import type { Thought } from "./Thought";
 import type { ToolCall } from "./ToolCall";
 import type { ToolResult } from "./ToolResult";
 import type { JsonValue } from "./serde_json/JsonValue";
 
 export type ResolvedInputMessageContent =
-  | { type: "text"; text: string }
+  | ({ type: "text" } & Text)
   | ({ type: "template" } & TemplateInput)
   | ({ type: "tool_call" } & ToolCall)
   | ({ type: "tool_result" } & ToolResult)

internal/tensorzero-node/lib/bindings/StoredInput.ts

@@ -7,6 +7,8 @@ import type { JsonValue } from "./serde_json/JsonValue";
  * This is almost identical to `ResolvedInput`, but without `File` data.
  * Only the object-storage path is actually stored in clickhouse
  * (which can be used to re-fetch the file and produce a `ResolvedInput`).
+ *
+ * `StoredInputMessage` has a custom deserializer that addresses legacy data formats in the database.
  */
 export type StoredInput = {
   system?: JsonValue;

internal/tensorzero-node/lib/bindings/StoredInputMessage.ts

@@ -2,6 +2,9 @@
 import type { Role } from "./Role";
 import type { StoredInputMessageContent } from "./StoredInputMessageContent";
 
+/**
+ * `StoredInputMessage` has a custom deserializer that addresses legacy data formats in the database.
+ */
 export type StoredInputMessage = {
   role: Role;
   content: Array<StoredInputMessageContent>;

internal/tensorzero-node/lib/bindings/StoredInputMessageContent.ts

@@ -1,13 +1,14 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { StoredFile } from "./StoredFile";
 import type { TemplateInput } from "./TemplateInput";
+import type { Text } from "./Text";
 import type { Thought } from "./Thought";
 import type { ToolCall } from "./ToolCall";
 import type { ToolResult } from "./ToolResult";
 import type { JsonValue } from "./serde_json/JsonValue";
 
 export type StoredInputMessageContent =
-  | { type: "text"; value: JsonValue }
+  | ({ type: "text" } & Text)
   | ({ type: "template" } & TemplateInput)
   | ({ type: "tool_call" } & ToolCall)
   | ({ type: "tool_result" } & ToolResult)

tensorzero-core/src/db/clickhouse/dataset_queries.rs

@@ -384,7 +384,7 @@ impl DatasetQueries for ClickHouseConnectionInfo {
         params: &CountDatapointsForDatasetFunctionParams,
     ) -> Result<u32, Error> {
         let query = "
-        SELECT toUInt32(count()) as count 
+        SELECT toUInt32(count()) as count
         FROM {table:Identifier}
         WHERE dataset_name = {dataset_name:String}
             AND function_name = {function_name:String}";
@@ -3124,18 +3124,18 @@ mod tests {
                     '' as output_schema,");
                 assert_query_contains(query,
                     "tags,
-                    auxiliary, 
-                    source_inference_id, 
-                    is_deleted, 
-                    is_custom, 
-                    staled_at, 
+                    auxiliary,
+                    source_inference_id,
+                    is_deleted,
+                    is_custom,
+                    staled_at,
                     formatDateTime(updated_at, '%Y-%m-%dT%H:%i:%SZ') AS updated_at");
                 assert_query_contains(query, "FROM ChatInferenceDatapoint AS i FINAL
                     WHERE true
                     AND dataset_name = {dataset_name:String}
                     AND id IN ['123e4567-e89b-12d3-a456-426614174000']
                     AND staled_at IS NULL");
-                assert_query_contains(query, "ORDER BY updated_at DESC, id DESC 
+                assert_query_contains(query, "ORDER BY updated_at DESC, id DESC
                     LIMIT {subquery_page_size:UInt32}");
                 assert_query_contains(query, "UNION ALL");
                 assert_query_contains(query, "

tensorzero-core/src/endpoints/datasets/v1/update_datapoints.rs

@@ -629,9 +629,9 @@ mod tests {
                     system: None,
                     messages: vec![crate::inference::types::StoredInputMessage {
                         role: Role::User,
-                        content: vec![StoredInputMessageContent::Text {
-                            value: json!("original input"),
-                        }],
+                        content: vec![StoredInputMessageContent::Text(Text {
+                            text: "original input".to_string(),
+                        })],
                     }],
                 },
                 output: Some(vec![ContentBlockChatOutput::Text(Text {
@@ -666,9 +666,9 @@ mod tests {
                     system: None,
                     messages: vec![crate::inference::types::StoredInputMessage {
                         role: Role::User,
-                        content: vec![StoredInputMessageContent::Text {
-                            value: json!("original input"),
-                        }],
+                        content: vec![StoredInputMessageContent::Text(Text {
+                            text: "original input".to_string(),
+                        })],
                     }],
                 },
                 output: Some(JsonInferenceOutput {
@@ -800,9 +800,8 @@ mod tests {
             // Input should be updated
             assert_eq!(updated.input.messages.len(), 1);
             match &updated.input.messages[0].content[0] {
-                StoredInputMessageContent::Text { value } => {
-                    let text: String = serde_json::from_value(value.clone()).unwrap();
-                    assert_eq!(text, "new input text");
+                StoredInputMessageContent::Text(text) => {
+                    assert_eq!(text.text, "new input text");
                 }
                 _ => panic!("Expected text content"),
             }

tensorzero-core/src/inference/types/mod.rs

@@ -416,7 +416,7 @@ impl LazyResolvedInputMessageContent {
     pub async fn resolve(self) -> Result<ResolvedInputMessageContent, Error> {
         Ok(match self {
             LazyResolvedInputMessageContent::Text { text } => {
-                ResolvedInputMessageContent::Text { text }
+                ResolvedInputMessageContent::Text(Text { text })
             }
             LazyResolvedInputMessageContent::Template(template) => {
                 ResolvedInputMessageContent::Template(template)
@@ -1467,7 +1467,7 @@ impl From<String> for InputMessageContent {
 #[cfg(test)]
 impl From<String> for ResolvedInputMessageContent {
     fn from(text: String) -> Self {
-        ResolvedInputMessageContent::Text { text }
+        ResolvedInputMessageContent::Text(Text { text })
     }
 }