diff --git a/docs/readme.md b/docs/readme.md index 50f309e5..2f9be63f 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -109,6 +109,10 @@ Support Release Notes ------------- +### Upcoming +- Changes + - Firebase AI: Add support for enabling the model to use Code Execution. + ### 13.2.0 - Changes - General: Update to Firebase C++ SDK version 13.1.0. diff --git a/firebaseai/src/FunctionCalling.cs b/firebaseai/src/FunctionCalling.cs index 247597ce..633d4055 100644 --- a/firebaseai/src/FunctionCalling.cs +++ b/firebaseai/src/FunctionCalling.cs @@ -82,6 +82,14 @@ internal Dictionary ToJson() { /// public readonly struct GoogleSearch {} +/// +/// A tool that allows the model to execute code. +/// +/// This tool can be used to solve complex problems, for example, by generating and executing Python +/// code to solve a math problem. +/// +public readonly struct CodeExecution {} + /// /// A helper tool that the model may use when generating responses. /// @@ -93,6 +101,7 @@ public readonly struct Tool { private List FunctionDeclarations { get; } private GoogleSearch? GoogleSearch { get; } + private CodeExecution? CodeExecution { get; } /// /// Creates a tool that allows the model to perform function calling. @@ -102,6 +111,7 @@ public readonly struct Tool { public Tool(params FunctionDeclaration[] functionDeclarations) { FunctionDeclarations = new List(functionDeclarations); GoogleSearch = null; + CodeExecution = null; } /// /// Creates a tool that allows the model to perform function calling. @@ -111,6 +121,7 @@ public Tool(params FunctionDeclaration[] functionDeclarations) { public Tool(IEnumerable functionDeclarations) { FunctionDeclarations = new List(functionDeclarations); GoogleSearch = null; + CodeExecution = null; } /// @@ -121,6 +132,18 @@ public Tool(IEnumerable functionDeclarations) { public Tool(GoogleSearch googleSearch) { FunctionDeclarations = null; GoogleSearch = googleSearch; + CodeExecution = null; + } + + /// + /// Creates a tool that allows the model to use Code Execution. + /// + /// An empty `CodeExecution` object. The presence of this object + /// in the list of tools enables the model to use Code Execution. + public Tool(CodeExecution codeExecution) { + FunctionDeclarations = null; + GoogleSearch = null; + CodeExecution = codeExecution; } /// @@ -135,6 +158,9 @@ internal Dictionary ToJson() { if (GoogleSearch.HasValue) { json["googleSearch"] = new Dictionary(); } + if (CodeExecution.HasValue) { + json["codeExecution"] = new Dictionary(); + } return json; } } diff --git a/firebaseai/src/ModelContent.cs b/firebaseai/src/ModelContent.cs index 4e695709..7fa0f2bb 100644 --- a/firebaseai/src/ModelContent.cs +++ b/firebaseai/src/ModelContent.cs @@ -379,6 +379,153 @@ Dictionary Part.ToJson() { }; } } + + /// + /// A part containing code that was executed by the model. + /// + public readonly struct ExecutableCodePart : Part { + public enum CodeLanguage { + Unspecified = 0, + Python + } + + /// + /// The language + /// + public CodeLanguage Language { get; } + /// + /// The code that was executed. + /// + public string Code { get; } + + private readonly bool? _isThought; + public bool IsThought { get { return _isThought ?? false; } } + + private readonly string _thoughtSignature; + + private static CodeLanguage ParseLanguage(string str) { + return str switch { + "PYTHON" => CodeLanguage.Python, + _ => CodeLanguage.Unspecified, + }; + } + + private string LanguageAsString { + get { + return Language switch { + CodeLanguage.Python => "PYTHON", + _ => "LANGUAGE_UNSPECIFIED" + }; + } + } + + /// + /// Intended for internal use only. + /// + internal ExecutableCodePart(string language, string code, + bool? isThought, string thoughtSignature) { + Language = ParseLanguage(language); + Code = code; + _isThought = isThought; + _thoughtSignature = thoughtSignature; + } + + Dictionary Part.ToJson() { + var jsonDict = new Dictionary() { + { "executableCode", new Dictionary() { + { "language", LanguageAsString }, + { "code", Code } + } + } + }; + jsonDict.AddIfHasValue("thought", _isThought); + jsonDict.AddIfHasValue("thoughtSignature", _thoughtSignature); + return jsonDict; + } + } + + /// + /// A part containing the result of executing code. + /// + public readonly struct CodeExecutionResultPart : Part { + /// + /// The outcome of a code execution. + /// + public enum ExecutionOutcome { + Unspecified = 0, + /// + /// The code executed without errors. + /// + Ok, + /// + /// The code failed to execute. + /// + Failed, + /// + /// The code took too long to execute. + /// + DeadlineExceeded + } + + /// + /// The outcome of the code execution. + /// + public ExecutionOutcome Outcome { get; } + /// + /// The output of the code execution. + /// + public string Output { get; } + + private readonly bool? _isThought; + public bool IsThought { get { return _isThought ?? false; } } + + private readonly string _thoughtSignature; + + private static ExecutionOutcome ParseOutcome(string str) { + return str switch { + "OUTCOME_UNSPECIFIED" => ExecutionOutcome.Unspecified, + "OUTCOME_OK" => ExecutionOutcome.Ok, + "OUTCOME_FAILED" => ExecutionOutcome.Failed, + "OUTCOME_DEADLINE_EXCEEDED" => ExecutionOutcome.DeadlineExceeded, + _ => ExecutionOutcome.Unspecified, + }; + } + + private string OutcomeAsString { + get { + return Outcome switch { + ExecutionOutcome.Ok => "OUTCOME_OK", + ExecutionOutcome.Failed => "OUTCOME_FAILED", + ExecutionOutcome.DeadlineExceeded => "OUTCOME_DEADLINE_EXCEEDED", + _ => "OUTCOME_UNSPECIFIED" + }; + } + } + + /// + /// Intended for internal use only. + /// + internal CodeExecutionResultPart(string outcome, string output, + bool? isThought, string thoughtSignature) { + Outcome = ParseOutcome(outcome); + Output = output; + _isThought = isThought; + _thoughtSignature = thoughtSignature; + } + + Dictionary Part.ToJson() { + var jsonDict = new Dictionary() { + { "codeExecutionResult", new Dictionary() { + { "outcome", OutcomeAsString }, + { "output", Output } + } + } + }; + jsonDict.AddIfHasValue("thought", _isThought); + jsonDict.AddIfHasValue("thoughtSignature", _thoughtSignature); + return jsonDict; + } + } #endregion @@ -413,6 +560,24 @@ private static InlineDataPart InlineDataPartFromJson(Dictionary isThought, thoughtSignature); } + + private static ExecutableCodePart ExecutableCodePartFromJson(Dictionary jsonDict, + bool? isThought, string thoughtSignature) { + return new ExecutableCodePart( + jsonDict.ParseValue("language", JsonParseOptions.ThrowEverything), + jsonDict.ParseValue("code", JsonParseOptions.ThrowEverything), + isThought, + thoughtSignature); + } + + private static CodeExecutionResultPart CodeExecutionResultPartFromJson(Dictionary jsonDict, + bool? isThought, string thoughtSignature) { + return new CodeExecutionResultPart( + jsonDict.ParseValue("outcome", JsonParseOptions.ThrowEverything), + jsonDict.ParseValue("output", JsonParseOptions.ThrowEverything), + isThought, + thoughtSignature); + } private static Part PartFromJson(Dictionary jsonDict) { bool? isThought = jsonDict.ParseNullableValue("thought"); @@ -427,6 +592,14 @@ private static Part PartFromJson(Dictionary jsonDict) { innerDict => InlineDataPartFromJson(innerDict, isThought, thoughtSignature), out var inlineDataPart)) { return inlineDataPart; + } else if (jsonDict.TryParseObject("executableCode", + innerDict => ExecutableCodePartFromJson(innerDict, isThought, thoughtSignature), + out var executableCodePart)) { + return executableCodePart; + } else if (jsonDict.TryParseObject("codeExecutionResult", + innerDict => CodeExecutionResultPartFromJson(innerDict, isThought, thoughtSignature), + out var codeExecutionResultPart)) { + return codeExecutionResultPart; } else { #if FIREBASEAI_DEBUG_LOGGING UnityEngine.Debug.LogWarning($"Received unknown part, with keys: {string.Join(',', jsonDict.Keys)}"); diff --git a/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs b/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs index 3465a126..880bfca9 100644 --- a/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs +++ b/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs @@ -75,6 +75,7 @@ protected override void Start() { TestImagenGenerateImageOptions, TestThinkingBudget, TestIncludeThoughts, + TestCodeExecution, }; // Set of tests that only run the single time. Func[] singleTests = { @@ -100,6 +101,7 @@ protected override void Start() { InternalTestGenerateImagesAllFiltered, InternalTestGenerateImagesBase64SomeFiltered, InternalTestThoughtSummary, + InternalTestCodeExecution, }; // Create the set of tests, combining the above lists. @@ -819,11 +821,6 @@ async Task TestThinkingBudget(Backend backend) { // Test requesting thought summaries. async Task TestIncludeThoughts(Backend backend) { // Thinking Budget requires at least the 2.5 model. - var tool = new Tool(new FunctionDeclaration( - "GetKeyword", "Call to retrieve a special keyword.", - new Dictionary() { - { "input", Schema.String("Input string") }, - })); var model = GetFirebaseAI(backend).GetGenerativeModel( modelName: "gemini-2.5-flash", generationConfig: new GenerationConfig( @@ -844,6 +841,26 @@ async Task TestIncludeThoughts(Backend backend) { Assert("ThoughtSummary was missing", !string.IsNullOrWhiteSpace(response.ThoughtSummary)); } + async Task TestCodeExecution(Backend backend) { + var model = GetFirebaseAI(backend).GetGenerativeModel( + modelName: ModelName, + tools: new Tool[] { new Tool(new CodeExecution()) } + ); + + var prompt = "What is the sum of the first 50 prime numbers? Generate and run code for the calculation."; + var chat = model.StartChat(); + var response = await chat.SendMessageAsync(prompt); + + string result = response.Text; + Assert("Response text was missing", !string.IsNullOrWhiteSpace(result)); + + var executableCodeParts = response.Candidates.First().Content.Parts.OfType(); + Assert("Missing ExecutableCodeParts", executableCodeParts.Any()); + + var codeExecutionResultParts = response.Candidates.First().Content.Parts.OfType(); + Assert("Missing CodeExecutionResultParts", codeExecutionResultParts.Any()); + } + // Test providing a file from a GCS bucket (Firebase Storage) to the model. async Task TestReadFile() { // GCS is currently only supported with VertexAI. @@ -1519,5 +1536,23 @@ async Task InternalTestThoughtSummary() { ValidateUsageMetadata(response.UsageMetadata, 13, 2, 39, 54); } + + async Task InternalTestCodeExecution() { + Dictionary json = await GetVertexJsonTestData("unary-success-code-execution.json"); + GenerateContentResponse response = GenerateContentResponse.FromJson(json, FirebaseAI.Backend.InternalProvider.VertexAI); + + AssertEq("Candidate count", response.Candidates.Count(), 1); + var candidate = response.Candidates.First(); + + var executableCodeParts = candidate.Content.Parts.OfType().ToList(); + AssertEq("ExecutableCodePart count", executableCodeParts.Count(), 1); + AssertEq("ExecutableCodePart language", executableCodeParts[0].Language, ModelContent.ExecutableCodePart.CodeLanguage.Python); + AssertEq("ExecutableCodePart code", executableCodeParts[0].Code, "prime_numbers = [2, 3, 5, 7, 11]\nsum_of_primes = sum(prime_numbers)\nprint(f'The sum of the first 5 prime numbers is: {sum_of_primes}')\n"); + + var codeExecutionResultParts = candidate.Content.Parts.OfType().ToList(); + AssertEq("CodeExecutionResultPart count", codeExecutionResultParts.Count(), 1); + AssertEq("CodeExecutionResultPart outcome", codeExecutionResultParts[0].Outcome, ModelContent.CodeExecutionResultPart.ExecutionOutcome.Ok); + AssertEq("CodeExecutionResultPart output", codeExecutionResultParts[0].Output, "The sum of the first 5 prime numbers is: 28\n"); + } } } diff --git a/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-code-execution.json b/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-code-execution.json new file mode 100644 index 00000000..e5be3bd5 --- /dev/null +++ b/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-code-execution.json @@ -0,0 +1,59 @@ +{ + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "To find the sum of the first 5 prime numbers, we first need to identify them.\nPrime numbers are natural numbers greater than 1 that have no positive divisors other than 1 and themselves.\n\nThe first few prime numbers are:\n1. 2\n2. 3\n3. 5\n4. 7\n5. 11\n\nNow, we will sum these numbers using a Python tool.\n\n" + }, + { + "executableCode": { + "language": "PYTHON", + "code": "prime_numbers = [2, 3, 5, 7, 11]\nsum_of_primes = sum(prime_numbers)\nprint(f'The sum of the first 5 prime numbers is: {sum_of_primes}')\n" + } + }, + { + "codeExecutionResult": { + "outcome": "OUTCOME_OK", + "output": "The sum of the first 5 prime numbers is: 28\n" + } + }, + { + "text": "The sum of the first 5 prime numbers (2, 3, 5, 7, and 11) is 28." + } + ] + }, + "finishReason": "STOP" + } + ], + "usageMetadata": { + "promptTokenCount": 20, + "candidatesTokenCount": 192, + "totalTokenCount": 775, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 20 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 192 + } + ], + "toolUsePromptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 181 + } + ], + "toolUsePromptTokenCount": 371, + "thoughtsTokenCount": 192 + }, + "modelVersion": "gemini-2.5-flash", + "createTime": "2025-09-03T14:12:08.303054Z", + "responseId": "uEy4aM6_Euqawu8P6LOzyQc" +} diff --git a/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-code-execution.json.meta b/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-code-execution.json.meta new file mode 100644 index 00000000..e8deca7b --- /dev/null +++ b/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-code-execution.json.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: a098576373d984373a20dc6a8278adde +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: