diff --git a/docs/readme.md b/docs/readme.md index 9e05648f..7fbf9a6c 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -112,6 +112,8 @@ Release Notes ### Upcoming - Changes - Firebase AI: Add support for Gemini's URL context tool. + - Firebase AI: Add more specific methods for sending realtime data to + the LiveSession. Deprecate the previous SendMediaChunksAsync method. ### 13.3.0 - Changes diff --git a/firebaseai/src/LiveSession.cs b/firebaseai/src/LiveSession.cs index 730fa043..ce93b408 100644 --- a/firebaseai/src/LiveSession.cs +++ b/firebaseai/src/LiveSession.cs @@ -125,10 +125,10 @@ public async Task SendAsync( if (functionParts.Count > 0) { Dictionary toolResponse = new() { - { "toolResponse", new Dictionary() { - { "functionResponses", functionParts.Select(frPart => (frPart as ModelContent.Part).ToJson()["functionResponse"]).ToList() } - }} - }; + { "toolResponse", new Dictionary() { + { "functionResponses", functionParts.Select(frPart => (frPart as ModelContent.Part).ToJson()["functionResponse"]).ToList() } + }} + }; var toolResponseBytes = Encoding.UTF8.GetBytes(Json.Serialize(toolResponse)); await InternalSendBytesAsync(new ArraySegment(toolResponseBytes), cancellationToken); @@ -147,15 +147,15 @@ public async Task SendAsync( // Prepare the message payload Dictionary contentDict = new() { - { "turnComplete", turnComplete } - }; + { "turnComplete", turnComplete } + }; if (content.HasValue) { contentDict["turns"] = new List(new[] { content?.ToJson() }); } Dictionary jsonDict = new() { - { "clientContent", contentDict } - }; + { "clientContent", contentDict } + }; var byteArray = Encoding.UTF8.GetBytes(Json.Serialize(jsonDict)); await InternalSendBytesAsync(new ArraySegment(byteArray), cancellationToken); @@ -166,23 +166,84 @@ public async Task SendAsync( /// /// A list of media chunks to send. /// A token to cancel the send operation. + /// + /// Use SendAudioRealtimeAsync, SendVideoRealtimeAsync, or SendTextRealtimeAsync instead. + /// + /// @deprecated Use SendAudioRealtimeAsync, SendVideoRealtimeAsync, or SendTextRealtimeAsync instead. + [Obsolete("Use SendAudioRealtimeAsync, SendVideoRealtimeAsync, or SendTextRealtimeAsync instead.")] public async Task SendMediaChunksAsync( List mediaChunks, CancellationToken cancellationToken = default) { if (mediaChunks == null) return; + await InternalSendRealtimeInputAsync("mediaChunks", + mediaChunks.Select(mc => (mc as ModelContent.Part).ToJson()["inlineData"]).ToList(), + cancellationToken); + } + + /// + /// Sends text data to the server in realtime. + /// + /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for + /// details about the realtime input usage. + /// + /// The text data to send. + /// A token to cancel the send operation. + public async Task SendTextRealtimeAsync( + string text, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(text)) return; + + await InternalSendRealtimeInputAsync("text", text, cancellationToken); + } + + /// + /// Sends audio data to the server in realtime. + /// + /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for + /// details about the realtime input usage. + /// + /// The audio data to send. + /// A token to cancel the send operation. + public async Task SendAudioRealtimeAsync( + ModelContent.InlineDataPart audio, + CancellationToken cancellationToken = default) + { + await InternalSendRealtimeInputAsync("audio", + (audio as ModelContent.Part).ToJson()["inlineData"], cancellationToken); + } + + /// + /// Sends video data to the server in realtime. + /// + /// Check https://ai.google.dev/api/live#bidigeneratecontentrealtimeinput for + /// details about the realtime input usage. + /// + /// The video data to send. + /// A token to cancel the send operation. + public async Task SendVideoRealtimeAsync( + ModelContent.InlineDataPart video, + CancellationToken cancellationToken = default) + { + await InternalSendRealtimeInputAsync("video", + (video as ModelContent.Part).ToJson()["inlineData"], cancellationToken); + } + + private async Task InternalSendRealtimeInputAsync( + string key, object data, CancellationToken cancellationToken) + { // Prepare the message payload. Dictionary jsonDict = new() { - { - "realtimeInput", new Dictionary() { - { - // InlineDataPart inherits from Part, so this conversion should be safe. - "mediaChunks", mediaChunks.Select(mc => (mc as ModelContent.Part).ToJson()["inlineData"]).ToList() + { + "realtimeInput", new Dictionary() { + { + key, data + } } } - } - }; + }; var byteArray = Encoding.UTF8.GetBytes(Json.Serialize(jsonDict)); await InternalSendBytesAsync(new ArraySegment(byteArray), cancellationToken); @@ -214,7 +275,7 @@ private static byte[] ConvertTo16BitPCM(float[] samples) public Task SendAudioAsync(float[] audioData, CancellationToken cancellationToken = default) { ModelContent.InlineDataPart inlineDataPart = new("audio/pcm", ConvertTo16BitPCM(audioData)); - return SendMediaChunksAsync(new List(new[] { inlineDataPart }), cancellationToken); + return SendAudioRealtimeAsync(inlineDataPart, cancellationToken); } ///