Merge branch 'main' into ep/unit-test-report-action

Author: emilypgoogle

firebase-ai/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Unreleased
 
+- [fixed] Fixed an issue causing streaming chat interactions to drop thought signatures. (#7562)
+
+# 17.6.0
+
+- [feature] Added support for server templates via `TemplateGenerativeModel` and
+  `TemplateImagenModel`. (#7503)
+
+# 17.5.0
+
 - [changed] Added better scheduling and louder output for Live API.
 - [changed] Added support for input and output transcription. (#7482)
 - [feature] Added support for sending realtime audio and video in a `LiveSession`.

firebase-ai/api.txt

@@ -36,6 +36,10 @@ package com.google.firebase.ai {
     method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.ai.type.Tool>? tools = null);
     method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.ai.type.Tool>? tools = null, com.google.firebase.ai.type.Content? systemInstruction = null);
     method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.LiveGenerativeModel liveModel(String modelName, com.google.firebase.ai.type.LiveGenerationConfig? generationConfig = null, java.util.List<com.google.firebase.ai.type.Tool>? tools = null, com.google.firebase.ai.type.Content? systemInstruction = null, com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions());
+    method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.TemplateGenerativeModel templateGenerativeModel();
+    method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.TemplateGenerativeModel templateGenerativeModel(com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions());
+    method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.TemplateImagenModel templateImagenModel();
+    method @com.google.firebase.ai.type.PublicPreviewAPI public com.google.firebase.ai.TemplateImagenModel templateImagenModel(com.google.firebase.ai.type.RequestOptions requestOptions = com.google.firebase.ai.type.RequestOptions());
     property public static final com.google.firebase.ai.FirebaseAI instance;
     field public static final com.google.firebase.ai.FirebaseAI.Companion Companion;
   }
@@ -83,6 +87,15 @@ package com.google.firebase.ai {
     method public suspend Object? connect(kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.LiveSession>);
   }
 
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateGenerativeModel {
+    method public suspend Object? generateContent(String templateId, java.util.Map<java.lang.String,?> inputs, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.GenerateContentResponse>);
+    method public kotlinx.coroutines.flow.Flow<com.google.firebase.ai.type.GenerateContentResponse> generateContentStream(String templateId, java.util.Map<java.lang.String,?> inputs);
+  }
+
+  @com.google.firebase.ai.type.PublicPreviewAPI public final class TemplateImagenModel {
+    method public suspend Object? generateImages(String templateId, java.util.Map<java.lang.String,?> inputs, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
+  }
+
 }
 
 package com.google.firebase.ai.java {
@@ -166,6 +179,29 @@ package com.google.firebase.ai.java {
     method public com.google.firebase.ai.java.LiveSessionFutures from(com.google.firebase.ai.type.LiveSession session);
   }
 
+  public abstract class TemplateGenerativeModelFutures {
+    method public static final com.google.firebase.ai.java.TemplateGenerativeModelFutures from(com.google.firebase.ai.TemplateGenerativeModel model);
+    method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.ai.type.GenerateContentResponse> generateContent(String templateId, java.util.Map<java.lang.String,?> inputs);
+    method public abstract org.reactivestreams.Publisher<com.google.firebase.ai.type.GenerateContentResponse> generateContentStream(String templateId, java.util.Map<java.lang.String,?> inputs);
+    method public abstract com.google.firebase.ai.TemplateGenerativeModel getGenerativeModel();
+    field public static final com.google.firebase.ai.java.TemplateGenerativeModelFutures.Companion Companion;
+  }
+
+  public static final class TemplateGenerativeModelFutures.Companion {
+    method public com.google.firebase.ai.java.TemplateGenerativeModelFutures from(com.google.firebase.ai.TemplateGenerativeModel model);
+  }
+
+  public abstract class TemplateImagenModelFutures {
+    method public static final com.google.firebase.ai.java.TemplateImagenModelFutures from(com.google.firebase.ai.TemplateImagenModel model);
+    method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>> generateImages(String templateId, java.util.Map<java.lang.String,?> inputs);
+    method public abstract com.google.firebase.ai.TemplateImagenModel getImageModel();
+    field public static final com.google.firebase.ai.java.TemplateImagenModelFutures.Companion Companion;
+  }
+
+  public static final class TemplateImagenModelFutures.Companion {
+    method public com.google.firebase.ai.java.TemplateImagenModelFutures from(com.google.firebase.ai.TemplateImagenModel model);
+  }
+
 }
 
 package com.google.firebase.ai.type {

firebase-ai/gradle.properties

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version=17.5.0
-latestReleasedVersion=17.4.0
+version=17.6.1
+latestReleasedVersion=17.6.0

firebase-ai/src/main/kotlin/com/google/firebase/ai/Chat.kt

@@ -22,6 +22,7 @@ import com.google.firebase.ai.type.GenerateContentResponse
 import com.google.firebase.ai.type.ImagePart
 import com.google.firebase.ai.type.InlineDataPart
 import com.google.firebase.ai.type.InvalidStateException
+import com.google.firebase.ai.type.Part
 import com.google.firebase.ai.type.TextPart
 import com.google.firebase.ai.type.content
 import java.util.LinkedList
@@ -133,6 +134,7 @@ public class Chat(
     val bitmaps = LinkedList<Bitmap>()
     val inlineDataParts = LinkedList<InlineDataPart>()
     val text = StringBuilder()
+    val parts = mutableListOf<Part>()
 
     /**
      * TODO: revisit when images and inline data are returned. This will cause issues with how
@@ -147,22 +149,17 @@ public class Chat(
             is ImagePart -> bitmaps.add(part.image)
             is InlineDataPart -> inlineDataParts.add(part)
           }
+          parts.add(part)
         }
       }
       .onCompletion {
         lock.release()
         if (it == null) {
           val content =
             content("model") {
-              for (bitmap in bitmaps) {
-                image(bitmap)
-              }
-              for (inlineDataPart in inlineDataParts) {
-                inlineData(inlineDataPart.inlineData, inlineDataPart.mimeType)
-              }
-              if (text.isNotBlank()) {
-                text(text.toString())
-              }
+              setParts(
+                parts.filterNot { part -> part is TextPart && !part.hasContent() }.toMutableList()
+              )
             }
 
           history.add(prompt)
@@ -224,3 +221,12 @@ public class Chat(
     }
   }
 }
+
+/**
+ * Returns true if the [TextPart] contains any content, either in its [TextPart.text] property or
+ * its [TextPart.thoughtSignature] property.
+ */
+private fun TextPart.hasContent(): Boolean {
+  if (text.isNotEmpty()) return true
+  return !thoughtSignature.isNullOrBlank()
+}

firebase-ai/src/main/kotlin/com/google/firebase/ai/FirebaseAI.kt

@@ -106,6 +106,29 @@ internal constructor(
     )
   }
 
+  /**
+   * Instantiates a new [TemplateGenerativeModel] given the provided parameters.
+   *
+   * @param requestOptions Configuration options for sending requests to the backend.
+   * @return The initialized [TemplateGenerativeModel] instance.
+   */
+  @JvmOverloads
+  @PublicPreviewAPI
+  public fun templateGenerativeModel(
+    requestOptions: RequestOptions = RequestOptions(),
+  ): TemplateGenerativeModel {
+    val templateUri = getTemplateUri(backend)
+    return TemplateGenerativeModel(
+      templateUri,
+      firebaseApp.options.apiKey,
+      firebaseApp,
+      useLimitedUseAppCheckTokens,
+      requestOptions,
+      appCheckProvider.get(),
+      internalAuthProvider.get(),
+    )
+  }
+
   /**
    * Instantiates a new [LiveGenerationConfig] given the provided parameters.
    *
@@ -205,6 +228,29 @@ internal constructor(
     )
   }
 
+  /**
+   * Instantiates a new [TemplateImagenModel] given the provided parameters.
+   *
+   * @param requestOptions Configuration options for sending requests to the backend.
+   * @return The initialized [TemplateImagenModel] instance.
+   */
+  @JvmOverloads
+  @PublicPreviewAPI
+  public fun templateImagenModel(
+    requestOptions: RequestOptions = RequestOptions(),
+  ): TemplateImagenModel {
+    val templateUri = getTemplateUri(backend)
+    return TemplateImagenModel(
+      templateUri,
+      firebaseApp.options.apiKey,
+      firebaseApp,
+      useLimitedUseAppCheckTokens,
+      requestOptions,
+      appCheckProvider.get(),
+      internalAuthProvider.get(),
+    )
+  }
+
   public companion object {
     /** The [FirebaseAI] instance for the default [FirebaseApp] using the Google AI Backend. */
     @JvmStatic
@@ -258,6 +304,13 @@ internal constructor(
 
     private val TAG = FirebaseAI::class.java.simpleName
   }
+
+  private fun getTemplateUri(backend: GenerativeBackend): String =
+    when (backend.backend) {
+      GenerativeBackendEnum.VERTEX_AI ->
+        "projects/${firebaseApp.options.projectId}/locations/${backend.location}/templates/"
+      GenerativeBackendEnum.GOOGLE_AI -> "projects/${firebaseApp.options.projectId}/templates/"
+    }
 }
 
 /** The [FirebaseAI] instance for the default [FirebaseApp] using the Google AI Backend. */

firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt

@@ -233,7 +233,7 @@ internal constructor(
 }
 
 @OptIn(PublicPreviewAPI::class)
-private fun ImagenGenerationResponse.Internal.validate(): ImagenGenerationResponse.Internal {
+internal fun ImagenGenerationResponse.Internal.validate(): ImagenGenerationResponse.Internal {
   if (predictions.none { it.mimeType != null }) {
     throw ContentBlockedException(
       message = predictions.first { it.raiFilteredReason != null }.raiFilteredReason

firebase-ai/src/main/kotlin/com/google/firebase/ai/TemplateGenerativeModel.kt

@@ -0,0 +1,141 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.firebase.ai
+
+import com.google.firebase.FirebaseApp
+import com.google.firebase.ai.common.APIController
+import com.google.firebase.ai.common.AppCheckHeaderProvider
+import com.google.firebase.ai.common.TemplateGenerateContentRequest
+import com.google.firebase.ai.type.Content
+import com.google.firebase.ai.type.FinishReason
+import com.google.firebase.ai.type.FirebaseAIException
+import com.google.firebase.ai.type.GenerateContentResponse
+import com.google.firebase.ai.type.PromptBlockedException
+import com.google.firebase.ai.type.PublicPreviewAPI
+import com.google.firebase.ai.type.RequestOptions
+import com.google.firebase.ai.type.ResponseStoppedException
+import com.google.firebase.ai.type.SerializationException
+import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider
+import com.google.firebase.auth.internal.InternalAuthProvider
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.catch
+import kotlinx.coroutines.flow.map
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.jsonObject
+import org.json.JSONObject
+
+/**
+ * Represents a multimodal model (like Gemini), capable of generating content based on various
+ * templated input types.
+ */
+@PublicPreviewAPI
+public class TemplateGenerativeModel
+internal constructor(
+  private val templateUri: String,
+  private val controller: APIController,
+) {
+
+  internal constructor(
+    templateUri: String,
+    apiKey: String,
+    firebaseApp: FirebaseApp,
+    useLimitedUseAppCheckTokens: Boolean,
+    requestOptions: RequestOptions = RequestOptions(),
+    appCheckTokenProvider: InteropAppCheckTokenProvider? = null,
+    internalAuthProvider: InternalAuthProvider? = null
+  ) : this(
+    templateUri,
+    APIController(
+      apiKey,
+      "",
+      requestOptions,
+      "gl-kotlin/${KotlinVersion.CURRENT}-ai fire/${BuildConfig.VERSION_NAME}",
+      firebaseApp,
+      AppCheckHeaderProvider(
+        TAG,
+        useLimitedUseAppCheckTokens,
+        appCheckTokenProvider,
+        internalAuthProvider
+      ),
+    ),
+  )
+
+  /**
+   * Generates content from a prompt template and inputs.
+   *
+   * @param templateId The ID of the prompt template to use.
+   * @param inputs A map of variables to substitute into the template.
+   * @return The content generated by the model.
+   * @throws [FirebaseAIException] if the request failed.
+   * @see [FirebaseAIException] for types of errors.
+   */
+  public suspend fun generateContent(
+    templateId: String,
+    inputs: Map<String, Any>,
+  ): GenerateContentResponse =
+    try {
+      controller
+        .templateGenerateContent("$templateUri$templateId", constructRequest(inputs))
+        .toPublic()
+        .validate()
+    } catch (e: Throwable) {
+      throw FirebaseAIException.from(e)
+    }
+
+  /**
+   * Generates content as a stream from a prompt template and inputs.
+   *
+   * @param templateId The ID of the prompt template to use.
+   * @param inputs A map of variables to substitute into the template.
+   * @return A [Flow] which will emit responses as they are returned by the model.
+   * @throws [FirebaseAIException] if the request failed.
+   * @see [FirebaseAIException] for types of errors.
+   */
+  public fun generateContentStream(
+    templateId: String,
+    inputs: Map<String, Any>
+  ): Flow<GenerateContentResponse> =
+    controller
+      .templateGenerateContentStream("$templateUri$templateId", constructRequest(inputs))
+      .catch { throw FirebaseAIException.from(it) }
+      .map { it.toPublic().validate() }
+
+  internal fun constructRequest(
+    inputs: Map<String, Any>,
+    history: List<Content>? = null
+  ): TemplateGenerateContentRequest {
+    return TemplateGenerateContentRequest(
+      Json.parseToJsonElement(JSONObject(inputs).toString()).jsonObject,
+      history?.let { it.map { it.toTemplateInternal() } }
+    )
+  }
+
+  private fun GenerateContentResponse.validate() = apply {
+    if (candidates.isEmpty() && promptFeedback == null) {
+      throw SerializationException("Error deserializing response, found no valid fields")
+    }
+    promptFeedback?.blockReason?.let { throw PromptBlockedException(this) }
+    candidates
+      .mapNotNull { it.finishReason }
+      .firstOrNull { it != FinishReason.STOP }
+      ?.let { throw ResponseStoppedException(this) }
+  }
+
+  private companion object {
+    private val TAG = TemplateGenerativeModel::class.java.simpleName
+  }
+}