From d23a093b3d68871f91dfe143a24082ee0ff4162a Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 10:13:58 -0700 Subject: [PATCH 01/12] add safety scores to image generation --- .../com/google/firebase/ai/ImagenModel.kt | 2 ++ .../com/google/firebase/ai/common/Request.kt | 1 + .../ai/type/ImagenGenerationResponse.kt | 19 +++++++++++++++++-- .../firebase/ai/type/ImagenInlineImage.kt | 8 ++++++-- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index b82113bbf91..bbadb9199a1 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -177,6 +177,7 @@ internal constructor( GenerateImageRequest.ImagenParameters( sampleCount = generationConfig?.numberOfImages ?: 1, includeRaiReason = true, + includeSafetyAttributes = true, addWatermark = generationConfig?.addWatermark, personGeneration = safetySettings?.personFilterLevel?.internalVal, negativePrompt = generationConfig?.negativePrompt, @@ -206,6 +207,7 @@ internal constructor( GenerateImageRequest.ImagenParameters( sampleCount = generationConfig?.numberOfImages ?: 1, includeRaiReason = true, + includeSafetyAttributes = true, addWatermark = generationConfig?.addWatermark, personGeneration = safetySettings?.personFilterLevel?.internalVal, negativePrompt = generationConfig?.negativePrompt, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt index 3f8e0ae079d..43a6f648aad 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt @@ -93,6 +93,7 @@ internal data class GenerateImageRequest( internal data class ImagenParameters( val sampleCount: Int, val includeRaiReason: Boolean, + val includeSafetyAttributes: Boolean, val storageUri: String?, val negativePrompt: String?, val aspectRatio: String?, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt index 67f13cff199..4dc3428a839 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt @@ -42,7 +42,7 @@ internal constructor(public val images: List, public val filteredReason: Stri internal fun toPublicInline() = ImagenGenerationResponse( images = predictions.filter { it.mimeType != null }.map { it.toPublicInline() }, - null, + predictions.firstNotNullOfOrNull { it.raiFilteredReason }, ) } @@ -52,10 +52,25 @@ internal constructor(public val images: List, public val filteredReason: Stri val gcsUri: String? = null, val mimeType: String? = null, val raiFilteredReason: String? = null, + val safetyAttributes: ImagenSafetyAttributes? = null, ) { internal fun toPublicInline() = - ImagenInlineImage(Base64.decode(bytesBase64Encoded!!, Base64.NO_WRAP), mimeType!!) + ImagenInlineImage( + Base64.decode(bytesBase64Encoded!!, Base64.NO_WRAP), + mimeType!!, + safetyAttributes?.toPublic() ?: emptyMap() + ) internal fun toPublicGCS() = ImagenGCSImage(gcsUri!!, mimeType!!) } + + @Serializable + internal data class ImagenSafetyAttributes( + val categories: List? = null, + val scores: List? = null + ){ + internal fun toPublic(): Map { + return categories?.zip(scores!!)?.toMap() ?: emptyMap() + } + } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt index fc6033d5390..de6874dae9c 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt @@ -28,10 +28,14 @@ import kotlinx.serialization.Serializable * @property data The raw image bytes in JPEG or PNG format, as specified by [mimeType]. * @property mimeType The IANA standard MIME type of the image data; either `"image/png"` or * `"image/jpeg"`; to request a different format, see [ImagenGenerationConfig.imageFormat]. + * @property safetyAttributes a set of safety attributes with their associated score. */ @PublicPreviewAPI public class ImagenInlineImage -internal constructor(public val data: ByteArray, public val mimeType: String) { +internal constructor( + public val data: ByteArray, + public val mimeType: String, + public val safetyAttributes: Map) { /** * Returns the image as an Android OS native [Bitmap] so that it can be saved or sent to the UI. @@ -53,5 +57,5 @@ public fun Bitmap.toImagenInlineImage(): ImagenInlineImage { val byteArrayOutputStream = ByteArrayOutputStream() this.compress(Bitmap.CompressFormat.JPEG, 80, byteArrayOutputStream) val byteArray = byteArrayOutputStream.toByteArray() - return ImagenInlineImage(data = byteArray, mimeType = "image/jpeg") + return ImagenInlineImage(data = byteArray, mimeType = "image/jpeg", safetyAttributes = emptyMap()) } From e79db7173f5dbf12ac1ec992854597d7621bbd45 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 10:15:55 -0700 Subject: [PATCH 02/12] formatting and changelog --- firebase-ai/CHANGELOG.md | 1 + .../com/google/firebase/ai/type/ImagenGenerationResponse.kt | 4 ++-- .../kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/firebase-ai/CHANGELOG.md b/firebase-ai/CHANGELOG.md index ea6bb088f63..1c5883fa158 100644 --- a/firebase-ai/CHANGELOG.md +++ b/firebase-ai/CHANGELOG.md @@ -1,4 +1,5 @@ # Unreleased +- [feature] Added support for `ImagenGenerationResponse.safetyAttributes` (#7322) # 17.2.0 diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt index 4dc3428a839..f9a632a4507 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt @@ -42,7 +42,7 @@ internal constructor(public val images: List, public val filteredReason: Stri internal fun toPublicInline() = ImagenGenerationResponse( images = predictions.filter { it.mimeType != null }.map { it.toPublicInline() }, - predictions.firstNotNullOfOrNull { it.raiFilteredReason }, + predictions.firstNotNullOfOrNull { it.raiFilteredReason }, ) } @@ -68,7 +68,7 @@ internal constructor(public val images: List, public val filteredReason: Stri internal data class ImagenSafetyAttributes( val categories: List? = null, val scores: List? = null - ){ + ) { internal fun toPublic(): Map { return categories?.zip(scores!!)?.toMap() ?: emptyMap() } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt index de6874dae9c..0766b35adf8 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt @@ -35,7 +35,8 @@ public class ImagenInlineImage internal constructor( public val data: ByteArray, public val mimeType: String, - public val safetyAttributes: Map) { + public val safetyAttributes: Map +) { /** * Returns the image as an Android OS native [Bitmap] so that it can be saved or sent to the UI. From 1219d9cc3e94e82fdc9e9141d90f4c3ac5b72e5e Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 10:17:27 -0700 Subject: [PATCH 03/12] improved null handling --- .../com/google/firebase/ai/type/ImagenGenerationResponse.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt index f9a632a4507..6a76bec6006 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt @@ -70,7 +70,10 @@ internal constructor(public val images: List, public val filteredReason: Stri val scores: List? = null ) { internal fun toPublic(): Map { - return categories?.zip(scores!!)?.toMap() ?: emptyMap() + if (categories == null || scores == null) { + return emptyMap() + } + return categories.zip(scores).toMap() } } } From 0da743bcd16ccbb6edf11fa1a24b642e70a299d2 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 10:22:50 -0700 Subject: [PATCH 04/12] add api.txt file --- firebase-ai/api.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index 2a49552828a..dbf689d698d 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -709,8 +709,10 @@ package com.google.firebase.ai.type { method public android.graphics.Bitmap asBitmap(); method public byte[] getData(); method public String getMimeType(); + method public java.util.Map getSafetyAttributes(); property public final byte[] data; property public final String mimeType; + property public final java.util.Map safetyAttributes; } public final class ImagenInlineImageKt { From 1f599fea85373d7ade6e0bb401c4ea6bdc6aba4e Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 10:49:47 -0700 Subject: [PATCH 05/12] fix changelog formatting --- firebase-ai/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/firebase-ai/CHANGELOG.md b/firebase-ai/CHANGELOG.md index 39ae2d9665f..6b96834bef9 100644 --- a/firebase-ai/CHANGELOG.md +++ b/firebase-ai/CHANGELOG.md @@ -1,4 +1,5 @@ # Unreleased + - [feature] Added support for `ImagenGenerationResponse.safetyAttributes` (#7322) - [feature] Introduced `MissingPermissionsException`, which is thrown when the necessary permissions From 71a280e6e4c7c0e1559efdc00e4b9fcb7ca77c3e Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 10:51:25 -0700 Subject: [PATCH 06/12] fix formatting (again) --- firebase-ai/CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/firebase-ai/CHANGELOG.md b/firebase-ai/CHANGELOG.md index 6b96834bef9..5a61544e701 100644 --- a/firebase-ai/CHANGELOG.md +++ b/firebase-ai/CHANGELOG.md @@ -1,7 +1,6 @@ # Unreleased - [feature] Added support for `ImagenGenerationResponse.safetyAttributes` (#7322) - - [feature] Introduced `MissingPermissionsException`, which is thrown when the necessary permissions have not been granted by the user. - [feature] Added helper functions to `LiveSession` to allow developers to track the status of the From 556701edd15e1c656eed36fd3530c47446d5d25e Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 13:48:25 -0700 Subject: [PATCH 07/12] add test --- .../google/firebase/ai/VertexAIUnarySnapshotTests.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt index e34a1e1db0d..4c14cf912d0 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt @@ -591,6 +591,18 @@ internal class VertexAIUnarySnapshotTests { } } + @Test + fun `generateImages should contain safety data`() = + goldenVertexUnaryFile("unary-success-generate-images-safety_info.json") { + withTimeout(testTimeout) { + val response = imagenModel.generateImages("prompt") + response.images[0].safetyAttributes shouldNotBe emptyMap() + response.images[1].safetyAttributes shouldNotBe emptyMap() + response.images[2].safetyAttributes shouldNotBe emptyMap() + response.images[3].safetyAttributes shouldBe emptyMap() + } + } + @Test fun `google search grounding metadata is parsed correctly`() = goldenVertexUnaryFile("unary-success-google-search-grounding.json") { From bada683acc1a55f05f982cd3953def595d7e236d Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 15:48:12 -0700 Subject: [PATCH 08/12] remove public API --- firebase-ai/api.txt | 2 -- .../com/google/firebase/ai/type/ImagenInlineImage.kt | 3 +-- .../google/firebase/ai/VertexAIUnarySnapshotTests.kt | 12 ------------ 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index 9d8b5cfe33d..6edae50559c 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -713,10 +713,8 @@ package com.google.firebase.ai.type { method public android.graphics.Bitmap asBitmap(); method public byte[] getData(); method public String getMimeType(); - method public java.util.Map getSafetyAttributes(); property public final byte[] data; property public final String mimeType; - property public final java.util.Map safetyAttributes; } public final class ImagenInlineImageKt { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt index 0766b35adf8..3795b5d3f96 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt @@ -35,7 +35,6 @@ public class ImagenInlineImage internal constructor( public val data: ByteArray, public val mimeType: String, - public val safetyAttributes: Map ) { /** @@ -58,5 +57,5 @@ public fun Bitmap.toImagenInlineImage(): ImagenInlineImage { val byteArrayOutputStream = ByteArrayOutputStream() this.compress(Bitmap.CompressFormat.JPEG, 80, byteArrayOutputStream) val byteArray = byteArrayOutputStream.toByteArray() - return ImagenInlineImage(data = byteArray, mimeType = "image/jpeg", safetyAttributes = emptyMap()) + return ImagenInlineImage(data = byteArray, mimeType = "image/jpeg") } diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt index 4c14cf912d0..e34a1e1db0d 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt @@ -591,18 +591,6 @@ internal class VertexAIUnarySnapshotTests { } } - @Test - fun `generateImages should contain safety data`() = - goldenVertexUnaryFile("unary-success-generate-images-safety_info.json") { - withTimeout(testTimeout) { - val response = imagenModel.generateImages("prompt") - response.images[0].safetyAttributes shouldNotBe emptyMap() - response.images[1].safetyAttributes shouldNotBe emptyMap() - response.images[2].safetyAttributes shouldNotBe emptyMap() - response.images[3].safetyAttributes shouldBe emptyMap() - } - } - @Test fun `google search grounding metadata is parsed correctly`() = goldenVertexUnaryFile("unary-success-google-search-grounding.json") { From 8086ccbbc6e44f66a12aa38c22b77fbace05cb33 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 15:48:32 -0700 Subject: [PATCH 09/12] remove changelog --- firebase-ai/CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/firebase-ai/CHANGELOG.md b/firebase-ai/CHANGELOG.md index 53c3bfe56ac..d63e4781242 100644 --- a/firebase-ai/CHANGELOG.md +++ b/firebase-ai/CHANGELOG.md @@ -1,6 +1,5 @@ # Unreleased -- [feature] Added support for `ImagenGenerationResponse.safetyAttributes` (#7322) - [feature] Introduced `MissingPermissionsException`, which is thrown when the necessary permissions have not been granted by the user. - [feature] Added helper functions to `LiveSession` to allow developers to track the status of the From e5be3bf6dcc550e6bf8e2f1ccc0c3a441b0b2853 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 3 Sep 2025 15:50:14 -0700 Subject: [PATCH 10/12] fix constructor --- .../com/google/firebase/ai/type/ImagenGenerationResponse.kt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt index 6a76bec6006..f93854851e6 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenGenerationResponse.kt @@ -55,11 +55,7 @@ internal constructor(public val images: List, public val filteredReason: Stri val safetyAttributes: ImagenSafetyAttributes? = null, ) { internal fun toPublicInline() = - ImagenInlineImage( - Base64.decode(bytesBase64Encoded!!, Base64.NO_WRAP), - mimeType!!, - safetyAttributes?.toPublic() ?: emptyMap() - ) + ImagenInlineImage(Base64.decode(bytesBase64Encoded!!, Base64.NO_WRAP), mimeType!!) internal fun toPublicGCS() = ImagenGCSImage(gcsUri!!, mimeType!!) } From 9c113c61ad96444bc19257cc191656486d0a4303 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Mon, 8 Sep 2025 10:44:20 -0700 Subject: [PATCH 11/12] re-added truncated test --- .../com/google/firebase/ai/VertexAIUnarySnapshotTests.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt index e34a1e1db0d..a57f2f14c96 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/VertexAIUnarySnapshotTests.kt @@ -591,6 +591,15 @@ internal class VertexAIUnarySnapshotTests { } } + @Test + fun `generateImages should contain safety data`() = + goldenVertexUnaryFile("unary-success-generate-images-safety_info.json") { + withTimeout(testTimeout) { + val response = imagenModel.generateImages("prompt") + // There is no public API, but if it parses then success + } + } + @Test fun `google search grounding metadata is parsed correctly`() = goldenVertexUnaryFile("unary-success-google-search-grounding.json") { From a30f7236f4431b82eb2d3d8fbdfec771af50dd1a Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Mon, 8 Sep 2025 13:07:25 -0700 Subject: [PATCH 12/12] remove unneeded doc --- .../main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt index 3795b5d3f96..a83b29ec135 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt @@ -28,7 +28,6 @@ import kotlinx.serialization.Serializable * @property data The raw image bytes in JPEG or PNG format, as specified by [mimeType]. * @property mimeType The IANA standard MIME type of the image data; either `"image/png"` or * `"image/jpeg"`; to request a different format, see [ImagenGenerationConfig.imageFormat]. - * @property safetyAttributes a set of safety attributes with their associated score. */ @PublicPreviewAPI public class ImagenInlineImage