Gradio components in gr.Chatbot()

.changeset/bitter-goats-chew.md

@@ -0,0 +1,9 @@
+---
+"@gradio/chatbot": minor
+"@gradio/image": minor
+"@gradio/multimodaltextbox": minor
+"@gradio/plot": minor
+"gradio": minor
+---
+
+feat:Gradio components in `gr.Chatbot()`

demo/chatbot_multimodal/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: chatbot_multimodal"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/avatar.png https://github.com/gradio-app/gradio/raw/main/demo/chatbot_multimodal/files/avatar.png\n", "!wget -q -O files/lion.jpg https://github.com/gradio-app/gradio/raw/main/demo/chatbot_multimodal/files/lion.jpg"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import os\n", "import time\n", "\n", "# Chatbot demo with multimodal input (text, markdown, LaTeX, code blocks, image, audio, & video). Plus shows support for streaming text.\n", "\n", "\n", "def print_like_dislike(x: gr.LikeData):\n", "    print(x.index, x.value, x.liked)\n", "\n", "def add_message(history, message):\n", "    for x in message[\"files\"]:\n", "        history.append(((x,), None))\n", "    if message[\"text\"] is not None:\n", "        history.append((message[\"text\"], None))\n", "    return history, gr.MultimodalTextbox(value=None, interactive=False)\n", "\n", "def bot(history):\n", "    response = \"**That's cool!**\"\n", "    history[-1][1] = \"\"\n", "    for character in response:\n", "        history[-1][1] += character\n", "        time.sleep(0.05)\n", "        yield history\n", "\n", "with gr.Blocks() as demo:\n", "    chatbot = gr.Chatbot(\n", "        [],\n", "        elem_id=\"chatbot\",\n", "        bubble_full_width=False\n", "    )\n", "\n", "    chat_input = gr.MultimodalTextbox(interactive=True, file_types=[\"image\"], placeholder=\"Enter message or upload file...\", show_label=False)\n", "\n", "    chat_msg = chat_input.submit(add_message, [chatbot, chat_input], [chatbot, chat_input])\n", "    bot_msg = chat_msg.then(bot, chatbot, chatbot, api_name=\"bot_response\")\n", "    bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input])\n", "\n", "    chatbot.like(print_like_dislike, None, None)\n", "\n", "demo.queue()\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: chatbot_multimodal"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/avatar.png https://github.com/gradio-app/gradio/raw/main/demo/chatbot_multimodal/files/avatar.png\n", "!wget -q -O files/cantina.wav https://github.com/gradio-app/gradio/raw/main/demo/chatbot_multimodal/files/cantina.wav\n", "!wget -q -O files/cheetah.jpg https://github.com/gradio-app/gradio/raw/main/demo/chatbot_multimodal/files/cheetah.jpg\n", "!wget -q -O files/lion.jpg https://github.com/gradio-app/gradio/raw/main/demo/chatbot_multimodal/files/lion.jpg\n", "!wget -q -O files/world.mp4 https://github.com/gradio-app/gradio/raw/main/demo/chatbot_multimodal/files/world.mp4\n", "!wget -q -O files/zebra.jpg https://github.com/gradio-app/gradio/raw/main/demo/chatbot_multimodal/files/zebra.jpg"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import plotly.express as px\n", "\n", "# Chatbot demo with multimodal input (text, markdown, LaTeX, code blocks, image, audio, & video). Plus shows support for streaming text.\n", "\n", "def random_plot():\n", "    df = px.data.iris()\n", "    fig = px.scatter(df, x=\"sepal_width\", y=\"sepal_length\", color=\"species\",\n", "                    size='petal_length', hover_data=['petal_width'])\n", "    return fig\n", "\n", "def print_like_dislike(x: gr.LikeData):\n", "    print(x.index, x.value, x.liked)\n", "\n", "def add_message(history, message):\n", "    for x in message[\"files\"]:\n", "        history.append(((x,), None))\n", "    if message[\"text\"] is not None:\n", "        history.append((message[\"text\"], None))\n", "    return history, gr.MultimodalTextbox(value=None, interactive=False)\n", "\n", "def bot(history):\n", "    history[-1][1] = \"Cool!\"\n", "    return history\n", "\n", "fig = random_plot()\n", "\n", "with gr.Blocks(fill_height=True) as demo:\n", "    chatbot = gr.Chatbot(\n", "        [[\"Image\", gr.Image(value=\"files/avatar.png\", render=False)],\n", "         [\"Video\", gr.Video(value=\"files/world.mp4\", render=False)],\n", "         [\"Audio\", gr.Audio(value=\"files/cantina.wav\", render=False)],\n", "         [\"Plot\", gr.Plot(value=fig, render=False)],\n", "         [\"Gallery\", gr.Gallery(value=[\"files/lion.jpg\", \"files/cheetah.jpg\", \"files/zebra.jpg\"], render=False)]],\n", "        elem_id=\"chatbot\",\n", "        bubble_full_width=False,\n", "        scale=1,\n", "    )\n", "\n", "    chat_input = gr.MultimodalTextbox(interactive=True, file_types=[\"image\"], placeholder=\"Enter message or upload file...\", show_label=False)\n", "\n", "    chat_msg = chat_input.submit(add_message, [chatbot, chat_input], [chatbot, chat_input])\n", "    bot_msg = chat_msg.then(bot, chatbot, chatbot, api_name=\"bot_response\")\n", "    bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input])\n", "\n", "    chatbot.like(print_like_dislike, None, None)\n", "\n", "demo.queue()\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/chatbot_multimodal/run.py

@@ -1,9 +1,13 @@
 import gradio as gr
-import os
-import time
+import plotly.express as px
 
 # Chatbot demo with multimodal input (text, markdown, LaTeX, code blocks, image, audio, & video). Plus shows support for streaming text.
 
+def random_plot():
+    df = px.data.iris()
+    fig = px.scatter(df, x="sepal_width", y="sepal_length", color="species",
+                    size='petal_length', hover_data=['petal_width'])
+    return fig
 
 def print_like_dislike(x: gr.LikeData):
     print(x.index, x.value, x.liked)
@@ -16,18 +20,21 @@ def add_message(history, message):
     return history, gr.MultimodalTextbox(value=None, interactive=False)
 
 def bot(history):
-    response = "**That's cool!**"
-    history[-1][1] = ""
-    for character in response:
-        history[-1][1] += character
-        time.sleep(0.05)
-        yield history
-
-with gr.Blocks() as demo:
+    history[-1][1] = "Cool!"
+    return history
+
+fig = random_plot()
+
+with gr.Blocks(fill_height=True) as demo:
     chatbot = gr.Chatbot(
-        [],
+        [["Image", gr.Image(value="files/avatar.png", render=False)],
+         ["Video", gr.Video(value="files/world.mp4", render=False)],
+         ["Audio", gr.Audio(value="files/cantina.wav", render=False)],
+         ["Plot", gr.Plot(value=fig, render=False)],
+         ["Gallery", gr.Gallery(value=["files/lion.jpg", "files/cheetah.jpg", "files/zebra.jpg"], render=False)]],
         elem_id="chatbot",
-        bubble_full_width=False
+        bubble_full_width=False,
+        scale=1,
     )
 
     chat_input = gr.MultimodalTextbox(interactive=True, file_types=["image"], placeholder="Enter message or upload file...", show_label=False)

gradio/components/chatbot.py

@@ -10,7 +10,12 @@
 from gradio_client.documentation import document
 
 from gradio import utils
+from gradio.components import (
+    Component as GradioComponent,
+)
 from gradio.components.base import Component
+from gradio.components.gallery import Gallery, GalleryData
+from gradio.components.plot import Plot, PlotData
 from gradio.data_classes import FileData, GradioModel, GradioRootModel
 from gradio.events import Events
 
@@ -21,7 +26,12 @@ class FileMessage(GradioModel):
 
 
 class ChatbotData(GradioRootModel):
-    root: List[Tuple[Union[str, FileMessage, None], Union[str, FileMessage, None]]]
+    root: List[
+        Tuple[
+            Union[str, FileMessage, List[FileMessage], PlotData, GalleryData, None],
+            Union[str, FileMessage, List[FileMessage], PlotData, GalleryData, None],
+        ]
+    ]
 
 
 @document()
@@ -40,7 +50,9 @@ class Chatbot(Component):
 
     def __init__(
         self,
-        value: list[list[str | tuple[str] | tuple[str | Path, str] | None]]
+        value: list[
+            list[str | GradioComponent | tuple[str] | tuple[str | Path, str] | None]
+        ]
         | Callable
         | None = None,
         *,
@@ -139,24 +151,37 @@ def __init__(
         self.placeholder = placeholder
 
     def _preprocess_chat_messages(
-        self, chat_message: str | FileMessage | None
-    ) -> str | tuple[str | None] | tuple[str | None, str] | None:
+        self,
+        chat_message: str
+        | FileMessage
+        | List[FileMessage]
+        | GalleryData
+        | PlotData
+        | None,
+    ) -> str | GradioComponent | tuple[str | None] | tuple[str | None, str] | None:
         if chat_message is None:
             return None
         elif isinstance(chat_message, FileMessage):
             if chat_message.alt_text is not None:
                 return (chat_message.file.path, chat_message.alt_text)
             else:
                 return (chat_message.file.path,)
-        elif isinstance(chat_message, str):
+        elif isinstance(chat_message, (str)):
             return chat_message
+        elif isinstance(chat_message, GalleryData):
+            value = Gallery().preprocess(chat_message)
+            gallery = Gallery(value=value)
+            return gallery
+        elif isinstance(chat_message, PlotData):
+            plot = Plot(value=chat_message)
+            return plot
         else:
             raise ValueError(f"Invalid message for Chatbot component: {chat_message}")
 
     def preprocess(
         self,
         payload: ChatbotData | None,
-    ) -> list[list[str | tuple[str] | tuple[str, str] | None]] | None:
+    ) -> list[list[str | GradioComponent | tuple[str] | tuple[str, str] | None]] | None:
         """
         Parameters:
             payload: data as a ChatbotData object
@@ -184,18 +209,36 @@ def preprocess(
         return processed_messages
 
     def _postprocess_chat_messages(
-        self, chat_message: str | tuple | list | None
-    ) -> str | FileMessage | None:
-        if chat_message is None:
-            return None
-        elif isinstance(chat_message, (tuple, list)):
-            filepath = str(chat_message[0])
-
+        self, chat_message: str | tuple | list | GradioComponent | None
+    ) -> str | FileMessage | List[FileMessage] | PlotData | GalleryData | None:
+        def create_file_message(chat_message, filepath):
             mime_type = client_utils.get_mimetype(filepath)
             return FileMessage(
                 file=FileData(path=filepath, mime_type=mime_type),
-                alt_text=chat_message[1] if len(chat_message) > 1 else None,
+                alt_text=chat_message[1]
+                if not isinstance(chat_message, GradioComponent)
+                and len(chat_message) > 1
+                else None,
             )
+
+        if chat_message is None:
+            return None
+        elif isinstance(chat_message, (tuple, list)):
+            if isinstance(chat_message[0], GradioComponent):
+                return type(chat_message[0]).postprocess(
+                    chat_message[0], chat_message[0]._constructor_args[1]["value"]
+                )
+            else:
+                filepath = str(chat_message[0])
+                return create_file_message(chat_message, filepath)
+        elif isinstance(chat_message, GradioComponent):
+            if type(chat_message) == Plot or type(chat_message) == Gallery:
+                return type(chat_message).postprocess(
+                    chat_message, chat_message._constructor_args[1]["value"]
+                )  # type: ignore
+            else:
+                filepath = chat_message._constructor_args[1]["value"]
+                return create_file_message(chat_message, filepath)
         elif isinstance(chat_message, str):
             chat_message = inspect.cleandoc(chat_message)
             return chat_message
@@ -204,7 +247,10 @@ def _postprocess_chat_messages(
 
     def postprocess(
         self,
-        value: list[list[str | tuple[str] | tuple[str, str] | None] | tuple] | None,
+        value: list[
+            list[str | GradioComponent | tuple[str] | tuple[str, str] | None] | tuple
+        ]
+        | None,
     ) -> ChatbotData:
         """
         Parameters:
@@ -214,6 +260,7 @@ def postprocess(
         """
         if value is None:
             return ChatbotData(root=[])
+
         processed_messages = []
         for message_pair in value:
             if not isinstance(message_pair, (tuple, list)):

gradio/components/plot.py

@@ -124,6 +124,8 @@ def postprocess(self, value: Any) -> PlotData | None:
 
         if value is None:
             return None
+        if isinstance(value, PlotData):
+            return value
         if isinstance(value, (ModuleType, matplotlib.figure.Figure)):  # type: ignore
             dtype = "matplotlib"
             out_y = processing_utils.encode_plot_to_base64(value, self.format)

js/chatbot/Index.svelte

@@ -16,8 +16,18 @@
 	export let elem_classes: string[] = [];
 	export let visible = true;
 	export let value: [
-		string | { file: FileData; alt_text: string | null } | null,
-		string | { file: FileData; alt_text: string | null } | null
+		(
+			| string
+			| { file: FileData | FileData[]; alt_text: string | null }
+			| { type: string; plot: string | null }
+			| null
+		),
+		(
+			| string
+			| { file: FileData | FileData[]; alt_text: string | null }
+			| { type: string; plot: string | null }
+			| null
+		)
 	][] = [];
 	export let scale: number | null = null;
 	export let min_width: number | undefined = undefined;
@@ -50,19 +60,35 @@
 	export let avatar_images: [FileData | null, FileData | null] = [null, null];
 
 	let _value: [
-		string | { file: FileData; alt_text: string | null } | null,
-		string | { file: FileData; alt_text: string | null } | null
+		(
+			| string
+			| { file: FileData | FileData[]; alt_text: string | null }
+			| { type: string; plot: string | null }
+			| null
+		),
+		(
+			| string
+			| { file: FileData | FileData[]; alt_text: string | null }
+			| { type: string; plot: string | null }
+			| null
+		)
 	][];
 
 	const redirect_src_url = (src: string): string =>
 		src.replace('src="/file', `src="${root}file`);
 
 	function normalize_messages(
-		message: { file: FileData; alt_text: string | null } | null
-	): { file: FileData; alt_text: string | null } | null {
+		message: { file: FileData | FileData[]; alt_text: string | null } | null
+	): { file: FileData | FileData[]; alt_text: string | null } | null {
 		if (message === null) {
 			return message;
 		}
+		if (Array.isArray(message)) {
+			return {
+				file: message.map((file: FileData) => file as FileData),
+				alt_text: message?.alt_text
+			};
+		}
 		return {
 			file: message?.file as FileData,
 			alt_text: message?.alt_text
@@ -73,10 +99,14 @@
 		? value.map(([user_msg, bot_msg]) => [
 				typeof user_msg === "string"
 					? redirect_src_url(user_msg)
-					: normalize_messages(user_msg),
-				typeof bot_msg === "string"
+					: user_msg != null && "plot" in user_msg
+						? user_msg
+						: normalize_messages(user_msg),
+				typeof bot_msg === "string" && bot_msg != null
 					? redirect_src_url(bot_msg)
-					: normalize_messages(bot_msg)
+					: bot_msg != null && "plot" in bot_msg
+						? bot_msg
+						: normalize_messages(bot_msg)
 			])
 		: [];
 
@@ -137,6 +167,7 @@
 			{line_breaks}
 			{layout}
 			{placeholder}
+			upload={gradio.client.upload}
 		/>
 	</div>
 </Block>

js/chatbot/package.json

@@ -10,9 +10,11 @@
 		"@gradio/atoms": "workspace:^",
 		"@gradio/audio": "workspace:^",
 		"@gradio/client": "workspace:^",
+		"@gradio/gallery": "workspace:^",
 		"@gradio/icons": "workspace:^",
 		"@gradio/image": "workspace:^",
 		"@gradio/markdown": "workspace:^",
+		"@gradio/plot": "workspace:^",
 		"@gradio/statustracker": "workspace:^",
 		"@gradio/theme": "workspace:^",
 		"@gradio/upload": "workspace:^",